From 7199d68ba078148ff76a38f2c483da73edd62747 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 3 Dec 2020 21:39:19 -0800
Subject: EPUB writer: include title page in landmarks.

Closes #6919.

Note that the toc is also included if `--toc` is specified.
---
 test/command/5986.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'test/command')

diff --git a/test/command/5986.md b/test/command/5986.md
index e24aebb64..ea0ca70c1 100644
--- a/test/command/5986.md
+++ b/test/command/5986.md
@@ -6,7 +6,8 @@
 <p><span id="nav.xhtml"></span></p>
 <nav type="landmarks" id="landmarks" hidden="hidden">
 <ol>
-<li><a href="#nav.xhtml#toc">Table of contents</a></li>
+<li><a href="text/title_page.xhtml">Title Page</a></li>
+<li><a href="#nav.xhtml#toc">Table of Contents</a></li>
 </ol>
 </nav>
 <p><span id="ch001.xhtml"></span></p>
-- 
cgit v1.2.3


From dc3ef5201f9531bc405ac07e763d9f004bb6bc91 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Fri, 4 Dec 2020 10:55:48 -0800
Subject: Markdown writer: ensure that a new csl-block begins on a new line.

This just looks better and doesn't affect the semantics.
See #6921.
---
 src/Text/Pandoc/Writers/Markdown.hs | 7 ++++++-
 test/command/pandoc-citeproc-53.md  | 6 ++++--
 2 files changed, 10 insertions(+), 3 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/Markdown.hs b/src/Text/Pandoc/Writers/Markdown.hs
index 6aec6b244..5eb47b261 100644
--- a/src/Text/Pandoc/Writers/Markdown.hs
+++ b/src/Text/Pandoc/Writers/Markdown.hs
@@ -1058,7 +1058,12 @@ inlineToMarkdown opts (Span ("",["emoji"],kvs) [Str s]) =
 inlineToMarkdown opts (Span attrs ils) = do
   variant <- asks envVariant
   contents <- inlineListToMarkdown opts ils
-  return $ case variant of
+  return $ case attrs of
+             (_,["csl-block"],_) -> (cr <>)
+             (_,["csl-left-margin"],_) -> (cr <>)
+             (_,["csl-indent"],_) -> (cr <>)
+             _ -> id
+         $ case variant of
                 PlainText -> contents
                 _     | attrs == nullAttr -> contents
                       | isEnabled Ext_bracketed_spans opts ->
diff --git a/test/command/pandoc-citeproc-53.md b/test/command/pandoc-citeproc-53.md
index 295f52049..fb8d5c35e 100644
--- a/test/command/pandoc-citeproc-53.md
+++ b/test/command/pandoc-citeproc-53.md
@@ -28,11 +28,13 @@ Doe[^1] Doe[^2] Roe[^3] Roe[^4] Doe[^5] Doe[^6] Roe[^7] Roe[^8]
 
 ::: {#refs .references .csl-bib-body}
 ::: {#ref-a .csl-entry}
-[[Doe J.]{.smallcaps} ]{.csl-block}[2000, *Work A*,.]{.csl-left-margin}
+[[Doe J.]{.smallcaps} ]{.csl-block}
+[2000, *Work A*,.]{.csl-left-margin}
 :::
 
 ::: {#ref-b .csl-entry}
-[[Roe J.]{.smallcaps} ]{.csl-block}[1990, *Work B*,.]{.csl-left-margin}
+[[Roe J.]{.smallcaps} ]{.csl-block}
+[1990, *Work B*,.]{.csl-left-margin}
 :::
 :::
 
-- 
cgit v1.2.3


From ddb76cb356a82f6a9e51a6f3626dd154816e9205 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sat, 5 Dec 2020 09:53:39 -0800
Subject: LaTeX reader: don't apply theorem default styling to a figure inside.

If we put an image in italics, then when rendering to Markdown
we no longer get an implicit figure.

Closes #6925.
---
 src/Text/Pandoc/Readers/LaTeX.hs |  1 +
 test/command/6925.md             | 34 ++++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 test/command/6925.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/LaTeX.hs b/src/Text/Pandoc/Readers/LaTeX.hs
index 15a1a19fc..afe960454 100644
--- a/src/Text/Pandoc/Readers/LaTeX.hs
+++ b/src/Text/Pandoc/Readers/LaTeX.hs
@@ -1812,6 +1812,7 @@ theoremEnvironment name = do
                   _          -> bs
 
 italicize :: Block -> Block
+italicize x@(Para [Image{}]) = x -- see #6925
 italicize (Para ils) = Para [Emph ils]
 italicize (Plain ils) = Plain [Emph ils]
 italicize x = x
diff --git a/test/command/6925.md b/test/command/6925.md
new file mode 100644
index 000000000..458a0b91d
--- /dev/null
+++ b/test/command/6925.md
@@ -0,0 +1,34 @@
+```
+% pandoc -f latex -t markdown
+\documentclass{amsart}
+\newtheorem{thm}{Theorem}[section]
+\theoremstyle{definition}
+\newtheorem{thm2}[section]{Theorem}
+\begin{document}
+\begin{thm}
+a
+\begin{figure}
+\includegraphics[]{1.png}
+\end{figure}
+\end{thm}
+
+\begin{thm2}
+a
+\begin{figure}
+\includegraphics[]{1.png}
+\end{figure}
+\end{thm2}
+\end{document}
+^D
+::: {.thm}
+**Theorem 1**. *a*
+
+![image](1.png)
+:::
+
+::: {.thm2}
+**Theorem 1**. a
+
+![image](1.png)
+:::
+```
-- 
cgit v1.2.3


From c161893f442a3e001b64af1421e9f62376d71c92 Mon Sep 17 00:00:00 2001
From: Nils Carlson <nils@nilscarlson.se>
Date: Sat, 5 Dec 2020 18:00:04 +0000
Subject: OpenDocument writer: Allow references for internal links (#6774)

This commit adds two extensions to the OpenDocument writer,
`xrefs_name` and `xrefs_number`.

Links to headings, figures and tables inside the document are
substituted with cross-references that will use the name or caption
of the referenced item for `xrefs_name` or the number for `xrefs_number`.

For the `xrefs_number` to be useful heading numbers must be enabled
in the generated document and table and figure captions must be enabled using for example the `native_numbering` extension.

In order for numbers and reference text to be updated the generated
document must be refreshed.

Co-authored-by: Nils Carlson <nils.carlson@ludd.ltu.se>
---
 MANUAL.txt                              | 37 ++++++++++++++
 src/Text/Pandoc/Extensions.hs           |  4 ++
 src/Text/Pandoc/Writers/OpenDocument.hs | 91 ++++++++++++++++++++++++++-------
 test/command/6774.md                    | 63 +++++++++++++++++++++++
 4 files changed, 177 insertions(+), 18 deletions(-)
 create mode 100644 test/command/6774.md

(limited to 'test/command')

diff --git a/MANUAL.txt b/MANUAL.txt
index fad885fed..4e1615ff2 100644
--- a/MANUAL.txt
+++ b/MANUAL.txt
@@ -3027,6 +3027,43 @@ This extension can be enabled/disabled for the following formats:
 output formats
 :  `odt`, `opendocument`
 
+#### Extension: `xrefs_name` ####
+
+Links to headings, figures and tables inside the document are
+substituted with cross-references that will use the name or caption
+of the referenced item. The original link text is replaced once
+the generated document is refreshed. This extension can be combined
+with `xrefs_number` in which case numbers will appear before the
+name.
+
+Text in cross-references is only made consistent with the referenced
+item once the document has been refreshed.
+
+This extension can be enabled/disabled for the following formats:
+
+output formats
+:  `odt`, `opendocument`
+
+#### Extension: `xrefs_number` ####
+
+Links to headings, figures and tables inside the document are
+substituted with cross-references that will use the number
+of the referenced item. The original link text is discarded.
+This extension can be combined with `xrefs_name` in which case
+the name or caption numbers will appear after the number.
+
+For the `xrefs_number` to be useful heading numbers must be enabled
+in the generated document, also table and figure captions must be enabled
+using for example the `native_numbering` extension.
+
+Numbers in cross-references are only visible in the final document once
+it has been refreshed.
+
+This extension can be enabled/disabled for the following formats:
+
+output formats
+:  `odt`, `opendocument`
+
 #### Extension: `styles` #### {#ext-styles}
 
 When converting from docx, read all docx styles as divs (for
diff --git a/src/Text/Pandoc/Extensions.hs b/src/Text/Pandoc/Extensions.hs
index 646f7abfb..a94e24e2c 100644
--- a/src/Text/Pandoc/Extensions.hs
+++ b/src/Text/Pandoc/Extensions.hs
@@ -149,6 +149,8 @@ data Extension =
     | Ext_tex_math_dollars    -- ^ TeX math between $..$ or $$..$$
     | Ext_tex_math_double_backslash  -- ^ TeX math btw \\(..\\) \\[..\\]
     | Ext_tex_math_single_backslash  -- ^ TeX math btw \(..\) \[..\]
+    | Ext_xrefs_name          -- ^ Use xrefs with names
+    | Ext_xrefs_number        -- ^ Use xrefs with numbers
     | Ext_yaml_metadata_block -- ^ YAML metadata block
     | Ext_gutenberg           -- ^ Use Project Gutenberg conventions for plain
     | Ext_attributes          -- ^ Generic attribute syntax
@@ -465,6 +467,8 @@ getAllExtensions f = universalExtensions <> getAll f
   getAll "opendocument"    = extensionsFromList
     [ Ext_empty_paragraphs
     , Ext_native_numbering
+    , Ext_xrefs_name
+    , Ext_xrefs_number
     ]
   getAll "odt"             = getAll "opendocument" <> autoIdExtensions
   getAll "muse"            = autoIdExtensions <>
diff --git a/src/Text/Pandoc/Writers/OpenDocument.hs b/src/Text/Pandoc/Writers/OpenDocument.hs
index 071a5542f..cf42f2228 100644
--- a/src/Text/Pandoc/Writers/OpenDocument.hs
+++ b/src/Text/Pandoc/Writers/OpenDocument.hs
@@ -17,6 +17,7 @@ module Text.Pandoc.Writers.OpenDocument ( writeOpenDocument ) where
 import Control.Arrow ((***), (>>>))
 import Control.Monad.State.Strict hiding (when)
 import Data.Char (chr)
+import Data.Foldable (find)
 import Data.List (sortOn, sortBy, foldl')
 import qualified Data.Map as Map
 import Data.Maybe (fromMaybe, isNothing)
@@ -35,6 +36,7 @@ import Text.DocLayout
 import Text.Pandoc.Shared (linesToPara, tshow, blocksToInlines)
 import Text.Pandoc.Templates (renderTemplate)
 import qualified Text.Pandoc.Translations as Term (Term(Figure, Table))
+import Text.Pandoc.Walk
 import Text.Pandoc.Writers.Math
 import Text.Pandoc.Writers.Shared
 import qualified Text.Pandoc.Writers.AnnotatedTable as Ann
@@ -54,6 +56,11 @@ plainToPara x         = x
 
 type OD m = StateT WriterState m
 
+data ReferenceType
+  = HeaderRef
+  | TableRef
+  | ImageRef
+
 data WriterState =
     WriterState { stNotes          :: [Doc Text]
                 , stTableStyles    :: [Doc Text]
@@ -69,6 +76,7 @@ data WriterState =
                 , stImageId        :: Int
                 , stTableCaptionId :: Int
                 , stImageCaptionId :: Int
+                , stIdentTypes     :: [(Text,ReferenceType)]
                 }
 
 defaultWriterState :: WriterState
@@ -86,6 +94,7 @@ defaultWriterState =
                 , stImageId        = 1
                 , stTableCaptionId = 1
                 , stImageCaptionId = 1
+                , stIdentTypes     = []
                 }
 
 when :: Bool -> Doc Text -> Doc Text
@@ -243,6 +252,12 @@ writeOpenDocument opts (Pandoc meta blocks) = do
                         meta
   ((body, metadata),s) <- flip runStateT
         defaultWriterState $ do
+           let collectInlineIdent (Image (ident,_,_) _ _) = [(ident,ImageRef)]
+               collectInlineIdent _                       = []
+           let collectBlockIdent (Header _ (ident,_,_) _)      = [(ident,HeaderRef)]
+               collectBlockIdent (Table (ident,_,_) _ _ _ _ _) = [(ident,TableRef)]
+               collectBlockIdent _                             = []
+           modify $ \s -> s{ stIdentTypes = query collectBlockIdent blocks ++ query collectInlineIdent blocks }
            m <- metaToContext opts
                   (blocksToOpenDocument opts)
                   (fmap chomp . inlinesToOpenDocument opts)
@@ -411,7 +426,7 @@ blockToOpenDocument o bs
                            inTags True "text:list" [ ("text:style-name", "L" <> tshow ln)]
                                       <$> orderedListToOpenDocument o pn b
       table :: PandocMonad m => Ann.Table -> OD m (Doc Text)
-      table (Ann.Table _ (Caption _ c) colspecs thead tbodies _) = do
+      table (Ann.Table (ident, _, _) (Caption _ c) colspecs thead tbodies _) = do
         tn <- length <$> gets stTableStyles
         pn <- length <$> gets stParaStyles
         let  genIds      = map chr [65..]
@@ -433,7 +448,7 @@ blockToOpenDocument o bs
                       then return empty
                       else inlinesToOpenDocument o (blocksToInlines c) >>=
                              if isEnabled Ext_native_numbering o
-                                then numberedTableCaption
+                                then numberedTableCaption ident
                                 else unNumberedCaption "TableCaption"
         th <- colHeadsToOpenDocument o (map fst paraHStyles) thead
         tr <- mapM (tableBodyToOpenDocument o (map fst paraStyles)) tbodies
@@ -442,36 +457,39 @@ blockToOpenDocument o bs
                           , ("table:style-name", name)
                           ] (vcat columns $$ th $$ vcat tr)
         return $ captionDoc $$ tableDoc
-      figure attr caption source title | null caption =
+      figure attr@(ident, _, _) caption source title | null caption =
         withParagraphStyle o "Figure" [Para [Image attr caption (source,title)]]
                                   | otherwise    = do
         imageDoc <- withParagraphStyle o "FigureWithCaption" [Para [Image attr caption (source,title)]]
         captionDoc <- inlinesToOpenDocument o caption >>=
                          if isEnabled Ext_native_numbering o
-                            then numberedFigureCaption
+                            then numberedFigureCaption ident
                             else unNumberedCaption "FigureCaption"
         return $ imageDoc $$ captionDoc
 
 
-numberedTableCaption :: PandocMonad m => Doc Text -> OD m (Doc Text)
-numberedTableCaption caption = do
+numberedTableCaption :: PandocMonad m => Text -> Doc Text -> OD m (Doc Text)
+numberedTableCaption ident caption = do
     id' <- gets stTableCaptionId
     modify (\st -> st{ stTableCaptionId = id' + 1 })
     capterm <- translateTerm Term.Table
-    return $ numberedCaption "TableCaption" capterm "Table" id' caption
+    return $ numberedCaption "TableCaption" capterm "Table" id' ident caption
 
-numberedFigureCaption :: PandocMonad m => Doc Text -> OD m (Doc Text)
-numberedFigureCaption caption = do
+numberedFigureCaption :: PandocMonad m => Text -> Doc Text -> OD m (Doc Text)
+numberedFigureCaption ident caption = do
     id' <- gets stImageCaptionId
     modify (\st -> st{ stImageCaptionId = id' + 1 })
     capterm <- translateTerm Term.Figure
-    return $ numberedCaption "FigureCaption" capterm "Illustration" id' caption
+    return $ numberedCaption "FigureCaption" capterm  "Illustration" id' ident caption
 
-numberedCaption :: Text -> Text -> Text -> Int -> Doc Text -> Doc Text
-numberedCaption style term name num caption =
+numberedCaption :: Text -> Text -> Text -> Int -> Text -> Doc Text -> Doc Text
+numberedCaption style term name num ident caption =
     let t = text $ T.unpack term
         r = num - 1
-        s = inTags False "text:sequence" [ ("text:ref-name", "ref" <> name <> tshow r),
+        ident' = case ident of
+          "" -> "ref" <> name <> tshow r
+          _ -> ident
+        s = inTags False "text:sequence" [ ("text:ref-name", ident'),
                                            ("text:name", name),
                                            ("text:formula", "ooow:" <> name <> "+1"),
                                            ("style:num-format", "1") ] $ text $ show num
@@ -607,7 +625,9 @@ inlineToOpenDocument o ils
                        else do
                          report $ InlineNotRendered ils
                          return empty
-    Link _ l (s,t) ->  mkLink s t <$> inlinesToOpenDocument o l
+    Link _ l (s,t) -> do
+      identTypes <- gets stIdentTypes
+      mkLink o identTypes s t <$> inlinesToOpenDocument o l
     Image attr _ (s,t) -> mkImg attr s t
     Note        l  -> mkNote l
     where
@@ -619,10 +639,6 @@ inlineToOpenDocument o ils
       unhighlighted s = inlinedCode $ preformatted s
       preformatted s = handleSpaces $ escapeStringForXML s
       inlinedCode s = return $ inTags False "text:span" [("text:style-name", "Source_Text")] s
-      mkLink   s t = inTags False "text:a" [ ("xlink:type" , "simple")
-                                           , ("xlink:href" , s       )
-                                           , ("office:name", t       )
-                                           ] . inSpanTags "Definition"
       mkImg (_, _, kvs) s _ = do
                id' <- gets stImageId
                modify (\st -> st{ stImageId = id' + 1 })
@@ -659,6 +675,45 @@ inlineToOpenDocument o ils
         addNote nn
         return nn
 
+mkLink :: WriterOptions -> [(Text,ReferenceType)] -> Text -> Text -> Doc Text -> Doc Text
+mkLink o identTypes s t d =
+  let maybeIdentAndType = case T.uncons s of
+                            Just ('#', ident) -> find ((ident ==) . fst) identTypes
+                            _                 -> Nothing
+      d' = inSpanTags "Definition" d
+      ref refType format ident       = inTags False refType
+                                       [ ("text:reference-format", format ),
+                                         ("text:ref-name", ident) ]
+      inlineSpace                    = selfClosingTag "text:s" []
+      bookmarkRef                    = ref "text:bookmark-ref"
+      bookmarkRefNumber ident        = bookmarkRef "number" ident mempty
+      bookmarkRefName ident          = bookmarkRef "text" ident d
+      bookmarkRefNameNumber ident    = bookmarkRefNumber ident <> inlineSpace <> bookmarkRefName ident
+      bookmarkRef'
+        | isEnabled Ext_xrefs_number o && isEnabled Ext_xrefs_name o = bookmarkRefNameNumber
+        | isEnabled Ext_xrefs_name o                                 = bookmarkRefName
+        | otherwise                                                  = bookmarkRefNumber
+      sequenceRef                    = ref "text:sequence-ref"
+      sequenceRefNumber ident        = sequenceRef "value" ident mempty
+      sequenceRefName ident          = sequenceRef "caption" ident d
+      sequenceRefNameNumber ident    = sequenceRefNumber ident <> inlineSpace <> sequenceRefName ident
+      sequenceRef'
+        | isEnabled Ext_xrefs_number o && isEnabled Ext_xrefs_name o = sequenceRefNameNumber
+        | isEnabled Ext_xrefs_name o                                 = sequenceRefName
+        | otherwise                                                  = sequenceRefNumber
+      link = inTags False "text:a" [ ("xlink:type" , "simple")
+                                          , ("xlink:href" , s       )
+                                          , ("office:name", t       )
+                                          ] d'
+      linkOrReference = case maybeIdentAndType of
+                          Just (ident, HeaderRef) -> bookmarkRef' ident
+                          Just (ident, TableRef)  -> sequenceRef' ident
+                          Just (ident, ImageRef)  -> sequenceRef' ident
+                          _                       -> link
+      in if isEnabled Ext_xrefs_name o || isEnabled Ext_xrefs_number o
+            then linkOrReference
+            else link
+
 bulletListStyle :: PandocMonad m => Int -> OD m (Int,(Int,[Doc Text]))
 bulletListStyle l = do
   let doStyles  i = inTags True "text:list-level-style-bullet"
diff --git a/test/command/6774.md b/test/command/6774.md
new file mode 100644
index 000000000..66549c0f2
--- /dev/null
+++ b/test/command/6774.md
@@ -0,0 +1,63 @@
+```
+% pandoc -f native -t opendocument --quiet
+[Header 1 ("chapter1",[],[]) [Str "The",Space,Str "Chapter"]
+,Para [Str "Chapter",Space,Str "1",Space,Str "references",Space,Link ("",[],[]) [Str "The",Space,Str "Chapter"] ("#chapter1","")]]
+^D
+<text:h text:style-name="Heading_20_1" text:outline-level="1"><text:bookmark-start text:name="chapter1" />The
+Chapter<text:bookmark-end text:name="chapter1" /></text:h>
+<text:p text:style-name="First_20_paragraph">Chapter 1 references
+<text:a xlink:type="simple" xlink:href="#chapter1" office:name=""><text:span text:style-name="Definition">The
+Chapter</text:span></text:a></text:p>
+```
+```
+% pandoc -f native -t opendocument+xrefs_name --quiet
+[Header 1 ("chapter1",[],[]) [Str "The",Space,Str "Chapter"]
+,Para [Str "Chapter",Space,Str "1",Space,Str "references",Space,Link ("",[],[]) [Str "The",Space,Str "Chapter"] ("#chapter1","")]
+,Para [Image ("lalune",[],[]) [Str "lalune"] ("lalune.jpg","fig:Voyage dans la Lune")]
+,Para [Str "Image",Space,Str "1",Space,Str "references",Space,Link ("",[],[]) [Str "La",Space,Str "Lune"] ("#lalune","")]]
+^D
+<text:h text:style-name="Heading_20_1" text:outline-level="1"><text:bookmark-start text:name="chapter1" />The
+Chapter<text:bookmark-end text:name="chapter1" /></text:h>
+<text:p text:style-name="First_20_paragraph">Chapter 1 references
+<text:bookmark-ref text:reference-format="text" text:ref-name="chapter1">The
+Chapter</text:bookmark-ref></text:p>
+<text:p text:style-name="FigureWithCaption"><draw:frame draw:name="img1"><draw:image xlink:href="lalune.jpg" xlink:type="simple" xlink:show="embed" xlink:actuate="onLoad" /></draw:frame></text:p>
+<text:p text:style-name="FigureCaption">lalune</text:p>
+<text:p text:style-name="Text_20_body">Image 1 references
+<text:sequence-ref text:reference-format="caption" text:ref-name="lalune">La
+Lune</text:sequence-ref></text:p>
+```
+```
+% pandoc -f native -t opendocument+xrefs_number --quiet
+[Header 1 ("chapter1",[],[]) [Str "The",Space,Str "Chapter"]
+,Para [Str "Chapter",Space,Str "1",Space,Str "references",Space,Link ("",[],[]) [Str "The",Space,Str "Chapter"] ("#chapter1","")]
+,Para [Image ("lalune",[],[]) [Str "lalune"] ("lalune.jpg","fig:Voyage dans la Lune")]
+,Para [Str "Image",Space,Str "1",Space,Str "references",Space,Link ("",[],[]) [Str "La",Space,Str "Lune"] ("#lalune","")]]
+^D
+<text:h text:style-name="Heading_20_1" text:outline-level="1"><text:bookmark-start text:name="chapter1" />The
+Chapter<text:bookmark-end text:name="chapter1" /></text:h>
+<text:p text:style-name="First_20_paragraph">Chapter 1 references
+<text:bookmark-ref text:reference-format="number" text:ref-name="chapter1"></text:bookmark-ref></text:p>
+<text:p text:style-name="FigureWithCaption"><draw:frame draw:name="img1"><draw:image xlink:href="lalune.jpg" xlink:type="simple" xlink:show="embed" xlink:actuate="onLoad" /></draw:frame></text:p>
+<text:p text:style-name="FigureCaption">lalune</text:p>
+<text:p text:style-name="Text_20_body">Image 1 references
+<text:sequence-ref text:reference-format="value" text:ref-name="lalune"></text:sequence-ref></text:p>
+```
+```
+% pandoc -f native -t opendocument+xrefs_number+xrefs_name --quiet
+[Header 1 ("chapter1",[],[]) [Str "The",Space,Str "Chapter"]
+,Para [Str "Chapter",Space,Str "1",Space,Str "references",Space,Link ("",[],[]) [Str "The",Space,Str "Chapter"] ("#chapter1","")]
+,Para [Image ("lalune",[],[]) [Str "lalune"] ("lalune.jpg","fig:Voyage dans la Lune")]
+,Para [Str "Image",Space,Str "1",Space,Str "references",Space,Link ("",[],[]) [Str "La",Space,Str "Lune"] ("#lalune","")]]
+^D
+<text:h text:style-name="Heading_20_1" text:outline-level="1"><text:bookmark-start text:name="chapter1" />The
+Chapter<text:bookmark-end text:name="chapter1" /></text:h>
+<text:p text:style-name="First_20_paragraph">Chapter 1 references
+<text:bookmark-ref text:reference-format="number" text:ref-name="chapter1"></text:bookmark-ref><text:s /><text:bookmark-ref text:reference-format="text" text:ref-name="chapter1">The
+Chapter</text:bookmark-ref></text:p>
+<text:p text:style-name="FigureWithCaption"><draw:frame draw:name="img1"><draw:image xlink:href="lalune.jpg" xlink:type="simple" xlink:show="embed" xlink:actuate="onLoad" /></draw:frame></text:p>
+<text:p text:style-name="FigureCaption">lalune</text:p>
+<text:p text:style-name="Text_20_body">Image 1 references
+<text:sequence-ref text:reference-format="value" text:ref-name="lalune"></text:sequence-ref><text:s /><text:sequence-ref text:reference-format="caption" text:ref-name="lalune">La
+Lune</text:sequence-ref></text:p>
+```
-- 
cgit v1.2.3


From 0a502e5ff52b251bbf3da69fd1f9a88d5e0fe92c Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 10 Dec 2020 15:44:10 -0800
Subject: HTML reader:  retain attribute prefixes and avoid duplicates.

Previously we stripped attribute prefixes, reading
`xml:lang` as `lang` for example. This resulted in
two duplicate `lang` attributes when `xml:lang` and
`lang` were both used.  This commit causes the prefixes
to be retained, and also avoids invald duplicate
attributes.

Closes #6938.
---
 src/Text/Pandoc/Readers/HTML.hs         | 28 +++++++++++-----------------
 src/Text/Pandoc/Readers/HTML/Parsing.hs | 20 +++++++++++++-------
 test/command/5986.md                    |  2 +-
 test/epub/wasteland.native              |  8 ++++----
 4 files changed, 29 insertions(+), 29 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/HTML.hs b/src/Text/Pandoc/Readers/HTML.hs
index eb78979a3..f870a241d 100644
--- a/src/Text/Pandoc/Readers/HTML.hs
+++ b/src/Text/Pandoc/Readers/HTML.hs
@@ -74,7 +74,7 @@ readHtml :: PandocMonad m
          -> Text        -- ^ String to parse (assumes @'\n'@ line endings)
          -> m Pandoc
 readHtml opts inp = do
-  let tags = stripPrefixes . canonicalizeTags $
+  let tags = stripPrefixes $ canonicalizeTags $
              parseTagsOptions parseOptions{ optTagPosition = True }
              (crFilter inp)
       parseDoc = do
@@ -95,6 +95,15 @@ readHtml opts inp = do
     Right doc -> return doc
     Left  err -> throwError $ PandocParseError $ T.pack $ getError err
 
+-- Strip namespace prefixes on tags (not attributes)
+stripPrefixes :: [Tag Text] -> [Tag Text]
+stripPrefixes = map stripPrefix
+
+stripPrefix :: Tag Text -> Tag Text
+stripPrefix (TagOpen s as) = TagOpen (T.takeWhileEnd (/=':') s) as
+stripPrefix (TagClose s)   = TagClose (T.takeWhileEnd (/=':') s)
+stripPrefix x = x
+
 replaceNotes :: PandocMonad m => [Block] -> TagParser m [Block]
 replaceNotes bs = do
   st <- getState
@@ -114,7 +123,7 @@ setInPlain = local (\s -> s {inPlain = True})
 pHtml :: PandocMonad m => TagParser m Blocks
 pHtml = try $ do
   (TagOpen "html" attr) <- lookAhead pAny
-  for_ (lookup "lang" attr) $
+  for_ (lookup "lang" attr <|> lookup "xml:lang" attr) $
     updateState . B.setMeta "lang" . B.text
   pInTags "html" block
 
@@ -1024,21 +1033,6 @@ htmlTag f = try $ do
          handleTag tagname
        _ -> mzero
 
--- Strip namespace prefixes
-stripPrefixes :: [Tag Text] -> [Tag Text]
-stripPrefixes = map stripPrefix
-
-stripPrefix :: Tag Text -> Tag Text
-stripPrefix (TagOpen s as) =
-    TagOpen (stripPrefix' s) (map (first stripPrefix') as)
-stripPrefix (TagClose s) = TagClose (stripPrefix' s)
-stripPrefix x = x
-
-stripPrefix' :: Text -> Text
-stripPrefix' s =
-  if T.null t then s else T.drop 1 t
-  where (_, t) = T.span (/= ':') s
-
 -- Utilities
 
 -- | Adjusts a url according to the document's base URL.
diff --git a/src/Text/Pandoc/Readers/HTML/Parsing.hs b/src/Text/Pandoc/Readers/HTML/Parsing.hs
index 2d58319da..e28ebe77b 100644
--- a/src/Text/Pandoc/Readers/HTML/Parsing.hs
+++ b/src/Text/Pandoc/Readers/HTML/Parsing.hs
@@ -193,14 +193,20 @@ t1 `closes` t2 |
 _ `closes` _ = False
 
 toStringAttr :: [(Text, Text)] -> [(Text, Text)]
-toStringAttr = map go
+toStringAttr = foldr go []
   where
-   go (x,y) =
-     case T.stripPrefix "data-" x of
-       Just x' | x' `Set.notMember` (html5Attributes <>
-                                     html4Attributes <> rdfaAttributes)
-         -> (x',y)
-       _ -> (x,y)
+   go :: (Text, Text) -> [(Text, Text)] -> [(Text, Text)]
+   -- treat xml:lang as lang
+   go ("xml:lang",y) ats = go ("lang",y) ats
+   -- prevent duplicate attributes
+   go (x,y) ats
+     | any (\(x',_) -> x == x') ats = ats
+     | otherwise      =
+        case T.stripPrefix "data-" x of
+          Just x' | x' `Set.notMember` (html5Attributes <>
+                                        html4Attributes <> rdfaAttributes)
+            -> go (x',y) ats
+          _ -> (x,y):ats
 
 -- Unlike fromAttrib from tagsoup, this distinguishes
 -- between a missing attribute and an attribute with empty content.
diff --git a/test/command/5986.md b/test/command/5986.md
index ea0ca70c1..ed8dd30c9 100644
--- a/test/command/5986.md
+++ b/test/command/5986.md
@@ -4,7 +4,7 @@
 ^D
 <p><span id="title_page.xhtml"></span></p>
 <p><span id="nav.xhtml"></span></p>
-<nav type="landmarks" id="landmarks" hidden="hidden">
+<nav epub:type="landmarks" id="landmarks" hidden="hidden">
 <ol>
 <li><a href="text/title_page.xhtml">Title Page</a></li>
 <li><a href="#nav.xhtml#toc">Table of Contents</a></li>
diff --git a/test/epub/wasteland.native b/test/epub/wasteland.native
index 0ed23eba2..ff59182a6 100644
--- a/test/epub/wasteland.native
+++ b/test/epub/wasteland.native
@@ -1,8 +1,8 @@
 [Para [Image ("",[],[]) [] ("wasteland-cover.jpg","")]
 ,Para [Span ("wasteland-content.xhtml",[],[]) []]
-,Div ("wasteland-content.xhtml#frontmatter",["section"],[("type","frontmatter")])
+,Div ("wasteland-content.xhtml#frontmatter",["section","frontmatter"],[])
  []
-,Div ("wasteland-content.xhtml#bodymatter",["section"],[("type","bodymatter")])
+,Div ("wasteland-content.xhtml#bodymatter",["section","bodymatter"],[])
  [Div ("wasteland-content.xhtml#ch1",["section"],[])
   [Header 2 ("",[],[]) [Str "I.",Space,Str "THE",Space,Str "BURIAL",Space,Str "OF",Space,Str "THE",Space,Str "DEAD"]
   ,Div ("",["linegroup"],[])
@@ -922,8 +922,8 @@
     [Plain [Str "Datta.",Space,Str "Dayadhvam.",Space,Str "Damyata."]]
    ,Div ("wasteland-content.xhtml#ln434",["linegroup","indent"],[])
     [Plain [Span ("",[],[("lang","sa")]) [Str "Shantih",Space,Str "shantih",Space,Str "shantih",Note [Para [Link ("",[],[]) [Str "434."] ("#wasteland-content.xhtml#ln434",""),Space,Str "Shantih.",Space,Str "Repeated",Space,Str "as",Space,Str "here,",Space,Str "a",Space,Str "formal",Space,Str "ending",Space,Str "to",Space,Str "an",Space,Str "Upanishad.",Space,Str "'The",SoftBreak,Str "Peace",Space,Str "which",Space,Str "passeth",Space,Str "understanding'",Space,Str "is",Space,Str "a",Space,Str "feeble",Space,Str "translation",Space,Str "of",Space,Str "the",SoftBreak,Str "content",Space,Str "of",Space,Str "this",Space,Str "word."]],SoftBreak]]]]]]
-,Div ("wasteland-content.xhtml#backmatter",["section"],[("type","backmatter")])
- [Div ("wasteland-content.xhtml#rearnotes",["section"],[("type","rearnotes")])
+,Div ("wasteland-content.xhtml#backmatter",["section","backmatter"],[])
+ [Div ("wasteland-content.xhtml#rearnotes",["section","rearnotes"],[])
   [Header 2 ("",[],[]) [Str "NOTES",Space,Str "ON",Space,Str "\"THE",Space,Str "WASTE",Space,Str "LAND\""]
   ,Para [Str "Not",Space,Str "only",Space,Str "the",Space,Str "title,",Space,Str "but",Space,Str "the",Space,Str "plan",Space,Str "and",Space,Str "a",Space,Str "good",Space,Str "deal",Space,Str "of",Space,Str "the",Space,Str "incidental",Space,Str "symbolism",Space,Str "of",SoftBreak,Str "the",Space,Str "poem",Space,Str "were",Space,Str "suggested",Space,Str "by",Space,Str "Miss",Space,Str "Jessie",Space,Str "L.",Space,Str "Weston's",Space,Str "book",Space,Str "on",Space,Str "the",Space,Str "Grail",Space,Str "legend:",SoftBreak,Str "From",Space,Str "Ritual",Space,Str "to",Space,Str "Romance"]
   ,Para [Str "Indeed,",Space,Str "so",Space,Str "deeply",Space,Str "am",Space,Str "I",Space,Str "indebted,",Space,Str "Miss",Space,Str "Weston's",Space,Str "book",Space,Str "will",Space,Str "elucidate",Space,Str "the",SoftBreak,Str "difficulties",Space,Str "of",Space,Str "the",Space,Str "poem",Space,Str "much",Space,Str "better",Space,Str "than",Space,Str "my",Space,Str "notes",Space,Str "can",Space,Str "do;",Space,Str "and",Space,Str "I",Space,Str "recommend",Space,Str "it",SoftBreak,Str "(apart",Space,Str "from",Space,Str "the",Space,Str "great",Space,Str "interest",Space,Str "of",Space,Str "the",Space,Str "book",Space,Str "itself)",Space,Str "to",Space,Str "any",Space,Str "who",Space,Str "think",Space,Str "such",SoftBreak,Str "elucidation",Space,Str "of",Space,Str "the",Space,Str "poem",Space,Str "worth",Space,Str "the",Space,Str "trouble.",Space,Str "To",Space,Str "another",Space,Str "work",Space,Str "of",Space,Str "anthropology",Space,Str "I",Space,Str "am",SoftBreak,Str "indebted",Space,Str "in",Space,Str "general,",Space,Str "one",Space,Str "which",Space,Str "has",Space,Str "influenced",Space,Str "our",Space,Str "generation",Space,Str "profoundly;",Space,Str "I",Space,Str "mean",SoftBreak,Str "The",Space,Str "Golden",Space,Str "Bough;",Space,Str "I",Space,Str "have",Space,Str "used",Space,Str "especially",Space,Str "the",Space,Str "two",Space,Str "volumes",Space,Str "Adonis,",Space,Str "Attis,",Space,Str "Osiris.",SoftBreak,Str "Anyone",Space,Str "who",Space,Str "is",Space,Str "acquainted",Space,Str "with",Space,Str "these",Space,Str "works",Space,Str "will",Space,Str "immediately",Space,Str "recognise",Space,Str "in",Space,Str "the",Space,Str "poem",SoftBreak,Str "certain",Space,Str "references",Space,Str "to",Space,Str "vegetation",Space,Str "ceremonies."]
-- 
cgit v1.2.3


From 208cb9619632ee9490b75158e44213ee6d4ab93e Mon Sep 17 00:00:00 2001
From: mb21 <mb21@users.noreply.github.com>
Date: Sat, 12 Dec 2020 14:12:39 +0100
Subject: ICML writer: fix image bounding box for custom widths/heights

fixes #6936
---
 src/Text/Pandoc/Writers/ICML.hs | 7 ++++++-
 test/command/5541-nesting.md    | 1 +
 test/command/svg.md             | 4 ++++
 test/writer.icml                | 2 ++
 4 files changed, 13 insertions(+), 1 deletion(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/ICML.hs b/src/Text/Pandoc/Writers/ICML.hs
index dcf5acfef..c254fbc58 100644
--- a/src/Text/Pandoc/Writers/ICML.hs
+++ b/src/Text/Pandoc/Writers/ICML.hs
@@ -621,7 +621,12 @@ imageICML opts style attr (src, _) = do
       image  = inTags True "Image"
                    [("Self","ue6"), ("ItemTransform", scale<>" -"<>hw<>" -"<>hh)]
                  $ vcat [
-                     inTags True "Properties" [] $ inTags True "Profile" [("type","string")] $ text "$ID/Embedded"
+                     inTags True "Properties" [] $ vcat [
+                         inTags True "Profile" [("type","string")] $ text "$ID/Embedded"
+                       , selfClosingTag "GraphicBounds" [("Left","0"), ("Top","0")
+                         , ("Right",  showFl $ ow*ow / imgWidth)
+                         , ("Bottom", showFl $ oh*oh / imgHeight)]
+                       ]
                    , selfClosingTag "Link" [("Self", "ueb"), ("LinkResourceURI", src')]
                    ]
       doc    = inTags True "CharacterStyleRange" attrs
diff --git a/test/command/5541-nesting.md b/test/command/5541-nesting.md
index 194b79164..5abe41a9a 100644
--- a/test/command/5541-nesting.md
+++ b/test/command/5541-nesting.md
@@ -80,6 +80,7 @@
           <Profile type="string">
             $ID/Embedded
           </Profile>
+          <GraphicBounds Left="0" Top="0" Right="150" Bottom="150" />
         </Properties>
         <Link Self="ueb" LinkResourceURI="file:lalune.jpg" />
       </Image>
diff --git a/test/command/svg.md b/test/command/svg.md
index 26a8213f6..57506570b 100644
--- a/test/command/svg.md
+++ b/test/command/svg.md
@@ -23,6 +23,7 @@
           <Profile type="string">
             $ID/Embedded
           </Profile>
+          <GraphicBounds Left="0" Top="0" Right="300" Bottom="200" />
         </Properties>
         <Link Self="ueb" LinkResourceURI="file:command/corrupt.svg" />
       </Image>
@@ -56,6 +57,7 @@
           <Profile type="string">
             $ID/Embedded
           </Profile>
+          <GraphicBounds Left="0" Top="0" Right="300" Bottom="200" />
         </Properties>
         <Link Self="ueb" LinkResourceURI="file:command/SVG_logo.svg" />
       </Image>
@@ -89,6 +91,7 @@
           <Profile type="string">
             $ID/Embedded
           </Profile>
+          <GraphicBounds Left="0" Top="0" Right="300" Bottom="200" />
         </Properties>
         <Link Self="ueb" LinkResourceURI="file:command/SVG_logo-without-xml-declaration.svg" />
       </Image>
@@ -122,6 +125,7 @@
           <Profile type="string">
             $ID/Embedded
           </Profile>
+          <GraphicBounds Left="0" Top="0" Right="109.5" Bottom="130.5" />
         </Properties>
         <Link Self="ueb" LinkResourceURI="file:command/inkscape-cube.svg" />
       </Image>
diff --git a/test/writer.icml b/test/writer.icml
index a4b40056a..0601ba494 100644
--- a/test/writer.icml
+++ b/test/writer.icml
@@ -2800,6 +2800,7 @@ These should not be escaped:  \$ \\ \&gt; \[ \{</Content>
           <Profile type="string">
             $ID/Embedded
           </Profile>
+          <GraphicBounds Left="0" Top="0" Right="150" Bottom="150" />
         </Properties>
         <Link Self="ueb" LinkResourceURI="file:lalune.jpg" />
       </Image>
@@ -2836,6 +2837,7 @@ These should not be escaped:  \$ \\ \&gt; \[ \{</Content>
           <Profile type="string">
             $ID/Embedded
           </Profile>
+          <GraphicBounds Left="0" Top="0" Right="20" Bottom="22" />
         </Properties>
         <Link Self="ueb" LinkResourceURI="file:movie.jpg" />
       </Image>
-- 
cgit v1.2.3


From c43e2dc0f4ce639c8d0bd156afea2fd07ffc91ff Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sun, 13 Dec 2020 10:45:03 -0800
Subject: RST writer: better image handling.

- An image alone in its paragraph (but not a figure) is now
  rendered as an independent image, with an `alt` attribute
  if a description is supplied.
- An inline image that is not alone in its paragraph will
  be rendered, as before, using a substitution.
  Such an image cannot have a "center", "left", or
  "right" alignment, so the classes `align-center`,
  `align-left`, or `align-right` are ignored.
  However, `align-top`, `align-middle`, `align-bottom`
  will generate a corresponding `align` attribute.

Closes #6948.
---
 src/Text/Pandoc/Writers/RST.hs | 30 +++++++++++++++++++++---------
 test/command/4420.md           | 11 -----------
 test/command/6948.md           | 31 +++++++++++++++++++++++++++++++
 3 files changed, 52 insertions(+), 20 deletions(-)
 create mode 100644 test/command/6948.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/RST.hs b/src/Text/Pandoc/Writers/RST.hs
index 43bf382b7..8beeef46a 100644
--- a/src/Text/Pandoc/Writers/RST.hs
+++ b/src/Text/Pandoc/Writers/RST.hs
@@ -143,9 +143,12 @@ pictToRST (label, (attr, src, _, mbtarget)) = do
   let (_, cls, _) = attr
       classes = case cls of
                    []               -> empty
-                   ["align-right"]  -> ":align: right"
-                   ["align-left"]   -> ":align: left"
-                   ["align-center"] -> ":align: center"
+                   ["align-top"]    -> ":align: top"
+                   ["align-middle"] -> ":align: middle"
+                   ["align-bottom"] -> ":align: bottom"
+                   ["align-center"] -> empty
+                   ["align-right"]  -> empty
+                   ["align-left"]   -> empty
                    _                -> ":class: " <> literal (T.unwords cls)
   return $ nowrap
          $ ".. |" <> label' <> "| image:: " <> literal src $$ hang 3 empty (classes $$ dims)
@@ -215,19 +218,28 @@ blockToRST (Div (ident,classes,_kvs) bs) = do
            nest 3 contents $$
            blankline
 blockToRST (Plain inlines) = inlineListToRST inlines
--- title beginning with fig: indicates that the image is a figure
-blockToRST (Para [Image attr txt (src,T.stripPrefix "fig:" -> Just tit)]) = do
-  capt <- inlineListToRST txt
+blockToRST (Para [Image attr txt (src, rawtit)]) = do
+  description <- inlineListToRST txt
   dims <- imageDimsToRST attr
-  let fig = "figure:: " <> literal src
-      alt = ":alt: " <> if T.null tit then capt else literal tit
+  -- title beginning with fig: indicates that the image is a figure
+  let (isfig, tit) = case T.stripPrefix "fig:" rawtit of
+                          Nothing   -> (False, rawtit)
+                          Just tit' -> (True, tit')
+  let fig | isfig = "figure:: " <> literal src
+          | otherwise = "image:: " <> literal src
+      alt | isfig = ":alt: " <> if T.null tit then description else literal tit
+          | null txt = empty
+          | otherwise = ":alt: " <> description
+      capt | isfig = description
+           | otherwise = empty
       (_,cls,_) = attr
       classes = case cls of
                    []               -> empty
                    ["align-right"]  -> ":align: right"
                    ["align-left"]   -> ":align: left"
                    ["align-center"] -> ":align: center"
-                   _                -> ":figclass: " <> literal (T.unwords cls)
+                   _ | isfig        -> ":figclass: " <> literal (T.unwords cls)
+                     | otherwise    -> ":class: " <> literal (T.unwords cls)
   return $ hang 3 ".. " (fig $$ alt $$ classes $$ dims $+$ capt) $$ blankline
 blockToRST (Para inlines)
   | LineBreak `elem` inlines =
diff --git a/test/command/4420.md b/test/command/4420.md
index 3e7008935..36d697234 100644
--- a/test/command/4420.md
+++ b/test/command/4420.md
@@ -1,14 +1,3 @@
-```
-% pandoc -f native -t rst
-[Image ("",["align-right"],[("width","100px")]) [Str "image"] ("foo.png","")]
-^D
-|image|
-
-.. |image| image:: foo.png
-   :align: right
-   :width: 100px
-```
-
 ```
 % pandoc -f native -t rst
 [Para [Image ("",["align-right"],[("width","100px")]) [Str "image"] ("foo.png","fig:test")]]
diff --git a/test/command/6948.md b/test/command/6948.md
new file mode 100644
index 000000000..8803aebe9
--- /dev/null
+++ b/test/command/6948.md
@@ -0,0 +1,31 @@
+Treat an image alone in its paragraph (but not a figure)
+as an independent image:
+```
+% pandoc -f native -t rst
+[Para [Image ("",["align-center"],[]) [Str "https://pandoc.org/diagram.jpg"] ("https://pandoc.org/diagram.jpg","")]]
+^D
+.. image:: https://pandoc.org/diagram.jpg
+   :alt: https://pandoc.org/diagram.jpg
+   :align: center
+```
+
+Here we just omit the center attribute as it's not valid:
+```
+% pandoc -f native -t rst
+[Para [Str "hi",Space,Image ("",["align-center"],[]) [Str "https://pandoc.org/diagram.jpg"] ("https://pandoc.org/diagram.jpg","")]]
+^D
+hi |https://pandoc.org/diagram.jpg|
+
+.. |https://pandoc.org/diagram.jpg| image:: https://pandoc.org/diagram.jpg
+```
+
+But we can use top, middle, or bottom alignment:
+```
+% pandoc -f native -t rst
+[Para [Str "hi",Space,Image ("",["align-top"],[]) [Str "https://pandoc.org/diagram.jpg"] ("https://pandoc.org/diagram.jpg","")]]
+^D
+hi |https://pandoc.org/diagram.jpg|
+
+.. |https://pandoc.org/diagram.jpg| image:: https://pandoc.org/diagram.jpg
+   :align: top
+```
-- 
cgit v1.2.3


From 7d799bfcda749e4a3ad6fcae59ac5ccb80b77ffd Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Tue, 15 Dec 2020 08:51:43 -0800
Subject: Allow both inline and external references to be used

with `--citeproc`.  This fixes a regression, since pandoc-citeproc
allowed these to be combined.

Closes #6951.
---
 src/Text/Pandoc/Citeproc.hs | 29 +++++++++++++++--------------
 test/command/6951.md        | 18 ++++++++++++++++++
 2 files changed, 33 insertions(+), 14 deletions(-)
 create mode 100644 test/command/6951.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Citeproc.hs b/src/Text/Pandoc/Citeproc.hs
index 0de2882ae..82d1dc32f 100644
--- a/src/Text/Pandoc/Citeproc.hs
+++ b/src/Text/Pandoc/Citeproc.hs
@@ -105,20 +105,21 @@ processCitations (Pandoc meta bs) = do
   let idpred = if "*" `Set.member` nocites
                   then const True
                   else (`Set.member` citeIds)
-  refs <- map (linkifyVariables . legacyDateRanges) <$>
-          case lookupMeta "references" meta of
-            Just (MetaList rs) -> return $ mapMaybe metaValueToReference rs
-            _                  ->
-              case lookupMeta "bibliography" meta of
-                 Just (MetaList xs) ->
-                   mconcat <$>
-                     mapM (getRefsFromBib locale idpred)
-                       (mapMaybe metaValueToText xs)
-                 Just x ->
-                   case metaValueToText x of
-                     Just fp -> getRefsFromBib locale idpred fp
-                     Nothing -> return []
-                 Nothing -> return []
+  let inlineRefs = case lookupMeta "references" meta of
+                    Just (MetaList rs) -> mapMaybe metaValueToReference rs
+                    _                  -> []
+  externalRefs <- case lookupMeta "bibliography" meta of
+                    Just (MetaList xs) ->
+                      mconcat <$>
+                        mapM (getRefsFromBib locale idpred)
+                          (mapMaybe metaValueToText xs)
+                    Just x ->
+                      case metaValueToText x of
+                        Just fp -> getRefsFromBib locale idpred fp
+                        Nothing -> return []
+                    Nothing -> return []
+  let refs = map (linkifyVariables . legacyDateRanges)
+                 (inlineRefs ++ externalRefs)
   let otherIdsMap = foldr (\ref m ->
                              case T.words . extractText <$>
                                   M.lookup "other-ids"
diff --git a/test/command/6951.md b/test/command/6951.md
new file mode 100644
index 000000000..4a6301d80
--- /dev/null
+++ b/test/command/6951.md
@@ -0,0 +1,18 @@
+```
+% pandoc --citeproc -t plain --bibliography command/biblio.bib
+---
+references:
+- id: foo
+  title: Crazy
+  type: book
+...
+
+[@foo; @item1]
+
+^D
+(Crazy, n.d.; Doe 2005)
+
+Crazy. n.d.
+
+Doe, John. 2005. First Book. Cambridge: Cambridge University Press.
+```
-- 
cgit v1.2.3


From b4b4e32307499aafa2e6c1c713ee41f1c787ea76 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Tue, 15 Dec 2020 23:45:34 -0800
Subject: Properly handle boolean values in writing YAML metadata.

(Markdown writer.)
This requires doctemplates >= 0.9.
Closes #6388.
---
 pandoc.cabal                        |  4 ++--
 src/Text/Pandoc/Writers/Markdown.hs |  2 ++
 src/Text/Pandoc/Writers/Shared.hs   |  3 +--
 stack.yaml                          |  5 ++++-
 test/command/6388.md                | 16 ++++++++++++++++
 5 files changed, 25 insertions(+), 5 deletions(-)
 create mode 100644 test/command/6388.md

(limited to 'test/command')

diff --git a/pandoc.cabal b/pandoc.cabal
index 6483690b9..9244d25cd 100644
--- a/pandoc.cabal
+++ b/pandoc.cabal
@@ -410,7 +410,7 @@ library
                  deepseq               >= 1.3      && < 1.5,
                  directory             >= 1.2.3    && < 1.4,
                  doclayout             >= 0.3      && < 0.4,
-                 doctemplates          >= 0.8.2    && < 0.9,
+                 doctemplates          >= 0.9      && < 0.10,
                  emojis                >= 0.1      && < 0.2,
                  exceptions            >= 0.8      && < 0.11,
                  file-embed            >= 0.0      && < 0.1,
@@ -767,7 +767,7 @@ test-suite test-pandoc
                   bytestring        >= 0.9     && < 0.12,
                   containers        >= 0.4.2.1 && < 0.7,
                   directory         >= 1.2.3   && < 1.4,
-                  doctemplates      >= 0.8.2   && < 0.9,
+                  doctemplates      >= 0.9     && < 0.10,
                   exceptions        >= 0.8     && < 0.11,
                   executable-path   >= 0.0     && < 0.1,
                   filepath          >= 1.1     && < 1.5,
diff --git a/src/Text/Pandoc/Writers/Markdown.hs b/src/Text/Pandoc/Writers/Markdown.hs
index 5eb47b261..c349fd713 100644
--- a/src/Text/Pandoc/Writers/Markdown.hs
+++ b/src/Text/Pandoc/Writers/Markdown.hs
@@ -174,6 +174,8 @@ valToYaml :: Val Text -> Doc Text
 valToYaml (ListVal xs) =
   vcat $ map (\v -> hang 2 "- " (valToYaml v)) xs
 valToYaml (MapVal c) = contextToYaml c
+valToYaml (BoolVal True) = "true"
+valToYaml (BoolVal False) = "false"
 valToYaml (SimpleVal x)
   | isEmpty x = empty
   | otherwise =
diff --git a/src/Text/Pandoc/Writers/Shared.hs b/src/Text/Pandoc/Writers/Shared.hs
index b399afbf3..129e45e9d 100644
--- a/src/Text/Pandoc/Writers/Shared.hs
+++ b/src/Text/Pandoc/Writers/Shared.hs
@@ -109,8 +109,7 @@ metaValueToVal blockWriter inlineWriter (MetaMap metamap) =
   MapVal . Context <$> mapM (metaValueToVal blockWriter inlineWriter) metamap
 metaValueToVal blockWriter inlineWriter (MetaList xs) = ListVal <$>
   mapM (metaValueToVal blockWriter inlineWriter) xs
-metaValueToVal _ _ (MetaBool True) = return $ SimpleVal "true"
-metaValueToVal _ _ (MetaBool False) = return NullVal
+metaValueToVal _ _ (MetaBool b) = return $ BoolVal b
 metaValueToVal _ inlineWriter (MetaString s) =
    SimpleVal <$> inlineWriter (Builder.toList (Builder.text s))
 metaValueToVal blockWriter _ (MetaBlocks bs) = SimpleVal <$> blockWriter bs
diff --git a/stack.yaml b/stack.yaml
index ef0bc7a83..7c1bde8ae 100644
--- a/stack.yaml
+++ b/stack.yaml
@@ -16,7 +16,10 @@ extra-deps:
 - commonmark-0.1.1.2
 - commonmark-extensions-0.2.0.4
 - commonmark-pandoc-0.2.0.1
-- doctemplates-0.8.3
+# - doctemplates-0.8.3
+- doctemplates:
+  git: https://github.com/jgm/doctemplates.git
+  commit: 6cb8d09264f7c1f4b59ac67ff98b1a664a85e36a
 - citeproc-0.2.0.1
 # - citeproc:
 #   git: https://github.com/jgm/citeproc.git
diff --git a/test/command/6388.md b/test/command/6388.md
new file mode 100644
index 000000000..29a9156d7
--- /dev/null
+++ b/test/command/6388.md
@@ -0,0 +1,16 @@
+```
+% pandoc -t markdown -s
+---
+nvalue: false
+value: true
+---
+
+text
+^D
+---
+nvalue: false
+value: true
+---
+
+text
+```
-- 
cgit v1.2.3


From 95b15fe6d3eb89a18ff143e32a0fb958f9e8ab8f Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Fri, 18 Dec 2020 11:14:58 -0800
Subject: Remove some test files that are no longer used.

---
 test/command/custom-attributes.html | 16 ----------------
 test/command/empty-inline-code.txt  |  6 ------
 test/grofftest.sh                   | 26 --------------------------
 3 files changed, 48 deletions(-)
 delete mode 100644 test/command/custom-attributes.html
 delete mode 100644 test/command/empty-inline-code.txt
 delete mode 100644 test/grofftest.sh

(limited to 'test/command')

diff --git a/test/command/custom-attributes.html b/test/command/custom-attributes.html
deleted file mode 100644
index 67dccc1b8..000000000
--- a/test/command/custom-attributes.html
+++ /dev/null
@@ -1,16 +0,0 @@
-Custom attributes should automatically have data- added
-in HTML5:
-```
-% pandoc -t html5
-[hello]{foo="bar"}
-^D
-<span data-foo="bar">hello</span>
-```
-
-but not in HTML4:
-```
-% pandoc -t html4
-[hello]{foo="bar"}
-^D
-<span foo="bar">hello</span>
-```
diff --git a/test/command/empty-inline-code.txt b/test/command/empty-inline-code.txt
deleted file mode 100644
index b57072a44..000000000
--- a/test/command/empty-inline-code.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-```
-% pandoc -t native
-` `
-^D
-[Code ("",[],[]) ""]
-```
diff --git a/test/grofftest.sh b/test/grofftest.sh
deleted file mode 100644
index ac652112b..000000000
--- a/test/grofftest.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/bash
-
-# iterates recursively over specified directory, tries to convert
-# man pages and prints to stderr on errors.
-
-# if called with two arguments, the first is the path to pandoc,
-# and the second is the directory.  if with one argument, it
-# is the directory, and pandoc is used from path.
-
-if [ $# -eq 2 ]; then
-        PANDOC=$1
-        DIR=$2
-elif [ $# -eq 1 ]; then
-        PANDOC=pandoc
-        DIR=$1
-else
-	echo "Not enough arguments"
-	exit 1
-fi
-
-$PANDOC --version > /dev/null || { echo "pandoc executable error" >&2 ; exit 1 ; }
-
-for f in `find "$DIR" -name '*.[0-9]'`; do
-    ( iconv -f utf8 -t utf8 $f 2>/dev/null || iconv -f latin1 -t utf8 $f ) | \
-        $PANDOC --resource-path "$DIR":"$(dirname $f)" -f man -o /dev/null || echo "Failed to convert $f"
-done
-- 
cgit v1.2.3


From 668596cc89f8f6bddafd35fd031638bc0c416183 Mon Sep 17 00:00:00 2001
From: timo-a <timo-a@gmx.ch>
Date: Mon, 28 Dec 2020 03:42:28 +0100
Subject: Add support for writing nested tables to asciidoc (#6972)

Added field to WriterState that denotes the current nesting level for traversing tables.
Depending on the value of that field nested tables are recognized and written.
Asciidoc supports one level of nesting. If deeper tables are to be written, they are
omitted and a warning is issued.
---
 src/Text/Pandoc/Writers/AsciiDoc.hs           | 39 ++++++++++---
 test/command/nested-table-to-asciidoc-6942.md | 82 +++++++++++++++++++++++++++
 2 files changed, 114 insertions(+), 7 deletions(-)
 create mode 100644 test/command/nested-table-to-asciidoc-6942.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/AsciiDoc.hs b/src/Text/Pandoc/Writers/AsciiDoc.hs
index e742577b6..0a312d1d1 100644
--- a/src/Text/Pandoc/Writers/AsciiDoc.hs
+++ b/src/Text/Pandoc/Writers/AsciiDoc.hs
@@ -37,6 +37,7 @@ import Text.Pandoc.Shared
 import Text.Pandoc.Templates (renderTemplate)
 import Text.Pandoc.Writers.Shared
 
+
 data WriterState = WriterState { defListMarker       :: Text
                                , orderedListLevel    :: Int
                                , bulletListLevel     :: Int
@@ -45,6 +46,10 @@ data WriterState = WriterState { defListMarker       :: Text
                                , asciidoctorVariant  :: Bool
                                , inList              :: Bool
                                , hasMath             :: Bool
+                               -- |0 is no table
+                               -- 1 is top level table
+                               -- 2 is a table in a table
+                               , tableNestingLevel   :: Int
                                }
 
 defaultWriterState :: WriterState
@@ -56,6 +61,7 @@ defaultWriterState = WriterState { defListMarker      = "::"
                                  , asciidoctorVariant = False
                                  , inList             = False
                                  , hasMath            = False
+                                 , tableNestingLevel  = 0
                                  }
 
 -- | Convert Pandoc to AsciiDoc.
@@ -194,7 +200,7 @@ blockToAsciiDoc opts (BlockQuote blocks) = do
                      else contents
   let bar = text "____"
   return $ bar $$ chomp contents' $$ bar <> blankline
-blockToAsciiDoc opts (Table _ blkCapt specs thead tbody tfoot) = do
+blockToAsciiDoc opts block@(Table _ blkCapt specs thead tbody tfoot) = do
   let (caption, aligns, widths, headers, rows) =
         toLegacyTable blkCapt specs thead tbody tfoot
   caption' <- inlineListToAsciiDoc opts caption
@@ -236,23 +242,42 @@ blockToAsciiDoc opts (Table _ blkCapt specs thead tbody tfoot) = do
              $ zipWith colspec aligns widths')
          <> text ","
          <> headerspec <> text "]"
+         
+  -- construct cells and recurse in case of nested tables
+  parentTableLevel <- gets tableNestingLevel
+  let currentNestingLevel = parentTableLevel + 1
+  
+  modify $ \st -> st{ tableNestingLevel = currentNestingLevel }
+  
+  let separator = text (if parentTableLevel == 0
+                          then "|"  -- top level separator
+                          else "!") -- nested separator
+
   let makeCell [Plain x] = do d <- blockListToAsciiDoc opts [Plain x]
-                              return $ text "|" <> chomp d
+                              return $ separator <> chomp d
       makeCell [Para x]  = makeCell [Plain x]
-      makeCell []        = return $ text "|"
-      makeCell bs        = do d <- blockListToAsciiDoc opts bs
-                              return $ text "a|" $$ d
+      makeCell []        = return separator
+      makeCell bs        = if currentNestingLevel == 2
+                             then do
+                               --asciidoc only supports nesting once
+                               report $ BlockNotRendered block
+                               return separator
+                             else do
+                               d <- blockListToAsciiDoc opts bs
+                               return $ (text "a" <> separator) $$ d
+
   let makeRow cells = hsep `fmap` mapM makeCell cells
   rows' <- mapM makeRow rows
   head' <- makeRow headers
+  modify $ \st -> st{ tableNestingLevel = parentTableLevel }
   let head'' = if all null headers then empty else head'
   let colwidth = if writerWrapText opts == WrapAuto
                     then writerColumns opts
                     else 100000
   let maxwidth = maximum $ map offset (head':rows')
   let body = if maxwidth > colwidth then vsep rows' else vcat rows'
-  let border = text "|==="
-  return $
+  let border = separator <> text "==="
+  return $ 
     caption'' $$ tablespec $$ border $$ head'' $$ body $$ border $$ blankline
 blockToAsciiDoc opts (BulletList items) = do
   inlist <- gets inList
diff --git a/test/command/nested-table-to-asciidoc-6942.md b/test/command/nested-table-to-asciidoc-6942.md
new file mode 100644
index 000000000..baf11fdf7
--- /dev/null
+++ b/test/command/nested-table-to-asciidoc-6942.md
@@ -0,0 +1,82 @@
+A table within a table should be convertet into a table within table
+
+```
+% pandoc -f html -t asciidoc
+<!doctype html>
+<html>
+<head>
+<meta charset="utf-8">
+<title> NestedTables </title>
+</head>
+<body>
+<table>
+ <tr>
+  <td >
+   <table>  <tr> <td> a1 </td> <td> a2 </td> </tr>  </table>
+  </td>
+  <td>b</td>
+ </tr>
+ <tr>
+   <td>c</td> <td>d </td>
+ </tr>
+</table>
+</body>
+</html>
+^D
+[width="100%",cols="50%,50%",]
+|===
+a|
+[cols=",",]
+!===
+!a1 !a2
+!===
+
+|b
+|c |d
+|===
+```
+
+A table within a table within a table cannot be converted because asciidoc only
+supports two levels of tables.
+The table on level 3 is thus converted to level 2 and a warning is produced
+```
+% pandoc -f html -t asciidoc --verbose
+<!doctype html>
+<html>
+<head>
+<meta charset="utf-8">
+<title> NestedTables </title>
+</head>
+<body>
+<table>
+ <tr>
+  <td>
+    <table>  <tr>
+	<td> a1 </td>
+	<td>
+	  <table>  <tr> <td> 1 </td> <td> 2 </td> </tr>  </table>
+	</td>
+    </tr>  </table>
+  </td>
+  <td>b</td>
+ </tr>
+ <tr>
+   <td>c</td> <td>d </td>
+ </tr>
+</table>
+</body>
+</html>
+^D
+[INFO] Not rendering Table ("",[],[]) (Caption Nothing []) [(AlignDefault,ColWidth 0.5),(AlignDefault,ColWidth 0.5)] (TableHead ("",[],[]) []) [TableBody ("",[],[]) (RowHeadColumns 0) [] [Row ("",[],[]) [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) [Plain [Str "a1"]],Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) [Table ("",[],[]) (Caption Nothing []) [(AlignDefault,ColWidthDefault),(AlignDefault,ColWidthDefault)] (TableHead ("",[],[]) []) [TableBody ("",[],[]) (RowHeadColumns 0) [] [Row ("",[],[]) [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) [Plain [Str "1"]],Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1) [Plain [Str "2"]]]]] (TableFoot ("",[],[]) [])]]]] (TableFoot ("",[],[]) [])
+[width="100%",cols="50%,50%",]
+|===
+a|
+[width="100%",cols="50%,50%",]
+!===
+!a1 !
+!===
+
+|b
+|c |d
+|===
+```
-- 
cgit v1.2.3


From a7a162ea556c0a17062a4784995c7bb42fd88d77 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Mon, 28 Dec 2020 14:40:03 -0800
Subject: Update test for new citeproc and require it in cabal.

---
 pandoc.cabal                                        | 2 +-
 test/command/pandoc-citeproc-chicago-author-date.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'test/command')

diff --git a/pandoc.cabal b/pandoc.cabal
index f3ae5b74e..79ca7dfc9 100644
--- a/pandoc.cabal
+++ b/pandoc.cabal
@@ -405,7 +405,7 @@ library
                  blaze-markup          >= 0.8      && < 0.9,
                  bytestring            >= 0.9      && < 0.12,
                  case-insensitive      >= 1.2      && < 1.3,
-                 citeproc              >= 0.3.0.1  && < 0.4,
+                 citeproc              >= 0.3.0.2  && < 0.4,
                  commonmark            >= 0.1.1.2  && < 0.2,
                  commonmark-extensions >= 0.2.0.4  && < 0.3,
                  commonmark-pandoc     >= 0.2      && < 0.3,
diff --git a/test/command/pandoc-citeproc-chicago-author-date.md b/test/command/pandoc-citeproc-chicago-author-date.md
index ad49d2ca3..a70bc5de8 100644
--- a/test/command/pandoc-citeproc-chicago-author-date.md
+++ b/test/command/pandoc-citeproc-chicago-author-date.md
@@ -60,7 +60,7 @@ References {#references .unnumbered}
 
 [Doe](#ref-item1) ([2005, 30](#ref-item1), with suffix) says blah.
 
-[Doe](#ref-item1) ([2005](#ref-item1), [2006, 30](#ref-item2); see also
+[Doe](#ref-item1) ([2005](#ref-item1); [2006, 30](#ref-item2); see also
 [Doe and Roe 2007](#ref-пункт3)) says blah.
 
 In a note.[^1]
-- 
cgit v1.2.3


From 23f964b9076f9795b00e740d20feebeb1d6d91db Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Wed, 30 Dec 2020 21:31:28 -0800
Subject: Mediawiki reader: allow space around storng/emph delimiters.

Closes #6993.
---
 src/Text/Pandoc/Readers/MediaWiki.hs | 10 ++++------
 test/command/6993.md                 | 21 +++++++++++++++++++++
 2 files changed, 25 insertions(+), 6 deletions(-)
 create mode 100644 test/command/6993.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/MediaWiki.hs b/src/Text/Pandoc/Readers/MediaWiki.hs
index e8985ab2c..d712b1120 100644
--- a/src/Text/Pandoc/Readers/MediaWiki.hs
+++ b/src/Text/Pandoc/Readers/MediaWiki.hs
@@ -678,19 +678,17 @@ url = do
 -- | Parses a list of inlines between start and end delimiters.
 inlinesBetween :: (PandocMonad m, Show b) => MWParser m a -> MWParser m b -> MWParser m Inlines
 inlinesBetween start end =
-  trimInlines . mconcat <$> try (start >> many1Till inner end)
-    where inner      = innerSpace <|> (notFollowedBy' (() <$ whitespace) >> inline)
-          innerSpace = try $ whitespace <* notFollowedBy' end
+  trimInlines . mconcat <$> try (start >> many1Till inline end)
 
 emph :: PandocMonad m => MWParser m Inlines
 emph = B.emph <$> nested (inlinesBetween start end)
-    where start = sym "''" >> lookAhead nonspaceChar
+    where start = sym "''"
           end   = try $ notFollowedBy' (() <$ strong) >> sym "''"
 
 strong :: PandocMonad m => MWParser m Inlines
 strong = B.strong <$> nested (inlinesBetween start end)
-    where start = sym "'''" >> lookAhead nonspaceChar
-          end   = try $ sym "'''"
+    where start = sym "'''"
+          end   = sym "'''"
 
 doubleQuotes :: PandocMonad m => MWParser m Inlines
 doubleQuotes = do
diff --git a/test/command/6993.md b/test/command/6993.md
new file mode 100644
index 000000000..b7f7f5384
--- /dev/null
+++ b/test/command/6993.md
@@ -0,0 +1,21 @@
+```
+% pandoc -f mediawiki -t native
+'''Should be bold '''
+^D
+[Para [Strong [Str "Should",Space,Str "be",Space,Str "bold"]]]
+```
+
+```
+% pandoc -f mediawiki -t native
+''' Should be bold'''
+^D
+[Para [Strong [Str "Should",Space,Str "be",Space,Str "bold"]]]
+```
+
+```
+% pandoc -f mediawiki -t native
+'' Should be emph ''
+^D
+[Para [Emph [Str "Should",Space,Str "be",Space,Str "emph"]]]
+```
+
-- 
cgit v1.2.3


From 9a18cf4b591364e7f0dc9b976628e91f8b31a6cf Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sat, 2 Jan 2021 07:56:07 -0800
Subject: LaTeX writer: revert table line height increase in 2.11.3.

In 2.11.3 we started adding `\addlinespace`, which produced less
dense tables.  This wasn't an intentional change; I misunderstood
a comment in the discussion leading up to the change. This commit
restores the earlier default table appearance.

Note that if you want a less dense table, you can use something like
`\def\arraystretch{1.5}` in your header.

Closes #6996.
---
 src/Text/Pandoc/Writers/LaTeX/Table.hs |  2 +-
 test/command/2378.md                   |  6 ++--
 test/command/5367.md                   |  6 ++--
 test/tables.latex                      | 61 +++++++++++++++-------------------
 test/tables/nordics.latex              | 18 +++++-----
 test/tables/planets.latex              | 28 ++++++----------
 test/tables/students.latex             | 22 ++++++------
 7 files changed, 63 insertions(+), 80 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/LaTeX/Table.hs b/src/Text/Pandoc/Writers/LaTeX/Table.hs
index 9dd66c8a3..8411d9f80 100644
--- a/src/Text/Pandoc/Writers/LaTeX/Table.hs
+++ b/src/Text/Pandoc/Writers/LaTeX/Table.hs
@@ -169,7 +169,7 @@ rowToLaTeX :: PandocMonad m
            -> LW m (Doc Text)
 rowToLaTeX blocksWriter celltype row = do
   cellsDocs <- mapM (cellToLaTeX blocksWriter celltype) (fillRow row)
-  return $ hsep (intersperse "&" cellsDocs) <> " \\\\ \\addlinespace"
+  return $ hsep (intersperse "&" cellsDocs) <> " \\\\"
 
 -- | Pads row with empty cells to adjust for rowspans above this row.
 fillRow :: [Ann.Cell] -> [Ann.Cell]
diff --git a/test/command/2378.md b/test/command/2378.md
index e81855367..59359c653 100644
--- a/test/command/2378.md
+++ b/test/command/2378.md
@@ -14,14 +14,14 @@ is used.
 \begin{longtable}[]{@{}ll@{}}
 \caption{a table}\tabularnewline
 \toprule
-x & y\footnote{a footnote} \\ \addlinespace
+x & y\footnote{a footnote} \\
 \midrule
 \endfirsthead
 \toprule
-x & y{} \\ \addlinespace
+x & y{} \\
 \midrule
 \endhead
-1 & 2 \\ \addlinespace
+1 & 2 \\
 \bottomrule
 \end{longtable}
 ```
diff --git a/test/command/5367.md b/test/command/5367.md
index 4dbcca771..2d3a5e52e 100644
--- a/test/command/5367.md
+++ b/test/command/5367.md
@@ -24,14 +24,14 @@ hello\footnote{doc footnote}
   >{\centering\arraybackslash}p{(\columnwidth - 0\tabcolsep) * \real{0.17}}@{}}
 \caption[Sample table.]{Sample table.\footnote{caption footnote}}\tabularnewline
 \toprule
-Fruit\footnote{header footnote} \\ \addlinespace
+Fruit\footnote{header footnote} \\
 \midrule
 \endfirsthead
 \toprule
-Fruit{} \\ \addlinespace
+Fruit{} \\
 \midrule
 \endhead
-Bans\footnote{table cell footnote} \\ \addlinespace
+Bans\footnote{table cell footnote} \\
 \bottomrule
 \end{longtable}
 
diff --git a/test/tables.latex b/test/tables.latex
index 38c571ba6..afa14d845 100644
--- a/test/tables.latex
+++ b/test/tables.latex
@@ -3,16 +3,16 @@ Simple table with caption:
 \begin{longtable}[]{@{}rlcl@{}}
 \caption{Demonstration of simple table syntax.}\tabularnewline
 \toprule
-Right & Left & Center & Default \\ \addlinespace
+Right & Left & Center & Default \\
 \midrule
 \endfirsthead
 \toprule
-Right & Left & Center & Default \\ \addlinespace
+Right & Left & Center & Default \\
 \midrule
 \endhead
-12 & 12 & 12 & 12 \\ \addlinespace
-123 & 123 & 123 & 123 \\ \addlinespace
-1 & 1 & 1 & 1 \\ \addlinespace
+12 & 12 & 12 & 12 \\
+123 & 123 & 123 & 123 \\
+1 & 1 & 1 & 1 \\
 \bottomrule
 \end{longtable}
 
@@ -20,12 +20,12 @@ Simple table without caption:
 
 \begin{longtable}[]{@{}rlcl@{}}
 \toprule
-Right & Left & Center & Default \\ \addlinespace
+Right & Left & Center & Default \\
 \midrule
 \endhead
-12 & 12 & 12 & 12 \\ \addlinespace
-123 & 123 & 123 & 123 \\ \addlinespace
-1 & 1 & 1 & 1 \\ \addlinespace
+12 & 12 & 12 & 12 \\
+123 & 123 & 123 & 123 \\
+1 & 1 & 1 & 1 \\
 \bottomrule
 \end{longtable}
 
@@ -34,16 +34,16 @@ Simple table indented two spaces:
 \begin{longtable}[]{@{}rlcl@{}}
 \caption{Demonstration of simple table syntax.}\tabularnewline
 \toprule
-Right & Left & Center & Default \\ \addlinespace
+Right & Left & Center & Default \\
 \midrule
 \endfirsthead
 \toprule
-Right & Left & Center & Default \\ \addlinespace
+Right & Left & Center & Default \\
 \midrule
 \endhead
-12 & 12 & 12 & 12 \\ \addlinespace
-123 & 123 & 123 & 123 \\ \addlinespace
-1 & 1 & 1 & 1 \\ \addlinespace
+12 & 12 & 12 & 12 \\
+123 & 123 & 123 & 123 \\
+1 & 1 & 1 & 1 \\
 \bottomrule
 \end{longtable}
 
@@ -56,19 +56,15 @@ Multiline table with caption:
   >{\raggedright\arraybackslash}p{(\columnwidth - 6\tabcolsep) * \real{0.35}}@{}}
 \caption{Here's the caption. It may span multiple lines.}\tabularnewline
 \toprule
-Centered Header & Left Aligned & Right Aligned & Default
-aligned \\ \addlinespace
+Centered Header & Left Aligned & Right Aligned & Default aligned \\
 \midrule
 \endfirsthead
 \toprule
-Centered Header & Left Aligned & Right Aligned & Default
-aligned \\ \addlinespace
+Centered Header & Left Aligned & Right Aligned & Default aligned \\
 \midrule
 \endhead
-First & row & 12.0 & Example of a row that spans multiple
-lines. \\ \addlinespace
-Second & row & 5.0 & Here's another one. Note the blank line between
-rows. \\ \addlinespace
+First & row & 12.0 & Example of a row that spans multiple lines. \\
+Second & row & 5.0 & Here's another one. Note the blank line between rows. \\
 \bottomrule
 \end{longtable}
 
@@ -80,14 +76,11 @@ Multiline table without caption:
   >{\raggedleft\arraybackslash}p{(\columnwidth - 6\tabcolsep) * \real{0.16}}
   >{\raggedright\arraybackslash}p{(\columnwidth - 6\tabcolsep) * \real{0.35}}@{}}
 \toprule
-Centered Header & Left Aligned & Right Aligned & Default
-aligned \\ \addlinespace
+Centered Header & Left Aligned & Right Aligned & Default aligned \\
 \midrule
 \endhead
-First & row & 12.0 & Example of a row that spans multiple
-lines. \\ \addlinespace
-Second & row & 5.0 & Here's another one. Note the blank line between
-rows. \\ \addlinespace
+First & row & 12.0 & Example of a row that spans multiple lines. \\
+Second & row & 5.0 & Here's another one. Note the blank line between rows. \\
 \bottomrule
 \end{longtable}
 
@@ -96,9 +89,9 @@ Table without column headers:
 \begin{longtable}[]{@{}rlcr@{}}
 \toprule
 \endhead
-12 & 12 & 12 & 12 \\ \addlinespace
-123 & 123 & 123 & 123 \\ \addlinespace
-1 & 1 & 1 & 1 \\ \addlinespace
+12 & 12 & 12 & 12 \\
+123 & 123 & 123 & 123 \\
+1 & 1 & 1 & 1 \\
 \bottomrule
 \end{longtable}
 
@@ -111,9 +104,7 @@ Multiline table without column headers:
   >{\raggedright\arraybackslash}p{(\columnwidth - 6\tabcolsep) * \real{0.35}}@{}}
 \toprule
 \endhead
-First & row & 12.0 & Example of a row that spans multiple
-lines. \\ \addlinespace
-Second & row & 5.0 & Here's another one. Note the blank line between
-rows. \\ \addlinespace
+First & row & 12.0 & Example of a row that spans multiple lines. \\
+Second & row & 5.0 & Here's another one. Note the blank line between rows. \\
 \bottomrule
 \end{longtable}
diff --git a/test/tables/nordics.latex b/test/tables/nordics.latex
index d1a0b4aed..1b5929bad 100644
--- a/test/tables/nordics.latex
+++ b/test/tables/nordics.latex
@@ -6,21 +6,19 @@
 \caption{States belonging to the \emph{Nordics.}}\tabularnewline
 \toprule
 Name & Capital & \vtop{\hbox{\strut Population}\hbox{\strut (in 2018)}} &
-\vtop{\hbox{\strut Area}\hbox{\strut (in
-km\textsuperscript{2})}} \\ \addlinespace
+\vtop{\hbox{\strut Area}\hbox{\strut (in km\textsuperscript{2})}} \\
 \midrule
 \endfirsthead
 \toprule
 Name & Capital & \vtop{\hbox{\strut Population}\hbox{\strut (in 2018)}} &
-\vtop{\hbox{\strut Area}\hbox{\strut (in
-km\textsuperscript{2})}} \\ \addlinespace
+\vtop{\hbox{\strut Area}\hbox{\strut (in km\textsuperscript{2})}} \\
 \midrule
 \endhead
-Denmark & Copenhagen & 5,809,502 & 43,094 \\ \addlinespace
-Finland & Helsinki & 5,537,364 & 338,145 \\ \addlinespace
-Iceland & Reykjavik & 343,518 & 103,000 \\ \addlinespace
-Norway & Oslo & 5,372,191 & 323,802 \\ \addlinespace
-Sweden & Stockholm & 10,313,447 & 450,295 \\ \addlinespace
-Total & & 27,376,022 & 1,258,336 \\ \addlinespace
+Denmark & Copenhagen & 5,809,502 & 43,094 \\
+Finland & Helsinki & 5,537,364 & 338,145 \\
+Iceland & Reykjavik & 343,518 & 103,000 \\
+Norway & Oslo & 5,372,191 & 323,802 \\
+Sweden & Stockholm & 10,313,447 & 450,295 \\
+Total & & 27,376,022 & 1,258,336 \\
 \bottomrule
 \end{longtable}
diff --git a/test/tables/planets.latex b/test/tables/planets.latex
index 9457b6821..8238c43f3 100644
--- a/test/tables/planets.latex
+++ b/test/tables/planets.latex
@@ -3,34 +3,28 @@
 \toprule
 \multicolumn{2}{l}{} & Name & Mass (10\^{}24kg) & Diameter (km) & Density
 (kg/m\^{}3) & Gravity (m/s\^{}2) & Length of day (hours) & Distance from Sun
-(10\^{}6km) & Mean temperature (C) & Number of moons & Notes \\ \addlinespace
+(10\^{}6km) & Mean temperature (C) & Number of moons & Notes \\
 \midrule
 \endfirsthead
 \toprule
 \multicolumn{2}{l}{} & Name & Mass (10\^{}24kg) & Diameter (km) & Density
 (kg/m\^{}3) & Gravity (m/s\^{}2) & Length of day (hours) & Distance from Sun
-(10\^{}6km) & Mean temperature (C) & Number of moons & Notes \\ \addlinespace
+(10\^{}6km) & Mean temperature (C) & Number of moons & Notes \\
 \midrule
 \endhead
 \multicolumn{2}{l}{\multirow{4}{*}{Terrestrial planets}} & Mercury & 0.330 &
-4,879 & 5427 & 3.7 & 4222.6 & 57.9 & 167 & 0 & Closest to the
-Sun \\ \addlinespace
-& & Venus & 4.87 & 12,104 & 5243 & 8.9 & 2802.0 & 108.2 & 464 & 0
-& \\ \addlinespace
-& & Earth & 5.97 & 12,756 & 5514 & 9.8 & 24.0 & 149.6 & 15 & 1 & Our
-world \\ \addlinespace
+4,879 & 5427 & 3.7 & 4222.6 & 57.9 & 167 & 0 & Closest to the Sun \\
+& & Venus & 4.87 & 12,104 & 5243 & 8.9 & 2802.0 & 108.2 & 464 & 0 & \\
+& & Earth & 5.97 & 12,756 & 5514 & 9.8 & 24.0 & 149.6 & 15 & 1 & Our world \\
 & & Mars & 0.642 & 6,792 & 3933 & 3.7 & 24.7 & 227.9 & -65 & 2 & The red
-planet \\ \addlinespace
+planet \\
 \multirow{4}{*}{Jovian planets} & \multirow{2}{*}{Gas giants} & Jupiter & 1898
-& 142,984 & 1326 & 23.1 & 9.9 & 778.6 & -110 & 67 & The largest
-planet \\ \addlinespace
-& & Saturn & 568 & 120,536 & 687 & 9.0 & 10.7 & 1433.5 & -140 & 62
-& \\ \addlinespace
+& 142,984 & 1326 & 23.1 & 9.9 & 778.6 & -110 & 67 & The largest planet \\
+& & Saturn & 568 & 120,536 & 687 & 9.0 & 10.7 & 1433.5 & -140 & 62 & \\
 & \multirow{2}{*}{Ice giants} & Uranus & 86.8 & 51,118 & 1271 & 8.7 & 17.2 &
-2872.5 & -195 & 27 & \\ \addlinespace
-& & Neptune & 102 & 49,528 & 1638 & 11.0 & 16.1 & 4495.1 & -200 & 14
-& \\ \addlinespace
+2872.5 & -195 & 27 & \\
+& & Neptune & 102 & 49,528 & 1638 & 11.0 & 16.1 & 4495.1 & -200 & 14 & \\
 \multicolumn{2}{l}{Dwarf planets} & Pluto & 0.0146 & 2,370 & 2095 & 0.7 &
-153.3 & 5906.4 & -225 & 5 & Declassified as a planet in 2006. \\ \addlinespace
+153.3 & 5906.4 & -225 & 5 & Declassified as a planet in 2006. \\
 \bottomrule
 \end{longtable}
diff --git a/test/tables/students.latex b/test/tables/students.latex
index 680a539d3..87efb0851 100644
--- a/test/tables/students.latex
+++ b/test/tables/students.latex
@@ -3,21 +3,21 @@
   >{\raggedright\arraybackslash}p{(\columnwidth - 2\tabcolsep) * \real{0.50}}@{}}
 \caption{List of Students}\tabularnewline
 \toprule
-Student ID & Name \\ \addlinespace
+Student ID & Name \\
 \midrule
 \endfirsthead
 \toprule
-Student ID & Name \\ \addlinespace
+Student ID & Name \\
 \midrule
 \endhead
-\multicolumn{2}{l}{Computer Science} \\ \addlinespace
-3741255 & Jones, Martha \\ \addlinespace
-4077830 & Pierce, Benjamin \\ \addlinespace
-5151701 & Kirk, James \\ \addlinespace
-\multicolumn{2}{l}{Russian Literature} \\ \addlinespace
-3971244 & Nim, Victor \\ \addlinespace
-\multicolumn{2}{l}{Astrophysics} \\ \addlinespace
-4100332 & Petrov, Alexandra \\ \addlinespace
-4100332 & Toyota, Hiroko \\ \addlinespace
+\multicolumn{2}{l}{Computer Science} \\
+3741255 & Jones, Martha \\
+4077830 & Pierce, Benjamin \\
+5151701 & Kirk, James \\
+\multicolumn{2}{l}{Russian Literature} \\
+3971244 & Nim, Victor \\
+\multicolumn{2}{l}{Astrophysics} \\
+4100332 & Petrov, Alexandra \\
+4100332 & Toyota, Hiroko \\
 \bottomrule
 \end{longtable}
-- 
cgit v1.2.3


From ea479bf28a4031f408af12ea92d3e19f9a838820 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Mon, 4 Jan 2021 14:05:03 -0800
Subject: LaTeX reader: handle filecontents environment.

Closes #7003.
---
 src/Text/Pandoc/Readers/LaTeX.hs         | 32 +++++++++++++++++++++------
 src/Text/Pandoc/Readers/LaTeX/Parsing.hs |  2 ++
 test/command/7003.md                     | 37 ++++++++++++++++++++++++++++++++
 3 files changed, 65 insertions(+), 6 deletions(-)
 create mode 100644 test/command/7003.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/LaTeX.hs b/src/Text/Pandoc/Readers/LaTeX.hs
index d9fe0f502..14a41a911 100644
--- a/src/Text/Pandoc/Readers/LaTeX.hs
+++ b/src/Text/Pandoc/Readers/LaTeX.hs
@@ -1219,6 +1219,7 @@ preamble :: PandocMonad m => LP m Blocks
 preamble = mconcat <$> many preambleBlock
   where preambleBlock =  (mempty <$ spaces1)
                      <|> macroDef (rawBlock "latex")
+                     <|> filecontents
                      <|> (mempty <$ blockCommand)
                      <|> (mempty <$ braced)
                      <|> (do notFollowedBy (begin_ "document")
@@ -1272,6 +1273,16 @@ include name = do
   mapM_ (insertIncluded defaultExt) fs
   return mempty
 
+readFileFromTexinputs :: PandocMonad m => FilePath -> LP m (Maybe Text)
+readFileFromTexinputs fp = do
+  fileContentsMap <- sFileContents <$> getState
+  case M.lookup (T.pack fp) fileContentsMap of
+    Just t -> return (Just t)
+    Nothing -> do
+      dirs <- map T.unpack . splitTextBy (==':') . fromMaybe "."
+               <$> lookupEnv "TEXINPUTS"
+      readFileFromDirs dirs fp
+
 insertIncluded :: PandocMonad m
                => FilePath
                -> FilePath
@@ -1281,13 +1292,12 @@ insertIncluded defaultExtension f' = do
                 ".tex" -> f'
                 ".sty" -> f'
                 _      -> addExtension f' defaultExtension
-  dirs <- map T.unpack . splitTextBy (==':') . fromMaybe "." <$> lookupEnv "TEXINPUTS"
   pos <- getPosition
   containers <- getIncludeFiles <$> getState
   when (T.pack f `elem` containers) $
     throwError $ PandocParseError $ T.pack $ "Include file loop at " ++ show pos
   updateState $ addIncludeFile $ T.pack f
-  mbcontents <- readFileFromDirs dirs f
+  mbcontents <- readFileFromTexinputs f
   contents <- case mbcontents of
                    Just s -> return s
                    Nothing -> do
@@ -1695,6 +1705,18 @@ environments = M.fromList
    , ("iftoggle", try $ ifToggle >> block)
    ]
 
+filecontents :: PandocMonad m => LP m Blocks
+filecontents = try $ do
+  controlSeq "begin"
+  name <- untokenize <$> braced
+  guard $ name == "filecontents" || name == "filecontents*"
+  skipopts
+  fp <- untokenize <$> braced
+  txt <- verbEnv name
+  updateState $ \st ->
+    st{ sFileContents = M.insert fp txt (sFileContents st) }
+  return mempty
+
 theoremstyle :: PandocMonad m => LP m Blocks
 theoremstyle = do
   stylename <- untokenize <$> braced
@@ -1894,8 +1916,7 @@ inputMinted = do
   pos <- getPosition
   attr <- mintedAttr
   f <- T.filter (/='"') . untokenize <$> braced
-  dirs <- map T.unpack . splitTextBy (==':') . fromMaybe "." <$> lookupEnv "TEXINPUTS"
-  mbCode <- readFileFromDirs dirs (T.unpack f)
+  mbCode <- readFileFromTexinputs (T.unpack f)
   rawcode <- case mbCode of
                   Just s -> return s
                   Nothing -> do
@@ -1981,8 +2002,7 @@ inputListing = do
   pos <- getPosition
   options <- option [] keyvals
   f <- T.filter (/='"') . untokenize <$> braced
-  dirs <- map T.unpack . splitTextBy (==':') . fromMaybe "." <$> lookupEnv "TEXINPUTS"
-  mbCode <- readFileFromDirs dirs (T.unpack f)
+  mbCode <- readFileFromTexinputs (T.unpack f)
   codeLines <- case mbCode of
                       Just s -> return $ T.lines s
                       Nothing -> do
diff --git a/src/Text/Pandoc/Readers/LaTeX/Parsing.hs b/src/Text/Pandoc/Readers/LaTeX/Parsing.hs
index 563d32883..e92ed387c 100644
--- a/src/Text/Pandoc/Readers/LaTeX/Parsing.hs
+++ b/src/Text/Pandoc/Readers/LaTeX/Parsing.hs
@@ -152,6 +152,7 @@ data LaTeXState = LaTeXState{ sOptions       :: ReaderOptions
                             , sHasChapters   :: Bool
                             , sToggles       :: M.Map Text Bool
                             , sExpanded      :: Bool
+                            , sFileContents  :: M.Map Text Text
                             }
      deriving Show
 
@@ -177,6 +178,7 @@ defaultLaTeXState = LaTeXState{ sOptions       = def
                               , sHasChapters   = False
                               , sToggles       = M.empty
                               , sExpanded      = False
+                              , sFileContents  = M.empty
                               }
 
 instance PandocMonad m => HasQuoteContext LaTeXState m where
diff --git a/test/command/7003.md b/test/command/7003.md
new file mode 100644
index 000000000..1f6275943
--- /dev/null
+++ b/test/command/7003.md
@@ -0,0 +1,37 @@
+```
+% pandoc -f latex -t native
+\documentclass{article}
+\usepackage{listings}
+
+\lstset{basicstyle=\ttfamily}
+
+\begin{filecontents*}[overwrite]{example.tex}
+\documentclass{article}
+\begin{document}
+\section{Bar}
+This a Bar section
+\end{document}
+\end{filecontents*}
+
+\begin{document}
+
+\section{With lstlisting environment}
+
+\begin{lstlisting}
+\documentclass{article}
+\begin{document}
+\section{Foo}
+This a Foo section
+\end{document}
+\end{lstlisting}
+
+\section{With lstinputlisting command}
+
+\lstinputlisting{example.tex}
+\end{document}
+^D
+[Header 1 ("with-lstlisting-environment",[],[]) [Str "With",Space,Str "lstlisting",Space,Str "environment"]
+,CodeBlock ("",[],[]) "\\documentclass{article}\n\\begin{document}\n\\section{Foo}\nThis a Foo section\n\\end{document}"
+,Header 1 ("with-lstinputlisting-command",[],[]) [Str "With",Space,Str "lstinputlisting",Space,Str "command"]
+,CodeBlock ("",["latex"],[]) "\\documentclass{article}\n\\begin{document}\n\\section{Bar}\nThis a Bar section\n\\end{document}"]
+```
-- 
cgit v1.2.3


From 385b6a3b215124fd2dfa044b8847d69a6cf14a73 Mon Sep 17 00:00:00 2001
From: David Martschenko <62178322+davidmrt98@users.noreply.github.com>
Date: Tue, 5 Jan 2021 19:15:59 +0100
Subject: Implement defaults file inheritance (#6924)

Allow defaults files to inherit options from other defaults files by
specifying them with the following syntax:
`defaults: [list of defaults files or single defaults file]`.
---
 MANUAL.txt                                |  10 +++
 pandoc.cabal                              |   7 ++
 src/Text/Pandoc/App/CommandLineOptions.hs |  36 +++-----
 src/Text/Pandoc/App/Opt.hs                | 136 ++++++++++++++++++++++++++++--
 test/command/defaults-inheritance-1.md    |   6 ++
 test/command/defaults-inheritance-2.md    |   5 ++
 test/command/defaults-inheritance-3.md    |   6 ++
 test/command/defaults3.yaml               |   4 +
 test/command/defaults4.yaml               |   3 +
 test/command/defaults5.yaml               |   2 +
 test/command/defaults6.yaml               |   2 +
 test/command/defaults7.yaml               |   2 +
 test/command/defaults8.yaml               |   2 +
 test/command/defaults9.yaml               |   1 +
 14 files changed, 189 insertions(+), 33 deletions(-)
 create mode 100644 test/command/defaults-inheritance-1.md
 create mode 100644 test/command/defaults-inheritance-2.md
 create mode 100644 test/command/defaults-inheritance-3.md
 create mode 100644 test/command/defaults3.yaml
 create mode 100644 test/command/defaults4.yaml
 create mode 100644 test/command/defaults5.yaml
 create mode 100644 test/command/defaults6.yaml
 create mode 100644 test/command/defaults7.yaml
 create mode 100644 test/command/defaults8.yaml
 create mode 100644 test/command/defaults9.yaml

(limited to 'test/command')

diff --git a/MANUAL.txt b/MANUAL.txt
index 53cd54e5f..831ffb0b0 100644
--- a/MANUAL.txt
+++ b/MANUAL.txt
@@ -1507,6 +1507,16 @@ input-files:
 - content.md
 # or you may use input-file: with a single value
 
+# Include options from the specified defaults files.
+# The files will be searched for first in the working directory
+# and then in the defaults subdirectory of the user data directory.
+# The files are included in the same order in which they appear in
+# the list. Options specified in this defaults file always have
+# priority over the included ones.
+defaults:
+- defsA
+- defsB
+
 template: letter
 standalone: true
 self-contained: false
diff --git a/pandoc.cabal b/pandoc.cabal
index 794eef91d..cce3c1a58 100644
--- a/pandoc.cabal
+++ b/pandoc.cabal
@@ -216,6 +216,13 @@ extra-source-files:
                  test/command/01.csv
                  test/command/defaults1.yaml
                  test/command/defaults2.yaml
+                 test/command/defaults3.yaml
+                 test/command/defaults4.yaml
+                 test/command/defaults5.yaml
+                 test/command/defaults6.yaml
+                 test/command/defaults7.yaml
+                 test/command/defaults8.yaml
+                 test/command/defaults9.yaml
                  test/command/3533-rst-csv-tables.csv
                  test/command/3880.txt
                  test/command/5182.txt
diff --git a/src/Text/Pandoc/App/CommandLineOptions.hs b/src/Text/Pandoc/App/CommandLineOptions.hs
index 906fcc4c0..21ee47b7b 100644
--- a/src/Text/Pandoc/App/CommandLineOptions.hs
+++ b/src/Text/Pandoc/App/CommandLineOptions.hs
@@ -25,6 +25,7 @@ module Text.Pandoc.App.CommandLineOptions (
 import Control.Monad
 import Control.Monad.Trans
 import Control.Monad.Except (throwError)
+import Control.Monad.State.Strict
 import Data.Aeson.Encode.Pretty (encodePretty', Config(..), keyOrder,
          defConfig, Indent(..), NumberFormat(..))
 import Data.Bifunctor (second)
@@ -46,10 +47,12 @@ import System.FilePath
 import System.IO (stdout)
 import Text.DocTemplates (Context (..), ToContext (toVal), Val (..))
 import Text.Pandoc
-import Text.Pandoc.App.Opt (Opt (..), LineEnding (..), IpynbOutput (..), addMeta)
+import Text.Pandoc.App.Opt (Opt (..), LineEnding (..), IpynbOutput (..),
+                            DefaultsState (..), addMeta, applyDefaults,
+                            fullDefaultsPath)
 import Text.Pandoc.Filter (Filter (..))
 import Text.Pandoc.Highlighting (highlightingStyles)
-import Text.Pandoc.Shared (ordNub, elemText, safeStrRead, defaultUserDataDirs, findM)
+import Text.Pandoc.Shared (ordNub, elemText, safeStrRead, defaultUserDataDirs)
 import Text.Printf
 
 #ifdef EMBED_DATA_FILES
@@ -64,7 +67,6 @@ import qualified Data.ByteString as BS
 import qualified Data.ByteString.Lazy as B
 import qualified Data.Map as M
 import qualified Data.Text as T
-import qualified Data.YAML as Y
 import qualified Text.Pandoc.UTF8 as UTF8
 
 parseOptions :: [OptDescr (Opt -> IO Opt)] -> Opt -> IO Opt
@@ -166,7 +168,11 @@ options =
 
     , Option "d" ["defaults"]
                  (ReqArg
-                  (\arg opt -> applyDefaults opt arg
+                  (\arg opt -> runIOorExplode $ do
+                     let defsState = DefaultsState { curDefaults = Nothing,
+                                                     inheritanceGraph = [] }
+                     fp <- fullDefaultsPath (optDataDir opt) arg
+                     evalStateT (applyDefaults opt fp) defsState
                   )
                   "FILE")
                 ""
@@ -1012,28 +1018,6 @@ writersNames = sort
 splitField :: String -> (String, String)
 splitField = second (tailDef "true") . break (`elemText` ":=")
 
--- | Apply defaults from --defaults file.
-applyDefaults :: Opt -> FilePath -> IO Opt
-applyDefaults opt file = runIOorExplode $ do
-  let fp = if null (takeExtension file)
-              then addExtension file "yaml"
-              else file
-  setVerbosity $ optVerbosity opt
-  dataDirs <- liftIO defaultUserDataDirs
-  let fps = fp : case optDataDir opt of
-              Nothing -> map (</> ("defaults" </> fp))
-                               dataDirs
-              Just dd -> [dd </> "defaults" </> fp]
-  fp' <- fromMaybe fp <$> findM fileExists fps
-  inp <- readFileLazy fp'
-  case Y.decode1 inp of
-      Right (f :: Opt -> Opt) -> return $ f opt
-      Left (errpos, errmsg)  -> throwError $
-         PandocParseError $ T.pack $
-         "Error parsing " ++ fp' ++ " line " ++
-          show (Y.posLine errpos) ++ " column " ++
-          show (Y.posColumn errpos) ++ ":\n" ++ errmsg
-
 lookupHighlightStyle :: PandocMonad m => String -> m Style
 lookupHighlightStyle s
   | takeExtension s == ".theme" = -- attempt to load KDE theme
diff --git a/src/Text/Pandoc/App/Opt.hs b/src/Text/Pandoc/App/Opt.hs
index 00b4b5523..6dd19758e 100644
--- a/src/Text/Pandoc/App/Opt.hs
+++ b/src/Text/Pandoc/App/Opt.hs
@@ -20,10 +20,17 @@ module Text.Pandoc.App.Opt (
             Opt(..)
           , LineEnding (..)
           , IpynbOutput (..)
+          , DefaultsState (..)
           , defaultOpts
           , addMeta
+          , applyDefaults
+          , fullDefaultsPath
           ) where
+import Control.Monad.Except (MonadIO, liftIO, throwError, (>=>), foldM)
+import Control.Monad.State.Strict (StateT, modify, gets)
+import System.FilePath ( addExtension, (</>), takeExtension )
 import Data.Char (isLower, toLower)
+import Data.Maybe (fromMaybe)
 import GHC.Generics hiding (Meta)
 import Text.Pandoc.Builder (setMeta)
 import Text.Pandoc.Filter (Filter (..))
@@ -34,7 +41,9 @@ import Text.Pandoc.Options (TopLevelDivision (TopLevelDefault),
                             ReferenceLocation (EndOfDocument),
                             ObfuscationMethod (NoObfuscation),
                             CiteMethod (Citeproc))
-import Text.Pandoc.Shared (camelCaseStrToHyphenated)
+import Text.Pandoc.Class (readFileLazy, fileExists, setVerbosity, PandocMonad)
+import Text.Pandoc.Error (PandocError (PandocParseError, PandocSomeError))
+import Text.Pandoc.Shared (camelCaseStrToHyphenated, defaultUserDataDirs, findM, ordNub)
 import qualified Text.Pandoc.Parsing as P
 import Text.Pandoc.Readers.Metadata (yamlMap)
 import Text.Pandoc.Class.PandocPure
@@ -150,16 +159,77 @@ data Opt = Opt
     } deriving (Generic, Show)
 
 instance FromYAML (Opt -> Opt) where
-  parseYAML (Mapping _ _ m) =
-    foldr (.) id <$> mapM doOpt (M.toList m)
+  parseYAML (Mapping _ _ m) = chain doOpt (M.toList m)
   parseYAML n = failAtNode n "Expected a mapping"
 
+data DefaultsState = DefaultsState
+    {
+      curDefaults      :: Maybe FilePath -- currently parsed file
+    , inheritanceGraph :: [[FilePath]]   -- defaults file inheritance graph
+    } deriving (Show)
+
+instance (PandocMonad m, MonadIO m)
+      => FromYAML (Opt -> StateT DefaultsState m Opt) where
+  parseYAML (Mapping _ _ m) = do
+    let opts = M.mapKeys toText m
+    dataDir <- case M.lookup "data-dir" opts of
+      Nothing -> return Nothing
+      Just v -> Just . unpack <$> parseYAML v
+    f <- parseOptions $ M.toList m
+    case M.lookup "defaults" opts of
+      Just v -> do
+        g <- parseDefaults v dataDir
+        return  $ g >=> f
+      Nothing -> return f
+    where
+      toText (Scalar _ (SStr s)) = s
+      toText _ = ""
+  parseYAML n = failAtNode n "Expected a mapping"
+
+parseDefaults :: (PandocMonad m, MonadIO m)
+              => Node Pos
+              -> Maybe FilePath
+              -> Parser (Opt -> StateT DefaultsState m Opt)
+parseDefaults n dataDir = parseDefsNames n >>= \ds -> return $ \o -> do
+  -- get parent defaults:
+  defsParent <- gets $ fromMaybe "" . curDefaults
+  -- get child defaults:
+  defsChildren <- mapM (fullDefaultsPath dataDir) ds
+  -- expand parent in defaults inheritance graph by children:
+  defsGraph <- gets inheritanceGraph
+  let defsGraphExp = expand defsGraph defsChildren defsParent
+  modify $ \defsState -> defsState{ inheritanceGraph = defsGraphExp }
+  -- check for cyclic inheritance:
+  if cyclic defsGraphExp
+    then throwError $
+      PandocSomeError $ T.pack $
+        "Error: Circular defaults file reference in " ++
+        "'" ++ defsParent ++ "'"
+    else foldM applyDefaults o defsChildren
+  where parseDefsNames x = (parseYAML x >>= \xs -> return $ map unpack xs)
+                       <|> (parseYAML x >>= \x' -> return [unpack x'])
+
+parseOptions :: Monad m
+             => [(Node Pos, Node Pos)]
+             -> Parser (Opt -> StateT DefaultsState m Opt)
+parseOptions ns = do
+  f <- chain doOpt' ns
+  return $ return . f
+
+chain :: Monad m => (a -> m (b -> b)) -> [a] -> m (b -> b)
+chain f = foldM g id
+  where g o n = f n >>= \o' -> return $ o' . o
+
+doOpt' :: (Node Pos, Node Pos) -> Parser (Opt -> Opt)
+doOpt' (k',v) = do
+  k <- parseStringKey k'
+  case k of
+    "defaults" -> return id
+    _ -> doOpt (k',v)
+
 doOpt :: (Node Pos, Node Pos) -> Parser (Opt -> Opt)
 doOpt (k',v) = do
-  k <- case k' of
-         Scalar _ (SStr t) -> return t
-         Scalar _ _ -> failAtNode k' "Non-string key"
-         _ -> failAtNode k' "Non-scalar key"
+  k <- parseStringKey k'
   case k of
     "tab-stop" ->
       parseYAML v >>= \x -> return (\o -> o{ optTabStop = x })
@@ -494,6 +564,12 @@ defaultOpts = Opt
     , optStripComments         = False
     }
 
+parseStringKey ::  Node Pos -> Parser Text
+parseStringKey k = case k of
+  Scalar _ (SStr t) -> return t
+  Scalar _ _ -> failAtNode k "Non-string key"
+  _ -> failAtNode k "Non-scalar key"
+
 yamlToMeta :: Node Pos -> Parser Meta
 yamlToMeta (Mapping _ _ m) =
     either (fail . show) return $ runEverything (yamlMap pMetaString m)
@@ -524,6 +600,52 @@ readMetaValue s
   | s == "FALSE" = MetaBool False
   | otherwise    = MetaString $ T.pack s
 
+-- | Apply defaults from --defaults file.
+applyDefaults :: (PandocMonad m, MonadIO m)
+              => Opt
+              -> FilePath
+              -> StateT DefaultsState m Opt
+applyDefaults opt file = do
+  setVerbosity $ optVerbosity opt
+  modify $ \defsState -> defsState{ curDefaults = Just file }
+  inp <- readFileLazy file
+  case decode1 inp of
+      Right f -> f opt
+      Left (errpos, errmsg)  -> throwError $
+         PandocParseError $ T.pack $
+         "Error parsing " ++ file ++ " line " ++
+          show (posLine errpos) ++ " column " ++
+          show (posColumn errpos) ++ ":\n" ++ errmsg
+
+fullDefaultsPath :: (PandocMonad m, MonadIO m)
+                 => Maybe FilePath
+                 -> FilePath
+                 -> m FilePath
+fullDefaultsPath dataDir file = do
+  let fp = if null (takeExtension file)
+              then addExtension file "yaml"
+              else file
+  dataDirs <- liftIO defaultUserDataDirs
+  let fps = fp : case dataDir of
+                   Nothing -> map (</> ("defaults" </> fp))
+                                    dataDirs
+                   Just dd -> [dd </> "defaults" </> fp]
+  fromMaybe fp <$> findM fileExists fps
+
+-- | In a list of lists, append another list in front of every list which
+-- starts with specific element.
+expand :: Ord a => [[a]] -> [a] -> a -> [[a]]
+expand [] ns n = fmap (\x -> x : [n]) ns
+expand ps ns n = concatMap (ext n ns) ps
+  where
+    ext x xs p = case p of
+      (l : _) | x == l -> fmap (: p) xs
+      _ -> [p]
+
+cyclic :: Ord a => [[a]] -> Bool
+cyclic = any hasDuplicate
+  where
+    hasDuplicate xs = length (ordNub xs) /= length xs
 
 -- see https://github.com/jgm/pandoc/pull/4083
 -- using generic deriving caused long compilation times
diff --git a/test/command/defaults-inheritance-1.md b/test/command/defaults-inheritance-1.md
new file mode 100644
index 000000000..0760e2ec0
--- /dev/null
+++ b/test/command/defaults-inheritance-1.md
@@ -0,0 +1,6 @@
+```
+% pandoc -d command/defaults3
+# Header
+^D
+# Header
+```
diff --git a/test/command/defaults-inheritance-2.md b/test/command/defaults-inheritance-2.md
new file mode 100644
index 000000000..8b26a2613
--- /dev/null
+++ b/test/command/defaults-inheritance-2.md
@@ -0,0 +1,5 @@
+```
+% pandoc -d command/defaults6
+^D
+Error: Circular defaults file reference in 'command/defaults7.yaml'
+```
diff --git a/test/command/defaults-inheritance-3.md b/test/command/defaults-inheritance-3.md
new file mode 100644
index 000000000..81cac7baa
--- /dev/null
+++ b/test/command/defaults-inheritance-3.md
@@ -0,0 +1,6 @@
+```
+% pandoc -d command/defaults8
+<h1>Header</h1>
+^D
+# Header
+```
diff --git a/test/command/defaults3.yaml b/test/command/defaults3.yaml
new file mode 100644
index 000000000..d8b6f9144
--- /dev/null
+++ b/test/command/defaults3.yaml
@@ -0,0 +1,4 @@
+defaults:
+  - command/defaults4
+  - command/defaults5
+to: markdown
diff --git a/test/command/defaults4.yaml b/test/command/defaults4.yaml
new file mode 100644
index 000000000..a6caa985b
--- /dev/null
+++ b/test/command/defaults4.yaml
@@ -0,0 +1,3 @@
+from: html
+defaults:
+  - command/defaults5
diff --git a/test/command/defaults5.yaml b/test/command/defaults5.yaml
new file mode 100644
index 000000000..bb48b9708
--- /dev/null
+++ b/test/command/defaults5.yaml
@@ -0,0 +1,2 @@
+from: markdown
+to: html
diff --git a/test/command/defaults6.yaml b/test/command/defaults6.yaml
new file mode 100644
index 000000000..ac4a819e5
--- /dev/null
+++ b/test/command/defaults6.yaml
@@ -0,0 +1,2 @@
+defaults:
+  - command/defaults7
diff --git a/test/command/defaults7.yaml b/test/command/defaults7.yaml
new file mode 100644
index 000000000..19ca8f09e
--- /dev/null
+++ b/test/command/defaults7.yaml
@@ -0,0 +1,2 @@
+defaults:
+  - command/defaults6
diff --git a/test/command/defaults8.yaml b/test/command/defaults8.yaml
new file mode 100644
index 000000000..e418d33b2
--- /dev/null
+++ b/test/command/defaults8.yaml
@@ -0,0 +1,2 @@
+from: html
+defaults: command/defaults9
diff --git a/test/command/defaults9.yaml b/test/command/defaults9.yaml
new file mode 100644
index 000000000..d732eb066
--- /dev/null
+++ b/test/command/defaults9.yaml
@@ -0,0 +1 @@
+to: markdown
-- 
cgit v1.2.3


From 15ba184e6e7825898ebae5a59e2c3a60e0992da0 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Tue, 5 Jan 2021 12:07:02 -0800
Subject: HTML writer: fix implicit_figure at end of footnotes.

Closes #7006.
---
 src/Text/Pandoc/Writers/HTML.hs | 10 +++++++---
 test/command/7006.md            | 20 ++++++++++++++++++++
 2 files changed, 27 insertions(+), 3 deletions(-)
 create mode 100644 test/command/7006.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/HTML.hs b/src/Text/Pandoc/Writers/HTML.hs
index c93322953..0b051c8f0 100644
--- a/src/Text/Pandoc/Writers/HTML.hs
+++ b/src/Text/Pandoc/Writers/HTML.hs
@@ -1455,11 +1455,15 @@ blockListToNote opts ref blocks = do
                     else let lastBlock   = last blocks
                              otherBlocks = init blocks
                          in  case lastBlock of
-                                  (Para lst)  -> otherBlocks ++
+                                  Para [Image _ _ (_,tit)]
+                                      | "fig:" `T.isPrefixOf` tit
+                                            -> otherBlocks ++ [lastBlock,
+                                                  Plain backlink]
+                                  Para lst  -> otherBlocks ++
                                                  [Para (lst ++ backlink)]
-                                  (Plain lst) -> otherBlocks ++
+                                  Plain lst -> otherBlocks ++
                                                  [Plain (lst ++ backlink)]
-                                  _           -> otherBlocks ++ [lastBlock,
+                                  _         -> otherBlocks ++ [lastBlock,
                                                  Plain backlink]
   contents <- blockListToHtml opts blocks'
   let noteItem = H.li ! prefixedId opts ("fn" <> ref) $ contents
diff --git a/test/command/7006.md b/test/command/7006.md
new file mode 100644
index 000000000..e7951fb1a
--- /dev/null
+++ b/test/command/7006.md
@@ -0,0 +1,20 @@
+```
+% pandoc -t html
+Test.[^fn]
+
+[^fn]: Foo:
+
+    ![Caption.](/image.jpg)
+^D
+<p>Test.<a href="#fn1" class="footnote-ref" id="fnref1" role="doc-noteref"><sup>1</sup></a></p>
+<section class="footnotes" role="doc-endnotes">
+<hr />
+<ol>
+<li id="fn1" role="doc-endnote"><p>Foo:</p>
+<figure>
+<img src="/image.jpg" alt="Caption." /><figcaption aria-hidden="true">Caption.</figcaption>
+</figure>
+<a href="#fnref1" class="footnote-back" role="doc-backlink">↩︎</a></li>
+</ol>
+</section>
+```
-- 
cgit v1.2.3


From c0d8b186d142eb5e9f845de0a4ebcadd04c32dce Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 7 Jan 2021 11:06:25 -0800
Subject: T.P.Parsing: modify gridTableWith' for headerless tables.

If the table lacks a header, the header row should be an empty
list. Previously we got a list of empty cells, which caused
an empty header to be emitted instead of no header.  In LaTeX/PDF
output that meant we got a double top line with space between.

@tarleb @despres - please let me know if this is problematic
for some reason I'm not grasping.
---
 src/Text/Pandoc/Parsing.hs       | 22 +++++++++++-----------
 test/command/3516.md             |  6 +-----
 test/command/5708.md             |  6 +-----
 test/markdown-reader-more.native | 38 +++++---------------------------------
 test/rst-reader.native           | 24 +++---------------------
 test/tables-rstsubset.native     | 10 +---------
 6 files changed, 22 insertions(+), 84 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Parsing.hs b/src/Text/Pandoc/Parsing.hs
index 979344f63..fd14341ad 100644
--- a/src/Text/Pandoc/Parsing.hs
+++ b/src/Text/Pandoc/Parsing.hs
@@ -1069,24 +1069,24 @@ gridTableHeader :: (Stream s m Char, Monad mf, IsString s, HasLastStrPosition st
                 => Bool -- ^ Headerless table
                 -> ParserT s st m (mf Blocks)
                 -> ParserT s st m (mf [Blocks], [Alignment], [Int])
-gridTableHeader headless blocks = try $ do
+gridTableHeader True _ = do
   optional blanklines
   dashes <- gridDashedLines '-'
-  rawContent  <- if headless
-                    then return $ repeat ""
-                    else many1
-                         (notFollowedBy (gridTableSep '=') >> char '|' >>
+  let aligns = map snd dashes
+  let lines'   = map (snd . fst) dashes
+  let indices  = scanl (+) 0 lines'
+  return (return [], aligns, indices)
+gridTableHeader False blocks = try $ do
+  optional blanklines
+  dashes <- gridDashedLines '-'
+  rawContent  <- many1 (notFollowedBy (gridTableSep '=') >> char '|' >>
                            T.pack <$> many1Till anyChar newline)
-  underDashes <- if headless
-                    then return dashes
-                    else gridDashedLines '='
+  underDashes <- gridDashedLines '='
   guard $ length dashes == length underDashes
   let lines'   = map (snd . fst) underDashes
   let indices  = scanl (+) 0 lines'
   let aligns   = map snd underDashes
-  let rawHeads = if headless
-                    then replicate (length underDashes) ""
-                    else map (T.unlines . map trim) $ transpose
+  let rawHeads = map (T.unlines . map trim) $ transpose
                        $ map (gridTableSplitLine indices) rawContent
   heads <- sequence <$> mapM (parseFromString' blocks . trim) rawHeads
   return (heads, aligns, indices)
diff --git a/test/command/3516.md b/test/command/3516.md
index 1cb805afe..dc8770bad 100644
--- a/test/command/3516.md
+++ b/test/command/3516.md
@@ -29,11 +29,7 @@ on Windows builds.
  [(AlignDefault,ColWidth 5.555555555555555e-2)
  ,(AlignDefault,ColWidth 5.555555555555555e-2)]
  (TableHead ("",[],[])
- [Row ("",[],[])
-  [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []]])
+ [])
  [(TableBody ("",[],[]) (RowHeadColumns 0)
   []
   [Row ("",[],[])
diff --git a/test/command/5708.md b/test/command/5708.md
index e4b99e4db..e5ba72c69 100644
--- a/test/command/5708.md
+++ b/test/command/5708.md
@@ -9,11 +9,7 @@
  [(AlignDefault,ColWidth 0.125)
  ,(AlignDefault,ColWidth 0.2361111111111111)]
  (TableHead ("",[],[])
- [Row ("",[],[])
-  [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []]])
+ [])
  [(TableBody ("",[],[]) (RowHeadColumns 0)
   []
   [Row ("",[],[])
diff --git a/test/markdown-reader-more.native b/test/markdown-reader-more.native
index 5ac4facb8..b4a16f7cf 100644
--- a/test/markdown-reader-more.native
+++ b/test/markdown-reader-more.native
@@ -134,13 +134,7 @@ Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "Author",S
  ,(AlignDefault,ColWidth 0.16666666666666666)
  ,(AlignDefault,ColWidth 0.18055555555555555)]
  (TableHead ("",[],[])
- [Row ("",[],[])
-  [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []]])
+ [])
  [(TableBody ("",[],[]) (RowHeadColumns 0)
   []
   [Row ("",[],[])
@@ -198,13 +192,7 @@ Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "Author",S
  ,(AlignLeft,ColWidth 0.16666666666666666)
  ,(AlignCenter,ColWidth 0.18055555555555555)]
  (TableHead ("",[],[])
- [Row ("",[],[])
-  [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []]])
+ [])
  [(TableBody ("",[],[]) (RowHeadColumns 0)
   []
   [Row ("",[],[])
@@ -230,13 +218,7 @@ Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "Author",S
  ,(AlignDefault,ColWidth 0.16666666666666666)
  ,(AlignDefault,ColWidth 0.18055555555555555)]
  (TableHead ("",[],[])
- [Row ("",[],[])
-  [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []]])
+ [])
  [(TableBody ("",[],[]) (RowHeadColumns 0)
   []
   [Row ("",[],[])
@@ -262,13 +244,7 @@ Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "Author",S
  ,(AlignDefault,ColWidth 0.16666666666666666)
  ,(AlignDefault,ColWidth 0.18055555555555555)]
  (TableHead ("",[],[])
- [Row ("",[],[])
-  [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []]])
+ [])
  [(TableBody ("",[],[]) (RowHeadColumns 0)
   []
   [Row ("",[],[])
@@ -300,11 +276,7 @@ Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "Author",S
  [(AlignDefault,ColWidth 5.555555555555555e-2)
  ,(AlignDefault,ColWidth 5.555555555555555e-2)]
  (TableHead ("",[],[])
- [Row ("",[],[])
-  [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []]])
+ [])
  [(TableBody ("",[],[]) (RowHeadColumns 0)
   []
   [Row ("",[],[])
diff --git a/test/rst-reader.native b/test/rst-reader.native
index 93855cc00..9f8d5ab3e 100644
--- a/test/rst-reader.native
+++ b/test/rst-reader.native
@@ -346,13 +346,7 @@ Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "John",Spa
  ,(AlignDefault,ColWidth 0.15)
  ,(AlignDefault,ColWidth 0.1625)]
  (TableHead ("",[],[])
- [Row ("",[],[])
-  [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []]])
+ [])
  [(TableBody ("",[],[]) (RowHeadColumns 0)
   []
   [Row ("",[],[])
@@ -378,13 +372,7 @@ Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "John",Spa
  ,(AlignDefault,ColWidth 0.15)
  ,(AlignDefault,ColWidth 0.1625)]
  (TableHead ("",[],[])
- [Row ("",[],[])
-  [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []]])
+ [])
  [(TableBody ("",[],[]) (RowHeadColumns 0)
   []
   [Row ("",[],[])
@@ -410,13 +398,7 @@ Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "John",Spa
  ,(AlignDefault,ColWidth 0.15)
  ,(AlignDefault,ColWidth 0.1625)]
  (TableHead ("",[],[])
- [Row ("",[],[])
-  [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []]])
+ [])
  [(TableBody ("",[],[]) (RowHeadColumns 0)
   []
   [Row ("",[],[])
diff --git a/test/tables-rstsubset.native b/test/tables-rstsubset.native
index b584bb7ea..2536ff954 100644
--- a/test/tables-rstsubset.native
+++ b/test/tables-rstsubset.native
@@ -276,15 +276,7 @@
  ,(AlignDefault,ColWidth 0.15)
  ,(AlignDefault,ColWidth 0.3375)]
  (TableHead ("",[],[])
- [Row ("",[],[])
-  [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []
-  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
-   []]])
+ [])
  [(TableBody ("",[],[]) (RowHeadColumns 0)
   []
   [Row ("",[],[])
-- 
cgit v1.2.3


From 327e1428c5cdcb62beef6d2e00ab6f1d699256f0 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 7 Jan 2021 16:41:25 -0800
Subject: gfm/commonmark writer: implement start number on ordered lists.

Previously they always started at 1, but according to the spec
the start number is respected. Closes #7009.
---
 src/Text/Pandoc/Writers/Markdown.hs | 5 ++++-
 test/command/7009.md                | 8 ++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)
 create mode 100644 test/command/7009.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/Markdown.hs b/src/Text/Pandoc/Writers/Markdown.hs
index c349fd713..1b5c00468 100644
--- a/src/Text/Pandoc/Writers/Markdown.hs
+++ b/src/Text/Pandoc/Writers/Markdown.hs
@@ -674,7 +674,10 @@ blockToMarkdown' opts (BulletList items) = do
   contents <- inList $ mapM (bulletListItemToMarkdown opts) items
   return $ (if isTightList items then vcat else vsep) contents <> blankline
 blockToMarkdown' opts (OrderedList (start,sty,delim) items) = do
-  let start' = if isEnabled Ext_startnum opts then start else 1
+  variant <- asks envVariant
+  let start' = if variant == Commonmark || isEnabled Ext_startnum opts
+                  then start
+                  else 1
   let sty'   = if isEnabled Ext_fancy_lists opts then sty else DefaultStyle
   let delim' = if isEnabled Ext_fancy_lists opts then delim else DefaultDelim
   let attribs = (start', sty', delim')
diff --git a/test/command/7009.md b/test/command/7009.md
new file mode 100644
index 000000000..a060c6076
--- /dev/null
+++ b/test/command/7009.md
@@ -0,0 +1,8 @@
+```
+% pandoc -t gfm
+3. a
+4. b
+^D
+3.  a
+4.  b
+```
-- 
cgit v1.2.3


From 68fa43799963fff11a980d5d3959184c3d34a723 Mon Sep 17 00:00:00 2001
From: Albert Krewinkel <albert@zeitkraut.de>
Date: Mon, 11 Jan 2021 00:35:48 +0100
Subject: JATS writer: fix citations (#7018)

* JATS writer: keep code lines at 80 chars or below

* JATS writer: fix citations
---
 COPYRIGHT                       |  11 +--
 data/jats.csl                   | 203 ----------------------------------------
 pandoc.cabal                    |   2 -
 src/Text/Pandoc/App.hs          |  16 +---
 src/Text/Pandoc/Writers/JATS.hs |  33 ++++---
 test/command/7016.md            |  48 ++++++++++
 6 files changed, 72 insertions(+), 241 deletions(-)
 delete mode 100644 data/jats.csl
 create mode 100644 test/command/7016.md

(limited to 'test/command')

diff --git a/COPYRIGHT b/COPYRIGHT
index 9992e5680..a6e3a897c 100644
--- a/COPYRIGHT
+++ b/COPYRIGHT
@@ -181,7 +181,7 @@ http://github.com/paulrouget/dzslides
 Released under the Do What the Fuck You Want To Public License.
 
 ------------------------------------------------------------------------
-Pandoc embeds a lua interpreter (via hslua).
+Pandoc embeds a Lua interpreter (via hslua).
 
 Copyright © 1994–2020 Lua.org, PUC-Rio.
 
@@ -203,12 +203,3 @@ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-------------------------------------------------------------------------
-The template pandoc.jats is Copyright 2013--2015 Martin Fenner,
-released under GPL version 2 or later.
-
-The file data/jats.csl is derived from a csl file by Martin Fenner,
-revised by Martin Paul Eve and then John MacFarlane.
-"This work is licensed under a Creative Commons Attribution-ShareAlike 3.0
-License. Originally by Martin Fenner."
diff --git a/data/jats.csl b/data/jats.csl
deleted file mode 100644
index 6972cb3f8..000000000
--- a/data/jats.csl
+++ /dev/null
@@ -1,203 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<style xmlns="http://purl.org/net/xbiblio/csl" class="in-text" version="1.0" default-locale="en-US">
-  <info>
-    <title>Journal Article Tag Suite</title>
-    <title-short>JATS</title-short>
-    <id>http://www.zotero.org/styles/journal-article-tag-suite</id>
-    <link href="https://github.com/MartinPaulEve/JATS-CSL/blob/master/jats.csl" rel="self"/>
-    <link rel="documentation" href="http://jats.nlm.nih.gov/archiving/tag-library/1.0/index.html"/>
-    <author>
-      <name>Martin Paul Eve</name>
-      <email>martin@martineve.com</email>
-    </author>
-    <category citation-format="numeric"/>
-    <category field="medicine"/>
-    <category field="biology"/>
-    <summary>Use this style to generate bibliographic data in Journal Article Tagging Suite (JATS) 1.0 XML format</summary>
-    <updated>2014-06-21T17:41:26+00:00</updated>
-    <rights license="http://creativecommons.org/licenses/by-sa/3.0/">This work is licensed under a Creative Commons Attribution-ShareAlike 3.0 License. Originally by Martin Fenner.</rights>
-  </info>
-  <locale xml:lang="en">
-    <terms>
-        <term name="et-al">{{jats}}&lt;etal/&gt;{{/jats}}</term>
-    </terms>
-  </locale>
-  <macro name="citation-number">
-    <text variable="citation-number" prefix="{{jats}}id=&quot;ref-{{/jats}}" suffix="{{jats}}&quot;&gt;{{/jats}}"/>
-  </macro>
-  <macro name="author">
-    <names variable="author" delimiter=" ">
-        <name prefix="{{jats}}&lt;name&gt;{{/jats}}" suffix="{{jats}}&lt;/name&gt;{{/jats}}" name-as-sort-order="all" sort-separator=" ">
-            <name-part name="family" text-case="capitalize-first" prefix="{{jats}}&lt;surname&gt;{{/jats}}" suffix="{{jats}}&lt;/surname&gt;{{/jats}}"/>
-            <name-part name="given" text-case="capitalize-first" prefix="{{jats}}&lt;given-names&gt;{{/jats}}" suffix="{{jats}}&lt;/given-names&gt;{{/jats}}"/>
-      </name>
-      <substitute>
-        <names variable="editor"/>
-      </substitute>
-    </names>
-  </macro>
-
-  <macro name="editor" delimiter=" ">
-    <names variable="editor" prefix="{{jats}}&lt;person-group person-group-type=&quot;editor&quot;&gt;{{/jats}}" suffix="{{jats}}&lt;/person-group&gt;{{/jats}}">
-      <name prefix="{{jats}}&lt;name&gt;{{/jats}}" suffix="{{jats}}&lt;/name&gt;{{/jats}}" name-as-sort-order="all" sort-separator=" ">
-        <name-part name="family" text-case="capitalize-first" prefix="{{jats}}&lt;surname&gt;{{/jats}}" suffix="{{jats}}&lt;/surname&gt;{{/jats}}"/>
-        <name-part name="given" text-case="capitalize-first" prefix="{{jats}}&lt;given-names&gt;{{/jats}}" suffix="{{jats}}&lt;/given-names&gt;{{/jats}}"/>
-      </name>
-      <substitute>
-        <names variable="editor"/>
-      </substitute>
-    </names>
-  </macro>
-
-  <macro name="editor">
-    <group delimiter=": ">
-      <names variable="editor">
-        <name prefix="{{jats}}&lt;name&gt;{{/jats}}" suffix="{{jats}}&lt;/name&gt;{{/jats}}" name-as-sort-order="all" sort-separator="">
-          <name-part name="family" text-case="capitalize-first" prefix="{{jats}}&lt;surname&gt;{{/jats}}" suffix="{{jats}}&lt;/surname&gt;{{/jats}}"/>
-          <name-part name="given" text-case="capitalize-first" prefix="{{jats}}&lt;given-names&gt;{{/jats}}" suffix="{{jats}}&lt;given-names&gt;{{/jats}}"/>
-        </name>
-      </names>
-    </group>
-  </macro>
-  <macro name="title">
-    <choose>
-      <if type="book" match="any">
-        <group prefix="{{jats}}&lt;source&gt;{{/jats}}" suffix="{{jats}}&lt;/source&gt;{{/jats}}">
-	        <text variable="title"/>
-				</group>
-			</if>
-      <else>
-        <group prefix="{{jats}}&lt;article-title&gt;{{/jats}}" suffix="{{jats}}&lt;/article-title&gt;{{/jats}}">
-	        <text variable="title"/>
-				</group>
-      </else>
-    </choose>
-  </macro>
-  <macro name="container-title">
-    <text variable="container-title" form="short" prefix="{{jats}}&lt;source&gt;{{/jats}}" suffix="{{jats}}&lt;/source&gt;{{/jats}}"/>
-  </macro>
-  <macro name="publisher">
-    <text variable="publisher" prefix="{{jats}}&lt;publisher-name&gt;{{/jats}}" suffix="{{jats}}&lt;/publisher-name&gt;{{/jats}}"/>
-    <text variable="publisher-place" prefix="{{jats}}&lt;publisher-loc&gt;{{/jats}}" suffix="{{jats}}&lt;/publisher-loc&gt;{{/jats}}"/>
-  </macro>
-  <macro name="link">
-    <choose>
-      <if match="any" variable="DOI">
-        <group prefix="{{jats}}&lt;pub-id pub-id-type=&quot;doi&quot;&gt;{{/jats}}" suffix="{{jats}}&lt;/pub-id&gt;{{/jats}}">
-          <text variable="DOI"/>
-        </group>
-      </if>
-    </choose>
-    <choose>
-      <if match="any" variable="PMID">
-        <group prefix="{{jats}}&lt;pub-id pub-id-type=&quot;pmid&quot;&gt;{{/jats}}" suffix="{{jats}}&lt;/pub-id&gt;{{/jats}}">
-          <text variable="PMID"/>
-        </group>
-      </if>
-    </choose>
-    <choose>
-      <if variable="URL" match="any">
-        <text variable="URL" />
-      </if>
-    </choose>
-  </macro>
-  <macro name="date">
-    <choose>
-      <if type="article-journal article-magazine article-newspaper report patent book" match="any">
-        <group prefix="{{jats}}&lt;date&gt;{{/jats}}" suffix="{{jats}}&lt;/date&gt;{{/jats}}">
-          <date variable="issued">
-            <date-part name="day" form="numeric-leading-zeros" prefix="{{jats}}&lt;day&gt;{{/jats}}" suffix="{{jats}}&lt;/day&gt;{{/jats}}"/>
-            <date-part name="month" form="numeric-leading-zeros" prefix="{{jats}}&lt;month&gt;{{/jats}}" suffix="{{jats}}&lt;/month&gt;{{/jats}}"/>
-            <date-part name="year" prefix="{{jats}}&lt;year&gt;{{/jats}}" suffix="{{jats}}&lt;/year&gt;{{/jats}}"/>
-          </date>
-        </group>
-      </if>
-      <else>
-        <group prefix="{{jats}}&lt;date-in-citation content-type=&quot;access-date&quot;{{/jats}}" suffix="{{jats}}&lt;/date-in-citation&gt;{{/jats}}">
-          <date variable="accessed" prefix="{{jats}} iso-8601-date=&quot;{{/jats}}" suffix="{{jats}}&quot;&gt;{{/jats}}">
-            <date-part name="year"/>
-            <date-part name="month" form="numeric-leading-zeros" prefix="{{jats}}-{{/jats}}"/>
-            <date-part name="day" form="numeric-leading-zeros" prefix="{{jats}}-{{/jats}}"/>
-          </date>
-          <date variable="accessed">
-            <date-part name="day" prefix="{{jats}}&lt;day&gt;{{/jats}}" suffix="{{jats}}&lt;/day&gt;{{/jats}}"/>
-            <date-part name="month" form="numeric-leading-zeros" prefix="{{jats}}&lt;month&gt;{{/jats}}" suffix="{{jats}}&lt;/month&gt;{{/jats}}"/>
-            <date-part name="year" prefix="{{jats}}&lt;year&gt;{{/jats}}" suffix="{{jats}}&lt;/year&gt;{{/jats}}"/>
-          </date>
-        </group>
-      </else>
-    </choose>
-  </macro>
-  <macro name="location">
-    <choose>
-      <if type="article-journal article-magazine" match="any">
-        <text variable="volume" prefix="{{jats}}&lt;volume&gt;{{/jats}}" suffix="{{jats}}&lt;/volume&gt;{{/jats}}"/>
-        <text variable="issue" prefix="{{jats}}&lt;issue&gt;{{/jats}}" suffix="{{jats}}&lt;/issue&gt;{{/jats}}"/>
-      </if>
-    </choose>
-    <choose>
-      <if type="article-journal article-magazine article-newspaper chapter" match="any">
-        <text variable="page-first" prefix="{{jats}}&lt;fpage&gt;{{/jats}}" suffix="{{jats}}&lt;/fpage&gt;{{/jats}}"/>
-      </if>
-    </choose>
-  </macro>
-  <macro name="publication-type">
-    <group prefix="{{jats}} publication-type=&quot;{{/jats}}" suffix="{{jats}}&quot;&gt;{{/jats}}">
-      <choose>
-        <if type="article-journal article-magazine article-newspaper" match="any">
-          <text value="journal"/>
-        </if>
-        <else-if type="book" match="any">
-          <text value="book"/>
-        </else-if>
-        <else-if type="chapter" match="any">
-          <text value="bookchapter"/>
-        </else-if>
-        <else-if type="dataset" match="any">
-          <text value="dataset"/>
-        </else-if>
-        <else-if type="patent" match="any">
-          <text value="patent"/>
-        </else-if>
-        <else-if type="report" match="any">
-          <text value="report"/>
-        </else-if>
-        <else-if type="review" match="any">
-          <text value="review"/>
-        </else-if>
-        <else>
-          <text value="standard"/>
-        </else>
-      </choose>
-    </group>
-  </macro>
-  <citation collapse="citation-number">
-    <sort>
-      <key variable="citation-number"/>
-    </sort>
-    <layout delimiter=",">
-      <group prefix="{{jats}}&lt;xref ref-type=&quot;bibr&quot; rid=&quot;{{/jats}}" suffix="{{jats}}&lt;/xref&gt;{{/jats}}">
-        <text variable="citation-number" prefix="{{jats}}ref-{{/jats}}" suffix="{{jats}}&quot;&gt;{{/jats}}"/>
-        <text variable="citation-number"/>
-      </group>
-    </layout>
-  </citation>
-  <bibliography sort-separator="">
-    <layout>
-      <group prefix="{{jats}}&lt;ref {{/jats}}" suffix="{{jats}}&lt;/ref&gt;{{/jats}}">
-        <text macro="citation-number"/>
-        <group prefix="{{jats}}&lt;element-citation{{/jats}}" suffix="{{jats}}&lt;/element-citation&gt;{{/jats}}">
-          <text macro="publication-type"/>
-          <text macro="author" prefix="{{jats}}&lt;person-group person-group-type=&quot;author&quot;&gt;{{/jats}}" suffix="{{jats}}&lt;/person-group&gt;{{/jats}}"/>
-          <text macro="title" />
-          <text macro="container-title"/>
-          <text macro="editor"/>
-          <text macro="publisher"/>
-          <text macro="date"/>
-          <text macro="location"/>
-          <text macro="link"/>
-        </group>
-      </group>
-    </layout>
-  </bibliography>
-</style>
diff --git a/pandoc.cabal b/pandoc.cabal
index ede9af6f0..db8dab491 100644
--- a/pandoc.cabal
+++ b/pandoc.cabal
@@ -180,8 +180,6 @@ data-files:
                  data/pandoc.List.lua
                  -- bash completion template
                  data/bash_completion.tpl
-                 -- jats csl
-                 data/jats.csl
                  -- citeproc
                  data/default.csl
                  citeproc/biblatex-localization/*.lbx.strings
diff --git a/src/Text/Pandoc/App.hs b/src/Text/Pandoc/App.hs
index 725c76424..437af3257 100644
--- a/src/Text/Pandoc/App.hs
+++ b/src/Text/Pandoc/App.hs
@@ -50,10 +50,9 @@ import Text.Pandoc.App.Opt (Opt (..), LineEnding (..), defaultOpts,
 import Text.Pandoc.App.CommandLineOptions (parseOptions, options)
 import Text.Pandoc.App.OutputSettings (OutputSettings (..), optToOutputSettings)
 import Text.Pandoc.BCP47 (Lang (..), parseBCP47)
-import Text.Pandoc.Builder (setMeta)
 import Text.Pandoc.Filter (Filter (JSONFilter, LuaFilter), applyFilters)
 import Text.Pandoc.PDF (makePDF)
-import Text.Pandoc.SelfContained (makeDataURI, makeSelfContained)
+import Text.Pandoc.SelfContained (makeSelfContained)
 import Text.Pandoc.Shared (eastAsianLineBreakFilter, stripEmptyParagraphs,
          headerShift, isURI, tabFilter, uriPathToPath, filterIpynbOutput,
          defaultUserDataDirs, tshow, findM)
@@ -190,17 +189,6 @@ convertWithOpts opts = do
                     Nothing -> readDataFile "abbreviations"
                     Just f  -> readFileStrict f
 
-    metadata <- if format == "jats" &&
-                   isNothing (lookupMeta "csl" (optMetadata opts)) &&
-                   isNothing (lookupMeta "citation-style"
-                                               (optMetadata opts))
-                   then do
-                     jatsCSL <- readDataFile "jats.csl"
-                     let jatsEncoded = makeDataURI
-                                         ("application/xml", jatsCSL)
-                     return $ setMeta "csl" jatsEncoded $ optMetadata opts
-                   else return $ optMetadata opts
-
     case lookupMetaString "lang" (optMetadata opts) of
            ""      -> setTranslations $ Lang "en" "" "US" []
            l       -> case parseBCP47 l of
@@ -286,7 +274,7 @@ convertWithOpts opts = do
                       then fillMediaBag
                       else return)
               >=> return . adjustMetadata (metadataFromFile <>)
-              >=> return . adjustMetadata (<> metadata)
+              >=> return . adjustMetadata (<> optMetadata opts)
               >=> applyTransforms transforms
               >=> applyFilters readerOpts filters [T.unpack format]
               >=> maybe return extractMedia (optExtractMedia opts)
diff --git a/src/Text/Pandoc/Writers/JATS.hs b/src/Text/Pandoc/Writers/JATS.hs
index e8d93b8d5..b2266d179 100644
--- a/src/Text/Pandoc/Writers/JATS.hs
+++ b/src/Text/Pandoc/Writers/JATS.hs
@@ -3,7 +3,7 @@
 {-# LANGUAGE ViewPatterns #-}
 {- |
    Module      : Text.Pandoc.Writers.JATS
-   Copyright   : Copyright (C) 2017-2021 John MacFarlane
+   Copyright   : 2017-2021 John MacFarlane
    License     : GNU GPL, version 2 or above
 
    Maintainer  : John MacFarlane <jgm@berkeley.edu>
@@ -168,13 +168,15 @@ plainToPara x         = x
 -- | Convert a list of pairs of terms and definitions into a list of
 -- JATS varlistentrys.
 deflistItemsToJATS :: PandocMonad m
-                      => WriterOptions -> [([Inline],[[Block]])] -> JATS m (Doc Text)
+                   => WriterOptions
+                   -> [([Inline],[[Block]])] -> JATS m (Doc Text)
 deflistItemsToJATS opts items =
   vcat <$> mapM (uncurry (deflistItemToJATS opts)) items
 
 -- | Convert a term and a list of blocks into a JATS varlistentry.
 deflistItemToJATS :: PandocMonad m
-                     => WriterOptions -> [Inline] -> [[Block]] -> JATS m (Doc Text)
+                  => WriterOptions
+                  -> [Inline] -> [[Block]] -> JATS m (Doc Text)
 deflistItemToJATS opts term defs = do
   term' <- inlinesToJATS opts term
   def' <- wrappedBlocksToJATS (not . isPara)
@@ -186,7 +188,8 @@ deflistItemToJATS opts term defs = do
 
 -- | Convert a list of lists of blocks to a list of JATS list items.
 listItemsToJATS :: PandocMonad m
-                => WriterOptions -> Maybe [Text] -> [[Block]] -> JATS m (Doc Text)
+                => WriterOptions
+                -> Maybe [Text] -> [[Block]] -> JATS m (Doc Text)
 listItemsToJATS opts markers items =
   case markers of
        Nothing -> vcat <$> mapM (listItemToJATS opts Nothing) items
@@ -194,12 +197,13 @@ listItemsToJATS opts markers items =
 
 -- | Convert a list of blocks into a JATS list item.
 listItemToJATS :: PandocMonad m
-               => WriterOptions -> Maybe Text -> [Block] -> JATS m (Doc Text)
+               => WriterOptions
+               -> Maybe Text -> [Block] -> JATS m (Doc Text)
 listItemToJATS opts mbmarker item = do
   contents <- wrappedBlocksToJATS (not . isParaOrList) opts
                  (walk demoteHeaderAndRefs item)
   return $ inTagsIndented "list-item" $
-           maybe empty (\lbl -> inTagsSimple "label" (text $ T.unpack lbl)) mbmarker
+           maybe empty (inTagsSimple "label" . text . T.unpack) mbmarker
            $$ contents
 
 imageMimeType :: Text -> [(Text, Text)] -> (Text, Text)
@@ -247,7 +251,9 @@ blockToJATS opts (Div (id',"section":_,kvs) (Header _lvl _ ils : xs)) = do
   return $ inTags True "sec" attribs $
       inTagsSimple "title" title' $$ contents
 -- Bibliography reference:
-blockToJATS opts (Div (T.stripPrefix "ref-" -> Just _,_,_) [Para lst]) =
+blockToJATS opts (Div (ident,_,_) [Para lst]) | "ref-" `T.isPrefixOf` ident =
+  inTags True "ref" [("id", ident)] .
+  inTagsSimple "mixed-citation" <$>
   inlinesToJATS opts lst
 blockToJATS opts (Div ("refs",_,_) xs) = do
   contents <- blocksToJATS opts xs
@@ -470,10 +476,13 @@ inlineToJATS _ (Link _attr [Str t] (T.stripPrefix "mailto:" -> Just email, _))
   | escapeURI t == email =
   return $ inTagsSimple "email" $ literal (escapeStringForXML email)
 inlineToJATS opts (Link (ident,_,kvs) txt (T.uncons -> Just ('#', src), _)) = do
-  let attr = [("id", ident) | not (T.null ident)] ++
-             [("alt", stringify txt) | not (null txt)] ++
-             [("rid", src)] ++
-             [(k,v) | (k,v) <- kvs, k `elem` ["ref-type", "specific-use"]]
+  let attr = mconcat
+             [ [("id", ident) | not (T.null ident)]
+             , [("alt", stringify txt) | not (null txt)]
+             , [("rid", src)]
+             , [(k,v) | (k,v) <- kvs, k `elem` ["ref-type", "specific-use"]]
+             , [("ref-type", "bibr") | "ref-" `T.isPrefixOf` src]
+             ]
   if null txt
      then return $ selfClosingTag "xref" attr
      else do
@@ -529,7 +538,7 @@ demoteHeaderAndRefs (Div ("refs",cls,kvs) bs) =
 demoteHeaderAndRefs x = x
 
 parseDate :: Text -> Maybe Day
-parseDate s = msum (map (\fs -> parsetimeWith fs $ T.unpack s) formats) :: Maybe Day
+parseDate s = msum (map (`parsetimeWith` T.unpack s) formats)
   where parsetimeWith = parseTimeM True defaultTimeLocale
         formats = ["%x","%m/%d/%Y", "%D","%F", "%d %b %Y",
                     "%e %B %Y", "%b. %e, %Y", "%B %e, %Y",
diff --git a/test/command/7016.md b/test/command/7016.md
new file mode 100644
index 000000000..c2d791ce9
--- /dev/null
+++ b/test/command/7016.md
@@ -0,0 +1,48 @@
+```
+% pandoc --citeproc --to=jats_archiving --standalone
+---
+csl: command/apa.csl
+references:
+- id: doe
+  type: article
+  author:
+  - family: Doe
+    given: Jane
+  container-title: Proceedings of the Academy of Test Inputs
+  doi: 10.x/nope
+  issued: 2021
+  title: Another article
+...
+Blah [@doe].
+^D
+<?xml version="1.0" encoding="utf-8" ?>
+<!DOCTYPE article PUBLIC "-//NLM//DTD JATS (Z39.96) Journal Archiving and Interchange DTD v1.2 20190208//EN"
+                  "JATS-archivearticle1.dtd">
+<article xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" dtd-version="1.2" article-type="other">
+<front>
+<journal-meta>
+<journal-id></journal-id>
+<journal-title-group>
+</journal-title-group>
+<issn></issn>
+<publisher>
+<publisher-name></publisher-name>
+</publisher>
+</journal-meta>
+<article-meta>
+</article-meta>
+</front>
+<body>
+<p>Blah (Doe, 2021).</p>
+</body>
+<back>
+<ref-list>
+  <ref id="ref-doe">
+    <mixed-citation>Doe, J. (2021). Another article. <italic>Proceedings
+    of the Academy of Test Inputs</italic>.
+    doi:<ext-link ext-link-type="uri" xlink:href="https://doi.org/10.x/nope">10.x/nope</ext-link></mixed-citation>
+  </ref>
+</ref-list>
+</back>
+</article>
+```
-- 
cgit v1.2.3


From 3d6ebc905195592897e3a03ef648384ee01f5155 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Fri, 15 Jan 2021 11:51:07 -0800
Subject: Use dev version of citeproc.

Change a citation test which had wrong disambiguation
(see jgm/citeproc#44).
---
 cabal.project                                        |  4 ++++
 stack.yaml                                           |  8 ++++----
 .../pandoc-citeproc-chicago-fullnote-bibliography.md | 20 +++++++++-----------
 3 files changed, 17 insertions(+), 15 deletions(-)

(limited to 'test/command')

diff --git a/cabal.project b/cabal.project
index 348bedc7e..34b3837de 100644
--- a/cabal.project
+++ b/cabal.project
@@ -6,3 +6,7 @@ package pandoc
   flags: +embed_data_files -trypandoc
   ghc-options: -j +RTS -A64m -RTS
 
+source-repository-package
+ type: git
+ location: https://github.com/jgm/citeproc
+ tag: d44e24696ab444090d0e63e321c3a573f68b2e74
diff --git a/stack.yaml b/stack.yaml
index b8bdd372a..b2f8cf8fc 100644
--- a/stack.yaml
+++ b/stack.yaml
@@ -17,11 +17,11 @@ extra-deps:
 - commonmark-extensions-0.2.0.4
 - commonmark-pandoc-0.2.0.1
 - doctemplates-0.9
-- citeproc-0.3.0.3
 - texmath-0.12.1
-# - citeproc:
-#   git: https://github.com/jgm/citeproc.git
-#   commit: a8193fe375fa2354049bf9a967ba3bad4b1ba053
+- citeproc:
+  git: https://github.com/jgm/citeproc.git
+  commit: d44e24696ab444090d0e63e321c3a573f68b2e74
+# - citeproc-0.3.0.3
 ghc-options:
    "$locals": -fhide-source-paths -Wno-missing-home-modules
 resolver: lts-16.23
diff --git a/test/command/pandoc-citeproc-chicago-fullnote-bibliography.md b/test/command/pandoc-citeproc-chicago-fullnote-bibliography.md
index 4536342c6..d997d818b 100644
--- a/test/command/pandoc-citeproc-chicago-fullnote-bibliography.md
+++ b/test/command/pandoc-citeproc-chicago-fullnote-bibliography.md
@@ -107,7 +107,7 @@ Sam Smith. Oxford: Oxford University Press, 2007.
 
 [^5]: [30](#ref-item1), with suffix.
 
-[^6]: [*First Book*, 2005](#ref-item1); ["Article," *Journal of Generic
+[^6]: [*First Book*](#ref-item1); ["Article," *Journal of Generic
     Studies* 6 (2006): 30](#ref-item2); see also [John Doe and Jenny
     Roe, "Why Water Is Wet," in *Third Book*, ed. Sam Smith (Oxford:
     Oxford University Press, 2007)](#ref-пункт3).
@@ -116,22 +116,20 @@ Sam Smith. Oxford: Oxford University Press, 2007.
     and a citation without locators ([Doe and Roe, "Why Water Is
     Wet"](#ref-пункт3)).
 
-[^8]: See [Doe, *First Book*, 2005, chap. 3](#ref-item1); also [Doe and
-    Roe, "Why Water Is Wet," 34--35](#ref-пункт3).
+[^8]: See [Doe, *First Book*, chap. 3](#ref-item1); also [Doe and Roe,
+    "Why Water Is Wet," 34--35](#ref-пункт3).
 
-[^9]: See [Doe, *First Book*, 2005, 34--35](#ref-item1).
+[^9]: See [Doe, *First Book*, 34--35](#ref-item1).
 
 [^10]: Some citations (see [Doe, chap. 3](#ref-item1); [Doe and Roe,
-    "Why Water Is Wet"](#ref-пункт3); [Doe, "Article,"
-    2006](#ref-item2)).
+    "Why Water Is Wet"](#ref-пункт3); [Doe, "Article"](#ref-item2)).
 
-[^11]: [Doe, *First Book*, 2005, 33, 35--37](#ref-item1), and nowhere
-    else.
+[^11]: [Doe, *First Book*, 33, 35--37](#ref-item1), and nowhere else.
 
-[^12]: [Doe, *First Book*, 2005](#ref-item1) and nowhere else.
+[^12]: [Doe, *First Book*](#ref-item1) and nowhere else.
 
 [^13]: Like a citation without author: (), and again (), and now Doe
-    with a locator (["Article," 2006, 44](#ref-item2)).
+    with a locator (["Article," 44](#ref-item2)).
 
-[^14]: *See* [Doe, *First Book*, 2005, 32](#ref-item1).
+[^14]: *See* [Doe, *First Book*, 32](#ref-item1).
 ```
-- 
cgit v1.2.3


From b4b3560191b3699dd4db9d069244925a3c6074db Mon Sep 17 00:00:00 2001
From: Albert Krewinkel <albert@zeitkraut.de>
Date: Wed, 20 Jan 2021 19:09:36 +0100
Subject: JATS writer: allow to use element-citation

---
 MANUAL.txt                                 |   7 ++
 pandoc.cabal                               |   1 +
 src/Text/Pandoc/Extensions.hs              |  14 +++
 src/Text/Pandoc/Writers/JATS.hs            |  21 ++--
 src/Text/Pandoc/Writers/JATS/References.hs | 160 +++++++++++++++++++++++++++++
 src/Text/Pandoc/Writers/JATS/Types.hs      |   4 +-
 test/command/7042.md                       | 146 ++++++++++++++++++++++++++
 7 files changed, 346 insertions(+), 7 deletions(-)
 create mode 100644 src/Text/Pandoc/Writers/JATS/References.hs
 create mode 100644 test/command/7042.md

(limited to 'test/command')

diff --git a/MANUAL.txt b/MANUAL.txt
index b9c4ef637..7bf74a8f9 100644
--- a/MANUAL.txt
+++ b/MANUAL.txt
@@ -3093,6 +3093,13 @@ output format.
 Some aspects of [Pandoc's Markdown citation syntax](#citations)
 are also accepted in `org` input.
 
+#### Extension: `element_citations` ####
+
+In the `jats` output formats, this causes reference items to
+be replaced with `<element-citation>` elements. These
+elements are not influenced by CSL styles, but all information
+on the item is included in tags.
+
 #### Extension: `ntb` ####
 
 In the `context` output format this enables the use of [Natural Tables
diff --git a/pandoc.cabal b/pandoc.cabal
index 0d63cbe35..07feb10dd 100644
--- a/pandoc.cabal
+++ b/pandoc.cabal
@@ -638,6 +638,7 @@ library
                    Text.Pandoc.Readers.Metadata,
                    Text.Pandoc.Readers.Roff,
                    Text.Pandoc.Writers.Docx.StyleMap,
+                   Text.Pandoc.Writers.JATS.References,
                    Text.Pandoc.Writers.JATS.Table,
                    Text.Pandoc.Writers.JATS.Types,
                    Text.Pandoc.Writers.LaTeX.Caption,
diff --git a/src/Text/Pandoc/Extensions.hs b/src/Text/Pandoc/Extensions.hs
index 39c2a0489..7aa32c52c 100644
--- a/src/Text/Pandoc/Extensions.hs
+++ b/src/Text/Pandoc/Extensions.hs
@@ -88,6 +88,7 @@ data Extension =
                                   --   does not affect readers/writers directly; it causes
                                   --   the eastAsianLineBreakFilter to be applied after
                                   --   parsing, in Text.Pandoc.App.convertWithOpts.
+    | Ext_element_citations   -- ^ Use element-citation elements for JATS citations
     | Ext_emoji               -- ^ Support emoji like :smile:
     | Ext_empty_paragraphs -- ^ Allow empty paragraphs
     | Ext_epub_html_exts      -- ^ Recognise the EPUB extended version of HTML
@@ -412,6 +413,11 @@ getDefaultExtensions "textile"         = extensionsFromList
                                            Ext_smart,
                                            Ext_raw_html,
                                            Ext_auto_identifiers]
+getDefaultExtensions "jats"            = extensionsFromList
+                                          [Ext_auto_identifiers]
+getDefaultExtensions "jats_archiving"  = getDefaultExtensions "jats"
+getDefaultExtensions "jats_publishing" = getDefaultExtensions "jats"
+getDefaultExtensions "jats_articleauthoring" = getDefaultExtensions "jats"
 getDefaultExtensions "opml"            = pandocExtensions -- affects notes
 getDefaultExtensions _                 = extensionsFromList
                                           [Ext_auto_identifiers]
@@ -554,6 +560,14 @@ getAllExtensions f = universalExtensions <> getAll f
     , Ext_smart
     , Ext_raw_tex
     ]
+  getAll "jats"            =
+    extensionsFromList
+    [ Ext_auto_identifiers
+    , Ext_element_citations
+    ]
+  getAll "jats_archiving"  = getAll "jats"
+  getAll "jats_publishing" = getAll "jats"
+  getAll "jats_articleauthoring" = getAll "jats"
   getAll "opml"            = allMarkdownExtensions -- affects notes
   getAll "twiki"           = autoIdExtensions <>
     extensionsFromList
diff --git a/src/Text/Pandoc/Writers/JATS.hs b/src/Text/Pandoc/Writers/JATS.hs
index c75d40745..a9369db7a 100644
--- a/src/Text/Pandoc/Writers/JATS.hs
+++ b/src/Text/Pandoc/Writers/JATS.hs
@@ -29,6 +29,7 @@ import Data.Maybe (fromMaybe)
 import Data.Time (toGregorian, Day, parseTimeM, defaultTimeLocale, formatTime)
 import qualified Data.Text as T
 import Data.Text (Text)
+import Text.Pandoc.Citeproc (getReferences)
 import Text.Pandoc.Class.PandocMonad (PandocMonad, report)
 import Text.Pandoc.Definition
 import Text.Pandoc.Highlighting (languages, languagesByExtension)
@@ -40,6 +41,7 @@ import Text.DocLayout
 import Text.Pandoc.Shared
 import Text.Pandoc.Templates (renderTemplate)
 import Text.DocTemplates (Context(..), Val(..))
+import Text.Pandoc.Writers.JATS.References (referencesToJATS)
 import Text.Pandoc.Writers.JATS.Table (tableToJATS)
 import Text.Pandoc.Writers.JATS.Types
 import Text.Pandoc.Writers.Math
@@ -71,15 +73,19 @@ writeJATS = writeJatsArchiving
 
 -- | Convert a @'Pandoc'@ document to JATS.
 writeJats :: PandocMonad m => JATSTagSet -> WriterOptions -> Pandoc -> m Text
-writeJats tagSet opts d =
-  runReaderT (evalStateT (docToJATS opts d) initialState)
-             environment
-  where initialState = JATSState { jatsNotes = [] }
-        environment = JATSEnv
+writeJats tagSet opts d = do
+  refs <- if extensionEnabled Ext_element_citations $ writerExtensions opts
+          then getReferences Nothing d
+          else pure []
+  let environment = JATSEnv
           { jatsTagSet = tagSet
           , jatsInlinesWriter = inlinesToJATS
           , jatsBlockWriter = blockToJATS
+          , jatsReferences = refs
           }
+  let initialState = JATSState { jatsNotes = [] }
+  runReaderT (evalStateT (docToJATS opts d) initialState)
+             environment
 
 -- | Convert Pandoc document to string in JATS format.
 docToJATS :: PandocMonad m => WriterOptions -> Pandoc -> JATS m Text
@@ -258,7 +264,10 @@ blockToJATS opts (Div (ident,_,_) [Para lst]) | "ref-" `T.isPrefixOf` ident =
   inTagsSimple "mixed-citation" <$>
   inlinesToJATS opts lst
 blockToJATS opts (Div ("refs",_,_) xs) = do
-  contents <- blocksToJATS opts xs
+  refs <- asks jatsReferences
+  contents <- if null refs
+              then blocksToJATS opts xs
+              else referencesToJATS opts refs
   return $ inTagsIndented "ref-list" contents
 blockToJATS opts (Div (ident,[cls],kvs) bs) | cls `elem` ["fig", "caption", "table-wrap"] = do
   contents <- blocksToJATS opts bs
diff --git a/src/Text/Pandoc/Writers/JATS/References.hs b/src/Text/Pandoc/Writers/JATS/References.hs
new file mode 100644
index 000000000..4ee7eb9dd
--- /dev/null
+++ b/src/Text/Pandoc/Writers/JATS/References.hs
@@ -0,0 +1,160 @@
+{-# LANGUAGE LambdaCase        #-}
+{-# LANGUAGE OverloadedStrings #-}
+{- |
+   Module      : Text.Pandoc.Writers.JATS.References
+   Copyright   : © 2021 Albert Krewinkel
+   License     : GNU GPL, version 2 or above
+
+   Maintainer  : Albert Krewinkel <tarleb@zeitkraut.de>
+   Stability   : alpha
+   Portability : portable
+
+Creation of a bibliography list using @<element-citation>@ elements in
+reference items.
+-}
+module Text.Pandoc.Writers.JATS.References
+  ( referencesToJATS
+  , referenceToJATS
+  ) where
+
+import Citeproc.Pandoc ()
+import Citeproc.Types
+  ( Date (..), DateParts (..), ItemId (..), Name (..), Reference (..)
+  , Val (..) , lookupVariable, valToText
+  )
+import Data.Text (Text)
+import Text.DocLayout (Doc, empty, isEmpty, literal, vcat)
+import Text.Pandoc.Class.PandocMonad (PandocMonad)
+import Text.Pandoc.Builder (Inlines)
+import Text.Pandoc.Options (WriterOptions)
+import Text.Pandoc.Shared (tshow)
+import Text.Pandoc.Writers.JATS.Types
+import Text.Pandoc.XML (inTags)
+import qualified Data.Text as T
+
+referencesToJATS :: PandocMonad m
+                 => WriterOptions
+                 -> [Reference Inlines]
+                 -> JATS m (Doc Text)
+referencesToJATS opts =
+  fmap (inTags True "ref-list" [] . vcat) . mapM (referenceToJATS opts)
+
+referenceToJATS :: PandocMonad m
+                => WriterOptions
+                -> Reference Inlines
+                -> JATS m (Doc Text)
+referenceToJATS _opts ref = do
+  let refType = referenceType ref
+  let pubType = [("publication-type", refType) | not (T.null refType)]
+  let wrap = inTags True "ref" [("id", "ref-" <> unItemId (referenceId ref))]
+           . inTags True "element-citation" pubType
+  return . wrap . vcat $
+    [ authors
+    , "title" `varInTag`
+      if refType == "book"
+      then "source"
+      else "article-title"
+    , if refType == "book"
+      then empty
+      else "container-title" `varInTag` "source"
+    , editors
+    , "publisher"       `varInTag` "publisher-name"
+    , "publisher-place" `varInTag` "publisher-loc"
+    , yearTag
+    , accessed
+    , "volume"          `varInTag` "volume"
+    , "issue"           `varInTag` "issue"
+    , "page-first"      `varInTag` "fpage"
+    , "page-last"       `varInTag` "lpage"
+    , "pages"           `varInTag` "page-range"
+    , "ISBN"            `varInTag` "isbn"
+    , "ISSN"            `varInTag` "issn"
+    , varInTagWith "doi"  "pub-id" [("pub-id-type", "doi")]
+    , varInTagWith "pmid" "pub-id" [("pub-id-type", "pmid")]
+    ]
+  where
+    varInTag var tagName = varInTagWith var tagName []
+
+    varInTagWith var tagName tagAttribs =
+      case lookupVariable var ref >>= valToText of
+        Nothing  -> mempty
+        Just val -> inTags' tagName tagAttribs $ literal val
+
+    authors = case lookupVariable "author" ref of
+      Just (NamesVal names) ->
+        inTags True "person-group" [("person-group-type", "author")] . vcat $
+        map toNameElements names
+      _                     -> empty
+
+    editors = case lookupVariable "editor" ref of
+      Just (NamesVal names) ->
+        inTags True "person-group" [("person-group-type", "editor")] . vcat $
+        map toNameElements names
+      _                     -> empty
+
+    yearTag =
+      case lookupVariable "issued" ref of
+        Just (DateVal date) -> toDateElements date
+        _ -> empty
+
+    accessed =
+      case lookupVariable "accessed" ref of
+        Just (DateVal d) -> inTags' "date-in-citation"
+                                    [("content-type", "access-date")]
+                                    (toDateElements d)
+        _ -> empty
+
+toDateElements :: Date -> Doc Text
+toDateElements date =
+  case dateParts date of
+    dp@(DateParts (y:m:d:_)):_ -> yearElement y dp <>
+                                  monthElement m <>
+                                  dayElement d
+    dp@(DateParts (y:m:_)):_   -> yearElement y dp <> monthElement m
+    dp@(DateParts (y:_)):_     -> yearElement y dp
+    _                          -> empty
+
+yearElement :: Int -> DateParts -> Doc Text
+yearElement year dp =
+  inTags' "year" [("iso-8601-date", iso8601 dp)] $ literal (fourDigits year)
+
+monthElement :: Int -> Doc Text
+monthElement month = inTags' "month" [] . literal $ twoDigits month
+
+dayElement :: Int -> Doc Text
+dayElement day = inTags' "day" [] . literal $ twoDigits day
+
+iso8601 :: DateParts -> Text
+iso8601 = T.intercalate "-" . \case
+  DateParts (y:m:d:_) -> [fourDigits y, twoDigits m, twoDigits d]
+  DateParts (y:m:_)   -> [fourDigits y, twoDigits m]
+  DateParts (y:_)     -> [fourDigits y]
+  _                   -> []
+
+twoDigits :: Int -> Text
+twoDigits n = T.takeEnd 2 $ '0' `T.cons` tshow n
+
+fourDigits :: Int -> Text
+fourDigits n = T.takeEnd 4 $ "000" <> tshow n
+
+toNameElements :: Name -> Doc Text
+toNameElements name =
+  if not (isEmpty nameTags)
+  then inTags' "name" [] nameTags
+  else nameLiteral name `inNameTag` "string-name"
+    where
+      inNameTag val tag = maybe empty (inTags' tag [] . literal) val
+      surnamePrefix = maybe mempty (`T.snoc` ' ') $
+                      nameNonDroppingParticle name
+      givenSuffix = maybe mempty (T.cons ' ') $
+                    nameDroppingParticle name
+      nameTags = mconcat
+        [ ((surnamePrefix <>) <$> nameFamily name) `inNameTag` "surname"
+        , ((<> givenSuffix) <$> nameGiven name) `inNameTag` "given-names"
+        , nameSuffix name `inNameTag` "suffix"
+        ]
+
+-- | Put the supplied contents between start and end tags of tagType,
+--   with specified attributes.
+inTags' :: Text -> [(Text, Text)] -> Doc Text -> Doc Text
+inTags' = inTags False
diff --git a/src/Text/Pandoc/Writers/JATS/Types.hs b/src/Text/Pandoc/Writers/JATS/Types.hs
index 54ed4a8bd..6fdddc0b5 100644
--- a/src/Text/Pandoc/Writers/JATS/Types.hs
+++ b/src/Text/Pandoc/Writers/JATS/Types.hs
@@ -17,11 +17,12 @@ module Text.Pandoc.Writers.JATS.Types
   )
 where
 
+import Citeproc.Types (Reference)
 import Control.Monad.Reader (ReaderT)
 import Control.Monad.State (StateT)
 import Data.Text (Text)
 import Text.DocLayout (Doc)
-import Text.Pandoc.Definition (Block, Inline)
+import Text.Pandoc.Builder (Block, Inline, Inlines)
 import Text.Pandoc.Options (WriterOptions)
 
 -- | JATS tag set variant
@@ -40,6 +41,7 @@ data JATSEnv m = JATSEnv
   { jatsTagSet :: JATSTagSet
   , jatsInlinesWriter :: WriterOptions -> [Inline] -> JATS m (Doc Text)
   , jatsBlockWriter   :: WriterOptions -> Block    -> JATS m (Doc Text)
+  , jatsReferences    :: [Reference Inlines]
   }
 
 -- | JATS writer type
diff --git a/test/command/7042.md b/test/command/7042.md
new file mode 100644
index 000000000..de0294da3
--- /dev/null
+++ b/test/command/7042.md
@@ -0,0 +1,146 @@
+```
+% pandoc -f markdown -t jats_publishing+element_citations --citeproc -s
+---
+nocite: "[@*]"
+references:
+- author:
+  - family: Jane
+    given: Doe
+  container-title: Public Library of Tests
+  id: year-month
+  issued: 1999-08
+  title: Year and month
+  type: article-journal
+- accessed: 1999-01-22
+  author:
+  - family: Negidius
+    given: Numerius
+  container-title: Public Library of Tests
+  id: access-date
+  issued: 1911-10-03
+  title: Entry with access date
+  type: article-journal
+- author:
+  - family: Beethoven
+    given: Ludwig
+    dropping-particle: van
+  - family: Bray
+    given: Jan
+    non-dropping-particle: de
+  container-title: Public Library of Tests
+  id: name-particles
+  issued: 1820
+  title: Name particles, dropping and non-dropping
+  type: article-journal
+- author:
+  - 宮水 三葉
+  - 立花 瀧
+  title: Big Book of Tests
+  id: book-with-japanese-authors
+  issued: 2016
+  type: book
+- author:
+  - family: Watson
+    given: J. D.
+  - family: Crick
+    given: F. H. C.
+  container-title: Nature
+  doi: '10.1038/171737a0'
+  id: full-journal-article-entry
+  issue: 4356
+  issued: '1953-04-01'
+  pages: 737-738
+  pmid: 13054692
+  title: 'Molecular Structure of Nucleic Acids: A Structure for Deoxyribose Nucleic Acid'
+  type: article-journal
+  volume: 171
+...
+^D
+<?xml version="1.0" encoding="utf-8" ?>
+<!DOCTYPE article PUBLIC "-//NLM//DTD JATS (Z39.96) Journal Publishing DTD v1.2 20190208//EN"
+                  "JATS-publishing1.dtd">
+<article xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" dtd-version="1.2" article-type="other">
+<front>
+<journal-meta>
+<journal-id></journal-id>
+<journal-title-group>
+</journal-title-group>
+<issn></issn>
+<publisher>
+<publisher-name></publisher-name>
+</publisher>
+</journal-meta>
+<article-meta>
+</article-meta>
+</front>
+<body>
+
+</body>
+<back>
+<ref-list>
+  <ref-list>
+    <ref id="ref-year-month">
+      <element-citation publication-type="article-journal">
+        <person-group person-group-type="author">
+          <name><surname>Jane</surname><given-names>Doe</given-names></name>
+        </person-group>
+        <article-title>Year and month</article-title>
+        <source>Public Library of Tests</source>
+        <year iso-8601-date="1999-08">1999</year><month>08</month>
+      </element-citation>
+    </ref>
+    <ref id="ref-access-date">
+      <element-citation publication-type="article-journal">
+        <person-group person-group-type="author">
+          <name><surname>Negidius</surname><given-names>Numerius</given-names></name>
+        </person-group>
+        <article-title>Entry with access date</article-title>
+        <source>Public Library of Tests</source>
+        <year iso-8601-date="1911-10-03">1911</year><month>10</month><day>03</day>
+        <date-in-citation content-type="access-date"><year iso-8601-date="1999-01-22">1999</year><month>01</month><day>22</day></date-in-citation>
+      </element-citation>
+    </ref>
+    <ref id="ref-name-particles">
+      <element-citation publication-type="article-journal">
+        <person-group person-group-type="author">
+          <name><surname>Beethoven</surname><given-names>Ludwig van</given-names></name>
+          <name><surname>de Bray</surname><given-names>Jan</given-names></name>
+        </person-group>
+        <article-title>Name particles, dropping and non-dropping</article-title>
+        <source>Public Library of Tests</source>
+        <year iso-8601-date="1820">1820</year>
+      </element-citation>
+    </ref>
+    <ref id="ref-book-with-japanese-authors">
+      <element-citation publication-type="book">
+        <person-group person-group-type="author">
+          <string-name>宮水 三葉</string-name>
+          <string-name>立花 瀧</string-name>
+        </person-group>
+        <source>Big Book of Tests</source>
+        <year iso-8601-date="2016">2016</year>
+      </element-citation>
+    </ref>
+    <ref id="ref-full-journal-article-entry">
+      <element-citation publication-type="article-journal">
+        <person-group person-group-type="author">
+          <name><surname>Watson</surname><given-names>J. D.</given-names></name>
+          <name><surname>Crick</surname><given-names>F. H. C.</given-names></name>
+        </person-group>
+        <article-title>Molecular Structure of Nucleic Acids: A Structure for Deoxyribose Nucleic Acid</article-title>
+        <source>Nature</source>
+        <year iso-8601-date="1953-04-01">1953</year><month>04</month><day>01</day>
+        <volume>171</volume>
+        <issue>4356</issue>
+        <fpage>737</fpage>
+        <page-range>737-738</page-range>
+        <pub-id pub-id-type="doi">10.1038/171737a0</pub-id>
+        <pub-id pub-id-type="pmid">13054692</pub-id>
+      </element-citation>
+    </ref>
+  </ref-list>
+</ref-list>
+</back>
+</article>
+
+```
-- 
cgit v1.2.3


From 198ce0cde99731680b62294a7dff1b218b29bd4d Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Fri, 22 Jan 2021 20:49:41 -0800
Subject: ImageSize:   use viewBox for svg if no length, width.

This change allows pandoc to extract size information
from more SVGs.  Closes #7045.
---
 src/Text/Pandoc/ImageSize.hs |  8 ++++++--
 test/command/svg.md          | 30 ++++++++++++++----------------
 2 files changed, 20 insertions(+), 18 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/ImageSize.hs b/src/Text/Pandoc/ImageSize.hs
index 098c16721..e19958f6a 100644
--- a/src/Text/Pandoc/ImageSize.hs
+++ b/src/Text/Pandoc/ImageSize.hs
@@ -328,12 +328,16 @@ getSize img =
 svgSize :: WriterOptions -> ByteString -> Maybe ImageSize
 svgSize opts img = do
   doc <- Xml.parseXMLDoc $ UTF8.toString img
+  let viewboxSize = do
+        vb <- Xml.findAttrBy (== Xml.QName "viewBox" Nothing Nothing) doc
+        [_,_,w,h] <- mapM safeRead (T.words (T.pack vb))
+        return (w,h)
   let dpi = fromIntegral $ writerDpi opts
   let dirToInt dir = do
         dim <- Xml.findAttrBy (== Xml.QName dir Nothing Nothing) doc >>= lengthToDim . T.pack
         return $ inPixel opts dim
-  w <- dirToInt "width"
-  h <- dirToInt "height"
+  w <- dirToInt "width" <|> (fst <$> viewboxSize)
+  h <- dirToInt "height" <|> (snd <$> viewboxSize)
   return ImageSize {
     pxX  = w
   , pxY  = h
diff --git a/test/command/svg.md b/test/command/svg.md
index 57506570b..4ba836b20 100644
--- a/test/command/svg.md
+++ b/test/command/svg.md
@@ -36,28 +36,27 @@
 % pandoc -f latex -t icml
 \includegraphics{command/SVG_logo.svg}
 ^D
-[WARNING] Could not determine image size for 'command/SVG_logo.svg': could not determine SVG size
 <ParagraphStyleRange AppliedParagraphStyle="ParagraphStyle/Paragraph">
   <CharacterStyleRange AppliedCharacterStyle="$ID/NormalCharacterStyle">
-    <Rectangle Self="uec" StrokeWeight="0" ItemTransform="1 0 0 1 150 -100">
+    <Rectangle Self="uec" StrokeWeight="0" ItemTransform="1 0 0 1 37.5 -37.5">
       <Properties>
         <PathGeometry>
           <GeometryPathType PathOpen="false">
             <PathPointArray>
-              <PathPointType Anchor="-150 -100" LeftDirection="-150 -100" RightDirection="-150 -100" />
-              <PathPointType Anchor="-150 100" LeftDirection="-150 100" RightDirection="-150 100" />
-              <PathPointType Anchor="150 100" LeftDirection="150 100" RightDirection="150 100" />
-              <PathPointType Anchor="150 -100" LeftDirection="150 -100" RightDirection="150 -100" />
+              <PathPointType Anchor="-37.5 -37.5" LeftDirection="-37.5 -37.5" RightDirection="-37.5 -37.5" />
+              <PathPointType Anchor="-37.5 37.5" LeftDirection="-37.5 37.5" RightDirection="-37.5 37.5" />
+              <PathPointType Anchor="37.5 37.5" LeftDirection="37.5 37.5" RightDirection="37.5 37.5" />
+              <PathPointType Anchor="37.5 -37.5" LeftDirection="37.5 -37.5" RightDirection="37.5 -37.5" />
             </PathPointArray>
           </GeometryPathType>
         </PathGeometry>
       </Properties>
-      <Image Self="ue6" ItemTransform="1 0 0 1 -150 -100">
+      <Image Self="ue6" ItemTransform="1 0 0 1 -37.5 -37.5">
         <Properties>
           <Profile type="string">
             $ID/Embedded
           </Profile>
-          <GraphicBounds Left="0" Top="0" Right="300" Bottom="200" />
+          <GraphicBounds Left="0" Top="0" Right="75" Bottom="75" />
         </Properties>
         <Link Self="ueb" LinkResourceURI="file:command/SVG_logo.svg" />
       </Image>
@@ -70,28 +69,27 @@
 % pandoc -f latex -t icml
 \includegraphics{command/SVG_logo-without-xml-declaration.svg}
 ^D
-[WARNING] Could not determine image size for 'command/SVG_logo-without-xml-declaration.svg': could not determine SVG size
 <ParagraphStyleRange AppliedParagraphStyle="ParagraphStyle/Paragraph">
   <CharacterStyleRange AppliedCharacterStyle="$ID/NormalCharacterStyle">
-    <Rectangle Self="uec" StrokeWeight="0" ItemTransform="1 0 0 1 150 -100">
+    <Rectangle Self="uec" StrokeWeight="0" ItemTransform="1 0 0 1 37.5 -37.5">
       <Properties>
         <PathGeometry>
           <GeometryPathType PathOpen="false">
             <PathPointArray>
-              <PathPointType Anchor="-150 -100" LeftDirection="-150 -100" RightDirection="-150 -100" />
-              <PathPointType Anchor="-150 100" LeftDirection="-150 100" RightDirection="-150 100" />
-              <PathPointType Anchor="150 100" LeftDirection="150 100" RightDirection="150 100" />
-              <PathPointType Anchor="150 -100" LeftDirection="150 -100" RightDirection="150 -100" />
+              <PathPointType Anchor="-37.5 -37.5" LeftDirection="-37.5 -37.5" RightDirection="-37.5 -37.5" />
+              <PathPointType Anchor="-37.5 37.5" LeftDirection="-37.5 37.5" RightDirection="-37.5 37.5" />
+              <PathPointType Anchor="37.5 37.5" LeftDirection="37.5 37.5" RightDirection="37.5 37.5" />
+              <PathPointType Anchor="37.5 -37.5" LeftDirection="37.5 -37.5" RightDirection="37.5 -37.5" />
             </PathPointArray>
           </GeometryPathType>
         </PathGeometry>
       </Properties>
-      <Image Self="ue6" ItemTransform="1 0 0 1 -150 -100">
+      <Image Self="ue6" ItemTransform="1 0 0 1 -37.5 -37.5">
         <Properties>
           <Profile type="string">
             $ID/Embedded
           </Profile>
-          <GraphicBounds Left="0" Top="0" Right="300" Bottom="200" />
+          <GraphicBounds Left="0" Top="0" Right="75" Bottom="75" />
         </Properties>
         <Link Self="ueb" LinkResourceURI="file:command/SVG_logo-without-xml-declaration.svg" />
       </Image>
-- 
cgit v1.2.3


From 98c2a52b4ee6c833c0a2f2652386cec024e377eb Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Tue, 26 Jan 2021 22:45:57 -0800
Subject: Clean up BibTeX parsing.

Previously there was a messy code path that gave strange
results in some cases, not passing through raw tex but
trying to extract a string content.  This was an artefact
of trying to handle some special bibtex-specific commands
in the BibTeX reader. Now we just handle these in the
LaTeX reader and simplify parsing in the BibTeX reader.
This does mean that more raw tex will be passed through
(and currently this is not sensitive to the `raw_tex`
extension; this should be fixed).

Closes #7049.
---
 src/Text/Pandoc/Citeproc/BibTeX.hs | 33 +--------------------------------
 src/Text/Pandoc/Readers/LaTeX.hs   | 18 ++++++++++++++++++
 test/command/biblatex-cotton.md    |  6 +++---
 test/command/biblatex-murray.md    |  5 +++--
 4 files changed, 25 insertions(+), 37 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Citeproc/BibTeX.hs b/src/Text/Pandoc/Citeproc/BibTeX.hs
index 10730a1e9..5b9068378 100644
--- a/src/Text/Pandoc/Citeproc/BibTeX.hs
+++ b/src/Text/Pandoc/Citeproc/BibTeX.hs
@@ -750,41 +750,10 @@ blocksToInlines bs =
        _          -> B.fromList $ Walk.query (:[]) bs
 
 adjustSpans :: Lang -> Inline -> Inline
-adjustSpans lang (RawInline (Format "latex") s)
-  | s == "\\hyphen" || s == "\\hyphen " = Str "-"
-  | otherwise = parseRawLaTeX lang s
+adjustSpans lang (Span ("",[],[("bibstring",s)]) _) = Str $ resolveKey' lang s
 adjustSpans _ SoftBreak = Space
 adjustSpans _ x = x
 
-parseRawLaTeX :: Lang -> Text -> Inline
-parseRawLaTeX lang t@(T.stripPrefix "\\" -> Just xs) =
-  case parseLaTeX lang contents of
-          Right [Para ys]  -> f command ys
-          Right [Plain ys] -> f command ys
-          Right []         -> f command []
-          _                -> RawInline (Format "latex") t
-   where (command', contents') = T.break (\c -> c =='{' || c =='\\') xs
-         command  = T.strip command'
-         contents = T.drop 1 $ T.dropEnd 1 contents'
-         f "mkbibquote"    ils = Span nullAttr [Quoted DoubleQuote ils]
-         f "mkbibemph"     ils = Span nullAttr [Emph ils]
-         f "mkbibitalic"   ils = Span nullAttr [Emph ils]
-         f "mkbibbold"     ils = Span nullAttr [Strong ils]
-         f "mkbibparens"   ils = Span nullAttr $
-                                  [Str "("] ++ ils ++ [Str ")"]
-         f "mkbibbrackets" ils = Span nullAttr $
-                                  [Str "["] ++ ils ++ [Str "]"]
-         -- ... both should be nestable & should work in year fields
-         f "autocap"    ils    = Span nullAttr ils
-           -- TODO: should work in year fields
-         f "textnormal" ils    = Span ("",["nodecor"],[]) ils
-         f "bibstring" [Str s] = Str $ resolveKey' lang s
-         f "adddot"    []      = Str "."
-         f "adddotspace" []    = Span nullAttr [Str ".", Space]
-         f "addabbrvspace" []  = Space
-         f _            ils    = Span nullAttr ils
-parseRawLaTeX _ t = RawInline (Format "latex") t
-
 latex' :: Text -> Bib [Block]
 latex' t = do
   lang <- gets localeLang
diff --git a/src/Text/Pandoc/Readers/LaTeX.hs b/src/Text/Pandoc/Readers/LaTeX.hs
index f49323996..91c71c000 100644
--- a/src/Text/Pandoc/Readers/LaTeX.hs
+++ b/src/Text/Pandoc/Readers/LaTeX.hs
@@ -940,6 +940,24 @@ inlineCommands = M.union inlineLanguageCommands $ M.fromList
   , ("uline", underline <$> tok)
   -- plain tex stuff that should just be passed through as raw tex
   , ("ifdim", ifdim)
+  -- bibtex
+  , ("mkbibquote", spanWith nullAttr . doubleQuoted <$> tok)
+  , ("mkbibemph", spanWith nullAttr . emph <$> tok)
+  , ("mkbibitalic", spanWith nullAttr . emph <$> tok)
+  , ("mkbibbold", spanWith nullAttr . strong <$> tok)
+  , ("mkbibparens",
+       spanWith nullAttr . (\x -> str "(" <> x <> str ")") <$> tok)
+  , ("mkbibbrackets",
+       spanWith nullAttr . (\x -> str "[" <> x <> str "]") <$> tok)
+  , ("autocap", spanWith nullAttr <$> tok)
+  , ("textnormal", spanWith ("",["nodecor"],[]) <$> tok)
+  , ("bibstring",
+       (\x -> spanWith ("",[],[("bibstring",x)]) (str x)) . untokenize
+         <$> braced)
+  , ("adddot", pure (str "."))
+  , ("adddotspace", pure (spanWith nullAttr (str "." <> space)))
+  , ("addabbrvspace", pure space)
+  , ("hyphen", pure (str "-"))
   ]
 
 accent :: PandocMonad m => Char -> Maybe Char -> LP m Inlines
diff --git a/test/command/biblatex-cotton.md b/test/command/biblatex-cotton.md
index 51c3ee2f4..7a7a2e296 100644
--- a/test/command/biblatex-cotton.md
+++ b/test/command/biblatex-cotton.md
@@ -43,9 +43,9 @@ Cotton, F. A., Wilkinson, G., Murillio, C. A., & Bochmann, M. (1999).
 ---
 nocite: "[@*]"
 references:
-- annote: A book entry with author authors and an edition field. By
-    default, long author and editor lists are automatically truncated.
-    This is configurable
+- annote: A book entry with `\arabic{author}`{=latex} authors and an
+    edition field. By default, long author and editor lists are
+    automatically truncated. This is configurable
   author:
   - family: Cotton
     given: Frank Albert
diff --git a/test/command/biblatex-murray.md b/test/command/biblatex-murray.md
index 78081c6fe..8242e4822 100644
--- a/test/command/biblatex-murray.md
+++ b/test/command/biblatex-murray.md
@@ -55,8 +55,9 @@ properties as a function of core size. *Langmuir*, *14*(1), 17–30.
 ---
 nocite: "[@*]"
 references:
-- annote: An article entry with author authors. By default, long author
-    and editor lists are automatically truncated. This is configurable
+- annote: An article entry with `\arabic{author}`{=latex} authors. By
+    default, long author and editor lists are automatically truncated.
+    This is configurable
   author:
   - family: Hostetler
     given: Michael J.
-- 
cgit v1.2.3


From 9223788a05fe619d567bcbdf4cb31db63de86f32 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Fri, 29 Jan 2021 18:29:17 -0800
Subject: Markdown writer: handle math right before digit.

We insert an HTML comment to avoid a `$` right before
a digit, which pandoc will not recognize as a math delimiter.
---
 src/Text/Pandoc/Writers/Markdown.hs | 6 +++++-
 test/command/7058.md                | 6 ++++++
 2 files changed, 11 insertions(+), 1 deletion(-)
 create mode 100644 test/command/7058.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/Markdown.hs b/src/Text/Pandoc/Writers/Markdown.hs
index de9075ac4..898905603 100644
--- a/src/Text/Pandoc/Writers/Markdown.hs
+++ b/src/Text/Pandoc/Writers/Markdown.hs
@@ -22,7 +22,7 @@ module Text.Pandoc.Writers.Markdown (
   writePlain) where
 import Control.Monad.Reader
 import Control.Monad.State.Strict
-import Data.Char (isAlphaNum)
+import Data.Char (isAlphaNum, isDigit)
 import Data.Default
 import Data.List (find, intersperse, sortOn, transpose)
 import qualified Data.Map as M
@@ -987,6 +987,10 @@ inlineListToMarkdown opts lst = do
   inlist <- asks envInList
   go (if inlist then avoidBadWrapsInList lst else lst)
   where go [] = return empty
+        go (x@Math{}:y@(Str t):zs)
+          | T.all isDigit (T.take 1 t) -- starts with digit -- see #7058
+          = liftM2 (<>) (inlineToMarkdown opts x)
+              (go (RawInline (Format "html") "<!-- -->" : y : zs))
         go (i:is) = case i of
             Link {} -> case is of
                 -- If a link is followed by another link, or '[', '(' or ':'
diff --git a/test/command/7058.md b/test/command/7058.md
new file mode 100644
index 000000000..69e5dd445
--- /dev/null
+++ b/test/command/7058.md
@@ -0,0 +1,6 @@
+```
+% pandoc -f latex -t markdown
+5\(-\)8 \(x\)
+^D
+5$-$`<!-- -->`{=html}8 $x$
+```
-- 
cgit v1.2.3


From d1875b69ec136c19d9b2336ff57e360ff7b7bef6 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sun, 31 Jan 2021 12:05:46 -0800
Subject: RST reader: fix handling of header in CSV tables.

The interpretation of this line is not affected
by the delim option. Closes #7064.
---
 src/Text/Pandoc/Readers/RST.hs |  9 +++++----
 test/command/7064.md           | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 4 deletions(-)
 create mode 100644 test/command/7064.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/RST.hs b/src/Text/Pandoc/Readers/RST.hs
index 707af905f..0f32d993c 100644
--- a/src/Text/Pandoc/Readers/RST.hs
+++ b/src/Text/Pandoc/Readers/RST.hs
@@ -877,10 +877,11 @@ csvTableDirective top fields rawcsv = do
                     (bs, _) <- fetchItem u
                     return $ UTF8.toText bs
                   Nothing -> return rawcsv
-  let res = parseCSV opts (case explicitHeader of
-                              Just h  -> h <> "\n" <> rawcsv'
-                              Nothing -> rawcsv')
-  case res of
+  let header' = case explicitHeader of
+                  Just h  -> parseCSV defaultCSVOptions h
+                  Nothing -> Right []
+  let res = parseCSV opts rawcsv'
+  case (<>) <$> header' <*> res of
        Left e  ->
          throwError $ PandocParsecError "csv table" e
        Right rawrows -> do
diff --git a/test/command/7064.md b/test/command/7064.md
new file mode 100644
index 000000000..58b72e363
--- /dev/null
+++ b/test/command/7064.md
@@ -0,0 +1,32 @@
+```
+% pandoc -f rst -t html
+.. csv-table:: Changes
+    :header: "Version", "Date", "Description"
+    :widths: 15, 15, 70
+    :delim: $
+
+    0.1.0 $ 18/02/2013 $ Initial Release
+^D
+<table>
+<caption>Changes</caption>
+<colgroup>
+<col style="width: 15%" />
+<col style="width: 15%" />
+<col style="width: 70%" />
+</colgroup>
+<thead>
+<tr class="header">
+<th>Version</th>
+<th>Date</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr class="odd">
+<td>0.1.0</td>
+<td>18/02/2013</td>
+<td>Initial Release</td>
+</tr>
+</tbody>
+</table>
+```
-- 
cgit v1.2.3


From b239c89a82b66abc55bf7c08e37492938c817c56 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Mon, 1 Feb 2021 08:43:26 -0800
Subject: BibTeX writer fixes.  Closes #7067.

+ Require citeproc 0.3.0.7, which correctly titlecases when titles
  contain non-ASCII characters.
+ Correctly handle 'pages' (= 'page' in CSL).
+ Correctly handle BibLaTeX 'langid' (= 'language' in CSL).
+ In BibTeX output, protect foreign titles since there's no language
  field.
---
 pandoc.cabal                       |  2 +-
 src/Text/Pandoc/Citeproc/BibTeX.hs | 21 ++++++---
 stack.yaml                         |  4 +-
 test/command/7067.md               | 90 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 108 insertions(+), 9 deletions(-)
 create mode 100644 test/command/7067.md

(limited to 'test/command')

diff --git a/pandoc.cabal b/pandoc.cabal
index 0890318e3..1bdd2a6b3 100644
--- a/pandoc.cabal
+++ b/pandoc.cabal
@@ -410,7 +410,7 @@ library
                  blaze-markup          >= 0.8      && < 0.9,
                  bytestring            >= 0.9      && < 0.12,
                  case-insensitive      >= 1.2      && < 1.3,
-                 citeproc              >= 0.3.0.6  && < 0.4,
+                 citeproc              >= 0.3.0.7  && < 0.4,
                  commonmark            >= 0.1.1.3  && < 0.2,
                  commonmark-extensions >= 0.2.0.4  && < 0.3,
                  commonmark-pandoc     >= 0.2      && < 0.3,
diff --git a/src/Text/Pandoc/Citeproc/BibTeX.hs b/src/Text/Pandoc/Citeproc/BibTeX.hs
index 5b9068378..2b43fffb6 100644
--- a/src/Text/Pandoc/Citeproc/BibTeX.hs
+++ b/src/Text/Pandoc/Citeproc/BibTeX.hs
@@ -115,7 +115,7 @@ writeBibtexString opts variant mblang ref =
       "motion_picture"    | variant == Biblatex -> "movie"
       "review"             | variant == Biblatex -> "review"
       _                   -> "misc"
-  
+
   mbSubtype =
     case referenceType ref of
       "article-magazine"  -> Just "magazine"
@@ -149,7 +149,7 @@ writeBibtexString opts variant mblang ref =
            , "type"
            , "entrysubtype"
            , "note"
-           , "language"
+           , "langid"
            , "abstract"
            , "keywords"
            ]
@@ -202,12 +202,19 @@ writeBibtexString opts variant mblang ref =
                    [ (", " <>) <$> nameGiven name,
                      nameDroppingParticle name ]
 
-  titlecase = case mblang of
+  mblang' = (parseLang <$> getVariableAsText "language") <|> mblang
+
+  titlecase = case mblang' of
                 Just (Lang "en" _) -> titlecase'
                 Nothing            -> titlecase'
-                _                  -> id
-
-  titlecase' = addTextCase mblang TitleCase .
+                _                  ->
+                  case variant of
+                    Bibtex         -> B.spanWith nullAttr
+                     -- BibTex lacks a language field, so we wrap non-English
+                     -- titles in {} to protect case.
+                    Biblatex       -> id
+
+  titlecase' = addTextCase mblang' TitleCase .
     (\ils -> B.fromList
                (case B.toList ils of
                   Str t : xs -> Str t : Walk.walk spanAroundCapitalizedWords xs
@@ -299,6 +306,8 @@ writeBibtexString opts variant mblang ref =
   getContentsFor "urldate"  = getVariable "accessed" >>= toLaTeX . valToInlines
   getContentsFor "year"  = getVariable "issued" >>= getYear
   getContentsFor "month"  = getVariable "issued" >>= getMonth
+  getContentsFor "pages"  = getVariable "page" >>= toLaTeX . valToInlines
+  getContentsFor "langid"  = getVariable "language" >>= toLaTeX . valToInlines
   getContentsFor "number" = (getVariable "number"
                          <|> getVariable "collection-number"
                          <|> getVariable "issue") >>= toLaTeX . valToInlines
diff --git a/stack.yaml b/stack.yaml
index 6c8c39fa7..dc6fa6517 100644
--- a/stack.yaml
+++ b/stack.yaml
@@ -19,10 +19,10 @@ extra-deps:
 - doctemplates-0.9
 - texmath-0.12.1
 - tasty-bench-0.1
-- citeproc-0.3.0.6
+- citeproc-0.3.0.7
 #- citeproc:
 #  git: https://github.com/jgm/citeproc.git
-#  commit: da1f9702fa70d7e1bad9a796f897e920e2d578f7
+#  commit: feb3b7580c6738eec3b23921f7c1739cfba611aa
 ghc-options:
    "$locals": -fhide-source-paths -Wno-missing-home-modules
 resolver: lts-16.23
diff --git a/test/command/7067.md b/test/command/7067.md
new file mode 100644
index 000000000..a66e3aa1f
--- /dev/null
+++ b/test/command/7067.md
@@ -0,0 +1,90 @@
+```
+% pandoc -t biblatex
+---
+references:
+- id: garaud
+  author:
+    - family: Garaud
+      given: Marcel
+  container-title: Bulletin de la Societé des antiquaires de l’Ouest
+  collection-title: 4
+  issued:
+    - year: 1967
+  language: fr-FR
+  page: 11-27
+  title: Recherches sur les défrichements dans la Gâtine poitevine aux XI^e^ et XII^e^ siècles
+  type: article-journal
+  volume: 9
+...
+^D
+@article{garaud,
+  author = {Garaud, Marcel},
+  title = {Recherches sur les défrichements dans la Gâtine poitevine aux
+XI\textsuperscript{e} et XII\textsuperscript{e} siècles},
+  journal = {Bulletin de la Societé des antiquaires de l’Ouest},
+  series = {4},
+  volume = {9},
+  pages = {11-27},
+  date = {1967},
+  langid = {fr-FR}
+}
+```
+
+```
+% pandoc -t bibtex
+---
+references:
+- id: garaud
+  author:
+    - family: Garaud
+      given: Marcel
+  container-title: Bulletin de la Société des antiquaires de l’Ouest
+  collection-title: 4
+  issued:
+    - year: 1967
+  language: fr-FR
+  page: 11-27
+  title: Recherches sur les défrichements dans la Gâtine poitevine aux XI^e^ et XII^e^ siècles
+  type: article-journal
+  volume: 9
+...
+^D
+@article{garaud,
+  author = {Garaud, Marcel},
+  title = {{Recherches sur les défrichements dans la Gâtine poitevine aux
+XI\textsuperscript{e} et XII\textsuperscript{e} siècles}},
+  journal = {Bulletin de la Société des antiquaires de l’Ouest},
+  series = {4},
+  volume = {9},
+  pages = {11-27},
+  year = {1967}
+}
+```
+
+This tests the titlecasing of a word with an accented second letter:
+```
+% pandoc -t bibtex
+---
+references:
+- id: garaud
+  author:
+    - family: Garaud
+      given: Marcel
+  container-title: English Journal
+  issued:
+    - year: 1967
+  language: en-US
+  title: Research on the défrichements in the Gâtine poitevine
+  type: article-journal
+  volume: 9
+...
+^D
+@article{garaud,
+  author = {Garaud, Marcel},
+  title = {Research on the Défrichements in the {Gâtine} Poitevine},
+  journal = {English Journal},
+  volume = {9},
+  year = {1967}
+}
+```
+
-- 
cgit v1.2.3


From 02d3c71e7224853ecabaa9ac4cd947ec2ac1e579 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Mon, 1 Feb 2021 18:02:17 -0800
Subject: BibTeX writer: use doclayout and doctemplate.

This change allows bibtex/biblatex output to wrap as other
formats do, depending on the settings of `--wrap` and `--columns`.

It also introduces default templates for bibtex and biblatex,
which allow for using the variables `header-include`, `include-before`
or `include-after` (or alternatively the command line options
`--include-in-header`, `--include-before-body`, `--include-after-body`)
to insert content into the generated bibtex/biblatex.

This change requires a change in the return type of the unexported
`T.P.Citeproc.writeBibTeXString` from `Text` to `Doc Text`.

Closes #7068.
---
 data/templates/default.biblatex    | 10 ++++++++++
 pandoc.cabal                       |  2 ++
 src/Text/Pandoc/Citeproc/BibTeX.hs | 41 +++++++++++++++++++++-----------------
 src/Text/Pandoc/Templates.hs       |  2 --
 src/Text/Pandoc/Writers/BibTeX.hs  | 19 +++++++++++++++---
 test/command/7067.md               |  6 +++---
 6 files changed, 54 insertions(+), 26 deletions(-)
 create mode 100644 data/templates/default.biblatex

(limited to 'test/command')

diff --git a/data/templates/default.biblatex b/data/templates/default.biblatex
new file mode 100644
index 000000000..6bf2632d8
--- /dev/null
+++ b/data/templates/default.biblatex
@@ -0,0 +1,10 @@
+$for(header-includes)$
+$header-includes$
+$endfor$
+$for(include-before)$
+$include-before$
+$endfor$
+$body$
+$for(include-after)$
+$include-after$
+$endfor$
diff --git a/pandoc.cabal b/pandoc.cabal
index 1bdd2a6b3..2f8c255a8 100644
--- a/pandoc.cabal
+++ b/pandoc.cabal
@@ -56,6 +56,8 @@ data-files:
                  data/templates/default.icml
                  data/templates/default.opml
                  data/templates/default.latex
+                 data/templates/default.bibtex
+                 data/templates/default.biblatex
                  data/templates/default.context
                  data/templates/default.texinfo
                  data/templates/default.jira
diff --git a/src/Text/Pandoc/Citeproc/BibTeX.hs b/src/Text/Pandoc/Citeproc/BibTeX.hs
index 2b43fffb6..416fe439e 100644
--- a/src/Text/Pandoc/Citeproc/BibTeX.hs
+++ b/src/Text/Pandoc/Citeproc/BibTeX.hs
@@ -52,6 +52,8 @@ import           Data.Char              (isAlphaNum, isDigit, isLetter,
 import           Data.List              (foldl', intercalate, intersperse)
 import           Safe                   (readMay)
 import           Text.Printf            (printf)
+import           Text.DocLayout         (literal, hsep, nest, hang, Doc(..),
+                                         braces, ($$), cr)
 
 data Variant = Bibtex | Biblatex
   deriving (Show, Eq, Ord)
@@ -77,10 +79,11 @@ writeBibtexString :: WriterOptions       -- ^ options (for writing LaTex)
                   -> Variant             -- ^ bibtex or biblatex
                   -> Maybe Lang          -- ^ Language
                   -> Reference Inlines   -- ^ Reference to write
-                  -> Text
+                  -> Doc Text
 writeBibtexString opts variant mblang ref =
-  "@" <> bibtexType <> "{" <> unItemId (referenceId ref) <> ",\n  " <>
-  renderFields fs <> "\n}\n"
+  "@" <> bibtexType <> "{" <> literal (unItemId (referenceId ref)) <> ","
+  $$ nest 2 (renderFields fs)
+  $$ "}" <> cr
 
  where
   bibtexType =
@@ -231,10 +234,12 @@ writeBibtexString opts variant mblang ref =
   toLaTeX x =
     case runPure (writeLaTeX opts $ doc (B.plain x)) of
            Left _  -> Nothing
-           Right t -> Just t
+           Right t -> Just $ hsep . map literal $ T.words t
 
-  renderField name = (\contents -> name <> " = {" <> contents <> "}")
-                      <$> getContentsFor name
+  renderField :: Text -> Maybe (Doc Text)
+  renderField name =
+    (((literal name) <>) . hang 2 " = " . braces)
+      <$> getContentsFor name
 
   getVariable v = lookupVariable (toVariable v) ref
 
@@ -248,10 +253,10 @@ writeBibtexString opts variant mblang ref =
            Nothing ->
              case dateParts date of
                [DateParts (y1:_), DateParts (y2:_)] ->
-                 Just (T.pack (printf "%04d" y1) <> "--" <>
+                 Just $ literal (T.pack (printf "%04d" y1) <> "--" <>
                         T.pack (printf "%04d" y2))
                [DateParts (y1:_)] ->
-                 Just (T.pack (printf "%04d" y1))
+                 Just $ literal (T.pack (printf "%04d" y1))
                _ -> Nothing
        _ -> Nothing
 
@@ -274,19 +279,19 @@ writeBibtexString opts variant mblang ref =
        DateVal date ->
          case dateParts date of
            [DateParts (_:m1:_), DateParts (_:m2:_)] ->
-             Just (toMonth m1 <> "--" <> toMonth m2)
-           [DateParts (_:m1:_)] -> Just (toMonth m1)
+             Just $ literal (toMonth m1 <> "--" <> toMonth m2)
+           [DateParts (_:m1:_)] -> Just $ literal (toMonth m1)
            _ -> Nothing
        _ -> Nothing
 
-  getContentsFor :: Text -> Maybe Text
+  getContentsFor :: Text -> Maybe (Doc Text)
   getContentsFor "type" =
     getVariableAsText "genre" >>=
        \case
           "mathesis"  -> Just "mastersthesis"
           "phdthesis" -> Just "phdthesis"
           _           -> Nothing
-  getContentsFor "entrysubtype" = mbSubtype
+  getContentsFor "entrysubtype" = literal <$> mbSubtype
   getContentsFor "journal"
     | bibtexType `elem` ["article", "periodical", "suppperiodical", "review"]
       = getVariable "container-title" >>= toLaTeX . valToInlines
@@ -314,7 +319,7 @@ writeBibtexString opts variant mblang ref =
 
   getContentsFor x = getVariable x >>=
     if isURL x
-       then Just . stringify . valToInlines
+       then Just . literal . stringify . valToInlines
        else toLaTeX .
             (if x == "title"
                 then titlecase
@@ -323,7 +328,7 @@ writeBibtexString opts variant mblang ref =
 
   isURL x = x `elem` ["url","doi","issn","isbn"]
 
-  renderFields = T.intercalate ",\n  " . mapMaybe renderField
+  renderFields = mconcat . intersperse ("," <> cr) . mapMaybe renderField
 
 defaultLang :: Lang
 defaultLang = Lang "en" (Just "US")
@@ -1038,14 +1043,14 @@ getOldDate prefix = do
   let dateparts = filter (\x -> x /= DateParts [])
                   $ map toDateParts [(year',month',day'),
                                      (endyear',endmonth',endday')]
-  literal <- if null dateparts
-                then Just <$> getRawField (prefix <> "year")
-                else return Nothing
+  literal' <- if null dateparts
+                 then Just <$> getRawField (prefix <> "year")
+                 else return Nothing
   return $
     Date { dateParts = dateparts
          , dateCirca = False
          , dateSeason = Nothing
-         , dateLiteral = literal }
+         , dateLiteral = literal' }
 
 getRawField :: Text -> Bib Text
 getRawField f = do
diff --git a/src/Text/Pandoc/Templates.hs b/src/Text/Pandoc/Templates.hs
index 3e539bff7..e83f26329 100644
--- a/src/Text/Pandoc/Templates.hs
+++ b/src/Text/Pandoc/Templates.hs
@@ -81,8 +81,6 @@ getDefaultTemplate writer = do
   case format of
        "native"  -> return ""
        "csljson" -> return ""
-       "bibtex"  -> return ""
-       "biblatex" -> return ""
        "json"    -> return ""
        "docx"    -> return ""
        "fb2"     -> return ""
diff --git a/src/Text/Pandoc/Writers/BibTeX.hs b/src/Text/Pandoc/Writers/BibTeX.hs
index e1cb47ca1..b9ae0c13a 100644
--- a/src/Text/Pandoc/Writers/BibTeX.hs
+++ b/src/Text/Pandoc/Writers/BibTeX.hs
@@ -25,7 +25,11 @@ import Citeproc (parseLang)
 import Text.Pandoc.Class (PandocMonad)
 import Text.Pandoc.Citeproc.BibTeX as BibTeX
 import Text.Pandoc.Citeproc.MetaValue (metaValueToReference)
-import Text.Pandoc.Writers.Shared (lookupMetaString)
+import Text.Pandoc.Writers.Shared (lookupMetaString, defField,
+                                    addVariablesToContext)
+import Text.DocLayout (render, vcat)
+import Text.DocTemplates (Context(..))
+import Text.Pandoc.Templates (renderTemplate)
 
 -- | Write BibTeX based on the references metadata from a Pandoc document.
 writeBibTeX :: PandocMonad m => WriterOptions -> Pandoc -> m Text
@@ -43,6 +47,15 @@ writeBibTeX' variant opts (Pandoc meta _) = do
   let refs = case lookupMeta "references" meta of
                Just (MetaList xs) -> mapMaybe metaValueToReference xs
                _ -> []
-  return $ mconcat $
-    map (BibTeX.writeBibtexString opts variant mblang) refs
+  let main = vcat $ map (BibTeX.writeBibtexString opts variant mblang) refs
+  let context  = defField "body" main
+                 $ addVariablesToContext opts (mempty :: Context Text)
+  let colwidth = if writerWrapText opts == WrapAuto
+                    then Just $ writerColumns opts
+                    else Nothing
+  return $ render colwidth $
+    case writerTemplate opts of
+      Nothing  -> main
+      Just tpl -> renderTemplate tpl context
+
 
diff --git a/test/command/7067.md b/test/command/7067.md
index a66e3aa1f..34b3bb8ea 100644
--- a/test/command/7067.md
+++ b/test/command/7067.md
@@ -20,7 +20,7 @@ references:
 @article{garaud,
   author = {Garaud, Marcel},
   title = {Recherches sur les défrichements dans la Gâtine poitevine aux
-XI\textsuperscript{e} et XII\textsuperscript{e} siècles},
+    XI\textsuperscript{e} et XII\textsuperscript{e} siècles},
   journal = {Bulletin de la Societé des antiquaires de l’Ouest},
   series = {4},
   volume = {9},
@@ -51,8 +51,8 @@ references:
 ^D
 @article{garaud,
   author = {Garaud, Marcel},
-  title = {{Recherches sur les défrichements dans la Gâtine poitevine aux
-XI\textsuperscript{e} et XII\textsuperscript{e} siècles}},
+  title = {{Recherches sur les défrichements dans la Gâtine poitevine
+    aux XI\textsuperscript{e} et XII\textsuperscript{e} siècles}},
   journal = {Bulletin de la Société des antiquaires de l’Ouest},
   series = {4},
   volume = {9},
-- 
cgit v1.2.3


From 8e9131db4edfda6deafdf94939b907e4dce3cbaa Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sat, 6 Feb 2021 21:52:12 -0800
Subject: Markdown reader: improved handling of mmd link attributes in
 references.

Previously they only worked for links that had titles.  Closes #7080.
---
 src/Text/Pandoc/Readers/Markdown.hs | 2 ++
 test/command/7080.md                | 8 ++++++++
 2 files changed, 10 insertions(+)
 create mode 100644 test/command/7080.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/Markdown.hs b/src/Text/Pandoc/Readers/Markdown.hs
index 5c3a21bb7..e46553dd8 100644
--- a/src/Text/Pandoc/Readers/Markdown.hs
+++ b/src/Text/Pandoc/Readers/Markdown.hs
@@ -334,6 +334,8 @@ referenceKey = try $ do
                     skipMany spaceChar
                     notFollowedBy' referenceTitle
                     notFollowedBy' $ guardEnabled Ext_link_attributes >> attributes
+                    notFollowedBy' $ guardEnabled Ext_mmd_link_attributes >>
+                                     try (spnl <* keyValAttr)
                     notFollowedBy' (() <$ reference)
                     many1Char $ notFollowedBy space >> litChar
   let betweenAngles = try $ char '<' >> manyTillChar litChar (char '>')
diff --git a/test/command/7080.md b/test/command/7080.md
new file mode 100644
index 000000000..0727c1083
--- /dev/null
+++ b/test/command/7080.md
@@ -0,0 +1,8 @@
+```
+% pandoc -f markdown_mmd -t native
+![][image]
+
+[image]: image.png width=100px height=150px
+^D
+[Para [Image ("",[],[("width","100px"),("height","150px")]) [] ("image.png","")]]
+```
-- 
cgit v1.2.3


From 8ca191604dcd13af27c11d2da225da646ebce6fc Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Mon, 8 Feb 2021 23:35:19 -0800
Subject: Add new unexported module T.P.XMLParser.

This exports functions that uses xml-conduit's parser to
produce an xml-light Element or [Content].  This allows
existing pandoc code to use a better parser without
much modification.

The new parser is used in all places where xml-light's
parser was previously used.  Benchmarks show a significant
performance improvement in parsing XML-based formats
(especially ODT and FB2).

Note that the xml-light types use String, so the
conversion from xml-conduit types involves a lot
of extra allocation.  It would be desirable to
avoid that in the future by gradually switching
to using xml-conduit directly. This can be done
module by module.

The new parser also reports errors, which we report
when possible.

A new constructor PandocXMLError has been added to
PandocError in T.P.Error [API change].

Closes #7091, which was the main stimulus.

These changes revealed the need for some changes
in the tests.  The docbook-reader.docbook test
lacked definitions for the entities it used; these
have been added. And the docx golden tests have been
updated, because the new parser does not preserve
the order of attributes.

Add entity defs to docbook-reader.docbook.

Update golden tests for docx.
---
 MANUAL.txt                                         |   1 +
 pandoc.cabal                                       |   2 +
 src/Text/Pandoc/Error.hs                           |   3 +
 src/Text/Pandoc/ImageSize.hs                       |   5 +-
 src/Text/Pandoc/Readers/DocBook.hs                 |  52 ++++++++++------
 src/Text/Pandoc/Readers/Docx/Parse.hs              |  21 ++++---
 src/Text/Pandoc/Readers/Docx/Parse/Styles.hs       |  28 +++++----
 src/Text/Pandoc/Readers/EPUB.hs                    |  17 ++++--
 src/Text/Pandoc/Readers/FB2.hs                     |  10 ++--
 src/Text/Pandoc/Readers/JATS.hs                    |   9 ++-
 src/Text/Pandoc/Readers/OPML.hs                    |  10 +++-
 src/Text/Pandoc/Readers/Odt.hs                     |  24 ++++----
 src/Text/Pandoc/Writers/EPUB.hs                    |  13 +++-
 src/Text/Pandoc/Writers/FB2.hs                     |  11 +++-
 src/Text/Pandoc/Writers/ODT.hs                     |  29 +++++----
 src/Text/Pandoc/Writers/OOXML.hs                   |   9 +--
 src/Text/Pandoc/Writers/Powerpoint/Output.hs       |   4 +-
 src/Text/Pandoc/XMLParser.hs                       |  66 +++++++++++++++++++++
 test/Tests/Readers/JATS.hs                         |   1 +
 test/command/5321.md                               |   4 +-
 test/docbook-reader.docbook                        |   7 ++-
 test/docx/golden/block_quotes.docx                 | Bin 10092 -> 10098 bytes
 test/docx/golden/codeblock.docx                    | Bin 9944 -> 9950 bytes
 test/docx/golden/comments.docx                     | Bin 10279 -> 10285 bytes
 test/docx/golden/custom_style_no_reference.docx    | Bin 10042 -> 10048 bytes
 test/docx/golden/custom_style_preserve.docx        | Bin 10666 -> 10673 bytes
 test/docx/golden/custom_style_reference.docx       | Bin 12434 -> 12434 bytes
 test/docx/golden/definition_list.docx              | Bin 9941 -> 9947 bytes
 .../golden/document-properties-short-desc.docx     | Bin 9947 -> 9953 bytes
 test/docx/golden/document-properties.docx          | Bin 10423 -> 10429 bytes
 test/docx/golden/headers.docx                      | Bin 10080 -> 10086 bytes
 test/docx/golden/image.docx                        | Bin 26758 -> 26764 bytes
 test/docx/golden/inline_code.docx                  | Bin 9880 -> 9886 bytes
 test/docx/golden/inline_formatting.docx            | Bin 10060 -> 10066 bytes
 test/docx/golden/inline_images.docx                | Bin 26816 -> 26822 bytes
 test/docx/golden/link_in_notes.docx                | Bin 10101 -> 10107 bytes
 test/docx/golden/links.docx                        | Bin 10276 -> 10282 bytes
 test/docx/golden/lists.docx                        | Bin 10352 -> 10358 bytes
 test/docx/golden/lists_continuing.docx             | Bin 10143 -> 10149 bytes
 test/docx/golden/lists_multiple_initial.docx       | Bin 10232 -> 10238 bytes
 test/docx/golden/lists_restarting.docx             | Bin 10144 -> 10150 bytes
 test/docx/golden/nested_anchors_in_header.docx     | Bin 10239 -> 10245 bytes
 test/docx/golden/notes.docx                        | Bin 10046 -> 10052 bytes
 test/docx/golden/raw-blocks.docx                   | Bin 9980 -> 9986 bytes
 test/docx/golden/raw-bookmarks.docx                | Bin 10115 -> 10121 bytes
 test/docx/golden/table_one_row.docx                | Bin 9932 -> 9938 bytes
 test/docx/golden/table_with_list_cell.docx         | Bin 10249 -> 10255 bytes
 test/docx/golden/tables.docx                       | Bin 10266 -> 10272 bytes
 test/docx/golden/track_changes_deletion.docx       | Bin 9924 -> 9930 bytes
 test/docx/golden/track_changes_insertion.docx      | Bin 9907 -> 9913 bytes
 test/docx/golden/track_changes_move.docx           | Bin 9941 -> 9947 bytes
 .../golden/track_changes_scrubbed_metadata.docx    | Bin 10053 -> 10059 bytes
 test/docx/golden/unicode.docx                      | Bin 9865 -> 9871 bytes
 test/docx/golden/verbatim_subsuper.docx            | Bin 9913 -> 9919 bytes
 test/jats-reader.native                            |   2 +-
 test/jats-reader.xml                               |   1 +
 test/pptx/code-custom.pptx                         | Bin 28230 -> 28221 bytes
 test/pptx/code-custom_templated.pptx               | Bin 395524 -> 395516 bytes
 test/pptx/code.pptx                                | Bin 28229 -> 28220 bytes
 test/pptx/code_templated.pptx                      | Bin 395522 -> 395514 bytes
 test/pptx/document-properties-short-desc.pptx      | Bin 27012 -> 27004 bytes
 .../document-properties-short-desc_templated.pptx  | Bin 394298 -> 394288 bytes
 test/pptx/document-properties.pptx                 | Bin 27417 -> 27408 bytes
 test/pptx/document-properties_templated.pptx       | Bin 394701 -> 394691 bytes
 test/pptx/endnotes.pptx                            | Bin 26969 -> 26962 bytes
 test/pptx/endnotes_templated.pptx                  | Bin 394262 -> 394253 bytes
 test/pptx/endnotes_toc.pptx                        | Bin 27892 -> 27789 bytes
 test/pptx/endnotes_toc_templated.pptx              | Bin 395186 -> 395083 bytes
 test/pptx/images.pptx                              | Bin 44626 -> 44619 bytes
 test/pptx/images_templated.pptx                    | Bin 411916 -> 411909 bytes
 test/pptx/inline_formatting.pptx                   | Bin 26156 -> 26148 bytes
 test/pptx/inline_formatting_templated.pptx         | Bin 393447 -> 393438 bytes
 test/pptx/lists.pptx                               | Bin 27056 -> 27049 bytes
 test/pptx/lists_templated.pptx                     | Bin 394349 -> 394340 bytes
 test/pptx/raw_ooxml.pptx                           | Bin 26948 -> 26940 bytes
 test/pptx/raw_ooxml_templated.pptx                 | Bin 394240 -> 394231 bytes
 test/pptx/remove_empty_slides.pptx                 | Bin 44073 -> 44065 bytes
 test/pptx/remove_empty_slides_templated.pptx       | Bin 411359 -> 411352 bytes
 test/pptx/slide_breaks.pptx                        | Bin 28582 -> 28575 bytes
 test/pptx/slide_breaks_slide_level_1.pptx          | Bin 27751 -> 27744 bytes
 .../pptx/slide_breaks_slide_level_1_templated.pptx | Bin 395045 -> 395038 bytes
 test/pptx/slide_breaks_templated.pptx              | Bin 395875 -> 395868 bytes
 test/pptx/slide_breaks_toc.pptx                    | Bin 29539 -> 29532 bytes
 test/pptx/slide_breaks_toc_templated.pptx          | Bin 396833 -> 396826 bytes
 test/pptx/speaker_notes.pptx                       | Bin 35444 -> 35436 bytes
 test/pptx/speaker_notes_after_metadata.pptx        | Bin 31683 -> 31675 bytes
 .../speaker_notes_after_metadata_templated.pptx    | Bin 398964 -> 398955 bytes
 test/pptx/speaker_notes_afterheader.pptx           | Bin 30700 -> 30691 bytes
 test/pptx/speaker_notes_afterheader_templated.pptx | Bin 397988 -> 397979 bytes
 test/pptx/speaker_notes_afterseps.pptx             | Bin 51612 -> 51604 bytes
 test/pptx/speaker_notes_afterseps_templated.pptx   | Bin 418903 -> 418896 bytes
 test/pptx/speaker_notes_templated.pptx             | Bin 402736 -> 402728 bytes
 test/pptx/start_numbering_at.pptx                  | Bin 27031 -> 27023 bytes
 test/pptx/start_numbering_at_templated.pptx        | Bin 394323 -> 394314 bytes
 test/pptx/tables.pptx                              | Bin 27573 -> 27566 bytes
 test/pptx/tables_templated.pptx                    | Bin 394868 -> 394859 bytes
 test/pptx/two_column.pptx                          | Bin 26075 -> 26065 bytes
 test/pptx/two_column_templated.pptx                | Bin 393366 -> 393355 bytes
 98 files changed, 238 insertions(+), 91 deletions(-)
 create mode 100644 src/Text/Pandoc/XMLParser.hs

(limited to 'test/command')

diff --git a/MANUAL.txt b/MANUAL.txt
index 5b90d039a..dc3b4ca77 100644
--- a/MANUAL.txt
+++ b/MANUAL.txt
@@ -1464,6 +1464,7 @@ Nonzero exit codes have the following meanings:
    24 PandocCiteprocError
    31 PandocEpubSubdirectoryError
    43 PandocPDFError
+   44 PandocXMLError
    47 PandocPDFProgramNotFoundError
    61 PandocHttpError
    62 PandocShouldNeverHappenError
diff --git a/pandoc.cabal b/pandoc.cabal
index 72e7c2da5..e56456c68 100644
--- a/pandoc.cabal
+++ b/pandoc.cabal
@@ -493,6 +493,7 @@ library
                  unicode-transforms    >= 0.3      && < 0.4,
                  unordered-containers  >= 0.2      && < 0.3,
                  xml                   >= 1.3.12   && < 1.4,
+                 xml-conduit           >= 1.7      && < 1.10,
                  zip-archive           >= 0.2.3.4  && < 0.5,
                  zlib                  >= 0.5      && < 0.7
   if os(windows) && arch(i386)
@@ -686,6 +687,7 @@ library
                    Text.Pandoc.Lua.PandocLua,
                    Text.Pandoc.Lua.Util,
                    Text.Pandoc.Lua.Walk,
+                   Text.Pandoc.XMLParser,
                    Text.Pandoc.CSS,
                    Text.Pandoc.CSV,
                    Text.Pandoc.RoffChar,
diff --git a/src/Text/Pandoc/Error.hs b/src/Text/Pandoc/Error.hs
index 204cf15ca..831405f42 100644
--- a/src/Text/Pandoc/Error.hs
+++ b/src/Text/Pandoc/Error.hs
@@ -48,6 +48,7 @@ data PandocError = PandocIOError Text IOError
                  | PandocFailOnWarningError
                  | PandocPDFProgramNotFoundError Text
                  | PandocPDFError Text
+                 | PandocXMLError Text Text
                  | PandocFilterError Text Text
                  | PandocLuaError Text
                  | PandocCouldNotFindDataFileError Text
@@ -103,6 +104,8 @@ handleError (Left e) =
     PandocPDFProgramNotFoundError pdfprog -> err 47 $
         pdfprog <> " not found. Please select a different --pdf-engine or install " <> pdfprog
     PandocPDFError logmsg -> err 43 $ "Error producing PDF.\n" <> logmsg
+    PandocXMLError fp logmsg -> err 44 $ "Invalid XML" <>
+        (if T.null fp then "" else " in " <> fp) <> ":\n" <> logmsg
     PandocFilterError filtername msg -> err 83 $ "Error running filter " <>
         filtername <> ":\n" <> msg
     PandocLuaError msg -> err 84 $ "Error running Lua:\n" <> msg
diff --git a/src/Text/Pandoc/ImageSize.hs b/src/Text/Pandoc/ImageSize.hs
index e19958f6a..e0a1af8e8 100644
--- a/src/Text/Pandoc/ImageSize.hs
+++ b/src/Text/Pandoc/ImageSize.hs
@@ -45,7 +45,9 @@ import Text.Pandoc.Definition
 import Text.Pandoc.Options
 import qualified Text.Pandoc.UTF8 as UTF8
 import qualified Text.XML.Light as Xml
+import Text.Pandoc.XMLParser (parseXMLElement)
 import qualified Data.Text as T
+import qualified Data.Text.Lazy as TL
 import qualified Data.Text.Encoding as TE
 import Control.Applicative
 import qualified Data.Attoparsec.ByteString.Char8 as A
@@ -327,7 +329,8 @@ getSize img =
 
 svgSize :: WriterOptions -> ByteString -> Maybe ImageSize
 svgSize opts img = do
-  doc <- Xml.parseXMLDoc $ UTF8.toString img
+  doc <- either (const mzero) return $ parseXMLElement
+                                     $ TL.fromStrict $ UTF8.toText img
   let viewboxSize = do
         vb <- Xml.findAttrBy (== Xml.QName "viewBox" Nothing Nothing) doc
         [_,_,w,h] <- mapM safeRead (T.words (T.pack vb))
diff --git a/src/Text/Pandoc/Readers/DocBook.hs b/src/Text/Pandoc/Readers/DocBook.hs
index ada3e98ec..ad0108843 100644
--- a/src/Text/Pandoc/Readers/DocBook.hs
+++ b/src/Text/Pandoc/Readers/DocBook.hs
@@ -12,7 +12,7 @@ Conversion of DocBook XML to 'Pandoc' document.
 -}
 module Text.Pandoc.Readers.DocBook ( readDocBook ) where
 import Control.Monad.State.Strict
-import Data.Char (isSpace, toUpper)
+import Data.Char (isSpace, toUpper, isLetter)
 import Data.Default
 import Data.Either (rights)
 import Data.Foldable (asum)
@@ -21,7 +21,10 @@ import Data.List (intersperse,elemIndex)
 import Data.Maybe (fromMaybe,mapMaybe)
 import Data.Text (Text)
 import qualified Data.Text as T
+import qualified Data.Text.Lazy as TL
+import Control.Monad.Except (throwError)
 import Text.HTML.TagSoup.Entity (lookupEntity)
+import Text.Pandoc.Error (PandocError(..))
 import Text.Pandoc.Builder
 import Text.Pandoc.Class.PandocMonad (PandocMonad, report)
 import Text.Pandoc.Options
@@ -29,6 +32,7 @@ import Text.Pandoc.Logging (LogMessage(..))
 import Text.Pandoc.Shared (crFilter, safeRead, extractSpaces)
 import Text.TeXMath (readMathML, writeTeX)
 import Text.XML.Light
+import Text.Pandoc.XMLParser (parseXMLContents)
 
 {-
 
@@ -537,22 +541,25 @@ instance Default DBState where
 
 readDocBook :: PandocMonad m => ReaderOptions -> Text -> m Pandoc
 readDocBook _ inp = do
-  let tree = normalizeTree . parseXML . handleInstructions $ crFilter inp
+  tree <- either (throwError . PandocXMLError "") (return . normalizeTree) $
+            parseXMLContents (TL.fromStrict . handleInstructions $ crFilter inp)
   (bs, st') <- flip runStateT (def{ dbContent = tree }) $ mapM parseBlock tree
   return $ Pandoc (dbMeta st') (toList . mconcat $ bs)
 
--- We treat <?asciidoc-br?> specially (issue #1236), converting it
--- to <br/>, since xml-light doesn't parse the instruction correctly.
--- Other xml instructions are simply removed from the input stream.
+-- We treat certain processing instructions by converting them to tags
+-- beginning "pi-".
 handleInstructions :: Text -> Text
-handleInstructions = T.pack . handleInstructions' . T.unpack
-
-handleInstructions' :: String -> String
-handleInstructions' ('<':'?':'a':'s':'c':'i':'i':'d':'o':'c':'-':'b':'r':'?':'>':xs) = '<':'b':'r':'/':'>': handleInstructions' xs
-handleInstructions' xs = case break (=='<') xs of
-                             (ys, [])     -> ys
-                             ([], '<':zs) -> '<' : handleInstructions' zs
-                             (ys, zs)     -> ys ++ handleInstructions' zs
+handleInstructions t =
+  let (x,y) = T.breakOn "<?" t
+   in if T.null y
+         then x
+         else
+           let (w,z) = T.breakOn "?>" y
+            in (if T.takeWhile (\c -> isLetter c || c == '-')
+                    (T.drop 2 w) `elem` ["asciidoc-br", "dbfo"]
+                   then x <> "<pi-" <> T.drop 2 w <> "/>"
+                   else x <> w <> T.take 2 z) <>
+               handleInstructions (T.drop 2 z)
 
 getFigure :: PandocMonad m => Element -> DB m Blocks
 getFigure e = do
@@ -892,7 +899,11 @@ parseBlock (Elem e) =
         "subtitle" -> return mempty  -- handled in parent element
         _       -> skip >> getBlocks e
    where skip = do
-           lift $ report $ IgnoredElement $ T.pack $ qName (elName e)
+           let qn = T.pack $ qName $ elName e
+           let name = if "pi-" `T.isPrefixOf` qn
+                         then "<?" <> qn <> "?>"
+                         else qn
+           lift $ report $ IgnoredElement name
            return mempty
 
          codeBlockWithLang = do
@@ -964,7 +975,7 @@ parseBlock (Elem e) =
                                      cs -> map toAlignment cs
                       let parseWidth s = safeRead (T.filter (\x -> (x >= '0' && x <= '9')
                                                                    || x == '.') s)
-                      let textWidth = case filterChild (named "?dbfo") e of
+                      let textWidth = case filterChild (named "pi-dbfo") e of
                                         Just d  -> case attrValue "table-width" d of
                                                      "" -> 1.0
                                                      w  -> fromMaybe 100.0 (parseWidth w) / 100.0
@@ -1165,12 +1176,15 @@ parseInline (Elem e) =
         "title" -> return mempty
         "affiliation" -> skip
         -- Note: this isn't a real docbook tag; it's what we convert
-        -- <?asciidor-br?> to in handleInstructions, above.  A kludge to
-        -- work around xml-light's inability to parse an instruction.
-        "br" -> return linebreak
+        -- <?asciidor-br?> to in handleInstructions, above.
+        "pi-asciidoc-br" -> return linebreak
         _          -> skip >> innerInlines id
    where skip = do
-           lift $ report $ IgnoredElement $ T.pack $ qName (elName e)
+           let qn = T.pack $ qName $ elName e
+           let name = if "pi-" `T.isPrefixOf` qn
+                         then "<?" <> qn <> "?>"
+                         else qn
+           lift $ report $ IgnoredElement name
            return mempty
 
          innerInlines f = extractSpaces f . mconcat <$>
diff --git a/src/Text/Pandoc/Readers/Docx/Parse.hs b/src/Text/Pandoc/Readers/Docx/Parse.hs
index fdcffcc3f..056dab6c2 100644
--- a/src/Text/Pandoc/Readers/Docx/Parse.hs
+++ b/src/Text/Pandoc/Readers/Docx/Parse.hs
@@ -74,6 +74,7 @@ import Text.TeXMath.Readers.OMML (readOMML)
 import Text.TeXMath.Unicode.Fonts (Font (..), getUnicode, textToFont)
 import Text.XML.Light
 import qualified Text.XML.Light.Cursor as XMLC
+import Text.Pandoc.XMLParser (parseXMLElement)
 
 data ReaderEnv = ReaderEnv { envNotes         :: Notes
                            , envComments      :: Comments
@@ -343,10 +344,16 @@ archiveToDocxWithWarnings archive = do
     Right doc -> Right (Docx doc, stateWarnings st)
     Left e    -> Left e
 
+parseXMLFromEntry :: Entry -> Maybe Element
+parseXMLFromEntry entry =
+  case parseXMLElement (UTF8.toTextLazy (fromEntry entry)) of
+    Left _   -> Nothing
+    Right el -> Just el
+
 getDocumentXmlPath :: Archive -> Maybe FilePath
 getDocumentXmlPath zf = do
   entry <- findEntryByPath "_rels/.rels" zf
-  relsElem <- (parseXMLDoc . UTF8.toStringLazy . fromEntry) entry
+  relsElem <- parseXMLFromEntry entry
   let rels = filterChildrenName (\n -> qName n == "Relationship") relsElem
   rel <- find (\e -> findAttr (QName "Type" Nothing Nothing) e ==
                        Just "http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument")
@@ -362,7 +369,7 @@ archiveToDocument :: Archive -> D Document
 archiveToDocument zf = do
   docPath <- asks envDocXmlPath
   entry <- maybeToD $ findEntryByPath docPath zf
-  docElem <- maybeToD $ (parseXMLDoc . UTF8.toStringLazy . fromEntry) entry
+  docElem <- maybeToD $ parseXMLFromEntry entry
   let namespaces = elemToNameSpaces docElem
   bodyElem <- maybeToD $ findChildByName namespaces "w" "body" docElem
   let bodyElem' = fromMaybe bodyElem (walkDocument namespaces bodyElem)
@@ -401,9 +408,9 @@ constructBogusParStyleData stName = ParStyle
 archiveToNotes :: Archive -> Notes
 archiveToNotes zf =
   let fnElem = findEntryByPath "word/footnotes.xml" zf
-               >>= (parseXMLDoc . UTF8.toStringLazy . fromEntry)
+               >>= parseXMLFromEntry
       enElem = findEntryByPath "word/endnotes.xml" zf
-               >>= (parseXMLDoc . UTF8.toStringLazy . fromEntry)
+               >>= parseXMLFromEntry
       fn_namespaces = maybe [] elemToNameSpaces fnElem
       en_namespaces = maybe [] elemToNameSpaces enElem
       ns = unionBy (\x y -> fst x == fst y) fn_namespaces en_namespaces
@@ -415,7 +422,7 @@ archiveToNotes zf =
 archiveToComments :: Archive -> Comments
 archiveToComments zf =
   let cmtsElem = findEntryByPath "word/comments.xml" zf
-               >>= (parseXMLDoc . UTF8.toStringLazy . fromEntry)
+               >>= parseXMLFromEntry
       cmts_namespaces = maybe [] elemToNameSpaces cmtsElem
       cmts = elemToComments cmts_namespaces <$> (cmtsElem >>= walkDocument cmts_namespaces)
   in
@@ -445,7 +452,7 @@ filePathToRelationships :: Archive -> FilePath -> FilePath ->  [Relationship]
 filePathToRelationships ar docXmlPath fp
   | Just relType <- filePathToRelType fp docXmlPath
   , Just entry <- findEntryByPath fp ar
-  , Just relElems <- (parseXMLDoc . UTF8.toStringLazy . fromEntry) entry =
+  , Just relElems <- parseXMLFromEntry entry =
   mapMaybe (relElemToRelationship relType) $ elChildren relElems
 filePathToRelationships _ _ _ = []
 
@@ -527,7 +534,7 @@ archiveToNumbering' zf =
   case findEntryByPath "word/numbering.xml" zf of
     Nothing -> Just $ Numbering [] [] []
     Just entry -> do
-      numberingElem <- (parseXMLDoc . UTF8.toStringLazy . fromEntry) entry
+      numberingElem <- parseXMLFromEntry entry
       let namespaces = elemToNameSpaces numberingElem
           numElems = findChildrenByName namespaces "w" "num" numberingElem
           absNumElems = findChildrenByName namespaces "w" "abstractNum" numberingElem
diff --git a/src/Text/Pandoc/Readers/Docx/Parse/Styles.hs b/src/Text/Pandoc/Readers/Docx/Parse/Styles.hs
index 236167187..edade8654 100644
--- a/src/Text/Pandoc/Readers/Docx/Parse/Styles.hs
+++ b/src/Text/Pandoc/Readers/Docx/Parse/Styles.hs
@@ -53,6 +53,7 @@ import Data.Coerce
 import Text.Pandoc.Readers.Docx.Util
 import qualified Text.Pandoc.UTF8 as UTF8
 import Text.XML.Light
+import Text.Pandoc.XMLParser (parseXMLElement)
 
 newtype CharStyleId   = CharStyleId T.Text
   deriving (Show, Eq, Ord, IsString, FromStyleId)
@@ -135,19 +136,22 @@ defaultRunStyle = RunStyle { isBold = Nothing
                            , rParentStyle = Nothing
                            }
 
-archiveToStyles' :: (Ord k1, Ord k2, ElemToStyle a1, ElemToStyle a2) =>
-                    (a1 -> k1) -> (a2 -> k2) -> Archive -> (M.Map k1 a1, M.Map k2 a2)
+archiveToStyles'
+  :: (Ord k1, Ord k2, ElemToStyle a1, ElemToStyle a2)
+  => (a1 -> k1) -> (a2 -> k2) -> Archive -> (M.Map k1 a1, M.Map k2 a2)
 archiveToStyles' conv1 conv2 zf =
-  let stylesElem = findEntryByPath "word/styles.xml" zf >>=
-                   (parseXMLDoc . UTF8.toStringLazy . fromEntry)
-  in
-   case stylesElem of
-     Nothing -> (M.empty, M.empty)
-     Just styElem ->
-       let namespaces = elemToNameSpaces styElem
-       in
-        ( M.fromList $ map (\r -> (conv1 r, r)) $ buildBasedOnList namespaces styElem Nothing,
-          M.fromList $ map (\p -> (conv2 p, p)) $ buildBasedOnList namespaces styElem Nothing)
+  case findEntryByPath "word/styles.xml" zf of
+    Nothing -> (M.empty, M.empty)
+    Just entry ->
+      case parseXMLElement . UTF8.toTextLazy . fromEntry $ entry of
+        Left _ -> (M.empty, M.empty)
+        Right styElem ->
+          let namespaces = elemToNameSpaces styElem
+          in
+           ( M.fromList $ map (\r -> (conv1 r, r)) $
+               buildBasedOnList namespaces styElem Nothing,
+             M.fromList $ map (\p -> (conv2 p, p)) $
+               buildBasedOnList namespaces styElem Nothing)
 
 isBasedOnStyle :: (ElemToStyle a, FromStyleId (StyleId a)) => NameSpaces -> Element -> Maybe a -> Bool
 isBasedOnStyle ns element parentStyle
diff --git a/src/Text/Pandoc/Readers/EPUB.hs b/src/Text/Pandoc/Readers/EPUB.hs
index 5e3326e6d..369c4f0c9 100644
--- a/src/Text/Pandoc/Readers/EPUB.hs
+++ b/src/Text/Pandoc/Readers/EPUB.hs
@@ -17,7 +17,7 @@ module Text.Pandoc.Readers.EPUB
   (readEPUB)
   where
 
-import Codec.Archive.Zip (Archive (..), Entry, findEntryByPath, fromEntry,
+import Codec.Archive.Zip (Archive (..), Entry(..), findEntryByPath, fromEntry,
                           toArchiveOrFail)
 import Control.DeepSeq (NFData, deepseq)
 import Control.Monad (guard, liftM, liftM2, mplus)
@@ -41,9 +41,10 @@ import Text.Pandoc.MIME (MimeType)
 import Text.Pandoc.Options (ReaderOptions (..))
 import Text.Pandoc.Readers.HTML (readHtml)
 import Text.Pandoc.Shared (addMetaField, collapseFilePath, escapeURI)
-import qualified Text.Pandoc.UTF8 as UTF8 (toStringLazy)
+import qualified Text.Pandoc.UTF8 as UTF8 (toTextLazy)
 import Text.Pandoc.Walk (query, walk)
 import Text.XML.Light
+import Text.Pandoc.XMLParser (parseXMLElement)
 
 type Items = M.Map String (FilePath, MimeType)
 
@@ -181,7 +182,7 @@ renameMeta s         = T.pack s
 getManifest :: PandocMonad m => Archive -> m (String, Element)
 getManifest archive = do
   metaEntry <- findEntryByPathE ("META-INF" </> "container.xml") archive
-  docElem <- (parseXMLDocE . UTF8.toStringLazy . fromEntry) metaEntry
+  docElem <- parseXMLDocE metaEntry
   let namespaces = mapMaybe attrToNSPair (elAttribs docElem)
   ns <- mkE "xmlns not in namespaces" (lookup "xmlns" namespaces)
   as <- fmap (map attrToPair . elAttribs)
@@ -190,7 +191,7 @@ getManifest archive = do
   let rootdir = dropFileName manifestFile
   --mime <- lookup "media-type" as
   manifest <- findEntryByPathE manifestFile archive
-  (rootdir,) <$> (parseXMLDocE . UTF8.toStringLazy . fromEntry $ manifest)
+  (rootdir,) <$> parseXMLDocE manifest
 
 -- Fixup
 
@@ -284,8 +285,12 @@ findEntryByPathE :: PandocMonad m => FilePath -> Archive -> m Entry
 findEntryByPathE (normalise . unEscapeString -> path) a =
   mkE ("No entry on path: " ++ path) $ findEntryByPath path a
 
-parseXMLDocE :: PandocMonad m => String -> m Element
-parseXMLDocE doc = mkE "Unable to parse XML doc" $ parseXMLDoc doc
+parseXMLDocE :: PandocMonad m => Entry -> m Element
+parseXMLDocE entry =
+  either (throwError . PandocXMLError fp) return $ parseXMLElement doc
+ where
+  doc = UTF8.toTextLazy . fromEntry $ entry
+  fp  = T.pack $ eRelativePath entry
 
 findElementE :: PandocMonad m => QName -> Element -> m Element
 findElementE e x = mkE ("Unable to find element: " ++ show e) $ findElement e x
diff --git a/src/Text/Pandoc/Readers/FB2.hs b/src/Text/Pandoc/Readers/FB2.hs
index b0d2f092b..b804eab4f 100644
--- a/src/Text/Pandoc/Readers/FB2.hs
+++ b/src/Text/Pandoc/Readers/FB2.hs
@@ -32,6 +32,7 @@ import Data.List (intersperse)
 import qualified Data.Map as M
 import Data.Text (Text)
 import qualified Data.Text as T
+import qualified Data.Text.Lazy as TL
 import Data.Default
 import Data.Maybe
 import Text.HTML.TagSoup.Entity (lookupEntity)
@@ -42,6 +43,7 @@ import Text.Pandoc.Logging
 import Text.Pandoc.Options
 import Text.Pandoc.Shared (crFilter)
 import Text.XML.Light
+import Text.Pandoc.XMLParser (parseXMLElement)
 
 type FB2 m = StateT FB2State m
 
@@ -64,10 +66,10 @@ instance HasMeta FB2State where
 
 readFB2 :: PandocMonad m => ReaderOptions -> Text -> m Pandoc
 readFB2 _ inp =
-  case parseXMLDoc $ crFilter inp of
-    Nothing -> throwError $ PandocParseError "Not an XML document"
-    Just e ->  do
-      (bs, st) <- runStateT (parseRootElement e) def
+  case parseXMLElement $ TL.fromStrict $ crFilter inp of
+    Left msg -> throwError $ PandocXMLError "" msg
+    Right el ->  do
+      (bs, st) <- runStateT (parseRootElement el) def
       let authors = if null $ fb2Authors st
                     then id
                     else setMeta "author" (map text $ reverse $ fb2Authors st)
diff --git a/src/Text/Pandoc/Readers/JATS.hs b/src/Text/Pandoc/Readers/JATS.hs
index c638da519..dfd343b7a 100644
--- a/src/Text/Pandoc/Readers/JATS.hs
+++ b/src/Text/Pandoc/Readers/JATS.hs
@@ -14,6 +14,8 @@ Conversion of JATS XML to 'Pandoc' document.
 
 module Text.Pandoc.Readers.JATS ( readJATS ) where
 import Control.Monad.State.Strict
+import Control.Monad.Except (throwError)
+import Text.Pandoc.Error (PandocError(..))
 import Data.Char (isDigit, isSpace, toUpper)
 import Data.Default
 import Data.Generics
@@ -22,6 +24,7 @@ import qualified Data.Map as Map
 import Data.Maybe (maybeToList, fromMaybe)
 import Data.Text (Text)
 import qualified Data.Text as T
+import qualified Data.Text.Lazy as TL
 import Text.HTML.TagSoup.Entity (lookupEntity)
 import Text.Pandoc.Builder
 import Text.Pandoc.Class.PandocMonad (PandocMonad)
@@ -29,6 +32,7 @@ import Text.Pandoc.Options
 import Text.Pandoc.Shared (crFilter, safeRead, extractSpaces)
 import Text.TeXMath (readMathML, writeTeX)
 import Text.XML.Light
+import Text.Pandoc.XMLParser (parseXMLContents)
 import qualified Data.Set as S (fromList, member)
 import Data.Set ((\\))
 
@@ -51,8 +55,9 @@ instance Default JATSState where
 
 readJATS :: PandocMonad m => ReaderOptions -> Text -> m Pandoc
 readJATS _ inp = do
-  let tree = normalizeTree . parseXML
-               $ T.unpack $ crFilter inp
+  tree <- either (throwError . PandocXMLError "")
+                 (return . normalizeTree) $
+            parseXMLContents (TL.fromStrict $ crFilter inp)
   (bs, st') <- flip runStateT (def{ jatsContent = tree }) $ mapM parseBlock tree
   return $ Pandoc (jatsMeta st') (toList . mconcat $ bs)
 
diff --git a/src/Text/Pandoc/Readers/OPML.hs b/src/Text/Pandoc/Readers/OPML.hs
index 5b8996025..bdadc4dd9 100644
--- a/src/Text/Pandoc/Readers/OPML.hs
+++ b/src/Text/Pandoc/Readers/OPML.hs
@@ -19,14 +19,18 @@ import Data.Generics
 import Data.Maybe (fromMaybe)
 import Data.Text (Text)
 import qualified Data.Text as T
+import qualified Data.Text.Lazy as TL
 import Text.HTML.TagSoup.Entity (lookupEntity)
 import Text.Pandoc.Builder
 import Text.Pandoc.Class.PandocMonad (PandocMonad)
 import Text.Pandoc.Options
+import Text.Pandoc.Error (PandocError(..))
 import Text.Pandoc.Readers.HTML (readHtml)
 import Text.Pandoc.Readers.Markdown (readMarkdown)
 import Text.Pandoc.Shared (crFilter, blocksToInlines')
 import Text.XML.Light
+import Text.Pandoc.XMLParser (parseXMLContents)
+import Control.Monad.Except (throwError)
 
 type OPML m = StateT OPMLState m
 
@@ -49,8 +53,10 @@ instance Default OPMLState where
 readOPML :: PandocMonad m => ReaderOptions -> Text -> m Pandoc
 readOPML opts inp  = do
   (bs, st') <- runStateT
-                 (mapM parseBlock $ normalizeTree $
-                    parseXML (T.unpack (crFilter inp))) def{ opmlOptions = opts }
+                 (case parseXMLContents (TL.fromStrict (crFilter inp)) of
+                     Left msg -> throwError $ PandocXMLError "" msg
+                     Right ns -> mapM parseBlock $ normalizeTree ns)
+                 def{ opmlOptions = opts }
   return $
     setTitle (opmlDocTitle st') $
     setAuthors (opmlDocAuthors st') $
diff --git a/src/Text/Pandoc/Readers/Odt.hs b/src/Text/Pandoc/Readers/Odt.hs
index 9943d3147..85308deb1 100644
--- a/src/Text/Pandoc/Readers/Odt.hs
+++ b/src/Text/Pandoc/Readers/Odt.hs
@@ -15,6 +15,7 @@ module Text.Pandoc.Readers.Odt ( readOdt ) where
 
 import Codec.Archive.Zip
 import qualified Text.XML.Light as XML
+import Text.Pandoc.XMLParser (parseXMLElement)
 
 import qualified Data.ByteString.Lazy as B
 
@@ -66,18 +67,18 @@ bytesToOdt bytes = case toArchiveOrFail bytes of
 
 --
 archiveToOdt :: Archive -> Either PandocError (Pandoc, MediaBag)
-archiveToOdt archive = either (Left. PandocParseError) Right $ do
-  let onFailure msg Nothing = Left msg
+archiveToOdt archive = do
+  let onFailure msg Nothing = Left $ PandocParseError msg
       onFailure _   (Just x) = Right x
   contentEntry <- onFailure "Could not find content.xml"
                    (findEntryByPath "content.xml" archive)
   stylesEntry <- onFailure "Could not find styles.xml"
                    (findEntryByPath "styles.xml" archive)
-  contentElem <- onFailure "Could not find content element"
-                   (entryToXmlElem contentEntry)
-  stylesElem <- onFailure "Could not find styles element"
-                   (entryToXmlElem stylesEntry)
-  styles <- either (\_ -> Left "Could not read styles") Right
+  contentElem <- entryToXmlElem contentEntry
+  stylesElem <- entryToXmlElem stylesEntry
+  styles <- either
+               (\_ -> Left $ PandocParseError "Could not read styles")
+               Right
                (chooseMax (readStylesAt stylesElem ) (readStylesAt contentElem))
   let filePathIsOdtMedia :: FilePath -> Bool
       filePathIsOdtMedia fp =
@@ -85,10 +86,13 @@ archiveToOdt archive = either (Left. PandocParseError) Right $ do
         in  (dir == "Pictures/") || (dir /= "./" && name == "content.xml")
   let media = filteredFilesFromArchive archive filePathIsOdtMedia
   let startState = readerState styles media
-  either (\_ -> Left "Could not convert opendocument") Right
+  either (\_ -> Left $ PandocParseError "Could not convert opendocument") Right
     (runConverter' read_body startState contentElem)
 
 
 --
-entryToXmlElem :: Entry -> Maybe XML.Element
-entryToXmlElem = XML.parseXMLDoc . UTF8.toStringLazy . fromEntry
+entryToXmlElem :: Entry -> Either PandocError XML.Element
+entryToXmlElem entry =
+  case parseXMLElement . UTF8.toTextLazy . fromEntry $ entry of
+    Right x  -> Right x
+    Left msg -> Left $ PandocXMLError (T.pack $ eRelativePath entry) msg
diff --git a/src/Text/Pandoc/Writers/EPUB.hs b/src/Text/Pandoc/Writers/EPUB.hs
index 1f16f6772..e99fa2567 100644
--- a/src/Text/Pandoc/Writers/EPUB.hs
+++ b/src/Text/Pandoc/Writers/EPUB.hs
@@ -55,8 +55,9 @@ import Text.Pandoc.Walk (query, walk, walkM)
 import Text.Pandoc.Writers.HTML (writeHtmlStringForEPUB)
 import Text.Printf (printf)
 import Text.XML.Light (Attr (..), Element (..), Node (..), QName (..),
-                       add_attrs, lookupAttr, node, onlyElems, parseXML,
+                       add_attrs, lookupAttr, node, onlyElems,
                        ppElement, showElement, strContent, unode, unqual)
+import Text.Pandoc.XMLParser (parseXMLContents)
 import Text.Pandoc.XML (escapeStringForXML)
 import Text.DocTemplates (FromContext(lookupContext), Context(..),
                           ToContext(toVal), Val(..))
@@ -160,7 +161,12 @@ mkEntry path content = do
 getEPUBMetadata :: PandocMonad m => WriterOptions -> Meta -> E m EPUBMetadata
 getEPUBMetadata opts meta = do
   let md = metadataFromMeta opts meta
-  let elts = maybe [] (onlyElems . parseXML) $ writerEpubMetadata opts
+  elts <- case writerEpubMetadata opts of
+            Nothing -> return []
+            Just t -> case parseXMLContents (TL.fromStrict t) of
+                          Left msg -> throwError $
+                            PandocXMLError "epub metadata" msg
+                          Right ns -> return (onlyElems ns)
   let md' = foldr addMetadataFromXML md elts
   let addIdentifier m =
        if null (epubIdentifier m)
@@ -836,7 +842,8 @@ pandocToEPUB version opts doc = do
                                             : case subs of
                                                  []    -> []
                                                  (_:_) -> [unode "ol" ! [("class","toc")] $ subs]
-          where titElements = parseXML titRendered
+          where titElements = either (const []) id $
+                                parseXMLContents (TL.fromStrict titRendered)
                 titRendered = case P.runPure
                                (writeHtmlStringForEPUB version
                                  opts{ writerTemplate = Nothing
diff --git a/src/Text/Pandoc/Writers/FB2.hs b/src/Text/Pandoc/Writers/FB2.hs
index 25b1f28d1..9334d6e9a 100644
--- a/src/Text/Pandoc/Writers/FB2.hs
+++ b/src/Text/Pandoc/Writers/FB2.hs
@@ -19,7 +19,7 @@ FictionBook is an XML-based e-book format. For more information see:
 module Text.Pandoc.Writers.FB2 (writeFB2)  where
 
 import Control.Monad (zipWithM)
-import Control.Monad.Except (catchError)
+import Control.Monad.Except (catchError, throwError)
 import Control.Monad.State.Strict (StateT, evalStateT, get, gets, lift, liftM, modify)
 import Data.ByteString.Base64 (encode)
 import Data.Char (isAscii, isControl, isSpace)
@@ -27,16 +27,18 @@ import Data.Either (lefts, rights)
 import Data.List (intercalate)
 import Data.Text (Text, pack)
 import qualified Data.Text as T
+import qualified Data.Text.Lazy as TL
 import qualified Data.Text.Encoding as TE
 import Network.HTTP (urlEncode)
 import Text.XML.Light
 import qualified Text.XML.Light as X
 import qualified Text.XML.Light.Cursor as XC
-import qualified Text.XML.Light.Input as XI
+import Text.Pandoc.XMLParser (parseXMLContents)
 
 import Text.Pandoc.Class.PandocMonad (PandocMonad, report)
 import qualified Text.Pandoc.Class.PandocMonad as P
 import Text.Pandoc.Definition
+import Text.Pandoc.Error (PandocError(..))
 import Text.Pandoc.Logging
 import Text.Pandoc.Options (HTMLMathMethod (..), WriterOptions (..), def)
 import Text.Pandoc.Shared (capitalize, isURI, orderedListMarkers,
@@ -307,7 +309,10 @@ blockToXml (CodeBlock _ s) = return . spaceBeforeAfter .
                              map (el "p" . el "code" . T.unpack) . T.lines $ s
 blockToXml (RawBlock f str) =
   if f == Format "fb2"
-    then return $ XI.parseXML str
+    then
+      case parseXMLContents (TL.fromStrict str) of
+         Left msg  -> throwError $ PandocXMLError "" msg
+         Right nds -> return nds
     else return []
 blockToXml (Div _ bs) = cMapM blockToXml bs
 blockToXml (BlockQuote bs) = list . el "cite" <$> cMapM blockToXml bs
diff --git a/src/Text/Pandoc/Writers/ODT.hs b/src/Text/Pandoc/Writers/ODT.hs
index 05dfad5eb..a32ff618c 100644
--- a/src/Text/Pandoc/Writers/ODT.hs
+++ b/src/Text/Pandoc/Writers/ODT.hs
@@ -13,7 +13,7 @@ Conversion of 'Pandoc' documents to ODT.
 -}
 module Text.Pandoc.Writers.ODT ( writeODT ) where
 import Codec.Archive.Zip
-import Control.Monad.Except (catchError)
+import Control.Monad.Except (catchError, throwError)
 import Control.Monad.State.Strict
 import qualified Data.ByteString.Lazy as B
 import Data.Generics (everywhere', mkT)
@@ -27,6 +27,7 @@ import Text.Pandoc.BCP47 (Lang (..), getLang, renderLang)
 import Text.Pandoc.Class.PandocMonad (PandocMonad, report, toLang)
 import qualified Text.Pandoc.Class.PandocMonad as P
 import Text.Pandoc.Definition
+import Text.Pandoc.Error (PandocError(..))
 import Text.Pandoc.ImageSize
 import Text.Pandoc.Logging
 import Text.Pandoc.MIME (extensionFromMimeType, getMimeType)
@@ -35,10 +36,11 @@ import Text.DocLayout
 import Text.Pandoc.Shared (stringify, pandocVersion, tshow)
 import Text.Pandoc.Writers.Shared (lookupMetaString, lookupMetaBlocks,
                                    fixDisplayMath)
-import Text.Pandoc.UTF8 (fromStringLazy, fromTextLazy, toStringLazy)
+import Text.Pandoc.UTF8 (fromStringLazy, fromTextLazy, toTextLazy)
 import Text.Pandoc.Walk
 import Text.Pandoc.Writers.OpenDocument (writeOpenDocument)
 import Text.Pandoc.XML
+import Text.Pandoc.XMLParser (parseXMLElement)
 import Text.TeXMath
 import Text.XML.Light
 
@@ -172,17 +174,18 @@ updateStyleWithLang :: PandocMonad m => Maybe Lang -> Archive -> O m Archive
 updateStyleWithLang Nothing arch = return arch
 updateStyleWithLang (Just lang) arch = do
   epochtime <- floor `fmap` lift P.getPOSIXTime
-  return arch{ zEntries = [if eRelativePath e == "styles.xml"
-                              then case parseXMLDoc
-                                      (toStringLazy (fromEntry e)) of
-                                      Nothing -> e
-                                      Just d ->
-                                        toEntry "styles.xml" epochtime
-                                        ( fromStringLazy
-                                        . ppTopElement
-                                        . addLang lang $ d )
-                              else e
-                            | e <- zEntries arch] }
+  entries <- mapM (\e -> if eRelativePath e == "styles.xml"
+                            then case parseXMLElement
+                                    (toTextLazy (fromEntry e)) of
+                                    Left msg -> throwError $
+                                        PandocXMLError "styles.xml" msg
+                                    Right d -> return $
+                                      toEntry "styles.xml" epochtime
+                                      ( fromStringLazy
+                                      . ppTopElement
+                                      . addLang lang $ d )
+                            else return e) (zEntries arch)
+  return arch{ zEntries = entries }
 
 addLang :: Lang -> Element -> Element
 addLang lang = everywhere' (mkT updateLangAttr)
diff --git a/src/Text/Pandoc/Writers/OOXML.hs b/src/Text/Pandoc/Writers/OOXML.hs
index 3ac007f4e..8f60e70d5 100644
--- a/src/Text/Pandoc/Writers/OOXML.hs
+++ b/src/Text/Pandoc/Writers/OOXML.hs
@@ -35,6 +35,7 @@ import qualified Data.Text as T
 import Text.Pandoc.Class.PandocMonad (PandocMonad)
 import qualified Text.Pandoc.UTF8 as UTF8
 import Text.XML.Light as XML
+import Text.Pandoc.XMLParser (parseXMLElement)
 
 mknode :: Node t => String -> [(String,String)] -> t -> Element
 mknode s attrs =
@@ -62,10 +63,10 @@ parseXml refArchive distArchive relpath =
          findEntryByPath relpath distArchive of
             Nothing -> throwError $ PandocSomeError $
                         T.pack relpath <> " missing in reference file"
-            Just e  -> case parseXMLDoc . UTF8.toStringLazy . fromEntry $ e of
-                       Nothing -> throwError $ PandocSomeError $
-                                   T.pack relpath <> " corrupt in reference file"
-                       Just d  -> return d
+            Just e  -> case parseXMLElement . UTF8.toTextLazy . fromEntry $ e of
+                       Left msg ->
+                         throwError $ PandocXMLError (T.pack relpath) msg
+                       Right d  -> return d
 
 -- Copied from Util
 
diff --git a/src/Text/Pandoc/Writers/Powerpoint/Output.hs b/src/Text/Pandoc/Writers/Powerpoint/Output.hs
index 8554db622..cd092969b 100644
--- a/src/Text/Pandoc/Writers/Powerpoint/Output.hs
+++ b/src/Text/Pandoc/Writers/Powerpoint/Output.hs
@@ -29,6 +29,7 @@ import Data.Time.Clock (UTCTime)
 import Data.Time.Clock.POSIX (utcTimeToPOSIXSeconds, posixSecondsToUTCTime)
 import System.FilePath.Posix (splitDirectories, splitExtension, takeExtension)
 import Text.XML.Light
+import Text.Pandoc.XMLParser (parseXMLElement)
 import Text.Pandoc.Definition
 import qualified Text.Pandoc.UTF8 as UTF8
 import Text.Pandoc.Class.PandocMonad (PandocMonad)
@@ -77,7 +78,8 @@ getPresentationSize :: Archive -> Archive -> Maybe (Integer, Integer)
 getPresentationSize refArchive distArchive = do
   entry <- findEntryByPath "ppt/presentation.xml" refArchive  `mplus`
            findEntryByPath "ppt/presentation.xml" distArchive
-  presElement <- parseXMLDoc $ UTF8.toStringLazy $ fromEntry entry
+  presElement <- either (const Nothing) return $
+                   parseXMLElement $ UTF8.toTextLazy $ fromEntry entry
   let ns = elemToNameSpaces presElement
   sldSize <- findChild (elemName ns "p" "sldSz") presElement
   cxS <- findAttr (QName "cx" Nothing Nothing) sldSize
diff --git a/src/Text/Pandoc/XMLParser.hs b/src/Text/Pandoc/XMLParser.hs
new file mode 100644
index 000000000..8ad22a66a
--- /dev/null
+++ b/src/Text/Pandoc/XMLParser.hs
@@ -0,0 +1,66 @@
+{-# LANGUAGE OverloadedStrings #-}
+{- |
+   Module      : Text.Pandoc.XMLParser
+   Copyright   : Copyright (C) 2021 John MacFarlane
+   License     : GNU GPL, version 2 or above
+
+   Maintainer  : John MacFarlane <jgm@berkeley.edu>
+   Stability   : alpha
+   Portability : portable
+
+Bridge to allow using xml-conduit's parser with xml-light's types.
+-}
+module Text.Pandoc.XMLParser
+  ( parseXMLElement
+  , parseXMLContents
+  , module Text.XML.Light.Types
+  ) where
+
+import qualified Control.Exception as E
+import qualified Text.XML as Conduit
+import Text.XML.Unresolved (InvalidEventStream(..))
+import qualified Text.XML.Light as Light
+import Text.XML.Light.Types
+import qualified Data.Text as T
+import qualified Data.Text.Lazy as TL
+import qualified Data.Map as M
+import Data.Maybe (mapMaybe)
+
+-- Drop in replacement for parseXMLDoc in xml-light.
+parseXMLElement :: TL.Text -> Either T.Text Light.Element
+parseXMLElement t =
+  elementToElement .  Conduit.documentRoot <$>
+    either (Left . T.pack . E.displayException) Right
+    (Conduit.parseText Conduit.def{ Conduit.psRetainNamespaces = True } t)
+
+parseXMLContents :: TL.Text -> Either T.Text [Light.Content]
+parseXMLContents t =
+  case Conduit.parseText Conduit.def{ Conduit.psRetainNamespaces = True } t of
+    Left e ->
+      case E.fromException e of
+        Just (ContentAfterRoot _) ->
+          elContent <$> parseXMLElement ("<wrapper>" <> t <> "</wrapper>")
+        _ -> Left . T.pack . E.displayException $ e
+    Right x -> Right [Light.Elem . elementToElement . Conduit.documentRoot $ x]
+
+elementToElement :: Conduit.Element -> Light.Element
+elementToElement (Conduit.Element name attribMap nodes) =
+  Light.Element (nameToQname name) attrs (mapMaybe nodeToContent nodes) Nothing
+ where
+  attrs = map (\(n,v) -> Light.Attr (nameToQname n) (T.unpack v)) $
+              M.toList attribMap
+  nameToQname (Conduit.Name localName mbns mbpref) =
+    case mbpref of
+      Nothing | "xmlns:" `T.isPrefixOf` localName ->
+           Light.QName (T.unpack $ T.drop 6 localName)  (T.unpack <$> mbns)
+                       (Just "xmlns")
+      _ -> Light.QName (T.unpack localName) (T.unpack <$> mbns)
+                       (T.unpack <$> mbpref)
+
+nodeToContent :: Conduit.Node -> Maybe Light.Content
+nodeToContent (Conduit.NodeElement el) =
+  Just (Light.Elem (elementToElement el))
+nodeToContent (Conduit.NodeContent t) =
+  Just (Light.Text (Light.CData Light.CDataText (T.unpack t) Nothing))
+nodeToContent _ = Nothing
+
diff --git a/test/Tests/Readers/JATS.hs b/test/Tests/Readers/JATS.hs
index 525499c86..a9c9a0586 100644
--- a/test/Tests/Readers/JATS.hs
+++ b/test/Tests/Readers/JATS.hs
@@ -88,6 +88,7 @@ tests = [ testGroup "inline code"
             "<p>\n\
             \  <inline-formula><alternatives>\n\
             \  <tex-math><![CDATA[\\sigma|_{\\{x\\}}]]></tex-math>\n\
+            \  </alternatives></inline-formula>\n\
             \</p>"
             =?> para (math "\\sigma|_{\\{x\\}}")
           , test jats "math ml only" $
diff --git a/test/command/5321.md b/test/command/5321.md
index 081abe2a0..83404632a 100644
--- a/test/command/5321.md
+++ b/test/command/5321.md
@@ -4,7 +4,7 @@
   <caption>
     <p>bar</p>
   </caption>
-  <graphic xlink:href="foo.png" xlink:alt-text="baz">
+  <graphic xlink:href="foo.png" xlink:alt-text="baz" />
 </fig>
 ^D
 [Para [Image ("fig-1",[],[]) [Str "bar"] ("foo.png","fig:")]]
@@ -17,7 +17,7 @@
     <title>foo</title>
     <p>bar</p>
   </caption>
-  <graphic xlink:href="foo.png" xlink:alt-text="baz">
+  <graphic xlink:href="foo.png" xlink:alt-text="baz" />
 </fig>
 ^D
 [Para [Image ("fig-1",[],[]) [Str "foo",LineBreak,Str "bar"] ("foo.png","fig:")]]
diff --git a/test/docbook-reader.docbook b/test/docbook-reader.docbook
index 02568d8de..5717d78d0 100644
--- a/test/docbook-reader.docbook
+++ b/test/docbook-reader.docbook
@@ -1,6 +1,11 @@
 <?xml version="1.0" encoding="utf-8" ?>
 <!DOCTYPE article PUBLIC "-//OASIS//DTD DocBook XML V4.4//EN"
-                  "http://www.oasis-open.org/docbook/xml/4.4/docbookx.dtd">
+"http://www.oasis-open.org/docbook/xml/4.4/docbookx.dtd"
+[
+<!ENTITY GHC "GHC" >
+<!ENTITY let "LET" >
+<!ENTITY case "CASE" >
+]>
 <article>
   <articleinfo>
     <title>Pandoc Test Suite</title>
diff --git a/test/docx/golden/block_quotes.docx b/test/docx/golden/block_quotes.docx
index 3e1bf16e7..d3b16d0f2 100644
Binary files a/test/docx/golden/block_quotes.docx and b/test/docx/golden/block_quotes.docx differ
diff --git a/test/docx/golden/codeblock.docx b/test/docx/golden/codeblock.docx
index 66f055063..6293ef493 100644
Binary files a/test/docx/golden/codeblock.docx and b/test/docx/golden/codeblock.docx differ
diff --git a/test/docx/golden/comments.docx b/test/docx/golden/comments.docx
index fb3a02a0a..4205a1516 100644
Binary files a/test/docx/golden/comments.docx and b/test/docx/golden/comments.docx differ
diff --git a/test/docx/golden/custom_style_no_reference.docx b/test/docx/golden/custom_style_no_reference.docx
index bc6c2702a..adb3f23db 100644
Binary files a/test/docx/golden/custom_style_no_reference.docx and b/test/docx/golden/custom_style_no_reference.docx differ
diff --git a/test/docx/golden/custom_style_preserve.docx b/test/docx/golden/custom_style_preserve.docx
index 8c555a5bd..92c8137fe 100644
Binary files a/test/docx/golden/custom_style_preserve.docx and b/test/docx/golden/custom_style_preserve.docx differ
diff --git a/test/docx/golden/custom_style_reference.docx b/test/docx/golden/custom_style_reference.docx
index 5f96cc911..f53470617 100644
Binary files a/test/docx/golden/custom_style_reference.docx and b/test/docx/golden/custom_style_reference.docx differ
diff --git a/test/docx/golden/definition_list.docx b/test/docx/golden/definition_list.docx
index c21b3a5b3..d6af90a72 100644
Binary files a/test/docx/golden/definition_list.docx and b/test/docx/golden/definition_list.docx differ
diff --git a/test/docx/golden/document-properties-short-desc.docx b/test/docx/golden/document-properties-short-desc.docx
index 92ce144e9..e18dbe853 100644
Binary files a/test/docx/golden/document-properties-short-desc.docx and b/test/docx/golden/document-properties-short-desc.docx differ
diff --git a/test/docx/golden/document-properties.docx b/test/docx/golden/document-properties.docx
index d21b67309..820299043 100644
Binary files a/test/docx/golden/document-properties.docx and b/test/docx/golden/document-properties.docx differ
diff --git a/test/docx/golden/headers.docx b/test/docx/golden/headers.docx
index 3558a47bf..ae0f41d12 100644
Binary files a/test/docx/golden/headers.docx and b/test/docx/golden/headers.docx differ
diff --git a/test/docx/golden/image.docx b/test/docx/golden/image.docx
index 606df92a3..94cd35dfa 100644
Binary files a/test/docx/golden/image.docx and b/test/docx/golden/image.docx differ
diff --git a/test/docx/golden/inline_code.docx b/test/docx/golden/inline_code.docx
index 759269cac..879f2a25b 100644
Binary files a/test/docx/golden/inline_code.docx and b/test/docx/golden/inline_code.docx differ
diff --git a/test/docx/golden/inline_formatting.docx b/test/docx/golden/inline_formatting.docx
index c37777080..93f86478f 100644
Binary files a/test/docx/golden/inline_formatting.docx and b/test/docx/golden/inline_formatting.docx differ
diff --git a/test/docx/golden/inline_images.docx b/test/docx/golden/inline_images.docx
index 9450b1a73..967d297f2 100644
Binary files a/test/docx/golden/inline_images.docx and b/test/docx/golden/inline_images.docx differ
diff --git a/test/docx/golden/link_in_notes.docx b/test/docx/golden/link_in_notes.docx
index 6f0b830e6..c5614e2fa 100644
Binary files a/test/docx/golden/link_in_notes.docx and b/test/docx/golden/link_in_notes.docx differ
diff --git a/test/docx/golden/links.docx b/test/docx/golden/links.docx
index e53889cfb..0f39a831f 100644
Binary files a/test/docx/golden/links.docx and b/test/docx/golden/links.docx differ
diff --git a/test/docx/golden/lists.docx b/test/docx/golden/lists.docx
index 5dbe298b7..07046f223 100644
Binary files a/test/docx/golden/lists.docx and b/test/docx/golden/lists.docx differ
diff --git a/test/docx/golden/lists_continuing.docx b/test/docx/golden/lists_continuing.docx
index 194181288..3656618e6 100644
Binary files a/test/docx/golden/lists_continuing.docx and b/test/docx/golden/lists_continuing.docx differ
diff --git a/test/docx/golden/lists_multiple_initial.docx b/test/docx/golden/lists_multiple_initial.docx
index 6e0b634f7..8798253d5 100644
Binary files a/test/docx/golden/lists_multiple_initial.docx and b/test/docx/golden/lists_multiple_initial.docx differ
diff --git a/test/docx/golden/lists_restarting.docx b/test/docx/golden/lists_restarting.docx
index 477178e77..0a24d1840 100644
Binary files a/test/docx/golden/lists_restarting.docx and b/test/docx/golden/lists_restarting.docx differ
diff --git a/test/docx/golden/nested_anchors_in_header.docx b/test/docx/golden/nested_anchors_in_header.docx
index 51110356e..52bb7a217 100644
Binary files a/test/docx/golden/nested_anchors_in_header.docx and b/test/docx/golden/nested_anchors_in_header.docx differ
diff --git a/test/docx/golden/notes.docx b/test/docx/golden/notes.docx
index b6206cdf5..182c06c64 100644
Binary files a/test/docx/golden/notes.docx and b/test/docx/golden/notes.docx differ
diff --git a/test/docx/golden/raw-blocks.docx b/test/docx/golden/raw-blocks.docx
index 07b576080..7b69a56a3 100644
Binary files a/test/docx/golden/raw-blocks.docx and b/test/docx/golden/raw-blocks.docx differ
diff --git a/test/docx/golden/raw-bookmarks.docx b/test/docx/golden/raw-bookmarks.docx
index d46095eb7..3d3a35701 100644
Binary files a/test/docx/golden/raw-bookmarks.docx and b/test/docx/golden/raw-bookmarks.docx differ
diff --git a/test/docx/golden/table_one_row.docx b/test/docx/golden/table_one_row.docx
index 7caba4e93..5ae37b406 100644
Binary files a/test/docx/golden/table_one_row.docx and b/test/docx/golden/table_one_row.docx differ
diff --git a/test/docx/golden/table_with_list_cell.docx b/test/docx/golden/table_with_list_cell.docx
index 6aaa6da61..c29aa6716 100644
Binary files a/test/docx/golden/table_with_list_cell.docx and b/test/docx/golden/table_with_list_cell.docx differ
diff --git a/test/docx/golden/tables.docx b/test/docx/golden/tables.docx
index 5746c5ad0..664493246 100644
Binary files a/test/docx/golden/tables.docx and b/test/docx/golden/tables.docx differ
diff --git a/test/docx/golden/track_changes_deletion.docx b/test/docx/golden/track_changes_deletion.docx
index 5f22dccc6..b6d15340e 100644
Binary files a/test/docx/golden/track_changes_deletion.docx and b/test/docx/golden/track_changes_deletion.docx differ
diff --git a/test/docx/golden/track_changes_insertion.docx b/test/docx/golden/track_changes_insertion.docx
index ab5c4f56d..f8e1092d2 100644
Binary files a/test/docx/golden/track_changes_insertion.docx and b/test/docx/golden/track_changes_insertion.docx differ
diff --git a/test/docx/golden/track_changes_move.docx b/test/docx/golden/track_changes_move.docx
index 085f33162..b4cda82f2 100644
Binary files a/test/docx/golden/track_changes_move.docx and b/test/docx/golden/track_changes_move.docx differ
diff --git a/test/docx/golden/track_changes_scrubbed_metadata.docx b/test/docx/golden/track_changes_scrubbed_metadata.docx
index 1ac86d5c8..ee222efa0 100644
Binary files a/test/docx/golden/track_changes_scrubbed_metadata.docx and b/test/docx/golden/track_changes_scrubbed_metadata.docx differ
diff --git a/test/docx/golden/unicode.docx b/test/docx/golden/unicode.docx
index c2c443b19..c6f8d9c96 100644
Binary files a/test/docx/golden/unicode.docx and b/test/docx/golden/unicode.docx differ
diff --git a/test/docx/golden/verbatim_subsuper.docx b/test/docx/golden/verbatim_subsuper.docx
index 5ea18d32e..ea8146690 100644
Binary files a/test/docx/golden/verbatim_subsuper.docx and b/test/docx/golden/verbatim_subsuper.docx differ
diff --git a/test/jats-reader.native b/test/jats-reader.native
index ab77dd1a0..0715ea8cc 100644
--- a/test/jats-reader.native
+++ b/test/jats-reader.native
@@ -1,4 +1,4 @@
-Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "John",Space,Str "MacFarlane"]]),("title",MetaInlines [Str "Pandoc",Space,Str "Test",Space,Str "Suite"])]})
+Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "John",Space,Str "MacFarlane"],MetaInlines [Str "Anonymous"]]),("title",MetaInlines [Str "Pandoc",Space,Str "Test",Space,Str "Suite"])]})
 [Para [Str "This",Space,Str "is",Space,Str "a",Space,Str "set",Space,Str "of",Space,Str "tests",Space,Str "for",Space,Str "pandoc.",Space,Str "Most",Space,Str "of",Space,Str "them",Space,Str "are",Space,Str "adapted",Space,Str "from",Space,Str "John",Space,Str "Gruber's",Space,Str "markdown",Space,Str "test",Space,Str "suite."]
 ,Header 1 ("headers",[],[]) [Str "Headers"]
 ,Header 2 ("level-2-with-an-embedded-link",[],[]) [Str "Level",Space,Str "2",Space,Str "with",Space,Str "an",SoftBreak,Link ("",[],[]) [Str "embedded",SoftBreak,Str "link"] ("/url","")]
diff --git a/test/jats-reader.xml b/test/jats-reader.xml
index f75b3e95a..f33cb9ab3 100644
--- a/test/jats-reader.xml
+++ b/test/jats-reader.xml
@@ -20,6 +20,7 @@
       <surname>MacFarlane</surname>
       <given-names>John</given-names>
     </name>
+  </contrib>
   <contrib contrib-type="author">
     <name>
       <surname>Anonymous</surname>
diff --git a/test/pptx/code-custom.pptx b/test/pptx/code-custom.pptx
index aa9b7692a..58070eb3f 100644
Binary files a/test/pptx/code-custom.pptx and b/test/pptx/code-custom.pptx differ
diff --git a/test/pptx/code-custom_templated.pptx b/test/pptx/code-custom_templated.pptx
index 9aaef4cb5..db9b7e371 100644
Binary files a/test/pptx/code-custom_templated.pptx and b/test/pptx/code-custom_templated.pptx differ
diff --git a/test/pptx/code.pptx b/test/pptx/code.pptx
index 1737ec757..c7b1ed7d5 100644
Binary files a/test/pptx/code.pptx and b/test/pptx/code.pptx differ
diff --git a/test/pptx/code_templated.pptx b/test/pptx/code_templated.pptx
index 87fb560ef..6944d92bf 100644
Binary files a/test/pptx/code_templated.pptx and b/test/pptx/code_templated.pptx differ
diff --git a/test/pptx/document-properties-short-desc.pptx b/test/pptx/document-properties-short-desc.pptx
index 961c31020..ae0d28429 100644
Binary files a/test/pptx/document-properties-short-desc.pptx and b/test/pptx/document-properties-short-desc.pptx differ
diff --git a/test/pptx/document-properties-short-desc_templated.pptx b/test/pptx/document-properties-short-desc_templated.pptx
index 894738ef7..37c74c69a 100644
Binary files a/test/pptx/document-properties-short-desc_templated.pptx and b/test/pptx/document-properties-short-desc_templated.pptx differ
diff --git a/test/pptx/document-properties.pptx b/test/pptx/document-properties.pptx
index 188e8d826..324e443a1 100644
Binary files a/test/pptx/document-properties.pptx and b/test/pptx/document-properties.pptx differ
diff --git a/test/pptx/document-properties_templated.pptx b/test/pptx/document-properties_templated.pptx
index 253e8c0a7..c81b983e3 100644
Binary files a/test/pptx/document-properties_templated.pptx and b/test/pptx/document-properties_templated.pptx differ
diff --git a/test/pptx/endnotes.pptx b/test/pptx/endnotes.pptx
index e230420d2..30ce33db6 100644
Binary files a/test/pptx/endnotes.pptx and b/test/pptx/endnotes.pptx differ
diff --git a/test/pptx/endnotes_templated.pptx b/test/pptx/endnotes_templated.pptx
index 49384fd65..d6c604968 100644
Binary files a/test/pptx/endnotes_templated.pptx and b/test/pptx/endnotes_templated.pptx differ
diff --git a/test/pptx/endnotes_toc.pptx b/test/pptx/endnotes_toc.pptx
index cdf1be4ad..000e17ecd 100644
Binary files a/test/pptx/endnotes_toc.pptx and b/test/pptx/endnotes_toc.pptx differ
diff --git a/test/pptx/endnotes_toc_templated.pptx b/test/pptx/endnotes_toc_templated.pptx
index c4fcbad45..fdcd2e29b 100644
Binary files a/test/pptx/endnotes_toc_templated.pptx and b/test/pptx/endnotes_toc_templated.pptx differ
diff --git a/test/pptx/images.pptx b/test/pptx/images.pptx
index 4a13b5b7f..e73126376 100644
Binary files a/test/pptx/images.pptx and b/test/pptx/images.pptx differ
diff --git a/test/pptx/images_templated.pptx b/test/pptx/images_templated.pptx
index 7a6e9700e..e3f968e9e 100644
Binary files a/test/pptx/images_templated.pptx and b/test/pptx/images_templated.pptx differ
diff --git a/test/pptx/inline_formatting.pptx b/test/pptx/inline_formatting.pptx
index 926c8ff3f..eadb9372e 100644
Binary files a/test/pptx/inline_formatting.pptx and b/test/pptx/inline_formatting.pptx differ
diff --git a/test/pptx/inline_formatting_templated.pptx b/test/pptx/inline_formatting_templated.pptx
index 16f48e182..8ca6bab2b 100644
Binary files a/test/pptx/inline_formatting_templated.pptx and b/test/pptx/inline_formatting_templated.pptx differ
diff --git a/test/pptx/lists.pptx b/test/pptx/lists.pptx
index f47b17a74..ae188ee68 100644
Binary files a/test/pptx/lists.pptx and b/test/pptx/lists.pptx differ
diff --git a/test/pptx/lists_templated.pptx b/test/pptx/lists_templated.pptx
index 88109a95e..60301fa50 100644
Binary files a/test/pptx/lists_templated.pptx and b/test/pptx/lists_templated.pptx differ
diff --git a/test/pptx/raw_ooxml.pptx b/test/pptx/raw_ooxml.pptx
index 84020708f..17124a50d 100644
Binary files a/test/pptx/raw_ooxml.pptx and b/test/pptx/raw_ooxml.pptx differ
diff --git a/test/pptx/raw_ooxml_templated.pptx b/test/pptx/raw_ooxml_templated.pptx
index a2f77e945..19ae7dd4e 100644
Binary files a/test/pptx/raw_ooxml_templated.pptx and b/test/pptx/raw_ooxml_templated.pptx differ
diff --git a/test/pptx/remove_empty_slides.pptx b/test/pptx/remove_empty_slides.pptx
index 48bf7bc8a..b650b7585 100644
Binary files a/test/pptx/remove_empty_slides.pptx and b/test/pptx/remove_empty_slides.pptx differ
diff --git a/test/pptx/remove_empty_slides_templated.pptx b/test/pptx/remove_empty_slides_templated.pptx
index 23b134a5f..0ab029614 100644
Binary files a/test/pptx/remove_empty_slides_templated.pptx and b/test/pptx/remove_empty_slides_templated.pptx differ
diff --git a/test/pptx/slide_breaks.pptx b/test/pptx/slide_breaks.pptx
index d6eebeffb..2a6e35080 100644
Binary files a/test/pptx/slide_breaks.pptx and b/test/pptx/slide_breaks.pptx differ
diff --git a/test/pptx/slide_breaks_slide_level_1.pptx b/test/pptx/slide_breaks_slide_level_1.pptx
index a6c76a187..a7bcf6a4b 100644
Binary files a/test/pptx/slide_breaks_slide_level_1.pptx and b/test/pptx/slide_breaks_slide_level_1.pptx differ
diff --git a/test/pptx/slide_breaks_slide_level_1_templated.pptx b/test/pptx/slide_breaks_slide_level_1_templated.pptx
index 1fbde815b..21b018c25 100644
Binary files a/test/pptx/slide_breaks_slide_level_1_templated.pptx and b/test/pptx/slide_breaks_slide_level_1_templated.pptx differ
diff --git a/test/pptx/slide_breaks_templated.pptx b/test/pptx/slide_breaks_templated.pptx
index cb3af4aa1..4ec4772a4 100644
Binary files a/test/pptx/slide_breaks_templated.pptx and b/test/pptx/slide_breaks_templated.pptx differ
diff --git a/test/pptx/slide_breaks_toc.pptx b/test/pptx/slide_breaks_toc.pptx
index dff386885..5983657b6 100644
Binary files a/test/pptx/slide_breaks_toc.pptx and b/test/pptx/slide_breaks_toc.pptx differ
diff --git a/test/pptx/slide_breaks_toc_templated.pptx b/test/pptx/slide_breaks_toc_templated.pptx
index 43b125f5e..dd54c7082 100644
Binary files a/test/pptx/slide_breaks_toc_templated.pptx and b/test/pptx/slide_breaks_toc_templated.pptx differ
diff --git a/test/pptx/speaker_notes.pptx b/test/pptx/speaker_notes.pptx
index 3314a1c65..b3e5ed5b9 100644
Binary files a/test/pptx/speaker_notes.pptx and b/test/pptx/speaker_notes.pptx differ
diff --git a/test/pptx/speaker_notes_after_metadata.pptx b/test/pptx/speaker_notes_after_metadata.pptx
index 27a136838..1078854bb 100644
Binary files a/test/pptx/speaker_notes_after_metadata.pptx and b/test/pptx/speaker_notes_after_metadata.pptx differ
diff --git a/test/pptx/speaker_notes_after_metadata_templated.pptx b/test/pptx/speaker_notes_after_metadata_templated.pptx
index 7aa3b6a87..5116c6c4e 100644
Binary files a/test/pptx/speaker_notes_after_metadata_templated.pptx and b/test/pptx/speaker_notes_after_metadata_templated.pptx differ
diff --git a/test/pptx/speaker_notes_afterheader.pptx b/test/pptx/speaker_notes_afterheader.pptx
index d43709ca7..0c8e49bd9 100644
Binary files a/test/pptx/speaker_notes_afterheader.pptx and b/test/pptx/speaker_notes_afterheader.pptx differ
diff --git a/test/pptx/speaker_notes_afterheader_templated.pptx b/test/pptx/speaker_notes_afterheader_templated.pptx
index 793ea10f6..68695939d 100644
Binary files a/test/pptx/speaker_notes_afterheader_templated.pptx and b/test/pptx/speaker_notes_afterheader_templated.pptx differ
diff --git a/test/pptx/speaker_notes_afterseps.pptx b/test/pptx/speaker_notes_afterseps.pptx
index 2f4d3b820..7ed9b946d 100644
Binary files a/test/pptx/speaker_notes_afterseps.pptx and b/test/pptx/speaker_notes_afterseps.pptx differ
diff --git a/test/pptx/speaker_notes_afterseps_templated.pptx b/test/pptx/speaker_notes_afterseps_templated.pptx
index 94a221398..79fc82345 100644
Binary files a/test/pptx/speaker_notes_afterseps_templated.pptx and b/test/pptx/speaker_notes_afterseps_templated.pptx differ
diff --git a/test/pptx/speaker_notes_templated.pptx b/test/pptx/speaker_notes_templated.pptx
index 22040c88c..9f943c279 100644
Binary files a/test/pptx/speaker_notes_templated.pptx and b/test/pptx/speaker_notes_templated.pptx differ
diff --git a/test/pptx/start_numbering_at.pptx b/test/pptx/start_numbering_at.pptx
index 18477380b..ac72d8ced 100644
Binary files a/test/pptx/start_numbering_at.pptx and b/test/pptx/start_numbering_at.pptx differ
diff --git a/test/pptx/start_numbering_at_templated.pptx b/test/pptx/start_numbering_at_templated.pptx
index 4b9d0ba4d..15c7b5469 100644
Binary files a/test/pptx/start_numbering_at_templated.pptx and b/test/pptx/start_numbering_at_templated.pptx differ
diff --git a/test/pptx/tables.pptx b/test/pptx/tables.pptx
index 1c5b54185..926c5e699 100644
Binary files a/test/pptx/tables.pptx and b/test/pptx/tables.pptx differ
diff --git a/test/pptx/tables_templated.pptx b/test/pptx/tables_templated.pptx
index 1314f4de4..a37e72d2c 100644
Binary files a/test/pptx/tables_templated.pptx and b/test/pptx/tables_templated.pptx differ
diff --git a/test/pptx/two_column.pptx b/test/pptx/two_column.pptx
index 9018be36e..7f86533fe 100644
Binary files a/test/pptx/two_column.pptx and b/test/pptx/two_column.pptx differ
diff --git a/test/pptx/two_column_templated.pptx b/test/pptx/two_column_templated.pptx
index 35e93af67..89e3db0ab 100644
Binary files a/test/pptx/two_column_templated.pptx and b/test/pptx/two_column_templated.pptx differ
-- 
cgit v1.2.3


From 59875185b3da8f57174f1b3da42c750f4c3ee910 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Fri, 12 Feb 2021 17:43:00 -0800
Subject: Add command test for #7092

---
 test/command/7092.md | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 test/command/7092.md

(limited to 'test/command')

diff --git a/test/command/7092.md b/test/command/7092.md
new file mode 100644
index 000000000..e7f127244
--- /dev/null
+++ b/test/command/7092.md
@@ -0,0 +1,8 @@
+```
+% pandoc -flatex+raw_tex -t native
+\newcommand{\em}[1]{\emph{#1}}
+
+\em{\parseMe{foo}}
+^D
+[Para [Emph [RawInline (Format "latex") "\\parseMe{foo}"]]]
+```
-- 
cgit v1.2.3


From 3be066b7d308ca7f103c7fabd80238b350bf4f94 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Fri, 12 Feb 2021 17:43:34 -0800
Subject: Fix command test 5686

---
 test/command/5686.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'test/command')

diff --git a/test/command/5686.md b/test/command/5686.md
index 67b4579ef..1c18109c3 100644
--- a/test/command/5686.md
+++ b/test/command/5686.md
@@ -4,6 +4,6 @@ FOO\t0BAR
 
 This part does not make it to the html output.
 ^D
-[Para [Str "FOO",RawInline (Format "tex") "\\t0BAR"]
+[Para [Str "FOO",RawInline (Format "tex") "\\t0",Str "BAR"]
 ,Para [Str "This",Space,Str "part",Space,Str "does",Space,Str "not",Space,Str "make",Space,Str "it",Space,Str "to",Space,Str "the",Space,Str "html",Space,Str "output."]]
 ```
-- 
cgit v1.2.3


From 6e73273916a55448c1a12ece343454ef139648a8 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sat, 13 Feb 2021 13:05:17 -0800
Subject: T.P.Error: export `renderError`.

Refactor `handleError` to use `renderError`. This allows us
render error messages without exiting.
---
 src/Text/Pandoc/Error.hs | 105 ++++++++++++++++++++++++++++++++---------------
 test/command/7099.md     |   8 ++++
 2 files changed, 80 insertions(+), 33 deletions(-)
 create mode 100644 test/command/7099.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Error.hs b/src/Text/Pandoc/Error.hs
index 831405f42..50ad3c0e3 100644
--- a/src/Text/Pandoc/Error.hs
+++ b/src/Text/Pandoc/Error.hs
@@ -16,9 +16,10 @@ encountered during parsing.
 -}
 module Text.Pandoc.Error (
   PandocError(..),
+  renderError,
   handleError) where
 
-import Control.Exception (Exception)
+import Control.Exception (Exception, displayException)
 import Data.Typeable (Typeable)
 import Data.Word (Word8)
 import Data.Text (Text)
@@ -68,19 +69,17 @@ data PandocError = PandocIOError Text IOError
 
 instance Exception PandocError
 
--- | Handle PandocError by exiting with an error message.
-handleError :: Either PandocError a -> IO a
-handleError (Right r) = return r
-handleError (Left e) =
+renderError :: PandocError -> Text
+renderError e =
   case e of
-    PandocIOError _ err' -> ioError err'
-    PandocHttpError u err' -> err 61 $
+    PandocIOError _ err' -> T.pack $ displayException err'
+    PandocHttpError u err' ->
       "Could not fetch " <> u <> "\n" <> tshow err'
-    PandocShouldNeverHappenError s -> err 62 $
+    PandocShouldNeverHappenError s ->
       "Something we thought was impossible happened!\n" <>
       "Please report this to pandoc's developers: " <> s
-    PandocSomeError s -> err 63 s
-    PandocParseError s -> err 64 s
+    PandocSomeError s -> s
+    PandocParseError s -> s
     PandocParsecError input err' ->
         let errPos = errorPos err'
             errLine = sourceLine errPos
@@ -91,41 +90,41 @@ handleError (Left e) =
                                           ,"\n", T.replicate (errColumn - 1) " "
                                           ,"^"]
                         else ""
-        in  err 65 $ "\nError at " <> tshow  err' <>
+        in  "\nError at " <> tshow  err' <>
                      -- if error comes from a chunk or included file,
                      -- then we won't get the right text this way:
                      if sourceName errPos == "source"
                         then errorInFile
                         else ""
-    PandocMakePDFError s -> err 66 s
-    PandocOptionError s -> err 6 s
-    PandocSyntaxMapError s -> err 67 s
-    PandocFailOnWarningError -> err 3 "Failing because there were warnings."
-    PandocPDFProgramNotFoundError pdfprog -> err 47 $
+    PandocMakePDFError s -> s
+    PandocOptionError s -> s
+    PandocSyntaxMapError s -> s
+    PandocFailOnWarningError -> "Failing because there were warnings."
+    PandocPDFProgramNotFoundError pdfprog ->
         pdfprog <> " not found. Please select a different --pdf-engine or install " <> pdfprog
-    PandocPDFError logmsg -> err 43 $ "Error producing PDF.\n" <> logmsg
-    PandocXMLError fp logmsg -> err 44 $ "Invalid XML" <>
+    PandocPDFError logmsg -> "Error producing PDF.\n" <> logmsg
+    PandocXMLError fp logmsg -> "Invalid XML" <>
         (if T.null fp then "" else " in " <> fp) <> ":\n" <> logmsg
-    PandocFilterError filtername msg -> err 83 $ "Error running filter " <>
+    PandocFilterError filtername msg -> "Error running filter " <>
         filtername <> ":\n" <> msg
-    PandocLuaError msg -> err 84 $ "Error running Lua:\n" <> msg
-    PandocCouldNotFindDataFileError fn -> err 97 $
+    PandocLuaError msg -> "Error running Lua:\n" <> msg
+    PandocCouldNotFindDataFileError fn ->
         "Could not find data file " <> fn
-    PandocResourceNotFound fn -> err 99 $
+    PandocResourceNotFound fn ->
         "File " <> fn <> " not found in resource path"
-    PandocTemplateError s -> err 5 $ "Error compiling template " <> s
-    PandocAppError s -> err 4 s
-    PandocEpubSubdirectoryError s -> err 31 $
+    PandocTemplateError s -> "Error compiling template " <> s
+    PandocAppError s -> s
+    PandocEpubSubdirectoryError s ->
       "EPUB subdirectory name '" <> s <> "' contains illegal characters"
-    PandocMacroLoop s -> err 91 $
+    PandocMacroLoop s ->
       "Loop encountered in expanding macro " <> s
-    PandocUTF8DecodingError f offset w -> err 92 $
+    PandocUTF8DecodingError f offset w ->
       "UTF-8 decoding error in " <> f <> " at byte offset " <> tshow offset <>
       " (" <> T.pack (printf "%2x" w) <> ").\n" <>
       "The input must be a UTF-8 encoded text."
-    PandocIpynbDecodingError w -> err 93 $
+    PandocIpynbDecodingError w ->
       "ipynb decoding error: " <> w
-    PandocUnknownReaderError r -> err 21 $
+    PandocUnknownReaderError r ->
       "Unknown input format " <> r <>
       case r of
         "doc" -> "\nPandoc can convert from DOCX, but not from DOC." <>
@@ -133,7 +132,7 @@ handleError (Left e) =
                  " and convert that with pandoc."
         "pdf" -> "\nPandoc can convert to PDF, but not from PDF."
         _     -> ""
-    PandocUnknownWriterError w -> err 22 $
+    PandocUnknownWriterError w ->
        "Unknown output format " <> w <>
        case w of
          "pdf" -> "To create a pdf using pandoc, use" <>
@@ -142,14 +141,54 @@ handleError (Left e) =
                  ".pdf extension (-o filename.pdf)."
          "doc" -> "\nPandoc can convert to DOCX, but not to DOC."
          _     -> ""
-    PandocUnsupportedExtensionError ext f -> err 23 $
+    PandocUnsupportedExtensionError ext f ->
       "The extension " <> ext <> " is not supported " <>
       "for " <> f
-    PandocCiteprocError e' -> err 24 $
+    PandocCiteprocError e' ->
       prettyCiteprocError e'
-    PandocBibliographyError fp msg -> err 25 $
+    PandocBibliographyError fp msg ->
       "Error reading bibliography file " <> fp <> ":\n" <> msg
 
+
+-- | Handle PandocError by exiting with an error message.
+handleError :: Either PandocError a -> IO a
+handleError (Right r) = return r
+handleError (Left e) =
+  case e of
+    PandocIOError _ err' -> ioError err'
+    _ -> err exitCode (renderError e)
+ where
+  exitCode =
+    case e of
+      PandocIOError{} -> 1
+      PandocHttpError{} -> 61
+      PandocShouldNeverHappenError{} -> 62
+      PandocSomeError{} -> 63
+      PandocParseError{} -> 64
+      PandocParsecError{} -> 65
+      PandocMakePDFError{} -> 66
+      PandocOptionError{} -> 6
+      PandocSyntaxMapError{} -> 67
+      PandocFailOnWarningError{} -> 3
+      PandocPDFProgramNotFoundError{} -> 47
+      PandocPDFError{} -> 43
+      PandocXMLError{} -> 44
+      PandocFilterError{} -> 83
+      PandocLuaError{} -> 84
+      PandocCouldNotFindDataFileError{} -> 97
+      PandocResourceNotFound fn -> 99
+      PandocTemplateError{} -> 5
+      PandocAppError{} -> 4
+      PandocEpubSubdirectoryError{} -> 31
+      PandocMacroLoop s -> 91
+      PandocUTF8DecodingError{} -> 92
+      PandocIpynbDecodingError{} -> 93
+      PandocUnknownReaderError{} -> 21
+      PandocUnknownWriterError{} -> 22
+      PandocUnsupportedExtensionError{} -> 23
+      PandocCiteprocError{} -> 24
+      PandocBibliographyError{} -> 25
+
 err :: Int -> Text -> IO a
 err exitCode msg = do
   UTF8.hPutStrLn stderr (T.unpack msg)
diff --git a/test/command/7099.md b/test/command/7099.md
new file mode 100644
index 000000000..d9ff8e5ff
--- /dev/null
+++ b/test/command/7099.md
@@ -0,0 +1,8 @@
+```
+% pandoc -f html -t native --verbose
+<iframe src=""></iframe>
+^D
+[INFO] Fetching ...
+[INFO] Skipped '<iframe src></iframe>' at input line 1 column 1
+[]
+```
-- 
cgit v1.2.3


From d84a6041e12547331f3a252ea4e8b4d229ba1159 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sat, 13 Feb 2021 13:06:22 -0800
Subject: HTML reader: fix bad handling of empty src attribute in iframe.

- If src is empty, we simply skip the iframe.
- If src is invalid or cannot be fetched, we issue a warning
  and skip instead of failing with an error.
- Closes #7099.
---
 src/Text/Pandoc/Readers/HTML.hs | 18 ++++++++++++------
 test/command/7099.md            | 14 ++++++++++++--
 2 files changed, 24 insertions(+), 8 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/HTML.hs b/src/Text/Pandoc/Readers/HTML.hs
index afc7a3e25..cc60b5501 100644
--- a/src/Text/Pandoc/Readers/HTML.hs
+++ b/src/Text/Pandoc/Readers/HTML.hs
@@ -26,7 +26,7 @@ module Text.Pandoc.Readers.HTML ( readHtml
 
 import Control.Applicative ((<|>))
 import Control.Monad (guard, msum, mzero, unless, void)
-import Control.Monad.Except (throwError)
+import Control.Monad.Except (throwError, catchError)
 import Control.Monad.Reader (ask, asks, lift, local, runReaderT)
 import Data.ByteString.Base64 (encode)
 import Data.Char (isAlphaNum, isLetter)
@@ -393,11 +393,17 @@ pIframe = try $ do
   tag <- pSatisfy (tagOpen (=="iframe") (isJust . lookup "src"))
   pCloses "iframe" <|> eof
   url <- canonicalizeUrl $ fromAttrib "src" tag
-  (bs, _) <- openURL url
-  let inp = UTF8.toText bs
-  opts <- readerOpts <$> getState
-  Pandoc _ contents <- readHtml opts inp
-  return $ B.divWith ("",["iframe"],[]) $ B.fromList contents
+  if T.null url
+     then ignore $ renderTags' [tag, TagClose "iframe"]
+     else catchError
+       (do (bs, _) <- openURL url
+           let inp = UTF8.toText bs
+           opts <- readerOpts <$> getState
+           Pandoc _ contents <- readHtml opts inp
+           return $ B.divWith ("",["iframe"],[]) $ B.fromList contents)
+       (\e -> do
+         logMessage $ CouldNotFetchResource url (renderError e)
+         ignore $ renderTags' [tag, TagClose "iframe"])
 
 pRawHtmlBlock :: PandocMonad m => TagParser m Blocks
 pRawHtmlBlock = do
diff --git a/test/command/7099.md b/test/command/7099.md
index d9ff8e5ff..33ac8aea1 100644
--- a/test/command/7099.md
+++ b/test/command/7099.md
@@ -2,7 +2,17 @@
 % pandoc -f html -t native --verbose
 <iframe src=""></iframe>
 ^D
-[INFO] Fetching ...
-[INFO] Skipped '<iframe src></iframe>' at input line 1 column 1
+[INFO] Skipped '<iframe src></iframe>' at input line 1 column 16
+[]
+```
+
+```
+% pandoc -f html -t native --verbose
+<iframe src="h:invalid@url"></iframe>
+^D
+[INFO] Fetching h:invalid@url...
+[WARNING] Could not fetch resource 'h:invalid@url': Could not fetch h:invalid@url
+  InvalidUrlException "h:invalid@url" "Invalid scheme"
+[INFO] Skipped '<iframe src="h:invalid@url"></iframe>' at input line 1 column 29
 []
 ```
-- 
cgit v1.2.3


From f0a991a22be9b82e192d63bf80bbe39679bb2e07 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Mon, 22 Feb 2021 21:17:22 -0800
Subject: T.P.CSV: fix parsing of unquoted values.

Previously we didn't allow unescaped quotes in unquoted values,
but they are allowed. Closes #7112.
---
 src/Text/Pandoc/CSV.hs |  3 +--
 test/command/7112.md   | 15 +++++++++++++++
 2 files changed, 16 insertions(+), 2 deletions(-)
 create mode 100644 test/command/7112.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/CSV.hs b/src/Text/Pandoc/CSV.hs
index ec212fa9a..2bd21bcfb 100644
--- a/src/Text/Pandoc/CSV.hs
+++ b/src/Text/Pandoc/CSV.hs
@@ -68,8 +68,7 @@ escaped opts = try $
 
 pCSVUnquotedCell :: CSVOptions -> Parser Text
 pCSVUnquotedCell opts = T.pack <$>
-  many (satisfy (\c -> c /= csvDelim opts && c /= '\r' && c /= '\n'
-                  && c /= csvQuote opts))
+  many (satisfy (\c -> c /= csvDelim opts && c /= '\r' && c /= '\n'))
 
 pCSVDelim :: CSVOptions -> Parser ()
 pCSVDelim opts = do
diff --git a/test/command/7112.md b/test/command/7112.md
new file mode 100644
index 000000000..a75b9250a
--- /dev/null
+++ b/test/command/7112.md
@@ -0,0 +1,15 @@
+```
+% pandoc -f rst
+.. csv-table::
+
+   setting, ``echo PATH="path"``
+^D
+<table>
+<tbody>
+<tr class="odd">
+<td>setting</td>
+<td><code>echo PATH="path"</code></td>
+</tr>
+</tbody>
+</table>
+```
-- 
cgit v1.2.3


From a9cc5d261603e2e84961744c14f1b12aaaefc49d Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Fri, 26 Feb 2021 18:00:45 -0800
Subject: Update tests for changes to https URLs.

---
 test/command/biblatex-baez-article.md | 2 +-
 test/command/biblatex-baez-online.md  | 2 +-
 test/command/biblatex-itzhaki.md      | 2 +-
 test/command/biblatex-wassenberg.md   | 2 +-
 test/command/biblatex-wilde.md        | 2 +-
 test/txt2tags.native                  | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

(limited to 'test/command')

diff --git a/test/command/biblatex-baez-article.md b/test/command/biblatex-baez-article.md
index 07d293c24..ed3f830c5 100644
--- a/test/command/biblatex-baez-article.md
+++ b/test/command/biblatex-baez-article.md
@@ -67,7 +67,7 @@ references:
   title: "Higher-dimensional algebra V: 2-groups"
   title-short: Higher-dimensional algebra V
   type: article-journal
-  url: "http://arxiv.org/abs/math/0307200v3"
+  url: "https://arxiv.org/abs/math/0307200v3"
   version: 3
   volume: 12
 ---
diff --git a/test/command/biblatex-baez-online.md b/test/command/biblatex-baez-online.md
index 6834ed22b..9b73b70f8 100644
--- a/test/command/biblatex-baez-online.md
+++ b/test/command/biblatex-baez-online.md
@@ -61,7 +61,7 @@ references:
   title: "Higher-dimensional algebra V: 2-groups"
   title-short: Higher-dimensional algebra V
   type: webpage
-  url: "http://arxiv.org/abs/math/0307200v3"
+  url: "https://arxiv.org/abs/math/0307200v3"
   version: 3
 ---
 
diff --git a/test/command/biblatex-itzhaki.md b/test/command/biblatex-itzhaki.md
index 24229fb2c..568e26215 100644
--- a/test/command/biblatex-itzhaki.md
+++ b/test/command/biblatex-itzhaki.md
@@ -74,7 +74,7 @@ references:
   language: en-US
   title: Some remarks on '[t Hooft's]{.nocase} S-matrix for black holes
   type: webpage
-  url: "http://arxiv.org/abs/hep-th/9603067"
+  url: "https://arxiv.org/abs/hep-th/9603067"
   version: 1
 ---
 
diff --git a/test/command/biblatex-wassenberg.md b/test/command/biblatex-wassenberg.md
index 9424abba3..0ae9e10c7 100644
--- a/test/command/biblatex-wassenberg.md
+++ b/test/command/biblatex-wassenberg.md
@@ -75,7 +75,7 @@ references:
   language: en-US
   title: Faster radix sort via virtual memory and write-combining
   type: webpage
-  url: "http://arxiv.org/abs/1008.2849v1"
+  url: "https://arxiv.org/abs/1008.2849v1"
   version: 1
 ---
 
diff --git a/test/command/biblatex-wilde.md b/test/command/biblatex-wilde.md
index 1db2d4579..d471041f3 100644
--- a/test/command/biblatex-wilde.md
+++ b/test/command/biblatex-wilde.md
@@ -58,7 +58,7 @@ references:
     people"
   title-short: The importance of being earnest
   type: book
-  url: "http://books.google.com?id=4HIWAAAAYAAJ"
+  url: "https://books.google.com?id=4HIWAAAAYAAJ"
 ---
 
 
diff --git a/test/txt2tags.native b/test/txt2tags.native
index 23ecf867e..c52c5929a 100644
--- a/test/txt2tags.native
+++ b/test/txt2tags.native
@@ -37,7 +37,7 @@ Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "author"]]
 ,Para [Link ("",[],[]) [Str "mailto:user@domain.com"] ("user@domain.com",""),SoftBreak,Link ("",[],[]) [Str "mailto:user@domain.com"] ("user@domain.com",""),Str ".",SoftBreak,Link ("",[],[]) [Str "mailto:user@domain.com"] ("user@domain.com",""),Str ".",Space,Str "any",Space,Str "text.",SoftBreak,Str "any",Space,Str "text:",Space,Link ("",[],[]) [Str "mailto:user@domain.com"] ("user@domain.com",""),Str ".",Space,Str "any",Space,Str "text.",SoftBreak,Link ("",[],[]) [Str "label"] ("user@domain.com",""),SoftBreak,Link ("",[],[]) [Str "mailto:user@domain.com?subject=bla"] ("user@domain.com?subject=bla",""),SoftBreak,Link ("",[],[]) [Str "mailto:user@domain.com?subject=bla"] ("user@domain.com?subject=bla",""),Str ".",SoftBreak,Link ("",[],[]) [Str "mailto:user@domain.com?subject=bla"] ("user@domain.com?subject=bla",""),Str ",",SoftBreak,Link ("",[],[]) [Str "mailto:user@domain.com?subject=bla&cc=otheruser@domain.com"] ("user@domain.com?subject=bla&cc=otheruser@domain.com",""),SoftBreak,Link ("",[],[]) [Str "mailto:user@domain.com?subject=bla&cc=otheruser@domain.com"] ("user@domain.com?subject=bla&cc=otheruser@domain.com",""),Str ".",SoftBreak,Link ("",[],[]) [Str "mailto:user@domain.com?subject=bla&cc=otheruser@domain.com"] ("user@domain.com?subject=bla&cc=otheruser@domain.com",""),Str ",",SoftBreak,Link ("",[],[]) [Str "label"] ("user@domain.com?subject=bla&cc=otheruser@domain.com",""),Str ".",SoftBreak,Link ("",[],[]) [Str "label"] ("user@domain.com?subject=bla&cc=otheruser@domain.com.",""),Str ".",SoftBreak,Link ("",[],[]) [Str "http://www.domain.com"] ("http://www.domain.com",""),SoftBreak,Link ("",[],[]) [Str "http://www.domain.com/dir/"] ("http://www.domain.com/dir/",""),SoftBreak,Link ("",[],[]) [Str "http://www.domain.com/dir///"] ("http://www.domain.com/dir///",""),SoftBreak,Link ("",[],[]) [Str "http://www.domain.com."] ("http://www.domain.com.",""),SoftBreak,Link ("",[],[]) [Str "http://www.domain.com,"] ("http://www.domain.com,",""),SoftBreak,Link ("",[],[]) [Str "http://www.domain.com."] ("http://www.domain.com.",""),Space,Str "any",Space,Str "text.",SoftBreak,Link ("",[],[]) [Str "http://www.domain.com,"] ("http://www.domain.com,",""),Space,Str "any",Space,Str "text.",SoftBreak,Link ("",[],[]) [Str "http://www.domain.com/dir/."] ("http://www.domain.com/dir/.",""),Space,Str "any",Space,Str "text.",SoftBreak,Str "any",Space,Str "text:",Space,Link ("",[],[]) [Str "http://www.domain.com."] ("http://www.domain.com.",""),Space,Str "any",Space,Str "text.",SoftBreak,Str "any",Space,Str "text:",Space,Link ("",[],[]) [Str "http://www.domain.com/dir/."] ("http://www.domain.com/dir/.",""),Space,Str "any",Space,Str "text.",SoftBreak,Str "any",Space,Str "text:",Space,Link ("",[],[]) [Str "http://www.domain.com/dir/index.html."] ("http://www.domain.com/dir/index.html.",""),Space,Str "any",Space,Str "text.",SoftBreak,Str "any",Space,Str "text:",Space,Link ("",[],[]) [Str "http://www.domain.com/dir/index.html,"] ("http://www.domain.com/dir/index.html,",""),Space,Str "any",Space,Str "text.",SoftBreak,Link ("",[],[]) [Str "http://www.domain.com/dir/#anchor"] ("http://www.domain.com/dir/#anchor",""),SoftBreak,Link ("",[],[]) [Str "http://www.domain.com/dir/index.html#anchor"] ("http://www.domain.com/dir/index.html#anchor",""),SoftBreak,Link ("",[],[]) [Str "http://www.domain.com/dir/index.html#anchor."] ("http://www.domain.com/dir/index.html#anchor.",""),SoftBreak,Link ("",[],[]) [Str "http://www.domain.com/dir/#anchor."] ("http://www.domain.com/dir/#anchor.",""),Space,Str "any",Space,Str "text.",SoftBreak,Link ("",[],[]) [Str "http://www.domain.com/dir/index.html#anchor."] ("http://www.domain.com/dir/index.html#anchor.",""),Space,Str "any",Space,Str "text.",SoftBreak,Str "any",Space,Str "text:",Space,Link ("",[],[]) [Str "http://www.domain.com/dir/#anchor."] ("http://www.domain.com/dir/#anchor.",""),Space,Str "any",Space,Str "text.",SoftBreak,Str "any",Space,Str "text:",Space,Link ("",[],[]) [Str "http://www.domain.com/dir/index.html#anchor."] ("http://www.domain.com/dir/index.html#anchor.",""),Space,Str "any",Space,Str "text.",SoftBreak,Link ("",[],[]) [Str "http://domain.com?a=a@a.a&b=a+b+c."] ("http://domain.com?a=a@a.a&b=a+b+c.",""),SoftBreak,Link ("",[],[]) [Str "http://domain.com?a=a@a.a&b=a+b+c,"] ("http://domain.com?a=a@a.a&b=a+b+c,",""),SoftBreak,Link ("",[],[]) [Str "http://domain.com/bla.cgi?a=a@a.a&b=a+b+c."] ("http://domain.com/bla.cgi?a=a@a.a&b=a+b+c.",""),SoftBreak,Link ("",[],[]) [Str "http://domain.com/bla.cgi?a=a@a.a&b=a+b+c@."] ("http://domain.com/bla.cgi?a=a@a.a&b=a+b+c@.",""),SoftBreak,Link ("",[],[]) [Str "http://domain.com?a=a@a.a&b=a+b+c.#anchor"] ("http://domain.com?a=a@a.a&b=a+b+c.#anchor",""),SoftBreak,Link ("",[],[]) [Str "http://domain.com/bla.cgi?a=a@a.a&b=a+b+c.#anchor"] ("http://domain.com/bla.cgi?a=a@a.a&b=a+b+c.#anchor",""),SoftBreak,Link ("",[],[]) [Str "http://domain.com/bla.cgi?a=a@a.a&b=a+b+c@.#anchor"] ("http://domain.com/bla.cgi?a=a@a.a&b=a+b+c@.#anchor",""),SoftBreak,Link ("",[],[]) [Str "http://user:password@domain.com/bla.html."] ("http://user:password@domain.com/bla.html.",""),SoftBreak,Link ("",[],[]) [Str "http://user:password@domain.com/dir/."] ("http://user:password@domain.com/dir/.",""),SoftBreak,Link ("",[],[]) [Str "http://user:password@domain.com."] ("http://user:password@domain.com.",""),SoftBreak,Link ("",[],[]) [Str "http://user:@domain.com."] ("http://user:@domain.com.",""),SoftBreak,Link ("",[],[]) [Str "http://user@domain.com."] ("http://user@domain.com.",""),SoftBreak,Link ("",[],[]) [Str "http://user:password@domain.com/bla.cgi?a=a@a.a&b=a+b+c.#anchor"] ("http://user:password@domain.com/bla.cgi?a=a@a.a&b=a+b+c.#anchor",""),SoftBreak,Link ("",[],[]) [Str "http://user:password@domain.com/bla.cgi?a=a@a.a&b=a+b+c@#anchor"] ("http://user:password@domain.com/bla.cgi?a=a@a.a&b=a+b+c@#anchor",""),SoftBreak,Link ("",[],[]) [Str "label"] ("www.domain.com",""),SoftBreak,Str "[",Space,Str "label",Space,Link ("",[],[]) [Str "www.domain.com"] ("www.domain.com",""),Str "]",SoftBreak,Link ("",[],[]) [Str "label",Space] ("www.domain.com",""),SoftBreak,Link ("",[],[]) [Str "anchor",Space] ("http://www.domain.com/dir/index.html#anchor.",""),SoftBreak,Link ("",[],[]) [Str "login",Space] ("http://user:password@domain.com/bla.html",""),SoftBreak,Link ("",[],[]) [Str "form",Space] ("http://www.domain.com/bla.cgi?a=a@a.a&b=a+b+c.",""),SoftBreak,Link ("",[],[]) [Str "form",Space,Str "&",Space,Str "anchor"] ("http://www.domain.com/bla.cgi?a=a@a.a&b=a+b+c.#anchor",""),SoftBreak,Link ("",[],[]) [Str "login",Space,Str "&",Space,Str "form",Space] ("http://user:password@domain.com/bla.cgi?a=a@a.a&b=a+b+c.",""),SoftBreak,Link ("",[],[]) [Str "local",Space,Str "link",Space,Str "up",Space] ("..",""),SoftBreak,Link ("",[],[]) [Str "local",Space,Str "link",Space,Str "file",Space] ("bla.html",""),SoftBreak,Link ("",[],[]) [Str "local",Space,Str "link",Space,Str "anchor",Space] ("#anchor",""),SoftBreak,Link ("",[],[]) [Str "local",Space,Str "link",Space,Str "file/anchor"] ("bla.html#anchor",""),SoftBreak,Link ("",[],[]) [Str "local",Space,Str "link",Space,Str "file/anchor"] ("bla.html#anchor.",""),SoftBreak,Link ("",[],[]) [Str "local",Space,Str "link",Space,Str "img",Space] ("abc.gif",""),SoftBreak,Link ("",[],[]) [Str "www.fake.com"] ("www.domain.com",""),SoftBreak,Link ("",[],[]) [Str "http://domain.com:8080/~user/_st-r@a=n$g,e/index%20new.htm"] ("http://domain.com:8080/~user/_st-r@a=n$g,e/index%20new.htm",""),SoftBreak,Link ("",[],[]) [Str "http://domain.com:8080/~user/_st-r@a=n$g,e/index%20new.htm?a=/%22&b=+.@*_-"] ("http://domain.com:8080/~user/_st-r@a=n$g,e/index%20new.htm?a=/%22&b=+.@*_-",""),SoftBreak,Link ("",[],[]) [Str "http://domain.com:8080/~user/_st-r@a=n$g,e/index%20new.htm?a=/%22&b=+.@*_-#anchor_"] ("http://domain.com:8080/~user/_st-r@a=n$g,e/index%20new.htm?a=/%22&b=+.@*_-#anchor_",""),Str "-1%.",SoftBreak,Link ("",[],[]) [Str "http://foo._user-9:pass!#$%&*()+word@domain.com:8080/~user/_st-r@a=n$g,e/index%20new.htm?a=/%22&b=+.@*_-#anchor_"] ("http://foo._user-9:pass!#$%&*()+word@domain.com:8080/~user/_st-r@a=n$g,e/index%20new.htm?a=/%22&b=+.@*_-#anchor_",""),Str "-1%.",SoftBreak,Link ("",[],[]) [Str "http://L1.com"] ("http://L1.com",""),Space,Str "!",Space,Link ("",[],[]) [Str "mailto:L2@www.com"] ("L2@www.com",""),Space,Str "!",Space,Link ("",[],[]) [Str "L3"] ("www.com",""),Space,Str "!",Space,Link ("",[],[]) [Str "L4"] ("w@ww.com",""),Space,Str "!",Space,Link ("",[],[]) [Str "www.L5.com"] ("www.L5.com",""),SoftBreak,Link ("",[],[]) [Str "www.domain.com"] ("www.domain.com",""),SoftBreak,Link ("",[],[]) [Str "www2.domain.com"] ("www2.domain.com",""),SoftBreak,Link ("",[],[]) [Str "ftp.domain.com"] ("ftp.domain.com",""),SoftBreak,Link ("",[],[]) [Str "WWW.DOMAIN.COM"] ("WWW.DOMAIN.COM",""),SoftBreak,Link ("",[],[]) [Str "FTP.DOMAIN.COM"] ("FTP.DOMAIN.COM",""),SoftBreak,Link ("",[],[]) [Str "label"] ("www.domain.com",""),SoftBreak,Link ("",[],[]) [Str "label"] ("ftp.domain.com",""),SoftBreak,Link ("",[],[]) [Str "label"] ("WWW.DOMAIN.COM",""),SoftBreak,Link ("",[],[]) [Str "label"] ("FTP.DOMAIN.COM",""),SoftBreak,Str "[label",Space,Link ("",[],[]) [Str "www.domain.com"] ("www.domain.com",""),Space,Str "]",SoftBreak,Str "[label]",Space,Link ("",[],[]) [Str "www.domain.com"] ("www.domain.com",""),Str "]"]
 ,Header 1 ("image",[],[]) [Str "Image"]
 ,Para [Image ("",[],[]) [] ("img.png","")]
-,Para [Link ("",[],[]) [Image ("",[],[]) [] ("img.png","")] ("http://txt2tags.org","")]
+,Para [Link ("",[],[]) [Image ("",[],[]) [] ("img.png","")] ("https://txt2tags.org","")]
 ,Para [Image ("",[],[]) [] ("img.png",""),Space,Str "Image",Space,Str "at",Space,Str "the",Space,Str "line",Space,Str "beginning."]
 ,Para [Str "Image",Space,Str "in",Space,Str "the",Space,Str "middle",Space,Image ("",[],[]) [] ("img.png",""),Space,Str "of",Space,Str "the",Space,Str "line."]
 ,Para [Str "Image",Space,Str "at",Space,Str "the",Space,Str "line",Space,Str "end.",Space,Image ("",[],[]) [] ("img.png","")]
-- 
cgit v1.2.3


From 5aa73bd0a2820a0c89b5990dbe53abfdd5ade32d Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sun, 7 Mar 2021 15:49:02 -0800
Subject: LaTeX reader: handle table cells containing `&` in `\verb`.

Closes #7129.
---
 src/Text/Pandoc/Readers/LaTeX/Table.hs |  7 ++++++-
 test/command/7129.md                   | 27 +++++++++++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 test/command/7129.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/LaTeX/Table.hs b/src/Text/Pandoc/Readers/LaTeX/Table.hs
index 2ea9caf58..7833da081 100644
--- a/src/Text/Pandoc/Readers/LaTeX/Table.hs
+++ b/src/Text/Pandoc/Readers/LaTeX/Table.hs
@@ -134,6 +134,11 @@ parseTableRow :: PandocMonad m
               -> LP m Row
 parseTableRow blocks inline envname prefsufs = do
   notFollowedBy (spaces *> end_ envname)
+  -- contexts that can contain & that is not colsep:
+  let canContainAmp (Tok _ (CtrlSeq "begin") _) = True
+      canContainAmp (Tok _ (CtrlSeq "verb") _)  = True
+      canContainAmp (Tok _ (CtrlSeq "Verb") _)  = True
+      canContainAmp _       = False
   -- add prefixes and suffixes in token stream:
   let celltoks (pref, suff) = do
         prefpos <- getPosition
@@ -142,7 +147,7 @@ parseTableRow blocks inline envname prefsufs = do
                      ((lookAhead (controlSeq "parbox") >>
                        void blocks) -- #5711
                       <|>
-                      (lookAhead (controlSeq "begin") >> void inline)
+                      (lookAhead (satisfyTok canContainAmp) >> void inline)
                       <|>
                       (lookAhead (symbol '$') >> void inline))
                   <|>
diff --git a/test/command/7129.md b/test/command/7129.md
new file mode 100644
index 000000000..fef4ca2c3
--- /dev/null
+++ b/test/command/7129.md
@@ -0,0 +1,27 @@
+```
+% pandoc -f latex -t native
+\begin{tabular}{ll} \hline
+  FOO & BAR         \\ \hline
+  foo & \verb|b&r|  \\ \hline
+\end{tabular}
+^D
+[Table ("",[],[]) (Caption Nothing
+ [])
+ [(AlignLeft,ColWidthDefault)
+ ,(AlignLeft,ColWidthDefault)]
+ (TableHead ("",[],[])
+ [Row ("",[],[])
+  [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
+   [Plain [Str "FOO"]]
+  ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
+   [Plain [Str "BAR"]]]])
+ [(TableBody ("",[],[]) (RowHeadColumns 0)
+  []
+  [Row ("",[],[])
+   [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
+    [Plain [Str "foo"]]
+   ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
+    [Plain [Code ("",[],[]) "b&r"]]]])]
+ (TableFoot ("",[],[])
+ [])]
+```
-- 
cgit v1.2.3


From b9b2586ed3e9aac9c5ba86127fbf984fb3149844 Mon Sep 17 00:00:00 2001
From: Albert Krewinkel <albert@zeitkraut.de>
Date: Tue, 9 Mar 2021 18:01:08 +0100
Subject: Org writer: prevent unintended creation of ordered list items

Adjust line wrapping if default wrapping would cause a line to be read
as an ordered list item.

Fixes #7132
---
 src/Text/Pandoc/Writers/Org.hs | 11 +++++++----
 test/command/7132.md           | 10 ++++++++++
 2 files changed, 17 insertions(+), 4 deletions(-)
 create mode 100644 test/command/7132.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/Org.hs b/src/Text/Pandoc/Writers/Org.hs
index 8dfc2749c..1b525831e 100644
--- a/src/Text/Pandoc/Writers/Org.hs
+++ b/src/Text/Pandoc/Writers/Org.hs
@@ -17,7 +17,7 @@ Org-Mode:  <http://orgmode.org>
 -}
 module Text.Pandoc.Writers.Org (writeOrg) where
 import Control.Monad.State.Strict
-import Data.Char (isAlphaNum)
+import Data.Char (isAlphaNum, isDigit)
 import Data.List (intersect, intersperse, partition, transpose)
 import Data.Text (Text)
 import qualified Data.Text as T
@@ -347,16 +347,19 @@ inlineListToOrg :: PandocMonad m
                 => [Inline]
                 -> Org m (Doc Text)
 inlineListToOrg lst = hcat <$> mapM inlineToOrg (fixMarkers lst)
-  where fixMarkers [] = []  -- prevent note refs and list markers from wrapping, see #4171
+  where -- Prevent note refs and list markers from wrapping, see #4171
+        -- and #7132.
+        fixMarkers [] = []
         fixMarkers (Space : x : rest) | shouldFix x =
           Str " " : x : fixMarkers rest
         fixMarkers (SoftBreak : x : rest) | shouldFix x =
           Str " " : x : fixMarkers rest
         fixMarkers (x : rest) = x : fixMarkers rest
 
-        shouldFix Note{} = True -- Prevent footnotes
+        shouldFix Note{} = True    -- Prevent footnotes
         shouldFix (Str "-") = True -- Prevent bullet list items
-        -- TODO: prevent ordered list items
+        shouldFix (Str x)          -- Prevent ordered list items
+          | Just (cs, c) <- T.unsnoc x = T.all isDigit cs && c == '.' || c == ')'
         shouldFix _ = False
 
 -- | Convert Pandoc inline element to Org.
diff --git a/test/command/7132.md b/test/command/7132.md
new file mode 100644
index 000000000..30d4c0b3b
--- /dev/null
+++ b/test/command/7132.md
@@ -0,0 +1,10 @@
+```
+% pandoc -f markdown -t org --columns=72
+- This line has exactly the wrong number of characters before the number 5.
+- Long line ending with a number (this time it is in parentheses and a 23)
+^D
+- This line has exactly the wrong number of characters before the
+  number 5.
+- Long line ending with a number (this time it is in parentheses and
+  a 23)
+```
-- 
cgit v1.2.3


From 1c23e3a824bc4d850a908193443ad1915a3ebc61 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Tue, 9 Mar 2021 13:03:27 -0800
Subject: RST reader: fix logic for ending comments.

Previously comments sometimes got extended too far.  Closes #7134.
---
 src/Text/Pandoc/Readers/RST.hs |  3 ++-
 test/command/7134.md           | 16 ++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)
 create mode 100644 test/command/7134.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/RST.hs b/src/Text/Pandoc/Readers/RST.hs
index 29f81b046..514e3b88d 100644
--- a/src/Text/Pandoc/Readers/RST.hs
+++ b/src/Text/Pandoc/Readers/RST.hs
@@ -613,8 +613,9 @@ comment = try $ do
   string ".."
   skipMany1 spaceChar <|> (() <$ lookAhead newline)
   -- notFollowedBy' directiveLabel -- comment comes after directive so unnec.
-  manyTill anyChar blanklines
+  _ <- anyLine
   optional indentedBlock
+  optional blanklines
   return mempty
 
 directiveLabel :: Monad m => RSTParser m Text
diff --git a/test/command/7134.md b/test/command/7134.md
new file mode 100644
index 000000000..c69ae4bd0
--- /dev/null
+++ b/test/command/7134.md
@@ -0,0 +1,16 @@
+```
+% pandoc -f rst -t native
+This is a paragraph.
+
+    This is a block quote.
+
+..
+
+    This should be a second block quote.
+^D
+[Para [Str "This",Space,Str "is",Space,Str "a",Space,Str "paragraph."]
+,BlockQuote
+ [Para [Str "This",Space,Str "is",Space,Str "a",Space,Str "block",Space,Str "quote."]]
+,BlockQuote
+ [Para [Str "This",Space,Str "should",Space,Str "be",Space,Str "a",Space,Str "second",Space,Str "block",Space,Str "quote."]]]
+```
-- 
cgit v1.2.3


From 5608dc01e5342d367fba3377042dec3944f4d86f Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Wed, 10 Mar 2021 10:19:40 -0800
Subject: HTML writer: Add warnings on duplicate attribute values.

This prevents emitting invalid HTML.

Ultimately it would be good to prevent this in the types
themselves, but this is better for now.

T.P.Logging: Add DuplicateAttribute constructor to LogMessage.
[API change]
---
 src/Text/Pandoc/Logging.hs           |  8 +++++++
 src/Text/Pandoc/Writers/HTML.hs      | 41 ++++++++++++++++++++----------------
 test/command/duplicate_attributes.md |  7 ++++++
 3 files changed, 38 insertions(+), 18 deletions(-)
 create mode 100644 test/command/duplicate_attributes.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Logging.hs b/src/Text/Pandoc/Logging.hs
index 1d7bc9d66..efd2188f1 100644
--- a/src/Text/Pandoc/Logging.hs
+++ b/src/Text/Pandoc/Logging.hs
@@ -36,6 +36,7 @@ import Data.Typeable (Typeable)
 import GHC.Generics (Generic)
 import Text.Pandoc.Definition
 import Text.Parsec.Pos
+import Text.Pandoc.Shared (tshow)
 
 -- | Verbosity level.
 data Verbosity = ERROR | WARNING | INFO
@@ -101,6 +102,7 @@ data LogMessage =
   | CiteprocWarning Text
   | ATXHeadingInLHS Int Text
   | EnvironmentVariableUndefined Text
+  | DuplicateAttribute Text Text
   deriving (Show, Eq, Data, Ord, Typeable, Generic)
 
 instance ToJSON LogMessage where
@@ -232,6 +234,9 @@ instance ToJSON LogMessage where
            ,"contents" .= contents]
       EnvironmentVariableUndefined var ->
            ["variable" .= var ]
+      DuplicateAttribute attr val ->
+           ["attribute" .= attr
+           ,"value" .= val]
 
 showPos :: SourcePos -> Text
 showPos pos = Text.pack $ sn ++ "line " ++
@@ -350,6 +355,8 @@ showLogMessage msg =
             else ""
        EnvironmentVariableUndefined var ->
          "Undefined environment variable " <> var <> " in defaults file."
+       DuplicateAttribute attr val ->
+         "Ignoring duplicate attribute " <> attr <> "=" <> tshow val <> "."
 
 messageVerbosity :: LogMessage -> Verbosity
 messageVerbosity msg =
@@ -397,3 +404,4 @@ messageVerbosity msg =
        CiteprocWarning{}             -> WARNING
        ATXHeadingInLHS{}             -> WARNING
        EnvironmentVariableUndefined{}-> WARNING
+       DuplicateAttribute{}          -> WARNING
diff --git a/src/Text/Pandoc/Writers/HTML.hs b/src/Text/Pandoc/Writers/HTML.hs
index 26df0325e..2f33cd467 100644
--- a/src/Text/Pandoc/Writers/HTML.hs
+++ b/src/Text/Pandoc/Writers/HTML.hs
@@ -32,7 +32,7 @@ import Control.Monad.State.Strict
 import Data.Char (ord)
 import Data.List (intercalate, intersperse, partition, delete, (\\))
 import Data.List.NonEmpty (NonEmpty((:|)))
-import Data.Maybe (fromMaybe, isJust, isNothing, mapMaybe)
+import Data.Maybe (fromMaybe, isJust, isNothing)
 import qualified Data.Set as Set
 import Data.Text (Text)
 import qualified Data.Text as T
@@ -551,23 +551,28 @@ toAttrs :: PandocMonad m
 toAttrs kvs = do
   html5 <- gets stHtml5
   mbEpubVersion <- gets stEPUBVersion
-  return $ mapMaybe (\(x,y) ->
-            if html5
-               then
-                  if x `Set.member` (html5Attributes <> rdfaAttributes)
-                     || T.any (== ':') x -- e.g. epub: namespace
-                     || "data-" `T.isPrefixOf` x
-                     || "aria-" `T.isPrefixOf` x
-                     then Just $ customAttribute (textTag x) (toValue y)
-                     else Just $ customAttribute (textTag ("data-" <> x))
-                                  (toValue y)
-               else
-                 if mbEpubVersion == Just EPUB2 &&
-                    not (x `Set.member` (html4Attributes <> rdfaAttributes) ||
-                         "xml:" `T.isPrefixOf` x)
-                    then Nothing
-                    else Just $ customAttribute (textTag x) (toValue y))
-            kvs
+  reverse . snd <$> foldM (go html5 mbEpubVersion) (Set.empty, []) kvs
+ where
+  go html5 mbEpubVersion (keys, attrs) (k,v) = do
+    if k `Set.member` keys
+       then do
+         report $ DuplicateAttribute k v
+         return (keys, attrs)
+       else return (Set.insert k keys, addAttr html5 mbEpubVersion k v attrs)
+  addAttr html5 mbEpubVersion x y
+    | html5
+      = if x `Set.member` (html5Attributes <> rdfaAttributes)
+             || T.any (== ':') x -- e.g. epub: namespace
+             || "data-" `T.isPrefixOf` x
+             || "aria-" `T.isPrefixOf` x
+           then (customAttribute (textTag x) (toValue y) :)
+           else (customAttribute (textTag ("data-" <> x)) (toValue y) :)
+    | mbEpubVersion == Just EPUB2
+    , not (x `Set.member` (html4Attributes <> rdfaAttributes) ||
+      "xml:" `T.isPrefixOf` x)
+      = id
+    | otherwise
+      = (customAttribute (textTag x) (toValue y) :)
 
 attrsToHtml :: PandocMonad m
             => WriterOptions -> Attr -> StateT WriterState m [Attribute]
diff --git a/test/command/duplicate_attributes.md b/test/command/duplicate_attributes.md
new file mode 100644
index 000000000..b6e8a4c21
--- /dev/null
+++ b/test/command/duplicate_attributes.md
@@ -0,0 +1,7 @@
+```
+% pandoc
+[span]{.foobar style="color:blue" class="zip" style="color:red"}
+^D
+[WARNING] Ignoring duplicate attribute style="color:red".
+<p><span class="foobar zip" style="color:blue">span</span></p>
+```
-- 
cgit v1.2.3


From 35b66a76718205c303f416bf0afc01c098e8a171 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sat, 13 Mar 2021 12:50:44 -0800
Subject: MediaWiki reader: Allow block-level content in notes (ref).

Closes #7145.
---
 src/Text/Pandoc/Readers/MediaWiki.hs | 10 +++++++++-
 test/command/7145.md                 | 12 ++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)
 create mode 100644 test/command/7145.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/MediaWiki.hs b/src/Text/Pandoc/Readers/MediaWiki.hs
index cdb746c67..9f4d5e170 100644
--- a/src/Text/Pandoc/Readers/MediaWiki.hs
+++ b/src/Text/Pandoc/Readers/MediaWiki.hs
@@ -112,12 +112,14 @@ newBlockTags = ["haskell","syntaxhighlight","source","gallery","references"]
 isBlockTag' :: Tag Text -> Bool
 isBlockTag' tag@(TagOpen t _) = (isBlockTag tag || t `elem` newBlockTags) &&
   t `notElem` eitherBlockOrInline
+isBlockTag' (TagClose "ref") = True -- needed so 'special' doesn't parse it
 isBlockTag' tag@(TagClose t) = (isBlockTag tag || t `elem` newBlockTags) &&
   t `notElem` eitherBlockOrInline
 isBlockTag' tag = isBlockTag tag
 
 isInlineTag' :: Tag Text -> Bool
 isInlineTag' (TagComment _) = True
+isInlineTag' (TagClose "ref") = False -- see below inlineTag
 isInlineTag' t              = not (isBlockTag' t)
 
 eitherBlockOrInline :: [Text]
@@ -554,11 +556,17 @@ variable = try $ do
   contents <- manyTillChar anyChar (try $ string "}}}")
   return $ "{{{" <> contents <> "}}}"
 
+singleParaToPlain :: Blocks -> Blocks
+singleParaToPlain bs =
+  case B.toList bs of
+    [Para ils] -> B.fromList [Plain ils]
+    _ -> bs
+
 inlineTag :: PandocMonad m => MWParser m Inlines
 inlineTag = do
   (tag, _) <- lookAhead $ htmlTag isInlineTag'
   case tag of
-       TagOpen "ref" _ -> B.note . B.plain <$> inlinesInTags "ref"
+       TagOpen "ref" _ -> B.note . singleParaToPlain <$> blocksInTags "ref"
        TagOpen "nowiki" _ -> try $ do
           (_,raw) <- htmlTag (~== tag)
           if T.any (== '/') raw
diff --git a/test/command/7145.md b/test/command/7145.md
new file mode 100644
index 000000000..a04345890
--- /dev/null
+++ b/test/command/7145.md
@@ -0,0 +1,12 @@
+```
+% pandoc -f mediawiki -t native
+Maecenas at sapien tempor, pretium turpis ut, imperdiet augue.<ref>This is a multiline
+
+reference
+<i>with</i>
+empty
+
+linebreaks</ref>  Nulla ut massa eget ex venenatis lobortis id in eros.
+^D
+[Para [Str "Maecenas",Space,Str "at",Space,Str "sapien",Space,Str "tempor,",Space,Str "pretium",Space,Str "turpis",Space,Str "ut,",Space,Str "imperdiet",Space,Str "augue.",Note [Para [Str "This",Space,Str "is",Space,Str "a",Space,Str "multiline"],Para [Str "reference",SoftBreak,RawInline (Format "html") "<i>",Str "with",RawInline (Format "html") "</i>",SoftBreak,Str "empty"],Para [Str "linebreaks"]],Space,Str "Nulla",Space,Str "ut",Space,Str "massa",Space,Str "eget",Space,Str "ex",Space,Str "venenatis",Space,Str "lobortis",Space,Str "id",Space,Str "in",Space,Str "eros."]]
+```
-- 
cgit v1.2.3


From e66bf891ec21cc09611c5be4e4847fccd39e0084 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Wed, 17 Mar 2021 09:02:20 -0700
Subject: Add test for #7155.

---
 test/command/7155.md | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 test/command/7155.md

(limited to 'test/command')

diff --git a/test/command/7155.md b/test/command/7155.md
new file mode 100644
index 000000000..49358bf97
--- /dev/null
+++ b/test/command/7155.md
@@ -0,0 +1,15 @@
+```
+% pandoc -f markdown+tex_math_single_backslash -t native
+\(x\) \[x\]
+\\(x\\) \\[x\\]
+^D
+[Para [Math InlineMath "x",Space,Math DisplayMath "x",SoftBreak,Str "\\(x\\)",Space,Str "\\[x\\]"]]
+```
+
+```
+% pandoc -f markdown+tex_math_double_backslash -t native
+\(x\) \[x\]
+\\(x\\) \\[x\\]
+^D
+[Para [Str "(x)",Space,Str "[x]",SoftBreak,Math InlineMath "x",Space,Math DisplayMath "x"]]
+```
-- 
cgit v1.2.3


From 56ce1fc12651e9cf60b882a18d72a410967d6540 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Wed, 24 Mar 2021 11:57:49 -0700
Subject: Fix DocBook reader mathml regression...

...caused by the switch in XML libraries.
Also fixed a similar issue in JATS.
Closes #7173.
---
 src/Text/Pandoc/Readers/DocBook.hs |   3 +-
 src/Text/Pandoc/Readers/JATS.hs    |   8 ++-
 test/command/7173.md               | 137 +++++++++++++++++++++++++++++++++++++
 test/docbook-reader.docbook        |   4 +-
 4 files changed, 147 insertions(+), 5 deletions(-)
 create mode 100644 test/command/7173.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/DocBook.hs b/src/Text/Pandoc/Readers/DocBook.hs
index 6f5bb0ad4..4f525cfb1 100644
--- a/src/Text/Pandoc/Readers/DocBook.hs
+++ b/src/Text/Pandoc/Readers/DocBook.hs
@@ -1239,7 +1239,8 @@ equation e constructor =
   where
     mathMLEquations :: [Text]
     mathMLEquations = map writeTeX $ rights $ readMath
-      (\x -> qName (elName x) == "math" && qPrefix (elName x) == Just "mml")
+      (\x -> qName (elName x) == "math" &&
+             qURI (elName x) == Just "http://www.w3.org/1998/Math/MathML")
       (readMathML . showElement)
 
     latexEquations :: [Text]
diff --git a/src/Text/Pandoc/Readers/JATS.hs b/src/Text/Pandoc/Readers/JATS.hs
index 602f3b4f2..c068f3774 100644
--- a/src/Text/Pandoc/Readers/JATS.hs
+++ b/src/Text/Pandoc/Readers/JATS.hs
@@ -491,7 +491,8 @@ parseInline (Elem e) =
 
         "disp-formula" -> formula displayMath
         "inline-formula" -> formula math
-        "math" | qPrefix (elName e) == Just "mml" -> return . math $ mathML e
+        "math" | qURI (elName e) == Just "http://www.w3.org/1998/Math/MathML"
+                   -> return . math $ mathML e
         "tex-math" -> return . math $ textContent e
 
         "email" -> return $ link ("mailto:" <> textContent e) ""
@@ -514,8 +515,9 @@ parseInline (Elem e) =
                             filterChildren isMathML whereToLook
             return . mconcat . take 1 . map constructor $ texMaths ++ mathMLs
 
-         isMathML x = qName   (elName x) == "math" &&
-                      qPrefix (elName x) == Just "mml"
+         isMathML x = qName (elName x) == "math" &&
+                      qURI  (elName x) ==
+                                      Just "http://www.w3.org/1998/Math/MathML"
          removePrefix elname = elname { qPrefix = Nothing }
          codeWithLang = do
            let classes' = case attrValue "language" e of
diff --git a/test/command/7173.md b/test/command/7173.md
new file mode 100644
index 000000000..2599dc19b
--- /dev/null
+++ b/test/command/7173.md
@@ -0,0 +1,137 @@
+```
+% pandoc -f docbook -t latex
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE article PUBLIC "-//W3C//DTD MathML 2.0//EN" "http://www.w3.org/Math/DTD/mathml2/mathml2.dtd">
+<?xml-model href="http://www.oxygenxml.com/docbook/xml/5.0/rng/dbmathml.rng" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<?xml-model href="http://docbook.org/xml/5.0/rng/docbook.rng" type="application/xml" schematypens="http://purl.oclc.org/dsdl/schematron"?>
+<article xmlns="http://docbook.org/ns/docbook"
+    xmlns:xlink="http://www.w3.org/1999/xlink"
+    version="5.1">
+    <title>Untitled Document</title>
+    <para>Word</para>
+    <informalequation>
+        <alt role='tex'>1+2</alt>
+        <m:math xmlns:m="http://www.w3.org/1998/Math/MathML">
+            <m:mrow>
+                <m:mrow><m:mn>1</m:mn><m:mo>+</m:mo><m:mn>2</m:mn>
+                </m:mrow>
+            </m:mrow>
+        </m:math>
+    </informalequation>
+</article>
+^D
+Word
+
+\[1 + 2\]
+```
+
+```
+% pandoc -f docbook -t latex
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE article PUBLIC "-//W3C//DTD MathML 2.0//EN" "http://www.w3.org/Math/DTD/mathml2/mathml2.dtd">
+<?xml-model href="http://www.oxygenxml.com/docbook/xml/5.0/rng/dbmathml.rng" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<?xml-model href="http://docbook.org/xml/5.0/rng/docbook.rng" type="application/xml" schematypens="http://purl.oclc.org/dsdl/schematron"?>
+<article xmlns="http://docbook.org/ns/docbook"
+    xmlns:xlink="http://www.w3.org/1999/xlink"
+    version="5.1">
+    <title>Untitled Document</title>
+    <para>Word</para>
+    <informalequation>
+        <alt role='tex'>1+2</alt>
+        <mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML">
+            <mml:mrow>
+                <mml:mrow><mml:mn>1</mml:mn><mml:mo>+</mml:mo><mml:mn>2</mml:mn>
+                </mml:mrow>
+            </mml:mrow>
+        </mml:math>
+    </informalequation>
+</article>
+^D
+Word
+
+\[1 + 2\]
+```
+
+```
+% pandoc -f docbook -t latex
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE article PUBLIC "-//W3C//DTD MathML 2.0//EN" "http://www.w3.org/Math/DTD/mathml2/mathml2.dtd">
+<?xml-model href="http://www.oxygenxml.com/docbook/xml/5.0/rng/dbmathml.rng" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<?xml-model href="http://docbook.org/xml/5.0/rng/docbook.rng" type="application/xml" schematypens="http://purl.oclc.org/dsdl/schematron"?>
+<article xmlns="http://docbook.org/ns/docbook"
+    xmlns:xlink="http://www.w3.org/1999/xlink"
+    version="5.1">
+    <title>Untitled Document</title>
+    <para>Word</para>
+    <informalequation>
+        <alt role='tex'>1+2</alt>
+        <math xmlns="http://www.w3.org/1998/Math/MathML">
+            <mrow>
+                <mrow><mn>1</mn><mo>+</mo><mn>2</mn>
+                </mrow>
+            </mrow>
+        </math>
+    </informalequation>
+</article>
+^D
+Word
+
+\[1 + 2\]
+```
+
+```
+% pandoc -f docbook -t latex
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE article PUBLIC "-//W3C//DTD MathML 2.0//EN" "http://www.w3.org/Math/DTD/mathml2/mathml2.dtd">
+<?xml-model href="http://www.oxygenxml.com/docbook/xml/5.0/rng/dbmathml.rng" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<?xml-model href="http://docbook.org/xml/5.0/rng/docbook.rng" type="application/xml" schematypens="http://purl.oclc.org/dsdl/schematron"?>
+<article xmlns="http://docbook.org/ns/docbook"
+    xmlns:xlink="http://www.w3.org/1999/xlink"
+    xmlns:m="http://www.w3.org/1998/Math/MathML"
+    version="5.1">
+    <title>Untitled Document</title>
+    <para>Word</para>
+    <informalequation>
+        <alt role='tex'>1+2</alt>
+        <m:math>
+            <m:mrow>
+                <m:mrow><m:mn>1</m:mn><m:mo>+</m:mo><m:mn>2</m:mn>
+                </m:mrow>
+            </m:mrow>
+        </m:math>
+    </informalequation>
+</article>
+^D
+Word
+
+\[1 + 2\]
+```
+
+```
+% pandoc -f docbook -t latex
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE article PUBLIC "-//W3C//DTD MathML 2.0//EN" "http://www.w3.org/Math/DTD/mathml2/mathml2.dtd">
+<?xml-model href="http://www.oxygenxml.com/docbook/xml/5.0/rng/dbmathml.rng" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<?xml-model href="http://docbook.org/xml/5.0/rng/docbook.rng" type="application/xml" schematypens="http://purl.oclc.org/dsdl/schematron"?>
+<article xmlns="http://docbook.org/ns/docbook"
+    xmlns:xlink="http://www.w3.org/1999/xlink"
+    xmlns:mml="http://www.w3.org/1998/Math/MathML"
+    version="5.1">
+    <title>Untitled Document</title>
+    <para>Word</para>
+    <informalequation>
+        <alt role='tex'>1+2</alt>
+        <mml:math>
+            <mml:mrow>
+                <mml:mrow><mml:mn>1</mml:mn><mml:mo>+</mml:mo><mml:mn>2</mml:mn>
+                </mml:mrow>
+            </mml:mrow>
+        </mml:math>
+    </informalequation>
+</article>
+^D
+Word
+
+\[1 + 2\]
+```
+
diff --git a/test/docbook-reader.docbook b/test/docbook-reader.docbook
index 5717d78d0..51e62942b 100644
--- a/test/docbook-reader.docbook
+++ b/test/docbook-reader.docbook
@@ -6,7 +6,9 @@
 <!ENTITY let "LET" >
 <!ENTITY case "CASE" >
 ]>
-<article>
+<article xmlns="http://docbook.org/ns/docbook"
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xmlns:mml="http://www.w3.org/1998/Math/MathML">
   <articleinfo>
     <title>Pandoc Test Suite</title>
     <authorgroup>
-- 
cgit v1.2.3


From 60974538b25657c9aa37e72cc66ca3957912ddec Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Mon, 5 Apr 2021 23:29:22 -0700
Subject: Commonmark writer: Use backslash escapes for `<` and `|`...

instead of entities.  Closes #7208.
---
 src/Text/Pandoc/Writers/Markdown.hs | 12 +++++++++++-
 test/command/7208.md                |  6 ++++++
 2 files changed, 17 insertions(+), 1 deletion(-)
 create mode 100644 test/command/7208.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/Markdown.hs b/src/Text/Pandoc/Writers/Markdown.hs
index 4d9f3d5b0..daf45ed53 100644
--- a/src/Text/Pandoc/Writers/Markdown.hs
+++ b/src/Text/Pandoc/Writers/Markdown.hs
@@ -67,7 +67,17 @@ writePlain opts document =
 -- | Convert Pandoc to Commonmark.
 writeCommonMark :: PandocMonad m => WriterOptions -> Pandoc -> m Text
 writeCommonMark opts document =
-  evalMD (pandocToMarkdown opts document) def{ envVariant = Commonmark } def
+  evalMD (pandocToMarkdown opts' document) def{ envVariant = Commonmark } def
+ where
+  opts' = opts{ writerExtensions =
+                   -- These extensions can't be enabled or disabled
+                   -- for commonmark because they're part of the core;
+                   -- we set them here so that escapeText will behave
+                   -- properly.
+                   enableExtension Ext_all_symbols_escapable $
+                   enableExtension Ext_pipe_tables $
+                   enableExtension Ext_intraword_underscores $
+                     writerExtensions opts }
 
 pandocTitleBlock :: Doc Text -> [Doc Text] -> Doc Text -> Doc Text
 pandocTitleBlock tit auths dat =
diff --git a/test/command/7208.md b/test/command/7208.md
new file mode 100644
index 000000000..e65943ade
--- /dev/null
+++ b/test/command/7208.md
@@ -0,0 +1,6 @@
+```
+% pandoc -t gfm
+\<hi\>
+^D
+\<hi\>
+```
-- 
cgit v1.2.3


From 20cd33e5a44810b68fed74da00f4f51eb2282147 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 8 Apr 2021 14:47:11 -0700
Subject: Fix regression in grid tables for wide characters.

In the translation from String to Text, a char-width-sensitive
splitAt' was dropped.  This commit reinstates it.
Closes #7214.
---
 src/Text/Pandoc/Shared.hs | 18 +++++++++++++-----
 test/command/7214.md      | 28 ++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+), 5 deletions(-)
 create mode 100644 test/command/7214.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Shared.hs b/src/Text/Pandoc/Shared.hs
index 3292b32f4..95cbdc8b8 100644
--- a/src/Text/Pandoc/Shared.hs
+++ b/src/Text/Pandoc/Shared.hs
@@ -153,12 +153,20 @@ splitTextBy isSep t
                 in  first : splitTextBy isSep (T.dropWhile isSep rest)
 
 splitTextByIndices :: [Int] -> T.Text -> [T.Text]
-splitTextByIndices ns = splitTextByRelIndices (zipWith (-) ns (0:ns))
+splitTextByIndices ns = splitTextByRelIndices (zipWith (-) ns (0:ns)) . T.unpack
  where
-  splitTextByRelIndices [] t = [t]
-  splitTextByRelIndices (x:xs) t =
-    let (first, rest) = T.splitAt x t
-     in first : splitTextByRelIndices xs rest
+  splitTextByRelIndices [] cs = [T.pack cs]
+  splitTextByRelIndices (x:xs) cs =
+    let (first, rest) = splitAt' x cs
+     in T.pack first : splitTextByRelIndices xs rest
+
+-- Note: don't replace this with T.splitAt, which is not sensitive
+-- to character widths!
+splitAt' :: Int -> [Char] -> ([Char],[Char])
+splitAt' _ []          = ([],[])
+splitAt' n xs | n <= 0 = ([],xs)
+splitAt' n (x:xs)      = (x:ys,zs)
+  where (ys,zs) = splitAt' (n - charWidth x) xs
 
 ordNub :: (Ord a) => [a] -> [a]
 ordNub l = go Set.empty l
diff --git a/test/command/7214.md b/test/command/7214.md
new file mode 100644
index 000000000..43bf9e4ca
--- /dev/null
+++ b/test/command/7214.md
@@ -0,0 +1,28 @@
+```
+% pandoc
++------------+----------+------------------+
+|日本語      | の文字列 | words in english |
++------------+----------+------------------+
+|abc defghij | def      | xyz              |
++------------+----------+------------------+
+^D
+<table style="width:60%;">
+<colgroup>
+<col style="width: 18%" />
+<col style="width: 15%" />
+<col style="width: 26%" />
+</colgroup>
+<tbody>
+<tr class="odd">
+<td>日本語</td>
+<td>の文字列</td>
+<td>words in english</td>
+</tr>
+<tr class="even">
+<td>abc defghij</td>
+<td>def</td>
+<td>xyz</td>
+</tr>
+</tbody>
+</table>
+```
-- 
cgit v1.2.3


From 5f79a66ed64e9b0cc326e467dcb17239f1596fcc Mon Sep 17 00:00:00 2001
From: Albert Krewinkel <albert@zeitkraut.de>
Date: Fri, 16 Apr 2021 22:13:29 +0200
Subject: JATS writer: reduce unnecessary use of <p> elements for wrapping

The `<p>` element is used for wrapping in cases were the contents would
otherwise not be allowed in a certain context. Unnecessary wrapping is
avoided, especially around quotes (`<disp-quote>` elements).

Closes: #7227
---
 src/Text/Pandoc/Writers/JATS.hs       | 22 +++++----
 src/Text/Pandoc/Writers/JATS/Table.hs | 26 +++++++++--
 src/Text/Pandoc/Writers/JATS/Types.hs | 15 ++++--
 test/command/7041.md                  | 23 +++++++++
 test/writer.jats_archiving            | 68 +++++++++++----------------
 test/writer.jats_articleauthoring     | 88 ++++++++++++++++-------------------
 test/writer.jats_publishing           | 68 +++++++++++----------------
 7 files changed, 166 insertions(+), 144 deletions(-)
 create mode 100644 test/command/7041.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/JATS.hs b/src/Text/Pandoc/Writers/JATS.hs
index 5b3e439d4..b58ff8aef 100644
--- a/src/Text/Pandoc/Writers/JATS.hs
+++ b/src/Text/Pandoc/Writers/JATS.hs
@@ -1,6 +1,7 @@
+{-# LANGUAGE LambdaCase          #-}
+{-# LANGUAGE OverloadedStrings   #-}
 {-# LANGUAGE ScopedTypeVariables #-}
-{-# LANGUAGE OverloadedStrings #-}
-{-# LANGUAGE ViewPatterns #-}
+{-# LANGUAGE ViewPatterns        #-}
 {- |
    Module      : Text.Pandoc.Writers.JATS
    Copyright   : 2017-2021 John MacFarlane
@@ -80,7 +81,7 @@ writeJats tagSet opts d = do
   let environment = JATSEnv
           { jatsTagSet = tagSet
           , jatsInlinesWriter = inlinesToJATS
-          , jatsBlockWriter = blockToJATS
+          , jatsBlockWriter = wrappedBlocksToJATS
           , jatsReferences = refs
           }
   let initialState = JATSState { jatsNotes = [] }
@@ -162,11 +163,9 @@ wrappedBlocksToJATS needsWrap opts =
     wrappedBlockToJATS b = do
       inner <- blockToJATS opts b
       return $
-        if needsWrap b || isBlockQuote b -- see #7041
+        if needsWrap b
            then inTags True "p" [("specific-use","wrapper")] inner
            else inner
-    isBlockQuote (BlockQuote _) = True
-    isBlockQuote _ = False
 
 -- | Auxiliary function to convert Plain block to Para.
 plainToPara :: Block -> Block
@@ -324,10 +323,13 @@ blockToJATS opts (LineBlock lns) =
   blockToJATS opts $ linesToPara lns
 blockToJATS opts (BlockQuote blocks) = do
   tagSet <- asks jatsTagSet
-  let blocksToJats' = if tagSet == TagSetArticleAuthoring
-                      then wrappedBlocksToJATS (not . isPara)
-                      else blocksToJATS
-  inTagsIndented "disp-quote" <$> blocksToJats' opts blocks
+  let needsWrap = if tagSet == TagSetArticleAuthoring
+                  then not . isPara
+                  else \case
+                    Header{}       -> True
+                    HorizontalRule -> True
+                    _              -> False
+  inTagsIndented "disp-quote" <$> wrappedBlocksToJATS needsWrap opts blocks
 blockToJATS _ (CodeBlock a str) = return $
   inTags False tag attr (flush (text (T.unpack $ escapeStringForXML str)))
     where (lang, attr) = codeAttr a
diff --git a/src/Text/Pandoc/Writers/JATS/Table.hs b/src/Text/Pandoc/Writers/JATS/Table.hs
index 2e34900d2..70569bdcd 100644
--- a/src/Text/Pandoc/Writers/JATS/Table.hs
+++ b/src/Text/Pandoc/Writers/JATS/Table.hs
@@ -34,13 +34,19 @@ tableToJATS :: PandocMonad m
             -> JATS m (Doc Text)
 tableToJATS opts (Ann.Table attr caption colspecs thead tbodies tfoot) = do
   let (Caption _maybeShortCaption captionBlocks) = caption
+  -- Only paragraphs are allowed in captions, all other blocks must be
+  -- wrapped in @<p>@ elements.
+  let needsWrapping = \case
+        Plain{} -> False
+        Para{}  -> False
+        _       -> True
   tbl <- captionlessTable opts attr colspecs thead tbodies tfoot
   captionDoc <- if null captionBlocks
                 then return empty
                 else do
                   blockToJATS <- asks jatsBlockWriter
-                  inTagsIndented "caption" . vcat <$>
-                    mapM (blockToJATS opts) captionBlocks
+                  inTagsIndented "caption" <$>
+                    blockToJATS needsWrapping opts captionBlocks
   return $ inTags True "table-wrap" [] $ captionDoc $$ tbl
 
 captionlessTable :: PandocMonad m
@@ -230,7 +236,7 @@ tableCellToJats opts ctype colAlign (Cell attr align rowspan colspan item) = do
   inlinesToJats <- asks jatsInlinesWriter
   let cellContents = \case
         [Plain inlines] -> inlinesToJats opts inlines
-        blocks          -> vcat <$> mapM (blockToJats opts) blocks
+        blocks          -> blockToJats needsWrapInCell opts blocks
   let tag' = case ctype of
         BodyCell   -> "td"
         HeaderCell -> "th"
@@ -246,3 +252,17 @@ tableCellToJats opts ctype colAlign (Cell attr align rowspan colspan item) = do
               . maybeCons (colspanAttrib colspan)
               $ toAttribs attr validAttribs
   inTags False tag' attribs <$> cellContents item
+
+-- | Whether the JATS produced from this block should be wrapped in a
+-- @<p>@ element when put directly below a @<td>@ element.
+needsWrapInCell :: Block -> Bool
+needsWrapInCell = \case
+  Plain{}          -> False  -- should be unwrapped anyway
+  Para{}           -> False
+  BulletList{}     -> False
+  OrderedList{}    -> False
+  DefinitionList{} -> False
+  HorizontalRule   -> False
+  CodeBlock{}      -> False
+  RawBlock{}       -> False  -- responsibility of the user
+  _                -> True
diff --git a/src/Text/Pandoc/Writers/JATS/Types.hs b/src/Text/Pandoc/Writers/JATS/Types.hs
index 6fdddc0b5..8d8673cf6 100644
--- a/src/Text/Pandoc/Writers/JATS/Types.hs
+++ b/src/Text/Pandoc/Writers/JATS/Types.hs
@@ -37,11 +37,20 @@ newtype JATSState = JATSState
   { jatsNotes :: [(Int, Doc Text)]
   }
 
+-- | Environment containing all information relevant for rendering.
 data JATSEnv m = JATSEnv
-  { jatsTagSet :: JATSTagSet
+  { jatsTagSet :: JATSTagSet  -- ^ The tag set that's being ouput
+
+  , jatsBlockWriter   :: (Block -> Bool)
+                      -> WriterOptions -> [Block]  -> JATS m (Doc Text)
+    -- ^ Converts a block list to JATS, wrapping top-level blocks into a
+    -- @<p>@ element if the property evaluates to @True@.
+    -- See #7227.
+
   , jatsInlinesWriter :: WriterOptions -> [Inline] -> JATS m (Doc Text)
-  , jatsBlockWriter   :: WriterOptions -> Block    -> JATS m (Doc Text)
-  , jatsReferences    :: [Reference Inlines]
+    -- ^ Converts an inline list to JATS.
+
+  , jatsReferences    :: [Reference Inlines] -- ^ List of references
   }
 
 -- | JATS writer type
diff --git a/test/command/7041.md b/test/command/7041.md
new file mode 100644
index 000000000..1773963b8
--- /dev/null
+++ b/test/command/7041.md
@@ -0,0 +1,23 @@
+```
+% pandoc -f html -t jats
+<table>
+  <tr><td><blockquote>Fly, you fools!</blockquote></td></tr>
+</table>
+^D
+<table-wrap>
+  <table>
+    <colgroup>
+      <col width="100%" />
+    </colgroup>
+    <tbody>
+      <tr>
+        <td><p specific-use="wrapper">
+          <disp-quote>
+            <p>Fly, you fools!</p>
+          </disp-quote>
+        </p></td>
+      </tr>
+    </tbody>
+  </table>
+</table-wrap>
+```
diff --git a/test/writer.jats_archiving b/test/writer.jats_archiving
index 332b5d3fd..70e15b6b8 100644
--- a/test/writer.jats_archiving
+++ b/test/writer.jats_archiving
@@ -78,39 +78,31 @@ Gruber’s markdown test suite.</p>
 <sec id="block-quotes">
   <title>Block Quotes</title>
   <p>E-mail style:</p>
-  <p specific-use="wrapper">
+  <disp-quote>
+    <p>This is a block quote. It is pretty short.</p>
+  </disp-quote>
+  <disp-quote>
+    <p>Code in a block quote:</p>
+    <preformat>sub status {
+    print &quot;working&quot;;
+}</preformat>
+    <p>A list:</p>
+    <list list-type="order">
+      <list-item>
+        <p>item one</p>
+      </list-item>
+      <list-item>
+        <p>item two</p>
+      </list-item>
+    </list>
+    <p>Nested block quotes:</p>
     <disp-quote>
-      <p>This is a block quote. It is pretty short.</p>
+      <p>nested</p>
     </disp-quote>
-  </p>
-  <p specific-use="wrapper">
     <disp-quote>
-      <p>Code in a block quote:</p>
-      <preformat>sub status {
-    print &quot;working&quot;;
-}</preformat>
-      <p>A list:</p>
-      <list list-type="order">
-        <list-item>
-          <p>item one</p>
-        </list-item>
-        <list-item>
-          <p>item two</p>
-        </list-item>
-      </list>
-      <p>Nested block quotes:</p>
-      <p specific-use="wrapper">
-        <disp-quote>
-          <p>nested</p>
-        </disp-quote>
-      </p>
-      <p specific-use="wrapper">
-        <disp-quote>
-          <p>nested</p>
-        </disp-quote>
-      </p>
+      <p>nested</p>
     </disp-quote>
-  </p>
+  </disp-quote>
   <p>This should not be a block quote: 2 &gt; 1.</p>
   <p>And a following paragraph.</p>
 </sec>
@@ -837,12 +829,10 @@ These should not be escaped:  \$ \\ \&gt; \[ \{</preformat>
       </list-item>
     </list>
     <p>An e-mail address: <email>nobody@nowhere.net</email></p>
-    <p specific-use="wrapper">
-      <disp-quote>
-        <p>Blockquoted:
-        <ext-link ext-link-type="uri" xlink:href="http://example.com/">http://example.com/</ext-link></p>
-      </disp-quote>
-    </p>
+    <disp-quote>
+      <p>Blockquoted:
+      <ext-link ext-link-type="uri" xlink:href="http://example.com/">http://example.com/</ext-link></p>
+    </disp-quote>
     <p>Auto-links should not occur here:
     <monospace>&lt;http://example.com/&gt;</monospace></p>
     <preformat>or here: &lt;http://example.com/&gt;</preformat>
@@ -866,11 +856,9 @@ These should not be escaped:  \$ \\ \&gt; \[ \{</preformat>
   <italic>not</italic> be a footnote reference, because it contains a
   space.[^my note] Here is an inline
   note.<xref ref-type="fn" rid="fn3">3</xref></p>
-  <p specific-use="wrapper">
-    <disp-quote>
-      <p>Notes can go in quotes.<xref ref-type="fn" rid="fn4">4</xref></p>
-    </disp-quote>
-  </p>
+  <disp-quote>
+    <p>Notes can go in quotes.<xref ref-type="fn" rid="fn4">4</xref></p>
+  </disp-quote>
   <list list-type="order">
     <list-item>
       <p>And in list items.<xref ref-type="fn" rid="fn5">5</xref></p>
diff --git a/test/writer.jats_articleauthoring b/test/writer.jats_articleauthoring
index 956a30faa..59485a114 100644
--- a/test/writer.jats_articleauthoring
+++ b/test/writer.jats_articleauthoring
@@ -67,43 +67,39 @@ Gruber’s markdown test suite.</p>
 <sec id="block-quotes">
   <title>Block Quotes</title>
   <p>E-mail style:</p>
-  <p specific-use="wrapper">
-    <disp-quote>
-      <p>This is a block quote. It is pretty short.</p>
-    </disp-quote>
-  </p>
-  <p specific-use="wrapper">
-    <disp-quote>
-      <p>Code in a block quote:</p>
-      <p specific-use="wrapper">
-        <preformat>sub status {
+  <disp-quote>
+    <p>This is a block quote. It is pretty short.</p>
+  </disp-quote>
+  <disp-quote>
+    <p>Code in a block quote:</p>
+    <p specific-use="wrapper">
+      <preformat>sub status {
     print &quot;working&quot;;
 }</preformat>
-      </p>
-      <p>A list:</p>
-      <p specific-use="wrapper">
-        <list list-type="order">
-          <list-item>
-            <p>item one</p>
-          </list-item>
-          <list-item>
-            <p>item two</p>
-          </list-item>
-        </list>
-      </p>
-      <p>Nested block quotes:</p>
-      <p specific-use="wrapper">
-        <disp-quote>
-          <p>nested</p>
-        </disp-quote>
-      </p>
-      <p specific-use="wrapper">
-        <disp-quote>
-          <p>nested</p>
-        </disp-quote>
-      </p>
-    </disp-quote>
-  </p>
+    </p>
+    <p>A list:</p>
+    <p specific-use="wrapper">
+      <list list-type="order">
+        <list-item>
+          <p>item one</p>
+        </list-item>
+        <list-item>
+          <p>item two</p>
+        </list-item>
+      </list>
+    </p>
+    <p>Nested block quotes:</p>
+    <p specific-use="wrapper">
+      <disp-quote>
+        <p>nested</p>
+      </disp-quote>
+    </p>
+    <p specific-use="wrapper">
+      <disp-quote>
+        <p>nested</p>
+      </disp-quote>
+    </p>
+  </disp-quote>
   <p>This should not be a block quote: 2 &gt; 1.</p>
   <p>And a following paragraph.</p>
 </sec>
@@ -817,12 +813,10 @@ These should not be escaped:  \$ \\ \&gt; \[ \{</preformat>
       </list-item>
     </list>
     <p>An e-mail address: <email>nobody@nowhere.net</email></p>
-    <p specific-use="wrapper">
-      <disp-quote>
-        <p>Blockquoted:
-        <ext-link ext-link-type="uri" xlink:href="http://example.com/">http://example.com/</ext-link></p>
-      </disp-quote>
-    </p>
+    <disp-quote>
+      <p>Blockquoted:
+      <ext-link ext-link-type="uri" xlink:href="http://example.com/">http://example.com/</ext-link></p>
+    </disp-quote>
     <p>Auto-links should not occur here:
     <monospace>&lt;http://example.com/&gt;</monospace></p>
     <preformat>or here: &lt;http://example.com/&gt;</preformat>
@@ -860,13 +854,11 @@ These should not be escaped:  \$ \\ \&gt; \[ \{</preformat>
     and <monospace>]</monospace> verbatim characters, as well as [bracketed
     text].</p>
   </fn></p>
-  <p specific-use="wrapper">
-    <disp-quote>
-      <p>Notes can go in quotes.<fn>
-        <p>In quote.</p>
-      </fn></p>
-    </disp-quote>
-  </p>
+  <disp-quote>
+    <p>Notes can go in quotes.<fn>
+      <p>In quote.</p>
+    </fn></p>
+  </disp-quote>
   <list list-type="order">
     <list-item>
       <p>And in list items.<fn>
diff --git a/test/writer.jats_publishing b/test/writer.jats_publishing
index f53fd554d..e6db4172a 100644
--- a/test/writer.jats_publishing
+++ b/test/writer.jats_publishing
@@ -78,39 +78,31 @@ Gruber’s markdown test suite.</p>
 <sec id="block-quotes">
   <title>Block Quotes</title>
   <p>E-mail style:</p>
-  <p specific-use="wrapper">
+  <disp-quote>
+    <p>This is a block quote. It is pretty short.</p>
+  </disp-quote>
+  <disp-quote>
+    <p>Code in a block quote:</p>
+    <preformat>sub status {
+    print &quot;working&quot;;
+}</preformat>
+    <p>A list:</p>
+    <list list-type="order">
+      <list-item>
+        <p>item one</p>
+      </list-item>
+      <list-item>
+        <p>item two</p>
+      </list-item>
+    </list>
+    <p>Nested block quotes:</p>
     <disp-quote>
-      <p>This is a block quote. It is pretty short.</p>
+      <p>nested</p>
     </disp-quote>
-  </p>
-  <p specific-use="wrapper">
     <disp-quote>
-      <p>Code in a block quote:</p>
-      <preformat>sub status {
-    print &quot;working&quot;;
-}</preformat>
-      <p>A list:</p>
-      <list list-type="order">
-        <list-item>
-          <p>item one</p>
-        </list-item>
-        <list-item>
-          <p>item two</p>
-        </list-item>
-      </list>
-      <p>Nested block quotes:</p>
-      <p specific-use="wrapper">
-        <disp-quote>
-          <p>nested</p>
-        </disp-quote>
-      </p>
-      <p specific-use="wrapper">
-        <disp-quote>
-          <p>nested</p>
-        </disp-quote>
-      </p>
+      <p>nested</p>
     </disp-quote>
-  </p>
+  </disp-quote>
   <p>This should not be a block quote: 2 &gt; 1.</p>
   <p>And a following paragraph.</p>
 </sec>
@@ -837,12 +829,10 @@ These should not be escaped:  \$ \\ \&gt; \[ \{</preformat>
       </list-item>
     </list>
     <p>An e-mail address: <email>nobody@nowhere.net</email></p>
-    <p specific-use="wrapper">
-      <disp-quote>
-        <p>Blockquoted:
-        <ext-link ext-link-type="uri" xlink:href="http://example.com/">http://example.com/</ext-link></p>
-      </disp-quote>
-    </p>
+    <disp-quote>
+      <p>Blockquoted:
+      <ext-link ext-link-type="uri" xlink:href="http://example.com/">http://example.com/</ext-link></p>
+    </disp-quote>
     <p>Auto-links should not occur here:
     <monospace>&lt;http://example.com/&gt;</monospace></p>
     <preformat>or here: &lt;http://example.com/&gt;</preformat>
@@ -866,11 +856,9 @@ These should not be escaped:  \$ \\ \&gt; \[ \{</preformat>
   <italic>not</italic> be a footnote reference, because it contains a
   space.[^my note] Here is an inline
   note.<xref ref-type="fn" rid="fn3">3</xref></p>
-  <p specific-use="wrapper">
-    <disp-quote>
-      <p>Notes can go in quotes.<xref ref-type="fn" rid="fn4">4</xref></p>
-    </disp-quote>
-  </p>
+  <disp-quote>
+    <p>Notes can go in quotes.<xref ref-type="fn" rid="fn4">4</xref></p>
+  </disp-quote>
   <list list-type="order">
     <list-item>
       <p>And in list items.<xref ref-type="fn" rid="fn5">5</xref></p>
-- 
cgit v1.2.3


From ff5a5048091b765ed15750ccb5ea30f9459ba33a Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sun, 4 Apr 2021 11:51:44 -0700
Subject: Use new citeproc + unicode-collation.

Add command test for unicode-collation.
---
 cabal.project                     |  10 +++
 stack.yaml                        |   6 +-
 test/command/unicode-collation.md | 130 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 145 insertions(+), 1 deletion(-)
 create mode 100644 test/command/unicode-collation.md

(limited to 'test/command')

diff --git a/cabal.project b/cabal.project
index a0406da3c..fa17a20a6 100644
--- a/cabal.project
+++ b/cabal.project
@@ -2,6 +2,16 @@ packages: pandoc.cabal
 tests: True
 flags: +embed_data_files
 
+source-repository-package
+  type: git
+  location: https://github.com/jgm/citeproc
+  tag: b42857be658b8f2649e989e061978e304986f853
+
+source-repository-package
+  type: git
+  location: https://github.com/jgm/unicode-collation
+  tag: 9d229a5c6bcbaf53d7022575234eb223cfa90d55
+
 -- source-repository-package
 --  type: git
 --  location: https://github.com/jgm/citeproc
diff --git a/stack.yaml b/stack.yaml
index bf8e76239..159bf74b9 100644
--- a/stack.yaml
+++ b/stack.yaml
@@ -13,10 +13,14 @@ extra-deps:
 - skylighting-core-0.10.5.1
 - skylighting-0.10.5.1
 - doclayout-0.3.0.2
-- citeproc-0.3.0.9
 - texmath-0.12.2
 - random-1.2.0
 - xml-conduit-1.9.1.1
+# - citeproc-0.3.0.9
+- git: https://github.com/jgm/citeproc
+  commit: b42857be658b8f2649e989e061978e304986f853
+- git: https://github.com/jgm/unicode-collation
+  commit: 9d229a5c6bcbaf53d7022575234eb223cfa90d55
 ghc-options:
    "$locals": -fhide-source-paths -Wno-missing-home-modules
 resolver: lts-17.5
diff --git a/test/command/unicode-collation.md b/test/command/unicode-collation.md
new file mode 100644
index 000000000..a53ed7577
--- /dev/null
+++ b/test/command/unicode-collation.md
@@ -0,0 +1,130 @@
+```
+% pandoc --citeproc -t plain
+---
+lang: en-US
+csl: command/apa.csl
+references:
+- id: a1
+  type: book
+  author:
+  - family: Ubina
+    given: A. John
+  issued: 1985
+- id: a2
+  type: book
+  author:
+  - family: Über
+    given: Aglaia
+  issued: 1996
+- id: a3
+  type: book
+  author:
+  - family: Oñate
+    given: José
+  issued: 1985
+- id: a4
+  type: book
+  author:
+  - family: Onush
+    given: Frank
+  issued: 2002
+- id: a5
+  type: book
+  author:
+  - family: O'Neil
+    given: Timothy
+  issued: 2010
+---
+
+[@a1;@a2;@a3;@a4;@a5]
+^D
+(O’Neil, 2010; Oñate, 1985; Onush, 2002; Über, 1996; Ubina, 1985)
+
+O’Neil, T. (2010).
+
+Oñate, J. (1985).
+
+Onush, F. (2002).
+
+Über, A. (1996).
+
+Ubina, A. J. (1985).
+```
+
+```
+% pandoc --citeproc -t plain
+---
+lang: es
+csl: command/apa.csl
+references:
+- id: a1
+  type: book
+  author:
+  - family: Ubina
+    given: A. John
+  issued: 1985
+- id: a2
+  type: book
+  author:
+  - family: Über
+    given: Aglaia
+  issued: 1996
+- id: a3
+  type: book
+  author:
+  - family: Oñate
+    given: José
+  issued: 1985
+- id: a4
+  type: book
+  author:
+  - family: Onush
+    given: Frank
+  issued: 2002
+- id: a5
+  type: book
+  author:
+  - family: O'Neil
+    given: Timothy
+  issued: 2010
+---
+
+[@a1;@a2;@a3;@a4;@a5]
+^D
+(O’Neil, 2010; Onush, 2002; Oñate, 1985; Über, 1996; Ubina, 1985)
+
+O’Neil, T. (2010).
+
+Onush, F. (2002).
+
+Oñate, J. (1985).
+
+Über, A. (1996).
+
+Ubina, A. J. (1985).
+```
+
+```
+% pandoc -C -t plain
+---
+nocite: '@*'
+lang: fr
+references:
+- id: cote
+  author: cote
+- id: côte
+  author: côte
+- id: coté
+  author: coté
+- id: côté
+  author: côté
+...
+^D
+cote. s. d.
+
+côte. s. d.
+
+coté. s. d.
+
+côté. s. d.
+```
-- 
cgit v1.2.3


From 099ac9985b93befb32e01718e5b242d5c7b4f080 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 15 Apr 2021 17:36:18 -0700
Subject: Use BCP47 language codes in citeproc tests.

---
 test/command/pandoc-citeproc-65.md | 2 +-
 test/command/pandoc-citeproc-70.md | 4 ++--
 test/command/unicode-collation.md  | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'test/command')

diff --git a/test/command/pandoc-citeproc-65.md b/test/command/pandoc-citeproc-65.md
index 7b07f7ecf..93335e901 100644
--- a/test/command/pandoc-citeproc-65.md
+++ b/test/command/pandoc-citeproc-65.md
@@ -14,7 +14,7 @@ references:
   id: 'stotz:1996handbuch'
   issued:
     literal: 1996_2004
-  language: German
+  language: de
   number-of-volumes: 5
   publisher: Beck
   publisher-place: Munich
diff --git a/test/command/pandoc-citeproc-70.md b/test/command/pandoc-citeproc-70.md
index fb96add82..e1132b98d 100644
--- a/test/command/pandoc-citeproc-70.md
+++ b/test/command/pandoc-citeproc-70.md
@@ -22,7 +22,7 @@ references:
   issued:
     date-parts:
     - - 2009
-  language: French
+  language: fr
   page: '155--174'
   publisher: 'Fédération Internationale des Instituts d''Études
     Médiévales'
@@ -42,7 +42,7 @@ references:
   issued:
     date-parts:
     - - 1955
-  language: Latin
+  language: la
   note: 'ArticleType: research-article / Full publication date: 1955 /
     Copyright © 1955 Fordham University'
   page: '163--193'
diff --git a/test/command/unicode-collation.md b/test/command/unicode-collation.md
index a53ed7577..190e383d6 100644
--- a/test/command/unicode-collation.md
+++ b/test/command/unicode-collation.md
@@ -108,7 +108,7 @@ Ubina, A. J. (1985).
 % pandoc -C -t plain
 ---
 nocite: '@*'
-lang: fr
+lang: fr-FR-u-kb-true
 references:
 - id: cote
   author: cote
-- 
cgit v1.2.3


From a478a5c4c8753fd0bf272cd540ca197ae146a196 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sat, 17 Apr 2021 11:47:54 -0700
Subject: Update to released unicode-collation, latest citeproc dev version.

Update citeproc test.
---
 cabal.project                           | 11 +----------
 src/Text/Pandoc/App.hs                  |  2 +-
 src/Text/Pandoc/Citeproc/Data.hs        |  2 +-
 src/Text/Pandoc/Class/CommonState.hs    |  2 +-
 src/Text/Pandoc/Class/PandocMonad.hs    |  2 +-
 src/Text/Pandoc/Readers/LaTeX.hs        |  2 +-
 src/Text/Pandoc/Readers/LaTeX/Lang.hs   |  2 +-
 src/Text/Pandoc/Writers/ConTeXt.hs      |  2 +-
 src/Text/Pandoc/Writers/Docbook.hs      |  2 +-
 src/Text/Pandoc/Writers/Docx.hs         |  2 +-
 src/Text/Pandoc/Writers/LaTeX.hs        |  2 +-
 src/Text/Pandoc/Writers/LaTeX/Lang.hs   |  2 +-
 src/Text/Pandoc/Writers/ODT.hs          |  2 +-
 src/Text/Pandoc/Writers/OpenDocument.hs |  2 +-
 stack.yaml                              |  5 ++---
 test/command/pandoc-citeproc-320a.md    |  8 ++++----
 16 files changed, 20 insertions(+), 30 deletions(-)

(limited to 'test/command')

diff --git a/cabal.project b/cabal.project
index fa17a20a6..77bc8ef88 100644
--- a/cabal.project
+++ b/cabal.project
@@ -5,14 +5,5 @@ flags: +embed_data_files
 source-repository-package
   type: git
   location: https://github.com/jgm/citeproc
-  tag: b42857be658b8f2649e989e061978e304986f853
+  tag: f9439e07e9271c7c2674a51efcad2fb8c663b2c8
 
-source-repository-package
-  type: git
-  location: https://github.com/jgm/unicode-collation
-  tag: 9d229a5c6bcbaf53d7022575234eb223cfa90d55
-
--- source-repository-package
---  type: git
---  location: https://github.com/jgm/citeproc
---  tag: d44e24696ab444090d0e63e321c3a573f68b2e74
diff --git a/src/Text/Pandoc/App.hs b/src/Text/Pandoc/App.hs
index 67d3cce7d..4e8c9f2ab 100644
--- a/src/Text/Pandoc/App.hs
+++ b/src/Text/Pandoc/App.hs
@@ -55,7 +55,7 @@ import Text.Pandoc.App.Opt (Opt (..), LineEnding (..), defaultOpts,
 import Text.Pandoc.App.CommandLineOptions (parseOptions, parseOptionsFromArgs,
                                            options)
 import Text.Pandoc.App.OutputSettings (OutputSettings (..), optToOutputSettings)
-import UnicodeCollation.Lang (Lang (..), parseLang)
+import Text.Collate.Lang (Lang (..), parseLang)
 import Text.Pandoc.Filter (Filter (JSONFilter, LuaFilter), applyFilters)
 import Text.Pandoc.PDF (makePDF)
 import Text.Pandoc.SelfContained (makeSelfContained)
diff --git a/src/Text/Pandoc/Citeproc/Data.hs b/src/Text/Pandoc/Citeproc/Data.hs
index 388b9ba62..848a83a1e 100644
--- a/src/Text/Pandoc/Citeproc/Data.hs
+++ b/src/Text/Pandoc/Citeproc/Data.hs
@@ -10,7 +10,7 @@ import qualified Data.Text.Encoding as TE
 import qualified Data.Text as T
 import Data.Text (Text)
 import Text.Pandoc.Citeproc.Util (toIETF)
-import UnicodeCollation.Lang (Lang(..), parseLang)
+import Text.Collate.Lang (Lang(..), parseLang)
 
 biblatexLocalizations :: [(FilePath, ByteString)]
 biblatexLocalizations = $(embedDir "citeproc/biblatex-localization")
diff --git a/src/Text/Pandoc/Class/CommonState.hs b/src/Text/Pandoc/Class/CommonState.hs
index 0fd094d99..796a4afd5 100644
--- a/src/Text/Pandoc/Class/CommonState.hs
+++ b/src/Text/Pandoc/Class/CommonState.hs
@@ -19,7 +19,7 @@ where
 
 import Data.Default (Default (def))
 import Data.Text (Text)
-import UnicodeCollation.Lang (Lang)
+import Text.Collate.Lang (Lang)
 import Text.Pandoc.MediaBag (MediaBag)
 import Text.Pandoc.Logging (LogMessage, Verbosity (WARNING))
 import Text.Pandoc.Translations (Translations)
diff --git a/src/Text/Pandoc/Class/PandocMonad.hs b/src/Text/Pandoc/Class/PandocMonad.hs
index 76f1fa32b..7559cd7cd 100644
--- a/src/Text/Pandoc/Class/PandocMonad.hs
+++ b/src/Text/Pandoc/Class/PandocMonad.hs
@@ -70,7 +70,7 @@ import Network.URI ( escapeURIString, nonStrictRelativeTo,
 import System.FilePath ((</>), (<.>), takeExtension, dropExtension,
                         isRelative, splitDirectories)
 import System.Random (StdGen)
-import UnicodeCollation.Lang (Lang(..), parseLang, renderLang)
+import Text.Collate.Lang (Lang(..), parseLang, renderLang)
 import Text.Pandoc.Class.CommonState (CommonState (..))
 import Text.Pandoc.Definition
 import Text.Pandoc.Error
diff --git a/src/Text/Pandoc/Readers/LaTeX.hs b/src/Text/Pandoc/Readers/LaTeX.hs
index 83caf742a..203dab83c 100644
--- a/src/Text/Pandoc/Readers/LaTeX.hs
+++ b/src/Text/Pandoc/Readers/LaTeX.hs
@@ -33,7 +33,7 @@ import qualified Data.Set as Set
 import Data.Text (Text)
 import qualified Data.Text as T
 import System.FilePath (addExtension, replaceExtension, takeExtension)
-import UnicodeCollation.Lang (renderLang)
+import Text.Collate.Lang (renderLang)
 import Text.Pandoc.Builder as B
 import Text.Pandoc.Class.PandocPure (PandocPure)
 import Text.Pandoc.Class.PandocMonad (PandocMonad (..), getResourcePath,
diff --git a/src/Text/Pandoc/Readers/LaTeX/Lang.hs b/src/Text/Pandoc/Readers/LaTeX/Lang.hs
index b92e6ab57..6a8327904 100644
--- a/src/Text/Pandoc/Readers/LaTeX/Lang.hs
+++ b/src/Text/Pandoc/Readers/LaTeX/Lang.hs
@@ -23,7 +23,7 @@ import qualified Data.Map as M
 import Data.Text (Text)
 import qualified Data.Text as T
 import Text.Pandoc.Shared (extractSpaces)
-import UnicodeCollation.Lang (Lang(..), renderLang)
+import Text.Collate.Lang (Lang(..), renderLang)
 import Text.Pandoc.Class (PandocMonad(..), setTranslations)
 import Text.Pandoc.Readers.LaTeX.Parsing
 import Text.Pandoc.Parsing (updateState, option, getState, QuoteContext(..),
diff --git a/src/Text/Pandoc/Writers/ConTeXt.hs b/src/Text/Pandoc/Writers/ConTeXt.hs
index f352c84bc..f14b1d894 100644
--- a/src/Text/Pandoc/Writers/ConTeXt.hs
+++ b/src/Text/Pandoc/Writers/ConTeXt.hs
@@ -21,7 +21,7 @@ import Data.Maybe (mapMaybe)
 import Data.Text (Text)
 import qualified Data.Text as T
 import Network.URI (unEscapeString)
-import UnicodeCollation.Lang (Lang(..))
+import Text.Collate.Lang (Lang(..))
 import Text.Pandoc.Class.PandocMonad (PandocMonad, report, toLang)
 import Text.Pandoc.Definition
 import Text.Pandoc.ImageSize
diff --git a/src/Text/Pandoc/Writers/Docbook.hs b/src/Text/Pandoc/Writers/Docbook.hs
index 1f10c9d04..02b141250 100644
--- a/src/Text/Pandoc/Writers/Docbook.hs
+++ b/src/Text/Pandoc/Writers/Docbook.hs
@@ -493,4 +493,4 @@ isSectionAttr DocBook4 ("os",_) = True
 isSectionAttr DocBook4 ("revision",_) = True
 isSectionAttr DocBook4 ("security",_) = True
 isSectionAttr DocBook4 ("vendor",_) = True
-isSectionAttr _ (_,_) = False
\ No newline at end of file
+isSectionAttr _ (_,_) = False
diff --git a/src/Text/Pandoc/Writers/Docx.hs b/src/Text/Pandoc/Writers/Docx.hs
index 7781df8e7..749ad9a21 100644
--- a/src/Text/Pandoc/Writers/Docx.hs
+++ b/src/Text/Pandoc/Writers/Docx.hs
@@ -36,7 +36,7 @@ import qualified Data.Text.Lazy as TL
 import Data.Time.Clock.POSIX
 import Data.Digest.Pure.SHA (sha1, showDigest)
 import Skylighting
-import UnicodeCollation.Lang (renderLang)
+import Text.Collate.Lang (renderLang)
 import Text.Pandoc.Class.PandocMonad (PandocMonad, report, toLang)
 import qualified Text.Pandoc.Class.PandocMonad as P
 import Data.Time
diff --git a/src/Text/Pandoc/Writers/LaTeX.hs b/src/Text/Pandoc/Writers/LaTeX.hs
index e99bad738..8b1f3df1d 100644
--- a/src/Text/Pandoc/Writers/LaTeX.hs
+++ b/src/Text/Pandoc/Writers/LaTeX.hs
@@ -29,7 +29,7 @@ import qualified Data.Text as T
 import Network.URI (unEscapeString)
 import Text.DocTemplates (FromContext(lookupContext), renderTemplate,
                           Val(..), Context(..))
-import UnicodeCollation.Lang (Lang (..), renderLang)
+import Text.Collate.Lang (Lang (..), renderLang)
 import Text.Pandoc.Class.PandocMonad (PandocMonad, report, toLang)
 import Text.Pandoc.Definition
 import Text.Pandoc.Highlighting (formatLaTeXBlock, formatLaTeXInline, highlight,
diff --git a/src/Text/Pandoc/Writers/LaTeX/Lang.hs b/src/Text/Pandoc/Writers/LaTeX/Lang.hs
index 437b84120..0ba68b74e 100644
--- a/src/Text/Pandoc/Writers/LaTeX/Lang.hs
+++ b/src/Text/Pandoc/Writers/LaTeX/Lang.hs
@@ -15,7 +15,7 @@ module Text.Pandoc.Writers.LaTeX.Lang
     toBabel
   ) where
 import Data.Text (Text)
-import UnicodeCollation.Lang (Lang(..))
+import Text.Collate.Lang (Lang(..))
 
 
 -- In environments \Arabic instead of \arabic is used
diff --git a/src/Text/Pandoc/Writers/ODT.hs b/src/Text/Pandoc/Writers/ODT.hs
index 6fd4cdeb4..e4eb4fd25 100644
--- a/src/Text/Pandoc/Writers/ODT.hs
+++ b/src/Text/Pandoc/Writers/ODT.hs
@@ -24,7 +24,7 @@ import qualified Data.Text as T
 import qualified Data.Text.Lazy as TL
 import Data.Time
 import System.FilePath (takeDirectory, takeExtension, (<.>))
-import UnicodeCollation.Lang (Lang (..), renderLang)
+import Text.Collate.Lang (Lang (..), renderLang)
 import Text.Pandoc.Class.PandocMonad (PandocMonad, report, toLang)
 import qualified Text.Pandoc.Class.PandocMonad as P
 import Text.Pandoc.Definition
diff --git a/src/Text/Pandoc/Writers/OpenDocument.hs b/src/Text/Pandoc/Writers/OpenDocument.hs
index 6c265090c..34a3a4aa5 100644
--- a/src/Text/Pandoc/Writers/OpenDocument.hs
+++ b/src/Text/Pandoc/Writers/OpenDocument.hs
@@ -25,7 +25,7 @@ import Data.Ord (comparing)
 import qualified Data.Set as Set
 import Data.Text (Text)
 import qualified Data.Text as T
-import UnicodeCollation.Lang (Lang (..), parseLang)
+import Text.Collate.Lang (Lang (..), parseLang)
 import Text.Pandoc.Class.PandocMonad (PandocMonad, report, translateTerm,
                                       setTranslations, toLang)
 import Text.Pandoc.Definition
diff --git a/stack.yaml b/stack.yaml
index 159bf74b9..7bc33fa43 100644
--- a/stack.yaml
+++ b/stack.yaml
@@ -16,11 +16,10 @@ extra-deps:
 - texmath-0.12.2
 - random-1.2.0
 - xml-conduit-1.9.1.1
+- unicode-collation-0.1
 # - citeproc-0.3.0.9
 - git: https://github.com/jgm/citeproc
-  commit: b42857be658b8f2649e989e061978e304986f853
-- git: https://github.com/jgm/unicode-collation
-  commit: 9d229a5c6bcbaf53d7022575234eb223cfa90d55
+  commit: f9439e07e9271c7c2674a51efcad2fb8c663b2c8
 ghc-options:
    "$locals": -fhide-source-paths -Wno-missing-home-modules
 resolver: lts-17.5
diff --git a/test/command/pandoc-citeproc-320a.md b/test/command/pandoc-citeproc-320a.md
index 1c3b47de0..79dacfa10 100644
--- a/test/command/pandoc-citeproc-320a.md
+++ b/test/command/pandoc-citeproc-320a.md
@@ -56,6 +56,10 @@ n.d.; al-'Udhrī, n.d.; Uch, n.d.; Uebel, n.d.; Zzz, n.d.).
 Uch, Ann. n.d.
 :::
 
+::: {#ref-item4 .csl-entry}
+'Udhrī, Jamīl al-. n.d.
+:::
+
 ::: {#ref-item1 .csl-entry}
 ʾUdhrī, Jamīl al-. n.d.
 :::
@@ -68,10 +72,6 @@ Uch, Ann. n.d.
 \'Udhrī, Jamīl al-. n.d.
 :::
 
-::: {#ref-item4 .csl-entry}
-'Udhrī, Jamīl al-. n.d.
-:::
-
 ::: {#ref-item5 .csl-entry}
 'Udhrī, Jamīl al-. n.d.
 :::
-- 
cgit v1.2.3


From 73d394ca2adff31a384404ae25665b36c7d0bba0 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sun, 18 Apr 2021 22:01:12 -0700
Subject: Use MetaInlines not MetaBlocks for multimarkdown metadata fields.

This gives better results in converting to e.g. pandoc markdown.

Ref: <https://groups.google.com/d/msgid/pandoc-discuss/9728d1f4-040e-4392-aa04-148f648a8dfdn%40googlegroups.com>
---
 src/Text/Pandoc/Readers/Markdown.hs |  2 +-
 test/command/mmd-metadata.md        | 20 ++++++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)
 create mode 100644 test/command/mmd-metadata.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/Markdown.hs b/src/Text/Pandoc/Readers/Markdown.hs
index 6c3947a81..4b20e3a8b 100644
--- a/src/Text/Pandoc/Readers/Markdown.hs
+++ b/src/Text/Pandoc/Readers/Markdown.hs
@@ -298,7 +298,7 @@ kvPair allowEmpty = try $ do
           (try $ newline >> lookAhead (blankline <|> nonspaceChar))
   guard $ allowEmpty || not (T.null val)
   let key' = T.concat $ T.words $ T.toLower key
-  let val' = MetaBlocks $ B.toList $ B.plain $ B.text val
+  let val' = MetaInlines $ B.toList $ B.text val
   return (key',val')
 
 parseMarkdown :: PandocMonad m => MarkdownParser m Pandoc
diff --git a/test/command/mmd-metadata.md b/test/command/mmd-metadata.md
new file mode 100644
index 000000000..3cda34873
--- /dev/null
+++ b/test/command/mmd-metadata.md
@@ -0,0 +1,20 @@
+```
+% pandoc -f markdown_mmd -t markdown -s
+Title: Blah blah blah
+Author: Doo de Doo
+Base Header Level: 1
+Bibliography: Pubs.bib
+Lang: en-GB
+
+body
+^D
+---
+author: Doo de Doo
+baseheaderlevel: 1
+bibliography: Pubs.bib
+lang: en-GB
+title: Blah blah blah
+---
+
+body
+```
-- 
cgit v1.2.3


From 7f4850c9de086a2f6df072bf15357cd05335cffd Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sun, 25 Apr 2021 10:29:03 -0700
Subject: Remove biblatex-nussbaum.md test.

It is basically the same as biblaetx-quotes.md.
---
 test/command/biblatex-nussbaum.md | 63 ---------------------------------------
 1 file changed, 63 deletions(-)
 delete mode 100644 test/command/biblatex-nussbaum.md

(limited to 'test/command')

diff --git a/test/command/biblatex-nussbaum.md b/test/command/biblatex-nussbaum.md
deleted file mode 100644
index 5be575a32..000000000
--- a/test/command/biblatex-nussbaum.md
+++ /dev/null
@@ -1,63 +0,0 @@
-```
-% pandoc -f biblatex -t markdown -s
-@comment{
-
-Adapted from biblatex-example.bib
-
-
-Formatted with pandoc and chicago-author-date.csl, 2013-10-23:
-
-(Nussbaum 1978)
-
-Nussbaum, Martha. 1978. *Aristotle’s “De Motu Animalium”*. Princeton:
-Princeton University Press.
-
-
-Formatted with pandoc and apa.csl, 2013-10-23:
-
-(Nussbaum, 1978)
-
-Nussbaum, M. (1978). *Aristotle’s “De Motu Animalium”*. Princeton:
-Princeton University Press.
-
-
-}
-
-@string{ pup     = {Princeton University Press} }
-
-@Book{nussbaum,
-  author       = {Nussbaum, Martha},
-  title        = {Aristotle's \mkbibquote{De Motu Animalium}},
-  date         = 1978,
-  publisher    = pup,
-  location     = {Princeton},
-  keywords     = {secondary},
-  hyphenation  = {american},
-  sorttitle    = {Aristotle's De Motu Animalium},
-  indexsorttitle= {Aristotle's De Motu Animalium},
-  annotation   = {A book entry. Note the sorttitle and
-                  indexsorttitle fields and the markup of the quotes in
-                  the database file},
-}
-
-^D
----
-nocite: "[@*]"
-references:
-- annote: A book entry. Note the sorttitle and indexsorttitle fields and
-    the markup of the quotes in the database file
-  author:
-  - family: Nussbaum
-    given: Martha
-  id: nussbaum
-  issued: 1978
-  keyword: secondary
-  language: en-US
-  publisher: Princeton University Press
-  publisher-place: Princeton
-  title: "Aristotle's \"De Motu Animalium\""
-  type: book
----
-
-
-```
-- 
cgit v1.2.3


From 547bc2cdf83b8be926de55521674c0e8fab12db5 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sun, 25 Apr 2021 10:31:33 -0700
Subject: Add quotes properly in markdown YAML metadata fields.

This fixes a bug, which caused the writer to look at the LAST
rather than the FIRST character in determining whether quotes
were needed.  So we got spurious quotes in some cases and
didn't get necessary quotes in others.

Closes #7245.  Updated a number of test cases accordingly.
---
 src/Text/Pandoc/Writers/Markdown.hs           | 11 +++++------
 test/command/1279.md                          |  2 +-
 test/command/biblatex-inproceedings.md        |  2 +-
 test/command/biblatex-jaffe.md                |  4 ++--
 test/command/biblatex-moraux.md               |  2 +-
 test/command/biblatex-quotes.md               |  2 +-
 test/command/biblatex-spiegelberg.md          |  4 ++--
 test/command/biblatex-test-case-conversion.md |  4 ++--
 test/command/biblatex-textnormal.md           |  2 +-
 9 files changed, 16 insertions(+), 17 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/Markdown.hs b/src/Text/Pandoc/Writers/Markdown.hs
index daf45ed53..3295d9e6c 100644
--- a/src/Text/Pandoc/Writers/Markdown.hs
+++ b/src/Text/Pandoc/Writers/Markdown.hs
@@ -26,7 +26,7 @@ import Data.Default
 import Data.List (intersperse, sortOn, transpose)
 import Data.List.NonEmpty (nonEmpty, NonEmpty(..))
 import qualified Data.Map as M
-import Data.Maybe (fromMaybe, mapMaybe)
+import Data.Maybe (fromMaybe, mapMaybe, isNothing)
 import qualified Data.Set as Set
 import Data.Text (Text)
 import qualified Data.Text as T
@@ -142,18 +142,17 @@ valToYaml (SimpleVal x)
   | otherwise =
       if hasNewlines x
          then hang 0 ("|" <> cr) x
-         else if fst $ foldr needsDoubleQuotes (False, True) x
+         else if isNothing $ foldM needsDoubleQuotes True x
            then "\"" <> fmap escapeInDoubleQuotes x <> "\""
            else x
     where
-      needsDoubleQuotes t (positive, isFirst)
+      needsDoubleQuotes isFirst t
         = if T.any isBadAnywhere t ||
              (isFirst && T.any isYamlPunct (T.take 1 t))
-              then (True, False)
-              else (positive, False)
+              then Nothing
+              else Just False
       isBadAnywhere '#' = True
       isBadAnywhere ':' = True
-      isBadAnywhere '`' = False
       isBadAnywhere _   = False
       hasNewlines NewLine = True
       hasNewlines BlankLines{} = True
diff --git a/test/command/1279.md b/test/command/1279.md
index acad6859e..d3572f245 100644
--- a/test/command/1279.md
+++ b/test/command/1279.md
@@ -9,7 +9,7 @@ title: My Article
 [^1]: Dept. of This and That
 ^D
 ---
-author: "John Doe[^1]"
+author: John Doe[^1]
 date: 2014
 title: My Article
 ---
diff --git a/test/command/biblatex-inproceedings.md b/test/command/biblatex-inproceedings.md
index 29a1e0298..76b584800 100644
--- a/test/command/biblatex-inproceedings.md
+++ b/test/command/biblatex-inproceedings.md
@@ -63,7 +63,7 @@ references:
   publisher-place: Cambridge
   title: "Le *De Anima* dans la tradition grècque: Quelques aspects de
     l'interpretation du traité, de Theophraste à Themistius"
-  title-short: *De Anima* dans la tradition grècque
+  title-short: "*De Anima* dans la tradition grècque"
   type: paper-conference
 - author:
   - family: Salam
diff --git a/test/command/biblatex-jaffe.md b/test/command/biblatex-jaffe.md
index f172c6ae7..e0381772b 100644
--- a/test/command/biblatex-jaffe.md
+++ b/test/command/biblatex-jaffe.md
@@ -72,8 +72,8 @@ references:
   language: la
   number-of-volumes: 2
   publisher-place: Leipzig
-  title: "Regesta Pontificum Romanorum ab condita ecclesia ad annum post
-    Christum natum [mcxcviii]{.smallcaps}"
+  title: Regesta Pontificum Romanorum ab condita ecclesia ad annum post
+    Christum natum [mcxcviii]{.smallcaps}
   title-short: Regesta Pontificum Romanorum
   type: book
 ---
diff --git a/test/command/biblatex-moraux.md b/test/command/biblatex-moraux.md
index 8a1b084f8..1624ad960 100644
--- a/test/command/biblatex-moraux.md
+++ b/test/command/biblatex-moraux.md
@@ -86,7 +86,7 @@ references:
   publisher-place: Cambridge
   title: "Le *De Anima* dans la tradition grècque: Quelques aspects de
     l'interpretation du traité, de Theophraste à Themistius"
-  title-short: *De Anima* dans la tradition grècque
+  title-short: "*De Anima* dans la tradition grècque"
   type: paper-conference
 ---
 
diff --git a/test/command/biblatex-quotes.md b/test/command/biblatex-quotes.md
index 9d868cb76..50055316f 100644
--- a/test/command/biblatex-quotes.md
+++ b/test/command/biblatex-quotes.md
@@ -36,7 +36,7 @@ references:
   language: en-US
   publisher: Princeton University Press
   publisher-place: Princeton
-  title: "Aristotle's \"De Motu Animalium\""
+  title: Aristotle's "De Motu Animalium"
   type: book
 ---
 
diff --git a/test/command/biblatex-spiegelberg.md b/test/command/biblatex-spiegelberg.md
index 59d651958..59338ae88 100644
--- a/test/command/biblatex-spiegelberg.md
+++ b/test/command/biblatex-spiegelberg.md
@@ -63,8 +63,8 @@ references:
   issued: 1969
   language: de-DE
   page: 189-216
-  title: "Intention" und "Intentionalität" in der Scholastik, bei
-    Brentano und Husserl
+  title: "\"Intention\" und \"Intentionalität\" in der Scholastik, bei
+    Brentano und Husserl"
   title-short: Intention und Intentionalität
   type: article-journal
   volume: 29
diff --git a/test/command/biblatex-test-case-conversion.md b/test/command/biblatex-test-case-conversion.md
index aa30767ec..e69cb9336 100644
--- a/test/command/biblatex-test-case-conversion.md
+++ b/test/command/biblatex-test-case-conversion.md
@@ -63,9 +63,9 @@ references:
   id: item1
   issued: 2013
   language: en-US
-  title: "A title, in English, with a Proper Name and an ACRONYM and a
+  title: A title, in English, with a Proper Name and an ACRONYM and a
     [camelCase]{.nocase} word and some units, 400 [nm]{.nocase},
-    3 [cm]{.nocase}, and a quote, *Alea [iacta est]{.nocase}*"
+    3 [cm]{.nocase}, and a quote, *Alea [iacta est]{.nocase}*
   type: article-journal
 ---
 
diff --git a/test/command/biblatex-textnormal.md b/test/command/biblatex-textnormal.md
index 0aa5550c3..a4e6107ec 100644
--- a/test/command/biblatex-textnormal.md
+++ b/test/command/biblatex-textnormal.md
@@ -9,7 +9,7 @@
 nocite: "[@*]"
 references:
 - id: item1
-  title: "The title [of this book]{.nodecor}"
+  title: The title [of this book]{.nodecor}
   type: book
 ---
 
-- 
cgit v1.2.3


From e9c0f9f97ba6459530c7bb2ffb55d432a1ba7884 Mon Sep 17 00:00:00 2001
From: Jan Tojnar <jtojnar@gmail.com>
Date: Sun, 25 Apr 2021 19:36:06 +0200
Subject: Markdown writer: Cleaner (code)blocks with single class (#7242)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When a block only has a single class and no other attributes,
it is not necessary to wrap the class attribute in curly braces –
the class name can be placed after the opening mark as is.

This will result in bit cleaner output when pandoc is used
as a markdown pretty-printer.
---
 src/Text/Pandoc/Writers/Markdown.hs | 10 ++++++++--
 test/command/5304.md                |  2 +-
 test/command/5519.md                |  2 +-
 test/command/6925.md                |  4 ++--
 test/command/toc.md                 |  4 ++--
 5 files changed, 14 insertions(+), 8 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/Markdown.hs b/src/Text/Pandoc/Writers/Markdown.hs
index 3295d9e6c..2ad9eabd9 100644
--- a/src/Text/Pandoc/Writers/Markdown.hs
+++ b/src/Text/Pandoc/Writers/Markdown.hs
@@ -278,6 +278,12 @@ attrsToMarkdown attribs = braces $ hsep [attribId, attribClasses, attribKeys]
               escAttrChar '\\' = literal "\\\\"
               escAttrChar c    = literal $ T.singleton c
 
+-- | (Code) blocks with a single class can just use it standalone,
+-- no need to bother with curly braces.
+classOrAttrsToMarkdown :: Attr -> Doc Text
+classOrAttrsToMarkdown ("",[cls],_) = literal cls
+classOrAttrsToMarkdown attrs = attrsToMarkdown attrs
+
 linkAttributes :: WriterOptions -> Attr -> Doc Text
 linkAttributes opts attr =
   if isEnabled Ext_link_attributes opts && attr /= nullAttr
@@ -343,7 +349,7 @@ blockToMarkdown' opts (Div attrs ils) = do
     case () of
          _ | isEnabled Ext_fenced_divs opts &&
              attrs /= nullAttr ->
-                nowrap (literal ":::" <+> attrsToMarkdown attrs) $$
+                nowrap (literal ":::" <+> classOrAttrsToMarkdown attrs) $$
                 chomp contents $$
                 literal ":::" <> blankline
            | isEnabled Ext_native_divs opts ||
@@ -512,7 +518,7 @@ blockToMarkdown' opts (CodeBlock attribs str) = do
      backticks = endline '`'
      tildes = endline '~'
      attrs  = if isEnabled Ext_fenced_code_attributes opts
-                 then nowrap $ " " <> attrsToMarkdown attribs
+                 then nowrap $ " " <> classOrAttrsToMarkdown attribs
                  else case attribs of
                             (_,cls:_,_) -> " " <> literal cls
                             _             -> empty
diff --git a/test/command/5304.md b/test/command/5304.md
index 62b2b9ddd..70f32a96a 100644
--- a/test/command/5304.md
+++ b/test/command/5304.md
@@ -11,7 +11,7 @@
 ... 
 ```
 ^D
-``` {.markdown}
+``` markdown
 `«sträng»`
 
 `` «sträng» ``
diff --git a/test/command/5519.md b/test/command/5519.md
index a175ce9f9..ecde184f1 100644
--- a/test/command/5519.md
+++ b/test/command/5519.md
@@ -6,7 +6,7 @@
  ```
 ``````
 ^D
-```` {.attr}
+```` attr
  ```
  code
  ```
diff --git a/test/command/6925.md b/test/command/6925.md
index 458a0b91d..e0d8e6870 100644
--- a/test/command/6925.md
+++ b/test/command/6925.md
@@ -20,13 +20,13 @@ a
 \end{thm2}
 \end{document}
 ^D
-::: {.thm}
+::: thm
 **Theorem 1**. *a*
 
 ![image](1.png)
 :::
 
-::: {.thm2}
+::: thm2
 **Theorem 1**. a
 
 ![image](1.png)
diff --git a/test/command/toc.md b/test/command/toc.md
index 794af7690..543f97ba9 100644
--- a/test/command/toc.md
+++ b/test/command/toc.md
@@ -31,7 +31,7 @@
 
 ## b
 
-::: {.interior}
+::: interior
 # C
 
 ## cc
@@ -39,7 +39,7 @@
 # D
 :::
 
-::: {.blue}
+::: blue
 # E
 
 ## e
-- 
cgit v1.2.3


From 80e2e88287f43d88ea92a77779b25e161c81f67b Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Wed, 28 Apr 2021 23:30:16 -0700
Subject: Smarter smart quotes.

Treat a leading " with no closing " as a left curly quote.
This supports the practice, in fiction, of continuing
paragraphs quoting the same speaker without an end quote.
It also helps with quotes that break over lines in line
blocks.

Closes #7216.
---
 src/Text/Pandoc/Parsing.hs           | 45 +++++++++++++++++++++++-------------
 src/Text/Pandoc/Readers/Markdown.hs  | 18 ++++++++-------
 src/Text/Pandoc/Readers/RST.hs       | 19 +--------------
 src/Text/Pandoc/Readers/TWiki.hs     | 22 +-----------------
 test/Tests/Readers/Markdown.hs       |  2 +-
 test/command/7216.md                 | 19 +++++++++++++++
 test/command/pandoc-citeproc-320a.md | 12 +++++-----
 7 files changed, 67 insertions(+), 70 deletions(-)
 create mode 100644 test/command/7216.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Parsing.hs b/src/Text/Pandoc/Parsing.hs
index 847fd2e05..2f6189104 100644
--- a/src/Text/Pandoc/Parsing.hs
+++ b/src/Text/Pandoc/Parsing.hs
@@ -105,8 +105,9 @@ module Text.Pandoc.Parsing ( take1WhileP,
                              singleQuoteEnd,
                              doubleQuoteStart,
                              doubleQuoteEnd,
-                             ellipses,
                              apostrophe,
+                             doubleCloseQuote,
+                             ellipses,
                              dash,
                              nested,
                              citeKey,
@@ -1398,10 +1399,7 @@ smartPunctuation :: (HasReaderOptions st, HasLastStrPosition st, HasQuoteContext
                  -> ParserT s st m Inlines
 smartPunctuation inlineParser = do
   guardEnabled Ext_smart
-  choice [ quoted inlineParser, apostrophe, dash, ellipses ]
-
-apostrophe :: Stream s m Char => ParserT s st m Inlines
-apostrophe = (char '\'' <|> char '\8217') >> return (B.str "\x2019")
+  choice [ quoted inlineParser, apostrophe, doubleCloseQuote, dash, ellipses ]
 
 quoted :: (HasLastStrPosition st, HasQuoteContext st m, Stream s m Char)
        => ParserT s st m Inlines
@@ -1411,16 +1409,22 @@ quoted inlineParser = doubleQuoted inlineParser <|> singleQuoted inlineParser
 singleQuoted :: (HasLastStrPosition st, HasQuoteContext st m, Stream s m Char)
              => ParserT s st m Inlines
              -> ParserT s st m Inlines
-singleQuoted inlineParser = try $ B.singleQuoted . mconcat
-  <$  singleQuoteStart
-  <*> withQuoteContext InSingleQuote (many1Till inlineParser singleQuoteEnd)
-
-doubleQuoted :: (HasQuoteContext st m, Stream s m Char)
+singleQuoted inlineParser = do
+  singleQuoteStart
+  (B.singleQuoted . mconcat <$>
+    try
+     (withQuoteContext InSingleQuote (many1Till inlineParser singleQuoteEnd)))
+   <|> pure "\8217"
+
+doubleQuoted :: (HasQuoteContext st m, HasLastStrPosition st, Stream s m Char)
              => ParserT s st m Inlines
              -> ParserT s st m Inlines
-doubleQuoted inlineParser = try $ B.doubleQuoted . mconcat
-  <$  doubleQuoteStart
-  <*> withQuoteContext InDoubleQuote (manyTill inlineParser doubleQuoteEnd)
+doubleQuoted inlineParser = do
+  doubleQuoteStart
+  (B.doubleQuoted . mconcat <$>
+    try
+     (withQuoteContext InDoubleQuote (manyTill inlineParser doubleQuoteEnd)))
+   <|> pure (B.str "\8220")
 
 failIfInQuoteContext :: (HasQuoteContext st m, Stream s m t)
                      => QuoteContext
@@ -1443,7 +1447,7 @@ singleQuoteStart = do
   guard =<< notAfterString
   try $ do
     charOrRef "'\8216\145"
-    notFollowedBy (oneOf [' ', '\t', '\n'])
+    notFollowedBy (satisfy isSpaceChar)
 
 singleQuoteEnd :: Stream s m Char
                => ParserT s st m ()
@@ -1451,17 +1455,26 @@ singleQuoteEnd = try $ do
   charOrRef "'\8217\146"
   notFollowedBy alphaNum
 
-doubleQuoteStart :: (HasQuoteContext st m, Stream s m Char)
+doubleQuoteStart :: (HasLastStrPosition st,
+                     HasQuoteContext st m,
+                     Stream s m Char)
                  => ParserT s st m ()
 doubleQuoteStart = do
   failIfInQuoteContext InDoubleQuote
+  guard =<< notAfterString
   try $ do charOrRef "\"\8220\147"
-           notFollowedBy (oneOf [' ', '\t', '\n'])
+           notFollowedBy (satisfy isSpaceChar)
 
 doubleQuoteEnd :: Stream s m Char
                => ParserT s st m ()
 doubleQuoteEnd = void (charOrRef "\"\8221\148")
 
+apostrophe :: Stream s m Char => ParserT s st m Inlines
+apostrophe = (char '\'' <|> char '\8217') >> return (B.str "\8217")
+
+doubleCloseQuote :: Stream s m Char => ParserT s st m Inlines
+doubleCloseQuote = B.str "\8221" <$ char '"'
+
 ellipses :: Stream s m Char
          => ParserT s st m Inlines
 ellipses = try (string "..." >> return (B.str "\8230"))
diff --git a/src/Text/Pandoc/Readers/Markdown.hs b/src/Text/Pandoc/Readers/Markdown.hs
index 4b20e3a8b..ba8ed147e 100644
--- a/src/Text/Pandoc/Readers/Markdown.hs
+++ b/src/Text/Pandoc/Readers/Markdown.hs
@@ -2199,25 +2199,27 @@ citation = try $ do
 smart :: PandocMonad m => MarkdownParser m (F Inlines)
 smart = do
   guardEnabled Ext_smart
-  doubleQuoted <|> singleQuoted <|>
-    choice (map (return <$>) [apostrophe, dash, ellipses])
+  doubleQuoted <|> singleQuoted <|> (return <$> doubleCloseQuote) <|>
+    (return <$> apostrophe) <|> (return <$> dash) <|> (return <$> ellipses)
 
 singleQuoted :: PandocMonad m => MarkdownParser m (F Inlines)
-singleQuoted = try $ do
+singleQuoted = do
   singleQuoteStart
-  withQuoteContext InSingleQuote $
+  (try (withQuoteContext InSingleQuote $
     fmap B.singleQuoted . trimInlinesF . mconcat <$>
-      many1Till inline singleQuoteEnd
+      many1Till inline singleQuoteEnd))
+    <|> (return (return (B.str "\8217")))
 
 -- doubleQuoted will handle regular double-quoted sections, as well
 -- as dialogues with an open double-quote without a close double-quote
 -- in the same paragraph.
 doubleQuoted :: PandocMonad m => MarkdownParser m (F Inlines)
-doubleQuoted = try $ do
+doubleQuoted = do
   doubleQuoteStart
-  withQuoteContext InDoubleQuote $
+  (try (withQuoteContext InDoubleQuote $
     fmap B.doubleQuoted . trimInlinesF . mconcat <$>
-      many1Till inline doubleQuoteEnd
+      many1Till inline doubleQuoteEnd))
+    <|> (return (return (B.str "\8220")))
 
 toRow :: [Blocks] -> Row
 toRow = Row nullAttr . map B.simpleCell
diff --git a/src/Text/Pandoc/Readers/RST.hs b/src/Text/Pandoc/Readers/RST.hs
index 514e3b88d..ac4c0b6cb 100644
--- a/src/Text/Pandoc/Readers/RST.hs
+++ b/src/Text/Pandoc/Readers/RST.hs
@@ -1658,21 +1658,4 @@ note = try $ do
       return $ B.note contents
 
 smart :: PandocMonad m => RSTParser m Inlines
-smart = do
-  guardEnabled Ext_smart
-  doubleQuoted <|> singleQuoted <|>
-    choice [apostrophe, dash, ellipses]
-
-singleQuoted :: PandocMonad m => RSTParser m Inlines
-singleQuoted = try $ do
-  singleQuoteStart
-  withQuoteContext InSingleQuote $
-    B.singleQuoted . trimInlines . mconcat <$>
-      many1Till inline singleQuoteEnd
-
-doubleQuoted :: PandocMonad m => RSTParser m Inlines
-doubleQuoted = try $ do
-  doubleQuoteStart
-  withQuoteContext InDoubleQuote $
-    B.doubleQuoted . trimInlines . mconcat <$>
-      many1Till inline doubleQuoteEnd
+smart = smartPunctuation inline
diff --git a/src/Text/Pandoc/Readers/TWiki.hs b/src/Text/Pandoc/Readers/TWiki.hs
index 484a6c923..c4d7bcc93 100644
--- a/src/Text/Pandoc/Readers/TWiki.hs
+++ b/src/Text/Pandoc/Readers/TWiki.hs
@@ -469,27 +469,7 @@ symbol :: PandocMonad m => TWParser m B.Inlines
 symbol = B.str <$> countChar 1 nonspaceChar
 
 smart :: PandocMonad m => TWParser m B.Inlines
-smart = do
-  guardEnabled Ext_smart
-  doubleQuoted <|> singleQuoted <|>
-    choice [ apostrophe
-           , dash
-           , ellipses
-           ]
-
-singleQuoted :: PandocMonad m => TWParser m B.Inlines
-singleQuoted = try $ do
-  singleQuoteStart
-  withQuoteContext InSingleQuote
-    (B.singleQuoted . B.trimInlines . mconcat <$> many1Till inline singleQuoteEnd)
-
-doubleQuoted :: PandocMonad m => TWParser m B.Inlines
-doubleQuoted = try $ do
-  doubleQuoteStart
-  contents <- mconcat <$> many (try $ notFollowedBy doubleQuoteEnd >> inline)
-  withQuoteContext InDoubleQuote (doubleQuoteEnd >>
-   return (B.doubleQuoted $ B.trimInlines contents))
-   <|> return (B.str "\8220" B.<> contents)
+smart = smartPunctuation inline
 
 link :: PandocMonad m => TWParser m B.Inlines
 link = try $ do
diff --git a/test/Tests/Readers/Markdown.hs b/test/Tests/Readers/Markdown.hs
index 0930deae6..6e38da21a 100644
--- a/test/Tests/Readers/Markdown.hs
+++ b/test/Tests/Readers/Markdown.hs
@@ -358,7 +358,7 @@ tests = [ testGroup "inline code"
               para (text "The value of the " <> math "x" <> text "\8217s and the systems\8217 condition.")
           , test markdownSmart "unclosed double quote"
             ("**this should \"be bold**"
-            =?> para (strong "this should \"be bold"))
+            =?> para (strong "this should \8220be bold"))
           ]
         , testGroup "footnotes"
           [ "indent followed by newline and flush-left text" =:
diff --git a/test/command/7216.md b/test/command/7216.md
new file mode 100644
index 000000000..cab3b9689
--- /dev/null
+++ b/test/command/7216.md
@@ -0,0 +1,19 @@
+```
+pandoc -t latex
+"This is some text in quotes. Another paragraph by the same speaker follows. The first paragraph should have no close quote.
+
+"The second paragraph should have open and close quotes."
+
+| "Open quote on this line,
+| Close quote on the next line."
+| "Quotes on the same line."
+^D
+``This is some text in quotes. Another paragraph by the same speaker
+follows. The first paragraph should have no close quote.
+
+``The second paragraph should have open and close quotes.''
+
+``Open quote on this line,\\
+Close quote on the next line.''\\
+``Quotes on the same line.''
+```
diff --git a/test/command/pandoc-citeproc-320a.md b/test/command/pandoc-citeproc-320a.md
index 79dacfa10..e894a2250 100644
--- a/test/command/pandoc-citeproc-320a.md
+++ b/test/command/pandoc-citeproc-320a.md
@@ -49,17 +49,13 @@ references:
 Foo [@item1; @item2; @item3; @item4; @item5; @item6; @item7; @item8].
 ^D
 Foo (al-ʾUdhrī, n.d.; al-ʿUdhrī, n.d.; al-\'Udhrī, n.d.; al-'Udhrī,
-n.d.; al-'Udhrī, n.d.; Uch, n.d.; Uebel, n.d.; Zzz, n.d.).
+n.d.a, n.d.b; Uch, n.d.; Uebel, n.d.; Zzz, n.d.).
 
 ::: {#refs .references .csl-bib-body .hanging-indent}
 ::: {#ref-item6 .csl-entry}
 Uch, Ann. n.d.
 :::
 
-::: {#ref-item4 .csl-entry}
-'Udhrī, Jamīl al-. n.d.
-:::
-
 ::: {#ref-item1 .csl-entry}
 ʾUdhrī, Jamīl al-. n.d.
 :::
@@ -72,8 +68,12 @@ Uch, Ann. n.d.
 \'Udhrī, Jamīl al-. n.d.
 :::
 
+::: {#ref-item4 .csl-entry}
+'Udhrī, Jamīl al-. n.d.a.
+:::
+
 ::: {#ref-item5 .csl-entry}
-'Udhrī, Jamīl al-. n.d.
+---------. n.d.b.
 :::
 
 ::: {#ref-item7 .csl-entry}
-- 
cgit v1.2.3


From d14c5f94df490996d12f8a56e29f927b9f9549e8 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 29 Apr 2021 08:48:49 -0700
Subject: Further improvements in smart quotes.

Improves heuristic for detection of an "open double quote."
Closes #2103.
---
 src/Text/Pandoc/Parsing.hs | 4 ++--
 test/command/2103.md       | 8 ++++++++
 2 files changed, 10 insertions(+), 2 deletions(-)
 create mode 100644 test/command/2103.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Parsing.hs b/src/Text/Pandoc/Parsing.hs
index 2f6189104..37ab0adaa 100644
--- a/src/Text/Pandoc/Parsing.hs
+++ b/src/Text/Pandoc/Parsing.hs
@@ -1447,7 +1447,7 @@ singleQuoteStart = do
   guard =<< notAfterString
   try $ do
     charOrRef "'\8216\145"
-    notFollowedBy (satisfy isSpaceChar)
+    void $ lookAhead (satisfy (not . isSpaceChar))
 
 singleQuoteEnd :: Stream s m Char
                => ParserT s st m ()
@@ -1463,7 +1463,7 @@ doubleQuoteStart = do
   failIfInQuoteContext InDoubleQuote
   guard =<< notAfterString
   try $ do charOrRef "\"\8220\147"
-           notFollowedBy (satisfy isSpaceChar)
+           void $ lookAhead (satisfy (not . isSpaceChar))
 
 doubleQuoteEnd :: Stream s m Char
                => ParserT s st m ()
diff --git a/test/command/2103.md b/test/command/2103.md
new file mode 100644
index 000000000..14a522a5e
--- /dev/null
+++ b/test/command/2103.md
@@ -0,0 +1,8 @@
+```
+pandoc -t latex
+| A happy pandoc user said "fix this bug please
+| or I'll go crazy!"
+^D
+A happy pandoc user said ``fix this bug please\\
+or I'll go crazy!''
+```
-- 
cgit v1.2.3


From 8357b835d9c6d17f32bded56aa24059c2f6e0678 Mon Sep 17 00:00:00 2001
From: Albert Krewinkel <albert@zeitkraut.de>
Date: Wed, 5 May 2021 19:00:53 +0200
Subject: App: allow tabs expansion even if file-scope is used

Tabs in plain-text inputs are now handled correctly, even if the
`--file-scope` flag is used.

Closes: #6709
---
 src/Text/Pandoc/App.hs | 18 +++++++++++-------
 test/command/6709.md   | 11 +++++++++++
 2 files changed, 22 insertions(+), 7 deletions(-)
 create mode 100644 test/command/6709.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/App.hs b/src/Text/Pandoc/App.hs
index 4e8c9f2ab..96e4b5f47 100644
--- a/src/Text/Pandoc/App.hs
+++ b/src/Text/Pandoc/App.hs
@@ -255,13 +255,17 @@ convertWithOpts opts = do
     let sourceToDoc :: [FilePath] -> PandocIO Pandoc
         sourceToDoc sources' =
            case reader of
-                TextReader r
-                  | optFileScope opts || readerNameBase == "json" ->
-                      mconcat <$> mapM (readSource >=> r readerOpts) sources'
-                  | otherwise ->
-                      readSources sources' >>= r readerOpts
-                ByteStringReader r ->
-                  mconcat <$> mapM (readFile' >=> r readerOpts) sources'
+             TextReader r
+               | readerNameBase == "json" ->
+                   mconcat <$> mapM (readSource >=> r readerOpts) sources'
+               | optFileScope opts  ->
+                   -- Read source and convert tabs (see #6709)
+                   let readSource' = fmap convertTabs . readSource
+                   in mconcat <$> mapM (readSource' >=> r readerOpts) sources'
+               | otherwise ->
+                   readSources sources' >>= r readerOpts
+             ByteStringReader r ->
+               mconcat <$> mapM (readFile' >=> r readerOpts) sources'
 
 
     when (readerNameBase == "markdown_github" ||
diff --git a/test/command/6709.md b/test/command/6709.md
new file mode 100644
index 000000000..d6d514552
--- /dev/null
+++ b/test/command/6709.md
@@ -0,0 +1,11 @@
+Tabs must be expanded even if --file-scope is used
+````
+% pandoc -t native --file-scope --tab-stop=2
+```
+if true; then
+	echo "yup"
+fi
+```
+^D
+[CodeBlock ("",[],[]) "if true; then\n  echo \"yup\"\nfi"]
+````
-- 
cgit v1.2.3


From 2bd5d0cafbe67fa723f49fd2d35d913594fdb935 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Tue, 11 May 2021 07:52:05 -0600
Subject: LaTeX writer: better handling of line breaks in simple tables.

Now we also handle the case where they're embedded in other
elements, e.g. spans. Closes #7272.
---
 src/Text/Pandoc/Writers/LaTeX/Table.hs |  4 +---
 test/command/7272.md                   | 24 ++++++++++++++++++++++++
 2 files changed, 25 insertions(+), 3 deletions(-)
 create mode 100644 test/command/7272.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/LaTeX/Table.hs b/src/Text/Pandoc/Writers/LaTeX/Table.hs
index 6f8386937..16f63314b 100644
--- a/src/Text/Pandoc/Writers/LaTeX/Table.hs
+++ b/src/Text/Pandoc/Writers/LaTeX/Table.hs
@@ -219,9 +219,7 @@ footRows (Ann.TableFoot _attr rows) = map headerRowCells rows
 -- we need to go to some lengths to get line breaks working:
 -- as LineBreak bs = \vtop{\hbox{\strut as}\hbox{\strut bs}}.
 fixLineBreaks :: Block -> Block
-fixLineBreaks (Para ils)  = Para $ fixLineBreaks' ils
-fixLineBreaks (Plain ils) = Plain $ fixLineBreaks' ils
-fixLineBreaks x           = x
+fixLineBreaks = walk fixLineBreaks'
 
 fixLineBreaks' :: [Inline] -> [Inline]
 fixLineBreaks' ils = case splitBy (== LineBreak) ils of
diff --git a/test/command/7272.md b/test/command/7272.md
new file mode 100644
index 000000000..d3a3b2137
--- /dev/null
+++ b/test/command/7272.md
@@ -0,0 +1,24 @@
+```
+% pandoc -t latex -f html
+<table>
+  <tbody>
+    <tr>
+      <td>
+        <span>
+          text
+          <br />
+          text2
+        </span>
+      </td>
+    </tr>
+  </tbody>
+</table>
+^D
+\begin{longtable}[]{@{}
+  >{\raggedright\arraybackslash}p{(\columnwidth - 0\tabcolsep) * \real{1.00}}@{}}
+\toprule
+\endhead
+{\vtop{\hbox{\strut  text}\hbox{\strut text2 }}} \\
+\bottomrule
+\end{longtable}
+```
-- 
cgit v1.2.3


From 0217ae2a4ffe33bee7cc53ba44817cd540b5f01e Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Wed, 12 May 2021 11:05:55 -0700
Subject: Hande 'annote' field in bibtex/biblatex writer.

Closes #7266.
---
 src/Text/Pandoc/Citeproc/BibTeX.hs |  2 ++
 test/command/7266.md               | 10 ++++++++++
 2 files changed, 12 insertions(+)
 create mode 100644 test/command/7266.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Citeproc/BibTeX.hs b/src/Text/Pandoc/Citeproc/BibTeX.hs
index f6833000c..b17240557 100644
--- a/src/Text/Pandoc/Citeproc/BibTeX.hs
+++ b/src/Text/Pandoc/Citeproc/BibTeX.hs
@@ -156,6 +156,7 @@ writeBibtexString opts variant mblang ref =
            , "langid"
            , "abstract"
            , "keywords"
+           , "annote"
            ]
       Bibtex ->
            [ "author"
@@ -175,6 +176,7 @@ writeBibtexString opts variant mblang ref =
            , "address"
            , "type"
            , "note"
+           , "annote"
            ]
 
   valToInlines (TextVal t) = B.text t
diff --git a/test/command/7266.md b/test/command/7266.md
new file mode 100644
index 000000000..9db833636
--- /dev/null
+++ b/test/command/7266.md
@@ -0,0 +1,10 @@
+```
+% pandoc -f biblatex -t biblatex -s
+@article{id,
+  annote = "annotation"
+}
+^D
+@article{id,
+  annote = {annotation}
+}
+```
-- 
cgit v1.2.3


From 3f09f53459b877f53072efbf57dec21fa37280b5 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 13 May 2021 21:11:52 -0700
Subject: Implement curly-brace syntax for Markdown citation keys.

The change provides a way to use citation keys that contain
special characters not usable with the standard citation
key syntax.  Example: `@{foo_bar{x}'}` for the key `foo_bar{x}`.
Closes #6026.

The change requires adding a new parameter to the `citeKey`
parser from Text.Pandoc.Parsing [API change].

Markdown reader: recognize @{..} syntax for citatinos.

Markdown writer:  use @{..} syntax for citations when needed.

Update manual with curly-brace syntax for citations.

Closes #6026.
---
 MANUAL.txt                                 | 15 ++++++++++-----
 src/Text/Pandoc/Parsing.hs                 | 18 +++++++++++++-----
 src/Text/Pandoc/Readers/Markdown.hs        |  6 +++---
 src/Text/Pandoc/Readers/Org/Inlines.hs     |  8 ++++----
 src/Text/Pandoc/Writers/Markdown/Inline.hs |  9 +++++++--
 test/command/6026.md                       | 19 +++++++++++++++++++
 6 files changed, 56 insertions(+), 19 deletions(-)
 create mode 100644 test/command/6026.md

(limited to 'test/command')

diff --git a/MANUAL.txt b/MANUAL.txt
index 75fe599e2..6f06d1e8a 100644
--- a/MANUAL.txt
+++ b/MANUAL.txt
@@ -4949,12 +4949,15 @@ Inline and regular footnotes may be mixed freely.
 #### Extension: `citations` ####
 
 Markdown citations go inside square brackets and are separated
-by semicolons.  Each citation must have a key, composed of '@' +
-the citation identifier from the database, and may optionally
-have a prefix, a locator, and a suffix.  The citation key must
-begin with a letter, digit, or `_`, and may contain
+by semicolons.  Each citation must have a key and may optionally
+have a prefix, a locator, and a suffix.  The citation key
+consists of `@` plus the citation identifier, possibly
+enclosed in curly braces.  If the identifier starts
+with a letter, digit, or `_`, followed by zero or more
 alphanumerics, `_`, and internal punctuation characters
-(`:.#$%&-+?<>~/`).  Here are some examples:
+(`:.#$%&-+?<>~/`), then the curly braces may be omitted.
+Identifiers may not contain whitespace characters or unbalanced
+curly braces.  Here are some examples:
 
     Blah blah [see @doe99, pp. 33-35; also @smith04, chap. 1].
 
@@ -4962,6 +4965,8 @@ alphanumerics, `_`, and internal punctuation characters
 
     Blah blah [@smith04; @doe99].
 
+    Blah blah [@{https://example.com/bib?name=foobar&date=2000}, p. 33].
+
 `pandoc` detects locator terms in the [CSL locale files].
 Either abbreviated or unabbreviated forms are accepted. In the `en-US`
 locale, locator terms can be written in either singular or plural forms,
diff --git a/src/Text/Pandoc/Parsing.hs b/src/Text/Pandoc/Parsing.hs
index cbe9993c6..0bb794ba1 100644
--- a/src/Text/Pandoc/Parsing.hs
+++ b/src/Text/Pandoc/Parsing.hs
@@ -1605,19 +1605,27 @@ nested p = do
   return res
 
 citeKey :: (Stream s m Char, UpdateSourcePos s Char, HasLastStrPosition st)
-        => ParserT s st m (Bool, Text)
-citeKey = try $ do
+        => Bool -- ^ If True, allow expanded @{..} syntax.
+        -> ParserT s st m (Bool, Text)
+citeKey allowBraced = try $ do
   guard =<< notAfterString
   suppress_author <- option False (True <$ char '-')
   char '@'
+  key <- simpleCiteIdentifier
+        <|> if allowBraced
+               then charsInBalanced '{' '}' (satisfy (not . isSpace))
+               else mzero
+  return (suppress_author, key)
+
+simpleCiteIdentifier :: (Stream s m Char, UpdateSourcePos s Char)
+                      => ParserT s st m Text
+simpleCiteIdentifier = do
   firstChar <- alphaNum <|> char '_' <|> char '*' -- @* for wildcard in nocite
   let regchar = satisfy (\c -> isAlphaNum c || c == '_')
   let internal p = try $ p <* lookAhead regchar
   rest <- many $ regchar <|> internal (oneOf ":.#$%&-+?<>~/") <|>
                  try (oneOf ":/" <* lookAhead (char '/'))
-  let key = firstChar:rest
-  return (suppress_author, T.pack key)
-
+  return $ T.pack $ firstChar:rest
 
 token :: (Stream s m t)
       => (t -> Text)
diff --git a/src/Text/Pandoc/Readers/Markdown.hs b/src/Text/Pandoc/Readers/Markdown.hs
index 2d20ff018..34f16ab4e 100644
--- a/src/Text/Pandoc/Readers/Markdown.hs
+++ b/src/Text/Pandoc/Readers/Markdown.hs
@@ -2094,7 +2094,7 @@ cite = do
 
 textualCite :: PandocMonad m => MarkdownParser m (F Inlines)
 textualCite = try $ do
-  (suppressAuthor, key) <- citeKey
+  (suppressAuthor, key) <- citeKey True
   -- If this is a reference to an earlier example list item,
   -- then don't parse it as a citation.  If the example list
   -- item comes later, we'll parse it here and figure out in
@@ -2174,7 +2174,7 @@ prefix = trimInlinesF . mconcat <$>
   manyTill inline (char ']'
    <|> lookAhead
          (try $ do optional (try (char ';' >> spnl))
-                   citeKey
+                   citeKey True
                    return ']'))
 
 citeList :: PandocMonad m => MarkdownParser m (F [Citation])
@@ -2183,7 +2183,7 @@ citeList = fmap sequence $ sepBy1 citation (try $ char ';' >> spnl)
 citation :: PandocMonad m => MarkdownParser m (F Citation)
 citation = try $ do
   pref <- prefix
-  (suppress_author, key) <- citeKey
+  (suppress_author, key) <- citeKey True
   suff <- suffix
   noteNum <- stateNoteNumber <$> getState
   return $ do
diff --git a/src/Text/Pandoc/Readers/Org/Inlines.hs b/src/Text/Pandoc/Readers/Org/Inlines.hs
index 054f2611a..6862dd71e 100644
--- a/src/Text/Pandoc/Readers/Org/Inlines.hs
+++ b/src/Text/Pandoc/Readers/Org/Inlines.hs
@@ -263,7 +263,7 @@ berkeleyCitationList = try $ do
  where
    citationListPart :: PandocMonad m => OrgParser m (F Inlines)
    citationListPart = fmap (trimInlinesF . mconcat) . try . many1 $ do
-     notFollowedBy' citeKey
+     notFollowedBy' $ citeKey False
      notFollowedBy (oneOf ";]")
      inline
 
@@ -278,7 +278,7 @@ berkeleyBareTag' = try $ void (string "cite")
 
 berkeleyTextualCite :: PandocMonad m => OrgParser m (F [Citation])
 berkeleyTextualCite = try $ do
-  (suppressAuthor, key) <- citeKey
+  (suppressAuthor, key) <- citeKey False
   returnF . return $ Citation
     { citationId      = key
     , citationPrefix  = mempty
@@ -351,7 +351,7 @@ citeList = sequence <$> sepEndBy1 citation (try $ char ';' *> skipSpaces)
 citation :: PandocMonad m => OrgParser m (F Citation)
 citation = try $ do
   pref <- prefix
-  (suppress_author, key) <- citeKey
+  (suppress_author, key) <- citeKey False
   suff <- suffix
   return $ do
     x <- pref
@@ -368,7 +368,7 @@ citation = try $ do
       }
  where
    prefix = trimInlinesF . mconcat <$>
-            manyTill inline (char ']' <|> (']' <$ lookAhead citeKey))
+            manyTill inline (char ']' <|> (']' <$ lookAhead (citeKey False)))
    suffix = try $ do
      hasSpace <- option False (notFollowedBy nonspaceChar >> return True)
      skipSpaces
diff --git a/src/Text/Pandoc/Writers/Markdown/Inline.hs b/src/Text/Pandoc/Writers/Markdown/Inline.hs
index e35e1a0b9..2062050e4 100644
--- a/src/Text/Pandoc/Writers/Markdown/Inline.hs
+++ b/src/Text/Pandoc/Writers/Markdown/Inline.hs
@@ -491,11 +491,16 @@ inlineToMarkdown opts (Cite (c:cs) lst)
            rest <- mapM convertOne cs
            let inbr = suffs <+> joincits rest
                br   = if isEmpty inbr then empty else char '[' <> inbr <> char ']'
-           return $ literal ("@" <> citationId c) <+> br
+           return $ literal ("@" <> maybeInBraces (citationId c)) <+> br
          else do
            cits <- mapM convertOne (c:cs)
            return $ literal "[" <> joincits cits <> literal "]"
   where
+        maybeInBraces key =
+          case readWith (citeKey False >> spaces >> eof)
+                 defaultParserState ("@" <> key) of
+            Left _  -> "{" <> key <> "}"
+            Right _ -> key
         joincits = hcat . intersperse (literal "; ") . filter (not . isEmpty)
         convertOne Citation { citationId      = k
                             , citationPrefix  = pinlines
@@ -504,7 +509,7 @@ inlineToMarkdown opts (Cite (c:cs) lst)
                                = do
            pdoc <- inlineListToMarkdown opts pinlines
            sdoc <- inlineListToMarkdown opts sinlines
-           let k' = literal (modekey m <> "@" <> k)
+           let k' = literal (modekey m <> "@" <> maybeInBraces k)
                r = case sinlines of
                         Str (T.uncons -> Just (y,_)):_ | y `elem` (",;]@" :: String) -> k' <> sdoc
                         _                                         -> k' <+> sdoc
diff --git a/test/command/6026.md b/test/command/6026.md
new file mode 100644
index 000000000..5e18a5f42
--- /dev/null
+++ b/test/command/6026.md
@@ -0,0 +1,19 @@
+```
+% pandoc -t native
+@{https://openreview.net/forum?id=HkwoSDPgg}
+
+@https://openreview.net/forum?id=HkwoSDPgg
+^D
+[Para [Cite [Citation {citationId = "https://openreview.net/forum?id=HkwoSDPgg", citationPrefix = [], citationSuffix = [], citationMode = AuthorInText, citationNoteNum = 1, citationHash = 0}] [Str "@https://openreview.net/forum?id=HkwoSDPgg"]]
+,Para [Cite [Citation {citationId = "https://openreview.net/forum?id", citationPrefix = [], citationSuffix = [], citationMode = AuthorInText, citationNoteNum = 2, citationHash = 0}] [Str "@https://openreview.net/forum?id"],Str "=HkwoSDPgg"]]
+```
+```
+% pandoc -t markdown
+@{https://openreview.net/forum?id=HkwoSDPgg}
+
+@https://openreview.net/forum?id=HkwoSDPgg
+^D
+@{https://openreview.net/forum?id=HkwoSDPgg}
+
+@https://openreview.net/forum?id=HkwoSDPgg
+```
-- 
cgit v1.2.3


From 875f8f36545d1c21fa3d29c52c40517a667f2574 Mon Sep 17 00:00:00 2001
From: Albert Krewinkel <albert@zeitkraut.de>
Date: Fri, 14 May 2021 11:58:58 +0200
Subject: HTML reader: don't fail on unmatched closing "script" tag.

Prevent the reader from crashing if the HTML input contains an unmatched
closing `</script>` tag.

Fixes: #7282
---
 src/Text/Pandoc/Readers/HTML.hs | 16 +++++++++-------
 test/command/7282.md            |  7 +++++++
 2 files changed, 16 insertions(+), 7 deletions(-)
 create mode 100644 test/command/7282.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/HTML.hs b/src/Text/Pandoc/Readers/HTML.hs
index f5c8a2277..0a9d67e35 100644
--- a/src/Text/Pandoc/Readers/HTML.hs
+++ b/src/Text/Pandoc/Readers/HTML.hs
@@ -942,13 +942,15 @@ getTagName (TagClose t)  = Just t
 getTagName _             = Nothing
 
 isInlineTag :: Tag Text -> Bool
-isInlineTag t =
-  isCommentTag t ||
-  case getTagName t of
-    Nothing  -> False
-    Just "script" -> "math/tex" `T.isPrefixOf` fromAttrib "type" t
-    Just x   -> x `Set.notMember` blockTags ||
-                T.take 1 x == "?" -- processing instr.
+isInlineTag t = isCommentTag t || case t of
+  TagOpen "script" _ -> "math/tex" `T.isPrefixOf` fromAttrib "type" t
+  TagClose "script"  -> True
+  TagOpen name _     -> isInlineTagName name
+  TagClose name      -> isInlineTagName name
+  _                  -> False
+ where isInlineTagName x =
+         x `Set.notMember` blockTags ||
+         T.take 1 x == "?" -- processing instr.
 
 isBlockTag :: Tag Text -> Bool
 isBlockTag t = isBlockTagName || isTagComment t
diff --git a/test/command/7282.md b/test/command/7282.md
new file mode 100644
index 000000000..4ed6b5b3e
--- /dev/null
+++ b/test/command/7282.md
@@ -0,0 +1,7 @@
+Don't crash on unmatched closing tag.
+```
+% pandoc -f html -t native
+</script>
+^D
+[]
+```
-- 
cgit v1.2.3


From 76a4e7127be1116b67ae531f56aadb05963813b5 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Fri, 14 May 2021 10:09:02 -0700
Subject: Beamer writer: support exampleblock and alertblock.

A block will be rendered as an exampleblock if the heading
has class `example` and alertblock if it has class `alert`.

Closes #7278.
---
 MANUAL.txt                       |  6 +++++-
 src/Text/Pandoc/Writers/LaTeX.hs | 11 ++++++++---
 test/command/7278.md             | 38 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 51 insertions(+), 4 deletions(-)
 create mode 100644 test/command/7278.md

(limited to 'test/command')

diff --git a/MANUAL.txt b/MANUAL.txt
index 6f06d1e8a..cdfb0e332 100644
--- a/MANUAL.txt
+++ b/MANUAL.txt
@@ -5666,7 +5666,11 @@ rules:
   * A heading at the slide level always starts a new slide.
 
   * Headings *below* the slide level in the hierarchy create
-    headings *within* a slide.
+    headings *within* a slide.  (In beamer, a "block" will be
+    created.  If the heading has the class `example`, an
+    `exampleblock` environment will be used; if it has the class
+    `alert`, an `alertblock` will be used; otherwise a regular
+    `block` will be used.)
 
   * Headings *above* the slide level in the hierarchy create
     "title slides," which just contain the section title
diff --git a/src/Text/Pandoc/Writers/LaTeX.hs b/src/Text/Pandoc/Writers/LaTeX.hs
index 8b1f3df1d..d8722876e 100644
--- a/src/Text/Pandoc/Writers/LaTeX.hs
+++ b/src/Text/Pandoc/Writers/LaTeX.hs
@@ -290,7 +290,12 @@ blockToLaTeX :: PandocMonad m
              => Block     -- ^ Block to convert
              -> LW m (Doc Text)
 blockToLaTeX Null = return empty
-blockToLaTeX (Div attr@(identifier,"block":_,_) (Header _ _ ils : bs)) = do
+blockToLaTeX (Div attr@(identifier,"block":dclasses,_)
+             (Header _ _ ils : bs)) = do
+  let blockname
+        | "example" `elem` dclasses = "exampleblock"
+        | "alert" `elem` dclasses = "alertblock"
+        | otherwise = "block"
   ref <- toLabel identifier
   let anchor = if T.null identifier
                   then empty
@@ -298,8 +303,8 @@ blockToLaTeX (Div attr@(identifier,"block":_,_) (Header _ _ ils : bs)) = do
                        braces (literal ref) <> braces empty
   title' <- inlineListToLaTeX ils
   contents <- blockListToLaTeX bs
-  wrapDiv attr $ ("\\begin{block}" <> braces title' <> anchor) $$
-                 contents $$ "\\end{block}"
+  wrapDiv attr $ ("\\begin" <> braces blockname <> braces title' <> anchor) $$
+                 contents $$ "\\end" <> braces blockname
 blockToLaTeX (Div (identifier,"slide":dclasses,dkvs)
                (Header _ (_,hclasses,hkvs) ils : bs)) = do
   -- note: [fragile] is required or verbatim breaks
diff --git a/test/command/7278.md b/test/command/7278.md
new file mode 100644
index 000000000..dcf71c29f
--- /dev/null
+++ b/test/command/7278.md
@@ -0,0 +1,38 @@
+```
+% pandoc -t beamer
+# Slide
+
+Some blocks:
+
+## example block title {.example}
+
+text in block
+
+## alert block title {.alert}
+
+text in block
+
+## block title
+
+text in block
+^D
+\begin{frame}{Slide}
+\protect\hypertarget{slide}{}
+Some blocks:
+
+\begin{exampleblock}{example block title}
+\protect\hypertarget{example-block-title}{}
+text in block
+\end{exampleblock}
+
+\begin{alertblock}{alert block title}
+\protect\hypertarget{alert-block-title}{}
+text in block
+\end{alertblock}
+
+\begin{block}{block title}
+\protect\hypertarget{block-title}{}
+text in block
+\end{block}
+\end{frame}
+```
-- 
cgit v1.2.3


From 5a6399d9f62c4306fa073ae1311675158dd6a203 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sun, 16 May 2021 12:23:34 -0700
Subject: Markdown writer: fewer unneeded escapes for `#`.

See #6259.
---
 src/Text/Pandoc/Writers/Markdown/Inline.hs | 13 +++++++++++--
 test/command/3792.md                       |  2 +-
 test/command/4164.md                       |  2 +-
 test/command/7208.md                       |  2 +-
 test/command/biblatex-sigfridsson.md       |  2 +-
 test/writer.opml                           |  2 +-
 6 files changed, 16 insertions(+), 7 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/Markdown/Inline.hs b/src/Text/Pandoc/Writers/Markdown/Inline.hs
index ced5fbacb..e6c6da5a9 100644
--- a/src/Text/Pandoc/Writers/Markdown/Inline.hs
+++ b/src/Text/Pandoc/Writers/Markdown/Inline.hs
@@ -43,7 +43,11 @@ import Text.Pandoc.Writers.Markdown.Types (MarkdownVariant(..),
 -- | Escape special characters for Markdown.
 escapeText :: WriterOptions -> Text -> Text
 escapeText opts = T.pack . go . T.unpack
-  where
+ where
+  startsWithSpace (' ':_) = True
+  startsWithSpace ('\t':_) = True
+  startsWithSpace [] = True
+  startsWithSpace _ = False
   go [] = []
   go (c:cs) =
     case c of
@@ -59,7 +63,10 @@ escapeText opts = T.pack . go . T.unpack
                       | isAlphaNum d || d == '_' || d == '{'
                          -> '\\':'@':go cs
                     _ -> '@':go cs
-       _ | c `elem` ['\\','`','*','_','[',']','#'] ->
+       '#' | isEnabled Ext_space_in_atx_header opts
+           , startsWithSpace cs
+           -> '\\':'#':go cs
+       _ | c `elem` ['\\','`','*','_','[',']'] ->
               '\\':c:go cs
        '|' | isEnabled Ext_pipe_tables opts -> '\\':'|':go cs
        '^' | isEnabled Ext_superscript opts -> '\\':'^':go cs
@@ -81,6 +88,8 @@ escapeText opts = T.pack . go . T.unpack
                   | isEnabled Ext_intraword_underscores opts
                   , isAlphaNum c
                   , isAlphaNum x -> c : '_' : x : go xs
+                '#':xs           -> c : '#' : go xs
+                '>':xs           -> c : '>' : go xs
                 _                -> c : go cs
 
 attrsToMarkdown :: Attr -> Doc Text
diff --git a/test/command/3792.md b/test/command/3792.md
index eb109b9cc..eff26d517 100644
--- a/test/command/3792.md
+++ b/test/command/3792.md
@@ -6,7 +6,7 @@ and properly escaped.
 ok
 ^D
 ---
-title: \<this\> \*that\*
+title: \<this> \*that\*
 ---
 
 ok
diff --git a/test/command/4164.md b/test/command/4164.md
index 4e7b7e285..68cbd0584 100644
--- a/test/command/4164.md
+++ b/test/command/4164.md
@@ -26,6 +26,6 @@ Here is inline html:
 
 Here is inline html:
 
-\<div\> \<balise\> bla bla \</div\>
+\<div> \<balise> bla bla \</div>
 
 ```
diff --git a/test/command/7208.md b/test/command/7208.md
index e65943ade..fe02ec32e 100644
--- a/test/command/7208.md
+++ b/test/command/7208.md
@@ -2,5 +2,5 @@
 % pandoc -t gfm
 \<hi\>
 ^D
-\<hi\>
+\<hi>
 ```
diff --git a/test/command/biblatex-sigfridsson.md b/test/command/biblatex-sigfridsson.md
index f83c35622..e042a8762 100644
--- a/test/command/biblatex-sigfridsson.md
+++ b/test/command/biblatex-sigfridsson.md
@@ -90,7 +90,7 @@ references:
   - family: Ryde
     given: Ulf
   container-title: Journal of Computational Chemistry
-  doi: "10.1002/(SICI)1096-987X(199803)19:4\\<377::AID-JCC1\\>3.0.CO;2-P"
+  doi: "10.1002/(SICI)1096-987X(199803)19:4\\<377::AID-JCC1>3.0.CO;2-P"
   id: sigfridsson
   issue: 4
   issued: 1998
diff --git a/test/writer.opml b/test/writer.opml
index 6bdcb882e..bfe1e5de6 100644
--- a/test/writer.opml
+++ b/test/writer.opml
@@ -52,7 +52,7 @@
 </outline>
 <outline text="LaTeX" _note="-   &#10;-   2 + 2 = 4&#10;-   *x* ∈ *y*&#10;-   *α* ∧ *ω*&#10;-   223&#10;-   *p*-Tree&#10;-   Here’s some display math:&#10;    $$\\frac{d}{dx}f(x)=\\lim\_{h\\to 0}\\frac{f(x+h)-f(x)}{h}$$&#10;-   Here’s one that has a line break in it: *α* + *ω* × *x*².&#10;&#10;These shouldn’t be math:&#10;&#10;-   To get the famous equation, write `$e = mc^2$`.&#10;-   $22,000 is a *lot* of money. So is $34,000. (It worked if “lot” is&#10;    emphasized.)&#10;-   Shoes ($20) and socks ($5).&#10;-   Escaped `$`: $73 *this should be emphasized* 23$.&#10;&#10;Here’s a LaTeX table:&#10;&#10;------------------------------------------------------------------------">
 </outline>
-<outline text="Special Characters" _note="Here is some unicode:&#10;&#10;-   I hat: Î&#10;-   o umlaut: ö&#10;-   section: §&#10;-   set membership: ∈&#10;-   copyright: ©&#10;&#10;AT&amp;T has an ampersand in their name.&#10;&#10;AT&amp;T is another way to write it.&#10;&#10;This &amp; that.&#10;&#10;4 &amp;lt; 5.&#10;&#10;6 &amp;gt; 5.&#10;&#10;Backslash: \\&#10;&#10;Backtick: \`&#10;&#10;Asterisk: \*&#10;&#10;Underscore: \_&#10;&#10;Left brace: {&#10;&#10;Right brace: }&#10;&#10;Left bracket: \[&#10;&#10;Right bracket: \]&#10;&#10;Left paren: (&#10;&#10;Right paren: )&#10;&#10;Greater-than: &amp;gt;&#10;&#10;Hash: \#&#10;&#10;Period: .&#10;&#10;Bang: !&#10;&#10;Plus: +&#10;&#10;Minus: -&#10;&#10;------------------------------------------------------------------------">
+<outline text="Special Characters" _note="Here is some unicode:&#10;&#10;-   I hat: Î&#10;-   o umlaut: ö&#10;-   section: §&#10;-   set membership: ∈&#10;-   copyright: ©&#10;&#10;AT&amp;T has an ampersand in their name.&#10;&#10;AT&amp;T is another way to write it.&#10;&#10;This &amp; that.&#10;&#10;4 &amp;lt; 5.&#10;&#10;6 &amp;gt; 5.&#10;&#10;Backslash: \\&#10;&#10;Backtick: \`&#10;&#10;Asterisk: \*&#10;&#10;Underscore: \_&#10;&#10;Left brace: {&#10;&#10;Right brace: }&#10;&#10;Left bracket: \[&#10;&#10;Right bracket: \]&#10;&#10;Left paren: (&#10;&#10;Right paren: )&#10;&#10;Greater-than: &amp;gt;&#10;&#10;Hash: #&#10;&#10;Period: .&#10;&#10;Bang: !&#10;&#10;Plus: +&#10;&#10;Minus: -&#10;&#10;------------------------------------------------------------------------">
 </outline>
 <outline text="Links">
   <outline text="Explicit" _note="Just a [URL](/url/).&#10;&#10;[URL and title](/url/ &quot;title&quot;).&#10;&#10;[URL and title](/url/ &quot;title preceded by two spaces&quot;).&#10;&#10;[URL and title](/url/ &quot;title preceded by a tab&quot;).&#10;&#10;[URL and title](/url/ &quot;title with &quot;quotes&quot; in it&quot;)&#10;&#10;[URL and title](/url/ &quot;title with single quotes&quot;)&#10;&#10;[with\_underscore](/url/with_underscore)&#10;&#10;[Email link](mailto:nobody@nowhere.net)&#10;&#10;[Empty]().">
-- 
cgit v1.2.3


From eb3dff148e67e84362632e63848d40ba808940f4 Mon Sep 17 00:00:00 2001
From: Albert Krewinkel <albert@zeitkraut.de>
Date: Tue, 18 May 2021 22:55:47 +0200
Subject: LaTeX writer: separate successive quote chars with thin space

Successive quote characters are separated with a thin space to improve
readability and to prevent unwanted ligatures. Detection of these quotes
sometimes had failed if the second quote was nested in a span element.

Closes: #6958
---
 src/Text/Pandoc/Writers/LaTeX.hs |  5 +++--
 test/command/6958.md             | 10 ++++++++++
 2 files changed, 13 insertions(+), 2 deletions(-)
 create mode 100644 test/command/6958.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/LaTeX.hs b/src/Text/Pandoc/Writers/LaTeX.hs
index bf57937bd..978f94ea0 100644
--- a/src/Text/Pandoc/Writers/LaTeX.hs
+++ b/src/Text/Pandoc/Writers/LaTeX.hs
@@ -894,8 +894,9 @@ inlineToLaTeX (Quoted qt lst) = do
                       then char '`' <> inner <> char '\''
                       else char '\x2018' <> inner <> char '\x2019'
     where
-      isQuoted (Quoted _ _) = True
-      isQuoted _            = False
+      isQuoted (Span _ (x:_)) = isQuoted x
+      isQuoted (Quoted _ _)   = True
+      isQuoted _              = False
 inlineToLaTeX (Str str) = do
   setEmptyLine False
   liftM literal $ stringToLaTeX TextString str
diff --git a/test/command/6958.md b/test/command/6958.md
new file mode 100644
index 000000000..230371d7d
--- /dev/null
+++ b/test/command/6958.md
@@ -0,0 +1,10 @@
+Add thin space between single and double quotes.
+```
+% pandoc -t latex+smart
+---
+lang: en-GB
+---
+'["On the Outside"]{}: Constructing Cycling Citizenship.'
+^D
+`\,{``On the Outside''}: Constructing Cycling Citizenship.'
+```
-- 
cgit v1.2.3


From 5736b331d8ecaa12cc3e2712211ada37c665a93a Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Wed, 19 May 2021 16:14:49 -0700
Subject: LaTeX reader: better support for `\xspace`.

Previously we only supported it in inline contexts; now
we support it in all contexts, including math.

Partially addresses #7299.
---
 src/Text/Pandoc/Readers/LaTeX.hs         | 12 ------------
 src/Text/Pandoc/Readers/LaTeX/Parsing.hs | 21 +++++++++++++++++++--
 test/command/4442.md                     |  2 +-
 test/command/7299.md                     | 23 +++++++++++++++++++++++
 4 files changed, 43 insertions(+), 15 deletions(-)
 create mode 100644 test/command/7299.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/LaTeX.hs b/src/Text/Pandoc/Readers/LaTeX.hs
index f90d562ae..2ace18d1b 100644
--- a/src/Text/Pandoc/Readers/LaTeX.hs
+++ b/src/Text/Pandoc/Readers/LaTeX.hs
@@ -232,16 +232,6 @@ mkImage options (T.unpack -> src) = do
                _  -> return src
    return $ imageWith attr (T.pack src') "" alt
 
-doxspace :: PandocMonad m => LP m Inlines
-doxspace =
-  (space <$ lookAhead (satisfyTok startsWithLetter)) <|> return mempty
-  where startsWithLetter (Tok _ Word t) =
-          case T.uncons t of
-               Just (c, _) | isLetter c -> True
-               _           -> False
-        startsWithLetter _ = False
-
-
 removeDoubleQuotes :: Text -> Text
 removeDoubleQuotes t =
   Data.Maybe.fromMaybe t $ T.stripPrefix "\"" t >>= T.stripSuffix "\""
@@ -417,8 +407,6 @@ inlineCommands = M.unions
     -- LaTeX colors
     , ("textcolor", coloredInline "color")
     , ("colorbox", coloredInline "background-color")
-    -- xspace
-    , ("xspace", doxspace)
     -- etoolbox
     , ("ifstrequal", ifstrequal)
     , ("newtoggle", braced >>= newToggle)
diff --git a/src/Text/Pandoc/Readers/LaTeX/Parsing.hs b/src/Text/Pandoc/Readers/LaTeX/Parsing.hs
index 35ce3509d..b6804a825 100644
--- a/src/Text/Pandoc/Readers/LaTeX/Parsing.hs
+++ b/src/Text/Pandoc/Readers/LaTeX/Parsing.hs
@@ -464,7 +464,7 @@ satisfyTok f = do
 doMacros :: PandocMonad m => LP m ()
 doMacros = do
   st <- getState
-  unless (sVerbatimMode st || M.null (sMacros st)) $ do
+  unless (sVerbatimMode st) $
     getInput >>= doMacros' 1 >>= setInput
 
 doMacros' :: PandocMonad m => Int -> [Tok] -> LP m [Tok]
@@ -526,7 +526,7 @@ doMacros' n inp =
         $ throwError $ PandocMacroLoop name
       macros <- sMacros <$> getState
       case M.lookup name macros of
-           Nothing -> mzero
+           Nothing -> trySpecialMacro name ts
            Just (Macro expansionPoint argspecs optarg newtoks) -> do
              let getargs' = do
                    args <-
@@ -554,6 +554,23 @@ doMacros' n inp =
                    ExpandWhenUsed    -> doMacros' (n' + 1) result
                    ExpandWhenDefined -> return result
 
+-- | Certain macros do low-level tex manipulations that can't
+-- be represented in our Macro type, so we handle them here.
+trySpecialMacro :: PandocMonad m => Text -> [Tok] -> LP m [Tok]
+trySpecialMacro "xspace" ts = do
+  ts' <- doMacros' 1 ts
+  case ts' of
+    Tok pos Word t : _
+      | startsWithAlphaNum t -> return $ Tok pos Spaces " " : ts'
+    _ -> return ts'
+trySpecialMacro _ _ = mzero
+
+startsWithAlphaNum :: Text -> Bool
+startsWithAlphaNum t =
+  case T.uncons t of
+       Just (c, _) | isAlphaNum c -> True
+       _           -> False
+
 setpos :: SourcePos -> Tok -> Tok
 setpos spos (Tok _ tt txt) = Tok spos tt txt
 
diff --git a/test/command/4442.md b/test/command/4442.md
index 8574fe759..447073406 100644
--- a/test/command/4442.md
+++ b/test/command/4442.md
@@ -5,5 +5,5 @@
 ^D
 \newcommand{\myFruit}{Mango\xspace}
 
-Mango\xspace is the king of fruits.
+Mango is the king of fruits.
 ```
diff --git a/test/command/7299.md b/test/command/7299.md
new file mode 100644
index 000000000..0847c40ce
--- /dev/null
+++ b/test/command/7299.md
@@ -0,0 +1,23 @@
+```
+% pandoc -f latex -t plain
+$1-{\ensuremath{r}\xspace}$
+^D
+1 − r
+```
+
+```
+% pandoc -f latex -t plain
+\newcommand{\foo}{Foo\xspace}
+
+$\text{\foo bar}$
+^D
+Foo bar
+```
+
+```
+% pandoc -f latex -t plain
+a\xspace b
+^D
+a b
+```
+
-- 
cgit v1.2.3


From 8437a4a002210a33ee721c58f5f95605898a8e1b Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 20 May 2021 08:15:48 -0700
Subject: LaTeX reader: support `\pm` in `SI{..}`.

Closes #6620.
---
 src/Text/Pandoc/Readers/LaTeX/SIunitx.hs | 4 +++-
 test/command/6620.md                     | 3 +++
 2 files changed, 6 insertions(+), 1 deletion(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs b/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs
index 1952f4e1a..c4fb06700 100644
--- a/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs
+++ b/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs
@@ -45,7 +45,9 @@ doSI tok = do
                       unit]
 
 doSInum :: PandocMonad m => LP m Inlines
-doSInum = skipopts *> (tonum . untokenize <$> braced)
+doSInum = skipopts *> (tonum . untokenize . map convertPM <$> braced)
+  where convertPM (Tok pos (CtrlSeq "pm") _) = Tok pos Word "\xb1\xa0"
+        convertPM t = t
 
 tonum :: Text -> Inlines
 tonum value =
diff --git a/test/command/6620.md b/test/command/6620.md
index 314200d30..00643e57c 100644
--- a/test/command/6620.md
+++ b/test/command/6620.md
@@ -7,9 +7,12 @@
 \SI{0.135(21)}{\m}
 
 \SI{12.3(60)}{\m}
+
+\SI{10.0 \pm 3.3}{\ms}
 ^D
 <p>23 ± 2 m</p>
 <p>125 ± 12 m</p>
 <p>0.135 ± 0.021 m</p>
 <p>12.3 ± 6 m</p>
+<p>10.0 ± 3.3 ms</p>
 ```
-- 
cgit v1.2.3


From a366bd6abccd18a49f0033de850ddd53830cc5ed Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 20 May 2021 09:03:29 -0700
Subject: LaTeX reader: Fix parsing of `+-` in siunitx numbers.

See #6658.
---
 src/Text/Pandoc/Readers/LaTeX/SIunitx.hs | 11 +++++++----
 test/command/6620.md                     |  5 ++++-
 2 files changed, 11 insertions(+), 5 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs b/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs
index c4fb06700..5e140ef7a 100644
--- a/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs
+++ b/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs
@@ -45,9 +45,7 @@ doSI tok = do
                       unit]
 
 doSInum :: PandocMonad m => LP m Inlines
-doSInum = skipopts *> (tonum . untokenize . map convertPM <$> braced)
-  where convertPM (Tok pos (CtrlSeq "pm") _) = Tok pos Word "\xb1\xa0"
-        convertPM t = t
+doSInum = skipopts *> (tonum . untokenize <$> braced)
 
 tonum :: Text -> Inlines
 tonum value =
@@ -74,12 +72,16 @@ parseNumPart =
   parseDecimalNum <|>
   parseComma <|>
   parsePlusMinus <|>
+  parsePM <|>
   parseI <|>
   parseExp <|>
   parseX <|>
   parseSpace
  where
-  parseDecimalNum = do
+  parseDecimalNum, parsePlusMinus, parsePM,
+    parseComma, parseI, parseX,
+    parseExp, parseSpace :: Parser Text () Inlines
+  parseDecimalNum = try $ do
     pref <- option mempty $ (mempty <$ char '+') <|> ("\x2212" <$ char '-')
     basenum <- (pref <>) . T.pack
                 <$> many1 (satisfy (\c -> isDigit c || c == '.'))
@@ -100,6 +102,7 @@ parseNumPart =
                                                | otherwise -> "." <> t
   parseComma = str "." <$ char ','
   parsePlusMinus = str "\xa0\xb1\xa0" <$ try (string "+-")
+  parsePM = str "\xa0\xb1\xa0" <$ try (string "\\pm")
   parseParens =
     char '(' *> many1 (satisfy (\c -> isDigit c || c == '.')) <* char ')'
   parseI = str "i" <$ char 'i'
diff --git a/test/command/6620.md b/test/command/6620.md
index 00643e57c..b19cd1972 100644
--- a/test/command/6620.md
+++ b/test/command/6620.md
@@ -9,10 +9,13 @@
 \SI{12.3(60)}{\m}
 
 \SI{10.0 \pm 3.3}{\ms}
+
+\SI{10.0 +- 3.3}{\ms}
 ^D
 <p>23 ± 2 m</p>
 <p>125 ± 12 m</p>
 <p>0.135 ± 0.021 m</p>
 <p>12.3 ± 6 m</p>
-<p>10.0 ± 3.3 ms</p>
+<p>10.0 ± 3.3 ms</p>
+<p>10.0 ± 3.3 ms</p>
 ```
-- 
cgit v1.2.3


From 5dc917da3ed997c6e48e22bde242f0f8e1ae5333 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 20 May 2021 09:11:26 -0700
Subject: LaTeX reader siunitx: add leading 0 to numbers starting with .

---
 src/Text/Pandoc/Readers/LaTeX/SIunitx.hs | 7 +++++--
 test/command/6620.md                     | 3 ---
 test/command/6658.md                     | 9 +++++++++
 3 files changed, 14 insertions(+), 5 deletions(-)
 create mode 100644 test/command/6658.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs b/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs
index 5e140ef7a..1474329d4 100644
--- a/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs
+++ b/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs
@@ -83,8 +83,11 @@ parseNumPart =
     parseExp, parseSpace :: Parser Text () Inlines
   parseDecimalNum = try $ do
     pref <- option mempty $ (mempty <$ char '+') <|> ("\x2212" <$ char '-')
-    basenum <- (pref <>) . T.pack
-                <$> many1 (satisfy (\c -> isDigit c || c == '.'))
+    basenum' <- many1 (satisfy (\c -> isDigit c || c == '.'))
+    let basenum = pref <> T.pack
+                    (case basenum' of
+                      '.':_ -> '0':basenum'
+                      _ -> basenum')
     uncertainty <- option mempty $ T.pack <$> parseParens
     if T.null uncertainty
        then return $ str basenum
diff --git a/test/command/6620.md b/test/command/6620.md
index b19cd1972..e448ca6b5 100644
--- a/test/command/6620.md
+++ b/test/command/6620.md
@@ -9,13 +9,10 @@
 \SI{12.3(60)}{\m}
 
 \SI{10.0 \pm 3.3}{\ms}
-
-\SI{10.0 +- 3.3}{\ms}
 ^D
 <p>23 ± 2 m</p>
 <p>125 ± 12 m</p>
 <p>0.135 ± 0.021 m</p>
 <p>12.3 ± 6 m</p>
 <p>10.0 ± 3.3 ms</p>
-<p>10.0 ± 3.3 ms</p>
 ```
diff --git a/test/command/6658.md b/test/command/6658.md
new file mode 100644
index 000000000..bcd174465
--- /dev/null
+++ b/test/command/6658.md
@@ -0,0 +1,9 @@
+```
+pandoc -f latex
+\SI{10.0 +- 3.3}{\ms}
+
+\num{.3e45}
+^D
+<p>10.0 ± 3.3 ms</p>
+<p>0.3 × 10<sup>45</sup></p>
+```
-- 
cgit v1.2.3


From bc5058234feab7646f58dc01379b4eadf95bf411 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 20 May 2021 09:18:23 -0700
Subject: LaTeX reader sinuitx: fix + sign on ang.

---
 src/Text/Pandoc/Readers/LaTeX/SIunitx.hs | 9 ++++++---
 test/command/6658.md                     | 3 +++
 2 files changed, 9 insertions(+), 3 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs b/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs
index 1474329d4..72f81dcde 100644
--- a/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs
+++ b/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs
@@ -118,11 +118,14 @@ doSIang :: PandocMonad m => LP m Inlines
 doSIang = do
   skipopts
   ps <- T.splitOn ";" . untokenize <$> braced
+  let dropPlus t = case T.uncons t of
+                     Just ('+',t') -> t'
+                     _ -> t
   case ps ++ repeat "" of
     (d:m:s:_) -> return $
-      (if T.null d then mempty else str d <> str "\xb0") <>
-      (if T.null m then mempty else str m <> str "\x2032") <>
-      (if T.null s then mempty else str s <> str "\x2033")
+      (if T.null d then mempty else str (dropPlus d) <> str "\xb0") <>
+      (if T.null m then mempty else str (dropPlus m) <> str "\x2032") <>
+      (if T.null s then mempty else str (dropPlus s) <> str "\x2033")
     _ -> return mempty
 
 -- converts e.g. \SIrange{100}{200}{\ms} to "100 ms--200 ms"
diff --git a/test/command/6658.md b/test/command/6658.md
index bcd174465..0a8512f85 100644
--- a/test/command/6658.md
+++ b/test/command/6658.md
@@ -3,7 +3,10 @@ pandoc -f latex
 \SI{10.0 +- 3.3}{\ms}
 
 \num{.3e45}
+
+\ang{+10;+3;}
 ^D
 <p>10.0 ± 3.3 ms</p>
 <p>0.3 × 10<sup>45</sup></p>
+<p>10°3′</p>
 ```
-- 
cgit v1.2.3


From 4e990a8cf9207f2315d6a55a45c93c2857663316 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 20 May 2021 10:12:44 -0700
Subject: LaTeX/siunitx: fix parsing of `\cubic` etc. See #6658.

---
 src/Text/Pandoc/Readers/LaTeX/SIunitx.hs | 85 +++++++++++++++++++-------------
 test/command/6658.md                     |  3 ++
 2 files changed, 53 insertions(+), 35 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs b/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs
index 72f81dcde..63ab7267d 100644
--- a/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs
+++ b/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs
@@ -1,3 +1,4 @@
+{-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE OverloadedStrings #-}
 module Text.Pandoc.Readers.LaTeX.SIunitx
   ( siunitxCommands )
@@ -154,40 +155,55 @@ doSIrange includeUnits tok = do
 emptyOr160 :: Inlines -> Inlines
 emptyOr160 x = if x == mempty then x else str "\160"
 
-siUnit :: PandocMonad m => LP m Inlines -> LP m Inlines
-siUnit tok = try (do
-  Tok _ (CtrlSeq name) _ <- anyControlSeq
-  case name of
-    "square" -> do
-       unit <- siUnit tok
-       return $ unit <> superscript "2"
-    "cubic" -> do
-       unit <- siUnit tok
-       return $ unit <> superscript "3"
-    "raisetothe" -> do
-       n <- tok
-       unit <- siUnit tok
-       return $ unit <> superscript n
-    _ ->
-       case M.lookup name siUnitMap of
-            Just il ->
-              option il $
-                choice
-                 [ (il <> superscript "2") <$ controlSeq "squared"
-                 , (il <> superscript "3") <$ controlSeq "cubed"
-                 , (\n -> il <> superscript n) <$> (controlSeq "tothe" *> tok)
-                 ]
-            Nothing -> fail "not an siunit unit command")
- <|> (lookAhead anyControlSeq >> tok)
- <|> (do Tok _ Word t <- satisfyTok isWordTok
-         return $ str t)
- <|> (symbol '^' *> (superscript <$> tok))
- <|> (symbol '_' *> (subscript <$> tok))
- <|> ("\xa0" <$ symbol '.')
- <|> ("\xa0" <$ symbol '~')
- <|> tok
- <|> (do Tok _ _ t <- anyTok
-         return (str t))
+siUnit :: forall m. PandocMonad m => LP m Inlines -> LP m Inlines
+siUnit tok = mconcat <$> many1 siUnitPart
+ where
+  siUnitPart :: LP m Inlines
+  siUnitPart =
+        (siPrefix <*> siUnitPart)
+    <|> (do u <- siBase <|> tok
+            option u $ siSuffix <*> pure u)
+  siPrefix :: LP m (Inlines -> Inlines)
+  siPrefix =
+       (do _ <- controlSeq "per"
+           skipopts -- TODO handle option
+           return (str "/" <>))
+   <|> (do _ <- controlSeq "square"
+           skipopts
+           return (<> superscript "2"))
+   <|> (do _ <- controlSeq "cubic"
+           skipopts
+           return (<> superscript "3"))
+   <|> (do _ <- controlSeq "raisetothe"
+           skipopts
+           n <- tok
+           return (<> superscript n))
+  siSuffix :: LP m (Inlines -> Inlines)
+  siSuffix =
+       (do _ <- controlSeq "squared"
+           skipopts
+           return (<> superscript "2"))
+   <|> (do _ <- controlSeq "cubed"
+           skipopts
+           return (<> superscript "3"))
+   <|> (do _ <- controlSeq "tothe"
+           skipopts
+           n <- tok
+           return (<> superscript n))
+  siBase :: LP m Inlines
+  siBase = mconcat <$> many1
+    ((try
+       (do Tok _ (CtrlSeq name) _ <- anyControlSeq
+           case M.lookup name siUnitMap of
+              Just il -> pure il
+              Nothing -> fail "not a unit command"))
+    <|> (do Tok _ Word t <- satisfyTok isWordTok
+            return $ str t)
+    <|> (symbol '^' *> (superscript <$> tok))
+    <|> (symbol '_' *> (subscript <$> tok))
+    <|> (str "\xa0" <$ symbol '.')
+    <|> (str "\xa0" <$ symbol '~')
+     )
 
 siUnitMap :: M.Map Text Inlines
 siUnitMap = M.fromList
@@ -347,7 +363,6 @@ siUnitMap = M.fromList
   , ("Pa", str "Pa")
   , ("pascal", str "Pa")
   , ("percent", str "%")
-  , ("per", str "/")
   , ("peta", str "P")
   , ("pico", str "p")
   , ("planckbar", emph (str "\x210f"))
diff --git a/test/command/6658.md b/test/command/6658.md
index 0a8512f85..549610992 100644
--- a/test/command/6658.md
+++ b/test/command/6658.md
@@ -5,8 +5,11 @@ pandoc -f latex
 \num{.3e45}
 
 \ang{+10;+3;}
+
+\si{\gram\per\cubic\centi\metre}
 ^D
 <p>10.0 ± 3.3 ms</p>
 <p>0.3 × 10<sup>45</sup></p>
 <p>10°3′</p>
+<p>g/cm<sup>3</sup></p>
 ```
-- 
cgit v1.2.3


From bb11f5fb86993559f9999d4795355b02ae78cc3d Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 20 May 2021 12:06:15 -0700
Subject: LaTeX reader: More siunitx improvements.  Closes #6658.

There's still one slight divergence from the siunitx behavior:
we get 'kg m/A/s' instead of 'kg m/(A s)'. At the moment I'm
not going to worry about that.
---
 src/Text/Pandoc/Readers/LaTeX/Parsing.hs |   3 +-
 src/Text/Pandoc/Readers/LaTeX/SIunitx.hs | 138 +++++++++++++++++++++----------
 test/command/6658.md                     |  72 ++++++++++++++--
 3 files changed, 161 insertions(+), 52 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/LaTeX/Parsing.hs b/src/Text/Pandoc/Readers/LaTeX/Parsing.hs
index b6804a825..1c77eb299 100644
--- a/src/Text/Pandoc/Readers/LaTeX/Parsing.hs
+++ b/src/Text/Pandoc/Readers/LaTeX/Parsing.hs
@@ -806,7 +806,8 @@ withRaw parser = do
 
 keyval :: PandocMonad m => LP m (Text, Text)
 keyval = try $ do
-  Tok _ Word key <- satisfyTok isWordTok
+  key <- untokenize <$> many1 (notFollowedBy (symbol '=') >>
+                         (symbol '-' <|> symbol '_' <|> satisfyTok isWordTok))
   sp
   val <- option mempty $ do
            symbol '='
diff --git a/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs b/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs
index 63ab7267d..b8bf0ce7f 100644
--- a/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs
+++ b/src/Text/Pandoc/Readers/LaTeX/SIunitx.hs
@@ -10,27 +10,32 @@ import Text.Pandoc.Class
 import Text.Pandoc.Parsing hiding (blankline, mathDisplay, mathInline,
                             optional, space, spaces, withRaw, (<|>))
 import Control.Applicative ((<|>))
+import Control.Monad (void)
 import qualified Data.Map as M
 import Data.Char (isDigit)
 import Data.Text (Text)
 import qualified Data.Text as T
 import Data.List (intersperse)
-
+import qualified Data.Sequence as Seq
+import Text.Pandoc.Walk (walk)
 
 siunitxCommands :: PandocMonad m
                  => LP m Inlines -> M.Map Text (LP m Inlines)
 siunitxCommands tok = M.fromList
-  [ ("si", skipopts *> dosi tok)
+  [ ("si", dosi tok)
   , ("SI", doSI tok)
   , ("SIrange", doSIrange True tok)
   , ("numrange", doSIrange False tok)
   , ("numlist", doSInumlist)
+  , ("SIlist", doSIlist tok)
   , ("num", doSInum)
   , ("ang", doSIang)
   ]
 
 dosi :: PandocMonad m => LP m Inlines -> LP m Inlines
-dosi tok = grouped (siUnit tok) <|> siUnit tok
+dosi tok = do
+  options <- option [] keyvals
+  grouped (siUnit options tok) <|> siUnit options tok
 
 -- converts e.g. \SI{1}[\$]{} to "$ 1" or \SI{1}{\euro} to "1 €"
 doSI :: PandocMonad m => LP m Inlines -> LP m Inlines
@@ -65,9 +70,29 @@ doSInumlist = do
              mconcat (intersperse (str "," <> space) (init xs)) <>
              text ", & " <> last xs
 
+doSIlist :: PandocMonad m => LP m Inlines -> LP m Inlines
+doSIlist tok = do
+  options <- option [] keyvals
+  nums <- map tonum . T.splitOn ";" . untokenize <$> braced
+  unit <- grouped (siUnit options tok) <|> siUnit options tok
+  let xs = map (<> (str "\xa0" <> unit)) nums
+  case xs of
+    []  -> return mempty
+    [x] -> return x
+    _   -> return $
+             mconcat (intersperse (str "," <> space) (init xs)) <>
+             text ", & " <> last xs
+
 parseNum :: Parser Text () Inlines
 parseNum = (mconcat <$> many parseNumPart) <* eof
 
+minus :: Text
+minus = "\x2212"
+
+hyphenToMinus :: Inline -> Inline
+hyphenToMinus (Str t) = Str (T.replace "-" minus t)
+hyphenToMinus x = x
+
 parseNumPart :: Parser Text () Inlines
 parseNumPart =
   parseDecimalNum <|>
@@ -83,7 +108,7 @@ parseNumPart =
     parseComma, parseI, parseX,
     parseExp, parseSpace :: Parser Text () Inlines
   parseDecimalNum = try $ do
-    pref <- option mempty $ (mempty <$ char '+') <|> ("\x2212" <$ char '-')
+    pref <- option mempty $ (mempty <$ char '+') <|> (minus <$ char '-')
     basenum' <- many1 (satisfy (\c -> isDigit c || c == '.'))
     let basenum = pref <> T.pack
                     (case basenum' of
@@ -155,20 +180,30 @@ doSIrange includeUnits tok = do
 emptyOr160 :: Inlines -> Inlines
 emptyOr160 x = if x == mempty then x else str "\160"
 
-siUnit :: forall m. PandocMonad m => LP m Inlines -> LP m Inlines
-siUnit tok = mconcat <$> many1 siUnitPart
+siUnit :: forall m. PandocMonad m => [(Text,Text)] -> LP m Inlines -> LP m Inlines
+siUnit options tok = mconcat . intersperse (str "\xa0") <$> many1 siUnitPart
  where
   siUnitPart :: LP m Inlines
-  siUnitPart =
-        (siPrefix <*> siUnitPart)
-    <|> (do u <- siBase <|> tok
-            option u $ siSuffix <*> pure u)
+  siUnitPart = try $ do
+    skipMany (void (symbol '.') <|> void (symbol '~') <|> spaces1)
+    x <- ((siPrefix <*> siBase)
+            <|> (do u <- siBase <|> tok
+                    option u $ siSuffix <*> pure u))
+    option x (siInfix x)
+  siInfix :: Inlines -> LP m Inlines
+  siInfix u1 = try $
+       (do _ <- controlSeq "per"
+           u2 <- siUnitPart
+           let useSlash = lookup "per-mode" options == Just "symbol"
+           if useSlash
+              then return (u1 <> str "/" <> u2)
+              else return (u1 <> str "\xa0" <> negateExponent u2))
+   <|> (do _ <- symbol '/'
+           u2 <- siUnitPart
+           return (u1 <> str "/" <> u2))
   siPrefix :: LP m (Inlines -> Inlines)
   siPrefix =
-       (do _ <- controlSeq "per"
-           skipopts -- TODO handle option
-           return (str "/" <>))
-   <|> (do _ <- controlSeq "square"
+       (do _ <- controlSeq "square"
            skipopts
            return (<> superscript "2"))
    <|> (do _ <- controlSeq "cubic"
@@ -176,7 +211,7 @@ siUnit tok = mconcat <$> many1 siUnitPart
            return (<> superscript "3"))
    <|> (do _ <- controlSeq "raisetothe"
            skipopts
-           n <- tok
+           n <- walk hyphenToMinus <$> tok
            return (<> superscript n))
   siSuffix :: LP m (Inlines -> Inlines)
   siSuffix =
@@ -188,23 +223,57 @@ siUnit tok = mconcat <$> many1 siUnitPart
            return (<> superscript "3"))
    <|> (do _ <- controlSeq "tothe"
            skipopts
-           n <- tok
+           n <- walk hyphenToMinus <$> tok
            return (<> superscript n))
+   <|> (symbol '^' *> (do n <- walk hyphenToMinus <$> tok
+                          return (<> superscript n)))
+   <|> (symbol '_' *> (do n <- walk hyphenToMinus <$> tok
+                          return (<> subscript n)))
+  negateExponent :: Inlines -> Inlines
+  negateExponent ils =
+    case Seq.viewr (unMany ils) of
+      xs Seq.:> Superscript ss -> (Many xs) <>
+                                     superscript (str minus <> fromList ss)
+      _ -> ils <> superscript (str (minus <> "1"))
   siBase :: LP m Inlines
-  siBase = mconcat <$> many1
+  siBase =
     ((try
        (do Tok _ (CtrlSeq name) _ <- anyControlSeq
-           case M.lookup name siUnitMap of
-              Just il -> pure il
-              Nothing -> fail "not a unit command"))
+           case M.lookup name siUnitModifierMap of
+              Just il -> (il <>) <$> siBase
+              Nothing ->
+                case M.lookup name siUnitMap of
+                   Just il -> pure il
+                   Nothing -> fail "not a unit command"))
     <|> (do Tok _ Word t <- satisfyTok isWordTok
             return $ str t)
-    <|> (symbol '^' *> (superscript <$> tok))
-    <|> (symbol '_' *> (subscript <$> tok))
-    <|> (str "\xa0" <$ symbol '.')
-    <|> (str "\xa0" <$ symbol '~')
      )
 
+siUnitModifierMap :: M.Map Text Inlines
+siUnitModifierMap = M.fromList
+  [ ("atto", str "a")
+  , ("centi", str "c")
+  , ("deca", str "d")
+  , ("deci", str "d")
+  , ("deka", str "d")
+  , ("exa", str "E")
+  , ("femto", str "f")
+  , ("giga", str "G")
+  , ("hecto", str "h")
+  , ("kilo", str "k")
+  , ("mega", str "M")
+  , ("micro", str "μ")
+  , ("milli", str "m")
+  , ("nano", str "n")
+  , ("peta", str "P")
+  , ("pico", str "p")
+  , ("tera", str "T")
+  , ("yocto", str "y")
+  , ("yotta", str "Y")
+  , ("zepto", str "z")
+  , ("zetta", str "Z")
+  ]
+
 siUnitMap :: M.Map Text Inlines
 siUnitMap = M.fromList
   [ ("fg", str "fg")
@@ -303,7 +372,6 @@ siUnitMap = M.fromList
   , ("arcsecond", str "″")
   , ("astronomicalunit", str "ua")
   , ("atomicmassunit", str "u")
-  , ("atto", str "a")
   , ("bar", str "bar")
   , ("barn", str "b")
   , ("becquerel", str "Bq")
@@ -311,51 +379,38 @@ siUnitMap = M.fromList
   , ("bohr", emph (str "a") <> subscript (str "0"))
   , ("candela", str "cd")
   , ("celsius", str "°C")
-  , ("centi", str "c")
   , ("clight", emph (str "c") <> subscript (str "0"))
   , ("coulomb", str "C")
   , ("dalton", str "Da")
   , ("day", str "d")
-  , ("deca", str "d")
-  , ("deci", str "d")
   , ("decibel", str "db")
   , ("degreeCelsius",str "°C")
   , ("degree", str "°")
-  , ("deka", str "d")
   , ("electronmass", emph (str "m") <> subscript (str "e"))
   , ("electronvolt", str "eV")
   , ("elementarycharge", emph (str "e"))
-  , ("exa", str "E")
   , ("farad", str "F")
-  , ("femto", str "f")
-  , ("giga", str "G")
   , ("gram", str "g")
   , ("gray", str "Gy")
   , ("hartree", emph (str "E") <> subscript (str "h"))
   , ("hectare", str "ha")
-  , ("hecto", str "h")
   , ("henry", str "H")
   , ("hertz", str "Hz")
   , ("hour", str "h")
   , ("joule", str "J")
   , ("katal", str "kat")
   , ("kelvin", str "K")
-  , ("kilo", str "k")
   , ("kilogram", str "kg")
   , ("knot", str "kn")
   , ("liter", str "L")
   , ("litre", str "l")
   , ("lumen", str "lm")
   , ("lux", str "lx")
-  , ("mega", str "M")
   , ("meter", str "m")
   , ("metre", str "m")
-  , ("micro", str "μ")
-  , ("milli", str "m")
   , ("minute", str "min")
   , ("mmHg", str "mmHg")
   , ("mole", str "mol")
-  , ("nano", str "n")
   , ("nauticalmile", str "M")
   , ("neper", str "Np")
   , ("newton", str "N")
@@ -363,24 +418,17 @@ siUnitMap = M.fromList
   , ("Pa", str "Pa")
   , ("pascal", str "Pa")
   , ("percent", str "%")
-  , ("peta", str "P")
-  , ("pico", str "p")
   , ("planckbar", emph (str "\x210f"))
   , ("radian", str "rad")
   , ("second", str "s")
   , ("siemens", str "S")
   , ("sievert", str "Sv")
   , ("steradian", str "sr")
-  , ("tera", str "T")
   , ("tesla", str "T")
   , ("tonne", str "t")
   , ("volt", str "V")
   , ("watt", str "W")
   , ("weber", str "Wb")
-  , ("yocto", str "y")
-  , ("yotta", str "Y")
-  , ("zepto", str "z")
-  , ("zetta", str "Z")
   ]
 
 
diff --git a/test/command/6658.md b/test/command/6658.md
index 549610992..96700c8fe 100644
--- a/test/command/6658.md
+++ b/test/command/6658.md
@@ -1,15 +1,75 @@
 ```
-pandoc -f latex
-\SI{10.0 +- 3.3}{\ms}
+pandoc -f latex -t html
+\num{12345,67890}
+
+\num{1+-2i}
 
 \num{.3e45}
 
-\ang{+10;+3;}
+\num{1.654 x 2.34 x 3.430}
+
+\si{kg.m.s^{-1}}
+
+\si{\kilogram\metre\per\second}
+
+\si[per-mode=symbol]{\kilogram\metre\per\second}
+
+\si[per-mode=symbol]{\kilogram\metre\per\ampere\per\second}
+
+\numlist{10;20;30}
+
+\SIlist{0.13;0.67;0.80}{\milli\metre}
+
+\numrange{10}{20}
+
+\SIrange{0.13}{0.67}{\milli\metre}
+
+\ang{10}
+
+\ang{1;2;3}
+
+\ang{;;1}
+
+\ang{+10;;}
+
+\ang{-0;1;}
+
+\si{kg.m/s^2}
+
+\si{g_{polymer}~mol_{cat}.s^{-1}}
+
+\si{\kilo\gram\metre\per\square\second}
 
 \si{\gram\per\cubic\centi\metre}
+
+\si{\square\volt\cubic\lumen\per\farad}
+
+\si{\metre\squared\per\gray\cubic\lux}
+
+\si{\henry\second}
 ^D
-<p>10.0 ± 3.3 ms</p>
+<p>12345.67890</p>
+<p>1 ± 2i</p>
 <p>0.3 × 10<sup>45</sup></p>
-<p>10°3′</p>
-<p>g/cm<sup>3</sup></p>
+<p>1.654 × 2.34 × 3.430</p>
+<p>kg m s<sup>−1</sup></p>
+<p>kg m s<sup>−1</sup></p>
+<p>kg m/s</p>
+<p>kg m/A/s</p>
+<p>10, 20, &amp; 30</p>
+<p>0.13 mm, 0.67 mm, &amp; 0.80 mm</p>
+<p>10–20</p>
+<p>0.13 mm–0.67 mm</p>
+<p>10°</p>
+<p>1°2′3″</p>
+<p>1″</p>
+<p>10°</p>
+<p>-0°1′</p>
+<p>kg m/s<sup>2</sup></p>
+<p>g<sub>polymer</sub> mol<sub>cat</sub> s<sup>−1</sup></p>
+<p>kg m s<sup>−2</sup></p>
+<p>g cm<sup>−3</sup></p>
+<p>V<sup>2</sup> lm<sup>3</sup> F<sup>−1</sup></p>
+<p>m<sup>2</sup> Gy<sup>−1</sup> lx<sup>3</sup></p>
+<p>H s</p>
 ```
-- 
cgit v1.2.3


From d7b5def287aefe91f881daeecc5f72121c843b66 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 20 May 2021 17:12:00 -0700
Subject: Ms writer: handle tables with multiple paragraphs.

Previously they overflowed the table cell width.
We now set line lengths per-cell and restore them
after the table has been written.

Closes #7288.
---
 src/Text/Pandoc/Writers/Ms.hs | 28 ++++++++++++++++++++++------
 test/command/7288.md          | 40 ++++++++++++++++++++++++++++++++++++++++
 test/tables.ms                | 30 ++++++++++++++++++++++++++++++
 3 files changed, 92 insertions(+), 6 deletions(-)
 create mode 100644 test/command/7288.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/Ms.hs b/src/Text/Pandoc/Writers/Ms.hs
index 0ed7a8a64..97c23f24d 100644
--- a/src/Text/Pandoc/Writers/Ms.hs
+++ b/src/Text/Pandoc/Writers/Ms.hs
@@ -245,13 +245,17 @@ blockToMs opts (Table _ blkCapt specs thead tbody tfoot) =
       aligncode AlignDefault = "l"
   in do
   caption' <- inlineListToMs' opts caption
-  let iwidths = if all (== 0) widths
-                   then repeat ""
-                   else map (T.pack . printf "w(%0.1fn)" . (70 *)) widths
+  let isSimple = all (== 0) widths
+  let totalWidth = 70
   -- 78n default width - 8n indent = 70n
   let coldescriptions = literal $ T.unwords
-                        (zipWith (\align width -> aligncode align <> width)
-                        alignments iwidths) <> "."
+                        (zipWith (\align width -> aligncode align <>
+                                    if width == 0
+                                       then ""
+                                       else T.pack $
+                                              printf "w(%0.1fn)"
+                                              (totalWidth * width))
+                        alignments widths) <> "."
   colheadings <- mapM (blockListToMs opts) headers
   let makeRow cols = literal "T{" $$
                      vcat (intersperse (literal "T}\tT{") cols) $$
@@ -260,13 +264,25 @@ blockToMs opts (Table _ blkCapt specs thead tbody tfoot) =
                         then empty
                         else makeRow colheadings $$ char '_'
   body <- mapM (\row -> do
-                         cols <- mapM (blockListToMs opts) row
+                         cols <- mapM (\(cell, w) ->
+                                   (if isSimple
+                                       then id
+                                       else (literal (".nr LL " <>
+                                              T.pack (printf "%0.1fn"
+                                                (w * totalWidth))) $$)) <$>
+                                   blockListToMs opts cell) (zip row widths)
                          return $ makeRow cols) rows
   setFirstPara
   return $ literal ".PP" $$ caption' $$
            literal ".na" $$ -- we don't want justification in table cells
+           (if isSimple
+               then ""
+               else ".nr LLold \\n[LL]") $$
            literal ".TS" $$ literal "delim(@@) tab(\t);" $$ coldescriptions $$
            colheadings' $$ vcat body $$ literal ".TE" $$
+           (if isSimple
+               then ""
+               else ".nr LL \\n[LLold]") $$
            literal ".ad"
 
 blockToMs opts (BulletList items) = do
diff --git a/test/command/7288.md b/test/command/7288.md
new file mode 100644
index 000000000..e94aeeeb3
--- /dev/null
+++ b/test/command/7288.md
@@ -0,0 +1,40 @@
+```
+% pandoc -f rst -t ms
+.. list-table::
+   :widths: 50 50
+   :header-rows: 1
+
+   * - Left
+     - Right
+   * - Long text that should be easy to break up into multiple lines
+     - Another long text that should be easy to break up into multiple lines
+
+       Bar
+^D
+.PP
+.na
+.nr LLold \n[LL]
+.TS
+delim(@@) tab(	);
+lw(35.0n) lw(35.0n).
+T{
+Left
+T}	T{
+Right
+T}
+_
+T{
+.nr LL 35.0n
+.LP
+Long text that should be easy to break up into multiple lines
+T}	T{
+.nr LL 35.0n
+.PP
+Another long text that should be easy to break up into multiple lines
+.PP
+Bar
+T}
+.TE
+.nr LL \n[LLold]
+.ad
+```
diff --git a/test/tables.ms b/test/tables.ms
index 7337db25f..21183a5eb 100644
--- a/test/tables.ms
+++ b/test/tables.ms
@@ -143,6 +143,7 @@ Multiline table with caption:
 .PP
 Here\[cq]s the caption. It may span multiple lines.
 .na
+.nr LLold \n[LL]
 .TS
 delim(@@) tab(	);
 cw(10.5n) lw(9.6n) rw(11.4n) lw(24.5n).
@@ -157,30 +158,40 @@ Default aligned
 T}
 _
 T{
+.nr LL 10.5n
 First
 T}	T{
+.nr LL 9.6n
 row
 T}	T{
+.nr LL 11.4n
 12.0
 T}	T{
+.nr LL 24.5n
 Example of a row that spans multiple lines.
 T}
 T{
+.nr LL 10.5n
 Second
 T}	T{
+.nr LL 9.6n
 row
 T}	T{
+.nr LL 11.4n
 5.0
 T}	T{
+.nr LL 24.5n
 Here\[cq]s another one.
 Note the blank line between rows.
 T}
 .TE
+.nr LL \n[LLold]
 .ad
 .LP
 Multiline table without caption:
 .PP
 .na
+.nr LLold \n[LL]
 .TS
 delim(@@) tab(	);
 cw(10.5n) lw(9.6n) rw(11.4n) lw(24.5n).
@@ -195,25 +206,34 @@ Default aligned
 T}
 _
 T{
+.nr LL 10.5n
 First
 T}	T{
+.nr LL 9.6n
 row
 T}	T{
+.nr LL 11.4n
 12.0
 T}	T{
+.nr LL 24.5n
 Example of a row that spans multiple lines.
 T}
 T{
+.nr LL 10.5n
 Second
 T}	T{
+.nr LL 9.6n
 row
 T}	T{
+.nr LL 11.4n
 5.0
 T}	T{
+.nr LL 24.5n
 Here\[cq]s another one.
 Note the blank line between rows.
 T}
 .TE
+.nr LL \n[LLold]
 .ad
 .LP
 Table without column headers:
@@ -255,27 +275,37 @@ T}
 Multiline table without column headers:
 .PP
 .na
+.nr LLold \n[LL]
 .TS
 delim(@@) tab(	);
 cw(10.5n) lw(9.6n) rw(11.4n) lw(24.5n).
 T{
+.nr LL 10.5n
 First
 T}	T{
+.nr LL 9.6n
 row
 T}	T{
+.nr LL 11.4n
 12.0
 T}	T{
+.nr LL 24.5n
 Example of a row that spans multiple lines.
 T}
 T{
+.nr LL 10.5n
 Second
 T}	T{
+.nr LL 9.6n
 row
 T}	T{
+.nr LL 11.4n
 5.0
 T}	T{
+.nr LL 24.5n
 Here\[cq]s another one.
 Note the blank line between rows.
 T}
 .TE
+.nr LL \n[LLold]
 .ad
-- 
cgit v1.2.3


From 07d299d353761a7c29aa7e7a51371ad7842ec767 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 20 May 2021 18:45:39 -0700
Subject: DocBook reader: ensure that first and last names are separated.

Closes #6541.
---
 src/Text/Pandoc/Readers/DocBook.hs | 20 ++++++++++++++------
 test/command/6541.md               | 27 +++++++++++++++++++++++++++
 2 files changed, 41 insertions(+), 6 deletions(-)
 create mode 100644 test/command/6541.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/DocBook.hs b/src/Text/Pandoc/Readers/DocBook.hs
index 3db459cfd..b01ad3252 100644
--- a/src/Text/Pandoc/Readers/DocBook.hs
+++ b/src/Text/Pandoc/Readers/DocBook.hs
@@ -600,16 +600,24 @@ addMetadataFromElement e = do
          Nothing -> return ()
          Just z  -> addMetaField "author" z
     addMetaField "subtitle" e
-    addMetaField "author" e
+    addAuthor e
     addMetaField "date" e
     addMetaField "release" e
     addMetaField "releaseinfo" e
     return mempty
-  where addMetaField fieldname elt =
-            case filterChildren (named fieldname) elt of
-                   []  -> return ()
-                   [z] -> getInlines z >>= addMeta fieldname
-                   zs  -> mapM getInlines zs >>= addMeta fieldname
+  where
+   addAuthor elt =
+     case filterChildren (named "author") elt of
+       [] -> return ()
+       [z] -> fromAuthor z >>= addMeta "author"
+       zs  -> mapM fromAuthor zs >>= addMeta "author"
+   fromAuthor elt =
+     mconcat . intersperse space <$> mapM getInlines (elChildren elt)
+   addMetaField fieldname elt =
+     case filterChildren (named fieldname) elt of
+       []  -> return ()
+       [z] -> getInlines z >>= addMeta fieldname
+       zs  -> mapM getInlines zs >>= addMeta fieldname
 
 addMeta :: PandocMonad m => ToMetaValue a => Text -> a -> DB m ()
 addMeta field val = modify (setMeta field val)
diff --git a/test/command/6541.md b/test/command/6541.md
new file mode 100644
index 000000000..956340d4c
--- /dev/null
+++ b/test/command/6541.md
@@ -0,0 +1,27 @@
+```
+% pandoc -f docbook -t markdown -s
+<?xml version="1.0"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
+  "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
+<book>
+
+<bookinfo>
+<title>Title</title>
+<author>
+<firstname>Firstname</firstname><surname>Lastname</surname>
+</author>
+<releaseinfo>1.17</releaseinfo>
+</bookinfo>
+
+<para>Text.</para>
+
+</book>
+^D
+---
+author: Firstname Lastname
+releaseinfo: 1.17
+title: Title
+---
+
+Text.
+```
-- 
cgit v1.2.3


From 1af2cfb2873c5bb6ddd9fc00d076088b2e62af30 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sat, 22 May 2021 21:56:10 -0700
Subject: Handle relative lengths (e.g. `2*`) in HTML column widths.

See <https://www.w3.org/TR/html4/types.html#h-6.6>.

"A relative length has the form "i*", where "i" is an integer. When
allotting space among elements competing for that space, user agents
allot pixel and percentage lengths first, then divide up remaining
available space among relative lengths. Each relative length receives a
portion of the available space that is proportional to the integer
preceding the "*". The value "*" is equivalent to "1*". Thus, if 60
pixels of space are available after the user agent allots pixel and
percentage space, and the competing relative lengths are 1*, 2*, and 3*,
the 1* will be alloted 10 pixels, the 2* will be alloted 20 pixels, and
the 3* will be alloted 30 pixels."

Closes #4063.
---
 src/Text/Pandoc/Readers/HTML/Table.hs | 47 ++++++++++++++++++++++++-----------
 test/command/4063.md                  | 29 +++++++++++++++++++++
 2 files changed, 62 insertions(+), 14 deletions(-)
 create mode 100644 test/command/4063.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/HTML/Table.hs b/src/Text/Pandoc/Readers/HTML/Table.hs
index ad0b51253..3a569dd0a 100644
--- a/src/Text/Pandoc/Readers/HTML/Table.hs
+++ b/src/Text/Pandoc/Readers/HTML/Table.hs
@@ -17,6 +17,7 @@ module Text.Pandoc.Readers.HTML.Table (pTable) where
 
 import Control.Applicative ((<|>))
 import Data.Maybe (fromMaybe)
+import Data.Either (lefts, rights)
 import Data.List.NonEmpty (nonEmpty)
 import Data.Text (Text)
 import Text.HTML.TagSoup
@@ -33,34 +34,51 @@ import Text.Pandoc.Shared (onlySimpleTableCells, safeRead)
 import qualified Data.Text as T
 import qualified Text.Pandoc.Builder as B
 
--- | Parses a @<col>@ element, returning the column's width. Defaults to
--- @'ColWidthDefault'@ if the width is not set or cannot be determined.
-pCol :: PandocMonad m => TagParser m ColWidth
+-- | Parses a @<col>@ element, returning the column's width.
+-- An Either value is used:  Left i means a "relative length" with
+-- integral value i (see https://www.w3.org/TR/html4/types.html#h-6.6);
+-- Right w means a regular width.  Defaults to @'Right ColWidthDefault'@
+-- if the width is not set or cannot be determined.
+pCol :: PandocMonad m => TagParser m (Either Int ColWidth)
 pCol = try $ do
   TagOpen _ attribs' <- pSatisfy (matchTagOpen "col" [])
   let attribs = toStringAttr attribs'
   skipMany pBlank
   optional $ pSatisfy (matchTagClose "col")
   skipMany pBlank
-  let width = case lookup "width" attribs of
+  return $ case lookup "width" attribs of
                 Nothing -> case lookup "style" attribs of
                   Just (T.stripPrefix "width:" -> Just xs) | T.any (== '%') xs ->
-                    fromMaybe 0.0 $ safeRead (T.filter
-                      (`notElem` (" \t\r\n%'\";" :: [Char])) xs)
-                  _ -> 0.0
+                    maybe (Right ColWidthDefault) (Right . ColWidth)
+                      $ safeRead (T.filter
+                                   (`notElem` (" \t\r\n%'\";" :: [Char])) xs)
+                  _ -> Right ColWidthDefault
+                Just (T.unsnoc -> Just (xs, '*')) ->
+                  maybe (Left 1) Left $ safeRead xs
                 Just (T.unsnoc -> Just (xs, '%')) ->
-                  fromMaybe 0.0 $ safeRead xs
-                _ -> 0.0
-  if width > 0.0
-    then return $ ColWidth $ width / 100.0
-    else return ColWidthDefault
+                  maybe (Right ColWidthDefault)
+                        (Right . ColWidth . (/ 100.0)) $ safeRead xs
+                _ -> Right ColWidthDefault
 
-pColgroup :: PandocMonad m => TagParser m [ColWidth]
+pColgroup :: PandocMonad m => TagParser m [Either Int ColWidth]
 pColgroup = try $ do
   pSatisfy (matchTagOpen "colgroup" [])
   skipMany pBlank
   manyTill pCol (pCloses "colgroup" <|> eof) <* skipMany pBlank
 
+resolveRelativeLengths :: [Either Int ColWidth] -> [ColWidth]
+resolveRelativeLengths ws =
+  let remaining = 1 - sum (map getColWidth $ rights ws)
+      relatives = sum $ lefts ws
+      relUnit = remaining / fromIntegral relatives
+      toColWidth (Right x) = x
+      toColWidth (Left i) = ColWidth (fromIntegral i * relUnit)
+  in  map toColWidth ws
+
+getColWidth :: ColWidth -> Double
+getColWidth ColWidthDefault = 0
+getColWidth (ColWidth w) = w
+
 data CellType
   = HeaderCell
   | BodyCell
@@ -182,7 +200,8 @@ pTable :: PandocMonad m
 pTable block = try $ do
   TagOpen _ attribs <- pSatisfy (matchTagOpen "table" [])  <* skipMany pBlank
   caption <- option mempty $ pInTags "caption" block       <* skipMany pBlank
-  widths  <- ((mconcat <$> many1 pColgroup) <|> many pCol) <* skipMany pBlank
+  widths <- resolveRelativeLengths <$>
+              ((mconcat <$> many1 pColgroup) <|> many pCol) <* skipMany pBlank
   thead   <- pTableHead block               <* skipMany pBlank
   topfoot <- optionMaybe (pTableFoot block) <* skipMany pBlank
   tbodies <- many (pTableBody block)        <* skipMany pBlank
diff --git a/test/command/4063.md b/test/command/4063.md
new file mode 100644
index 000000000..838472b46
--- /dev/null
+++ b/test/command/4063.md
@@ -0,0 +1,29 @@
+```
+% pandoc -f html -t native
+<table>
+<colgroup>
+  <col width="30%" />
+  <col width="*" />
+</colgroup>
+<tr>
+  <td>1</td>
+  <td>2</td>
+</tr>
+</table>
+^D
+[Table ("",[],[]) (Caption Nothing
+ [])
+ [(AlignDefault,ColWidth 0.3)
+ ,(AlignDefault,ColWidth 0.7)]
+ (TableHead ("",[],[])
+ [])
+ [(TableBody ("",[],[]) (RowHeadColumns 0)
+  []
+  [Row ("",[],[])
+   [Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
+    [Plain [Str "1"]]
+   ,Cell ("",[],[]) AlignDefault (RowSpan 1) (ColSpan 1)
+    [Plain [Str "2"]]]])]
+ (TableFoot ("",[],[])
+ [])]
+```
-- 
cgit v1.2.3


From 8d5014fdfc57b80be54a3d23358e92c3b45a7e7d Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Tue, 25 May 2021 10:36:51 -0700
Subject: Logging: remove single quotes around paths in messages.

We weren't doing it consistently and it seems unnecessary.
---
 src/Text/Pandoc/Logging.hs | 12 ++++++------
 test/command/7099.md       |  2 +-
 test/command/svg.md        |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Logging.hs b/src/Text/Pandoc/Logging.hs
index 2642d72ac..193b8b61c 100644
--- a/src/Text/Pandoc/Logging.hs
+++ b/src/Text/Pandoc/Logging.hs
@@ -282,7 +282,7 @@ showLogMessage msg =
        ParsingUnescaped s pos ->
          "Parsing unescaped '" <> s <> "' at " <> showPos pos
        CouldNotLoadIncludeFile fp pos ->
-         "Could not load include file '" <> fp <> "' at " <> showPos pos
+         "Could not load include file " <> fp <> " at " <> showPos pos
        MacroAlreadyDefined name pos ->
          "Macro '" <> name <> "' already defined, ignoring at " <> showPos pos
        InlineNotRendered il ->
@@ -294,18 +294,18 @@ showLogMessage msg =
        IgnoredIOError s ->
          "IO Error (ignored): " <> s
        CouldNotFetchResource fp s ->
-         "Could not fetch resource '" <> fp <> "'" <>
+         "Could not fetch resource " <> fp <>
            if Text.null s then "" else ": " <> s
        CouldNotDetermineImageSize fp s ->
-         "Could not determine image size for '" <> fp <> "'" <>
+         "Could not determine image size for " <> fp <>
            if Text.null s then "" else ": " <> s
        CouldNotConvertImage fp s ->
-         "Could not convert image '" <> fp <> "'" <>
+         "Could not convert image " <> fp <>
            if Text.null s then "" else ": " <> s
        CouldNotDetermineMimeType fp ->
-         "Could not determine mime type for '" <> fp <> "'"
+         "Could not determine mime type for " <> fp
        CouldNotConvertTeXMath s m ->
-         "Could not convert TeX math '" <> s <> "', rendering as TeX" <>
+         "Could not convert TeX math " <> s <> ", rendering as TeX" <>
            if Text.null m then "" else ":\n" <> m
        CouldNotParseCSS m ->
          "Could not parse CSS" <> if Text.null m then "" else ":\n" <> m
diff --git a/test/command/7099.md b/test/command/7099.md
index 33ac8aea1..467b22a16 100644
--- a/test/command/7099.md
+++ b/test/command/7099.md
@@ -11,7 +11,7 @@
 <iframe src="h:invalid@url"></iframe>
 ^D
 [INFO] Fetching h:invalid@url...
-[WARNING] Could not fetch resource 'h:invalid@url': Could not fetch h:invalid@url
+[WARNING] Could not fetch resource h:invalid@url: Could not fetch h:invalid@url
   InvalidUrlException "h:invalid@url" "Invalid scheme"
 [INFO] Skipped '<iframe src="h:invalid@url"></iframe>' at input line 1 column 29
 []
diff --git a/test/command/svg.md b/test/command/svg.md
index 4ba836b20..57c99db33 100644
--- a/test/command/svg.md
+++ b/test/command/svg.md
@@ -2,7 +2,7 @@
 % pandoc -f latex -t icml
 \includegraphics{command/corrupt.svg}
 ^D
-[WARNING] Could not determine image size for 'command/corrupt.svg': could not determine image type
+[WARNING] Could not determine image size for command/corrupt.svg: could not determine image type
 <ParagraphStyleRange AppliedParagraphStyle="ParagraphStyle/Paragraph">
   <CharacterStyleRange AppliedCharacterStyle="$ID/NormalCharacterStyle">
     <Rectangle Self="uec" StrokeWeight="0" ItemTransform="1 0 0 1 150 -100">
-- 
cgit v1.2.3


From 6804f47383a954fc1fb48e44ab0383f9f72d5dea Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Tue, 25 May 2021 21:36:40 -0700
Subject: Fix a command test so it writes to stdout not stderr.

The error message to stderr was appearing in test output
and confusing some users, who thought it indicated a failing
test rather than expected output.
---
 test/command/defaults-inheritance-2.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'test/command')

diff --git a/test/command/defaults-inheritance-2.md b/test/command/defaults-inheritance-2.md
index 8b26a2613..c639655d3 100644
--- a/test/command/defaults-inheritance-2.md
+++ b/test/command/defaults-inheritance-2.md
@@ -1,5 +1,5 @@
 ```
-% pandoc -d command/defaults6
+% pandoc -d command/defaults6 2>&1
 ^D
 Error: Circular defaults file reference in 'command/defaults7.yaml'
 ```
-- 
cgit v1.2.3


From e0a1f7d2cfa1b18210ea13e0aa03747e6c76e5c5 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Wed, 26 May 2021 09:52:23 -0700
Subject: Command tests: fail if a file contains no tests.

And fix a test that failed in that way!
---
 test/Tests/Command.hs          | 5 ++++-
 test/command/biblatex-basic.md | 8 +++++---
 2 files changed, 9 insertions(+), 4 deletions(-)

(limited to 'test/command')

diff --git a/test/Tests/Command.hs b/test/Tests/Command.hs
index 3ec6d7230..539be1a1a 100644
--- a/test/Tests/Command.hs
+++ b/test/Tests/Command.hs
@@ -130,4 +130,7 @@ extractCommandTest testExePath fp = unsafePerformIO $ do
                         def{ readerExtensions = pandocExtensions } contents)
   let codeblocks = map extractCode $ filter isCodeBlock blocks
   let cases = zipWith (runCommandTest testExePath fp) [1..] codeblocks
-  return $ testGroup fp cases
+  return $ testGroup fp
+         $ if null cases
+              then [testCase "!!" $ assertFailure "No command tests defined"]
+              else cases
diff --git a/test/command/biblatex-basic.md b/test/command/biblatex-basic.md
index 1110b0fb4..8a1176999 100644
--- a/test/command/biblatex-basic.md
+++ b/test/command/biblatex-basic.md
@@ -1,3 +1,5 @@
+```
+% pandoc -f biblatex -t markdown -s
 @Book{item1,
 author="John Doe",
 title="First Book",
@@ -27,7 +29,7 @@ year="2007"
 
 ^D
 ---
-nocite: '[@*]'
+nocite: "[@*]"
 references:
 - author:
   - family: Doe
@@ -44,9 +46,9 @@ references:
   container-title: Journal of Generic Studies
   id: item2
   issued: 2006
-  page: '33-34'
+  page: 33-34
   title: Article
-  type: 'article-journal'
+  type: article-journal
   volume: 6
 - author:
   - family: Doe
-- 
cgit v1.2.3


From 81eadfd99ad3e905b806cc6c80ab0fea0185286f Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Wed, 26 May 2021 22:50:35 -0700
Subject: LaTeX reader: improve `\def` and implement `\newif`.

- Improve parsing of `\def` macros.  We previously set "verbatim mode"
  even for parsing the initial `\def`; this caused problems for things
  like
  ```
  \def\foo{\def\bar{BAR}}
  \foo
  \bar
  ```
- Implement `\newif`.
- Add tests.
---
 src/Text/Pandoc/Readers/LaTeX/Macro.hs   | 59 ++++++++++++++++++++++++--------
 src/Text/Pandoc/Readers/LaTeX/Parsing.hs | 19 +++++++++-
 test/command/newif.md                    | 55 +++++++++++++++++++++++++++++
 3 files changed, 118 insertions(+), 15 deletions(-)
 create mode 100644 test/command/newif.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/LaTeX/Macro.hs b/src/Text/Pandoc/Readers/LaTeX/Macro.hs
index 607f5438c..5495a8e74 100644
--- a/src/Text/Pandoc/Readers/LaTeX/Macro.hs
+++ b/src/Text/Pandoc/Readers/LaTeX/Macro.hs
@@ -14,6 +14,7 @@ import Text.Pandoc.Parsing hiding (blankline, mathDisplay, mathInline,
 import Control.Applicative ((<|>), optional)
 import qualified Data.Map as M
 import Data.Text (Text)
+import qualified Data.Text as T
 
 macroDef :: (PandocMonad m, Monoid a) => (Text -> a) -> LP m a
 macroDef constructor = do
@@ -22,9 +23,11 @@ macroDef constructor = do
       guardDisabled Ext_latex_macros)
      <|> return mempty
   where commandDef = do
-          (name, macro') <- newcommand <|> letmacro <|> defmacro
+          nameMacroPairs <- newcommand <|> letmacro <|> defmacro <|> newif
           guardDisabled Ext_latex_macros <|>
-           updateState (\s -> s{ sMacros = M.insert name macro' (sMacros s) })
+           mapM_ (\(name, macro') ->
+                   updateState (\s -> s{ sMacros = M.insert name macro'
+                                          (sMacros s) })) nameMacroPairs
         environmentDef = do
           mbenv <- newenvironment
           case mbenv of
@@ -40,7 +43,7 @@ macroDef constructor = do
         -- @\newcommand{\envname}[n-args][default]{begin}@
         -- @\newcommand{\endenvname}@
 
-letmacro :: PandocMonad m => LP m (Text, Macro)
+letmacro :: PandocMonad m => LP m [(Text, Macro)]
 letmacro = do
   controlSeq "let"
   (name, contents) <- withVerbatimMode $ do
@@ -53,18 +56,47 @@ letmacro = do
     contents <- bracedOrToken
     return (name, contents)
   contents' <- doMacros' 0 contents
-  return (name, Macro ExpandWhenDefined [] Nothing contents')
+  return [(name, Macro ExpandWhenDefined [] Nothing contents')]
 
-defmacro :: PandocMonad m => LP m (Text, Macro)
-defmacro = try $
+defmacro :: PandocMonad m => LP m [(Text, Macro)]
+defmacro = do
   -- we use withVerbatimMode, because macros are to be expanded
   -- at point of use, not point of definition
+  controlSeq "def"
   withVerbatimMode $ do
-    controlSeq "def"
     Tok _ (CtrlSeq name) _ <- anyControlSeq
     argspecs <- many (argspecArg <|> argspecPattern)
     contents <- bracedOrToken
-    return (name, Macro ExpandWhenUsed argspecs Nothing contents)
+    return [(name, Macro ExpandWhenUsed argspecs Nothing contents)]
+
+-- \newif\iffoo' defines:
+-- \iffoo to be \iffalse
+-- \footrue to be a command that defines \iffoo to be \iftrue
+-- \foofalse to be a command that defines \iffoo to be \iffalse
+newif :: PandocMonad m => LP m [(Text, Macro)]
+newif = do
+  controlSeq "newif"
+  withVerbatimMode $ do
+    Tok pos (CtrlSeq name) _ <- anyControlSeq
+    -- \def\iffoo\iffalse
+    -- \def\footrue{\def\iffoo\iftrue}
+    -- \def\foofalse{\def\iffoo\iffalse}
+    let base = T.drop 2 name
+    return [ (name, Macro ExpandWhenUsed [] Nothing
+                    [Tok pos (CtrlSeq "iffalse") "\\iffalse"])
+           , (base <> "true",
+                   Macro ExpandWhenUsed [] Nothing
+                   [ Tok pos (CtrlSeq "def") "\\def"
+                   , Tok pos (CtrlSeq name) ("\\" <> name)
+                   , Tok pos (CtrlSeq "iftrue") "\\iftrue"
+                   ])
+           , (base <> "false",
+                   Macro ExpandWhenUsed [] Nothing
+                   [ Tok pos (CtrlSeq "def") "\\def"
+                   , Tok pos (CtrlSeq name) ("\\" <> name)
+                   , Tok pos (CtrlSeq "iffalse") "\\iffalse"
+                   ])
+           ]
 
 argspecArg :: PandocMonad m => LP m ArgSpec
 argspecArg = do
@@ -77,10 +109,9 @@ argspecPattern =
                               (toktype' == Symbol || toktype' == Word) &&
                               (txt /= "{" && txt /= "\\" && txt /= "}")))
 
-newcommand :: PandocMonad m => LP m (Text, Macro)
+newcommand :: PandocMonad m => LP m [(Text, Macro)]
 newcommand = do
-  pos <- getPosition
-  Tok _ (CtrlSeq mtype) _ <- controlSeq "newcommand" <|>
+  Tok pos (CtrlSeq mtype) _ <- controlSeq "newcommand" <|>
                              controlSeq "renewcommand" <|>
                              controlSeq "providecommand" <|>
                              controlSeq "DeclareMathOperator" <|>
@@ -112,9 +143,9 @@ newcommand = do
         Just macro
           | mtype == "newcommand" -> do
               report $ MacroAlreadyDefined txt pos
-              return (name, macro)
-          | mtype == "providecommand" -> return (name, macro)
-        _ -> return (name, Macro ExpandWhenUsed argspecs optarg contents)
+              return [(name, macro)]
+          | mtype == "providecommand" -> return [(name, macro)]
+        _ -> return [(name, Macro ExpandWhenUsed argspecs optarg contents)]
 
 newenvironment :: PandocMonad m => LP m (Maybe (Text, Macro, Macro))
 newenvironment = do
diff --git a/src/Text/Pandoc/Readers/LaTeX/Parsing.hs b/src/Text/Pandoc/Readers/LaTeX/Parsing.hs
index 1c77eb299..a17b1f324 100644
--- a/src/Text/Pandoc/Readers/LaTeX/Parsing.hs
+++ b/src/Text/Pandoc/Readers/LaTeX/Parsing.hs
@@ -113,7 +113,6 @@ import Text.Pandoc.Readers.LaTeX.Types (ExpansionPoint (..), Macro (..),
                                         ArgSpec (..), Tok (..), TokType (..))
 import Text.Pandoc.Shared
 import Text.Parsec.Pos
--- import Debug.Trace
 
 newtype DottedNum = DottedNum [Int]
   deriving (Show, Eq)
@@ -563,8 +562,26 @@ trySpecialMacro "xspace" ts = do
     Tok pos Word t : _
       | startsWithAlphaNum t -> return $ Tok pos Spaces " " : ts'
     _ -> return ts'
+trySpecialMacro "iftrue" ts = handleIf True ts
+trySpecialMacro "iffalse" ts = handleIf False ts
 trySpecialMacro _ _ = mzero
 
+handleIf :: PandocMonad m => Bool -> [Tok] -> LP m [Tok]
+handleIf b ts = do
+  res' <- lift $ runParserT (ifParser b) defaultLaTeXState "tokens" ts
+  case res' of
+    Left _ -> Prelude.fail "Could not parse conditional"
+    Right ts' -> return ts'
+
+ifParser :: PandocMonad m => Bool -> LP m [Tok]
+ifParser b = do
+  ifToks <- many (notFollowedBy (controlSeq "else" <|> controlSeq "fi")
+                    *> anyTok)
+  elseToks <- (controlSeq "else" >> manyTill anyTok (controlSeq "fi"))
+                 <|> ([] <$ controlSeq "fi")
+  rest <- getInput
+  return $ (if b then ifToks else elseToks) ++ rest
+
 startsWithAlphaNum :: Text -> Bool
 startsWithAlphaNum t =
   case T.uncons t of
diff --git a/test/command/newif.md b/test/command/newif.md
new file mode 100644
index 000000000..f444f14c9
--- /dev/null
+++ b/test/command/newif.md
@@ -0,0 +1,55 @@
+```
+% pandoc -f latex -t plain
+\iftrue
+should print
+\iftrue
+should print
+\else
+should not print
+\fi
+\else
+should not print
+\fi
+
+\iffalse
+should not print
+\else
+\iftrue
+should print
+\else
+should not print
+\fi
+\fi
+
+\newif\ifepub
+
+\ifepub
+should not print
+\fi
+
+\epubtrue
+
+\ifepub
+should print
+\else
+should not print
+\fi
+
+\epubfalse
+
+\ifepub
+should not print
+\else
+should print
+\fi
+^D
+should print
+
+should print
+
+should print
+
+should print
+
+should print
+```
-- 
cgit v1.2.3


From 834da53058069fe50da510fa86e0807a7ff7868f Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Mon, 24 May 2021 10:17:37 -0700
Subject: Add `rebase_relative_paths` extension.

- Add manual entry for (non-default) extension
  `rebase_relative_paths`.
- Add constructor `Ext_rebase_relative_paths` to `Extensions`
  in Text.Pandoc.Extensions [API change]. When enabled, this
  extension rewrites relative image and link paths by prepending
  the (relative) directory of the containing file.
- Make Markdown reader sensitive to the new extension.
- Add tests for #3752.

Closes #3752.

NB. currently the extension applies to markdown and associated
readers but not commonmark/gfm.
---
 MANUAL.txt                          |  28 ++++++++++++++++++++++++++--
 pandoc.cabal                        |   4 ++++
 src/Text/Pandoc/Extensions.hs       |   3 +++
 src/Text/Pandoc/Readers/Markdown.hs |  36 +++++++++++++++++++++++++++++-------
 test/command/3752.md                |  35 +++++++++++++++++++++++++++++++++++
 test/command/chap1/spider.png       | Bin 0 -> 63531 bytes
 test/command/chap1/text.md          |  11 +++++++++++
 test/command/chap2/spider.png       | Bin 0 -> 9861 bytes
 test/command/chap2/text.md          |   3 +++
 9 files changed, 111 insertions(+), 9 deletions(-)
 create mode 100644 test/command/3752.md
 create mode 100644 test/command/chap1/spider.png
 create mode 100644 test/command/chap1/text.md
 create mode 100644 test/command/chap2/spider.png
 create mode 100644 test/command/chap2/text.md

(limited to 'test/command')

diff --git a/MANUAL.txt b/MANUAL.txt
index 6dc783e8c..48bf60d6e 100644
--- a/MANUAL.txt
+++ b/MANUAL.txt
@@ -3755,7 +3755,7 @@ definition:
 Note that space between items in a definition list is required.
 (A variant that loosens this requirement, but disallows "lazy"
 hard wrapping, can be activated with `compact_definition_lists`: see
-[Non-pandoc extensions], below.)
+[Non-default extensions], below.)
 
 [^3]:  I have been influenced by the suggestions of [David
   Wheeler](https://justatheory.com/2009/02/modest-markdown-proposal/).
@@ -5051,13 +5051,37 @@ author-in-text style inside notes when using a note style.
 [finding and editing styles]: https://citationstyles.org/authors/
 [CSL locale files]: https://github.com/citation-style-language/locales
 
-## Non-pandoc extensions
+## Non-default extensions
 
 The following Markdown syntax extensions are not enabled by default
 in pandoc, but may be enabled by adding `+EXTENSION` to the format
 name, where `EXTENSION` is the name of the extension.  Thus, for
 example, `markdown+hard_line_breaks` is Markdown with hard line breaks.
 
+#### Extension: `rebase_relative_paths` ####
+
+Rewrite relative paths for Markdown links and images, depending
+on the path of the file containing the link or image link.  For
+each link or image, pandoc will compute the directory of the
+containing file, relative to the working directory, and prepend
+the resulting path to the link or image path.
+
+The use of this extension is best understood by example.
+Suppose you have a a subdirectory for each chapter of a book,
+`chap1`, `chap2`, `chap3`. Each contains a file `text.md` and a
+number of images used in the chapter.  You would like to have
+`![image](spider.jpg)` in `chap1/text.md` refer to
+`chap1/spider.jpg` and `![image](spider.jpg)` in `chap2/text.md`
+refer to `chap2/spider.jpg`.  To do this, use
+
+    pandoc chap*/*.md -f markdown+rebase_relative_paths
+
+Without this extension, you would have to use
+`![image](chap1/spider.jpg)` in `chap1/text.md` and
+`![image](chap2/spider.jpg)` in `chap2/text.md`.  Links with
+relative paths will be rewritten in the same way as images.
+*This option currently only affects Markdown input.*
+
 #### Extension: `attributes` ####
 
 Allows attributes to be attached to any inline or block-level
diff --git a/pandoc.cabal b/pandoc.cabal
index c74df3e69..c8343d16e 100644
--- a/pandoc.cabal
+++ b/pandoc.cabal
@@ -214,6 +214,10 @@ extra-source-files:
                  test/command/C.txt
                  test/command/D.txt
                  test/command/01.csv
+                 test/command/chap1/spider.png
+                 test/command/chap2/spider.png
+                 test/command/chap1/text.md
+                 test/command/chap2/text.md
                  test/command/defaults1.yaml
                  test/command/defaults2.yaml
                  test/command/defaults3.yaml
diff --git a/src/Text/Pandoc/Extensions.hs b/src/Text/Pandoc/Extensions.hs
index 6423d5f56..c4d54c06e 100644
--- a/src/Text/Pandoc/Extensions.hs
+++ b/src/Text/Pandoc/Extensions.hs
@@ -136,6 +136,8 @@ data Extension =
     | Ext_raw_html            -- ^ Allow raw HTML
     | Ext_raw_tex             -- ^ Allow raw TeX (other than math)
     | Ext_raw_markdown        -- ^ Parse markdown in ipynb as raw markdown
+    | Ext_rebase_relative_paths -- ^ Rebase relative image and link paths,
+                                -- relative to directory of containing file
     | Ext_shortcut_reference_links -- ^ Shortcut reference links
     | Ext_simple_tables       -- ^ Pandoc-style simple tables
     | Ext_smart               -- ^ "Smart" quotes, apostrophes, ellipses, dashes
@@ -462,6 +464,7 @@ getAllExtensions f = universalExtensions <> getAll f
        , Ext_gutenberg
        , Ext_smart
        , Ext_literate_haskell
+       , Ext_rebase_relative_paths
        ]
   getAll "markdown_strict"   = allMarkdownExtensions
   getAll "markdown_phpextra" = allMarkdownExtensions
diff --git a/src/Text/Pandoc/Readers/Markdown.hs b/src/Text/Pandoc/Readers/Markdown.hs
index 34f16ab4e..968c6c165 100644
--- a/src/Text/Pandoc/Readers/Markdown.hs
+++ b/src/Text/Pandoc/Readers/Markdown.hs
@@ -29,7 +29,7 @@ import qualified Data.Set as Set
 import Data.Text (Text)
 import qualified Data.Text as T
 import qualified Data.ByteString.Lazy as BL
-import System.FilePath (addExtension, takeExtension)
+import System.FilePath (addExtension, takeExtension, isAbsolute, takeDirectory)
 import Text.HTML.TagSoup hiding (Row)
 import Text.Pandoc.Builder (Blocks, Inlines)
 import qualified Text.Pandoc.Builder as B
@@ -1836,9 +1836,12 @@ regLink :: PandocMonad m
         -> MarkdownParser m (F Inlines)
 regLink constructor lab = try $ do
   (src, tit) <- source
+  rebase <- option False (True <$ guardEnabled Ext_rebase_relative_paths)
+  pos <- getPosition
+  let src' = if rebase then rebasePath pos src else src
   attr <- option nullAttr $
           guardEnabled Ext_link_attributes >> attributes
-  return $ constructor attr src tit <$> lab
+  return $ constructor attr src' tit <$> lab
 
 -- a link like [this][ref] or [this][] or [this]
 referenceLink :: PandocMonad m
@@ -1854,6 +1857,8 @@ referenceLink constructor (lab, raw) = do
                          return (mempty, "")))
       <|>
       try ((guardDisabled Ext_spaced_reference_links <|> spnl) >> reference)
+  rebase <- option False (True <$ guardEnabled Ext_rebase_relative_paths)
+  pos <- getPosition
   when (raw' == "") $ guardEnabled Ext_shortcut_reference_links
   let labIsRef = raw' == "" || raw' == "[]"
   let key = toKey $ if labIsRef then raw else raw'
@@ -1878,7 +1883,9 @@ referenceLink constructor (lab, raw) = do
                    Just ((src, tit), _) -> constructor nullAttr src tit <$> lab
                    Nothing              -> makeFallback
             else makeFallback
-       Just ((src,tit), attr) -> constructor attr src tit <$> lab
+       Just ((src,tit), attr) ->
+          let src' = if rebase then rebasePath pos src else src
+           in constructor attr src' tit <$> lab
 
 dropBrackets :: Text -> Text
 dropBrackets = dropRB . dropLB
@@ -1911,15 +1918,30 @@ autoLink = try $ do
   return $ return $ B.linkWith attr (src <> escapeURI extra) ""
                      (B.str $ orig <> extra)
 
+-- | Rebase a relative path, by adding the (relative) directory
+-- of the containing source position.  Absolute links and URLs
+-- are untouched.
+rebasePath :: SourcePos -> Text -> Text
+rebasePath pos path = do
+  let fp = sourceName pos
+   in if isAbsolute (T.unpack path) || isURI path
+         then path
+         else
+           case takeDirectory fp of
+             ""  -> path
+             "." -> path
+             d   -> T.pack d <> "/" <> path
+
 image :: PandocMonad m => MarkdownParser m (F Inlines)
 image = try $ do
   char '!'
   (lab,raw) <- reference
   defaultExt <- getOption readerDefaultImageExtension
-  let constructor attr' src = case takeExtension (T.unpack src) of
-                          "" -> B.imageWith attr' (T.pack $ addExtension (T.unpack src)
-                                                          $ T.unpack defaultExt)
-                          _  -> B.imageWith attr' src
+  let constructor attr' src =
+         case takeExtension (T.unpack src) of
+            "" -> B.imageWith attr' (T.pack $ addExtension (T.unpack src)
+                                            $ T.unpack defaultExt)
+            _  -> B.imageWith attr' src
   regLink constructor lab <|> referenceLink constructor (lab,raw)
 
 note :: PandocMonad m => MarkdownParser m (F Inlines)
diff --git a/test/command/3752.md b/test/command/3752.md
new file mode 100644
index 000000000..76d51989b
--- /dev/null
+++ b/test/command/3752.md
@@ -0,0 +1,35 @@
+```
+% pandoc command/chap1/text.md command/chap2/text.md -f markdown+rebase_relative_paths --verbose -t docx | pandoc -f docx -t plain
+^D
+[INFO] Loaded command/chap1/spider.png from ./command/chap1/spider.png
+[INFO] Loaded command/chap1/../../lalune.jpg from ./command/chap1/../../lalune.jpg
+[INFO] Loaded command/chap2/spider.png from ./command/chap2/spider.png
+Chapter one
+
+A spider: [spider]
+
+The moon: [moon]
+
+Link to spider picture.
+
+URL left alone: manual.
+
+Absolute path left alone: absolute.
+
+Chapter two
+
+A spider: [spider]
+```
+
+```
+% pandoc command/chap1/text.md command/chap2/text.md -f markdown+rebase_relative_paths -t html
+^D
+<h1 id="chapter-one">Chapter one</h1>
+<p>A spider: <img src="command/chap1/spider.png" alt="spider" /></p>
+<p>The moon: <img src="command/chap1/../../lalune.jpg" alt="moon" /></p>
+<p>Link to <a href="command/chap1/spider.png">spider picture</a>.</p>
+<p>URL left alone: <a href="https://pandoc.org/MANUAL.html">manual</a>.</p>
+<p>Absolute path left alone: <a href="/foo/bar/baz.png">absolute</a>.</p>
+<h1 id="chapter-two">Chapter two</h1>
+<p>A spider: <img src="command/chap2/spider.png" alt="spider" /></p>
+```
diff --git a/test/command/chap1/spider.png b/test/command/chap1/spider.png
new file mode 100644
index 000000000..7ee9fe339
Binary files /dev/null and b/test/command/chap1/spider.png differ
diff --git a/test/command/chap1/text.md b/test/command/chap1/text.md
new file mode 100644
index 000000000..88b30313d
--- /dev/null
+++ b/test/command/chap1/text.md
@@ -0,0 +1,11 @@
+# Chapter one
+
+A spider: ![spider](spider.png)
+
+The moon: ![moon](../../lalune.jpg)
+
+Link to [spider picture](spider.png).
+
+URL left alone: [manual](https://pandoc.org/MANUAL.html).
+
+Absolute path left alone: [absolute](/foo/bar/baz.png).
diff --git a/test/command/chap2/spider.png b/test/command/chap2/spider.png
new file mode 100644
index 000000000..5377e940b
Binary files /dev/null and b/test/command/chap2/spider.png differ
diff --git a/test/command/chap2/text.md b/test/command/chap2/text.md
new file mode 100644
index 000000000..435a266d7
--- /dev/null
+++ b/test/command/chap2/text.md
@@ -0,0 +1,3 @@
+# Chapter two
+
+A spider: ![spider](spider.png)
-- 
cgit v1.2.3


From cbe16b2866abd8d0e4c15f027562b4b2bed3f01e Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 27 May 2021 10:49:45 -0700
Subject: Citeproc: Don't detect math elements as locators.

Closes #7321.
---
 MANUAL.txt                          |  2 +-
 src/Text/Pandoc/Citeproc/Locator.hs |  7 +++++++
 test/command/7321.md                | 24 ++++++++++++++++++++++++
 3 files changed, 32 insertions(+), 1 deletion(-)
 create mode 100644 test/command/7321.md

(limited to 'test/command')

diff --git a/MANUAL.txt b/MANUAL.txt
index 48bf60d6e..2ca50e44a 100644
--- a/MANUAL.txt
+++ b/MANUAL.txt
@@ -5015,7 +5015,7 @@ the suffix as locator by prepending curly braces:
 
     [@smith{ii, A, D-Z}, with a suffix]
     [@smith, {pp. iv, vi-xi, (xv)-(xvii)} with suffix here]
-    [@smith{}, $a^2$ and following]
+    [@smith{}, 99 years later]
 
 A minus sign (`-`) before the `@` will suppress mention of
 the author in the citation.  This can be useful when the
diff --git a/src/Text/Pandoc/Citeproc/Locator.hs b/src/Text/Pandoc/Citeproc/Locator.hs
index 44416ca12..dbedc08d9 100644
--- a/src/Text/Pandoc/Citeproc/Locator.hs
+++ b/src/Text/Pandoc/Citeproc/Locator.hs
@@ -181,6 +181,7 @@ pPageUnit = roman <|> plainUnit
       plainUnit = do
           ts <- many1 (notFollowedBy pSpace >>
                        notFollowedBy pLocatorPunct >>
+                       notFollowedBy pMath >>
                        anyToken)
           let s = stringify ts
           -- otherwise look for actual digits or -s
@@ -211,6 +212,12 @@ pMatchChar msg f = satisfyTok f' <?> msg
 pSpace :: LocatorParser Inline
 pSpace = satisfyTok (\t -> isSpacey t || t == Str "\160") <?> "space"
 
+pMath :: LocatorParser Inline
+pMath = satisfyTok isMath
+ where
+  isMath (Math{}) = True
+  isMath _ = False
+
 satisfyTok :: (Inline -> Bool) -> LocatorParser Inline
 satisfyTok f = tokenPrim show (\sp _ _ -> sp) (\tok -> if f tok
                                                           then Just tok
diff --git a/test/command/7321.md b/test/command/7321.md
new file mode 100644
index 000000000..f5e644bee
--- /dev/null
+++ b/test/command/7321.md
@@ -0,0 +1,24 @@
+```
+% pandoc -t plain --citeproc --wrap=none
+---
+references:
+- id: fenner2012a
+  title: One-click science marketing
+  author:
+  - family: Fenner
+    given: Martin
+  container-title: Nature Materials
+  volume: 11
+  issue: 4
+  publisher: Nature Publishing Group
+  type: article-journal
+  issued:
+    year: 2012
+---
+
+[@fenner2012a, $a^2+b^2=c^2$]
+^D
+(Fenner 2012, a² + b² = c²)
+
+Fenner, Martin. 2012. “One-Click Science Marketing.” Nature Materials 11 (4).
+```
-- 
cgit v1.2.3


From 6972a7dc9158c1d56b2dc9a06f596386f6b30860 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 27 May 2021 11:26:38 -0700
Subject: Modify rebase_reference_links treatment of reference links/images.

The directory is based on the file containing the link
reference, not the file containing the link, if these differ.
---
 MANUAL.txt                          | 6 +++++-
 src/Text/Pandoc/Readers/Markdown.hs | 9 ++++-----
 test/command/3752.md                | 5 ++++-
 test/command/chap1/text.md          | 2 ++
 test/command/chap2/text.md          | 2 ++
 5 files changed, 17 insertions(+), 7 deletions(-)

(limited to 'test/command')

diff --git a/MANUAL.txt b/MANUAL.txt
index 2ca50e44a..d755c0c00 100644
--- a/MANUAL.txt
+++ b/MANUAL.txt
@@ -5080,7 +5080,11 @@ Without this extension, you would have to use
 `![image](chap1/spider.jpg)` in `chap1/text.md` and
 `![image](chap2/spider.jpg)` in `chap2/text.md`.  Links with
 relative paths will be rewritten in the same way as images.
-*This option currently only affects Markdown input.*
+
+Note that relative paths in reference links and images will
+be rewritten relative to the file containing the link
+reference definition, not the file containing the reference link
+or image itself, if these differ.
 
 #### Extension: `attributes` ####
 
diff --git a/src/Text/Pandoc/Readers/Markdown.hs b/src/Text/Pandoc/Readers/Markdown.hs
index 968c6c165..e5cbadc94 100644
--- a/src/Text/Pandoc/Readers/Markdown.hs
+++ b/src/Text/Pandoc/Readers/Markdown.hs
@@ -336,7 +336,9 @@ referenceKey = try $ do
                     notFollowedBy' (() <$ reference)
                     many1Char $ notFollowedBy space >> litChar
   let betweenAngles = try $ char '<' >> manyTillChar litChar (char '>')
-  src <- try betweenAngles <|> sourceURL
+  rebase <- option False (True <$ guardEnabled Ext_rebase_relative_paths)
+  src <- (if rebase then rebasePath pos else id) <$>
+             (try betweenAngles <|> sourceURL)
   tit <- option "" referenceTitle
   attr   <- option nullAttr $ try $
               do guardEnabled Ext_link_attributes
@@ -1857,8 +1859,6 @@ referenceLink constructor (lab, raw) = do
                          return (mempty, "")))
       <|>
       try ((guardDisabled Ext_spaced_reference_links <|> spnl) >> reference)
-  rebase <- option False (True <$ guardEnabled Ext_rebase_relative_paths)
-  pos <- getPosition
   when (raw' == "") $ guardEnabled Ext_shortcut_reference_links
   let labIsRef = raw' == "" || raw' == "[]"
   let key = toKey $ if labIsRef then raw else raw'
@@ -1884,8 +1884,7 @@ referenceLink constructor (lab, raw) = do
                    Nothing              -> makeFallback
             else makeFallback
        Just ((src,tit), attr) ->
-          let src' = if rebase then rebasePath pos src else src
-           in constructor attr src' tit <$> lab
+           constructor attr src tit <$> lab
 
 dropBrackets :: Text -> Text
 dropBrackets = dropRB . dropLB
diff --git a/test/command/3752.md b/test/command/3752.md
index 76d51989b..6d31eb450 100644
--- a/test/command/3752.md
+++ b/test/command/3752.md
@@ -2,12 +2,14 @@
 % pandoc command/chap1/text.md command/chap2/text.md -f markdown+rebase_relative_paths --verbose -t docx | pandoc -f docx -t plain
 ^D
 [INFO] Loaded command/chap1/spider.png from ./command/chap1/spider.png
-[INFO] Loaded command/chap1/../../lalune.jpg from ./command/chap1/../../lalune.jpg
 [INFO] Loaded command/chap2/spider.png from ./command/chap2/spider.png
+[INFO] Loaded command/chap1/../../lalune.jpg from ./command/chap1/../../lalune.jpg
 Chapter one
 
 A spider: [spider]
 
+Another spider: [another spider]
+
 The moon: [moon]
 
 Link to spider picture.
@@ -26,6 +28,7 @@ A spider: [spider]
 ^D
 <h1 id="chapter-one">Chapter one</h1>
 <p>A spider: <img src="command/chap1/spider.png" alt="spider" /></p>
+<p>Another spider: <img src="command/chap2/spider.png" alt="another spider" /></p>
 <p>The moon: <img src="command/chap1/../../lalune.jpg" alt="moon" /></p>
 <p>Link to <a href="command/chap1/spider.png">spider picture</a>.</p>
 <p>URL left alone: <a href="https://pandoc.org/MANUAL.html">manual</a>.</p>
diff --git a/test/command/chap1/text.md b/test/command/chap1/text.md
index 88b30313d..d25514241 100644
--- a/test/command/chap1/text.md
+++ b/test/command/chap1/text.md
@@ -2,6 +2,8 @@
 
 A spider: ![spider](spider.png)
 
+Another spider: ![another spider][refspider]
+
 The moon: ![moon](../../lalune.jpg)
 
 Link to [spider picture](spider.png).
diff --git a/test/command/chap2/text.md b/test/command/chap2/text.md
index 435a266d7..082a1d79e 100644
--- a/test/command/chap2/text.md
+++ b/test/command/chap2/text.md
@@ -1,3 +1,5 @@
 # Chapter two
 
 A spider: ![spider](spider.png)
+
+[refspider]: spider.png
-- 
cgit v1.2.3


From 0661ce699fe83c64dd5a5874e8ab17241b19550c Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 27 May 2021 13:53:26 -0700
Subject: rebase_relative_paths extension: don't change fragment paths.

We don't want a pure fragment path to be rewritten, since
these are used for cross-referencing.
---
 MANUAL.txt                          | 3 +++
 src/Text/Pandoc/Readers/Markdown.hs | 3 ++-
 test/command/3752.md                | 3 +++
 test/command/chap1/text.md          | 2 ++
 4 files changed, 10 insertions(+), 1 deletion(-)

(limited to 'test/command')

diff --git a/MANUAL.txt b/MANUAL.txt
index d755c0c00..fb1a76e9a 100644
--- a/MANUAL.txt
+++ b/MANUAL.txt
@@ -5081,6 +5081,9 @@ Without this extension, you would have to use
 `![image](chap2/spider.jpg)` in `chap2/text.md`.  Links with
 relative paths will be rewritten in the same way as images.
 
+Absolute paths and URLs are not changed.  Neither are paths
+consisting entirely of a fragment, e.g. `[section one](#foo)`.
+
 Note that relative paths in reference links and images will
 be rewritten relative to the file containing the link
 reference definition, not the file containing the reference link
diff --git a/src/Text/Pandoc/Readers/Markdown.hs b/src/Text/Pandoc/Readers/Markdown.hs
index e5cbadc94..74f2668e4 100644
--- a/src/Text/Pandoc/Readers/Markdown.hs
+++ b/src/Text/Pandoc/Readers/Markdown.hs
@@ -1923,7 +1923,8 @@ autoLink = try $ do
 rebasePath :: SourcePos -> Text -> Text
 rebasePath pos path = do
   let fp = sourceName pos
-   in if isAbsolute (T.unpack path) || isURI path
+      isFragment = T.take 1 path == "#"
+   in if isFragment || isAbsolute (T.unpack path) || isURI path
          then path
          else
            case takeDirectory fp of
diff --git a/test/command/3752.md b/test/command/3752.md
index 6d31eb450..2765d79ac 100644
--- a/test/command/3752.md
+++ b/test/command/3752.md
@@ -18,6 +18,8 @@ URL left alone: manual.
 
 Absolute path left alone: absolute.
 
+Link to fragment: chapter two.
+
 Chapter two
 
 A spider: [spider]
@@ -33,6 +35,7 @@ A spider: [spider]
 <p>Link to <a href="command/chap1/spider.png">spider picture</a>.</p>
 <p>URL left alone: <a href="https://pandoc.org/MANUAL.html">manual</a>.</p>
 <p>Absolute path left alone: <a href="/foo/bar/baz.png">absolute</a>.</p>
+<p>Link to fragment: <a href="#chapter-two">chapter two</a>.</p>
 <h1 id="chapter-two">Chapter two</h1>
 <p>A spider: <img src="command/chap2/spider.png" alt="spider" /></p>
 ```
diff --git a/test/command/chap1/text.md b/test/command/chap1/text.md
index d25514241..68a317161 100644
--- a/test/command/chap1/text.md
+++ b/test/command/chap1/text.md
@@ -11,3 +11,5 @@ Link to [spider picture](spider.png).
 URL left alone: [manual](https://pandoc.org/MANUAL.html).
 
 Absolute path left alone: [absolute](/foo/bar/baz.png).
+
+Link to fragment: [chapter two](#chapter-two).
-- 
cgit v1.2.3


From 4b16d181e7219ed161a0e03c0c5ee9dec4b526b4 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 27 May 2021 14:16:37 -0700
Subject: rebase_relative_paths: leave empty paths unchanged.

---
 MANUAL.txt                          | 4 ++--
 src/Text/Pandoc/Readers/Markdown.hs | 2 +-
 test/command/3752.md                | 3 +++
 test/command/chap1/text.md          | 2 ++
 4 files changed, 8 insertions(+), 3 deletions(-)

(limited to 'test/command')

diff --git a/MANUAL.txt b/MANUAL.txt
index fb1a76e9a..c234debc9 100644
--- a/MANUAL.txt
+++ b/MANUAL.txt
@@ -5081,8 +5081,8 @@ Without this extension, you would have to use
 `![image](chap2/spider.jpg)` in `chap2/text.md`.  Links with
 relative paths will be rewritten in the same way as images.
 
-Absolute paths and URLs are not changed.  Neither are paths
-consisting entirely of a fragment, e.g. `[section one](#foo)`.
+Absolute paths and URLs are not changed.  Neither are empty
+paths or paths consisting entirely of a fragment, e.g., `#foo`.
 
 Note that relative paths in reference links and images will
 be rewritten relative to the file containing the link
diff --git a/src/Text/Pandoc/Readers/Markdown.hs b/src/Text/Pandoc/Readers/Markdown.hs
index 74f2668e4..bc5e3e30f 100644
--- a/src/Text/Pandoc/Readers/Markdown.hs
+++ b/src/Text/Pandoc/Readers/Markdown.hs
@@ -1924,7 +1924,7 @@ rebasePath :: SourcePos -> Text -> Text
 rebasePath pos path = do
   let fp = sourceName pos
       isFragment = T.take 1 path == "#"
-   in if isFragment || isAbsolute (T.unpack path) || isURI path
+   in if T.null path || isFragment || isAbsolute (T.unpack path) || isURI path
          then path
          else
            case takeDirectory fp of
diff --git a/test/command/3752.md b/test/command/3752.md
index 2765d79ac..df8af0ba1 100644
--- a/test/command/3752.md
+++ b/test/command/3752.md
@@ -20,6 +20,8 @@ Absolute path left alone: absolute.
 
 Link to fragment: chapter two.
 
+Empty path: empty.
+
 Chapter two
 
 A spider: [spider]
@@ -36,6 +38,7 @@ A spider: [spider]
 <p>URL left alone: <a href="https://pandoc.org/MANUAL.html">manual</a>.</p>
 <p>Absolute path left alone: <a href="/foo/bar/baz.png">absolute</a>.</p>
 <p>Link to fragment: <a href="#chapter-two">chapter two</a>.</p>
+<p>Empty path: <a href="">empty</a>.</p>
 <h1 id="chapter-two">Chapter two</h1>
 <p>A spider: <img src="command/chap2/spider.png" alt="spider" /></p>
 ```
diff --git a/test/command/chap1/text.md b/test/command/chap1/text.md
index 68a317161..54f0104de 100644
--- a/test/command/chap1/text.md
+++ b/test/command/chap1/text.md
@@ -13,3 +13,5 @@ URL left alone: [manual](https://pandoc.org/MANUAL.html).
 Absolute path left alone: [absolute](/foo/bar/baz.png).
 
 Link to fragment: [chapter two](#chapter-two).
+
+Empty path: [empty]().
-- 
cgit v1.2.3


From 4842c5fb828c3c34d816fa7bccd4656857742a0b Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 27 May 2021 18:28:52 -0700
Subject: Two citeproc locator/suffix improvements:

- Recognize locators spelled with a capital letter.
  Closes #7323.
- Add a comma and a space in front of the suffix if it doesn't start
  with space or punctuation.  Closes #7324.
---
 src/Text/Pandoc/Citeproc/Locator.hs | 14 +++++++++++---
 test/command/7323.md                | 29 +++++++++++++++++++++++++++++
 test/command/7324.md                | 25 +++++++++++++++++++++++++
 3 files changed, 65 insertions(+), 3 deletions(-)
 create mode 100644 test/command/7323.md
 create mode 100644 test/command/7324.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Citeproc/Locator.hs b/src/Text/Pandoc/Citeproc/Locator.hs
index dbedc08d9..f8931d7b5 100644
--- a/src/Text/Pandoc/Citeproc/Locator.hs
+++ b/src/Text/Pandoc/Citeproc/Locator.hs
@@ -20,7 +20,7 @@ parseLocator :: Locale -> [Inline] -> (Maybe (Text, Text), [Inline])
 parseLocator locale inp =
   case parse (pLocatorWords (toLocatorMap locale)) "suffix" $ splitInp inp of
        Right r -> r
-       Left _  -> (Nothing, inp)
+       Left _  -> (Nothing, maybeAddComma inp)
 
 splitInp :: [Inline] -> [Inline]
 splitInp = splitStrWhen (\c -> isSpace c || (isPunctuation c && c /= ':'))
@@ -42,9 +42,17 @@ pLocatorWords locMap = do
   -- i.e. the first one will be " 9"
   return $
     if T.null la && T.null lo
-       then (Nothing, s)
+       then (Nothing, maybeAddComma s)
        else (Just (la, T.strip lo), s)
 
+maybeAddComma :: [Inline] -> [Inline]
+maybeAddComma [] = []
+maybeAddComma ils@(Space : _) = ils
+maybeAddComma ils@(Str t : _)
+  | Just (c, _) <- T.uncons t
+  , isPunctuation c = ils
+maybeAddComma ils = Str "," : Space : ils
+
 pLocatorDelimited :: LocatorMap -> LocatorParser (Text, Text)
 pLocatorDelimited locMap = try $ do
   _ <- pMatchChar "{" (== '{')
@@ -97,7 +105,7 @@ pLocatorLabel' locMap lim = go ""
           t <- anyToken
           ts <- manyTill anyToken (try $ lookAhead lim)
           let s = acc <> stringify (t:ts)
-          case M.lookup (T.strip s) locMap of
+          case M.lookup (T.toCaseFold $ T.strip s) locMap of
             -- try to find a longer one, or return this one
             Just l -> go s <|> return (l, False)
             Nothing -> go s
diff --git a/test/command/7323.md b/test/command/7323.md
new file mode 100644
index 000000000..7de29cfe2
--- /dev/null
+++ b/test/command/7323.md
@@ -0,0 +1,29 @@
+```
+% pandoc --citeproc -t plain
+---
+references:
+- id: smith
+  author: John Smith
+  issued: 2019
+  title: Insects
+  type: book
+...
+
+@smith [chap. 6]
+
+@smith [chapter 6]
+
+@smith [Chap. 6]
+
+@smith [Chapter 6]
+^D
+John Smith (2019, chap. 6)
+
+John Smith (2019, chap. 6)
+
+John Smith (2019, chap. 6)
+
+John Smith (2019, chap. 6)
+
+John Smith. 2019. Insects.
+```
diff --git a/test/command/7324.md b/test/command/7324.md
new file mode 100644
index 000000000..fae1b9923
--- /dev/null
+++ b/test/command/7324.md
@@ -0,0 +1,25 @@
+```
+% pandoc --citeproc -t plain
+---
+references:
+- id: smith
+  author: John Smith
+  issued: 2019
+  title: Insects
+  type: book
+...
+
+@smith [, among others]
+
+@smith [ among others]
+
+@smith [among others]
+^D
+John Smith (2019, among others)
+
+John Smith (2019 among others)
+
+John Smith (2019, among others)
+
+John Smith. 2019. Insects.
+```
-- 
cgit v1.2.3


From b6b2331fdcee37f1bfb3fcc21816d73d6d56cfae Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Fri, 28 May 2021 13:58:44 -0700
Subject: Support `rebase_relative_paths` for commonmark based formats.

(Including `gfm`.)
---
 src/Text/Pandoc/Extensions.hs         |  1 +
 src/Text/Pandoc/Readers/CommonMark.hs |  4 +++-
 test/command/3752.md                  | 16 ++++++++++++++++
 3 files changed, 20 insertions(+), 1 deletion(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Extensions.hs b/src/Text/Pandoc/Extensions.hs
index c4d54c06e..9c55d0a7a 100644
--- a/src/Text/Pandoc/Extensions.hs
+++ b/src/Text/Pandoc/Extensions.hs
@@ -517,6 +517,7 @@ getAllExtensions f = universalExtensions <> getAll f
     , Ext_attributes
     , Ext_sourcepos
     , Ext_yaml_metadata_block
+    , Ext_rebase_relative_paths
     ]
   getAll "commonmark_x"    = getAll "commonmark"
   getAll "org"             = autoIdExtensions <>
diff --git a/src/Text/Pandoc/Readers/CommonMark.hs b/src/Text/Pandoc/Readers/CommonMark.hs
index b099a9b50..228e65312 100644
--- a/src/Text/Pandoc/Readers/CommonMark.hs
+++ b/src/Text/Pandoc/Readers/CommonMark.hs
@@ -112,5 +112,7 @@ specFor opts = foldr ($) defaultSyntaxSpec exts
            | isEnabled Ext_implicit_header_references opts ] ++
          [ (footnoteSpec <>) | isEnabled Ext_footnotes opts ] ++
          [ (definitionListSpec <>) | isEnabled Ext_definition_lists opts ] ++
-         [ (taskListSpec <>) | isEnabled Ext_task_lists opts ]
+         [ (taskListSpec <>) | isEnabled Ext_task_lists opts ] ++
+         [ (rebaseRelativePathsSpec <>)
+           | isEnabled Ext_rebase_relative_paths opts ]
 
diff --git a/test/command/3752.md b/test/command/3752.md
index df8af0ba1..2e96b531e 100644
--- a/test/command/3752.md
+++ b/test/command/3752.md
@@ -42,3 +42,19 @@ A spider: [spider]
 <h1 id="chapter-two">Chapter two</h1>
 <p>A spider: <img src="command/chap2/spider.png" alt="spider" /></p>
 ```
+
+```
+% pandoc command/chap1/text.md command/chap2/text.md -f commonmark+rebase_relative_paths -t html
+^D
+<h1>Chapter one</h1>
+<p>A spider: <img src="command/chap1/spider.png" alt="spider" /></p>
+<p>Another spider: <img src="command/chap2/spider.png" alt="another spider" /></p>
+<p>The moon: <img src="command/chap1/../../lalune.jpg" alt="moon" /></p>
+<p>Link to <a href="command/chap1/spider.png">spider picture</a>.</p>
+<p>URL left alone: <a href="https://pandoc.org/MANUAL.html">manual</a>.</p>
+<p>Absolute path left alone: <a href="/foo/bar/baz.png">absolute</a>.</p>
+<p>Link to fragment: <a href="#chapter-two">chapter two</a>.</p>
+<p>Empty path: <a href="">empty</a>.</p>
+<h1>Chapter two</h1>
+<p>A spider: <img src="command/chap2/spider.png" alt="spider" /></p>
+```
-- 
cgit v1.2.3


From e86f6abc45b8192eb98ec6d0a3b65d2a3c6726ad Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sat, 29 May 2021 12:06:32 -0700
Subject: Further test image size reductions.

---
 test/command/chap1/spider.png | Bin 63531 -> 8758 bytes
 test/command/chap2/spider.png | Bin 9861 -> 2921 bytes
 2 files changed, 0 insertions(+), 0 deletions(-)

(limited to 'test/command')

diff --git a/test/command/chap1/spider.png b/test/command/chap1/spider.png
index 7ee9fe339..4c6bf13e9 100644
Binary files a/test/command/chap1/spider.png and b/test/command/chap1/spider.png differ
diff --git a/test/command/chap2/spider.png b/test/command/chap2/spider.png
index 5377e940b..5456a857a 100644
Binary files a/test/command/chap2/spider.png and b/test/command/chap2/spider.png differ
-- 
cgit v1.2.3


From 7aade73dce70f75aa0fe04b11b118acc3ea7854f Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sat, 29 May 2021 12:14:37 -0700
Subject: Replace biblatex-exmaples.bib with shorter averroes.bib in tests.

---
 pandoc.cabal                        |    2 +-
 test/command/averroes.bib           |   95 ++
 test/command/biblatex-examples.bib  | 1674 -----------------------------------
 test/command/pandoc-citeproc-119.md |    2 +-
 4 files changed, 97 insertions(+), 1676 deletions(-)
 create mode 100644 test/command/averroes.bib
 delete mode 100644 test/command/biblatex-examples.bib

(limited to 'test/command')

diff --git a/pandoc.cabal b/pandoc.cabal
index fb70a9de4..40b0db594 100644
--- a/pandoc.cabal
+++ b/pandoc.cabal
@@ -208,7 +208,7 @@ extra-source-files:
                  test/command/*.md
                  test/command/*.csl
                  test/command/biblio.bib
-                 test/command/biblatex-examples.bib
+                 test/command/averroes.bib
                  test/command/A.txt
                  test/command/B.txt
                  test/command/C.txt
diff --git a/test/command/averroes.bib b/test/command/averroes.bib
new file mode 100644
index 000000000..b95c8e354
--- /dev/null
+++ b/test/command/averroes.bib
@@ -0,0 +1,95 @@
+@string{anch-ie = {Angew.~Chem. Int.~Ed.}}
+@string{cup     = {Cambridge University Press}}
+@string{dtv     = {Deutscher Taschenbuch-Verlag}}
+@string{hup     = {Harvard University Press}}
+@string{jams    = {J.~Amer. Math. Soc.}}
+@string{jchph   = {J.~Chem. Phys.}}
+@string{jomch   = {J.~Organomet. Chem.}}
+@string{pup     = {Princeton University Press}}
+
+@book{averroes/bland,
+  author       = {Averroes},
+  title        = {The Epistle on the Possibility of Conjunction with the Active
+                  Intellect by {Ibn Rushd} with the Commentary of {Moses Narboni}},
+  date         = 1982,
+  editor       = {Bland, Kalman P.},
+  translator   = {Bland, Kalman P.},
+  series       = {Moreshet: Studies in {Jewish} History, Literature and Thought},
+  number       = 7,
+  publisher    = {Jewish Theological Seminary of America},
+  location     = {New York},
+  keywords     = {primary},
+  langid       = {english},
+  langidopts   = {variant=american},
+  indextitle   = {Epistle on the Possibility of Conjunction, The},
+  shorttitle   = {Possibility of Conjunction},
+  annotation   = {A \texttt{book} entry with a \texttt{series} and a
+                  \texttt{number}. Note the concatenation of the \texttt{editor}
+                  and \texttt{translator} fields as well as the
+                  \texttt{indextitle} field},
+}
+
+@book{averroes/hannes,
+  author       = {Averroes},
+  title        = {Des Averro{\"e}s Abhandlung: \mkbibquote{{\"U}ber die
+                  M{\"o}glichkeit der Conjunktion} oder \mkbibquote{{\"U}ber den
+                  materiellen Intellekt}},
+  date         = 1892,
+  editor       = {Hannes, Ludwig},
+  translator   = {Hannes, Ludwig},
+  annotator    = {Hannes, Ludwig},
+  publisher    = {C.~A. Kaemmerer},
+  location     = {Halle an der Saale},
+  keywords     = {primary},
+  langid       = {german},
+  sorttitle    = {Uber die Moglichkeit der Conjunktion},
+  indexsorttitle= {Uber die Moglichkeit der Conjunktion},
+  indextitle   = {{\"U}ber die M{\"o}glichkeit der Conjunktion},
+  shorttitle   = {{\"U}ber die M{\"o}glichkeit der Conjunktion},
+  annotation   = {An annotated edition. Note the concatenation of the
+                  \texttt{editor}, \texttt{translator}, and \texttt{annotator}
+                  fields. Also note the \texttt{shorttitle},
+                  \texttt{indextitle}, \texttt{sorttitle}, and
+                  \texttt{indexsorttitle} fields},
+}
+
+@book{averroes/hercz,
+  author       = {Averroes},
+  title        = {Drei Abhandlungen {\"u}ber die Conjunction des separaten
+                  Intellects mit dem Menschen},
+  date         = 1869,
+  editor       = {Hercz, J.},
+  translator   = {Hercz, J.},
+  publisher    = {S.~Hermann},
+  location     = {Berlin},
+  keywords     = {primary},
+  langid       = {german},
+  indexsorttitle= {Drei Abhandlungen uber die Conjunction},
+  indextitle   = {Drei Abhandlungen {\"u}ber die Conjunction},
+  subtitle     = {Von Averroes (Vater und Sohn), aus dem Arabischen
+                  {\"u}bersetzt von Samuel Ibn Tibbon},
+  shorttitle   = {Drei Abhandlungen},
+  annotation   = {A \texttt{book} entry. Note the concatenation of the
+                  \texttt{editor} and \texttt{translator} fields as well as the
+                  \texttt{indextitle} and \texttt{indexsorttitle} fields},
+}
+
+@book{cicero,
+  author       = {Cicero, Marcus Tullius},
+  title        = {De natura deorum. {\"U}ber das Wesen der G{\"o}tter},
+  date         = 1995,
+  editor       = {Blank-Sangmeister, Ursula},
+  translator   = {Blank-Sangmeister, Ursula},
+  afterword    = {Thraede, Klaus},
+  language     = {langlatin and langgerman},
+  publisher    = {Reclam},
+  location     = {Stuttgart},
+  langid       = {german},
+  indextitle   = {De natura deorum},
+  shorttitle   = {De natura deorum},
+  annotation   = {A bilingual edition of Cicero's \emph{De natura deorum}, with
+                  a German translation. Note the format of the \texttt{language}
+                  field in the database file, the concatenation of the
+                  \texttt{editor} and \texttt{translator} fields, and the
+                  \texttt{afterword} field},
+}
diff --git a/test/command/biblatex-examples.bib b/test/command/biblatex-examples.bib
deleted file mode 100644
index 36ac849dd..000000000
--- a/test/command/biblatex-examples.bib
+++ /dev/null
@@ -1,1674 +0,0 @@
-@string{anch-ie = {Angew.~Chem. Int.~Ed.}}
-@string{cup     = {Cambridge University Press}}
-@string{dtv     = {Deutscher Taschenbuch-Verlag}}
-@string{hup     = {Harvard University Press}}
-@string{jams    = {J.~Amer. Math. Soc.}}
-@string{jchph   = {J.~Chem. Phys.}}
-@string{jomch   = {J.~Organomet. Chem.}}
-@string{pup     = {Princeton University Press}}
-
-@incollection{westfahl:space,
-  author       = {Westfahl, Gary},
-  title        = {The True Frontier},
-  subtitle     = {Confronting and Avoiding the Realities of Space in {American}
-                  Science Fiction Films},
-  pages        = {55-65},
-  crossref     = {westfahl:frontier},
-  langid       = {english},
-  langidopts   = {variant=american},
-  indextitle   = {True Frontier, The},
-  annotation   = {A cross-referenced article from a \texttt{collection}. This is
-                  an \texttt{incollection} entry with a \texttt{crossref}
-                  field. Note the \texttt{subtitle} and \texttt{indextitle}
-                  fields},
-}
-
-@set{set,
-  entryset     = {herrmann,aksin,yoon},
-  annotation   = {A \texttt{set} with three members. The \texttt{crossref} field
-                  in the \texttt{@set} entry and the \texttt{entryset} field in
-                  each set member entry is needed only when using BibTeX as the
-                  backend},
-}
-
-@set{stdmodel,
-  entryset     = {glashow,weinberg,salam},
-  annotation   = {A \texttt{set} with three members discussing the standard
-                  model of particle physics. The \texttt{crossref} field
-                  in the \texttt{@set} entry and the \texttt{entryset} field in
-                  each set member entry is needed only when using BibTeX as the
-                  backend},
-}
-
-@article{aksin,
-  author       = {Aks{\i}n, {\"O}zge and T{\"u}rkmen, Hayati and Artok, Levent
-                  and {\c{C}}etinkaya, Bekir and Ni, Chaoying and
-                  B{\"u}y{\"u}kg{\"u}ng{\"o}r, Orhan and {\"O}zkal, Erhan},
-  title        = {Effect of immobilization on catalytic characteristics of
-                  saturated {Pd-N}-heterocyclic carbenes in {Mizoroki-Heck}
-                  reactions},
-  journaltitle = jomch,
-  date         = 2006,
-  volume       = 691,
-  number       = 13,
-  pages        = {3027-3036},
-  indextitle   = {Effect of immobilization on catalytic characteristics},
-}
-
-@article{angenendt,
-  author       = {Angenendt, Arnold},
-  title        = {In Honore Salvatoris~-- Vom Sinn und Unsinn der
-                  Patrozinienkunde},
-  journaltitle = {Revue d'Histoire Eccl{\'e}siastique},
-  date         = 2002,
-  volume       = 97,
-  pages        = {431--456, 791--823},
-  langid       = {german},
-  indextitle   = {In Honore Salvatoris},
-  shorttitle   = {In Honore Salvatoris},
-  annotation   = {A German article in a French journal. Apart from that, a
-                  typical \texttt{article} entry. Note the \texttt{indextitle}
-                  field},
-}
-
-@article{baez/article,
-  author       = {Baez, John C. and Lauda, Aaron D.},
-  title        = {Higher-Dimensional Algebra {V}: 2-Groups},
-  journaltitle = {Theory and Applications of Categories},
-  date         = 2004,
-  volume       = 12,
-  pages        = {423-491},
-  version      = 3,
-  eprint       = {math/0307200v3},
-  eprinttype   = {arxiv},
-  langid       = {english},
-  langidopts   = {variant=american},
-  annotation   = {An \texttt{article} with \texttt{eprint} and
-                  \texttt{eprinttype} fields. Note that the arXiv reference is
-                  transformed into a clickable link if \texttt{hyperref} support
-                  has been enabled.  Compare \texttt{baez\slash online}, which
-                  is the same item given as an \texttt{online} entry},
-}
-
-@article{bertram,
-  author       = {Bertram, Aaron and Wentworth, Richard},
-  title        = {Gromov invariants for holomorphic maps on {Riemann} surfaces},
-  journaltitle = jams,
-  date         = 1996,
-  volume       = 9,
-  number       = 2,
-  pages        = {529-571},
-  langid       = {english},
-  langidopts   = {variant=american},
-  shorttitle   = {Gromov invariants},
-  annotation   = {An \texttt{article} entry with a \texttt{volume} and a
-                  \texttt{number} field},
-}
-
-@article{doody,
-  author       = {Doody, Terrence},
-  title        = {Hemingway's Style and {Jake's} Narration},
-  year         = 1974,
-  volume       = 4,
-  number       = 3,
-  pages        = {212-225},
-  langid       = {english},
-  langidopts   = {variant=american},
-  related      = {matuz:doody},
-  relatedstring= {\autocap{e}xcerpt in},
-  journal      = {The Journal of Narrative Technique},
-  annotation   = {An \texttt{article} entry cited as an excerpt from a
-                  \texttt{collection} entry. Note the format of the
-                  \texttt{related} and \texttt{relatedstring} fields},
-}
-
-@collection{matuz:doody,
-  editor       = {Matuz, Roger},
-  title        = {Contemporary Literary Criticism},
-  year         = 1990,
-  volume       = 61,
-  publisher    = {Gale},
-  location     = {Detroit},
-  pages        = {204-208},
-  langid       = {english},
-  langidopts   = {variant=american},
-  annotation   = {A \texttt{collection} entry providing the excerpt information
-                  for the \texttt{doody} entry. Note the format of the
-                  \texttt{pages} field},
-}
-
-@article{gillies,
-  author       = {Gillies, Alexander},
-  title        = {Herder and the Preparation of {Goethe's} Idea of World
-                  Literature},
-  journaltitle = {Publications of the English Goethe Society},
-  date         = 1933,
-  series       = {newseries},
-  volume       = 9,
-  pages        = {46-67},
-  langid       = {english},
-  langidopts   = {variant=british},
-  annotation   = {An \texttt{article} entry with a \texttt{series} and a
-                  \texttt{volume} field. Note that format of the \texttt{series}
-                  field in the database file},
-}
-
-@article{glashow,
-  author       = {Glashow, Sheldon},
-  title        = {Partial Symmetries of Weak Interactions},
-  journaltitle = {Nucl.~Phys.},
-  date         = 1961,
-  volume       = 22,
-  pages        = {579-588},
-}
-
-@article{herrmann,
-  author       = {Herrmann, Wolfgang A. and {\"O}fele, Karl and Schneider,
-                  Sabine K.  and Herdtweck, Eberhardt and Hoffmann, Stephan D.},
-  title        = {A carbocyclic carbene as an efficient catalyst ligand for {C--C}
-                  coupling reactions},
-  journaltitle = anch-ie,
-  date         = 2006,
-  volume       = 45,
-  number       = 23,
-  pages        = {3859-3862},
-  indextitle   = {Carbocyclic carbene as an efficient catalyst, A},
-}
-
-@article{kastenholz,
-  author       = {Kastenholz, M. A. and H{\"u}nenberger, Philippe H.},
-  title        = {Computation of methodology\hyphen independent ionic solvation
-                  free energies from molecular simulations},
-  journaltitle = jchph,
-  date         = 2006,
-  subtitle     = {{I}. {The} electrostatic potential in molecular liquids},
-  volume       = 124,
-  eid          = 124106,
-  doi          = {10.1063/1.2172593},
-  langid       = {english},
-  langidopts   = {variant=american},
-  indextitle   = {Computation of ionic solvation free energies},
-  annotation   = {An \texttt{article} entry with an \texttt{eid} and a
-                  \texttt{doi} field. Note that the \textsc{doi} is transformed
-                  into a clickable link if \texttt{hyperref} support has been
-                  enabled},
-  abstract     = {The computation of ionic solvation free energies from
-                  atomistic simulations is a surprisingly difficult problem that
-                  has found no satisfactory solution for more than 15 years. The
-                  reason is that the charging free energies evaluated from such
-                  simulations are affected by very large errors. One of these is
-                  related to the choice of a specific convention for summing up
-                  the contributions of solvent charges to the electrostatic
-                  potential in the ionic cavity, namely, on the basis of point
-                  charges within entire solvent molecules (M scheme) or on the
-                  basis of individual point charges (P scheme). The use of an
-                  inappropriate convention may lead to a charge-independent
-                  offset in the calculated potential, which depends on the
-                  details of the summation scheme, on the quadrupole-moment
-                  trace of the solvent molecule, and on the approximate form
-                  used to represent electrostatic interactions in the
-                  system. However, whether the M or P scheme (if any) represents
-                  the appropriate convention is still a matter of on-going
-                  debate. The goal of the present article is to settle this
-                  long-standing controversy by carefully analyzing (both
-                  analytically and numerically) the properties of the
-                  electrostatic potential in molecular liquids (and inside
-                  cavities within them).},
-}
-
-@article{murray,
-  author       = {Hostetler, Michael J. and Wingate, Julia E. and Zhong,
-                  Chuan-Jian and Harris, Jay E. and Vachet, Richard W. and
-                  Clark, Michael R.  and Londono, J. David and Green, Stephen
-                  J. and Stokes, Jennifer J.  and Wignall, George D. and Glish,
-                  Gary L. and Porter, Marc D.  and Evans, Neal D. and Murray,
-                  Royce W.},
-  title        = {Alkanethiolate gold cluster molecules with core diameters from
-                  1.5 to 5.2~{nm}},
-  journaltitle = {Langmuir},
-  date         = 1998,
-  subtitle     = {Core and monolayer properties as a function of core size},
-  volume       = 14,
-  number       = 1,
-  pages        = {17-30},
-  langid       = {english},
-  langidopts   = {variant=american},
-  indextitle   = {Alkanethiolate gold cluster molecules},
-  shorttitle   = {Alkanethiolate gold cluster molecules},
-  annotation   = {An \texttt{article} entry with \arabic{author} authors. By
-                  default, long author and editor lists are automatically
-                  truncated. This is configurable},
-}
-
-@article{reese,
-  author       = {Reese, Trevor R.},
-  title        = {Georgia in {Anglo-Spanish} Diplomacy, 1736--1739},
-  journaltitle = {William and Mary Quarterly},
-  date         = 1958,
-  series       = 3,
-  volume       = 15,
-  pages        = {168-190},
-  langid       = {english},
-  langidopts   = {variant=american},
-  annotation   = {An \texttt{article} entry with a \texttt{series} and a
-                  \texttt{volume} field. Note the format of the series. If the
-                  value of the \texttt{series} field is an integer, this number
-                  is printed as an ordinal and the string \enquote*{series} is
-                  appended automatically},
-}
-
-@article{sarfraz,
-  author       = {M. Sarfraz and M. F. A. Razzak},
-  title        = {Technical section: {An} algorithm for automatic capturing of
-                  the font outlines},
-  year         = 2002,
-  volume       = 26,
-  number       = 5,
-  pages        = {795-804},
-  issn         = {0097-8493},
-  journal      = {Computers and Graphics},
-  annotation   = {An \texttt{article} entry with an \texttt{issn} field},
-}
-
-@article{shore,
-  author       = {Shore, Bradd},
-  title        = {Twice-Born, Once Conceived},
-  journaltitle = {American Anthropologist},
-  date         = {1991-03},
-  subtitle     = {Meaning Construction and Cultural Cognition},
-  series       = {newseries},
-  volume       = 93,
-  number       = 1,
-  pages        = {9-27},
-  annotation   = {An \texttt{article} entry with \texttt{series},
-                  \texttt{volume}, and \texttt{number} fields. Note the format
-                  of the \texttt{series} which is a localization key},
-}
-
-@article{sigfridsson,
-  author       = {Sigfridsson, Emma and Ryde, Ulf},
-  title        = {Comparison of methods for deriving atomic charges from the
-                  electrostatic potential and moments},
-  journaltitle = {Journal of Computational Chemistry},
-  date         = 1998,
-  volume       = 19,
-  number       = 4,
-  pages        = {377-395},
-  doi          = {10.1002/(SICI)1096-987X(199803)19:4<377::AID-JCC1>3.0.CO;2-P},
-  langid       = {english},
-  langidopts   = {variant=american},
-  indextitle   = {Methods for deriving atomic charges},
-  annotation   = {An \texttt{article} entry with \texttt{volume},
-                  \texttt{number}, and \texttt{doi} fields. Note that the
-                  \textsc{doi} is transformed into a clickable link if
-                  \texttt{hyperref} support has been enabled},
-  abstract     = {Four methods for deriving partial atomic charges from the
-                  quantum chemical electrostatic potential (CHELP, CHELPG,
-                  Merz-Kollman, and RESP) have been compared and critically
-                  evaluated. It is shown that charges strongly depend on how and
-                  where the potential points are selected. Two alternative
-                  methods are suggested to avoid the arbitrariness in the
-                  point-selection schemes and van der Waals exclusion radii:
-                  CHELP-BOW, which also estimates the charges from the
-                  electrostatic potential, but with potential points that are
-                  Boltzmann-weighted after their occurrence in actual
-                  simulations using the energy function of the program in which
-                  the charges will be used, and CHELMO, which estimates the
-                  charges directly from the electrostatic multipole
-                  moments. Different criteria for the quality of the charges are
-                  discussed.},
-}
-
-@article{spiegelberg,
-  author       = {Spiegelberg, Herbert},
-  title        = {\mkbibquote{Intention} und \mkbibquote{Intentionalit{\"a}t} in
-                  der Scholastik, bei Brentano und Husserl},
-  journaltitle = {Studia Philosophica},
-  date         = 1969,
-  volume       = 29,
-  pages        = {189-216},
-  langid       = {german},
-  sorttitle    = {Intention und Intentionalitat in der Scholastik, bei Brentano
-                  und Husserl},
-  indexsorttitle= {Intention und Intentionalitat in der Scholastik, bei Brentano
-                  und Husserl},
-  shorttitle   = {Intention und Intentionalit{\"a}t},
-  annotation   = {An \texttt{article} entry. Note the \texttt{sorttitle} and
-                  \texttt{indexsorttitle} fields and the markup of the quotes in
-                  the database file},
-}
-
-@article{springer,
-  author       = {Springer, Otto},
-  title        = {Mediaeval Pilgrim Routes from {Scandinavia} to {Rome}},
-  journaltitle = {Mediaeval Studies},
-  date         = 1950,
-  volume       = 12,
-  pages        = {92-122},
-  langid       = {english},
-  langidopts   = {variant=british},
-  shorttitle   = {Mediaeval Pilgrim Routes},
-  annotation   = {A plain \texttt{article} entry},
-}
-
-@article{weinberg,
-  author       = {Weinberg, Steven},
-  title        = {A Model of Leptons},
-  journaltitle = {Phys.~Rev.~Lett.},
-  date         = 1967,
-  volume       = 19,
-  pages        = {1264-1266},
-}
-
-@article{yoon,
-  author       = {Yoon, Myeong S. and Ryu, Dowook and Kim, Jeongryul and Ahn,
-                  Kyo Han},
-  title        = {Palladium pincer complexes with reduced bond angle strain:
-                  efficient catalysts for the {Heck} reaction},
-  journaltitle = {Organometallics},
-  date         = 2006,
-  volume       = 25,
-  number       = 10,
-  pages        = {2409-2411},
-  indextitle   = {Palladium pincer complexes},
-}
-
-@book{aristotle:anima,
-  author       = {Aristotle},
-  title        = {De Anima},
-  date         = 1907,
-  editor       = {Hicks, Robert Drew},
-  publisher    = cup,
-  location     = {Cambridge},
-  keywords     = {primary},
-  langid       = {english},
-  langidopts   = {variant=british},
-  annotation   = {A \texttt{book} entry with an \texttt{author} and an
-                  \texttt{editor}},
-}
-
-@book{aristotle:physics,
-  author       = {Aristotle},
-  title        = {Physics},
-  date         = 1929,
-  translator   = {Wicksteed, P. H. and Cornford, F. M.},
-  publisher    = {G. P. Putnam},
-  location     = {New York},
-  keywords     = {primary},
-  langid       = {english},
-  langidopts   = {variant=american},
-  shorttitle   = {Physics},
-  annotation   = {A \texttt{book} entry with a \texttt{translator} field},
-}
-
-@book{aristotle:poetics,
-  author       = {Aristotle},
-  title        = {Poetics},
-  date         = 1968,
-  editor       = {Lucas, D. W.},
-  series       = {Clarendon {Aristotle}},
-  publisher    = {Clarendon Press},
-  location     = {Oxford},
-  keywords     = {primary},
-  langid       = {english},
-  langidopts   = {variant=british},
-  shorttitle   = {Poetics},
-  annotation   = {A \texttt{book} entry with an \texttt{author} and an
-                  \texttt{editor} as well as a \texttt{series} field},
-}
-
-@book{aristotle:rhetoric,
-  author       = {Aristotle},
-  title        = {The Rhetoric of {Aristotle} with a commentary by the late {Edward
-                  Meredith Cope}},
-  date         = 1877,
-  editor       = {Cope, Edward Meredith},
-  commentator  = {Cope, Edward Meredith},
-  volumes      = 3,
-  publisher    = cup,
-  keywords     = {primary},
-  langid       = {english},
-  langidopts   = {variant=british},
-  sorttitle    = {Rhetoric of Aristotle},
-  indextitle   = {Rhetoric of {Aristotle}, The},
-  shorttitle   = {Rhetoric},
-  annotation   = {A commented edition. Note the concatenation of the
-                  \texttt{editor} and \texttt{commentator} fields as well as the
-                  \texttt{volumes}, \texttt{sorttitle}, and \texttt{indextitle}
-                  fields},
-}
-
-@book{augustine,
-  author       = {Augustine, Robert L.},
-  title        = {Heterogeneous catalysis for the synthetic chemist},
-  date         = 1995,
-  publisher    = {Marcel Dekker},
-  location     = {New York},
-  langid       = {english},
-  langidopts   = {variant=american},
-  shorttitle   = {Heterogeneous catalysis},
-  annotation   = {A plain \texttt{book} entry},
-}
-
-@book{averroes/bland,
-  author       = {Averroes},
-  title        = {The Epistle on the Possibility of Conjunction with the Active
-                  Intellect by {Ibn Rushd} with the Commentary of {Moses Narboni}},
-  date         = 1982,
-  editor       = {Bland, Kalman P.},
-  translator   = {Bland, Kalman P.},
-  series       = {Moreshet: Studies in {Jewish} History, Literature and Thought},
-  number       = 7,
-  publisher    = {Jewish Theological Seminary of America},
-  location     = {New York},
-  keywords     = {primary},
-  langid       = {english},
-  langidopts   = {variant=american},
-  indextitle   = {Epistle on the Possibility of Conjunction, The},
-  shorttitle   = {Possibility of Conjunction},
-  annotation   = {A \texttt{book} entry with a \texttt{series} and a
-                  \texttt{number}. Note the concatenation of the \texttt{editor}
-                  and \texttt{translator} fields as well as the
-                  \texttt{indextitle} field},
-}
-
-@book{averroes/hannes,
-  author       = {Averroes},
-  title        = {Des Averro{\"e}s Abhandlung: \mkbibquote{{\"U}ber die
-                  M{\"o}glichkeit der Conjunktion} oder \mkbibquote{{\"U}ber den
-                  materiellen Intellekt}},
-  date         = 1892,
-  editor       = {Hannes, Ludwig},
-  translator   = {Hannes, Ludwig},
-  annotator    = {Hannes, Ludwig},
-  publisher    = {C.~A. Kaemmerer},
-  location     = {Halle an der Saale},
-  keywords     = {primary},
-  langid       = {german},
-  sorttitle    = {Uber die Moglichkeit der Conjunktion},
-  indexsorttitle= {Uber die Moglichkeit der Conjunktion},
-  indextitle   = {{\"U}ber die M{\"o}glichkeit der Conjunktion},
-  shorttitle   = {{\"U}ber die M{\"o}glichkeit der Conjunktion},
-  annotation   = {An annotated edition. Note the concatenation of the
-                  \texttt{editor}, \texttt{translator}, and \texttt{annotator}
-                  fields. Also note the \texttt{shorttitle},
-                  \texttt{indextitle}, \texttt{sorttitle}, and
-                  \texttt{indexsorttitle} fields},
-}
-
-@book{averroes/hercz,
-  author       = {Averroes},
-  title        = {Drei Abhandlungen {\"u}ber die Conjunction des separaten
-                  Intellects mit dem Menschen},
-  date         = 1869,
-  editor       = {Hercz, J.},
-  translator   = {Hercz, J.},
-  publisher    = {S.~Hermann},
-  location     = {Berlin},
-  keywords     = {primary},
-  langid       = {german},
-  indexsorttitle= {Drei Abhandlungen uber die Conjunction},
-  indextitle   = {Drei Abhandlungen {\"u}ber die Conjunction},
-  subtitle     = {Von Averroes (Vater und Sohn), aus dem Arabischen
-                  {\"u}bersetzt von Samuel Ibn Tibbon},
-  shorttitle   = {Drei Abhandlungen},
-  annotation   = {A \texttt{book} entry. Note the concatenation of the
-                  \texttt{editor} and \texttt{translator} fields as well as the
-                  \texttt{indextitle} and \texttt{indexsorttitle} fields},
-}
-
-@book{cicero,
-  author       = {Cicero, Marcus Tullius},
-  title        = {De natura deorum. {\"U}ber das Wesen der G{\"o}tter},
-  date         = 1995,
-  editor       = {Blank-Sangmeister, Ursula},
-  translator   = {Blank-Sangmeister, Ursula},
-  afterword    = {Thraede, Klaus},
-  language     = {langlatin and langgerman},
-  publisher    = {Reclam},
-  location     = {Stuttgart},
-  langid       = {german},
-  indextitle   = {De natura deorum},
-  shorttitle   = {De natura deorum},
-  annotation   = {A bilingual edition of Cicero's \emph{De natura deorum}, with
-                  a German translation. Note the format of the \texttt{language}
-                  field in the database file, the concatenation of the
-                  \texttt{editor} and \texttt{translator} fields, and the
-                  \texttt{afterword} field},
-}
-
-@book{coleridge,
-  author       = {Coleridge, Samuel Taylor},
-  title        = {Biographia literaria, or {Biographical} sketches of my literary
-                  life and opinions},
-  date         = 1983,
-  editor       = {Coburn, Kathleen and Engell, James and Bate, W. Jackson},
-  maintitle    = {The collected works of {Samuel Taylor Coleridge}},
-  volume       = 7,
-  part         = 2,
-  series       = {Bollingen Series},
-  number       = 75,
-  publisher    = {Routledge {and} Kegan Paul},
-  location     = {London},
-  langid       = {english},
-  langidopts   = {variant=british},
-  indextitle   = {Biographia literaria},
-  shorttitle   = {Biographia literaria},
-  annotation   = {One (partial) volume of a multivolume book. This is a
-                  \texttt{book} entry with a \texttt{volume} and a \texttt{part}
-                  field which explicitly refers to the second (physical) part of
-                  the seventh (logical) volume. Also note the \texttt{series}
-                  and \texttt{number} fields},
-}
-
-@book{companion,
-  author       = {Goossens, Michel and Mittelbach, Frank and Samarin, Alexander},
-  title        = {The {LaTeX} Companion},
-  date         = 1994,
-  edition      = 1,
-  publisher    = {Addison-Wesley},
-  location     = {Reading, Mass.},
-  pagetotal    = 528,
-  langid       = {english},
-  langidopts   = {variant=american},
-  sorttitle    = {LaTeX Companion},
-  indextitle   = {LaTeX Companion, The},
-  shorttitle   = {LaTeX Companion},
-  annotation   = {A book with three authors. Note the formatting of the author
-                  list. By default, only the first name is reversed in the
-                  bibliography},
-}
-
-@book{cotton,
-  author       = {Cotton, Frank Albert and Wilkinson, Geoffrey and Murillio,
-                  Carlos A. and Bochmann, Manfred},
-  title        = {Advanced inorganic chemistry},
-  date         = 1999,
-  edition      = 6,
-  publisher    = {Wiley},
-  location     = {Chichester},
-  langid       = {english},
-  langidopts   = {variant=british},
-  annotation   = {A \texttt{book} entry with \arabic{author} authors and an
-                  \texttt{edition} field. By default, long \texttt{author} and
-                  \texttt{editor} lists are automatically truncated. This is
-                  configurable},
-}
-
-@book{gerhardt,
-  author       = {Gerhardt, Michael J.},
-  title        = {The Federal Appointments Process},
-  date         = 2000,
-  publisher    = {Duke University Press},
-  location     = {Durham and London},
-  langid       = {english},
-  langidopts   = {variant=american},
-  sorttitle    = {Federal Appointments Process},
-  indextitle   = {Federal Appointments Process, The},
-  subtitle     = {A Constitutional and Historical Analysis},
-  shorttitle   = {Federal Appointments Process},
-  annotation   = {This is a \texttt{book} entry. Note the format of the
-                  \texttt{location} field as well as the \texttt{sorttitle} and
-                  \texttt{indextitle} fields},
-}
-
-@book{gonzalez,
-  author       = {Gonzalez, Ray},
-  title        = {The Ghost of {John Wayne} and Other Stories},
-  date         = 2001,
-  publisher    = {The University of Arizona Press},
-  location     = {Tucson},
-  isbn         = {0-816-52066-6},
-  langid       = {english},
-  langidopts   = {variant=american},
-  sorttitle    = {Ghost of John Wayne and Other Stories},
-  indextitle   = {Ghost of {John Wayne} and Other Stories, The},
-  shorttitle   = {Ghost of {John Wayne}},
-  annotation   = {A collection of short stories. This is a \texttt{book} entry.
-                  Note the \texttt{sorttitle} and \texttt{indextitle} fields in
-                  the database file. There's also an \texttt{isbn} field},
-}
-
-@book{hammond,
-  author       = {Hammond, Christopher},
-  title        = {The basics of crystallography and diffraction},
-  date         = 1997,
-  publisher    = {International Union of Crystallography and Oxford University
-                  Press},
-  location     = {Oxford},
-  langid       = {english},
-  langidopts   = {variant=british},
-  sorttitle    = {Basics of crystallography and diffraction},
-  indextitle   = {Basics of crystallography and diffraction, The},
-  shorttitle   = {Crystallography and diffraction},
-  annotation   = {A \texttt{book} entry. Note the \texttt{sorttitle} and
-                  \texttt{indextitle} fields as well as the format of the
-                  \texttt{publisher} field},
-}
-
-@book{iliad,
-  author       = {Homer},
-  title        = {Die Ilias},
-  date         = 2004,
-  translator   = {Schadewaldt, Wolfgang},
-  introduction = {Latacz, Joachim},
-  edition      = 3,
-  publisher    = {Artemis \& Winkler},
-  location     = {D{\"u}sseldorf and Z{\"u}rich},
-  langid       = {german},
-  sorttitle    = {Ilias},
-  indextitle   = {Ilias, Die},
-  shorttitle   = {Ilias},
-  annotation   = {A German translation of the \emph{Iliad}. Note the
-                  \texttt{translator} and \texttt{introduction} fields and the
-                  format of the \texttt{location} field in the database
-                  file. Also note the \texttt{sorttitle} and \texttt{indextitle}
-                  fields},
-}
-
-@book{knuth:ct,
-  author       = {Knuth, Donald E.},
-  title        = {Computers \& Typesetting},
-  date         = {1984/1986},
-  volumes      = 5,
-  publisher    = {Addison-Wesley},
-  location     = {Reading, Mass.},
-  langid       = {english},
-  langidopts   = {variant=american},
-  sortyear     = {1984-0},
-  sorttitle    = {Computers & Typesetting},
-  indexsorttitle= {Computers & Typesetting},
-  annotation   = {A five-volume book cited as a whole. This is a \texttt{book}
-                  entry, note the \texttt{volumes} field},
-}
-
-@book{knuth:ct:a,
-  author       = {Knuth, Donald E.},
-  title        = {The {\TeX} book},
-  date         = 1984,
-  maintitle    = {Computers \& Typesetting},
-  volume       = {A},
-  publisher    = {Addison-Wesley},
-  location     = {Reading, Mass.},
-  langid       = {english},
-  langidopts   = {variant=american},
-  sortyear     = {1984-1},
-  sorttitle    = {Computers & Typesetting A},
-  indexsorttitle= {The TeXbook},
-  indextitle   = {\protect\TeX book, The},
-  shorttitle   = {\TeX book},
-  annotation   = {The first volume of a five-volume book. Note the
-                  \texttt{sorttitle} and \texttt{sortyear} fields. We want this
-                  volume to be listed after the entry referring to the entire
-                  five-volume set. Also note the \texttt{indextitle} and
-                  \texttt{indexsorttitle} fields. Indexing packages that don't
-                  generate robust index entries require some control sequences
-                  to be protected from expansion},
-}
-
-@book{knuth:ct:b,
-  author       = {Knuth, Donald E.},
-  title        = {\TeX: The Program},
-  date         = 1986,
-  maintitle    = {Computers \& Typesetting},
-  volume       = {B},
-  publisher    = {Addison-Wesley},
-  location     = {Reading, Mass.},
-  langid       = {english},
-  langidopts   = {variant=american},
-  sortyear     = {1986-1},
-  sorttitle    = {Computers & Typesetting B},
-  indexsorttitle= {TeX: The Program},
-  shorttitle   = {\TeX},
-  annotation   = {The second volume of a five-volume book. Note the
-                  \texttt{sorttitle} and \texttt{sortyear} fields. Also note the
-                  \texttt{indexsorttitle} field},
-}
-
-@book{knuth:ct:c,
-  author       = {Knuth, Donald E.},
-  title        = {The {METAFONTbook}},
-  date         = 1986,
-  maintitle    = {Computers \& Typesetting},
-  volume       = {C},
-  publisher    = {Addison-Wesley},
-  location     = {Reading, Mass.},
-  langid       = {english},
-  langidopts   = {variant=american},
-  sortyear     = {1986-2},
-  sorttitle    = {Computers & Typesetting C},
-  indextitle   = {METAFONTbook, The},
-  shorttitle   = {METAFONTbook},
-  annotation   = {The third volume of a five-volume book. Note the
-                  \texttt{sorttitle} and \texttt{sortyear} fields as well as the
-                  \texttt{indextitle} field},
-}
-
-@book{knuth:ct:d,
-  author       = {Knuth, Donald E.},
-  title        = {METAFONT: The Program},
-  date         = 1986,
-  maintitle    = {Computers \& Typesetting},
-  volume       = {D},
-  publisher    = {Addison-Wesley},
-  location     = {Reading, Mass.},
-  langid       = {english},
-  langidopts   = {variant=american},
-  sortyear     = {1986-3},
-  sorttitle    = {Computers & Typesetting D},
-  shorttitle   = {METAFONT},
-  annotation   = {The fourth volume of a five-volume book. Note the
-                  \texttt{sorttitle} and \texttt{sortyear} fields},
-}
-
-@book{knuth:ct:e,
-  author       = {Knuth, Donald E.},
-  title        = {{Computer Modern} Typefaces},
-  date         = 1986,
-  maintitle    = {Computers \& Typesetting},
-  volume       = {E},
-  publisher    = {Addison-Wesley},
-  location     = {Reading, Mass.},
-  langid       = {english},
-  langidopts   = {variant=american},
-  sortyear     = {1986-4},
-  sorttitle    = {Computers & Typesetting E},
-  annotation   = {The fifth volume of a five-volume book. Note the
-                  \texttt{sorttitle} and \texttt{sortyear} fields},
-}
-
-@book{knuth:ct:related,
-  author       = {Knuth, Donald E.},
-  title        = {Computers \& Typesetting},
-  date         = {1984/1986},
-  volumes      = 5,
-  publisher    = {Addison-Wesley},
-  location     = {Reading, Mass.},
-  langid       = {english},
-  langidopts   = {variant=american},
-  sortyear     = {1984-0},
-  sorttitle    = {Computers & Typesetting},
-  indexsorttitle= {Computers & Typesetting},
-  related      = {knuth:ct:a,knuth:ct:b,knuth:ct:c,knuth:ct:d,knuth:ct:e},
-  relatedtype  = {multivolume},
-  annotation   = {A five-volume book cited as a whole and related to its
-                  individual volumes. Note the \texttt{related} and
-                  \texttt{relatedtype} fields},
-}
-
-@book{kullback,
-  author       = {Kullback, Solomon},
-  title        = {Information Theory and Statistics},
-  year         = 1959,
-  publisher    = {John Wiley \& Sons},
-  location     = {New York},
-  langid       = {english},
-  langidopts   = {variant=american},
-}
-
-@book{kullback:reprint,
-  author       = {Kullback, Solomon},
-  title        = {Information Theory and Statistics},
-  year         = 1997,
-  publisher    = {Dover Publications},
-  location     = {New York},
-  origyear     = 1959,
-  origpublisher= {John Wiley \& Sons},
-  langid       = {english},
-  langidopts   = {variant=american},
-  annotation   = {A reprint of the \texttt{kullback} entry. Note the format of
-                  \texttt{origyear} and \texttt{origpublisher}. These fields are
-                  not used by the standard bibliography styles},
-}
-
-@book{kullback:related,
-  author       = {Kullback, Solomon},
-  title        = {Information Theory and Statistics},
-  year         = 1997,
-  publisher    = {Dover Publications},
-  location     = {New York},
-  langid       = {english},
-  langidopts   = {variant=american},
-  related      = {kullback},
-  relatedtype  = {origpubin},
-  annotation   = {A reprint of the \texttt{kullback} entry. Note the format of
-                  the \texttt{related} and \texttt{relatedtype} fields},
-}
-
-@book{malinowski,
-  author       = {Malinowski, Bronis{\l}aw},
-  title        = {Argonauts of the {Western Pacific}},
-  date         = 1972,
-  edition      = 8,
-  publisher    = {Routledge {and} Kegan Paul},
-  location     = {London},
-  langid       = {english},
-  langidopts   = {variant=british},
-  subtitle     = {An account of native enterprise and adventure in the
-                  Archipelagoes of {Melanesian New Guinea}},
-  shorttitle   = {Argonauts},
-  annotation   = {This is a \texttt{book} entry. Note the format of the
-                  \texttt{publisher} and \texttt{edition} fields as well as the
-                  \texttt{subtitle} field},
-}
-
-@book{maron,
-  author       = {Maron, Monika},
-  title        = {Animal Triste},
-  date         = 2000,
-  translator   = {Brigitte Goldstein},
-  origlanguage = {german},
-  publisher    = {University of Nebraska Press},
-  location     = {Lincoln},
-  langid       = {english},
-  langidopts   = {variant=american},
-  shorttitle   = {Animal Triste},
-  annotation   = {An English translation of a German novel with a French title.
-                  In other words: a \texttt{book} entry with a
-                  \texttt{translator} field.  Note the \texttt{origlanguage}
-                  field which is concatenated with the \texttt{translator}},
-}
-
-@book{massa,
-  author       = {Werner Massa},
-  title        = {Crystal structure determination},
-  date         = 2004,
-  edition      = 2,
-  publisher    = {Spinger},
-  location     = {Berlin},
-  langid       = {english},
-  langidopts   = {variant=british},
-  annotation   = {A \texttt{book} entry with an \texttt{edition} field},
-}
-
-@article{moore,
-  author       = {Moore, Gordon E.},
-  title        = {Cramming more components onto integrated circuits},
-  journaltitle = {Electronics},
-  year         = 1965,
-  volume       = 38,
-  number       = 8,
-  pages        = {114-117},
-  langid       = {english},
-  langidopts   = {variant=american},
-}
-
-@article{moore:related,
-  author       = {Moore, Gordon E.},
-  title        = {Cramming more components onto integrated circuits},
-  journaltitle = {Proceedings of the {IEEE}},
-  year         = 1998,
-  volume       = 86,
-  number       = 1,
-  pages        = {82-85},
-  langid       = {english},
-  langidopts   = {variant=american},
-  related      = {moore},
-  relatedtype  = {reprintfrom},
-  annotation   = {A reprint of Moore's law. Note the \texttt{related} and
-                  \texttt{relatedtype} fields},
-}
-
-@book{nietzsche:ksa,
-  author       = {Nietzsche, Friedrich},
-  title        = {S{\"a}mtliche Werke},
-  date         = 1988,
-  editor       = {Colli, Giorgio and Montinari, Mazzino},
-  edition      = 2,
-  volumes      = 15,
-  publisher    = dtv # { and Walter de Gruyter},
-  location     = {M{\"u}nchen and Berlin and New York},
-  langid       = {german},
-  sortyear     = {1988-0},
-  sorttitle    = {Werke-00-000},
-  indexsorttitle= {Samtliche Werke},
-  subtitle     = {Kritische Studienausgabe},
-  annotation   = {The critical edition of Nietzsche's works. This is a
-                  \texttt{book} entry referring to a 15-volume work as a
-                  whole. Note the \texttt{volumes} field and the format of the
-                  \texttt{publisher} and \texttt{location} fields in the
-                  database file. Also note the \texttt{sorttitle} and
-                  \texttt{sortyear} fields which are used to fine-tune the
-                  sorting order of the bibliography. We want this item listed
-                  first in the bibliography},
-}
-
-@book{nietzsche:ksa1,
-  author       = {Nietzsche, Friedrich},
-  title        = {Die Geburt der Trag{\"o}die. Unzeitgem{\"a}{\ss}e
-                  Betrachtungen I--IV. Nachgelassene Schriften 1870--1973},
-  date         = 1988,
-  editor       = {Colli, Giorgio and Montinari, Mazzino},
-  maintitle    = {S{\"a}mtliche Werke},
-  mainsubtitle = {Kritische Studienausgabe},
-  volume       = 1,
-  edition      = 2,
-  publisher    = dtv # { and Walter de Gruyter},
-  location     = {M{\"u}nchen and Berlin and New York},
-  langid       = {german},
-  sortyear     = {1988-1},
-  sorttitle    = {Werke-01-000},
-  indexsorttitle= {Samtliche Werke I},
-  bookauthor   = {Nietzsche, Friedrich},
-  indextitle   = {S{\"a}mtliche Werke I},
-  shorttitle   = {S{\"a}mtliche Werke I},
-  annotation   = {A single volume from the critical edition of Nietzsche's
-                  works. This \texttt{book} entry explicitly refers to the first
-                  volume only. Note the \texttt{title} and \texttt{maintitle}
-                  fields. Also note the \texttt{sorttitle} and \texttt{sortyear}
-                  fields. We want this entry to be listed after the entry
-                  referring to the entire edition},
-}
-
-@book{nussbaum,
-  author       = {Nussbaum, Martha},
-  title        = {Aristotle's \mkbibquote{De Motu Animalium}},
-  date         = 1978,
-  publisher    = pup,
-  location     = {Princeton},
-  keywords     = {secondary},
-  langid       = {english},
-  langidopts   = {variant=american},
-  sorttitle    = {Aristotle's De Motu Animalium},
-  indexsorttitle= {Aristotle's De Motu Animalium},
-  annotation   = {A \texttt{book} entry. Note the \texttt{sorttitle} and
-                  \texttt{indexsorttitle} fields and the markup of the quotes in
-                  the database file},
-}
-
-@book{piccato,
-  author       = {Piccato, Pablo},
-  title        = {City of Suspects},
-  date         = 2001,
-  publisher    = {Duke University Press},
-  location     = {Durham and London},
-  langid       = {english},
-  langidopts   = {variant=american},
-  subtitle     = {Crime in {Mexico City}, 1900--1931},
-  shorttitle   = {City of Suspects},
-  annotation   = {This is a \texttt{book} entry. Note the format of the
-                  \texttt{location} field in the database file},
-}
-
-@book{vangennep,
-  author       = {van Gennep, Arnold},
-  title        = {Les rites de passage},
-  date         = 1909,
-  publisher    = {Nourry},
-  location     = {Paris},
-  options      = {useprefix},
-  langid       = {french},
-  sorttitle    = {Rites de passage},
-  indextitle   = {Rites de passage, Les},
-  shorttitle   = {Rites de passage},
-  annotation   = {A \texttt{book} entry. Note the format of the printed name and
-                  compare the \texttt{useprefix} option in the \texttt{options}
-                  field as well as \texttt{brandt} and \texttt{geer}},
-}
-
-@book{vangennep:trans,
-  author       = {van Gennep, Arnold},
-  title        = {The Rites of Passage},
-  year         = 1960,
-  translator   = {Vizedom, Monika B. and Caffee, Gabrielle L.},
-  language     = {english},
-  origlanguage = {french},
-  publisher    = {University of Chicago Press},
-  options      = {useprefix},
-  indextitle   = {Rites of Passage, The},
-  sorttitle    = {Rites of Passage},
-  shorttitle   = {Rites of Passage},
-  langid       = {english},
-  langidopts   = {variant=american},
-  annotation   = {A translation of the \texttt{vangennep} entry. Note the
-                  \texttt{translator} and \texttt{origlanguage} fields. Compare
-                  with the \texttt{vangennep:related} entry.},
-}
-
-@book{vangennep:related,
-  author       = {van Gennep, Arnold},
-  title        = {Les rites de passage},
-  date         = 1909,
-  publisher    = {Nourry},
-  location     = {Paris},
-  options      = {useprefix},
-  langid       = {french},
-  related      = {vizedom:related},
-  relatedtype  = {bytranslator},
-  sorttitle    = {Rites de passage},
-  indextitle   = {Rites de passage, Les},
-  shorttitle   = {Rites de passage},
-  annotation   = {A variant of the \texttt{vangennep} entry related to its
-                  translation. Note the format of the \texttt{related} and
-                  \texttt{relatedtype} fields},
-}
-
-@book{vizedom:related,
-  title        = {The Rites of Passage},
-  year         = 1960,
-  translator   = {Vizedom, Monika B. and Caffee, Gabrielle L.},
-  language     = {english},
-  publisher    = {University of Chicago Press},
-  langid       = {english},
-  langidopts   = {variant=american},
-  options      = {usetranslator},
-  related      = {vangennep},
-  relatedtype  = {translationof},
-  indextitle   = {Rites of Passage, The},
-  sorttitle    = {Rites of Passage},
-  shorttitle   = {Rites of Passage},
-  annotation   = {A translated work from \texttt{vangennep}. Note the format of
-                  the \texttt{related} and \texttt{relatedtype} fields},
-}
-
-@book{vazques-de-parga,
-  author       = {V{\'a}zques{ de }Parga, Luis and Lacarra, Jos{\'e} Mar{\'i}a
-                  and Ur{\'i}a R{\'i}u, Juan},
-  title        = {Las Peregrinaciones a Santiago de Compostela},
-  date         = 1993,
-  volumes      = 3,
-  note         = {Ed. facs. de la realizada en 1948--49},
-  publisher    = {Iberdrola},
-  location     = {Pamplona},
-  langid       = {spanish},
-  sorttitle    = {Peregrinaciones a Santiago de Compostela},
-  indextitle   = {Peregrinaciones a Santiago de Compostela, Las},
-  shorttitle   = {Peregrinaciones},
-  annotation   = {A multivolume book cited as a whole. This is a \texttt{book}
-                  entry with \texttt{volumes}, \texttt{note},
-                  \texttt{sorttitle}, and \texttt{indextitle} fields},
-}
-
-@book{wilde,
-  author       = {Wilde, Oscar},
-  title        = {The Importance of Being Earnest: A Trivial Comedy for Serious
-                  People},
-  year         = 1899,
-  series       = {English and {American} drama of the Nineteenth Century},
-  publisher    = {Leonard Smithers {and} Company},
-  eprint       = {4HIWAAAAYAAJ},
-  eprinttype   = {googlebooks},
-  annotation   = {A \texttt{book} with \texttt{eprint} and \texttt{eprinttype}
-                  fields.},
-}
-
-@book{worman,
-  author       = {Worman, Nancy},
-  title        = {The Cast of Character},
-  date         = 2002,
-  publisher    = {University of Texas Press},
-  location     = {Austin},
-  langid       = {english},
-  langidopts   = {variant=american},
-  sorttitle    = {Cast of Character},
-  indextitle   = {Cast of Character, The},
-  subtitle     = {Style in {Greek} Literature},
-  shorttitle   = {Cast of Character},
-  annotation   = {A \texttt{book} entry. Note the \texttt{sorttitle} and
-                  \texttt{indextitle} fields},
-}
-
-@collection{britannica,
-  editor       = {Preece, Warren E.},
-  title        = {The {New Encyclop{\ae}dia Britannica}},
-  date         = 2003,
-  edition      = 15,
-  volumes      = 32,
-  publisher    = {Encyclop{\ae}dia Britannica},
-  location     = {Chicago, Ill.},
-  options      = {useeditor=false},
-  label        = {EB},
-  langid       = {english},
-  langidopts   = {variant=british},
-  sorttitle    = {Encyclop{\ae}dia Britannica},
-  indextitle   = {{Encyclop{\ae}dia Britannica}, The {New}},
-  shorttitle   = {{Encyclop{\ae}dia Britannica}},
-  annotation   = {This is a \texttt{collection} entry for an encyclopedia. Note
-                  the \texttt{useeditor} option in the \texttt{options} field as
-                  well as the \texttt{sorttitle} field. We want this entry to be
-                  cited and alphabetized by title even though there is an
-                  editor. In addition to that, we want the title to be
-                  alphabetized under \enquote*{E} rather than \enquote*{T}. Also
-                  note the \texttt{label} field which is provided for
-                  author-year citation styles},
-}
-
-@collection{gaonkar,
-  editor       = {Gaonkar, Dilip Parameshwar},
-  title        = {Alternative Modernities},
-  date         = 2001,
-  publisher    = {Duke University Press},
-  location     = {Durham and London},
-  isbn         = {0-822-32714-7},
-  langid       = {english},
-  langidopts   = {variant=american},
-  annotation   = {This is a \texttt{collection} entry. Note the format of the
-                  \texttt{location} field in the database file as well as the
-                  \texttt{isbn} field},
-}
-
-@InCollection{gaonkar:in,
-  author       = {Gaonkar, Dilip Parameshwar},
-  editor       = {Gaonkar, Dilip Parameshwar},
-  title        = {On Alternative Modernities},
-  date         = 2001,
-  booktitle    = {Alternative Modernities},
-  publisher    = {Duke University Press},
-  location     = {Durham and London},
-  isbn         = {0-822-32714-7},
-  pages        = {1-23},
-}
-
-@collection{jaffe,
-  editor       = {Jaff{\'e}, Philipp},
-  title        = {Regesta Pontificum Romanorum ab condita ecclesia ad annum post
-                  Christum natum \textsc{mcxcviii}},
-  date         = {1885/1888},
-  editora      = {Loewenfeld, Samuel and Kaltenbrunner, Ferdinand and Ewald,
-                  Paul},
-  edition      = 2,
-  volumes      = 2,
-  location     = {Leipzig},
-  langid       = {latin},
-  editoratype  = {redactor},
-  indextitle   = {Regesta Pontificum Romanorum},
-  shorttitle   = {Regesta Pontificum Romanorum},
-  annotation   = {A \texttt{collection} entry with \texttt{edition} and
-                  \texttt{volumes} fields. Note the \texttt{editora} and
-                  \texttt{editoratype} fields},
-}
-
-@collection{westfahl:frontier,
-  editor       = {Westfahl, Gary},
-  title        = {Space and Beyond},
-  date         = 2000,
-  subtitle     = {The Frontier Theme in Science Fiction},
-  publisher    = {Greenwood},
-  location     = {Westport, Conn. and London},
-  langid       = {english},
-  langidopts   = {variant=american},
-  booktitle    = {Space and Beyond},
-  booksubtitle = {The Frontier Theme in Science Fiction},
-  annotation   = {This is a \texttt{collection} entry. Note the format of the
-                  \texttt{location} field as well as the \texttt{subtitle} and
-                  \texttt{booksubtitle} fields},
-}
-
-@inbook{kant:kpv,
-  title        = {Kritik der praktischen Vernunft},
-  date         = 1968,
-  author       = {Kant, Immanuel},
-  booktitle    = {Kritik der praktischen Vernunft. Kritik der Urtheilskraft},
-  bookauthor   = {Kant, Immanuel},
-  maintitle    = {Kants Werke. Akademie Textausgabe},
-  volume       = 5,
-  publisher    = {Walter de Gruyter},
-  location     = {Berlin},
-  pages        = {1-163},
-  shorthand    = {KpV},
-  langid       = {german},
-  shorttitle   = {Kritik der praktischen Vernunft},
-  annotation   = {An edition of Kant's \emph{Collected Works}, volume five. This
-                  is an \texttt{inbook} entry which explicitly refers to the
-                  \emph{Critique of Practical Reason} only, not to the entire
-                  fifth volume. Note the \texttt{author} and \texttt{bookauthor}
-                  fields in the database file. By default, the
-                  \texttt{bookauthor} is omitted if the values of the
-                  \texttt{author} and \texttt{bookauthor} fields are identical},
-}
-
-@inbook{kant:ku,
-  title        = {Kritik der Urtheilskraft},
-  date         = 1968,
-  author       = {Kant, Immanuel},
-  booktitle    = {Kritik der praktischen Vernunft. Kritik der Urtheilskraft},
-  bookauthor   = {Kant, Immanuel},
-  maintitle    = {Kants Werke. Akademie Textausgabe},
-  volume       = 5,
-  publisher    = {Walter de Gruyter},
-  location     = {Berlin},
-  pages        = {165-485},
-  shorthand    = {KU},
-  langid       = {german},
-  annotation   = {An edition of Kant's \emph{Collected Works}, volume five. This
-                  is an \texttt{inbook} entry which explicitly refers to the
-                  \emph{Critique of Judgment} only, not to the entire fifth
-                  volume},
-}
-
-@inbook{nietzsche:historie,
-  title        = {Unzeitgem{\"a}sse Betrachtungen. Zweites St{\"u}ck},
-  date         = 1988,
-  author       = {Nietzsche, Friedrich},
-  booktitle    = {Die Geburt der Trag{\"o}die. Unzeitgem{\"a}{\ss}e
-                  Betrachtungen I--IV. Nachgelassene Schriften 1870--1973},
-  bookauthor   = {Nietzsche, Friedrich},
-  editor       = {Colli, Giorgio and Montinari, Mazzino},
-  subtitle     = {Vom Nutzen und Nachtheil der Historie f{\"u}r das Leben},
-  maintitle    = {S{\"a}mtliche Werke},
-  mainsubtitle = {Kritische Studienausgabe},
-  volume       = 1,
-  publisher    = dtv # { and Walter de Gruyter},
-  location     = {M{\"u}nchen and Berlin and New York},
-  pages        = {243-334},
-  langid       = {german},
-  sortyear     = {1988-2},
-  sorttitle    = {Werke-01-243},
-  indexsorttitle= {Vom Nutzen und Nachtheil der Historie fur das Leben},
-  indextitle   = {Vom Nutzen und Nachtheil der Historie f{\"u}r das Leben},
-  shorttitle   = {Vom Nutzen und Nachtheil der Historie},
-  annotation   = {A single essay from the critical edition of Nietzsche's works.
-                  This \texttt{inbook} entry explicitly refers to an essay found
-                  in the first volume. Note the \texttt{title},
-                  \texttt{booktitle}, and \texttt{maintitle} fields. Also note
-                  the \texttt{sorttitle} and \texttt{sortyear} fields. We want
-                  this entry to be listed after the entry referring to the
-                  entire first volume},
-}
-
-@incollection{brandt,
-  author       = {von Brandt, Ahasver and Erich Hoffmann},
-  editor       = {Ferdinand Seibt},
-  title        = {Die nordischen L{\"a}nder von der Mitte des 11.~Jahrhunderts
-                  bis 1448},
-  date         = 1987,
-  booktitle    = {Europa im Hoch- und Sp{\"a}tmittelalter},
-  series       = {Handbuch der europ{\"a}ischen Geschichte},
-  number       = 2,
-  publisher    = {Klett-Cotta},
-  location     = {Stuttgart},
-  pages        = {884-917},
-  options      = {useprefix=false},
-  langid       = {german},
-  indexsorttitle= {Nordischen Lander von der Mitte des 11. Jahrhunderts bis
-                  1448},
-  indextitle   = {Nordischen L{\"a}nder von der Mitte des 11.~Jahrhunderts bis
-                  1448, Die},
-  shorttitle   = {Die nordischen L{\"a}nder},
-  annotation   = {An \texttt{incollection} entry with a \texttt{series} and a
-                  \texttt{number}. Note the format of the printed name and
-                  compare the \texttt{useprefix} option in the \texttt{options}
-                  field as well as \texttt{vangennep}. Also note the
-                  \texttt{indextitle, and \texttt{indexsorttitle} fields}},
-}
-
-@incollection{hyman,
-  author       = {Arthur Hyman},
-  editor       = {O'Meara, Dominic J.},
-  title        = {Aristotle's Theory of the Intellect and its Interpretation by
-                  {Averroes}},
-  date         = 1981,
-  booktitle    = {Studies in {Aristotle}},
-  series       = {Studies in Philosophy and the History of Philosophy},
-  number       = 9,
-  publisher    = {The Catholic University of America Press},
-  location     = {Washington, D.C.},
-  pages        = {161-191},
-  keywords     = {secondary},
-  langid       = {english},
-  langidopts   = {variant=american},
-  indextitle   = {Aristotle's Theory of the Intellect},
-  shorttitle   = {Aristotle's Theory of the Intellect},
-  annotation   = {An \texttt{incollection} entry with a \texttt{series} and
-                  \texttt{number} field},
-}
-
-@incollection{pines,
-  author       = {Pines, Shlomo},
-  editor       = {Twersky, Isadore},
-  title        = {The Limitations of Human Knowledge According to {Al-Farabi}, {ibn
-                  Bajja}, and {Maimonides}},
-  date         = 1979,
-  booktitle    = {Studies in Medieval {Jewish} History and Literature},
-  publisher    = hup,
-  location     = {Cambridge, Mass.},
-  pages        = {82-109},
-  keywords     = {secondary},
-  langid       = {english},
-  langidopts   = {variant=american},
-  indextitle   = {Limitations of Human Knowledge According to {Al-Farabi}, {ibn
-                  Bajja}, and {Maimonides}, The},
-  shorttitle   = {Limitations of Human Knowledge},
-  annotation   = {A typical \texttt{incollection} entry. Note the
-                  \texttt{indextitle} field},
-}
-
-@inproceedings{moraux,
-  author       = {Moraux, Paul},
-  editor       = {Lloyd, G. E. R. and Owen, G. E. L.},
-  title        = {Le \emph{De Anima} dans la tradition gr{\`e}cque},
-  date         = 1979,
-  booktitle    = {Aristotle on Mind and the Senses},
-  subtitle     = {Quelques aspects de l'interpretation du trait{\'e}, de
-                  Theophraste {\`a} Themistius},
-  booktitleaddon= {Proceedings of the Seventh Symposium Aristotelicum},
-  eventdate    = 1975,
-  publisher    = cup,
-  location     = {Cambridge},
-  pages        = {281-324},
-  keywords     = {secondary},
-  langid       = {french},
-  indexsorttitle= {De Anima dans la tradition grecque},
-  indextitle   = {\emph{De Anima} dans la tradition gr{\`e}cque, Le},
-  shorttitle   = {\emph{De Anima} dans la tradition gr{\`e}cque},
-  annotation   = {This is a typical \texttt{inproceedings} entry. Note the
-                  \texttt{booksubtitle}, \texttt{shorttitle},
-                  \texttt{indextitle}, and \texttt{indexsorttitle} fields. Also
-                  note the \texttt{eventdate} field.},
-}
-
-@inproceedings{salam,
-  author       = {Salam, Abdus},
-  editor       = {Svartholm, Nils},
-  title        = {Weak and Electromagnetic Interactions},
-  date         = 1968,
-  booktitle    = {Elementary particle theory},
-  booksubtitle = {Relativistic groups and analyticity},
-  booktitleaddon= {Proceedings of the {Eighth Nobel Symposium}},
-  eventdate    = {1968-05-19/1968-05-25},
-  venue        = {Aspen{\"a}sgarden, Lerum},
-  publisher    = {Almquist \& Wiksell},
-  location     = {Stockholm},
-  pages        = {367-377},
-}
-
-@manual{cms,
-  title        = {The {Chicago} Manual of Style},
-  date         = 2003,
-  subtitle     = {The Essential Guide for Writers, Editors, and Publishers},
-  edition      = 15,
-  publisher    = {University of Chicago Press},
-  location     = {Chicago, Ill.},
-  isbn         = {0-226-10403-6},
-  label        = {CMS},
-  langid       = {english},
-  langidopts   = {variant=american},
-  sorttitle    = {Chicago Manual of Style},
-  indextitle   = {Chicago Manual of Style, The},
-  shorttitle   = {Chicago Manual of Style},
-  annotation   = {This is a \texttt{manual} entry without an \texttt{author} or
-                  \texttt{editor}. Note the \texttt{label} field in the database
-                  file which is provided for author-year citation styles. Also
-                  note the \texttt{sorttitle} and \texttt{indextitle} fields. By
-                  default, all entries without an \texttt{author} or
-                  \texttt{editor} are alphabetized by \texttt{title} but we want
-                  this entry to be alphabetized under \enquote*{C} rather than
-                  \enquote*{T}. There's also an \texttt{isbn} field},
-}
-
-@online{baez/online,
-  author       = {Baez, John C. and Lauda, Aaron D.},
-  title        = {Higher-Dimensional Algebra {V}: 2-Groups},
-  date         = {2004-10-27},
-  version      = 3,
-  langid       = {english},
-  langidopts   = {variant=american},
-  eprinttype   = {arxiv},
-  eprint       = {math/0307200v3},
-  annotation   = {An \texttt{online} reference from arXiv. Note the
-                  \texttt{eprint} and \texttt{eprinttype} fields. Compare
-                  \texttt{baez\slash article} which is the same item given as an
-                  \texttt{article} entry with eprint information},
-}
-
-@online{ctan,
-  title        = {CTAN},
-  date         = 2006,
-  url          = {http://www.ctan.org},
-  subtitle     = {The {Comprehensive TeX Archive Network}},
-  urldate      = {2006-10-01},
-  label        = {CTAN},
-  langid       = {english},
-  langidopts   = {variant=american},
-  annotation   = {This is an \texttt{online} entry. The \textsc{url}, which is
-                  given in the \texttt{url} field, is transformed into a
-                  clickable link if \texttt{hyperref} support has been
-                  enabled. Note the format of the \texttt{urldate} field
-                  (\texttt{yyyy-mm-dd}) in the database file. Also note the
-                  \texttt{label} field which may be used as a fallback by
-                  citation styles which need an \texttt{author} and\slash or a
-                  \texttt{year}},
-}
-
-@online{itzhaki,
-  author       = {Itzhaki, Nissan},
-  title        = {Some remarks on {'t Hooft's} {S}-matrix for black holes},
-  date         = {1996-03-11},
-  version      = 1,
-  langid       = {english},
-  langidopts   = {variant=american},
-  eprinttype   = {arxiv},
-  eprint       = {hep-th/9603067},
-  annotation   = {An \texttt{online} reference from arXiv. Note the
-                  \texttt{eprint} and \texttt{eprinttype} fields. Also note that
-                  the arXiv reference is transformed into a clickable link if
-                  \texttt{hyperref} support has been enabled},
-  abstract     = {We discuss the limitations of 't Hooft's proposal for the
-                  black hole S-matrix. We find that the validity of the S-matrix
-                  implies violation of the semi-classical approximation at
-                  scales large compared to the Planck scale. We also show that
-                  the effect of the centrifugal barrier on the S-matrix is
-                  crucial even for large transverse distances.},
-}
-
-@online{markey,
-  author       = {Markey, Nicolas},
-  title        = {Tame the {BeaST}},
-  date         = {2005-10-16},
-  url          = {http://mirror.ctan.org/info/bibtex/tamethebeast/ttb_en.pdf},
-  subtitle     = {The {B} to {X} of {BibTeX}},
-  version      = {1.3},
-  urldate      = {2006-10-01},
-  langid       = {english},
-  langidopts   = {variant=american},
-  sorttitle    = {Tame the Beast},
-  annotation   = {An \texttt{online} entry for a tutorial. Note the format of
-                  the \texttt{date} field (\texttt{yyyy-mm-dd}) in the database
-                  file.},
-}
-
-@online{wassenberg,
-  author       = {Wassenberg, Jan and Sanders, Peter},
-  title        = {Faster Radix Sort via Virtual Memory and Write-Combining},
-  date         = {2010-08-17},
-  version      = 1,
-  langid       = {english},
-  langidopts   = {variant=american},
-  eprinttype   = {arxiv},
-  eprintclass  = {cs.DS},
-  eprint       = {1008.2849v1},
-  annotation   = {A recent \texttt{online} reference from arXiv using the new
-                  (April 2007 onward) identifier format. Note the
-                  \texttt{eprint}, \texttt{eprinttype}, and \texttt{eprintclass}
-                  fields. Also note that the arXiv reference is transformed into
-                  a clickable link if \texttt{hyperref} support has been
-                  enabled},
-  abstract     = {Sorting algorithms are the deciding factor for the performance
-                  of common operations such as removal of duplicates or database
-                  sort-merge joins. This work focuses on 32-bit integer keys,
-                  optionally paired with a 32-bit value. We present a fast radix
-                  sorting algorithm that builds upon a microarchitecture-aware
-                  variant of counting sort},
-}
-
-@patent{almendro,
-  author       = {Almendro, Jos{\'e} L. and Mart{\'i}n, Jacinto and S{\'a}nchez,
-                  Alberto and Nozal, Fernando},
-  title        = {Elektromagnetisches Signalhorn},
-  number       = {EU-29702195U},
-  date         = 1998,
-  location     = {countryfr and countryuk and countryde},
-  langid       = {german},
-  annotation   = {This is a \texttt{patent} entry with a \texttt{location}
-                  field. The number is given in the \texttt{number} field. Note
-                  the format of the \texttt{location} field in the database
-                  file. Compare \texttt{laufenberg}, \texttt{sorace}, and
-                  \texttt{kowalik}},
-}
-
-@patent{kowalik,
-  author       = {Kowalik, F. and Isard, M.},
-  title        = {Estimateur d'un d{\'e}faut de fonctionnement d'un modulateur
-                  en quadrature et {\'e}tage de modulation l'utilisant},
-  number       = 9500261,
-  date         = {1995-01-11},
-  type         = {patreqfr},
-  langid       = {french},
-  indextitle   = {Estimateur d'un d{\'e}faut de fonctionnement},
-  annotation   = {This is a \texttt{patent} entry for a French patent request
-                  with a full date. The number is given in the \texttt{number}
-                  field. Note the format of the \texttt{type} and \texttt{date}
-                  fields in the database file. Compare \texttt{almendro},
-                  \texttt{laufenberg}, and \texttt{sorace}},
-}
-
-@patent{laufenberg,
-  author       = {Laufenberg, Xaver and Eynius, Dominique and Suelzle, Helmut
-                  and Usbeck, Stephan and Spaeth, Matthias and Neuser-Hoffmann,
-                  Miriam and Myrzik, Christian and Schmid, Manfred and Nietfeld,
-                  Franz and Thiel, Alexander and Braun, Harald and Ebner,
-                  Norbert},
-  title        = {Elektrische Einrichtung und Betriebsverfahren},
-  number       = 1700367,
-  date         = {2006-09-13},
-  holder       = {{Robert Bosch GmbH} and {Daimler Chrysler AG} and {Bayerische
-                  Motoren Werke AG}},
-  type         = {patenteu},
-  langid       = {german},
-  annotation   = {This is a \texttt{patent} entry with a \texttt{holder} field.
-                  Note the format of the \texttt{type} and \texttt{location}
-                  fields in the database file. Compare \texttt{almendro},
-                  \texttt{sorace}, and \texttt{kowalik}},
-  abstract     = {The invention relates to an electric device comprising a
-                  generator, in particular for use in the vehicle electric
-                  system of a motor vehicle and a controller for controlling the
-                  generator voltage. The device is equipped with a control zone,
-                  in which the voltage is controlled and zones, in which the
-                  torque is controlled. The invention also relates to methods
-                  for operating a device of this type.},
-  file         = {http://v3.espacenet.com/textdoc?IDX=EP1700367},
-}
-
-@patent{sorace,
-  author       = {Sorace, Ronald E. and Reinhardt, Victor S. and Vaughn, Steven
-                  A.},
-  title        = {High-Speed Digital-to-{RF} Converter},
-  number       = 5668842,
-  date         = {1997-09-16},
-  holder       = {{Hughes Aircraft Company}},
-  type         = {patentus},
-  langid       = {english},
-  langidopts   = {variant=american},
-  annotation   = {This is a \texttt{patent} entry with a \texttt{holder} field.
-                  Note the format of the \texttt{type} and \texttt{date} fields
-                  in the database file. Compare \texttt{almendro},
-                  \texttt{laufenberg}, and \texttt{kowalik}},
-}
-
-@periodical{jcg,
-  title        = {Computers and Graphics},
-  year         = 2011,
-  issuetitle   = {Semantic {3D} Media and Content},
-  volume       = 35,
-  number       = 4,
-  issn         = {0097-8493},
-  annotation   = {This is a \texttt{periodical} entry with an \texttt{issn}
-                  field.},
-}
-
-@report{chiu,
-  author       = {Chiu, Willy W. and Chow, We Min},
-  title        = {A Hybrid Hierarchical Model of a {Multiple Virtual Storage}
-                  ({MVS}) Operating System},
-  type         = {resreport},
-  institution  = {IBM},
-  date         = 1978,
-  number       = {RC-6947},
-  langid       = {english},
-  langidopts   = {variant=american},
-  sorttitle    = {Hybrid Hierarchical Model of a Multiple Virtual Storage (MVS)
-                  Operating System},
-  indextitle   = {Hybrid Hierarchical Model, A},
-  annotation   = {This is a \texttt{report} entry for a research report. Note
-                  the format of the \texttt{type} field in the database file
-                  which uses a localization key. The number of the report is
-                  given in the \texttt{number} field. Also note the
-                  \texttt{sorttitle} and \texttt{indextitle} fields},
-}
-
-@report{padhye,
-  author       = {Padhye, Jitendra and Firoiu, Victor and Towsley, Don},
-  title        = {A Stochastic Model of {TCP Reno} Congestion Avoidance and
-                  Control},
-  type         = {techreport},
-  institution  = {University of Massachusetts},
-  date         = 1999,
-  number       = {99-02},
-  location     = {Amherst, Mass.},
-  langid       = {english},
-  langidopts   = {variant=american},
-  sorttitle    = {A Stochastic Model of TCP Reno Congestion Avoidance and
-                  Control},
-  indextitle   = {Stochastic Model of {TCP Reno} Congestion Avoidance and Control,
-                  A},
-  annotation   = {This is a \texttt{report} entry for a technical report. Note
-                  the format of the \texttt{type} field in the database file
-                  which uses a localization key. The number of the report is
-                  given in the \texttt{number} field. Also note the
-                  \texttt{sorttitle} and \texttt{indextitle} fields},
-  abstract     = {The steady state performance of a bulk transfer TCP flow
-                  (i.e. a flow with a large amount of data to send, such as FTP
-                  transfers) may be characterized by three quantities. The first
-                  is the send rate, which is the amount of data sent by the
-                  sender in unit time. The second is the throughput, which is
-                  the amount of data received by the receiver in unit time. Note
-                  that the throughput will always be less than or equal to the
-                  send rate due to losses. Finally, the number of non-duplicate
-                  packets received by the receiver in unit time gives us the
-                  goodput of the connection. The goodput is always less than or
-                  equal to the throughput, since the receiver may receive two
-                  copies of the same packet due to retransmissions by the
-                  sender. In a previous paper, we presented a simple model for
-                  predicting the steady state send rate of a bulk transfer TCP
-                  flow as a function of loss rate and round trip time. In this
-                  paper, we extend that work in two ways. First, we analyze the
-                  performance of bulk transfer TCP flows using more precise,
-                  stochastic analysis. Second, we build upon the previous
-                  analysis to provide both an approximate formula as well as a
-                  more accurate stochastic model for the steady state throughput
-                  of a bulk transfer TCP flow.},
-  file         = {ftp://gaia.cs.umass.edu/pub/Padhey99-markov.ps},
-}
-
-@thesis{geer,
-  author       = {de Geer, Ingrid},
-  title        = {Earl, Saint, Bishop, Skald~-- and Music},
-  type         = {phdthesis},
-  institution  = {Uppsala Universitet},
-  date         = 1985,
-  subtitle     = {The {Orkney Earldom} of the Twelfth Century. {A} Musicological
-                  Study},
-  location     = {Uppsala},
-  options      = {useprefix=false},
-  langid       = {english},
-  langidopts   = {variant=british},
-  annotation   = {This is a typical \texttt{thesis} entry for a PhD thesis. Note
-                  the \texttt{type} field in the database file which uses a
-                  localization key. Also note the format of the printed name and
-                  compare the \texttt{useprefix} option in the \texttt{options}
-                  field as well as \texttt{vangennep}},
-}
-
-@thesis{loh,
-  author       = {Loh, Nin C.},
-  title        = {High-Resolution Micromachined Interferometric Accelerometer},
-  type         = {mathesis},
-  institution  = {Massachusetts Institute of Technology},
-  date         = 1992,
-  location     = {Cambridge, Mass.},
-  langid       = {english},
-  langidopts   = {variant=american},
-  annotation   = {This is a typical \texttt{thesis} entry for an MA thesis. Note
-                  the \texttt{type} field in the database file which uses a
-                  localization key},
-}
diff --git a/test/command/pandoc-citeproc-119.md b/test/command/pandoc-citeproc-119.md
index 34ffc93af..cbae7c8fe 100644
--- a/test/command/pandoc-citeproc-119.md
+++ b/test/command/pandoc-citeproc-119.md
@@ -1,7 +1,7 @@
 ```
 % pandoc --citeproc -t markdown-citations
 ---
-bibliography: 'command/biblatex-examples.bib'
+bibliography: 'command/averroes.bib'
 csl: command/apa.csl
 ---
 
-- 
cgit v1.2.3


From c210b983662c26c0b827c9199f6f1a2b34c1559e Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sat, 29 May 2021 14:36:49 -0700
Subject: Fix test #3752 (1) for Windows.

---
 test/command/3752.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'test/command')

diff --git a/test/command/3752.md b/test/command/3752.md
index 2e96b531e..6ac025ebe 100644
--- a/test/command/3752.md
+++ b/test/command/3752.md
@@ -1,5 +1,5 @@
 ```
-% pandoc command/chap1/text.md command/chap2/text.md -f markdown+rebase_relative_paths --verbose -t docx | pandoc -f docx -t plain
+% pandoc command/chap1/text.md command/chap2/text.md -f markdown+rebase_relative_paths --verbose -t docx -o - | pandoc -f docx -t plain
 ^D
 [INFO] Loaded command/chap1/spider.png from ./command/chap1/spider.png
 [INFO] Loaded command/chap2/spider.png from ./command/chap2/spider.png
-- 
cgit v1.2.3


From cc206af392a40dd7b01b714ae7f33b2fbf4925cc Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sun, 30 May 2021 10:22:02 -0700
Subject: Have LoadedResource use relative paths.

The immediate reason for this is to allow the test output of #3752
to work on both windows and linux.
---
 src/Text/Pandoc/Class/PandocMonad.hs | 4 ++--
 test/command/3752.md                 | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Class/PandocMonad.hs b/src/Text/Pandoc/Class/PandocMonad.hs
index b5f401619..4eb80df29 100644
--- a/src/Text/Pandoc/Class/PandocMonad.hs
+++ b/src/Text/Pandoc/Class/PandocMonad.hs
@@ -66,7 +66,7 @@ import Network.URI ( escapeURIString, nonStrictRelativeTo,
                      unEscapeString, parseURIReference, isAllowedInURI,
                      parseURI, URI(..) )
 import System.FilePath ((</>), takeExtension, dropExtension,
-                        isRelative, splitDirectories)
+                        isRelative, splitDirectories, makeRelative)
 import System.Random (StdGen)
 import Text.Collate.Lang (Lang(..), parseLang, renderLang)
 import Text.Pandoc.Class.CommonState (CommonState (..))
@@ -413,7 +413,7 @@ downloadOrRead s = do
              (fp', cont) <- if isRelative f
                                then withPaths resourcePath readFileStrict f
                                else (f,) <$> readFileStrict f
-             report $ LoadedResource f fp'
+             report $ LoadedResource f (makeRelative "." fp')
              return (cont, mime)
          httpcolon = URI{ uriScheme = "http:",
                           uriAuthority = Nothing,
diff --git a/test/command/3752.md b/test/command/3752.md
index 6ac025ebe..863e3f2d4 100644
--- a/test/command/3752.md
+++ b/test/command/3752.md
@@ -1,9 +1,9 @@
 ```
 % pandoc command/chap1/text.md command/chap2/text.md -f markdown+rebase_relative_paths --verbose -t docx -o - | pandoc -f docx -t plain
 ^D
-[INFO] Loaded command/chap1/spider.png from ./command/chap1/spider.png
-[INFO] Loaded command/chap2/spider.png from ./command/chap2/spider.png
-[INFO] Loaded command/chap1/../../lalune.jpg from ./command/chap1/../../lalune.jpg
+[INFO] Loaded command/chap1/spider.png from command/chap1/spider.png
+[INFO] Loaded command/chap2/spider.png from command/chap2/spider.png
+[INFO] Loaded command/chap1/../../lalune.jpg from command/chap1/../../lalune.jpg
 Chapter one
 
 A spider: [spider]
-- 
cgit v1.2.3


From 62f46b3995425c9a3ec87cba0eb8a4d736adec07 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Mon, 31 May 2021 21:34:51 -0600
Subject: Fix regression with commonmark/gfm yaml metdata block parsing.

A regression in 2.14 led to the document body being omitted
after YAML metadata in some cases.  This is now fixed.

Closes #7339.
---
 src/Text/Pandoc/Readers/CommonMark.hs | 10 +++++-----
 test/command/7339.md                  | 11 +++++++++++
 2 files changed, 16 insertions(+), 5 deletions(-)
 create mode 100644 test/command/7339.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/CommonMark.hs b/src/Text/Pandoc/Readers/CommonMark.hs
index 228e65312..411d64278 100644
--- a/src/Text/Pandoc/Readers/CommonMark.hs
+++ b/src/Text/Pandoc/Readers/CommonMark.hs
@@ -30,7 +30,7 @@ import Text.Pandoc.Readers.Metadata (yamlMetaBlock)
 import Control.Monad.Except
 import Data.Functor.Identity (runIdentity)
 import Data.Typeable
-import Text.Pandoc.Parsing (runParserT, getPosition,
+import Text.Pandoc.Parsing (runParserT, getInput,
                             runF, defaultParserState, option, many1, anyChar,
                             Sources(..), ToSources(..), ParserT, Future,
                             sourceName)
@@ -44,14 +44,14 @@ readCommonMark opts s
     let sources = toSources s
     let toks = concatMap sourceToToks (unSources sources)
     res <- runParserT (do meta <- yamlMetaBlock (metaValueParser opts)
-                          pos <- getPosition
-                          return (meta, pos))
+                          rest <- getInput
+                          return (meta, rest))
                       defaultParserState "YAML metadata" (toSources s)
     case res of
       Left _ -> readCommonMarkBody opts sources toks
-      Right (meta, pos) -> do
+      Right (meta, rest) -> do
         -- strip off metadata section and parse body
-        let body = dropWhile (\t -> tokPos t < pos) toks
+        let body = concatMap sourceToToks (unSources rest)
         Pandoc _ bs <- readCommonMarkBody opts sources body
         return $ Pandoc (runF meta defaultParserState) bs
   | otherwise = do
diff --git a/test/command/7339.md b/test/command/7339.md
new file mode 100644
index 000000000..9697c1c32
--- /dev/null
+++ b/test/command/7339.md
@@ -0,0 +1,11 @@
+```
+% pandoc -f gfm -s -t native
+---
+title: Test
+---
+
+Hi
+^D
+Pandoc (Meta {unMeta = fromList [("title",MetaInlines [Str "Test"])]})
+[Para [Str "Hi"]]
+```
-- 
cgit v1.2.3


From abb59bd58222c67fd36a8e447c01de3404a7ed1d Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Tue, 1 Jun 2021 13:54:51 -0600
Subject: LaTeX reader: don't allow optional * on symbol control sequences.

Generally we allow optional starred variants of LaTeX commands
(since many allow them, and if we don't accept these explicitly,
ignoring the star usually gives acceptable results).  But we
don't want to do this for `\(*\)` and similar cases.

Closes #7340.
---
 src/Text/Pandoc/Readers/LaTeX.hs | 6 ++++--
 test/command/7340.md             | 6 ++++++
 2 files changed, 10 insertions(+), 2 deletions(-)
 create mode 100644 test/command/7340.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/LaTeX.hs b/src/Text/Pandoc/Readers/LaTeX.hs
index 2ace18d1b..9e14c159a 100644
--- a/src/Text/Pandoc/Readers/LaTeX.hs
+++ b/src/Text/Pandoc/Readers/LaTeX.hs
@@ -24,7 +24,7 @@ module Text.Pandoc.Readers.LaTeX ( readLaTeX,
 import Control.Applicative (many, optional, (<|>))
 import Control.Monad
 import Control.Monad.Except (throwError)
-import Data.Char (isDigit, isLetter, toUpper, chr)
+import Data.Char (isDigit, isLetter, isAlphaNum, toUpper, chr)
 import Data.Default
 import Data.List (intercalate)
 import qualified Data.Map as M
@@ -300,7 +300,9 @@ inlineCommand' :: PandocMonad m => LP m Inlines
 inlineCommand' = try $ do
   Tok _ (CtrlSeq name) cmd <- anyControlSeq
   guard $ name /= "begin" && name /= "end" && name /= "and"
-  star <- option "" ("*" <$ symbol '*' <* sp)
+  star <- if T.all isAlphaNum name
+             then option "" ("*" <$ symbol '*' <* sp)
+             else pure ""
   overlay <- option "" overlaySpecification
   let name' = name <> star <> overlay
   let names = ordNub [name', name] -- check non-starred as fallback
diff --git a/test/command/7340.md b/test/command/7340.md
new file mode 100644
index 000000000..25decd732
--- /dev/null
+++ b/test/command/7340.md
@@ -0,0 +1,6 @@
+```
+% pandoc -f latex -t native
+\(*\)
+^D
+[Para [Math InlineMath "*"]]
+```
-- 
cgit v1.2.3


From 21cc52abe33997ea2f2c539f10d26684b7633bc0 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sat, 5 Jun 2021 14:13:58 -0600
Subject: LaTeX writer: Fix regression in table header position.

In recent versions the table headers were no longer bottom-aligned
(if more than one line).  This patch fixes that by using minipages
for table headers in non-simple tables.

Closes #7347.
---
 src/Text/Pandoc/Writers/LaTeX/Table.hs | 13 ++++++++++---
 test/command/5367.md                   |  8 ++++++--
 test/tables.latex                      | 30 +++++++++++++++++++++++++++---
 test/tables/nordics.latex              | 26 ++++++++++++++++++++++----
 test/tables/planets.latex              |  8 ++++----
 test/tables/students.latex             | 12 ++++++++++--
 6 files changed, 79 insertions(+), 18 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/LaTeX/Table.hs b/src/Text/Pandoc/Writers/LaTeX/Table.hs
index 16f63314b..8dc7d1162 100644
--- a/src/Text/Pandoc/Writers/LaTeX/Table.hs
+++ b/src/Text/Pandoc/Writers/LaTeX/Table.hs
@@ -16,6 +16,7 @@ module Text.Pandoc.Writers.LaTeX.Table
   ) where
 import Control.Monad.State.Strict
 import Data.List (intersperse)
+import qualified Data.List.NonEmpty as NonEmpty
 import Data.List.NonEmpty (NonEmpty ((:|)))
 import Data.Text (Text)
 import qualified Data.Text as T
@@ -243,8 +244,13 @@ cellToLaTeX :: PandocMonad m
             -> Ann.Cell
             -> LW m (Doc Text)
 cellToLaTeX blockListToLaTeX celltype annotatedCell = do
-  let (Ann.Cell _specs _colnum cell) = annotatedCell
-  let (Cell _attr align rowspan colspan blocks) = cell
+  let (Ann.Cell specs _colnum cell) = annotatedCell
+  let hasWidths = snd (NonEmpty.head specs) /= ColWidthDefault
+  let specAlign = fst (NonEmpty.head specs)
+  let (Cell _attr align' rowspan colspan blocks) = cell
+  let align = case align' of
+                AlignDefault -> specAlign
+                _            -> align'
   beamer <- gets stBeamer
   externalNotes <- gets stExternalNotes
   inMinipage <- gets stInMinipage
@@ -256,7 +262,7 @@ cellToLaTeX blockListToLaTeX celltype annotatedCell = do
         Plain{} -> True
         _       -> False
   result <-
-    if all isPlainOrPara blocks
+    if not hasWidths || (celltype /= HeaderCell && all isPlainOrPara blocks)
        then
          blockListToLaTeX $ walk fixLineBreaks $ walk displayMathToInline blocks
        else do
@@ -290,3 +296,4 @@ cellToLaTeX blockListToLaTeX celltype annotatedCell = do
 data CellType
   = HeaderCell
   | BodyCell
+  deriving Eq
diff --git a/test/command/5367.md b/test/command/5367.md
index 2d3a5e52e..a67011c2f 100644
--- a/test/command/5367.md
+++ b/test/command/5367.md
@@ -24,11 +24,15 @@ hello\footnote{doc footnote}
   >{\centering\arraybackslash}p{(\columnwidth - 0\tabcolsep) * \real{0.17}}@{}}
 \caption[Sample table.]{Sample table.\footnote{caption footnote}}\tabularnewline
 \toprule
-Fruit\footnote{header footnote} \\
+\begin{minipage}[b]{\linewidth}\centering
+Fruit\footnote{header footnote}
+\end{minipage} \\
 \midrule
 \endfirsthead
 \toprule
-Fruit{} \\
+\begin{minipage}[b]{\linewidth}\centering
+Fruit{}
+\end{minipage} \\
 \midrule
 \endhead
 Bans\footnote{table cell footnote} \\
diff --git a/test/tables.latex b/test/tables.latex
index afa14d845..9d111fa7a 100644
--- a/test/tables.latex
+++ b/test/tables.latex
@@ -56,11 +56,27 @@ Multiline table with caption:
   >{\raggedright\arraybackslash}p{(\columnwidth - 6\tabcolsep) * \real{0.35}}@{}}
 \caption{Here's the caption. It may span multiple lines.}\tabularnewline
 \toprule
-Centered Header & Left Aligned & Right Aligned & Default aligned \\
+\begin{minipage}[b]{\linewidth}\centering
+Centered Header
+\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright
+Left Aligned
+\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedleft
+Right Aligned
+\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright
+Default aligned
+\end{minipage} \\
 \midrule
 \endfirsthead
 \toprule
-Centered Header & Left Aligned & Right Aligned & Default aligned \\
+\begin{minipage}[b]{\linewidth}\centering
+Centered Header
+\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright
+Left Aligned
+\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedleft
+Right Aligned
+\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright
+Default aligned
+\end{minipage} \\
 \midrule
 \endhead
 First & row & 12.0 & Example of a row that spans multiple lines. \\
@@ -76,7 +92,15 @@ Multiline table without caption:
   >{\raggedleft\arraybackslash}p{(\columnwidth - 6\tabcolsep) * \real{0.16}}
   >{\raggedright\arraybackslash}p{(\columnwidth - 6\tabcolsep) * \real{0.35}}@{}}
 \toprule
-Centered Header & Left Aligned & Right Aligned & Default aligned \\
+\begin{minipage}[b]{\linewidth}\centering
+Centered Header
+\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright
+Left Aligned
+\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedleft
+Right Aligned
+\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright
+Default aligned
+\end{minipage} \\
 \midrule
 \endhead
 First & row & 12.0 & Example of a row that spans multiple lines. \\
diff --git a/test/tables/nordics.latex b/test/tables/nordics.latex
index 1b5929bad..1dcac7319 100644
--- a/test/tables/nordics.latex
+++ b/test/tables/nordics.latex
@@ -5,13 +5,31 @@
   >{\raggedright\arraybackslash}p{(\columnwidth - 6\tabcolsep) * \real{0.20}}@{}}
 \caption{States belonging to the \emph{Nordics.}}\tabularnewline
 \toprule
-Name & Capital & \vtop{\hbox{\strut Population}\hbox{\strut (in 2018)}} &
-\vtop{\hbox{\strut Area}\hbox{\strut (in km\textsuperscript{2})}} \\
+\begin{minipage}[b]{\linewidth}\centering
+Name
+\end{minipage} & \begin{minipage}[b]{\linewidth}\centering
+Capital
+\end{minipage} & \begin{minipage}[b]{\linewidth}\centering
+Population\\
+(in 2018)
+\end{minipage} & \begin{minipage}[b]{\linewidth}\centering
+Area\\
+(in km\textsuperscript{2})
+\end{minipage} \\
 \midrule
 \endfirsthead
 \toprule
-Name & Capital & \vtop{\hbox{\strut Population}\hbox{\strut (in 2018)}} &
-\vtop{\hbox{\strut Area}\hbox{\strut (in km\textsuperscript{2})}} \\
+\begin{minipage}[b]{\linewidth}\centering
+Name
+\end{minipage} & \begin{minipage}[b]{\linewidth}\centering
+Capital
+\end{minipage} & \begin{minipage}[b]{\linewidth}\centering
+Population\\
+(in 2018)
+\end{minipage} & \begin{minipage}[b]{\linewidth}\centering
+Area\\
+(in km\textsuperscript{2})
+\end{minipage} \\
 \midrule
 \endhead
 Denmark & Copenhagen & 5,809,502 & 43,094 \\
diff --git a/test/tables/planets.latex b/test/tables/planets.latex
index 8238c43f3..b22c3adeb 100644
--- a/test/tables/planets.latex
+++ b/test/tables/planets.latex
@@ -1,18 +1,18 @@
 \begin{longtable}[]{@{}cclrrrrrrrrl@{}}
 \caption{Data about the planets of our solar system.}\tabularnewline
 \toprule
-\multicolumn{2}{l}{} & Name & Mass (10\^{}24kg) & Diameter (km) & Density
+\multicolumn{2}{c}{} & Name & Mass (10\^{}24kg) & Diameter (km) & Density
 (kg/m\^{}3) & Gravity (m/s\^{}2) & Length of day (hours) & Distance from Sun
 (10\^{}6km) & Mean temperature (C) & Number of moons & Notes \\
 \midrule
 \endfirsthead
 \toprule
-\multicolumn{2}{l}{} & Name & Mass (10\^{}24kg) & Diameter (km) & Density
+\multicolumn{2}{c}{} & Name & Mass (10\^{}24kg) & Diameter (km) & Density
 (kg/m\^{}3) & Gravity (m/s\^{}2) & Length of day (hours) & Distance from Sun
 (10\^{}6km) & Mean temperature (C) & Number of moons & Notes \\
 \midrule
 \endhead
-\multicolumn{2}{l}{\multirow{4}{*}{Terrestrial planets}} & Mercury & 0.330 &
+\multicolumn{2}{c}{\multirow{4}{*}{Terrestrial planets}} & Mercury & 0.330 &
 4,879 & 5427 & 3.7 & 4222.6 & 57.9 & 167 & 0 & Closest to the Sun \\
 & & Venus & 4.87 & 12,104 & 5243 & 8.9 & 2802.0 & 108.2 & 464 & 0 & \\
 & & Earth & 5.97 & 12,756 & 5514 & 9.8 & 24.0 & 149.6 & 15 & 1 & Our world \\
@@ -24,7 +24,7 @@ planet \\
 & \multirow{2}{*}{Ice giants} & Uranus & 86.8 & 51,118 & 1271 & 8.7 & 17.2 &
 2872.5 & -195 & 27 & \\
 & & Neptune & 102 & 49,528 & 1638 & 11.0 & 16.1 & 4495.1 & -200 & 14 & \\
-\multicolumn{2}{l}{Dwarf planets} & Pluto & 0.0146 & 2,370 & 2095 & 0.7 &
+\multicolumn{2}{c}{Dwarf planets} & Pluto & 0.0146 & 2,370 & 2095 & 0.7 &
 153.3 & 5906.4 & -225 & 5 & Declassified as a planet in 2006. \\
 \bottomrule
 \end{longtable}
diff --git a/test/tables/students.latex b/test/tables/students.latex
index 87efb0851..3d4d287d9 100644
--- a/test/tables/students.latex
+++ b/test/tables/students.latex
@@ -3,11 +3,19 @@
   >{\raggedright\arraybackslash}p{(\columnwidth - 2\tabcolsep) * \real{0.50}}@{}}
 \caption{List of Students}\tabularnewline
 \toprule
-Student ID & Name \\
+\begin{minipage}[b]{\linewidth}\centering
+Student ID
+\end{minipage} & \begin{minipage}[b]{\linewidth}\centering
+Name
+\end{minipage} \\
 \midrule
 \endfirsthead
 \toprule
-Student ID & Name \\
+\begin{minipage}[b]{\linewidth}\centering
+Student ID
+\end{minipage} & \begin{minipage}[b]{\linewidth}\centering
+Name
+\end{minipage} \\
 \midrule
 \endhead
 \multicolumn{2}{l}{Computer Science} \\
-- 
cgit v1.2.3


From b0cd6c622494666add6bdd7674ec5b7791bc83d0 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sat, 12 Jun 2021 10:16:44 -0700
Subject: Fix regression in citeproc processing.

If inline references are used (in the metadata `references` field),
we should still only include in the bibliography items that are
actually cited -- unless `nocite` is used.

Closes #7376.
---
 src/Text/Pandoc/Citeproc.hs         |  4 +++-
 test/command/7376.md                | 16 ++++++++++++++++
 test/command/pandoc-citeproc-356.md |  9 ++-------
 3 files changed, 21 insertions(+), 8 deletions(-)
 create mode 100644 test/command/7376.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Citeproc.hs b/src/Text/Pandoc/Citeproc.hs
index ad3b26c0f..a5b26c9b4 100644
--- a/src/Text/Pandoc/Citeproc.hs
+++ b/src/Text/Pandoc/Citeproc.hs
@@ -194,7 +194,9 @@ getReferences mblocale (Pandoc meta bs) = do
                   then const True
                   else (`Set.member` citeIds)
   let inlineRefs = case lookupMeta "references" meta of
-                    Just (MetaList rs) -> mapMaybe metaValueToReference rs
+                    Just (MetaList rs) ->
+                      filter (idpred . unItemId . referenceId)
+                         $  mapMaybe metaValueToReference rs
                     _                  -> []
   externalRefs <- case lookupMeta "bibliography" meta of
                     Just (MetaList xs) ->
diff --git a/test/command/7376.md b/test/command/7376.md
new file mode 100644
index 000000000..229c61cfb
--- /dev/null
+++ b/test/command/7376.md
@@ -0,0 +1,16 @@
+```
+% pandoc --citeproc -t plain
+---
+references:
+- id: item1
+  type: book
+  author:
+    - family: Doe
+      given: Jane
+  issued: 2020
+  title: The title
+...
+^D
+
+
+```
diff --git a/test/command/pandoc-citeproc-356.md b/test/command/pandoc-citeproc-356.md
index 4463ef63f..b4f998dae 100644
--- a/test/command/pandoc-citeproc-356.md
+++ b/test/command/pandoc-citeproc-356.md
@@ -15,11 +15,6 @@ references:
 
 [@bar]
 ^D
-(Alice 2042)
-
-::: {#refs .references .csl-bib-body .hanging-indent}
-::: {#ref-foo .csl-entry}
-Alice. 2042.
-:::
-:::
+[WARNING] Citeproc: citation bar not found
+(**bar?**)
 ```
-- 
cgit v1.2.3


From 82ad855f38b8fa8dc1cbfc14fa294dfd5f9f02ab Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Mon, 21 Jun 2021 08:49:00 -0700
Subject: Markdown writer: Fix regression in code blocks with attributes.

Code blocks with a single class but nonempty attributes
were having attributes drop as a result of #7242.

Closes #7397.
---
 src/Text/Pandoc/Writers/Markdown.hs |  6 +++---
 test/command/7397.md                | 14 ++++++++++++++
 2 files changed, 17 insertions(+), 3 deletions(-)
 create mode 100644 test/command/7397.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/Markdown.hs b/src/Text/Pandoc/Writers/Markdown.hs
index 425ea07ca..b13ab57ee 100644
--- a/src/Text/Pandoc/Writers/Markdown.hs
+++ b/src/Text/Pandoc/Writers/Markdown.hs
@@ -256,10 +256,10 @@ noteToMarkdown opts num blocks = do
               then hang (writerTabStop opts) (marker <> spacer) contents
               else marker <> spacer <> contents
 
--- | (Code) blocks with a single class can just use it standalone,
--- no need to bother with curly braces.
+-- | (Code) blocks with a single class and no attributes can just use it
+-- standalone, no need to bother with curly braces.
 classOrAttrsToMarkdown :: Attr -> Doc Text
-classOrAttrsToMarkdown ("",[cls],_) = literal cls
+classOrAttrsToMarkdown ("",[cls],[]) = literal cls
 classOrAttrsToMarkdown attrs = attrsToMarkdown attrs
 
 -- | Ordered list start parser for use in Para below.
diff --git a/test/command/7397.md b/test/command/7397.md
new file mode 100644
index 000000000..ca8b6a482
--- /dev/null
+++ b/test/command/7397.md
@@ -0,0 +1,14 @@
+```
+% pandoc -t markdown
+~~~~ { .haskell startFrom="100"}
+qsort []     = []
+qsort (x:xs) = qsort (filter (< x) xs) ++ [x] ++
+               qsort (filter (>= x) xs)
+~~~~
+^D
+``` {.haskell startFrom="100"}
+qsort []     = []
+qsort (x:xs) = qsort (filter (< x) xs) ++ [x] ++
+               qsort (filter (>= x) xs)
+```
+```
-- 
cgit v1.2.3


From a39313eddbc84c7680d4bc7cef7770b18c89260a Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Mon, 21 Jun 2021 09:30:23 -0700
Subject: Fix test for #7397

---
 test/command/7397.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'test/command')

diff --git a/test/command/7397.md b/test/command/7397.md
index ca8b6a482..24ce391b0 100644
--- a/test/command/7397.md
+++ b/test/command/7397.md
@@ -1,4 +1,4 @@
-```
+````
 % pandoc -t markdown
 ~~~~ { .haskell startFrom="100"}
 qsort []     = []
@@ -11,4 +11,4 @@ qsort []     = []
 qsort (x:xs) = qsort (filter (< x) xs) ++ [x] ++
                qsort (filter (>= x) xs)
 ```
-```
+````
-- 
cgit v1.2.3


From ed3974a254c3e0c4e7a34d5d25ddef90c25d2092 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Mon, 21 Jun 2021 18:25:36 -0700
Subject: LaTeX writer: always use a minipage for cells with line breaks...

if width information is available.  Otherwise the way we treat them can
lead to content that overflows a cell.

Closes #7393.
---
 src/Text/Pandoc/Writers/LaTeX/Table.hs | 9 +++++++--
 test/command/7272.md                   | 5 ++++-
 2 files changed, 11 insertions(+), 3 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/LaTeX/Table.hs b/src/Text/Pandoc/Writers/LaTeX/Table.hs
index 8dc7d1162..abdc26649 100644
--- a/src/Text/Pandoc/Writers/LaTeX/Table.hs
+++ b/src/Text/Pandoc/Writers/LaTeX/Table.hs
@@ -26,7 +26,8 @@ import Text.DocLayout
   ( Doc, braces, cr, empty, hcat, hsep, isEmpty, literal, nest
   , text, vcat, ($$) )
 import Text.Pandoc.Shared (blocksToInlines, splitBy, tshow)
-import Text.Pandoc.Walk (walk)
+import Text.Pandoc.Walk (walk, query)
+import Data.Monoid (Any(..))
 import Text.Pandoc.Writers.LaTeX.Caption (getCaption)
 import Text.Pandoc.Writers.LaTeX.Notes (notesToLaTeX)
 import Text.Pandoc.Writers.LaTeX.Types
@@ -261,8 +262,12 @@ cellToLaTeX blockListToLaTeX celltype annotatedCell = do
         Para{}  -> True
         Plain{} -> True
         _       -> False
+  let hasLineBreak LineBreak = Any True
+      hasLineBreak _ = Any False
   result <-
-    if not hasWidths || (celltype /= HeaderCell && all isPlainOrPara blocks)
+    if not hasWidths || (celltype /= HeaderCell
+                           && all isPlainOrPara blocks
+                           && not (getAny (query hasLineBreak blocks)))
        then
          blockListToLaTeX $ walk fixLineBreaks $ walk displayMathToInline blocks
        else do
diff --git a/test/command/7272.md b/test/command/7272.md
index d3a3b2137..3b9064c9c 100644
--- a/test/command/7272.md
+++ b/test/command/7272.md
@@ -18,7 +18,10 @@
   >{\raggedright\arraybackslash}p{(\columnwidth - 0\tabcolsep) * \real{1.00}}@{}}
 \toprule
 \endhead
-{\vtop{\hbox{\strut  text}\hbox{\strut text2 }}} \\
+\begin{minipage}[t]{\linewidth}\raggedright
+{ text\\
+text2 }
+\end{minipage} \\
 \bottomrule
 \end{longtable}
 ```
-- 
cgit v1.2.3


From 2ef2049b4e94dc51961e75edb27af1d2f83acd3b Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Mon, 21 Jun 2021 22:34:38 -0700
Subject: Update command test for change to LaTeX LineBreak handling.

---
 test/command/2874.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'test/command')

diff --git a/test/command/2874.md b/test/command/2874.md
index 1fb530dc1..99f46d2fb 100644
--- a/test/command/2874.md
+++ b/test/command/2874.md
@@ -3,12 +3,12 @@
 <a></a>
 <br/>
 ^D
-{}~\\
+{}\strut \\
 ```
 
 ```
 % pandoc -f html -t latex
 <a name="foo"></a><br/>
 ^D
-\protect\hypertarget{foo}{}{}~\\
+\protect\hypertarget{foo}{}{}\strut \\
 ```
-- 
cgit v1.2.3


From 8eed5b90d09a4a0c2592c92215fa96c69cf35234 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Mon, 21 Jun 2021 23:31:27 -0700
Subject: LaTeX writer: add strut at end of minipage if it contains...

line breaks.  Without them, the last line is shorter
than it should be, at least in some cases.
---
 src/Text/Pandoc/Writers/LaTeX/Table.hs | 7 +++++--
 test/command/7272.md                   | 2 +-
 test/tables/nordics.latex              | 8 ++++----
 3 files changed, 10 insertions(+), 7 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/LaTeX/Table.hs b/src/Text/Pandoc/Writers/LaTeX/Table.hs
index abdc26649..27a8a0257 100644
--- a/src/Text/Pandoc/Writers/LaTeX/Table.hs
+++ b/src/Text/Pandoc/Writers/LaTeX/Table.hs
@@ -264,10 +264,11 @@ cellToLaTeX blockListToLaTeX celltype annotatedCell = do
         _       -> False
   let hasLineBreak LineBreak = Any True
       hasLineBreak _ = Any False
+  let hasLineBreaks = getAny $ query hasLineBreak blocks
   result <-
     if not hasWidths || (celltype /= HeaderCell
                            && all isPlainOrPara blocks
-                           && not (getAny (query hasLineBreak blocks)))
+                           && not hasLineBreaks)
        then
          blockListToLaTeX $ walk fixLineBreaks $ walk displayMathToInline blocks
        else do
@@ -280,7 +281,9 @@ cellToLaTeX blockListToLaTeX celltype annotatedCell = do
          let halign = literal $ alignCommand align
          return $ "\\begin{minipage}" <> valign <>
                   braces "\\linewidth" <> halign <> cr <>
-                  cellContents <> cr <>
+                  cellContents <>
+                  (if hasLineBreaks then "\\strut" else mempty)
+                  <> cr <>
                   "\\end{minipage}"
   modify $ \st -> st{ stExternalNotes = externalNotes }
   when (rowspan /= RowSpan 1) $
diff --git a/test/command/7272.md b/test/command/7272.md
index 3b9064c9c..d27b25143 100644
--- a/test/command/7272.md
+++ b/test/command/7272.md
@@ -20,7 +20,7 @@
 \endhead
 \begin{minipage}[t]{\linewidth}\raggedright
 { text\\
-text2 }
+text2 }\strut
 \end{minipage} \\
 \bottomrule
 \end{longtable}
diff --git a/test/tables/nordics.latex b/test/tables/nordics.latex
index 1dcac7319..6f17a163e 100644
--- a/test/tables/nordics.latex
+++ b/test/tables/nordics.latex
@@ -11,10 +11,10 @@ Name
 Capital
 \end{minipage} & \begin{minipage}[b]{\linewidth}\centering
 Population\\
-(in 2018)
+(in 2018)\strut
 \end{minipage} & \begin{minipage}[b]{\linewidth}\centering
 Area\\
-(in km\textsuperscript{2})
+(in km\textsuperscript{2})\strut
 \end{minipage} \\
 \midrule
 \endfirsthead
@@ -25,10 +25,10 @@ Name
 Capital
 \end{minipage} & \begin{minipage}[b]{\linewidth}\centering
 Population\\
-(in 2018)
+(in 2018)\strut
 \end{minipage} & \begin{minipage}[b]{\linewidth}\centering
 Area\\
-(in km\textsuperscript{2})
+(in km\textsuperscript{2})\strut
 \end{minipage} \\
 \midrule
 \endhead
-- 
cgit v1.2.3


From 1b07997f4a6870650f20702ed6d962f9471e3d40 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Tue, 22 Jun 2021 09:55:50 -0700
Subject: Fix regression with comment-only YAML metadata blocks.

Closes #7400.
---
 src/Text/Pandoc/Readers/Metadata.hs | 3 +++
 test/command/7400.md                | 9 +++++++++
 2 files changed, 12 insertions(+)
 create mode 100644 test/command/7400.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/Metadata.hs b/src/Text/Pandoc/Readers/Metadata.hs
index 45eddf25a..cbc523b25 100644
--- a/src/Text/Pandoc/Readers/Metadata.hs
+++ b/src/Text/Pandoc/Readers/Metadata.hs
@@ -45,6 +45,9 @@ yamlBsToMeta pMetaValue bstr = do
        Right [] -> return . return $ mempty
        Right [YAML.Doc (YAML.Scalar _ YAML.SNull)]
                 -> return . return $ mempty
+       -- the following is what we get from a comment:
+       Right [YAML.Doc (YAML.Scalar _ (YAML.SUnknown _ ""))]
+                -> return . return $ mempty
        Right _  -> Prelude.fail "expected YAML object"
        Left (yamlpos, err')
                 -> do pos <- getPosition
diff --git a/test/command/7400.md b/test/command/7400.md
new file mode 100644
index 000000000..d4be32d72
--- /dev/null
+++ b/test/command/7400.md
@@ -0,0 +1,9 @@
+```
+% pandoc -t native -s
+---
+# Comment only
+...
+^D
+Pandoc (Meta {unMeta = fromList []})
+[]
+```
-- 
cgit v1.2.3


From dd098d4e15090d12cc71301f91a159c5bfb29b50 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Mon, 28 Jun 2021 11:27:37 -0700
Subject: Markdown writer: put space between Plain and following fenced Div.

Closes #4465.
---
 src/Text/Pandoc/Writers/Markdown.hs |  3 +++
 test/command/4465.md                | 15 +++++++++++++++
 2 files changed, 18 insertions(+)
 create mode 100644 test/command/4465.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/Markdown.hs b/src/Text/Pandoc/Writers/Markdown.hs
index b13ab57ee..fda2bbcef 100644
--- a/src/Text/Pandoc/Writers/Markdown.hs
+++ b/src/Text/Pandoc/Writers/Markdown.hs
@@ -791,6 +791,9 @@ blockListToMarkdown opts blocks = do
            b1 : commentSep : fixBlocks (b2:bs)
       fixBlocks (Plain ils : bs@(RawBlock{}:_)) =
            Plain ils : fixBlocks bs
+      fixBlocks (Plain ils : bs@(Div{}:_))
+          | isEnabled Ext_fenced_divs opts =
+           Para ils : fixBlocks bs
       fixBlocks (Plain ils : bs) | inlist =
            Plain ils : fixBlocks bs
       fixBlocks (Plain ils : bs) =
diff --git a/test/command/4465.md b/test/command/4465.md
new file mode 100644
index 000000000..eaffcf7f3
--- /dev/null
+++ b/test/command/4465.md
@@ -0,0 +1,15 @@
+```
+% pandoc -f html -t markdown
+<ol>
+  <li>An ordered list can contain block-level elements ind html, it means that divs are also allowed.</li>
+  <li>Let's see the problem! <div class="example">This is an example.</div></li>
+</ol>
+^D
+1.  An ordered list can contain block-level elements ind html, it means
+    that divs are also allowed.
+2.  Let\'s see the problem!
+
+    ::: example
+    This is an example.
+    :::
+```
-- 
cgit v1.2.3


From 851d037b3eee4516fde50b81eb8a0fc9b2f1545b Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Mon, 28 Jun 2021 22:41:14 -0700
Subject: Improve punctuation moving with `--citeproc`.

Previously, using `--citeproc` could cause punctuation to move in
quotes even when there aer no citations. This has been changed;
now, punctuation moving is limited to citations.

In addition, we only move footnotes around punctuation if the
style is a note style, even if `notes-after-punctuation` is `true`.
---
 src/Text/Pandoc/Citeproc.hs         | 29 +++++++++++++++--------------
 test/command/6890.md                |  4 ++--
 test/command/pandoc-citeproc-322.md |  2 +-
 3 files changed, 18 insertions(+), 17 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Citeproc.hs b/src/Text/Pandoc/Citeproc.hs
index a5b26c9b4..a2fca106a 100644
--- a/src/Text/Pandoc/Citeproc.hs
+++ b/src/Text/Pandoc/Citeproc.hs
@@ -90,21 +90,15 @@ processCitations (Pandoc meta bs) = do
                          walk (convertQuotes locale) .
                          insertSpace $ out)
                       (resultBibliography result)
-  let moveNotes = maybe True truish $
-                        lookupMeta "notes-after-punctuation" meta
+  let moveNotes = styleIsNoteStyle sopts &&
+           maybe True truish (lookupMeta "notes-after-punctuation" meta)
   let cits = map (walk (convertQuotes locale)) $
                resultCitations result
 
-  let fixQuotes = case localePunctuationInQuote locale of
-                    Just True ->
-                      B.toList . movePunctuationInsideQuotes .  B.fromList
-                    _ -> id
-
   let metanocites = lookupMeta "nocite" meta
   let Pandoc meta'' bs' =
          maybe id (setMeta "nocite") metanocites .
-         walk (map capitalizeNoteCitation .
-                fixQuotes .  mvPunct moveNotes locale) .
+         walk (map capitalizeNoteCitation .  mvPunct moveNotes locale) .
          walk deNote .
          evalState (walkM insertResolvedCitations $ Pandoc meta' bs)
          $ cits
@@ -375,7 +369,6 @@ formatFromExtension fp = case dropWhile (== '.') $ takeExtension fp of
 
 
 isNote :: Inline -> Bool
-isNote (Note _)          = True
 isNote (Cite _ [Note _]) = True
  -- the following allows citation styles that are "in-text" but use superscript
  -- references to be treated as if they are "notes" for the purposes of moving
@@ -388,6 +381,12 @@ isSpacy Space     = True
 isSpacy SoftBreak = True
 isSpacy _         = False
 
+movePunctInsideQuotes :: Locale -> [Inline] -> [Inline]
+movePunctInsideQuotes locale
+  | localePunctuationInQuote locale == Just True
+    = B.toList . movePunctuationInsideQuotes . B.fromList
+  | otherwise
+    = id
 
 mvPunct :: Bool -> Locale -> [Inline] -> [Inline]
 mvPunct moveNotes locale (x : xs)
@@ -400,7 +399,8 @@ mvPunct moveNotes locale (q : s : x : ys)
     in  if moveNotes
            then if T.null spunct
                    then q : x : mvPunct moveNotes locale ys
-                   else q : Str spunct : x : mvPunct moveNotes locale
+                   else movePunctInsideQuotes locale
+                        [q , Str spunct , x] ++ mvPunct moveNotes locale
                         (B.toList
                           (dropTextWhile isPunctuation (B.fromList ys)))
            else q : x : mvPunct moveNotes locale ys
@@ -412,9 +412,10 @@ mvPunct moveNotes locale (Cite cs ils : ys)
    , moveNotes
    = let s = stringify ys
          spunct = T.takeWhile isPunctuation s
-     in  Cite cs (init ils
-                  ++ [Str spunct | not (endWithPunct False (init ils))]
-                  ++ [last ils]) :
+     in  Cite cs (movePunctInsideQuotes locale $
+                    init ils
+                    ++ [Str spunct | not (endWithPunct False (init ils))]
+                    ++ [last ils]) :
          mvPunct moveNotes locale
            (B.toList (dropTextWhile isPunctuation (B.fromList ys)))
 mvPunct moveNotes locale (s : x : ys) | isSpacy s, isNote x =
diff --git a/test/command/6890.md b/test/command/6890.md
index e4129e2a9..e36c12771 100644
--- a/test/command/6890.md
+++ b/test/command/6890.md
@@ -23,12 +23,12 @@ references:
 
 @fruchtel-sozialer-2013a
 
-Some text [^1].
+Some text.[^1]
 
 [^1]: @fruchtel-sozialer-2013a
 ^D
 [Para [Cite [Citation {citationId = "fruchtel-sozialer-2013a", citationPrefix = [], citationSuffix = [], citationMode = AuthorInText, citationNoteNum = 1, citationHash = 0}] [Str "Fr\252chtel,",Space,Str "Budde,",Space,Str "and",Space,Str "Cyprian",Space,Str "(2013)"]]
-,Para [Str "Some",Space,Str "text",Str ".",Note [Para [Cite [Citation {citationId = "fruchtel-sozialer-2013a", citationPrefix = [], citationSuffix = [], citationMode = AuthorInText, citationNoteNum = 2, citationHash = 0}] [Str "Fr\252chtel,",Space,Str "Budde,",Space,Str "and",Space,Str "Cyprian",Space,Str "(2013)"]]],Str ""]
+,Para [Str "Some",Space,Str "text.",Note [Para [Cite [Citation {citationId = "fruchtel-sozialer-2013a", citationPrefix = [], citationSuffix = [], citationMode = AuthorInText, citationNoteNum = 2, citationHash = 0}] [Str "Fr\252chtel,",Space,Str "Budde,",Space,Str "and",Space,Str "Cyprian",Space,Str "(2013)"]]]]
 ,Div ("refs",["references","csl-bib-body","hanging-indent"],[])
  [Div ("ref-fruchtel-sozialer-2013a",["csl-entry"],[])
   [Para [Str "Fr\252chtel,",Space,Str "Frank,",Space,Str "Wolfgang",Space,Str "Budde,",Space,Str "and",Space,Str "Gudrun",Space,Str "Cyprian.",Space,Str "2013.",Space,Emph [Str "Sozialer",Space,Str "Raum",Space,Str "und",Space,Str "Soziale",Space,Str "Arbeit",Space,Str "Fieldbook:",Space,Str "Methoden",Space,Str "und",Space,Str "Techniken"],Str ".",Space,Str "3rd",Space,Str "ed.",Space,Str "Wiesbaden,",Space,Str "Germany:",Space,Str "Springer",Space,Str "VS."]]]]
diff --git a/test/command/pandoc-citeproc-322.md b/test/command/pandoc-citeproc-322.md
index 78494f0c4..c70eae755 100644
--- a/test/command/pandoc-citeproc-322.md
+++ b/test/command/pandoc-citeproc-322.md
@@ -19,7 +19,7 @@ references:
   type: 'article-journal'
 ---
 
-Foo[@timmory__justice_1950].
+Foo [@timmory__justice_1950].
 ^D
 Foo.[^1]
 
-- 
cgit v1.2.3


From f4ef652a4165fefe4b587882cc75d1e24971ef1f Mon Sep 17 00:00:00 2001
From: Aner Lucero <4rgento@gmail.com>
Date: Tue, 29 Jun 2021 08:49:36 -0300
Subject: Remove duplicated alt text in HTML output.

---
 src/Text/Pandoc/Writers/HTML.hs |  5 +++--
 test/command/7416.md            | 19 +++++++++++++++++++
 2 files changed, 22 insertions(+), 2 deletions(-)
 create mode 100644 test/command/7416.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/HTML.hs b/src/Text/Pandoc/Writers/HTML.hs
index b99b1a413..df01fc35f 100644
--- a/src/Text/Pandoc/Writers/HTML.hs
+++ b/src/Text/Pandoc/Writers/HTML.hs
@@ -1433,7 +1433,7 @@ inlineToHtml opts inline = do
                         return $ if T.null tit
                                     then link'
                                     else link' ! A.title (toValue tit)
-    (Image attr txt (s,tit)) -> do
+    (Image attr@(_, _, attrList) txt (s, tit)) -> do
                         let alternate = stringify txt
                         slideVariant <- gets stSlideVariant
                         let isReveal = slideVariant == RevealJsSlides
@@ -1446,7 +1446,8 @@ inlineToHtml opts inline = do
                               [A.title $ toValue tit | not (T.null tit)] ++
                               attrs
                             imageTag = (if html5 then H5.img else H.img
-                              , [A.alt $ toValue alternate | not (null txt)] )
+                              , [A.alt $ toValue alternate | not (null txt) &&
+                                  isNothing (lookup "alt" attrList)] )
                             mediaTag tg fallbackTxt =
                               let linkTxt = if null txt
                                             then fallbackTxt
diff --git a/test/command/7416.md b/test/command/7416.md
new file mode 100644
index 000000000..70a0257fa
--- /dev/null
+++ b/test/command/7416.md
@@ -0,0 +1,19 @@
+```
+% pandoc -f markdown -t html
+![caption](../media/rId25.jpg "title"){alt="alt"}
+
+^D
+<figure>
+<img src="../media/rId25.jpg" title="title" alt="alt" /><figcaption aria-hidden="true">caption</figcaption>
+</figure>
+```
+
+```
+% pandoc -f markdown -t html
+![caption](../media/rId25.jpg "title")
+
+^D
+<figure>
+<img src="../media/rId25.jpg" title="title" alt="caption" /><figcaption aria-hidden="true">caption</figcaption>
+</figure>
+```
-- 
cgit v1.2.3


From b7572db224123ab6c193ccdf24d2c3dd4fc0b0dc Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Tue, 29 Jun 2021 09:18:00 -0700
Subject: Use dev version of citeproc.

This eliminates double hyperlinks in author-in-text citations.
Author-only citations are no longer hyperlinked.
See jgm/citeproc#77.
---
 cabal.project                                              |  2 +-
 stack.yaml                                                 |  2 +-
 test/command/pandoc-citeproc-chicago-author-date.md        | 14 +++++++-------
 .../pandoc-citeproc-chicago-fullnote-bibliography.md       | 13 ++++++-------
 4 files changed, 15 insertions(+), 16 deletions(-)

(limited to 'test/command')

diff --git a/cabal.project b/cabal.project
index ba7f4f4cb..99c7b3556 100644
--- a/cabal.project
+++ b/cabal.project
@@ -10,4 +10,4 @@ source-repository-package
 source-repository-package
   type: git
   location: https://github.com/jgm/citeproc
-  tag: d201ddc812b1550d70c3e448eec6d9c3af72d96b
+  tag: 85277c4baa6c0350baf29247583ae0f438f7c9c5
diff --git a/stack.yaml b/stack.yaml
index f67da4ec7..2db0a626f 100644
--- a/stack.yaml
+++ b/stack.yaml
@@ -14,7 +14,7 @@ extra-deps:
 - git: https://github.com/jgm/doctemplates
   commit: 428c26d5303cf7a2b1051fe1ffd9aafe9ba71c81
 - git: https://github.com/jgm/citeproc
-  commit: d201ddc812b1550d70c3e448eec6d9c3af72d96b
+  commit: 85277c4baa6c0350baf29247583ae0f438f7c9c5
 ghc-options:
    "$locals": -fhide-source-paths -Wno-missing-home-modules
 resolver: lts-18.0
diff --git a/test/command/pandoc-citeproc-chicago-author-date.md b/test/command/pandoc-citeproc-chicago-author-date.md
index a70bc5de8..503dfdc91 100644
--- a/test/command/pandoc-citeproc-chicago-author-date.md
+++ b/test/command/pandoc-citeproc-chicago-author-date.md
@@ -54,14 +54,14 @@ References {#references .unnumbered}
 
 ([**nonexistent?**](#ref-nonexistent))
 
-[Doe](#ref-item1) ([2005](#ref-item1)) says blah.
+Doe ([2005](#ref-item1)) says blah.
 
-[Doe](#ref-item1) ([2005, 30](#ref-item1)) says blah.
+Doe ([2005, 30](#ref-item1)) says blah.
 
-[Doe](#ref-item1) ([2005, 30](#ref-item1), with suffix) says blah.
+Doe ([2005, 30](#ref-item1), with suffix) says blah.
 
-[Doe](#ref-item1) ([2005](#ref-item1); [2006, 30](#ref-item2); see also
-[Doe and Roe 2007](#ref-пункт3)) says blah.
+Doe ([2005](#ref-item1); [2006, 30](#ref-item2); see also [Doe and Roe
+2007](#ref-пункт3)) says blah.
 
 In a note.[^1]
 
@@ -98,8 +98,8 @@ edited by Sam Smith. Oxford: Oxford University Press.
 :::
 :::
 
-[^1]: [Doe and Roe](#ref-пункт3) ([2007, 12](#ref-пункт3)) and a
-    citation without locators ([Doe and Roe 2007](#ref-пункт3)).
+[^1]: Doe and Roe ([2007, 12](#ref-пункт3)) and a citation without
+    locators ([Doe and Roe 2007](#ref-пункт3)).
 
 [^2]: Some citations (see [Doe 2005, chap. 3](#ref-item1);
     [2006](#ref-item2); [Doe and Roe 2007](#ref-пункт3)).
diff --git a/test/command/pandoc-citeproc-chicago-fullnote-bibliography.md b/test/command/pandoc-citeproc-chicago-fullnote-bibliography.md
index d997d818b..e5ab30a52 100644
--- a/test/command/pandoc-citeproc-chicago-fullnote-bibliography.md
+++ b/test/command/pandoc-citeproc-chicago-fullnote-bibliography.md
@@ -55,13 +55,13 @@ References {#references .unnumbered}
 
 [^2]
 
-[John Doe](#ref-item1)[^3] says blah.
+John Doe[^3] says blah.
 
-[Doe](#ref-item1)[^4] says blah.
+Doe[^4] says blah.
 
-[Doe](#ref-item1)[^5] says blah.
+Doe[^5] says blah.
 
-[Doe](#ref-item1)[^6] says blah.
+Doe[^6] says blah.
 
 In a note.[^7]
 
@@ -112,9 +112,8 @@ Sam Smith. Oxford: Oxford University Press, 2007.
     Roe, "Why Water Is Wet," in *Third Book*, ed. Sam Smith (Oxford:
     Oxford University Press, 2007)](#ref-пункт3).
 
-[^7]: [Doe and Roe](#ref-пункт3), ["Why Water Is Wet," 12](#ref-пункт3)
-    and a citation without locators ([Doe and Roe, "Why Water Is
-    Wet"](#ref-пункт3)).
+[^7]: Doe and Roe, ["Why Water Is Wet," 12](#ref-пункт3) and a citation
+    without locators ([Doe and Roe, "Why Water Is Wet"](#ref-пункт3)).
 
 [^8]: See [Doe, *First Book*, chap. 3](#ref-item1); also [Doe and Roe,
     "Why Water Is Wet," 34--35](#ref-пункт3).
-- 
cgit v1.2.3


From cb038bb3125028e17a5b05c32495219a17fb6537 Mon Sep 17 00:00:00 2001
From: Aner Lucero <4rgento@gmail.com>
Date: Fri, 2 Jul 2021 11:17:14 -0300
Subject: HTML5 writer, remove aria-hidden when explicit atl text is provided.

---
 src/Text/Pandoc/Writers/HTML.hs | 11 +++++++----
 test/command/7416.md            |  2 +-
 2 files changed, 8 insertions(+), 5 deletions(-)

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Writers/HTML.hs b/src/Text/Pandoc/Writers/HTML.hs
index df01fc35f..7eb8dfe12 100644
--- a/src/Text/Pandoc/Writers/HTML.hs
+++ b/src/Text/Pandoc/Writers/HTML.hs
@@ -659,17 +659,20 @@ dimensionsToAttrList attr = consolidateStyles $ go Width ++ go Height
 figure :: PandocMonad m
        => WriterOptions -> Attr -> [Inline] -> (Text, Text)
        -> StateT WriterState m Html
-figure opts attr txt (s,tit) = do
+figure opts attr@(_, _, attrList) txt (s,tit) = do
   html5 <- gets stHtml5
   -- Screen-readers will normally read the @alt@ text and the figure; we
   -- want to avoid them reading the same text twice. With HTML5 we can
   -- use aria-hidden for the caption; with HTML4, we use an empty
   -- alt-text instead.
+  -- When the alt text differs from the caption both should be read.
   let alt = if html5 then txt else [Str ""]
   let tocapt = if html5
-                  then H5.figcaption !
-                       H5.customAttribute (textTag "aria-hidden")
-                                          (toValue @Text "true")
+                  then (H5.figcaption !) $
+                       if isJust (lookup "alt" attrList)
+                          then mempty
+                          else H5.customAttribute (textTag "aria-hidden")
+                                                  (toValue @Text "true")
                   else H.p ! A.class_ "caption"
   img <- inlineToHtml opts (Image attr alt (s,tit))
   capt <- if null txt
diff --git a/test/command/7416.md b/test/command/7416.md
index 70a0257fa..2f9577f10 100644
--- a/test/command/7416.md
+++ b/test/command/7416.md
@@ -4,7 +4,7 @@
 
 ^D
 <figure>
-<img src="../media/rId25.jpg" title="title" alt="alt" /><figcaption aria-hidden="true">caption</figcaption>
+<img src="../media/rId25.jpg" title="title" alt="alt" /><figcaption>caption</figcaption>
 </figure>
 ```
 
-- 
cgit v1.2.3


From 3a31fe68efcf80888294a1d3d33751d266d4de07 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Mon, 5 Jul 2021 14:45:07 -0700
Subject: Add command test for #7394.

And fix a small bug in handling of citations in notes, which
led to commas at the end of sentences in some cases.
---
 src/Text/Pandoc/Citeproc.hs |  1 +
 test/command/7394.md        | 85 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 86 insertions(+)
 create mode 100644 test/command/7394.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Citeproc.hs b/src/Text/Pandoc/Citeproc.hs
index 6e2d9ce7e..246f54516 100644
--- a/src/Text/Pandoc/Citeproc.hs
+++ b/src/Text/Pandoc/Citeproc.hs
@@ -612,6 +612,7 @@ deNote (Note bs) =
   noteInParens x = [x]
 
   noteAfterComma needsPer (Span ("",["csl-note"],[]) ils)
+    | not (null ils)
        = Str "," : Space :
          if needsPer
             then ils
diff --git a/test/command/7394.md b/test/command/7394.md
new file mode 100644
index 000000000..d6eb769b9
--- /dev/null
+++ b/test/command/7394.md
@@ -0,0 +1,85 @@
+```
+% pandoc -f markdown -t plain --citeproc
+---
+csl: command/chicago-fullnote-bibliography.csl
+references:
+- author:
+  - family: Wandt
+    given: Manfred
+  edition: 6
+  id: wandt2014ges-sv
+  issued: 2014
+  publisher: Franz Vahlen
+  publisher-place: München
+  title: Gesetzliche schuldverhältnisse
+  title-short: Gesetzl SV
+  type: book
+- author:
+  - family: Smith
+    given: Zenda
+  edition: 6
+  id: smith2015
+  issued: 2015
+  publisher: Macmillan
+  publisher-place: New York
+  title: A verb and a noun
+  type: book
+---
+
+Hi^[@wandt2014ges-sv.].
+
+Hi^[[@wandt2014ges-sv].].
+
+Hi^[[See also @wandt2014ges-sv].].
+
+Hi^[See also @wandt2014ges-sv.].
+
+Hi^[@wandt2014ges-sv [@smith2015].].
+
+Hi^[[@wandt2014ges-sv; @smith2015].].
+
+Hi [@wandt2014ges-sv].
+
+Hi [see also @wandt2014ges-sv].
+
+^D
+Hi[1].
+
+Hi[2].
+
+Hi[3].
+
+Hi[4].
+
+Hi[5].
+
+Hi[6].
+
+Hi.[7]
+
+Hi.[8]
+
+Smith, Zenda. A Verb and a Noun. 6th ed. New York: Macmillan, 2015.
+
+Wandt, Manfred. Gesetzliche Schuldverhältnisse. 6th ed. München: Franz
+Vahlen, 2014.
+
+[1] Manfred Wandt, Gesetzliche Schuldverhältnisse, 6th ed. (München:
+Franz Vahlen, 2014).
+
+[2] Wandt.
+
+[3] See also Wandt.
+
+[4] See also Wandt.
+
+[5] Wandt, Zenda Smith, A Verb and a Noun, 6th ed. (New York: Macmillan,
+2015).
+
+[6] Wandt, Gesetzl SV; Smith, A Verb and a Noun.
+
+[7] Wandt, Gesetzl SV.
+
+[8] See also Wandt.
+```
+
-- 
cgit v1.2.3


From ae22b1e977cfb1357bb21fabc227e76a6adb0599 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Fri, 9 Jul 2021 12:27:41 -0700
Subject: RST reader: fix regression with code includes.

With the recent changes to include infrastructure,
included code blocks were getting an extra newline.

Closes #7436.  Added regression test.
---
 pandoc.cabal                   |  1 +
 src/Text/Pandoc/Readers/RST.hs |  6 +++++-
 test/command/7436.md           | 14 ++++++++++++++
 test/command/three.txt         |  3 +++
 4 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 test/command/7436.md
 create mode 100644 test/command/three.txt

(limited to 'test/command')

diff --git a/pandoc.cabal b/pandoc.cabal
index 84a04c6b6..e3cd7e54f 100644
--- a/pandoc.cabal
+++ b/pandoc.cabal
@@ -213,6 +213,7 @@ extra-source-files:
                  test/command/B.txt
                  test/command/C.txt
                  test/command/D.txt
+                 test/command/three.txt
                  test/command/01.csv
                  test/command/chap1/spider.png
                  test/command/chap2/spider.png
diff --git a/src/Text/Pandoc/Readers/RST.hs b/src/Text/Pandoc/Readers/RST.hs
index 35292d949..3990f0cb5 100644
--- a/src/Text/Pandoc/Readers/RST.hs
+++ b/src/Text/Pandoc/Readers/RST.hs
@@ -474,6 +474,7 @@ includeDirective top fields body = do
            case lookup "literal" fields of
              Just _  -> B.rawBlock "rst" . sourcesToText <$> getInput
              Nothing -> parseBlocks
+  let isLiteral = isJust (lookup "code" fields `mplus` lookup "literal" fields)
   let selectLines =
         (case trim <$> lookup "end-before" fields of
                          Just patt -> takeWhile (not . (patt `T.isInfixOf`))
@@ -482,8 +483,11 @@ includeDirective top fields body = do
                          Just patt -> drop 1 .
                                         dropWhile (not . (patt `T.isInfixOf`))
                          Nothing   -> id)
+
   let toStream t =
-        toSources [(f, T.unlines . selectLines . T.lines $ t)]
+        Sources [(initialPos f,
+                   (T.unlines . selectLines . T.lines $ t) <>
+                    if isLiteral then mempty else "\n")]  -- see #7436
   currentDir <- takeDirectory . sourceName <$> getPosition
   insertIncludedFile parser toStream [currentDir] f startLine endLine
 
diff --git a/test/command/7436.md b/test/command/7436.md
new file mode 100644
index 000000000..ad4cb8c2f
--- /dev/null
+++ b/test/command/7436.md
@@ -0,0 +1,14 @@
+```
+% pandoc -f rst -t native
+.. include:: command/three.txt
+   :code:
+
+.. include:: command/three.txt
+   :literal:
+
+.. include:: command/three.txt
+^D
+[CodeBlock ("",[""],[("code","")]) "1st line.\n2nd line.\n3rd line.\n"
+,RawBlock (Format "rst") "1st line.\n2nd line.\n3rd line.\n"
+,Para [Str "1st",Space,Str "line.",SoftBreak,Str "2nd",Space,Str "line.",SoftBreak,Str "3rd",Space,Str "line."]]
+```
diff --git a/test/command/three.txt b/test/command/three.txt
new file mode 100644
index 000000000..3ca3fdd4e
--- /dev/null
+++ b/test/command/three.txt
@@ -0,0 +1,3 @@
+1st line.
+2nd line.
+3rd line.
-- 
cgit v1.2.3


From ac0a9da6d85e9b7a73973a20019caa324b2c1aff Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sun, 11 Jul 2021 13:01:45 -0700
Subject: Improved parsing of raw LaTeX from Text streams (rawLaTeXParser).

We now use source positions from the token stream to tell us
how much of the text stream to consume.  Getting this to
work required a few other changes to make token source positions
accurate.

Closes #7434.
---
 src/Text/Pandoc/Readers/LaTeX.hs         |  9 +++-----
 src/Text/Pandoc/Readers/LaTeX/Parsing.hs | 39 ++++++++++++++++++++++++++++----
 test/command/7434.md                     | 15 ++++++++++++
 3 files changed, 52 insertions(+), 11 deletions(-)
 create mode 100644 test/command/7434.md

(limited to 'test/command')

diff --git a/src/Text/Pandoc/Readers/LaTeX.hs b/src/Text/Pandoc/Readers/LaTeX.hs
index 9e14c159a..31c8d9095 100644
--- a/src/Text/Pandoc/Readers/LaTeX.hs
+++ b/src/Text/Pandoc/Readers/LaTeX.hs
@@ -136,8 +136,7 @@ rawLaTeXBlock :: (PandocMonad m, HasMacros s, HasReaderOptions s)
               => ParserT Sources s m Text
 rawLaTeXBlock = do
   lookAhead (try (char '\\' >> letter))
-  inp <- getInput
-  let toks = tokenizeSources inp
+  toks <- getInputTokens
   snd <$> (rawLaTeXParser toks False (macroDef (const mempty)) blocks
       <|> rawLaTeXParser toks True
              (do choice (map controlSeq
@@ -167,8 +166,7 @@ rawLaTeXInline :: (PandocMonad m, HasMacros s, HasReaderOptions s)
                => ParserT Sources s m Text
 rawLaTeXInline = do
   lookAhead (try (char '\\' >> letter))
-  inp <- getInput
-  let toks = tokenizeSources inp
+  toks <- getInputTokens
   raw <- snd <$>
           (   rawLaTeXParser toks True
               (mempty <$ (controlSeq "input" >> skipMany rawopt >> braced))
@@ -182,8 +180,7 @@ rawLaTeXInline = do
 inlineCommand :: PandocMonad m => ParserT Sources ParserState m Inlines
 inlineCommand = do
   lookAhead (try (char '\\' >> letter))
-  inp <- getInput
-  let toks = tokenizeSources inp
+  toks <- getInputTokens
   fst <$> rawLaTeXParser toks True (inlineEnvironment <|> inlineCommand')
           inlines
 
diff --git a/src/Text/Pandoc/Readers/LaTeX/Parsing.hs b/src/Text/Pandoc/Readers/LaTeX/Parsing.hs
index a17b1f324..9dac4d6ef 100644
--- a/src/Text/Pandoc/Readers/LaTeX/Parsing.hs
+++ b/src/Text/Pandoc/Readers/LaTeX/Parsing.hs
@@ -1,4 +1,5 @@
 {-# LANGUAGE FlexibleInstances     #-}
+{-# LANGUAGE FlexibleContexts      #-}
 {-# LANGUAGE LambdaCase            #-}
 {-# LANGUAGE MultiParamTypeClasses #-}
 {-# LANGUAGE OverloadedStrings     #-}
@@ -28,6 +29,7 @@ module Text.Pandoc.Readers.LaTeX.Parsing
   , applyMacros
   , tokenize
   , tokenizeSources
+  , getInputTokens
   , untokenize
   , untoken
   , totoks
@@ -246,18 +248,23 @@ withVerbatimMode parser = do
        updateState $ \st -> st{ sVerbatimMode = False }
        return result
 
-rawLaTeXParser :: (PandocMonad m, HasMacros s, HasReaderOptions s)
+rawLaTeXParser :: (PandocMonad m, HasMacros s, HasReaderOptions s, Show a)
                => [Tok] -> Bool -> LP m a -> LP m a
                -> ParserT Sources s m (a, Text)
 rawLaTeXParser toks retokenize parser valParser = do
   pstate <- getState
   let lstate = def{ sOptions = extractReaderOptions pstate }
   let lstate' = lstate { sMacros = extractMacros pstate }
+  let setStartPos = case toks of
+                      Tok pos _ _ : _ -> setPosition pos
+                      _ -> return ()
+  let preparser = setStartPos >> parser
   let rawparser = (,) <$> withRaw valParser <*> getState
-  res' <- lift $ runParserT (snd <$> withRaw parser) lstate "chunk" toks
+  res' <- lift $ runParserT (withRaw (preparser >> getPosition))
+                            lstate "chunk" toks
   case res' of
        Left _    -> mzero
-       Right toks' -> do
+       Right (endpos, toks') -> do
          res <- lift $ runParserT (do when retokenize $ do
                                         -- retokenize, applying macros
                                         ts <- many (satisfyTok (const True))
@@ -268,7 +275,13 @@ rawLaTeXParser toks retokenize parser valParser = do
               Left _    -> mzero
               Right ((val, raw), st) -> do
                 updateState (updateMacros (sMacros st <>))
-                void $ count (T.length (untokenize toks')) anyChar
+                let skipTilPos stopPos = do
+                      anyChar
+                      pos <- getPosition
+                      if pos >= stopPos
+                         then return ()
+                         else skipTilPos stopPos
+                skipTilPos endpos
                 let result = untokenize raw
                 -- ensure we end with space if input did, see #4442
                 let result' =
@@ -306,6 +319,17 @@ tokenizeSources = concatMap tokenizeSource . unSources
  where
    tokenizeSource (pos, t) = totoks pos t
 
+-- Return tokens from input sources. Ensure that starting position is
+-- correct.
+getInputTokens :: PandocMonad m => ParserT Sources s m [Tok]
+getInputTokens = do
+  pos <- getPosition
+  ss <- getInput
+  return $
+    case ss of
+      Sources [] -> []
+      Sources ((_,t):rest) -> tokenizeSources $ Sources ((pos,t):rest)
+
 tokenize :: SourceName -> Text -> [Tok]
 tokenize sourcename = totoks (initialPos sourcename)
 
@@ -433,8 +457,13 @@ parseFromToks :: PandocMonad m => LP m a -> [Tok] -> LP m a
 parseFromToks parser toks = do
   oldInput <- getInput
   setInput toks
+  oldpos <- getPosition
+  case toks of
+     Tok pos _ _ : _ -> setPosition pos
+     _ -> return ()
   result <- disablingWithRaw parser
   setInput oldInput
+  setPosition oldpos
   return result
 
 disablingWithRaw :: PandocMonad m => LP m a -> LP m a
@@ -458,7 +487,7 @@ satisfyTok f = do
                   | otherwise = Nothing
         updatePos :: SourcePos -> Tok -> [Tok] -> SourcePos
         updatePos _spos _ (Tok pos _ _ : _) = pos
-        updatePos spos _ []                 = incSourceColumn spos 1
+        updatePos spos (Tok _ _ t)  []      = incSourceColumn spos (T.length t)
 
 doMacros :: PandocMonad m => LP m ()
 doMacros = do
diff --git a/test/command/7434.md b/test/command/7434.md
new file mode 100644
index 000000000..be8732bc2
--- /dev/null
+++ b/test/command/7434.md
@@ -0,0 +1,15 @@
+```
+% pandoc -f markdown -t native
+\begin{proof}
+\newcommand{\x}{\left.\right.}
+\x
+\end{proof}
+
+1234567890abcdefghi
+
+[\*\a](x)
+^D
+[RawBlock (Format "tex") "\\begin{proof}\n\\newcommand{\\x}{\\left.\\right.}\n\\left.\\right.\n\\end{proof}"
+,Para [Str "1234567890abcdefghi"]
+,Para [Link ("",[],[]) [Str "*",RawInline (Format "tex") "\\a"] ("x","")]]
+```
-- 
cgit v1.2.3