From 0feb7504b1c68cef76b30ea9987e2eae3101714c Mon Sep 17 00:00:00 2001 From: John MacFarlane Date: Sat, 1 Jul 2017 19:31:43 +0200 Subject: Rewrote LaTeX reader with proper tokenization. This rewrite is primarily motivated by the need to get macros working properly. A side benefit is that the reader is significantly faster (27s -> 19s in one benchmark, and there is a lot of room for further optimization). We now tokenize the input text, then parse the token stream. Macros modify the token stream, so they should now be effective in any context, including math. Thus, we no longer need the clunky macro processing capacities of texmath. A custom state LaTeXState is used instead of ParserState. This, plus the tokenization, will require some rewriting of the exported functions rawLaTeXInline, inlineCommand, rawLaTeXBlock. * Added Text.Pandoc.Readers.LaTeX.Types (new exported module). Exports Macro, Tok, TokType, Line, Column. [API change] * Text.Pandoc.Parsing: adjusted type of `insertIncludedFile` so it can be used with token parser. * Removed old texmath macro stuff from Parsing. Use Macro from Text.Pandoc.Readers.LaTeX.Types instead. * Removed texmath macro material from Markdown reader. * Changed types for Text.Pandoc.Readers.LaTeX's rawLaTeXInline and rawLaTeXBlock. (Both now return a String, and they are polymorphic in state.) * Added orgMacros field to OrgState. [API change] * Removed readerApplyMacros from ReaderOptions. Now we just check the `latex_macros` reader extension. * Allow `\newcommand\foo{blah}` without braces. Fixes #1390. Fixes #2118. Fixes #3236. Fixes #3779. Fixes #934. Fixes #982. --- test/Tests/Readers/LaTeX.hs | 3 ++- test/command/1390.md | 20 ++++++++++++++++++++ test/command/2118.md | 11 +++++++++++ test/command/3113.md | 2 +- test/command/3236.md | 9 +++++++++ test/command/3558.md | 8 ++++++-- test/command/3779.md | 28 ++++++++++++++++++++++++++++ test/command/934.md | 12 ++++++++++++ test/command/982.md | 11 +++++++++++ test/latex-reader.latex | 1 - test/latex-reader.native | 2 +- test/markdown-reader-more.native | 4 ++-- 12 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 test/command/1390.md create mode 100644 test/command/2118.md create mode 100644 test/command/3236.md create mode 100644 test/command/3779.md create mode 100644 test/command/934.md create mode 100644 test/command/982.md (limited to 'test') diff --git a/test/Tests/Readers/LaTeX.hs b/test/Tests/Readers/LaTeX.hs index afac9e8cb..f2be6de5f 100644 --- a/test/Tests/Readers/LaTeX.hs +++ b/test/Tests/Readers/LaTeX.hs @@ -58,7 +58,8 @@ tests = [ testGroup "basic" , "blank lines + space + comments" =: "% my comment\n\n \n % another\n\nhi" =?> para "hi" , "comment in paragraph" =: - "hi % this is a comment\nthere\n" =?> para "hi there" + "hi % this is a comment\nthere\n" =?> + para ("hi" <> softbreak <> "there") ] , testGroup "code blocks" diff --git a/test/command/1390.md b/test/command/1390.md new file mode 100644 index 000000000..ffd2cef8d --- /dev/null +++ b/test/command/1390.md @@ -0,0 +1,20 @@ +``` +% pandoc -f latex -t native +\newcommand\foo{+} +Testing: $\mu\foo\eta$. +^D +[Para [Str "Testing:",Space,Math InlineMath "\\mu+\\eta",Str "."]] +``` + + + diff --git a/test/command/2118.md b/test/command/2118.md new file mode 100644 index 000000000..d640e2e2b --- /dev/null +++ b/test/command/2118.md @@ -0,0 +1,11 @@ +``` +% pandoc -f latex -t native +\newcommand{\inclgraph}{\includegraphics[width=0.8\textwidth]} +\begin{figure}[ht] + \inclgraph{setminus.png} + \caption{Set subtraction} + \label{fig:setminus} +\end{figure} +^D +[Para [Image ("",[],[("width","80%")]) [Str "Set",Space,Str "subtraction",Span ("",[],[("data-label","fig:setminus")]) []] ("setminus.png","fig:")]] +``` diff --git a/test/command/3113.md b/test/command/3113.md index f44e25709..5ca171d97 100644 --- a/test/command/3113.md +++ b/test/command/3113.md @@ -8,6 +8,6 @@ C&=&D,\\ E&=&F \end{eqnarray} ^D -[Para [Math DisplayMath "\\begin{aligned}\nA&=&B,\\\\\nC&=&D,\\\\\nE&=&F\\end{aligned}"]] +[Para [Math DisplayMath "\\begin{aligned}\nA&=&B,\\\\\nC&=&D,\\\\\n%\\end{eqnarray}\n%\\begin{eqnarray}\nE&=&F\\end{aligned}"]] ``` diff --git a/test/command/3236.md b/test/command/3236.md new file mode 100644 index 000000000..1d1a9b2c3 --- /dev/null +++ b/test/command/3236.md @@ -0,0 +1,9 @@ +``` +pandoc -f latex -t native +\newcommand{\mycolor}{red} + +\includegraphics[width=17cm]{\mycolor /header} +Magnificent \mycolor{} header. +^D +[Para [Image ("",[],[("width","17cm")]) [Str "image"] ("red/header",""),SoftBreak,Str "Magnificent",Space,Str "red",Space,Str "header."]] +``` diff --git a/test/command/3558.md b/test/command/3558.md index 3f4079064..795858b78 100644 --- a/test/command/3558.md +++ b/test/command/3558.md @@ -1,8 +1,12 @@ ``` % pandoc -t native -\startmulti +\multi + hello + \endmulti ^D -[Para [RawInline (Format "tex") "\\startmulti\n",Str "hello",SoftBreak,RawInline (Format "tex") "\\endmulti"]] +[RawBlock (Format "latex") "\\multi" +,Para [Str "hello"] +,RawBlock (Format "latex") "\\endmulti"] ``` diff --git a/test/command/3779.md b/test/command/3779.md new file mode 100644 index 000000000..1097123f0 --- /dev/null +++ b/test/command/3779.md @@ -0,0 +1,28 @@ +``` +% pandoc -f latex -t native +\newcommand{\fakeitemize}[1]{ + \begin{itemize} + #1 + \end{itemize} +} +\newcommand{\testcmd}[1]{ + #1 +} +\fakeitemize{ + \item Pandoc is 100\% awesome. +} + +\begin{itemize} + \item Pandoc is 200\% awesome. +\end{itemize} + +\testcmd{ + Pandoc is 300\% awesome. +} +^D +[BulletList + [[Para [Str "Pandoc",Space,Str "is",Space,Str "100%",Space,Str "awesome."]]] +,BulletList + [[Para [Str "Pandoc",Space,Str "is",Space,Str "200%",Space,Str "awesome."]]] +,Para [Str "Pandoc",Space,Str "is",Space,Str "300%",Space,Str "awesome."]] +``` diff --git a/test/command/934.md b/test/command/934.md new file mode 100644 index 000000000..ef99abdf9 --- /dev/null +++ b/test/command/934.md @@ -0,0 +1,12 @@ +``` +% pandoc -f latex -t native +\newcommand{\ddb}[2]{ + \textit{``#1''} + + \textbf{#2} +} +\ddb{This should be italic and in quotes}{And this is the attribution} +^D +[Para [Emph [Quoted DoubleQuote [Str "This",Space,Str "should",Space,Str "be",Space,Str "italic",Space,Str "and",Space,Str "in",Space,Str "quotes"]]] +,Para [Strong [Str "And",Space,Str "this",Space,Str "is",Space,Str "the",Space,Str "attribution"]]] +``` diff --git a/test/command/982.md b/test/command/982.md new file mode 100644 index 000000000..5f54f7713 --- /dev/null +++ b/test/command/982.md @@ -0,0 +1,11 @@ +``` +% pandoc -f latex -t native +\newcommand{\BEQ}{\begin{equation}} +\newcommand{\EEQ}{\end{equation}} + +\BEQ +y=x^2 +\EEQ +^D +[Para [Math DisplayMath "y=x^2"]] +``` diff --git a/test/latex-reader.latex b/test/latex-reader.latex index 2ebdfed99..7cbcc9672 100644 --- a/test/latex-reader.latex +++ b/test/latex-reader.latex @@ -4,7 +4,6 @@ \setlength{\parindent}{0pt} \setlength{\parskip}{6pt plus 2pt minus 1pt} -\newcommand{\textsubscript}[1]{\ensuremath{_{\scriptsize\textrm{#1}}}} \usepackage[breaklinks=true,unicode=true]{hyperref} \usepackage[normalem]{ulem} % avoid problems with \sout in headers with hyperref: diff --git a/test/latex-reader.native b/test/latex-reader.native index d481a714d..04be2538e 100644 --- a/test/latex-reader.native +++ b/test/latex-reader.native @@ -261,7 +261,7 @@ Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "John",Spa ,Header 1 ("latex",[],[]) [Str "LaTeX"] ,BulletList [[Para [Cite [Citation {citationId = "smith.1899", citationPrefix = [], citationSuffix = [Str "22-23"], citationMode = NormalCitation, citationNoteNum = 0, citationHash = 0}] [RawInline (Format "latex") "\\cite[22-23]{smith.1899}"]]] - ,[Para [RawInline (Format "latex") "\\doublespacing\n"]] + ,[Para [RawInline (Format "latex") "\\doublespacing"]] ,[Para [Math InlineMath "2+2=4"]] ,[Para [Math InlineMath "x \\in y"]] ,[Para [Math InlineMath "\\alpha \\wedge \\omega"]] diff --git a/test/markdown-reader-more.native b/test/markdown-reader-more.native index 1007dbac7..5d63a21de 100644 --- a/test/markdown-reader-more.native +++ b/test/markdown-reader-more.native @@ -3,7 +3,7 @@ Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "Author",S ,Header 2 ("blank-line-before-url-in-link-reference",[],[]) [Str "Blank",Space,Str "line",Space,Str "before",Space,Str "URL",Space,Str "in",Space,Str "link",Space,Str "reference"] ,Para [Link ("",[],[]) [Str "foo"] ("/url",""),Space,Str "and",Space,Link ("",[],[]) [Str "bar"] ("/url","title")] ,Header 2 ("raw-context-environments",[],[]) [Str "Raw",Space,Str "ConTeXt",Space,Str "environments"] -,Plain [RawInline (Format "tex") "\\placeformula "] +,RawBlock (Format "latex") "\\placeformula " ,RawBlock (Format "context") "\\startformula\n L_{1} = L_{2}\n \\stopformula" ,RawBlock (Format "context") "\\start[a2]\n\\start[a2]\n\\stop[a2]\n\\stop[a2]" ,Header 2 ("raw-latex-environments",[],[]) [Str "Raw",Space,Str "LaTeX",Space,Str "environments"] @@ -56,7 +56,7 @@ Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "Author",S ,OrderedList (3,Example,TwoParens) [[Plain [Str "Third",Space,Str "example."]]] ,Header 2 ("macros",[],[]) [Str "Macros"] -,Para [Math InlineMath "{\\langle x,y \\rangle}"] +,Para [Math InlineMath "\\langle x,y \\rangle"] ,Header 2 ("case-insensitive-references",[],[]) [Str "Case-insensitive",Space,Str "references"] ,Para [Link ("",[],[]) [Str "Fum"] ("/fum","")] ,Para [Link ("",[],[]) [Str "FUM"] ("/fum","")] -- cgit v1.2.3