Added tests of latex tokenizer.

This should help prevent regressions like #4159.

1 files changed, 18 insertions, 1 deletions

diff --git a/test/Tests/Readers/LaTeX.hs b/test/Tests/Readers/LaTeX.hs
index 99b49fd45..4396d550f 100644
--- a/test/Tests/Readers/LaTeX.hs
+++ b/test/Tests/Readers/LaTeX.hs
@@ -3,7 +3,11 @@ module Tests.Readers.LaTeX (tests) where
 
 import Data.Text (Text)
 import qualified Data.Text as T
+import qualified Text.Pandoc.UTF8 as UTF8
+import Text.Pandoc.Readers.LaTeX (tokenize, untokenize)
 import Test.Tasty
+import Test.Tasty.HUnit
+import Test.Tasty.QuickCheck
 import Tests.Helpers
 import Text.Pandoc
 import Text.Pandoc.Arbitrary ()
@@ -22,8 +26,21 @@ simpleTable' :: [Alignment] -> [[Blocks]] -> Blocks
 simpleTable' aligns = table "" (zip aligns (repeat 0.0))
                       (map (const mempty) aligns)
 
+tokUntokRt :: String -> Bool
+tokUntokRt s = untokenize (tokenize "random" t) == t
+  where t = T.pack s
+
 tests :: [TestTree]
-tests = [ testGroup "basic"
+tests = [ testGroup "tokenization"
+          [ testCase "tokenizer round trip on test case" $ do
+                 orig <- T.pack <$> UTF8.readFile "../test/latex-reader.latex"
+                 let new = untokenize $ tokenize "../test/latex-reader.latex"
+                             orig
+                 assertEqual "untokenize . tokenize is identity" orig new
+          , testProperty "untokenize . tokenize is identity" tokUntokRt
+          ]
+
+        , testGroup "basic"
           [ "simple" =:
             "word" =?> para "word"
           , "space" =: