Removed `--normalize` option and normalization functions from Shared.

* Removed normalize, normalizeInlines, normalizeBlocks
  from Text.Pandoc.Shared.  These shouldn't now be necessary,
  since normalization is handled automatically by the Builder
  monoid instance.

* Remove `--normalize` command-line option.

* Don't use normalize in tests.

* A few revisions to readers so they work well without normalize.

5 files changed, 20 insertions, 159 deletions

diff --git a/src/Text/Pandoc/Readers/RST.hs b/src/Text/Pandoc/Readers/RST.hs
index 75cd03d30..57b6c6f6c 100644
--- a/src/Text/Pandoc/Readers/RST.hs
+++ b/src/Text/Pandoc/Readers/RST.hs
@@ -744,7 +744,7 @@ addNewRole roleString fields = do
           M.insert role (baseRole, fmt, attr) customRoles
     }
 
-    return $ B.singleton Null
+    return mempty
   where
     countKeys k = length . filter (== k) . map fst $ fields
     inheritedRole =
diff --git a/src/Text/Pandoc/Readers/Txt2Tags.hs b/src/Text/Pandoc/Readers/Txt2Tags.hs
index 4abe13827..d2459ba47 100644
--- a/src/Text/Pandoc/Readers/Txt2Tags.hs
+++ b/src/Text/Pandoc/Readers/Txt2Tags.hs
@@ -447,9 +447,13 @@ inlineMarkup p f c special = try $ do
       lastChar <- anyChar
       end <- many1 (char c)
       let parser inp = parseFromString (mconcat <$> many p) inp
-      let start' = special (drop 2 start)
+      let start' = case drop 2 start of
+                          "" -> mempty
+                          xs -> special xs
       body' <- parser (middle ++ [lastChar])
-      let end' = special (drop 2 end)
+      let end' = case drop 2 end of
+                          "" -> mempty
+                          xs -> special xs
       return $ f (start' <> body' <> end')
     Nothing -> do -- Either bad or case such as *****
       guard (l >= 5)
diff --git a/src/Text/Pandoc/Shared.hs b/src/Text/Pandoc/Shared.hs
index 3df016996..6f52a8290 100644
--- a/src/Text/Pandoc/Shared.hs
+++ b/src/Text/Pandoc/Shared.hs
@@ -55,9 +55,6 @@ module Text.Pandoc.Shared (
                      orderedListMarkers,
                      normalizeSpaces,
                      extractSpaces,
-                     normalize,
-                     normalizeInlines,
-                     normalizeBlocks,
                      removeFormatting,
                      stringify,
                      capitalize,
@@ -398,153 +395,6 @@ extractSpaces f is =
                     _                -> mempty in
   (left <> f (B.trimInlines . B.Many $ contents) <> right)
 
--- | Normalize @Pandoc@ document, consolidating doubled 'Space's,
--- combining adjacent 'Str's and 'Emph's, remove 'Null's and
--- empty elements, etc.
-normalize :: Pandoc -> Pandoc
-normalize (Pandoc (Meta meta) blocks) =
-  Pandoc (Meta $ M.map go meta) (normalizeBlocks blocks)
-  where go (MetaInlines xs) = MetaInlines $ normalizeInlines xs
-        go (MetaBlocks xs)  = MetaBlocks  $ normalizeBlocks xs
-        go (MetaList ms)    = MetaList $ map go ms
-        go (MetaMap m)      = MetaMap $ M.map go m
-        go x                = x
-
-normalizeBlocks :: [Block] -> [Block]
-normalizeBlocks (Null : xs) = normalizeBlocks xs
-normalizeBlocks (Div attr bs : xs) =
-  Div attr (normalizeBlocks bs) : normalizeBlocks xs
-normalizeBlocks (BlockQuote bs : xs) =
-  case normalizeBlocks bs of
-       []    -> normalizeBlocks xs
-       bs'   -> BlockQuote bs' : normalizeBlocks xs
-normalizeBlocks (BulletList [] : xs) = normalizeBlocks xs
-normalizeBlocks (BulletList items : xs) =
-  BulletList (map normalizeBlocks items) : normalizeBlocks xs
-normalizeBlocks (OrderedList _ [] : xs) = normalizeBlocks xs
-normalizeBlocks (OrderedList attr items : xs) =
-  OrderedList attr (map normalizeBlocks items) : normalizeBlocks xs
-normalizeBlocks (DefinitionList [] : xs) = normalizeBlocks xs
-normalizeBlocks (DefinitionList items : xs) =
-  DefinitionList (map go items) : normalizeBlocks xs
-  where go (ils, bs) = (normalizeInlines ils, map normalizeBlocks bs)
-normalizeBlocks (RawBlock _ "" : xs) = normalizeBlocks xs
-normalizeBlocks (RawBlock f x : xs) =
-   case normalizeBlocks xs of
-        (RawBlock f' x' : rest) | f' == f ->
-          RawBlock f (x ++ ('\n':x')) : rest
-        rest -> RawBlock f x : rest
-normalizeBlocks (Para ils : xs) =
-  case normalizeInlines ils of
-       []   -> normalizeBlocks xs
-       ils' -> Para ils' : normalizeBlocks xs
-normalizeBlocks (Plain ils : xs) =
-  case normalizeInlines ils of
-       []   -> normalizeBlocks xs
-       ils' -> Plain ils' : normalizeBlocks xs
-normalizeBlocks (Header lev attr ils : xs) =
-  Header lev attr (normalizeInlines ils) : normalizeBlocks xs
-normalizeBlocks (Table capt aligns widths hdrs rows : xs) =
-  Table (normalizeInlines capt) aligns widths
-    (map normalizeBlocks hdrs) (map (map normalizeBlocks) rows)
-  : normalizeBlocks xs
-normalizeBlocks (x:xs) = x : normalizeBlocks xs
-normalizeBlocks [] = []
-
-normalizeInlines :: [Inline] -> [Inline]
-normalizeInlines (Str x : ys) =
-  case concat (x : map fromStr strs) of
-        ""     -> rest
-        n      -> Str n : rest
-   where
-     (strs, rest)  = span isStr $ normalizeInlines ys
-     isStr (Str _) = True
-     isStr _       = False
-     fromStr (Str z) = z
-     fromStr _       = error "normalizeInlines - fromStr - not a Str"
-normalizeInlines (Space : SoftBreak : ys) =
-  SoftBreak : normalizeInlines ys
-normalizeInlines (Space : ys) =
-  if null rest
-     then []
-     else Space : rest
-   where isSp Space = True
-         isSp _     = False
-         rest       = dropWhile isSp $ normalizeInlines ys
-normalizeInlines (Emph xs : zs) =
-  case normalizeInlines zs of
-       (Emph ys : rest) -> normalizeInlines $
-         Emph (normalizeInlines $ xs ++ ys) : rest
-       rest -> case normalizeInlines xs of
-                    []  -> rest
-                    xs' -> Emph xs' : rest
-normalizeInlines (Strong xs : zs) =
-  case normalizeInlines zs of
-       (Strong ys : rest) -> normalizeInlines $
-         Strong (normalizeInlines $ xs ++ ys) : rest
-       rest -> case normalizeInlines xs of
-                    []  -> rest
-                    xs' -> Strong xs' : rest
-normalizeInlines (Subscript xs : zs) =
-  case normalizeInlines zs of
-       (Subscript ys : rest) -> normalizeInlines $
-         Subscript (normalizeInlines $ xs ++ ys) : rest
-       rest -> case normalizeInlines xs of
-                    []  -> rest
-                    xs' -> Subscript xs' : rest
-normalizeInlines (Superscript xs : zs) =
-  case normalizeInlines zs of
-       (Superscript ys : rest) -> normalizeInlines $
-         Superscript (normalizeInlines $ xs ++ ys) : rest
-       rest -> case normalizeInlines xs of
-                    []  -> rest
-                    xs' -> Superscript xs' : rest
-normalizeInlines (SmallCaps xs : zs) =
-  case normalizeInlines zs of
-       (SmallCaps ys : rest) -> normalizeInlines $
-         SmallCaps (normalizeInlines $ xs ++ ys) : rest
-       rest -> case normalizeInlines xs of
-                    []  -> rest
-                    xs' -> SmallCaps xs' : rest
-normalizeInlines (Strikeout xs : zs) =
-  case normalizeInlines zs of
-       (Strikeout ys : rest) -> normalizeInlines $
-         Strikeout (normalizeInlines $ xs ++ ys) : rest
-       rest -> case normalizeInlines xs of
-                    []  -> rest
-                    xs' -> Strikeout xs' : rest
-normalizeInlines (RawInline _ [] : ys) = normalizeInlines ys
-normalizeInlines (RawInline f xs : zs) =
-  case normalizeInlines zs of
-       (RawInline f' ys : rest) | f == f' -> normalizeInlines $
-         RawInline f (xs ++ ys) : rest
-       rest -> RawInline f xs : rest
-normalizeInlines (Code _ "" : ys) = normalizeInlines ys
-normalizeInlines (Code attr xs : zs) =
-  case normalizeInlines zs of
-       (Code attr' ys : rest) | attr == attr' -> normalizeInlines $
-         Code attr (xs ++ ys) : rest
-       rest -> Code attr xs : rest
--- allow empty spans, they may carry identifiers etc.
--- normalizeInlines (Span _ [] : ys) = normalizeInlines ys
-normalizeInlines (Span attr xs : zs) =
-  case normalizeInlines zs of
-       (Span attr' ys : rest) | attr == attr' -> normalizeInlines $
-         Span attr (normalizeInlines $ xs ++ ys) : rest
-       rest -> Span attr (normalizeInlines xs) : rest
-normalizeInlines (Note bs : ys) = Note (normalizeBlocks bs) :
-  normalizeInlines ys
-normalizeInlines (Quoted qt ils : ys) =
-  Quoted qt (normalizeInlines ils) : normalizeInlines ys
-normalizeInlines (Link attr ils t : ys) =
-  Link attr (normalizeInlines ils) t : normalizeInlines ys
-normalizeInlines (Image attr ils t : ys) =
-  Image attr (normalizeInlines ils) t : normalizeInlines ys
-normalizeInlines (Cite cs ils : ys) =
-  Cite cs (normalizeInlines ils) : normalizeInlines ys
-normalizeInlines (x : xs) = x : normalizeInlines xs
-normalizeInlines [] = []
-
 -- | Extract inlines, removing formatting.
 removeFormatting :: Walkable Inline a => a -> [Inline]
 removeFormatting = query go . walk deNote
diff --git a/src/Text/Pandoc/Writers/Docx.hs b/src/Text/Pandoc/Writers/Docx.hs
index 07aed0c9b..163b2f3af 100644
--- a/src/Text/Pandoc/Writers/Docx.hs
+++ b/src/Text/Pandoc/Writers/Docx.hs
@@ -141,7 +141,7 @@ defaultWriterState = WriterState{
       , stDelId          = 1
       , stStyleMaps      = defaultStyleMaps
       , stFirstPara      = False
-      , stTocTitle       = normalizeInlines [Str "Table of Contents"]
+      , stTocTitle       = [Str "Table of Contents"]
       , stDynamicParaProps = []
       , stDynamicTextProps = []
       }
@@ -207,7 +207,7 @@ isValidChar (ord -> c)
   | otherwise                     = False
 
 metaValueToInlines :: MetaValue -> [Inline]
-metaValueToInlines (MetaString s) = normalizeInlines [Str s]
+metaValueToInlines (MetaString s) = [Str s]
 metaValueToInlines (MetaInlines ils) = ils
 metaValueToInlines (MetaBlocks bs) = query return bs
 metaValueToInlines (MetaBool b) = [Str $ show b]
diff --git a/src/Text/Pandoc/Writers/DokuWiki.hs b/src/Text/Pandoc/Writers/DokuWiki.hs
index c7a09fe50..42cddcef8 100644
--- a/src/Text/Pandoc/Writers/DokuWiki.hs
+++ b/src/Text/Pandoc/Writers/DokuWiki.hs
@@ -45,7 +45,7 @@ import Text.Pandoc.Options ( WriterOptions(
                               , writerTemplate
                               , writerWrapText), WrapOption(..) )
 import Text.Pandoc.Shared ( escapeURI, linesToPara, removeFormatting
-                          , camelCaseToHyphenated, trimr, normalize, substitute )
+                          , camelCaseToHyphenated, trimr, substitute )
 import Text.Pandoc.Writers.Shared ( defField, metaToJSON )
 import Text.Pandoc.ImageSize
 import Text.Pandoc.Templates ( renderTemplate' )
@@ -80,7 +80,7 @@ type DokuWiki = ReaderT WriterEnvironment (State WriterState)
 -- | Convert Pandoc to DokuWiki.
 writeDokuWiki :: PandocMonad m => WriterOptions -> Pandoc -> m String
 writeDokuWiki opts document = return $
-  runDokuWiki (pandocToDokuWiki opts $ normalize document)
+  runDokuWiki (pandocToDokuWiki opts document)
 
 runDokuWiki :: DokuWiki a -> a
 runDokuWiki = flip evalState def . flip runReaderT def
@@ -394,9 +394,16 @@ blockListToDokuWiki :: WriterOptions -- ^ Options
                     -> DokuWiki String
 blockListToDokuWiki opts blocks = do
   backSlash <- stBackSlashLB <$> ask
+  let blocks' = consolidateRawBlocks blocks
   if backSlash
-    then (backSlashLineBreaks . vcat) <$> mapM (blockToDokuWiki opts) blocks
-    else vcat <$> mapM (blockToDokuWiki opts) blocks
+    then (backSlashLineBreaks . vcat) <$> mapM (blockToDokuWiki opts) blocks'
+    else vcat <$> mapM (blockToDokuWiki opts) blocks'
+
+consolidateRawBlocks :: [Block] -> [Block]
+consolidateRawBlocks [] = []
+consolidateRawBlocks (RawBlock f1 b1 : RawBlock f2 b2 : xs)
+  | f1 == f2 = consolidateRawBlocks (RawBlock f1 (b1 ++ "\n" ++ b2) : xs)
+consolidateRawBlocks (x:xs) = x : consolidateRawBlocks xs
 
 -- | Convert list of Pandoc inline elements to DokuWiki.
 inlineListToDokuWiki :: WriterOptions -> [Inline] -> DokuWiki String