Fix scope for LaTeX macros.

They should by default scope over the group in which they
are defined (except `\gdef` and `\xdef`, which are global).
In addition, environments must be treated as groups.

We handle this by making sMacros in the LaTeX parser state
a STACK of macro tables. Opening a group adds a table to
the stack, closing one removes one.  Only the top of the stack
is queried.

This commit adds a parameter for scope to the Macro constructor
(not exported).

Closes #7494.

4 files changed, 150 insertions, 55 deletions

diff --git a/src/Text/Pandoc/Readers/LaTeX/Macro.hs b/src/Text/Pandoc/Readers/LaTeX/Macro.hs
index 6a7b0eed3..5709cbb8c 100644
--- a/src/Text/Pandoc/Readers/LaTeX/Macro.hs
+++ b/src/Text/Pandoc/Readers/LaTeX/Macro.hs
@@ -15,6 +15,8 @@ import Control.Applicative ((<|>), optional)
 import qualified Data.Map as M
 import Data.Text (Text)
 import qualified Data.Text as T
+import qualified Data.List.NonEmpty as NonEmpty
+import Data.List.NonEmpty (NonEmpty(..))
 
 macroDef :: (PandocMonad m, Monoid a) => (Text -> a) -> LP m a
 macroDef constructor = do
@@ -26,24 +28,36 @@ macroDef constructor = do
           nameMacroPairs <- newcommand <|> letmacro <|>
             edefmacro <|> defmacro <|> newif
           guardDisabled Ext_latex_macros <|>
-           mapM_ (\(name, macro') ->
-                   updateState (\s -> s{ sMacros = M.insert name macro'
-                                          (sMacros s) })) nameMacroPairs
+            mapM_ insertMacro nameMacroPairs
         environmentDef = do
           mbenv <- newenvironment
           case mbenv of
             Nothing -> return ()
             Just (name, macro1, macro2) ->
               guardDisabled Ext_latex_macros <|>
-                do updateState $ \s -> s{ sMacros =
-                    M.insert name macro1 (sMacros s) }
-                   updateState $ \s -> s{ sMacros =
-                    M.insert ("end" <> name) macro2 (sMacros s) }
+                do insertMacro (name, macro1)
+                   insertMacro ("end" <> name, macro2)
         -- @\newenvironment{envname}[n-args][default]{begin}{end}@
         -- is equivalent to
         -- @\newcommand{\envname}[n-args][default]{begin}@
         -- @\newcommand{\endenvname}@
 
+insertMacro :: PandocMonad m => (Text, Macro) -> LP m ()
+insertMacro (name, macro'@(Macro GlobalScope _ _ _ _)) =
+  updateState $ \s ->
+     s{ sMacros = NonEmpty.map (M.insert name macro') (sMacros s) }
+insertMacro (name, macro'@(Macro GroupScope _ _ _ _)) =
+  updateState $ \s ->
+     s{ sMacros = M.insert name macro' (NonEmpty.head (sMacros s)) :|
+                      NonEmpty.tail (sMacros s) }
+
+lookupMacro :: PandocMonad m => Text -> LP m Macro
+lookupMacro name = do
+   macros :| _ <- sMacros <$> getState
+   case M.lookup name macros of
+     Just m -> return m
+     Nothing -> fail "Macro not found"
+
 letmacro :: PandocMonad m => LP m [(Text, Macro)]
 letmacro = do
   controlSeq "let"
@@ -54,20 +68,19 @@ letmacro = do
     -- we first parse in verbatim mode, and then expand macros,
     -- because we don't want \let\foo\bar to turn into
     -- \let\foo hello if we have previously \def\bar{hello}
-    macros <- sMacros <$> getState
     target <- anyControlSeq <|> singleChar
     case target of
       (Tok _ (CtrlSeq name') _) ->
-        case M.lookup name' macros of
-          Just m -> return [(name, m)]
-          Nothing -> return [(name, Macro ExpandWhenDefined [] Nothing [target])]
-      _ -> return [(name, Macro ExpandWhenDefined [] Nothing [target])]
+         (do m <- lookupMacro name'
+             pure [(name, m)])
+         <|> pure [(name,
+                    Macro GroupScope ExpandWhenDefined [] Nothing [target])]
+      _ -> pure [(name, Macro GroupScope ExpandWhenDefined [] Nothing [target])]
 
 edefmacro :: PandocMonad m => LP m [(Text, Macro)]
 edefmacro = do
-  controlSeq "edef" <|> controlSeq "xdef"
-  -- TODO Currently we don't distinguish these.  \edef should only
-  -- affect its own group, while \xdef sets a global macro.
+  scope <- (GroupScope <$ controlSeq "edef")
+       <|> (GlobalScope <$ controlSeq "xdef")
   (name, contents) <- withVerbatimMode $ do
     Tok _ (CtrlSeq name) _ <- anyControlSeq
     -- we first parse in verbatim mode, and then expand macros,
@@ -77,20 +90,19 @@ edefmacro = do
     return (name, contents)
   -- expand macros
   contents' <- parseFromToks (many anyTok) contents
-  return [(name, Macro ExpandWhenDefined [] Nothing contents')]
+  return [(name, Macro scope ExpandWhenDefined [] Nothing contents')]
 
 defmacro :: PandocMonad m => LP m [(Text, Macro)]
 defmacro = do
   -- we use withVerbatimMode, because macros are to be expanded
   -- at point of use, not point of definition
-  controlSeq "def" <|> controlSeq "gdef"
-  -- TODO Currently we don't distinguish these.  \def should only
-  -- affect its own group, while \gdef sets a global macro.
+  scope <- (GroupScope <$ controlSeq "def")
+       <|> (GlobalScope <$ controlSeq "gdef")
   withVerbatimMode $ do
     Tok _ (CtrlSeq name) _ <- anyControlSeq
     argspecs <- many (argspecArg <|> argspecPattern)
     contents <- bracedOrToken
-    return [(name, Macro ExpandWhenUsed argspecs Nothing contents)]
+    return [(name, Macro scope ExpandWhenUsed argspecs Nothing contents)]
 
 -- \newif\iffoo' defines:
 -- \iffoo to be \iffalse
@@ -105,16 +117,16 @@ newif = do
     -- \def\footrue{\def\iffoo\iftrue}
     -- \def\foofalse{\def\iffoo\iffalse}
     let base = T.drop 2 name
-    return [ (name, Macro ExpandWhenUsed [] Nothing
+    return [ (name, Macro GroupScope ExpandWhenUsed [] Nothing
                     [Tok pos (CtrlSeq "iffalse") "\\iffalse"])
            , (base <> "true",
-                   Macro ExpandWhenUsed [] Nothing
+                   Macro GroupScope ExpandWhenUsed [] Nothing
                    [ Tok pos (CtrlSeq "def") "\\def"
                    , Tok pos (CtrlSeq name) ("\\" <> name)
                    , Tok pos (CtrlSeq "iftrue") "\\iftrue"
                    ])
            , (base <> "false",
-                   Macro ExpandWhenUsed [] Nothing
+                   Macro GroupScope ExpandWhenUsed [] Nothing
                    [ Tok pos (CtrlSeq "def") "\\def"
                    , Tok pos (CtrlSeq name) ("\\" <> name)
                    , Tok pos (CtrlSeq "iffalse") "\\iffalse"
@@ -161,14 +173,13 @@ newcommand = do
                  : (contents' ++
                    [ Tok pos Symbol "}", Tok pos Symbol "}" ])
               _                     -> contents'
-    macros <- sMacros <$> getState
-    case M.lookup name macros of
-        Just macro
-          | mtype == "newcommand" -> do
-              report $ MacroAlreadyDefined txt pos
-              return [(name, macro)]
-          | mtype == "providecommand" -> return [(name, macro)]
-        _ -> return [(name, Macro ExpandWhenUsed argspecs optarg contents)]
+    let macro = Macro GroupScope ExpandWhenUsed argspecs optarg contents
+    (do lookupMacro name
+        case mtype of
+          "providecommand" -> return []
+          "renewcommand" -> return [(name, macro)]
+          _ -> [] <$ report (MacroAlreadyDefined txt pos))
+      <|> pure [(name, macro)]
 
 newenvironment :: PandocMonad m => LP m (Maybe (Text, Macro, Macro))
 newenvironment = do
@@ -187,17 +198,22 @@ newenvironment = do
     let argspecs = map (\i -> ArgNum i) [1..numargs]
     startcontents <- spaces >> bracedOrToken
     endcontents <- spaces >> bracedOrToken
-    macros <- sMacros <$> getState
-    case M.lookup name macros of
-         Just _
-           | mtype == "newenvironment" -> do
-               report $ MacroAlreadyDefined name pos
-               return Nothing
-           | mtype == "provideenvironment" ->
-               return Nothing
-         _ -> return $ Just (name,
-                      Macro ExpandWhenUsed argspecs optarg startcontents,
-                      Macro ExpandWhenUsed [] Nothing endcontents)
+    -- we need the environment to be in a group so macros defined
+    -- inside behave correctly:
+    let bg = Tok pos (CtrlSeq "bgroup") "\\bgroup "
+    let eg = Tok pos (CtrlSeq "egroup") "\\egroup "
+    let result = (name,
+                    Macro GroupScope ExpandWhenUsed argspecs optarg
+                      (bg:startcontents),
+                    Macro GroupScope ExpandWhenUsed [] Nothing (eg:endcontents))
+    (do lookupMacro name
+        case mtype of
+          "provideenvironment" -> return Nothing
+          "renewenvironment" -> return (Just result)
+          _ -> do
+             report $ MacroAlreadyDefined name pos
+             return Nothing)
+      <|> return (Just result)
 
 bracketedNum :: PandocMonad m => LP m Int
 bracketedNum = do
diff --git a/src/Text/Pandoc/Readers/LaTeX/Parsing.hs b/src/Text/Pandoc/Readers/LaTeX/Parsing.hs
index 8d6791abb..e1b72926f 100644
--- a/src/Text/Pandoc/Readers/LaTeX/Parsing.hs
+++ b/src/Text/Pandoc/Readers/LaTeX/Parsing.hs
@@ -102,6 +102,9 @@ import qualified Data.IntMap as IntMap
 import qualified Data.Map as M
 import qualified Data.Set as Set
 import Data.Text (Text)
+import Data.Maybe (fromMaybe)
+import Data.List.NonEmpty (NonEmpty(..))
+import qualified Data.List.NonEmpty as NonEmpty
 import qualified Data.Text as T
 import Text.Pandoc.Builder
 import Text.Pandoc.Class.PandocMonad (PandocMonad, report)
@@ -146,7 +149,7 @@ data TheoremSpec =
 data LaTeXState = LaTeXState{ sOptions       :: ReaderOptions
                             , sMeta          :: Meta
                             , sQuoteContext  :: QuoteContext
-                            , sMacros        :: M.Map Text Macro
+                            , sMacros        :: NonEmpty (M.Map Text Macro)
                             , sContainers    :: [Text]
                             , sLogMessages   :: [LogMessage]
                             , sIdentifiers   :: Set.Set Text
@@ -173,7 +176,7 @@ defaultLaTeXState :: LaTeXState
 defaultLaTeXState = LaTeXState{ sOptions       = def
                               , sMeta          = nullMeta
                               , sQuoteContext  = NoQuote
-                              , sMacros        = M.empty
+                              , sMacros        = M.empty :| []
                               , sContainers    = []
                               , sLogMessages   = []
                               , sIdentifiers   = Set.empty
@@ -220,8 +223,9 @@ instance HasIncludeFiles LaTeXState where
   dropLatestIncludeFile s = s { sContainers = drop 1 $ sContainers s }
 
 instance HasMacros LaTeXState where
-  extractMacros  st  = sMacros st
-  updateMacros f st  = st{ sMacros = f (sMacros st) }
+  extractMacros  st  = NonEmpty.head $ sMacros st
+  updateMacros f st  = st{ sMacros = f (NonEmpty.head $ sMacros st)
+                                     :| (NonEmpty.tail $ sMacros st) }
 
 instance HasReaderOptions LaTeXState where
   extractReaderOptions = sOptions
@@ -254,7 +258,7 @@ rawLaTeXParser :: (PandocMonad m, HasMacros s, HasReaderOptions s, Show a)
 rawLaTeXParser toks retokenize parser valParser = do
   pstate <- getState
   let lstate = def{ sOptions = extractReaderOptions pstate }
-  let lstate' = lstate { sMacros = extractMacros pstate }
+  let lstate' = lstate { sMacros = extractMacros pstate :| [] }
   let setStartPos = case toks of
                       Tok pos _ _ : _ -> setPosition pos
                       _ -> return ()
@@ -274,7 +278,7 @@ rawLaTeXParser toks retokenize parser valParser = do
          case res of
               Left _    -> mzero
               Right ((val, raw), st) -> do
-                updateState (updateMacros (sMacros st <>))
+                updateState (updateMacros ((NonEmpty.head (sMacros st)) <>))
                 let skipTilPos stopPos = do
                       anyChar
                       pos <- getPosition
@@ -299,7 +303,7 @@ applyMacros s = (guardDisabled Ext_latex_macros >> return s) <|>
    do let retokenize = untokenize <$> many anyTok
       pstate <- getState
       let lstate = def{ sOptions = extractReaderOptions pstate
-                      , sMacros  = extractMacros pstate }
+                      , sMacros  = extractMacros pstate :| [] }
       res <- runParserT retokenize lstate "math" (tokenize "math" s)
       case res of
            Left e   -> Prelude.fail (show e)
@@ -552,10 +556,10 @@ doMacros' n inp =
     handleMacros n' spos name ts = do
       when (n' > 20)  -- detect macro expansion loops
         $ throwError $ PandocMacroLoop name
-      macros <- sMacros <$> getState
+      (macros :| _ ) <- sMacros <$> getState
       case M.lookup name macros of
            Nothing -> trySpecialMacro name ts
-           Just (Macro expansionPoint argspecs optarg newtoks) -> do
+           Just (Macro _scope expansionPoint argspecs optarg newtoks) -> do
              let getargs' = do
                    args <-
                      (case expansionPoint of
@@ -745,10 +749,22 @@ primEscape = do
 bgroup :: PandocMonad m => LP m Tok
 bgroup = try $ do
   optional sp
-  symbol '{' <|> controlSeq "bgroup" <|> controlSeq "begingroup"
+  t <- symbol '{' <|> controlSeq "bgroup" <|> controlSeq "begingroup"
+  -- Add a copy of the macro table to the top of the macro stack,
+  -- private for this group. We inherit all the macros defined in
+  -- the parent group.
+  updateState $ \s -> s{ sMacros = NonEmpty.cons (NonEmpty.head (sMacros s))
+                                                 (sMacros s) }
+  return t
+
 
 egroup :: PandocMonad m => LP m Tok
-egroup = symbol '}' <|> controlSeq "egroup" <|> controlSeq "endgroup"
+egroup = do
+  t <- symbol '}' <|> controlSeq "egroup" <|> controlSeq "endgroup"
+  -- remove the group's macro table from the stack
+  updateState $ \s -> s{ sMacros = fromMaybe (sMacros s) $
+      NonEmpty.nonEmpty (NonEmpty.tail (sMacros s)) }
+  return t
 
 grouped :: (PandocMonad m,  Monoid a) => LP m a -> LP m a
 grouped parser = try $ do
@@ -1017,7 +1033,16 @@ resetCaption = updateState $ \st -> st{ sCaption   = Nothing
                                       , sLastLabel = Nothing }
 
 env :: PandocMonad m => Text -> LP m a -> LP m a
-env name p = p <* end_ name
+env name p = do
+  -- environments are groups as far as macros are concerned,
+  -- so we need a local copy of the macro table (see above, bgroup, egroup):
+  updateState $ \s -> s{ sMacros = NonEmpty.cons (NonEmpty.head (sMacros s))
+                                                 (sMacros s) }
+  result <- p
+  updateState $ \s -> s{ sMacros = fromMaybe (sMacros s) $
+      NonEmpty.nonEmpty (NonEmpty.tail (sMacros s)) }
+  end_ name
+  return result
 
 tokWith :: PandocMonad m => LP m Inlines -> LP m Inlines
 tokWith inlineParser = try $ spaces >>
diff --git a/src/Text/Pandoc/Readers/LaTeX/Types.hs b/src/Text/Pandoc/Readers/LaTeX/Types.hs
index c20b72bc5..a4eae56db 100644
--- a/src/Text/Pandoc/Readers/LaTeX/Types.hs
+++ b/src/Text/Pandoc/Readers/LaTeX/Types.hs
@@ -15,6 +15,7 @@ module Text.Pandoc.Readers.LaTeX.Types ( Tok(..)
                                        , Macro(..)
                                        , ArgSpec(..)
                                        , ExpansionPoint(..)
+                                       , MacroScope(..)
                                        , SourcePos
                                        )
 where
@@ -43,7 +44,10 @@ tokToText (Tok _ _ t) = t
 data ExpansionPoint = ExpandWhenDefined | ExpandWhenUsed
      deriving (Eq, Ord, Show)
 
-data Macro = Macro ExpansionPoint [ArgSpec] (Maybe [Tok]) [Tok]
+data MacroScope = GlobalScope | GroupScope
+  deriving (Eq, Ord, Show)
+
+data Macro = Macro MacroScope ExpansionPoint [ArgSpec] (Maybe [Tok]) [Tok]
      deriving Show
 
 data ArgSpec = ArgNum Int | Pattern [Tok]
diff --git a/test/command/7494.md b/test/command/7494.md
new file mode 100644
index 000000000..d8a2ff7a5
--- /dev/null
+++ b/test/command/7494.md
@@ -0,0 +1,50 @@
+```
+% pandoc -f latex -t plain
+\def\foo{BAR}
+{\foo
+\def\foo{BAZ}
+\foo
+}
+\foo
+^D
+BAR BAZ BAR
+```
+
+```
+% pandoc -f latex -t plain
+\def\foo{BAR}
+{\foo
+\gdef\foo{BAZ}
+\foo
+}
+\foo
+^D
+BAR BAZ BAZ
+```
+
+```
+% pandoc -f latex -t plain
+\newcommand{\aaa}{BBB}
+{
+\renewcommand{\aaa}{AAA}
+\aaa
+}
+\aaa
+^D
+AAA BBB
+```
+
+```
+% pandoc -f latex -t markdown
+\newcommand{\aaa}{BBB}
+\begin{quote}
+\renewcommand{\aaa}{AAA}
+\aaa
+\end{quote}
+\aaa
+^D
+> AAA
+
+BBB
+```
+