Populate permission database from a file

8 files changed, 234 insertions, 45 deletions

diff --git a/README.md b/README.md
index fcb57cc..40d3f4d 100644
--- a/README.md
+++ b/README.md
@@ -55,8 +55,27 @@ back-end server (if allowed).
 
 Permissions system
 ------------------
+Permissions are stored in internal SQLite3 database and imported
+from data sources, which can be a PostgreSQL database or a file.  See
+[sproxy.sql](./sproxy.sql) and [datafile.yml.example](./datafile.yml.example)
+for details.
+
+Do note that Sproxy2 fetches only `group_member`, `group_privilege`
+and `privilege_rule` tables, because only these tables are used for
+authorization. The other tables in PostgreSQL schema serve for data
+integrity. Data integrity of the data file is not verfied, though import
+may fail due to primary key restrictions.
+
+Only one data source can be used. The data in internal database, if any,
+is fully overwritten by the data from a data source. If no data source is
+specified, the data in internal database remains unchanged, even between
+restarts.  Broken data source is _not_ fatal. Sproxy will keep using existing
+internal database, or create a new empty one if missed. Broken data source
+means inability to connect to PostgreSQL database, missed datafile, etc.
+
+The data from a PostgreSQL database are periodically fetched into the internal
+database, while the data file is read once at startup.
 
-Permissions are stored in a PostgreSQL database. See sproxy.sql for details.
 Here are the main concepts:
 
 - A `group` is identified by a name. Every group has
@@ -72,14 +91,10 @@ Here are the main concepts:
   surprising, please see the following example:
 
 
-Do note that Sproxy2 fetches only `group_member`, `group_privilege` and `privilege_rule`
-tables, because only these tables are used for authorization. The other tables
-serve for data integrity.
-
 Keep in mind that:
 
-- Domains are converted into lower case (coming from PostgreSQL or HTTP requests).
-- Emails are converted into lower case (coming from PostgreSQL or OAuth2 providers).
+- Domains are converted into lower case (coming from a data source or HTTP requests).
+- Emails are converted into lower case (coming from a data source or OAuth2 providers).
 - Groups are case-sensitive and treated as is.
 - HTTP methods are *case-sensitive*.
 - HTTP query parameters are ignored when matching a request against the rules.
diff --git a/datafile.yml.example b/datafile.yml.example
new file mode 100644
index 0000000..85d88b9
--- /dev/null
+++ b/datafile.yml.example
@@ -0,0 +1,34 @@
+--- # Data file. Don't remove this line. This is YAML: https://en.wikipedia.org/wiki/YAML
+
+group_member:
+  - group: "devops"
+    email: "%"
+
+  - group: "foo"
+    email: "%"
+
+
+group_privilege:
+  - group: "foo"
+    domain: "example.com"
+    privilege: "full"
+
+  - group: "devops"
+    domain: "example.com"
+    privilege: "full"
+
+
+privilege_rule:
+  - domain: "example.com"
+    privilege: "full"
+    path: "%"
+    method: "GET"
+
+  - domain: "example.com"
+    privilege: "full"
+    path: "%"
+    method: "POST"
+
+
+... # End of data file. Don't remove this line. This is YAML: https://en.wikipedia.org/wiki/YAML
+
diff --git a/sproxy.yml.example b/sproxy.yml.example
index d539956..9fba77b 100644
--- a/sproxy.yml.example
+++ b/sproxy.yml.example
@@ -1,8 +1,13 @@
 --- # Sproxy configuration. Don't remove this line. This is YAML: https://en.wikipedia.org/wiki/YAML
 
+# Logging level: debug, info, warn, error.
+# Optional. Default is debug.
+#
+# log_level: debug
+
 # The port Sproxy listens on (HTTPS).
 # Optional. Default is 443.
-# 
+#
 # listen: 443
 
 # Listen on port 80 and redirect HTTP requests to HTTPS.
@@ -24,10 +29,32 @@
 #
 # home: "."
 
+
+# File with SSL certificate. Required.
+# It can be a bundle with the server certificate coming first:
+# cat me-cert.pem CA-cert.pem > cert.pem
+# Once again: most wanted certs go first ;-)
+# Or you can opt in using of `ssl_cert_chain`
+ssl_cert: /path/cert.pem
+
+# File with SSL key (secret!). Required.
+ssl_key: /path/key.pem
+
+# Chain SSL certificate files.
+# Optional. Default is an empty list
+# Example:
+# ssl_cert_chain:
+#   - /path/foo.pem
+#   - /path/bar.pem
+#
+# ssl_cert_chain: []
+
+
 # PostgreSQL database connection string.
 # Optional. If specified, sproxy will periodically pull the data from this
 # database into internal SQLite3 database. Define password in a file
 # referenced by the PGPASSFILE environment variable. Or use the "pgpassfile" option.
+# Cannot be used with the "datafile" option.
 # Example:
 # database: "user=sproxy-readonly dbname=sproxy port=6001"
 #
@@ -40,10 +67,16 @@
 #
 # pgpassfile:
 
-# Logging level: debug, info, warn, error.
-# Optional. Default is debug.
-# 
-# log_level: debug
+
+# YAML file used to fill internal SQLite3 database.
+# Optional. If specified, Sproxy will import it on start overwriting
+# and existing data in the internal database.
+# Useful for development or some simple deployments.
+# Cannot be used with the "database" option.
+# For example see the datafile.yml.example
+#
+# datafile: /path/data.yml
+
 
 # A file with arbitrary content used to sign sproxy cookie and other things (secret!).
 # Optional. If not specified, a random key is generated on startup, and
@@ -54,25 +87,6 @@
 # 
 # key: /run/keys/sproxy.secret
 
-# File with SSL certificate. Required.
-# It can be a bundle with the server certificate coming first:
-# cat me-cert.pem CA-cert.pem > cert.pem
-# Once again: most wanted certs go first ;-)
-# Or you can opt in using of `ssl_cert_chain`
-ssl_cert: /path/cert.pem
-
-# File with SSL key (secret!). Required.
-ssl_key: /path/key.pem
-
-# Chain SSL certificate files.
-# Optional. Default is an empty list
-# Example:
-# ssl_cert_chain:
-#   - /path/foo.pem
-#   - /path/bar.pem
-# 
-# ssl_cert_chain: []
-
 
 # Credentials for supported OAuth2 providers.
 # Currently supported: "google", "linkedin"
diff --git a/sproxy2.cabal b/sproxy2.cabal
index da4ed1f..89083ba 100644
--- a/sproxy2.cabal
+++ b/sproxy2.cabal
@@ -13,8 +13,13 @@ maintainer: Igor Pashev <pashev.igor@gmail.com>
 copyright: 2016, Zalora South East Asia Pte. Ltd
 category: Databases, Web
 build-type: Simple
-extra-source-files: README.md ChangeLog.md sproxy.yml.example sproxy.sql
 cabal-version: >= 1.20
+extra-source-files:
+  ChangeLog.md
+  README.md
+  datafile.yml.example
+  sproxy.sql
+  sproxy.yml.example
 
 source-repository head
   type: git
@@ -37,6 +42,7 @@ executable sproxy2
       Sproxy.Logging
       Sproxy.Server
       Sproxy.Server.DB
+      Sproxy.Server.DB.DataFile
     build-depends:
         base >= 4.8 && < 50
       , aeson
diff --git a/src/Sproxy/Config.hs b/src/Sproxy/Config.hs
index 30a8bae..e76b436 100644
--- a/src/Sproxy/Config.hs
+++ b/src/Sproxy/Config.hs
@@ -27,6 +27,7 @@ data ConfigFile = ConfigFile {
 , cfListen80     :: Maybe Bool
 , cfBackends     :: [BackendConf]
 , cfOAuth2       :: HashMap Text OAuth2Conf
+, cfDataFile     :: Maybe FilePath
 , cfDatabase     :: Maybe String
 , cfPgPassFile   :: Maybe FilePath
 , cfHTTP2        :: Bool
@@ -45,6 +46,7 @@ instance FromJSON ConfigFile where
     <*> m .:? "listen80"
     <*> m .:  "backends"
     <*> m .:  "oauth2"
+    <*> m .:? "datafile"
     <*> m .:? "database"
     <*> m .:? "pgpassfile"
     <*> m .:? "http2"          .!= True
diff --git a/src/Sproxy/Server.hs b/src/Sproxy/Server.hs
index 98e9d56..5c80e44 100644
--- a/src/Sproxy/Server.hs
+++ b/src/Sproxy/Server.hs
@@ -67,13 +67,8 @@ server configFile = do
     setGroupID $ userGroupID u
     setUserID $ userID u
 
-  case cfPgPassFile cf of
-    Nothing -> return ()
-    Just f  -> do
-      Log.info $ "pgpassfile: " ++ show f
-      setEnv "PGPASSFILE" f
-
-  db <- DB.start (cfHome cf) (newDataSource cf)
+  ds <- newDataSource cf
+  db <- DB.start (cfHome cf) ds
 
   key <- maybe
            (Log.info "using new random key" >> getEntropy 32)
@@ -112,11 +107,23 @@ server configFile = do
     (sproxy key db oauth2clients backends)
 
 
-newDataSource :: ConfigFile -> Maybe DB.DataSource
+newDataSource :: ConfigFile -> IO (Maybe DB.DataSource)
 newDataSource cf =
-  case cfDatabase cf of
-    Just str -> Just $ DB.PostgreSQL str
-    Nothing -> Nothing
+  case (cfDataFile cf, cfDatabase cf) of
+    (Nothing, Just str) -> do
+      case cfPgPassFile cf of
+        Nothing -> return ()
+        Just f  -> do
+          Log.info $ "pgpassfile: " ++ show f
+          setEnv "PGPASSFILE" f
+      return . Just $ DB.PostgreSQL str
+
+    (Just f, Nothing)   -> return . Just $ DB.File f
+
+    (Nothing, Nothing)  -> return Nothing
+    _ -> do
+      Log.error "only one data source can be used"
+      exitFailure
 
 
 newOAuth2Client :: (Text, OAuth2Conf) -> IO (Text, OAuth2Client)
diff --git a/src/Sproxy/Server/DB.hs b/src/Sproxy/Server/DB.hs
index 90e2abd..2823ba0 100644
--- a/src/Sproxy/Server/DB.hs
+++ b/src/Sproxy/Server/DB.hs
@@ -14,17 +14,20 @@ import Control.Monad (forever, void)
 import Data.ByteString.Char8 (pack)
 import Data.Pool (Pool, createPool, withResource)
 import Data.Text (Text, toLower, unpack)
+import Data.Yaml (decodeFileEither)
 import Database.SQLite.Simple (NamedParam((:=)))
 import Text.InterpolatedString.Perl6 (q, qc)
 import qualified Database.PostgreSQL.Simple as PG
 import qualified Database.SQLite.Simple as SQLite
 
+import Sproxy.Server.DB.DataFile ( DataFile(..), GroupMember(..),
+  GroupPrivilege(..), PrivilegeRule(..) )
 import qualified Sproxy.Logging as Log
 
 
 type Database = Pool SQLite.Connection
 
-data DataSource = PostgreSQL String -- | File FilePath
+data DataSource = PostgreSQL String | File FilePath
 
 {- TODO:
  - Hash remote tables and update the local only when the remote change
@@ -77,6 +80,12 @@ userGroups db email domain path method =
      , ":method" := method -- XXX case-sensitive by RFC2616
      ]
 
+-- FIXME short-cut for https://github.com/nurpax/sqlite-simple/issues/50
+-- FIXME nextRow is the only way to execute a prepared statement
+-- FIXME with bound parameters, but we don't expect any results.
+submit :: SQLite.Statement -> IO ()
+submit st = void (SQLite.nextRow st :: IO (Maybe [Int]))
+
 
 populate :: Database -> Maybe DataSource -> IO ()
 
@@ -87,6 +96,40 @@ populate db Nothing = do
     createGroupPrivilege c
     createPrivilegeRule c
 
+populate db (Just (File f)) = do
+  Log.info $ "db: reading " ++ show f
+  r <- decodeFileEither f
+  case r of
+    Left e   -> Log.error $ f ++ ": " ++ show e
+    Right df ->
+      withResource db $ \c -> SQLite.withTransaction c $ do
+        dropGroupMember c
+        createGroupMember c
+        SQLite.withStatement c
+          [q|INSERT INTO group_member("group", email) VALUES (?, ?)|]
+            $ \st -> mapM_ (\gm -> SQLite.withBind st
+                                  (gmGroup gm, toLower $ gmEmail gm)
+                                  (submit st)
+                          ) (groupMember df)
+
+        dropGroupPrivilege c
+        createGroupPrivilege c
+        SQLite.withStatement c
+          [q|INSERT INTO group_privilege("group", domain, privilege) VALUES (?, ?, ?)|]
+            $ \st -> mapM_ (\gp -> SQLite.withBind st
+                                  (gpGroup gp, toLower $ gpDomain gp, gpPrivilege gp)
+                                  (submit st)
+                          ) (groupPrivilege df)
+
+        dropPrivilegeRule c
+        createPrivilegeRule c
+        SQLite.withStatement c
+          [q|INSERT INTO privilege_rule(domain, privilege, path, method) VALUES (?, ?, ?, ?)|]
+            $ \st -> mapM_ (\pr -> SQLite.withBind st
+                                  (toLower $ prDomain pr, prPrivilege pr, prPath pr, prMethod pr)
+                                  (submit st)
+                          ) (privilegeRule df)
+
 -- XXX We keep only required minimum of the data, without any integrity check.
 -- XXX Integrity check should be done somewhere else, e. g. in the master PostgreSQL database,
 -- XXX or during importing the config file.
@@ -155,7 +198,6 @@ createGroupPrivilege c = SQLite.execute_ c [q|
   )
 |]
 
-
 dropPrivilegeRule :: SQLite.Connection -> IO ()
 dropPrivilegeRule c = SQLite.execute_ c "DROP TABLE IF EXISTS privilege_rule"
 
diff --git a/src/Sproxy/Server/DB/DataFile.hs b/src/Sproxy/Server/DB/DataFile.hs
new file mode 100644
index 0000000..efac923
--- /dev/null
+++ b/src/Sproxy/Server/DB/DataFile.hs
@@ -0,0 +1,69 @@
+{-# LANGUAGE OverloadedStrings #-}
+module Sproxy.Server.DB.DataFile (
+  DataFile(..)
+, GroupMember(..)
+, GroupPrivilege(..)
+, PrivilegeRule(..)
+) where
+
+import Control.Applicative (empty)
+import Data.Aeson (FromJSON, parseJSON)
+import Data.Text (Text)
+import Data.Yaml (Value(Object), (.:))
+
+
+data DataFile = DataFile {
+  groupMember    :: [GroupMember]
+, groupPrivilege :: [GroupPrivilege]
+, privilegeRule  :: [PrivilegeRule]
+} deriving (Show)
+
+instance FromJSON DataFile where
+  parseJSON (Object m) = DataFile <$>
+        m .: "group_member"
+    <*> m .: "group_privilege"
+    <*> m .: "privilege_rule"
+  parseJSON _ = empty
+
+
+data GroupMember = GroupMember {
+  gmGroup :: Text
+, gmEmail :: Text
+} deriving (Show)
+
+instance FromJSON GroupMember where
+  parseJSON (Object m) = GroupMember <$>
+        m .: "group"
+    <*> m .: "email"
+  parseJSON _ = empty
+
+
+data GroupPrivilege = GroupPrivilege {
+  gpGroup     :: Text
+, gpDomain    :: Text
+, gpPrivilege :: Text
+} deriving (Show)
+
+instance FromJSON GroupPrivilege where
+  parseJSON (Object m) = GroupPrivilege <$>
+        m .: "group"
+    <*> m .: "domain"
+    <*> m .: "privilege"
+  parseJSON _ = empty
+
+
+data PrivilegeRule = PrivilegeRule {
+  prDomain    :: Text
+, prPrivilege :: Text
+, prPath      :: Text
+, prMethod    :: Text
+} deriving (Show)
+
+instance FromJSON PrivilegeRule where
+  parseJSON (Object m) = PrivilegeRule <$>
+        m .: "domain"
+    <*> m .: "privilege"
+    <*> m .: "path"
+    <*> m .: "method"
+  parseJSON _ = empty
+