From 471aa27bf0a0e4c698303acb7fdf9cea6a75634b Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 18 Nov 2013 13:06:58 -0400
Subject: copied from git-annex

---
 Utility/Applicative.hs        |  16 ++
 Utility/Batch.hs              |  91 +++++++++++
 Utility/CoProcess.hs          |  93 +++++++++++
 Utility/Data.hs               |  17 ++
 Utility/Directory.hs          | 107 +++++++++++++
 Utility/Env.hs                |  63 ++++++++
 Utility/Exception.hs          |  59 +++++++
 Utility/FileMode.hs           | 135 ++++++++++++++++
 Utility/FileSystemEncoding.hs |  93 +++++++++++
 Utility/Format.hs             | 178 +++++++++++++++++++++
 Utility/Metered.hs            | 116 ++++++++++++++
 Utility/Misc.hs               | 138 ++++++++++++++++
 Utility/Monad.hs              |  69 ++++++++
 Utility/PartialPrelude.hs     |  68 ++++++++
 Utility/Path.hs               | 254 ++++++++++++++++++++++++++++++
 Utility/Process.hs            | 356 ++++++++++++++++++++++++++++++++++++++++++
 Utility/Rsync.hs              | 152 ++++++++++++++++++
 Utility/SafeCommand.hs        | 120 ++++++++++++++
 Utility/Tmp.hs                |  88 +++++++++++
 Utility/UserInfo.hs           |  55 +++++++
 20 files changed, 2268 insertions(+)
 create mode 100644 Utility/Applicative.hs
 create mode 100644 Utility/Batch.hs
 create mode 100644 Utility/CoProcess.hs
 create mode 100644 Utility/Data.hs
 create mode 100644 Utility/Directory.hs
 create mode 100644 Utility/Env.hs
 create mode 100644 Utility/Exception.hs
 create mode 100644 Utility/FileMode.hs
 create mode 100644 Utility/FileSystemEncoding.hs
 create mode 100644 Utility/Format.hs
 create mode 100644 Utility/Metered.hs
 create mode 100644 Utility/Misc.hs
 create mode 100644 Utility/Monad.hs
 create mode 100644 Utility/PartialPrelude.hs
 create mode 100644 Utility/Path.hs
 create mode 100644 Utility/Process.hs
 create mode 100644 Utility/Rsync.hs
 create mode 100644 Utility/SafeCommand.hs
 create mode 100644 Utility/Tmp.hs
 create mode 100644 Utility/UserInfo.hs

(limited to 'Utility')

diff --git a/Utility/Applicative.hs b/Utility/Applicative.hs
new file mode 100644
index 0000000..64400c8
--- /dev/null
+++ b/Utility/Applicative.hs
@@ -0,0 +1,16 @@
+{- applicative stuff
+ -
+ - Copyright 2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Applicative where
+
+{- Like <$> , but supports one level of currying.
+ - 
+ - foo v = bar <$> action v  ==  foo = bar <$$> action
+ -}
+(<$$>) :: Functor f => (a -> b) -> (c -> f a) -> c -> f b
+f <$$> v = fmap f . v
+infixr 4 <$$>
diff --git a/Utility/Batch.hs b/Utility/Batch.hs
new file mode 100644
index 0000000..035a2eb
--- /dev/null
+++ b/Utility/Batch.hs
@@ -0,0 +1,91 @@
+{- Running a long or expensive batch operation niced.
+ -
+ - Copyright 2013 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE CPP #-}
+
+module Utility.Batch where
+
+import Common
+#ifndef mingw32_HOST_OS
+import qualified Build.SysConfig
+#endif
+
+#if defined(linux_HOST_OS) || defined(__ANDROID__)
+import Control.Concurrent.Async
+import System.Posix.Process
+#endif
+import qualified Control.Exception as E
+import System.Process (env)
+
+{- Runs an operation, at batch priority.
+ -
+ - This is done by running it in a bound thread, which on Linux can be set
+ - to have a different nice level than the rest of the program. Note that
+ - due to running in a bound thread, some operations may be more expensive
+ - to perform. Also note that if the action calls forkIO or forkOS itself,
+ - that will make a new thread that does not have the batch priority.
+ -
+ - POSIX threads do not support separate nice levels, so on other operating
+ - systems, the action is simply ran.
+ -}
+batch :: IO a -> IO a
+#if defined(linux_HOST_OS) || defined(__ANDROID__)
+batch a = wait =<< batchthread
+  where
+  	batchthread = asyncBound $ do
+		setProcessPriority 0 maxNice
+		a
+#else
+batch a = a
+#endif
+
+maxNice :: Int
+maxNice = 19
+
+{- Converts a command to run niced. -}
+toBatchCommand :: (String, [CommandParam]) -> (String, [CommandParam])
+toBatchCommand (command, params) = (command', params')
+  where
+#ifndef mingw32_HOST_OS
+	commandline = unwords $ map shellEscape $ command : toCommand params
+	nicedcommand
+		| Build.SysConfig.nice = "nice " ++ commandline
+		| otherwise = commandline
+	command' = "sh"
+	params' =
+		[ Param "-c"
+		, Param $ "exec " ++ nicedcommand
+		]
+#else
+	command' = command
+	params' = params
+#endif
+
+{- Runs a command in a way that's suitable for batch jobs that can be
+ - interrupted.
+ -
+ - The command is run niced. If the calling thread receives an async
+ - exception, it sends the command a SIGTERM, and after the command
+ - finishes shuttting down, it re-raises the async exception. -}
+batchCommand :: String -> [CommandParam] -> IO Bool
+batchCommand command params = batchCommandEnv command params Nothing
+
+batchCommandEnv :: String -> [CommandParam] -> Maybe [(String, String)] -> IO Bool
+batchCommandEnv command params environ = do
+	(_, _, _, pid) <- createProcess $ p { env = environ }
+	r <- E.try (waitForProcess pid) :: IO (Either E.SomeException ExitCode)
+	case r of
+		Right ExitSuccess -> return True
+		Right _ -> return False
+		Left asyncexception -> do
+			terminateProcess pid
+			void $ waitForProcess pid
+			E.throwIO asyncexception
+  where
+  	(command', params') = toBatchCommand (command, params)
+  	p = proc command' $ toCommand params'
+
diff --git a/Utility/CoProcess.hs b/Utility/CoProcess.hs
new file mode 100644
index 0000000..710d2af
--- /dev/null
+++ b/Utility/CoProcess.hs
@@ -0,0 +1,93 @@
+{- Interface for running a shell command as a coprocess,
+ - sending it queries and getting back results.
+ -
+ - Copyright 2012-2013 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE CPP #-}
+
+module Utility.CoProcess (
+	CoProcessHandle,
+	start,
+	stop,
+	query,
+	rawMode
+) where
+
+import Common
+
+import Control.Concurrent.MVar
+
+type CoProcessHandle = MVar CoProcessState
+
+data CoProcessState = CoProcessState
+	{ coProcessPid :: ProcessHandle
+	, coProcessTo :: Handle
+	, coProcessFrom :: Handle
+	, coProcessSpec :: CoProcessSpec
+	}
+
+data CoProcessSpec = CoProcessSpec
+	{ coProcessRestartable :: Bool
+	, coProcessCmd :: FilePath
+	, coProcessParams :: [String]
+	, coProcessEnv :: Maybe [(String, String)]
+	}
+
+start :: Bool -> FilePath -> [String] -> Maybe [(String, String)] -> IO CoProcessHandle
+start restartable cmd params env = do
+	s <- start' $ CoProcessSpec restartable cmd params env
+	newMVar s
+
+start' :: CoProcessSpec -> IO CoProcessState
+start' s = do
+	(pid, from, to) <- startInteractiveProcess (coProcessCmd s) (coProcessParams s) (coProcessEnv s)
+	return $ CoProcessState pid to from s
+
+stop :: CoProcessHandle -> IO ()
+stop ch = do
+	s <- readMVar ch
+	hClose $ coProcessTo s
+	hClose $ coProcessFrom s
+	let p = proc (coProcessCmd $ coProcessSpec s) (coProcessParams $ coProcessSpec s)
+	forceSuccessProcess p (coProcessPid s)
+
+{- To handle a restartable process, any IO exception thrown by the send and
+ - receive actions are assumed to mean communication with the process
+ - failed, and the failed action is re-run with a new process. -}
+query :: CoProcessHandle -> (Handle -> IO a) -> (Handle -> IO b) -> IO b
+query ch send receive = do
+	s <- readMVar ch
+	restartable s (send $ coProcessTo s) $ const $
+		restartable s (hFlush $ coProcessTo s) $ const $
+			restartable s (receive $ coProcessFrom s) $
+				return
+  where
+  	restartable s a cont
+		| coProcessRestartable (coProcessSpec s) =
+			maybe restart cont =<< catchMaybeIO a
+		| otherwise = cont =<< a
+	restart = do
+		s <- takeMVar ch
+		void $ catchMaybeIO $ do
+			hClose $ coProcessTo s
+			hClose $ coProcessFrom s
+		void $ waitForProcess $ coProcessPid s
+		s' <- start' (coProcessSpec s)
+		putMVar ch s'
+		query ch send receive
+
+rawMode :: CoProcessHandle -> IO CoProcessHandle
+rawMode ch = do
+	s <- readMVar ch
+	raw $ coProcessFrom s
+	raw $ coProcessTo s
+	return ch
+  where
+  	raw h = do
+		fileEncoding h
+#ifdef mingw32_HOST_OS
+		hSetNewlineMode h noNewlineTranslation
+#endif
diff --git a/Utility/Data.hs b/Utility/Data.hs
new file mode 100644
index 0000000..3592582
--- /dev/null
+++ b/Utility/Data.hs
@@ -0,0 +1,17 @@
+{- utilities for simple data types
+ -
+ - Copyright 2013 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Data where
+
+{- First item in the list that is not Nothing. -}
+firstJust :: Eq a => [Maybe a] -> Maybe a
+firstJust ms = case dropWhile (== Nothing) ms of
+	[] -> Nothing
+	(md:_) -> md
+
+eitherToMaybe :: Either a b -> Maybe b
+eitherToMaybe = either (const Nothing) Just
diff --git a/Utility/Directory.hs b/Utility/Directory.hs
new file mode 100644
index 0000000..4918d20
--- /dev/null
+++ b/Utility/Directory.hs
@@ -0,0 +1,107 @@
+{- directory manipulation
+ -
+ - Copyright 2011 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE CPP #-}
+
+module Utility.Directory where
+
+import System.IO.Error
+import System.PosixCompat.Files
+import System.Directory
+import Control.Exception (throw)
+import Control.Monad
+import Control.Monad.IfElse
+import System.FilePath
+import Control.Applicative
+import System.IO.Unsafe (unsafeInterleaveIO)
+
+import Utility.SafeCommand
+import Utility.Tmp
+import Utility.Exception
+import Utility.Monad
+
+dirCruft :: FilePath -> Bool
+dirCruft "." = True
+dirCruft ".." = True
+dirCruft _ = False
+
+{- Lists the contents of a directory.
+ - Unlike getDirectoryContents, paths are not relative to the directory. -}
+dirContents :: FilePath -> IO [FilePath]
+dirContents d = map (d </>) . filter (not . dirCruft) <$> getDirectoryContents d
+
+{- Gets files in a directory, and then its subdirectories, recursively,
+ - and lazily. If the directory does not exist, no exception is thrown,
+ - instead, [] is returned. -}
+dirContentsRecursive :: FilePath -> IO [FilePath]
+dirContentsRecursive topdir = dirContentsRecursiveSkipping (const False) topdir
+
+{- Skips directories whose basenames match the skipdir. -}
+dirContentsRecursiveSkipping :: (FilePath -> Bool) -> FilePath -> IO [FilePath]
+dirContentsRecursiveSkipping skipdir topdir = go [topdir]
+  where
+  	go [] = return []
+	go (dir:dirs)
+		| skipdir (takeFileName dir) = go dirs
+		| otherwise = unsafeInterleaveIO $ do
+			(files, dirs') <- collect [] []
+				=<< catchDefaultIO [] (dirContents dir)
+			files' <- go (dirs' ++ dirs)
+			return (files ++ files')
+	collect files dirs' [] = return (reverse files, reverse dirs')
+	collect files dirs' (entry:entries)
+		| dirCruft entry = collect files dirs' entries
+		| otherwise = do
+			ifM (doesDirectoryExist entry)
+				( collect files (entry:dirs') entries
+				, collect (entry:files) dirs' entries
+				)			
+
+{- Moves one filename to another.
+ - First tries a rename, but falls back to moving across devices if needed. -}
+moveFile :: FilePath -> FilePath -> IO ()
+moveFile src dest = tryIO (rename src dest) >>= onrename
+  where
+	onrename (Right _) = noop
+	onrename (Left e)
+		| isPermissionError e = rethrow
+		| isDoesNotExistError e = rethrow
+		| otherwise = do
+			-- copyFile is likely not as optimised as
+			-- the mv command, so we'll use the latter.
+			-- But, mv will move into a directory if
+			-- dest is one, which is not desired.
+			whenM (isdir dest) rethrow
+			viaTmp mv dest undefined
+	  where
+		rethrow = throw e
+		mv tmp _ = do
+			ok <- boolSystem "mv" [Param "-f", Param src, Param tmp]
+			unless ok $ do
+				-- delete any partial
+				_ <- tryIO $ removeFile tmp
+				rethrow
+
+	isdir f = do
+		r <- tryIO $ getFileStatus f
+		case r of
+			(Left _) -> return False
+			(Right s) -> return $ isDirectory s
+
+{- Removes a file, which may or may not exist, and does not have to
+ - be a regular file.
+ -
+ - Note that an exception is thrown if the file exists but
+ - cannot be removed. -}
+nukeFile :: FilePath -> IO ()
+nukeFile file = void $ tryWhenExists go
+  where
+#ifndef mingw32_HOST_OS
+	go = removeLink file
+#else
+	go = removeFile file
+#endif
diff --git a/Utility/Env.hs b/Utility/Env.hs
new file mode 100644
index 0000000..cb73873
--- /dev/null
+++ b/Utility/Env.hs
@@ -0,0 +1,63 @@
+{- portable environment variables
+ -
+ - Copyright 2013 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE CPP #-}
+
+module Utility.Env where
+
+#ifdef mingw32_HOST_OS
+import Utility.Exception
+import Control.Applicative
+import Data.Maybe
+import qualified System.Environment as E
+#else
+import qualified System.Posix.Env as PE
+#endif
+
+getEnv :: String -> IO (Maybe String)
+#ifndef mingw32_HOST_OS
+getEnv = PE.getEnv
+#else
+getEnv = catchMaybeIO . E.getEnv
+#endif
+
+getEnvDefault :: String -> String -> IO String
+#ifndef mingw32_HOST_OS
+getEnvDefault = PE.getEnvDefault
+#else
+getEnvDefault var fallback = fromMaybe fallback <$> getEnv var
+#endif
+
+getEnvironment :: IO [(String, String)]
+#ifndef mingw32_HOST_OS
+getEnvironment = PE.getEnvironment
+#else
+getEnvironment = E.getEnvironment
+#endif
+
+{- Returns True if it could successfully set the environment variable.
+ -
+ - There is, apparently, no way to do this in Windows. Instead,
+ - environment varuables must be provided when running a new process. -}
+setEnv :: String -> String -> Bool -> IO Bool
+#ifndef mingw32_HOST_OS
+setEnv var val overwrite = do
+	PE.setEnv var val overwrite
+	return True
+#else
+setEnv _ _ _ = return False
+#endif
+
+{- Returns True if it could successfully unset the environment variable. -}
+unsetEnv :: String -> IO Bool
+#ifndef mingw32_HOST_OS
+unsetEnv var = do
+	PE.unsetEnv var
+	return True
+#else
+unsetEnv _ = return False
+#endif
diff --git a/Utility/Exception.hs b/Utility/Exception.hs
new file mode 100644
index 0000000..cf2c615
--- /dev/null
+++ b/Utility/Exception.hs
@@ -0,0 +1,59 @@
+{- Simple IO exception handling (and some more)
+ -
+ - Copyright 2011-2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE ScopedTypeVariables #-}
+
+module Utility.Exception where
+
+import Control.Exception
+import qualified Control.Exception as E
+import Control.Applicative
+import Control.Monad
+import System.IO.Error (isDoesNotExistError)
+import Utility.Data
+
+{- Catches IO errors and returns a Bool -}
+catchBoolIO :: IO Bool -> IO Bool
+catchBoolIO a = catchDefaultIO False a
+
+{- Catches IO errors and returns a Maybe -}
+catchMaybeIO :: IO a -> IO (Maybe a)
+catchMaybeIO a = catchDefaultIO Nothing $ Just <$> a
+
+{- Catches IO errors and returns a default value. -}
+catchDefaultIO :: a -> IO a -> IO a
+catchDefaultIO def a = catchIO a (const $ return def)
+
+{- Catches IO errors and returns the error message. -}
+catchMsgIO :: IO a -> IO (Either String a)
+catchMsgIO a = either (Left . show) Right <$> tryIO a
+
+{- catch specialized for IO errors only -}
+catchIO :: IO a -> (IOException -> IO a) -> IO a
+catchIO = E.catch
+
+{- try specialized for IO errors only -}
+tryIO :: IO a -> IO (Either IOException a)
+tryIO = try
+
+{- Catches all exceptions except for async exceptions.
+ - This is often better to use than catching them all, so that
+ - ThreadKilled and UserInterrupt get through.
+ -}
+catchNonAsync :: IO a -> (SomeException -> IO a) -> IO a
+catchNonAsync a onerr = a `catches`
+	[ Handler (\ (e :: AsyncException) -> throw e)
+	, Handler (\ (e :: SomeException) -> onerr e)
+	]
+
+tryNonAsync :: IO a -> IO (Either SomeException a)
+tryNonAsync a = (Right <$> a) `catchNonAsync` (return . Left)
+
+{- Catches only DoesNotExist exceptions, and lets all others through. -}
+tryWhenExists :: IO a -> IO (Maybe a)
+tryWhenExists a = eitherToMaybe <$>
+	tryJust (guard . isDoesNotExistError) a
diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs
new file mode 100644
index 0000000..d76fb57
--- /dev/null
+++ b/Utility/FileMode.hs
@@ -0,0 +1,135 @@
+{- File mode utilities.
+ -
+ - Copyright 2010-2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE CPP #-}
+
+module Utility.FileMode where
+
+import Common
+
+import Control.Exception (bracket)
+import System.PosixCompat.Types
+#ifndef mingw32_HOST_OS
+import System.Posix.Files
+#endif
+import Foreign (complement)
+
+{- Applies a conversion function to a file's mode. -}
+modifyFileMode :: FilePath -> (FileMode -> FileMode) -> IO ()
+modifyFileMode f convert = void $ modifyFileMode' f convert
+modifyFileMode' :: FilePath -> (FileMode -> FileMode) -> IO FileMode
+modifyFileMode' f convert = do
+	s <- getFileStatus f
+	let old = fileMode s
+	let new = convert old
+	when (new /= old) $
+		setFileMode f new
+	return old
+
+{- Adds the specified FileModes to the input mode, leaving the rest
+ - unchanged. -}
+addModes :: [FileMode] -> FileMode -> FileMode
+addModes ms m = combineModes (m:ms)
+
+{- Removes the specified FileModes from the input mode. -}
+removeModes :: [FileMode] -> FileMode -> FileMode
+removeModes ms m = m `intersectFileModes` complement (combineModes ms)
+
+{- Runs an action after changing a file's mode, then restores the old mode. -}
+withModifiedFileMode :: FilePath -> (FileMode -> FileMode) -> IO a -> IO a
+withModifiedFileMode file convert a = bracket setup cleanup go
+  where
+	setup = modifyFileMode' file convert
+	cleanup oldmode = modifyFileMode file (const oldmode)
+	go _ = a
+
+writeModes :: [FileMode]
+writeModes = [ownerWriteMode, groupWriteMode, otherWriteMode]
+
+readModes :: [FileMode]
+readModes = [ownerReadMode, groupReadMode, otherReadMode]
+
+executeModes :: [FileMode]
+executeModes = [ownerExecuteMode, groupExecuteMode, otherExecuteMode]
+
+{- Removes the write bits from a file. -}
+preventWrite :: FilePath -> IO ()
+preventWrite f = modifyFileMode f $ removeModes writeModes
+
+{- Turns a file's owner write bit back on. -}
+allowWrite :: FilePath -> IO ()
+allowWrite f = modifyFileMode f $ addModes [ownerWriteMode]
+
+{- Allows owner and group to read and write to a file. -}
+groupWriteRead :: FilePath -> IO ()
+groupWriteRead f = modifyFileMode f $ addModes
+	[ ownerWriteMode, groupWriteMode
+	, ownerReadMode, groupReadMode
+	]
+
+checkMode :: FileMode -> FileMode -> Bool
+checkMode checkfor mode = checkfor `intersectFileModes` mode == checkfor
+
+{- Checks if a file mode indicates it's a symlink. -}
+isSymLink :: FileMode -> Bool
+#ifdef mingw32_HOST_OS
+isSymLink _ = False
+#else
+isSymLink = checkMode symbolicLinkMode
+#endif
+
+{- Checks if a file has any executable bits set. -}
+isExecutable :: FileMode -> Bool
+isExecutable mode = combineModes executeModes `intersectFileModes` mode /= 0
+
+{- Runs an action without that pesky umask influencing it, unless the
+ - passed FileMode is the standard one. -}
+noUmask :: FileMode -> IO a -> IO a
+#ifndef mingw32_HOST_OS
+noUmask mode a
+	| mode == stdFileMode = a
+	| otherwise = bracket setup cleanup go
+  where
+	setup = setFileCreationMask nullFileMode
+	cleanup = setFileCreationMask
+	go _ = a
+#else
+noUmask _ a = a
+#endif
+
+combineModes :: [FileMode] -> FileMode
+combineModes [] = undefined
+combineModes [m] = m
+combineModes (m:ms) = foldl unionFileModes m ms
+
+isSticky :: FileMode -> Bool
+#ifdef mingw32_HOST_OS
+isSticky _ = False
+#else
+isSticky = checkMode stickyMode
+
+stickyMode :: FileMode
+stickyMode = 512
+
+setSticky :: FilePath -> IO ()
+setSticky f = modifyFileMode f $ addModes [stickyMode]
+#endif
+
+{- Writes a file, ensuring that its modes do not allow it to be read
+ - by anyone other than the current user, before any content is written.
+ -
+ - On a filesystem that does not support file permissions, this is the same
+ - as writeFile.
+ -}
+writeFileProtected :: FilePath -> String -> IO ()
+writeFileProtected file content = do
+	h <- openFile file WriteMode
+	void $ tryIO $
+		modifyFileMode file $
+			removeModes [groupReadMode, otherReadMode]
+	hPutStr h content
+	hClose h
diff --git a/Utility/FileSystemEncoding.hs b/Utility/FileSystemEncoding.hs
new file mode 100644
index 0000000..ac105e7
--- /dev/null
+++ b/Utility/FileSystemEncoding.hs
@@ -0,0 +1,93 @@
+{- GHC File system encoding handling.
+ -
+ - Copyright 2012-2013 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.FileSystemEncoding (
+	fileEncoding,
+	withFilePath,
+	md5FilePath,
+	decodeW8,
+	encodeW8,
+	truncateFilePath,
+) where
+
+import qualified GHC.Foreign as GHC
+import qualified GHC.IO.Encoding as Encoding
+import Foreign.C
+import System.IO
+import System.IO.Unsafe
+import qualified Data.Hash.MD5 as MD5
+import Data.Word
+import Data.Bits.Utils
+
+{- Sets a Handle to use the filesystem encoding. This causes data
+ - written or read from it to be encoded/decoded the same
+ - as ghc 7.4 does to filenames etc. This special encoding
+ - allows "arbitrary undecodable bytes to be round-tripped through it". -}
+fileEncoding :: Handle -> IO ()
+fileEncoding h = hSetEncoding h =<< Encoding.getFileSystemEncoding
+
+{- Marshal a Haskell FilePath into a NUL terminated C string using temporary
+ - storage. The FilePath is encoded using the filesystem encoding,
+ - reversing the decoding that should have been done when the FilePath
+ - was obtained. -}
+withFilePath :: FilePath -> (CString -> IO a) -> IO a
+withFilePath fp f = Encoding.getFileSystemEncoding
+	>>= \enc -> GHC.withCString enc fp f
+
+{- Encodes a FilePath into a String, applying the filesystem encoding.
+ -
+ - There are very few things it makes sense to do with such an encoded
+ - string. It's not a legal filename; it should not be displayed.
+ - So this function is not exported, but instead used by the few functions
+ - that can usefully consume it.
+ -
+ - This use of unsafePerformIO is belived to be safe; GHC's interface
+ - only allows doing this conversion with CStrings, and the CString buffer
+ - is allocated, used, and deallocated within the call, with no side
+ - effects.
+ -}
+{-# NOINLINE _encodeFilePath #-}
+_encodeFilePath :: FilePath -> String
+_encodeFilePath fp = unsafePerformIO $ do
+	enc <- Encoding.getFileSystemEncoding
+	GHC.withCString enc fp $ GHC.peekCString Encoding.char8
+
+{- Encodes a FilePath into a Md5.Str, applying the filesystem encoding. -}
+md5FilePath :: FilePath -> MD5.Str
+md5FilePath = MD5.Str . _encodeFilePath
+
+{- Converts a [Word8] to a FilePath, encoding using the filesystem encoding.
+ -
+ - w82c produces a String, which may contain Chars that are invalid
+ - unicode. From there, this is really a simple matter of applying the
+ - file system encoding, only complicated by GHC's interface to doing so.
+ -}
+{-# NOINLINE encodeW8 #-}
+encodeW8 :: [Word8] -> FilePath
+encodeW8 w8 = unsafePerformIO $ do
+	enc <- Encoding.getFileSystemEncoding
+	GHC.withCString Encoding.char8 (w82s w8) $ GHC.peekCString enc
+
+{- Useful when you want the actual number of bytes that will be used to
+ - represent the FilePath on disk. -}
+decodeW8 :: FilePath -> [Word8]
+decodeW8 = s2w8 . _encodeFilePath
+
+{- Truncates a FilePath to the given number of bytes (or less),
+ - as represented on disk.
+ -
+ - Avoids returning an invalid part of a unicode byte sequence, at the
+ - cost of efficiency when running on a large FilePath.
+ -}
+truncateFilePath :: Int -> FilePath -> FilePath
+truncateFilePath n = go . reverse
+  where
+  	go f =
+		let bytes = decodeW8 f
+		in if length bytes <= n
+			then reverse f
+			else go (drop 1 f)
diff --git a/Utility/Format.hs b/Utility/Format.hs
new file mode 100644
index 0000000..e7a2751
--- /dev/null
+++ b/Utility/Format.hs
@@ -0,0 +1,178 @@
+{- Formatted string handling.
+ -
+ - Copyright 2010, 2011 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Format (
+	Format,
+	gen,
+	format,
+	decode_c,
+	encode_c,
+	prop_idempotent_deencode
+) where
+
+import Text.Printf (printf)
+import Data.Char (isAlphaNum, isOctDigit, isHexDigit, isSpace, chr, ord)
+import Data.Maybe (fromMaybe)
+import Data.Word (Word8)
+import Data.List (isPrefixOf)
+import qualified Codec.Binary.UTF8.String
+import qualified Data.Map as M
+
+import Utility.PartialPrelude
+
+type FormatString = String
+
+{- A format consists of a list of fragments. -}
+type Format = [Frag]
+
+{- A fragment is either a constant string,
+ - or a variable, with a justification. -}
+data Frag = Const String | Var String Justify
+	deriving (Show)
+
+data Justify = LeftJustified Int | RightJustified Int | UnJustified
+	deriving (Show)
+
+type Variables = M.Map String String
+
+{- Expands a Format using some variables, generating a formatted string.
+ - This can be repeatedly called, efficiently. -}
+format :: Format -> Variables -> String
+format f vars = concatMap expand f
+  where
+	expand (Const s) = s
+	expand (Var name j)
+		| "escaped_" `isPrefixOf` name =
+			justify j $ encode_c_strict $
+				getvar $ drop (length "escaped_") name
+		| otherwise = justify j $ getvar name
+	getvar name = fromMaybe "" $ M.lookup name vars
+	justify UnJustified s        = s
+	justify (LeftJustified i) s  = s ++ pad i s
+	justify (RightJustified i) s = pad i s ++ s
+	pad i s = take (i - length s) spaces
+	spaces = repeat ' '
+
+{- Generates a Format that can be used to expand variables in a
+ - format string, such as "${foo} ${bar;10} ${baz;-10}\n"
+ -
+ - (This is the same type of format string used by dpkg-query.)
+ -}
+gen :: FormatString -> Format
+gen = filter (not . empty) . fuse [] . scan [] . decode_c
+  where
+	-- The Format is built up in reverse, for efficiency,
+	-- and can have many adjacent Consts. Fusing it fixes both
+	-- problems.
+	fuse f [] = f
+	fuse f (Const c1:Const c2:vs) = fuse f $ Const (c2++c1) : vs
+	fuse f (v:vs) = fuse (v:f) vs
+
+	scan f (a:b:cs)
+		| a == '$' && b == '{' = invar f [] cs
+		| otherwise = scan (Const [a] : f ) (b:cs)
+	scan f v = Const v : f
+
+	invar f var [] = Const (novar var) : f
+	invar f var (c:cs)
+		| c == '}' = foundvar f var UnJustified cs
+		| isAlphaNum c || c == '_' = invar f (c:var) cs
+		| c == ';' = inpad "" f var cs
+		| otherwise = scan ((Const $ novar $ c:var):f) cs
+
+	inpad p f var (c:cs)
+		| c == '}' = foundvar f var (readjustify $ reverse p) cs
+		| otherwise = inpad (c:p) f var cs
+	inpad p f var [] = Const (novar $ p++";"++var) : f
+	readjustify = getjustify . fromMaybe 0 . readish
+	getjustify i
+		| i == 0 = UnJustified
+		| i < 0 = LeftJustified (-1 * i)
+		| otherwise = RightJustified i
+	novar v = "${" ++ reverse v
+	foundvar f v p = scan (Var (reverse v) p : f)
+
+empty :: Frag -> Bool
+empty (Const "") = True
+empty _ = False
+
+{- Decodes a C-style encoding, where \n is a newline, \NNN is an octal
+ - encoded character, and \xNN is a hex encoded character.
+ -}
+decode_c :: FormatString -> FormatString
+decode_c [] = []
+decode_c s = unescape ("", s)
+  where
+	e = '\\'
+	unescape (b, []) = b
+	-- look for escapes starting with '\'
+	unescape (b, v) = b ++ fst pair ++ unescape (handle $ snd pair)
+	  where
+		pair = span (/= e) v
+	isescape x = x == e
+	handle (x:'x':n1:n2:rest)
+		| isescape x && allhex = (fromhex, rest)
+	  where
+	  	allhex = isHexDigit n1 && isHexDigit n2
+		fromhex = [chr $ readhex [n1, n2]]
+		readhex h = Prelude.read $ "0x" ++ h :: Int
+	handle (x:n1:n2:n3:rest)
+		| isescape x && alloctal = (fromoctal, rest)
+	  where
+		alloctal = isOctDigit n1 && isOctDigit n2 && isOctDigit n3
+		fromoctal = [chr $ readoctal [n1, n2, n3]]
+		readoctal o = Prelude.read $ "0o" ++ o :: Int
+	-- \C is used for a few special characters
+	handle (x:nc:rest)
+		| isescape x = ([echar nc], rest)
+	  where
+		echar 'a' = '\a'
+		echar 'b' = '\b'
+		echar 'f' = '\f'
+		echar 'n' = '\n'
+		echar 'r' = '\r'
+		echar 't' = '\t'
+		echar 'v' = '\v'
+		echar a = a
+	handle n = ("", n)
+
+{- Inverse of decode_c. -}
+encode_c :: FormatString -> FormatString
+encode_c = encode_c' (const False)
+
+{- Encodes more strictly, including whitespace. -}
+encode_c_strict :: FormatString -> FormatString
+encode_c_strict = encode_c' isSpace
+
+encode_c' :: (Char -> Bool) -> FormatString -> FormatString
+encode_c' p = concatMap echar
+  where
+	e c = '\\' : [c]
+	echar '\a' = e 'a'
+	echar '\b' = e 'b'
+	echar '\f' = e 'f'
+	echar '\n' = e 'n'
+	echar '\r' = e 'r'
+	echar '\t' = e 't'
+	echar '\v' = e 'v'
+	echar '\\' = e '\\'
+	echar '"'  = e '"'
+	echar c
+		| ord c < 0x20 = e_asc c -- low ascii
+		| ord c >= 256 = e_utf c -- unicode
+		| ord c > 0x7E = e_asc c -- high ascii
+		| p c          = e_asc c -- unprintable ascii
+		| otherwise    = [c]     -- printable ascii
+	-- unicode character is decomposed to individual Word8s,
+	-- and each is shown in octal
+	e_utf c = showoctal =<< (Codec.Binary.UTF8.String.encode [c] :: [Word8])
+	e_asc c = showoctal $ ord c
+	showoctal i = '\\' : printf "%03o" i
+
+{- for quickcheck -}
+prop_idempotent_deencode :: String -> Bool
+prop_idempotent_deencode s = s == decode_c (encode_c s)
diff --git a/Utility/Metered.hs b/Utility/Metered.hs
new file mode 100644
index 0000000..f33ad44
--- /dev/null
+++ b/Utility/Metered.hs
@@ -0,0 +1,116 @@
+{- Metered IO
+ -
+ - Copyright 2012, 2013 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE TypeSynonymInstances #-}
+
+module Utility.Metered where
+
+import Common
+
+import qualified Data.ByteString.Lazy as L
+import qualified Data.ByteString as S
+import System.IO.Unsafe
+import Foreign.Storable (Storable(sizeOf))
+import System.Posix.Types
+
+{- An action that can be run repeatedly, updating it on the bytes processed.
+ -
+ - Note that each call receives the total number of bytes processed, so
+ - far, *not* an incremental amount since the last call. -}
+type MeterUpdate = (BytesProcessed -> IO ())
+
+{- Total number of bytes processed so far. -}
+newtype BytesProcessed = BytesProcessed Integer
+	deriving (Eq, Ord)
+
+class AsBytesProcessed a where
+	toBytesProcessed :: a -> BytesProcessed
+	fromBytesProcessed :: BytesProcessed -> a
+
+instance AsBytesProcessed Integer where
+	toBytesProcessed i = BytesProcessed i
+	fromBytesProcessed (BytesProcessed i) = i
+
+instance AsBytesProcessed Int where
+	toBytesProcessed i = BytesProcessed $ toInteger i
+	fromBytesProcessed (BytesProcessed i) = fromInteger i
+
+instance AsBytesProcessed FileOffset where
+	toBytesProcessed sz = BytesProcessed $ toInteger sz
+	fromBytesProcessed (BytesProcessed sz) = fromInteger sz
+
+addBytesProcessed :: AsBytesProcessed v => BytesProcessed -> v -> BytesProcessed
+addBytesProcessed (BytesProcessed i) v = 
+	let (BytesProcessed n) = toBytesProcessed v
+	in BytesProcessed $! i + n
+
+zeroBytesProcessed :: BytesProcessed
+zeroBytesProcessed = BytesProcessed 0
+
+{- Sends the content of a file to an action, updating the meter as it's
+ - consumed. -}
+withMeteredFile :: FilePath -> MeterUpdate -> (L.ByteString -> IO a) -> IO a
+withMeteredFile f meterupdate a = withBinaryFile f ReadMode $ \h ->
+	hGetContentsMetered h meterupdate >>= a
+
+{- Sends the content of a file to a Handle, updating the meter as it's
+ - written. -}
+streamMeteredFile :: FilePath -> MeterUpdate -> Handle -> IO ()
+streamMeteredFile f meterupdate h = withMeteredFile f meterupdate $ L.hPut h
+
+{- Writes a ByteString to a Handle, updating a meter as it's written. -}
+meteredWrite :: MeterUpdate -> Handle -> L.ByteString -> IO ()
+meteredWrite meterupdate h = go zeroBytesProcessed . L.toChunks 
+  where
+	go _ [] = return ()
+	go sofar (c:cs) = do
+		S.hPut h c
+		let sofar' = addBytesProcessed sofar $ S.length c
+		meterupdate sofar'
+		go sofar' cs
+
+meteredWriteFile :: MeterUpdate -> FilePath -> L.ByteString -> IO ()
+meteredWriteFile meterupdate f b = withBinaryFile f WriteMode $ \h ->
+	meteredWrite meterupdate h b
+
+{- This is like L.hGetContents, but after each chunk is read, a meter
+ - is updated based on the size of the chunk.
+ -
+ - Note that the meter update is run in unsafeInterleaveIO, which means that
+ - it can be run at any time. It's even possible for updates to run out
+ - of order, as different parts of the ByteString are consumed.
+ -
+ - All the usual caveats about using unsafeInterleaveIO apply to the
+ - meter updates, so use caution.
+ -}
+hGetContentsMetered :: Handle -> MeterUpdate -> IO L.ByteString
+hGetContentsMetered h meterupdate = lazyRead zeroBytesProcessed
+  where
+	lazyRead sofar = unsafeInterleaveIO $ loop sofar
+
+	loop sofar = do
+		c <- S.hGetSome h defaultChunkSize
+		if S.null c
+			then do
+				hClose h
+				return $ L.empty
+			else do
+				let sofar' = addBytesProcessed sofar $
+					S.length c
+				meterupdate sofar'
+				{- unsafeInterleaveIO causes this to be
+				 - deferred until the data is read from the
+				 - ByteString. -}
+				cs <- lazyRead sofar'
+				return $ L.append (L.fromChunks [c]) cs
+
+{- Same default chunk size Lazy ByteStrings use. -}
+defaultChunkSize :: Int
+defaultChunkSize = 32 * k - chunkOverhead
+  where
+	k = 1024
+	chunkOverhead = 2 * sizeOf (undefined :: Int) -- GHC specific
diff --git a/Utility/Misc.hs b/Utility/Misc.hs
new file mode 100644
index 0000000..a2c9c81
--- /dev/null
+++ b/Utility/Misc.hs
@@ -0,0 +1,138 @@
+{- misc utility functions
+ -
+ - Copyright 2010-2011 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE CPP #-}
+
+module Utility.Misc where
+
+import System.IO
+import Control.Monad
+import Foreign
+import Data.Char
+import Data.List
+import Control.Applicative
+#ifndef mingw32_HOST_OS
+import System.Posix.Process (getAnyProcessStatus)
+import Utility.Exception
+#endif
+
+{- A version of hgetContents that is not lazy. Ensures file is 
+ - all read before it gets closed. -}
+hGetContentsStrict :: Handle -> IO String
+hGetContentsStrict = hGetContents >=> \s -> length s `seq` return s
+
+{- A version of readFile that is not lazy. -}
+readFileStrict :: FilePath -> IO String
+readFileStrict = readFile >=> \s -> length s `seq` return s
+
+{- Like break, but the item matching the condition is not included
+ - in the second result list.
+ -
+ - separate (== ':') "foo:bar" = ("foo", "bar")
+ - separate (== ':') "foobar" = ("foobar", "")
+ -}
+separate :: (a -> Bool) -> [a] -> ([a], [a])
+separate c l = unbreak $ break c l
+  where
+	unbreak r@(a, b)
+		| null b = r
+		| otherwise = (a, tail b)
+
+{- Breaks out the first line. -}
+firstLine :: String -> String
+firstLine = takeWhile (/= '\n')
+
+{- Splits a list into segments that are delimited by items matching
+ - a predicate. (The delimiters are not included in the segments.)
+ - Segments may be empty. -}
+segment :: (a -> Bool) -> [a] -> [[a]]
+segment p l = map reverse $ go [] [] l
+  where
+	go c r [] = reverse $ c:r
+	go c r (i:is)
+		| p i = go [] (c:r) is
+		| otherwise = go (i:c) r is
+
+prop_segment_regressionTest :: Bool
+prop_segment_regressionTest = all id
+	-- Even an empty list is a segment.
+	[ segment (== "--") [] == [[]]
+	-- There are two segements in this list, even though the first is empty.
+	, segment (== "--") ["--", "foo", "bar"] == [[],["foo","bar"]]
+	]
+
+{- Includes the delimiters as segments of their own. -}
+segmentDelim :: (a -> Bool) -> [a] -> [[a]]
+segmentDelim p l = map reverse $ go [] [] l
+  where
+	go c r [] = reverse $ c:r
+	go c r (i:is)
+		| p i = go [] ([i]:c:r) is
+		| otherwise = go (i:c) r is
+
+{- Replaces multiple values in a string.
+ -
+ - Takes care to skip over just-replaced values, so that they are not
+ - mangled. For example, massReplace [("foo", "new foo")] does not
+ - replace the "new foo" with "new new foo".
+ -}
+massReplace :: [(String, String)] -> String -> String
+massReplace vs = go [] vs
+  where
+
+	go acc _ [] = concat $ reverse acc
+	go acc [] (c:cs) = go ([c]:acc) vs cs
+	go acc ((val, replacement):rest) s
+		| val `isPrefixOf` s =
+			go (replacement:acc) vs (drop (length val) s)
+		| otherwise = go acc rest s
+
+{- Given two orderings, returns the second if the first is EQ and returns
+ - the first otherwise.
+ -
+ - Example use:
+ -
+ - compare lname1 lname2 `thenOrd` compare fname1 fname2
+ -}
+thenOrd :: Ordering -> Ordering -> Ordering
+thenOrd EQ x = x
+thenOrd x _ = x
+{-# INLINE thenOrd #-}
+
+{- Wrapper around hGetBufSome that returns a String.
+ -
+ - The null string is returned on eof, otherwise returns whatever
+ - data is currently available to read from the handle, or waits for
+ - data to be written to it if none is currently available.
+ - 
+ - Note on encodings: The normal encoding of the Handle is ignored;
+ - each byte is converted to a Char. Not unicode clean!
+ -}
+hGetSomeString :: Handle -> Int -> IO String
+hGetSomeString h sz = do
+	fp <- mallocForeignPtrBytes sz
+	len <- withForeignPtr fp $ \buf -> hGetBufSome h buf sz
+	map (chr . fromIntegral) <$> withForeignPtr fp (peekbytes len)
+  where
+	peekbytes :: Int -> Ptr Word8 -> IO [Word8]
+	peekbytes len buf = mapM (peekElemOff buf) [0..pred len]
+
+{- Reaps any zombie git processes. 
+ -
+ - Warning: Not thread safe. Anything that was expecting to wait
+ - on a process and get back an exit status is going to be confused
+ - if this reap gets there first. -}
+reapZombies :: IO ()
+#ifndef mingw32_HOST_OS
+reapZombies = do
+	-- throws an exception when there are no child processes
+	catchDefaultIO Nothing (getAnyProcessStatus False True)
+		>>= maybe (return ()) (const reapZombies)
+
+#else
+reapZombies = return ()
+#endif
diff --git a/Utility/Monad.hs b/Utility/Monad.hs
new file mode 100644
index 0000000..1ba43c5
--- /dev/null
+++ b/Utility/Monad.hs
@@ -0,0 +1,69 @@
+{- monadic stuff
+ -
+ - Copyright 2010-2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Monad where
+
+import Data.Maybe
+import Control.Monad
+
+{- Return the first value from a list, if any, satisfying the given
+ - predicate -}
+firstM :: Monad m => (a -> m Bool) -> [a] -> m (Maybe a)
+firstM _ [] = return Nothing
+firstM p (x:xs) = ifM (p x) (return $ Just x , firstM p xs)
+
+{- Runs the action on values from the list until it succeeds, returning
+ - its result. -}
+getM :: Monad m => (a -> m (Maybe b)) -> [a] -> m (Maybe b)
+getM _ [] = return Nothing
+getM p (x:xs) = maybe (getM p xs) (return . Just) =<< p x
+
+{- Returns true if any value in the list satisfies the predicate,
+ - stopping once one is found. -}
+anyM :: Monad m => (a -> m Bool) -> [a] -> m Bool
+anyM p = liftM isJust . firstM p
+
+allM :: Monad m => (a -> m Bool) -> [a] -> m Bool
+allM _ [] = return True
+allM p (x:xs) = p x <&&> allM p xs
+
+{- Runs an action on values from a list until it succeeds. -}
+untilTrue :: Monad m => [a] -> (a -> m Bool) -> m Bool
+untilTrue = flip anyM
+
+{- if with a monadic conditional. -}
+ifM :: Monad m => m Bool -> (m a, m a) -> m a
+ifM cond (thenclause, elseclause) = do
+	c <- cond
+	if c then thenclause else elseclause
+
+{- short-circuiting monadic || -}
+(<||>) :: Monad m => m Bool -> m Bool -> m Bool
+ma <||> mb = ifM ma ( return True , mb )
+
+{- short-circuiting monadic && -}
+(<&&>) :: Monad m => m Bool -> m Bool -> m Bool
+ma <&&> mb = ifM ma ( mb , return False )
+
+{- Same fixity as && and || -}
+infixr 3 <&&>
+infixr 2 <||>
+
+{- Runs an action, passing its value to an observer before returning it. -}
+observe :: Monad m => (a -> m b) -> m a -> m a
+observe observer a = do
+	r <- a
+	_ <- observer r
+	return r
+
+{- b `after` a runs first a, then b, and returns the value of a -}
+after :: Monad m => m b -> m a -> m a
+after = observe . const
+
+{- do nothing -}
+noop :: Monad m => m ()
+noop = return ()
diff --git a/Utility/PartialPrelude.hs b/Utility/PartialPrelude.hs
new file mode 100644
index 0000000..6efa093
--- /dev/null
+++ b/Utility/PartialPrelude.hs
@@ -0,0 +1,68 @@
+{- Parts of the Prelude are partial functions, which are a common source of
+ - bugs.
+ -
+ - This exports functions that conflict with the prelude, which avoids
+ - them being accidentially used.
+ -}
+
+module Utility.PartialPrelude where
+
+import qualified Data.Maybe
+
+{- read should be avoided, as it throws an error
+ - Instead, use: readish -}
+read :: Read a => String -> a
+read = Prelude.read
+
+{- head is a partial function; head [] is an error
+ - Instead, use: take 1 or headMaybe -}
+head :: [a] -> a
+head = Prelude.head
+
+{- tail is also partial
+ - Instead, use: drop 1 -}
+tail :: [a] -> [a]
+tail = Prelude.tail
+
+{- init too
+ - Instead, use: beginning -}
+init :: [a] -> [a]
+init = Prelude.init
+
+{- last too
+ - Instead, use: end or lastMaybe -}
+last :: [a] -> a
+last = Prelude.last
+
+{- Attempts to read a value from a String.
+ -
+ - Ignores leading/trailing whitespace, and throws away any trailing
+ - text after the part that can be read.
+ -
+ - readMaybe is available in Text.Read in new versions of GHC,
+ - but that one requires the entire string to be consumed.
+ -}
+readish :: Read a => String -> Maybe a
+readish s = case reads s of
+	((x,_):_) -> Just x
+	_ -> Nothing
+
+{- Like head but Nothing on empty list. -}
+headMaybe :: [a] -> Maybe a
+headMaybe = Data.Maybe.listToMaybe
+
+{- Like last but Nothing on empty list. -}
+lastMaybe :: [a] -> Maybe a
+lastMaybe [] = Nothing
+lastMaybe v = Just $ Prelude.last v
+
+{- All but the last element of a list.
+ - (Like init, but no error on an empty list.) -}
+beginning :: [a] -> [a]
+beginning [] = []
+beginning l = Prelude.init l
+
+{- Like last, but no error on an empty list. -}
+end :: [a] -> [a]
+end [] = []
+end l = [Prelude.last l]
diff --git a/Utility/Path.hs b/Utility/Path.hs
new file mode 100644
index 0000000..b6214b2
--- /dev/null
+++ b/Utility/Path.hs
@@ -0,0 +1,254 @@
+{- path manipulation
+ -
+ - Copyright 2010-2013 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE PackageImports, CPP #-}
+
+module Utility.Path where
+
+import Data.String.Utils
+import System.FilePath
+import System.Directory
+import Data.List
+import Data.Maybe
+import Data.Char
+import Control.Applicative
+
+#ifdef mingw32_HOST_OS
+import Data.Char
+import qualified System.FilePath.Posix as Posix
+#else
+import qualified "MissingH" System.Path as MissingH
+import System.Posix.Files
+#endif
+
+import Utility.Monad
+import Utility.UserInfo
+
+{- Makes a path absolute if it's not already.
+ - The first parameter is a base directory (ie, the cwd) to use if the path
+ - is not already absolute.
+ -
+ - On Unix, collapses and normalizes ".." etc in the path. May return Nothing
+ - if the path cannot be normalized.
+ -
+ - MissingH's absNormPath does not work on Windows, so on Windows
+ - no normalization is done.
+ -}
+absNormPath :: FilePath -> FilePath -> Maybe FilePath
+#ifndef mingw32_HOST_OS
+absNormPath dir path = MissingH.absNormPath dir path
+#else
+absNormPath dir path = Just $ combine dir path
+#endif
+
+{- Returns the parent directory of a path.
+ -
+ - To allow this to be easily used in loops, which terminate upon reaching the
+ - top, the parent of / is "" -}
+parentDir :: FilePath -> FilePath
+parentDir dir
+	| null dirs = ""
+	| otherwise = joinDrive drive (join s $ init dirs)
+  where
+	-- on Unix, the drive will be "/" when the dir is absolute, otherwise ""
+	(drive, path) = splitDrive dir
+	dirs = filter (not . null) $ split s path
+	s = [pathSeparator]
+
+prop_parentDir_basics :: FilePath -> Bool
+prop_parentDir_basics dir
+	| null dir = True
+	| dir == "/" = parentDir dir == ""
+	| otherwise = p /= dir
+  where
+	p = parentDir dir
+
+{- Checks if the first FilePath is, or could be said to contain the second.
+ - For example, "foo/" contains "foo/bar". Also, "foo", "./foo", "foo/" etc
+ - are all equivilant.
+ -}
+dirContains :: FilePath -> FilePath -> Bool
+dirContains a b = a == b || a' == b' || (a'++[pathSeparator]) `isPrefixOf` b'
+  where
+	norm p = fromMaybe "" $ absNormPath p "."
+	a' = norm a
+	b' = norm b
+
+{- Converts a filename into a normalized, absolute path.
+ -
+ - Unlike Directory.canonicalizePath, this does not require the path
+ - already exists. -}
+absPath :: FilePath -> IO FilePath
+absPath file = do
+	cwd <- getCurrentDirectory
+	return $ absPathFrom cwd file
+
+{- Converts a filename into a normalized, absolute path
+ - from the specified cwd. -}
+absPathFrom :: FilePath -> FilePath -> FilePath
+absPathFrom cwd file = fromMaybe bad $ absNormPath cwd file
+  where
+	bad = error $ "unable to normalize " ++ file
+
+{- Constructs a relative path from the CWD to a file.
+ -
+ - For example, assuming CWD is /tmp/foo/bar:
+ -    relPathCwdToFile "/tmp/foo" == ".."
+ -    relPathCwdToFile "/tmp/foo/bar" == "" 
+ -}
+relPathCwdToFile :: FilePath -> IO FilePath
+relPathCwdToFile f = relPathDirToFile <$> getCurrentDirectory <*> absPath f
+
+{- Constructs a relative path from a directory to a file.
+ -
+ - Both must be absolute, and normalized (eg with absNormpath).
+ -}
+relPathDirToFile :: FilePath -> FilePath -> FilePath
+relPathDirToFile from to = join s $ dotdots ++ uncommon
+  where
+	s = [pathSeparator]
+	pfrom = split s from
+	pto = split s to
+	common = map fst $ takeWhile same $ zip pfrom pto
+	same (c,d) = c == d
+	uncommon = drop numcommon pto
+	dotdots = replicate (length pfrom - numcommon) ".."
+	numcommon = length common
+
+prop_relPathDirToFile_basics :: FilePath -> FilePath -> Bool
+prop_relPathDirToFile_basics from to
+	| from == to = null r
+	| otherwise = not (null r)
+  where
+	r = relPathDirToFile from to 
+
+prop_relPathDirToFile_regressionTest :: Bool
+prop_relPathDirToFile_regressionTest = same_dir_shortcurcuits_at_difference
+  where
+	{- Two paths have the same directory component at the same
+	 - location, but it's not really the same directory.
+	 - Code used to get this wrong. -}
+	same_dir_shortcurcuits_at_difference =
+		relPathDirToFile (joinPath [pathSeparator : "tmp", "r", "lll", "xxx", "yyy", "18"])
+			(joinPath [pathSeparator : "tmp", "r", ".git", "annex", "objects", "18", "gk", "SHA256-foo", "SHA256-foo"])
+				== joinPath ["..", "..", "..", "..", ".git", "annex", "objects", "18", "gk", "SHA256-foo", "SHA256-foo"]
+
+{- Given an original list of paths, and an expanded list derived from it,
+ - generates a list of lists, where each sublist corresponds to one of the
+ - original paths. When the original path is a directory, any items
+ - in the expanded list that are contained in that directory will appear in
+ - its segment.
+ -}
+segmentPaths :: [FilePath] -> [FilePath] -> [[FilePath]]
+segmentPaths [] new = [new]
+segmentPaths [_] new = [new] -- optimisation
+segmentPaths (l:ls) new = [found] ++ segmentPaths ls rest
+  where
+	(found, rest)=partition (l `dirContains`) new
+
+{- This assumes that it's cheaper to call segmentPaths on the result,
+ - than it would be to run the action separately with each path. In
+ - the case of git file list commands, that assumption tends to hold.
+ -}
+runSegmentPaths :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [[FilePath]]
+runSegmentPaths a paths = segmentPaths paths <$> a paths
+
+{- Converts paths in the home directory to use ~/ -}
+relHome :: FilePath -> IO String
+relHome path = do
+	home <- myHomeDir
+	return $ if dirContains home path
+		then "~/" ++ relPathDirToFile home path
+		else path
+
+{- Checks if a command is available in PATH.
+ -
+ - The command may be fully-qualified, in which case, this succeeds as
+ - long as it exists. -}
+inPath :: String -> IO Bool
+inPath command = isJust <$> searchPath command
+
+{- Finds a command in PATH and returns the full path to it.
+ -
+ - The command may be fully qualified already, in which case it will
+ - be returned if it exists.
+ -}
+searchPath :: String -> IO (Maybe FilePath)
+searchPath command
+	| isAbsolute command = check command
+	| otherwise = getSearchPath >>= getM indir
+  where
+	indir d = check $ d </> command
+	check f = firstM doesFileExist
+#ifdef mingw32_HOST_OS
+		[f, f ++ ".exe"]
+#else
+		[f]
+#endif
+
+{- Checks if a filename is a unix dotfile. All files inside dotdirs
+ - count as dotfiles. -}
+dotfile :: FilePath -> Bool
+dotfile file
+	| f == "." = False
+	| f == ".." = False
+	| f == "" = False
+	| otherwise = "." `isPrefixOf` f || dotfile (takeDirectory file)
+  where
+	f = takeFileName file
+
+{- Converts a DOS style path to a Cygwin style path. Only on Windows.
+ - Any trailing '\' is preserved as a trailing '/' -}
+toCygPath :: FilePath -> FilePath
+#ifndef mingw32_HOST_OS
+toCygPath = id
+#else
+toCygPath p
+	| null drive = recombine parts
+	| otherwise = recombine $ "/cygdrive" : driveletter drive : parts
+  where
+  	(drive, p') = splitDrive p
+	parts = splitDirectories p'
+  	driveletter = map toLower . takeWhile (/= ':')
+	recombine = fixtrailing . Posix.joinPath
+  	fixtrailing s
+		| hasTrailingPathSeparator p = Posix.addTrailingPathSeparator s
+		| otherwise = s
+#endif
+
+{- Maximum size to use for a file in a specified directory.
+ -
+ - Many systems have a 255 byte limit to the name of a file, 
+ - so that's taken as the max if the system has a larger limit, or has no
+ - limit.
+ -}
+fileNameLengthLimit :: FilePath -> IO Int
+#ifdef mingw32_HOST_OS
+fileNameLengthLimit _ = return 255
+#else
+fileNameLengthLimit dir = do
+	l <- fromIntegral <$> getPathVar dir FileNameLimit
+	if l <= 0
+		then return 255
+		else return $ minimum [l, 255]
+  where
+#endif
+
+{- Given a string that we'd like to use as the basis for FilePath, but that
+ - was provided by a third party and is not to be trusted, returns the closest
+ - sane FilePath.
+ -
+ - All spaces and punctuation are replaced with '_', except for '.'
+ - "../" will thus turn into ".._", which is safe.
+ -}
+sanitizeFilePath :: String -> FilePath
+sanitizeFilePath = map sanitize
+  where
+  	sanitize c
+		| c == '.' = c
+		| isSpace c || isPunctuation c || c == '/' = '_'
+		| otherwise = c
diff --git a/Utility/Process.hs b/Utility/Process.hs
new file mode 100644
index 0000000..398e8a3
--- /dev/null
+++ b/Utility/Process.hs
@@ -0,0 +1,356 @@
+{- System.Process enhancements, including additional ways of running
+ - processes, and logging.
+ -
+ - Copyright 2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE CPP, Rank2Types #-}
+
+module Utility.Process (
+	module X,
+	CreateProcess,
+	StdHandle(..),
+	readProcess,
+	readProcessEnv,
+	writeReadProcessEnv,
+	forceSuccessProcess,
+	checkSuccessProcess,
+	ignoreFailureProcess,
+	createProcessSuccess,
+	createProcessChecked,
+	createBackgroundProcess,
+	processTranscript,
+	withHandle,
+	withBothHandles,
+	withQuietOutput,
+	withNullHandle,
+	createProcess,
+	startInteractiveProcess,
+	stdinHandle,
+	stdoutHandle,
+	stderrHandle,
+) where
+
+import qualified System.Process
+import System.Process as X hiding (CreateProcess(..), createProcess, runInteractiveProcess, readProcess, readProcessWithExitCode, system, rawSystem, runInteractiveCommand, runProcess)
+import System.Process hiding (createProcess, readProcess)
+import System.Exit
+import System.IO
+import System.Log.Logger
+import Control.Concurrent
+import qualified Control.Exception as E
+import Control.Monad
+#ifndef mingw32_HOST_OS
+import System.Posix.IO
+#else
+import Control.Applicative
+#endif
+import Data.Maybe
+
+import Utility.Misc
+import Utility.Exception
+
+type CreateProcessRunner = forall a. CreateProcess -> ((Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> IO a) -> IO a
+
+data StdHandle = StdinHandle | StdoutHandle | StderrHandle
+	deriving (Eq)
+
+{- Normally, when reading from a process, it does not need to be fed any
+ - standard input. -}
+readProcess :: FilePath	-> [String] -> IO String
+readProcess cmd args = readProcessEnv cmd args Nothing
+
+readProcessEnv :: FilePath -> [String] -> Maybe [(String, String)] -> IO String
+readProcessEnv cmd args environ =
+	withHandle StdoutHandle createProcessSuccess p $ \h -> do
+		output  <- hGetContentsStrict h
+		hClose h
+		return output
+  where
+	p = (proc cmd args)
+		{ std_out = CreatePipe
+		, env = environ
+		}
+
+{- Runs an action to write to a process on its stdin, 
+ - returns its output, and also allows specifying the environment.
+ -}
+writeReadProcessEnv
+	:: FilePath
+	-> [String]
+	-> Maybe [(String, String)]
+	-> (Maybe (Handle -> IO ()))
+	-> (Maybe (Handle -> IO ()))
+	-> IO String
+writeReadProcessEnv cmd args environ writestdin adjusthandle = do
+	(Just inh, Just outh, _, pid) <- createProcess p
+
+	maybe (return ()) (\a -> a inh) adjusthandle
+	maybe (return ()) (\a -> a outh) adjusthandle
+
+	-- fork off a thread to start consuming the output
+	output  <- hGetContents outh
+	outMVar <- newEmptyMVar
+	_ <- forkIO $ E.evaluate (length output) >> putMVar outMVar ()
+
+	-- now write and flush any input
+	maybe (return ()) (\a -> a inh >> hFlush inh) writestdin
+	hClose inh -- done with stdin
+
+	-- wait on the output
+	takeMVar outMVar
+	hClose outh
+
+	-- wait on the process
+	forceSuccessProcess p pid
+
+	return output
+
+  where
+	p = (proc cmd args)
+		{ std_in = CreatePipe
+		, std_out = CreatePipe
+		, std_err = Inherit
+		, env = environ
+		}
+
+{- Waits for a ProcessHandle, and throws an IOError if the process
+ - did not exit successfully. -}
+forceSuccessProcess :: CreateProcess -> ProcessHandle -> IO ()
+forceSuccessProcess p pid = do
+	code <- waitForProcess pid
+	case code of
+		ExitSuccess -> return ()
+		ExitFailure n -> fail $ showCmd p ++ " exited " ++ show n
+
+{- Waits for a ProcessHandle and returns True if it exited successfully.
+ - Note that using this with createProcessChecked will throw away
+ - the Bool, and is only useful to ignore the exit code of a process,
+ - while still waiting for it. -}
+checkSuccessProcess :: ProcessHandle -> IO Bool
+checkSuccessProcess pid = do
+	code <- waitForProcess pid
+	return $ code == ExitSuccess
+
+ignoreFailureProcess :: ProcessHandle -> IO Bool
+ignoreFailureProcess pid = do
+	void $ waitForProcess pid
+	return True
+
+{- Runs createProcess, then an action on its handles, and then
+ - forceSuccessProcess. -}
+createProcessSuccess :: CreateProcessRunner
+createProcessSuccess p a = createProcessChecked (forceSuccessProcess p) p a
+
+{- Runs createProcess, then an action on its handles, and then
+ - a checker action on its exit code, which must wait for the process. -}
+createProcessChecked :: (ProcessHandle -> IO b) -> CreateProcessRunner
+createProcessChecked checker p a = do
+	t@(_, _, _, pid) <- createProcess p
+	r <- tryNonAsync $ a t
+	_ <- checker pid
+	either E.throw return r
+
+{- Leaves the process running, suitable for lazy streaming.
+ - Note: Zombies will result, and must be waited on. -}
+createBackgroundProcess :: CreateProcessRunner
+createBackgroundProcess p a = a =<< createProcess p
+
+{- Runs a process, optionally feeding it some input, and
+ - returns a transcript combining its stdout and stderr, and
+ - whether it succeeded or failed. -}
+processTranscript :: String -> [String] -> (Maybe String) -> IO (String, Bool)
+#ifndef mingw32_HOST_OS
+{- This implementation interleves stdout and stderr in exactly the order
+ - the process writes them. -}
+processTranscript cmd opts input = do
+	(readf, writef) <- createPipe
+	readh <- fdToHandle readf
+	writeh <- fdToHandle writef
+	p@(_, _, _, pid) <- createProcess $
+		(proc cmd opts)
+			{ std_in = if isJust input then CreatePipe else Inherit
+			, std_out = UseHandle writeh
+			, std_err = UseHandle writeh
+			}
+	hClose writeh
+
+	get <- mkreader readh
+
+	-- now write and flush any input
+	case input of
+		Just s -> do
+			let inh = stdinHandle p
+			unless (null s) $ do
+				hPutStr inh s
+				hFlush inh
+			hClose inh
+		Nothing -> return ()
+
+	transcript <- get
+
+	ok <- checkSuccessProcess pid
+	return (transcript, ok)
+#else
+{- This implementation for Windows puts stderr after stdout. -}
+processTranscript cmd opts input = do
+	p@(_, _, _, pid) <- createProcess $
+		(proc cmd opts)
+			{ std_in = if isJust input then CreatePipe else Inherit
+			, std_out = CreatePipe
+			, std_err = CreatePipe
+			}
+
+	getout <- mkreader (stdoutHandle p)
+	geterr <- mkreader (stderrHandle p)
+
+	case input of
+		Just s -> do
+			let inh = stdinHandle p
+			unless (null s) $ do
+				hPutStr inh s
+				hFlush inh
+			hClose inh
+		Nothing -> return ()
+	
+	transcript <- (++) <$> getout <*> geterr
+	ok <- checkSuccessProcess pid
+	return (transcript, ok)
+#endif
+  where
+	mkreader h = do
+		s <- hGetContents h
+		v <- newEmptyMVar
+		void $ forkIO $ do
+			void $ E.evaluate (length s)
+			putMVar v ()
+		return $ do
+			takeMVar v
+			return s
+
+{- Runs a CreateProcessRunner, on a CreateProcess structure, that
+ - is adjusted to pipe only from/to a single StdHandle, and passes
+ - the resulting Handle to an action. -}
+withHandle
+	:: StdHandle
+	-> CreateProcessRunner
+	-> CreateProcess
+	-> (Handle -> IO a)
+	-> IO a
+withHandle h creator p a = creator p' $ a . select
+  where
+	base = p
+		{ std_in = Inherit
+		, std_out = Inherit
+		, std_err = Inherit
+		}
+	(select, p')
+		| h == StdinHandle  =
+			(stdinHandle, base { std_in = CreatePipe })
+		| h == StdoutHandle =
+			(stdoutHandle, base { std_out = CreatePipe })
+		| h == StderrHandle =
+			(stderrHandle, base { std_err = CreatePipe })
+
+{- Like withHandle, but passes (stdin, stdout) handles to the action. -}
+withBothHandles
+	:: CreateProcessRunner
+	-> CreateProcess
+	-> ((Handle, Handle) -> IO a)
+	-> IO a
+withBothHandles creator p a = creator p' $ a . bothHandles
+  where
+	p' = p
+		{ std_in = CreatePipe
+		, std_out = CreatePipe
+		, std_err = Inherit
+		}
+
+{- Forces the CreateProcessRunner to run quietly;
+ - both stdout and stderr are discarded. -}
+withQuietOutput
+	:: CreateProcessRunner
+	-> CreateProcess
+	-> IO ()
+withQuietOutput creator p = withNullHandle $ \nullh -> do
+	let p' = p
+		{ std_out = UseHandle nullh
+		, std_err = UseHandle nullh
+		}
+	creator p' $ const $ return ()
+
+withNullHandle :: (Handle -> IO a) -> IO a
+withNullHandle = withFile devnull WriteMode
+  where
+#ifndef mingw32_HOST_OS
+	devnull = "/dev/null"
+#else
+	devnull = "NUL"
+#endif
+
+{- Extract a desired handle from createProcess's tuple.
+ - These partial functions are safe as long as createProcess is run
+ - with appropriate parameters to set up the desired handle.
+ - Get it wrong and the runtime crash will always happen, so should be
+ - easily noticed. -}
+type HandleExtractor = (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> Handle
+stdinHandle :: HandleExtractor
+stdinHandle (Just h, _, _, _) = h
+stdinHandle _ = error "expected stdinHandle"
+stdoutHandle :: HandleExtractor
+stdoutHandle (_, Just h, _, _) = h
+stdoutHandle _ = error "expected stdoutHandle"
+stderrHandle :: HandleExtractor
+stderrHandle (_, _, Just h, _) = h
+stderrHandle _ = error "expected stderrHandle"
+bothHandles :: (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> (Handle, Handle)
+bothHandles (Just hin, Just hout, _, _) = (hin, hout)
+bothHandles _ = error "expected bothHandles"
+
+{- Debugging trace for a CreateProcess. -}
+debugProcess :: CreateProcess -> IO ()
+debugProcess p = do
+	debugM "Utility.Process" $ unwords
+		[ action ++ ":"
+		, showCmd p
+		]
+  where
+	action
+		| piped (std_in p) && piped (std_out p) = "chat"
+		| piped (std_in p)                      = "feed"
+		| piped (std_out p)                     = "read"
+		| otherwise                             = "call"
+	piped Inherit = False
+	piped _ = True
+
+{- Shows the command that a CreateProcess will run. -}
+showCmd :: CreateProcess -> String
+showCmd = go . cmdspec
+  where
+	go (ShellCommand s) = s
+	go (RawCommand c ps) = c ++ " " ++ show ps
+
+{- Starts an interactive process. Unlike runInteractiveProcess in
+ - System.Process, stderr is inherited. -}
+startInteractiveProcess
+	:: FilePath
+	-> [String]
+	-> Maybe [(String, String)]
+	-> IO (ProcessHandle, Handle, Handle)
+startInteractiveProcess cmd args environ = do
+	let p = (proc cmd args)
+		{ std_in = CreatePipe
+		, std_out = CreatePipe
+		, std_err = Inherit
+		, env = environ
+		}
+	(Just from, Just to, _, pid) <- createProcess p
+	return (pid, to, from)
+
+{- Wrapper around System.Process function that does debug logging. -}
+createProcess :: CreateProcess -> IO (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle)
+createProcess p = do
+	debugProcess p
+	System.Process.createProcess p
diff --git a/Utility/Rsync.hs b/Utility/Rsync.hs
new file mode 100644
index 0000000..5f322a0
--- /dev/null
+++ b/Utility/Rsync.hs
@@ -0,0 +1,152 @@
+{- various rsync stuff
+ -
+ - Copyright 2010-2013 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Rsync where
+
+import Common
+import Utility.Metered
+
+import Data.Char
+import System.Console.GetOpt
+import Data.Tuple.Utils
+
+{- Generates parameters to make rsync use a specified command as its remote
+ - shell. -}
+rsyncShell :: [CommandParam] -> [CommandParam]
+rsyncShell command = [Param "-e", Param $ unwords $ map escape (toCommand command)]
+  where
+	{- rsync requires some weird, non-shell like quoting in
+	- here. A doubled single quote inside the single quoted
+	- string is a single quote. -}
+	escape s = "'" ++  intercalate "''" (split "'" s) ++ "'"
+
+{- Runs rsync in server mode to send a file. -}
+rsyncServerSend :: [CommandParam] -> FilePath -> IO Bool
+rsyncServerSend options file = rsync $
+	rsyncServerParams ++ Param "--sender" : options ++ [File file]
+
+{- Runs rsync in server mode to receive a file. -}
+rsyncServerReceive :: [CommandParam] -> FilePath -> IO Bool
+rsyncServerReceive options file = rsync $
+	rsyncServerParams ++ options ++ [File file]
+
+rsyncServerParams :: [CommandParam]
+rsyncServerParams =
+	[ Param "--server"
+	-- preserve timestamps
+	, Param "-t"
+	-- allow resuming of transfers of big files
+	, Param "--inplace"
+	-- other options rsync normally uses in server mode
+	, Params "-e.Lsf ."
+	]
+
+rsyncUseDestinationPermissions :: CommandParam
+rsyncUseDestinationPermissions = Param "--chmod=ugo=rwX"
+
+rsync :: [CommandParam] -> IO Bool
+rsync = boolSystem "rsync" . rsyncParamsFixup
+
+{- On Windows, rsync is from Cygwin, and expects to get Cygwin formatted
+ - paths to files. (It thinks that C:foo refers to a host named "C").
+ - Fix up all Files in the Params appropriately. -}
+rsyncParamsFixup :: [CommandParam] -> [CommandParam]
+rsyncParamsFixup = map fixup
+  where
+  	fixup (File f) = File (toCygPath f)
+	fixup p = p
+
+{- Runs rsync, but intercepts its progress output and updates a meter.
+ - The progress output is also output to stdout. 
+ -
+ - The params must enable rsync's --progress mode for this to work.
+ -}
+rsyncProgress :: MeterUpdate -> [CommandParam] -> IO Bool
+rsyncProgress meterupdate params = do
+	r <- withHandle StdoutHandle createProcessSuccess p (feedprogress 0 [])
+	{- For an unknown reason, piping rsync's output like this does
+	 - causes it to run a second ssh process, which it neglects to wait
+	 - on. Reap the resulting zombie. -}
+	reapZombies
+	return r
+  where
+	p = proc "rsync" (toCommand $ rsyncParamsFixup params)
+	feedprogress prev buf h = do
+		s <- hGetSomeString h 80
+		if null s
+			then return True
+			else do
+				putStr s
+				hFlush stdout
+				let (mbytes, buf') = parseRsyncProgress (buf++s)
+				case mbytes of
+					Nothing -> feedprogress prev buf' h
+					(Just bytes) -> do
+						when (bytes /= prev) $
+							meterupdate $ toBytesProcessed bytes
+						feedprogress bytes buf' h
+
+{- Checks if an rsync url involves the remote shell (ssh or rsh).
+ - Use of such urls with rsync requires additional shell
+ - escaping. -}
+rsyncUrlIsShell :: String -> Bool
+rsyncUrlIsShell s
+	| "rsync://" `isPrefixOf` s = False
+	| otherwise = go s
+  where
+	-- host::dir is rsync protocol, while host:dir is ssh/rsh
+	go [] = False
+	go (c:cs)
+		| c == '/' = False -- got to directory with no colon
+		| c == ':' = not $ ":" `isPrefixOf` cs
+		| otherwise = go cs
+
+{- Checks if a rsync url is really just a local path. -}
+rsyncUrlIsPath :: String -> Bool
+rsyncUrlIsPath s
+	| rsyncUrlIsShell s = False
+	| otherwise = ':' `notElem` s
+
+{- Parses the String looking for rsync progress output, and returns
+ - Maybe the number of bytes rsynced so far, and any any remainder of the
+ - string that could be an incomplete progress output. That remainder
+ - should be prepended to future output, and fed back in. This interface
+ - allows the output to be read in any desired size chunk, or even one
+ - character at a time.
+ -
+ - Strategy: Look for chunks prefixed with \r (rsync writes a \r before
+ - the first progress output, and each thereafter). The first number
+ - after the \r is the number of bytes processed. After the number,
+ - there must appear some whitespace, or we didn't get the whole number,
+ - and return the \r and part we did get, for later processing.
+ -}
+parseRsyncProgress :: String -> (Maybe Integer, String)
+parseRsyncProgress = go [] . reverse . progresschunks
+  where
+	go remainder [] = (Nothing, remainder)
+	go remainder (x:xs) = case parsebytes (findbytesstart x) of
+		Nothing -> go (delim:x++remainder) xs
+		Just b -> (Just b, remainder)
+
+	delim = '\r'
+	{- Find chunks that each start with delim.
+	 - The first chunk doesn't start with it
+	 - (it's empty when delim is at the start of the string). -}
+	progresschunks = drop 1 . split [delim]
+	findbytesstart s = dropWhile isSpace s
+	parsebytes s = case break isSpace s of
+		([], _) -> Nothing
+		(_, []) -> Nothing
+		(b, _) -> readish b
+
+{- Filters options to those that are safe to pass to rsync in server mode,
+ - without causing it to eg, expose files. -}
+filterRsyncSafeOptions :: [String] -> [String]
+filterRsyncSafeOptions = fst3 . getOpt Permute
+	[ Option [] ["bwlimit"] (reqArgLong "bwlimit") "" ]
+  where
+	reqArgLong x = ReqArg (\v -> "--" ++ x ++ "=" ++ v) ""
diff --git a/Utility/SafeCommand.hs b/Utility/SafeCommand.hs
new file mode 100644
index 0000000..c8318ec
--- /dev/null
+++ b/Utility/SafeCommand.hs
@@ -0,0 +1,120 @@
+{- safely running shell commands
+ -
+ - Copyright 2010-2013 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.SafeCommand where
+
+import System.Exit
+import Utility.Process
+import System.Process (env)
+import Data.String.Utils
+import Control.Applicative
+import System.FilePath
+import Data.Char
+
+{- A type for parameters passed to a shell command. A command can
+ - be passed either some Params (multiple parameters can be included,
+ - whitespace-separated, or a single Param (for when parameters contain
+ - whitespace), or a File.
+ -}
+data CommandParam = Params String | Param String | File FilePath
+	deriving (Eq, Show, Ord)
+
+{- Used to pass a list of CommandParams to a function that runs
+ - a command and expects Strings. -}
+toCommand :: [CommandParam] -> [String]
+toCommand = concatMap unwrap
+  where
+	unwrap (Param s) = [s]
+	unwrap (Params s) = filter (not . null) (split " " s)
+	-- Files that start with a non-alphanumeric that is not a path
+	-- separator are modified to avoid the command interpreting them as
+	-- options or other special constructs.
+	unwrap (File s@(h:_))
+		| isAlphaNum h || h `elem` pathseps = [s]
+		| otherwise = ["./" ++ s]
+	unwrap (File s) = [s]
+	-- '/' is explicitly included because it's an alternative
+	-- path separator on Windows.
+	pathseps = pathSeparator:"./"
+
+{- Run a system command, and returns True or False
+ - if it succeeded or failed.
+ -}
+boolSystem :: FilePath -> [CommandParam] -> IO Bool
+boolSystem command params = boolSystemEnv command params Nothing
+
+boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool
+boolSystemEnv command params environ = dispatch <$> safeSystemEnv command params environ
+  where
+	dispatch ExitSuccess = True
+	dispatch _ = False
+
+{- Runs a system command, returning the exit status. -}
+safeSystem :: FilePath -> [CommandParam] -> IO ExitCode
+safeSystem command params = safeSystemEnv command params Nothing
+
+safeSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO ExitCode
+safeSystemEnv command params environ = do
+	(_, _, _, pid) <- createProcess (proc command $ toCommand params)
+		{ env = environ }
+	waitForProcess pid
+
+{- Wraps a shell command line inside sh -c, allowing it to be run in a
+ - login shell that may not support POSIX shell, eg csh. -}
+shellWrap :: String -> String
+shellWrap cmdline = "sh -c " ++ shellEscape cmdline
+
+{- Escapes a filename or other parameter to be safely able to be exposed to
+ - the shell.
+ -
+ - This method works for POSIX shells, as well as other shells like csh.
+ -}
+shellEscape :: String -> String
+shellEscape f = "'" ++ escaped ++ "'"
+  where
+	-- replace ' with '"'"'
+	escaped = join "'\"'\"'" $ split "'" f
+
+{- Unescapes a set of shellEscaped words or filenames. -}
+shellUnEscape :: String -> [String]
+shellUnEscape [] = []
+shellUnEscape s = word : shellUnEscape rest
+  where
+	(word, rest) = findword "" s
+	findword w [] = (w, "")
+	findword w (c:cs)
+		| c == ' ' = (w, cs)
+		| c == '\'' = inquote c w cs
+		| c == '"' = inquote c w cs
+		| otherwise = findword (w++[c]) cs
+	inquote _ w [] = (w, "")
+	inquote q w (c:cs)
+		| c == q = findword w cs
+		| otherwise = inquote q (w++[c]) cs
+
+{- For quickcheck. -}
+prop_idempotent_shellEscape :: String -> Bool
+prop_idempotent_shellEscape s = [s] == (shellUnEscape . shellEscape) s
+prop_idempotent_shellEscape_multiword :: [String] -> Bool
+prop_idempotent_shellEscape_multiword s = s == (shellUnEscape . unwords . map shellEscape) s
+
+{- Segements a list of filenames into groups that are all below the manximum
+ - command-line length limit. Does not preserve order. -}
+segmentXargs :: [FilePath] -> [[FilePath]]
+segmentXargs l = go l [] 0 []
+  where
+	go [] c _ r = c:r
+	go (f:fs) c accumlen r
+		| len < maxlen && newlen > maxlen = go (f:fs) [] 0 (c:r)
+		| otherwise = go fs (f:c) newlen r
+	  where
+		len = length f
+		newlen = accumlen + len
+
+	{- 10k of filenames per command, well under Linux's 20k limit;
+	 - allows room for other parameters etc. -}
+	maxlen = 10240
diff --git a/Utility/Tmp.hs b/Utility/Tmp.hs
new file mode 100644
index 0000000..186cd12
--- /dev/null
+++ b/Utility/Tmp.hs
@@ -0,0 +1,88 @@
+{- Temporary files and directories.
+ -
+ - Copyright 2010-2013 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Tmp where
+
+import Control.Exception (bracket)
+import System.IO
+import System.Directory
+import Control.Monad.IfElse
+
+import Utility.Exception
+import System.FilePath
+import Utility.FileSystemEncoding
+
+type Template = String
+
+{- Runs an action like writeFile, writing to a temp file first and
+ - then moving it into place. The temp file is stored in the same
+ - directory as the final file to avoid cross-device renames. -}
+viaTmp :: (FilePath -> String -> IO ()) -> FilePath -> String -> IO ()
+viaTmp a file content = do
+	let (dir, base) = splitFileName file
+	createDirectoryIfMissing True dir
+	(tmpfile, handle) <- openTempFile dir (base ++ ".tmp")
+	hClose handle
+	a tmpfile content
+	renameFile tmpfile file
+
+{- Runs an action with a tmp file located in the system's tmp directory
+ - (or in "." if there is none) then removes the file. -}
+withTmpFile :: Template -> (FilePath -> Handle -> IO a) -> IO a
+withTmpFile template a = do
+	tmpdir <- catchDefaultIO "." getTemporaryDirectory
+	withTmpFileIn tmpdir template a
+
+{- Runs an action with a tmp file located in the specified directory,
+ - then removes the file. -}
+withTmpFileIn :: FilePath -> Template -> (FilePath -> Handle -> IO a) -> IO a
+withTmpFileIn tmpdir template a = bracket create remove use
+  where
+	create = openTempFile tmpdir template
+	remove (name, handle) = do
+		hClose handle
+		catchBoolIO (removeFile name >> return True)
+	use (name, handle) = a name handle
+
+{- Runs an action with a tmp directory located within the system's tmp
+ - directory (or within "." if there is none), then removes the tmp
+ - directory and all its contents. -}
+withTmpDir :: Template -> (FilePath -> IO a) -> IO a
+withTmpDir template a = do
+	tmpdir <- catchDefaultIO "." getTemporaryDirectory
+	withTmpDirIn tmpdir template a
+
+{- Runs an action with a tmp directory located within a specified directory,
+ - then removes the tmp directory and all its contents. -}
+withTmpDirIn :: FilePath -> Template -> (FilePath -> IO a) -> IO a
+withTmpDirIn tmpdir template = bracket create remove
+  where
+	remove d = whenM (doesDirectoryExist d) $
+		removeDirectoryRecursive d
+	create = do
+		createDirectoryIfMissing True tmpdir
+		makenewdir (tmpdir </> template) (0 :: Int)
+	makenewdir t n = do
+		let dir = t ++ "." ++ show n
+		either (const $ makenewdir t $ n + 1) (const $ return dir)
+			=<< tryIO (createDirectory dir)
+
+{- It's not safe to use a FilePath of an existing file as the template
+ - for openTempFile, because if the FilePath is really long, the tmpfile
+ - will be longer, and may exceed the maximum filename length.
+ -
+ - This generates a template that is never too long.
+ - (Well, it allocates 20 characters for use in making a unique temp file,
+ - anyway, which is enough for the current implementation and any
+ - likely implementation.)
+ -}
+relatedTemplate :: FilePath -> FilePath
+relatedTemplate f
+	| len > 20 = truncateFilePath (len - 20) f
+	| otherwise = f
+  where
+	len = length f
diff --git a/Utility/UserInfo.hs b/Utility/UserInfo.hs
new file mode 100644
index 0000000..9c3bfd4
--- /dev/null
+++ b/Utility/UserInfo.hs
@@ -0,0 +1,55 @@
+{- user info
+ -
+ - Copyright 2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE CPP #-}
+
+module Utility.UserInfo (
+	myHomeDir,
+	myUserName,
+	myUserGecos,
+) where
+
+import Control.Applicative
+import System.PosixCompat
+
+import Utility.Env
+
+{- Current user's home directory.
+ -
+ - getpwent will fail on LDAP or NIS, so use HOME if set. -}
+myHomeDir :: IO FilePath
+myHomeDir = myVal env homeDirectory
+  where
+#ifndef mingw32_HOST_OS
+	env = ["HOME"]
+#else
+	env = ["USERPROFILE", "HOME"] -- HOME is used in Cygwin
+#endif
+
+{- Current user's user name. -}
+myUserName :: IO String
+myUserName = myVal env userName
+  where
+#ifndef mingw32_HOST_OS
+	env = ["USER", "LOGNAME"]
+#else
+	env = ["USERNAME", "USER", "LOGNAME"]
+#endif
+
+myUserGecos :: IO String
+#ifdef __ANDROID__
+myUserGecos = return "" -- userGecos crashes on Android
+#else
+myUserGecos = myVal [] userGecos
+#endif
+
+myVal :: [String] -> (UserEntry -> String) -> IO String
+myVal envvars extract = maybe (extract <$> getpwent) return =<< check envvars
+  where
+	check [] = return Nothing
+	check (v:vs) = maybe (check vs) (return . Just) =<< getEnv v
+	getpwent = getUserEntryForID =<< getEffectiveUserID
-- 
cgit v1.2.3