+{-# LANGUAGE GADTs,NoMonoLocalBinds #-}
+-- Norman likes local bindings
+-- If this module lives on I'd like to get rid of this flag in due course
+
+{-# OPTIONS_GHC -fno-warn-incomplete-patterns #-}
+#if __GLASGOW_HASKELL__ >= 701
+-- GHC 7.0.1 improved incomplete pattern warnings with GADTs
+{-# OPTIONS_GHC -fwarn-incomplete-patterns #-}
+#endif
module CmmSpillReload
( DualLive(..)
where
import BlockId
+import Cmm
import CmmExpr
-import CmmTx
-import CmmLiveZ
-import DFMonad
-import MkZipCfg
+import CmmLive
import OptimizationFuel
-import PprCmm()
-import ZipCfg
-import ZipCfgCmmRep
-import ZipDataflow
-import Monad
+import Control.Monad
import Outputable hiding (empty)
import qualified Outputable as PP
-import Panic
import UniqSet
-import Maybe
-import Prelude hiding (zip)
-
--- The point of this module is to insert spills and reloads to
--- establish the invariant that at a call (or at any proc point with
--- an established protocol) all live variables not expected in
--- registers are sitting on the stack. We use a backward analysis to
--- insert spills and reloads. It should be followed by a
--- forward transformation to sink reloads as deeply as possible, so as
--- to reduce register pressure.
-
--- A variable can be expected to be live in a register, live on the
--- stack, or both. This analysis ensures that spills and reloads are
--- inserted as needed to make sure that every live variable needed
--- after a call is available on the stack. Spills are pushed back to
--- their reaching definitions, but reloads are dropped wherever needed
--- and will have to be sunk by a later forward transformation.
+import Compiler.Hoopl
+import Data.Maybe
+import Prelude hiding (succ, zip)
+
+{- Note [Overview of spill/reload]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The point of this module is to insert spills and reloads to
+establish the invariant that at a call (or at any proc point with
+an established protocol) all live variables not expected in
+registers are sitting on the stack. We use a backward analysis to
+insert spills and reloads. It should be followed by a
+forward transformation to sink reloads as deeply as possible, so as
+to reduce register pressure.
+
+A variable can be expected to be live in a register, live on the
+stack, or both. This analysis ensures that spills and reloads are
+inserted as needed to make sure that every live variable needed
+after a call is available on the stack. Spills are pushed back to
+their reaching definitions, but reloads are dropped wherever needed
+and will have to be sunk by a later forward transformation.
+-}
data DualLive = DualLive { on_stack :: RegSet, in_regs :: RegSet }
dualLiveLattice :: DataflowLattice DualLive
-dualLiveLattice =
- DataflowLattice "variables live in registers and on stack" empty add False
+dualLiveLattice = DataflowLattice "variables live in registers and on stack" empty add
where empty = DualLive emptyRegSet emptyRegSet
- -- | compute in the Tx monad to track whether anything has changed
- add new old = do stack <- add1 (on_stack new) (on_stack old)
- regs <- add1 (in_regs new) (in_regs old)
- return $ DualLive stack regs
- add1 = fact_add_to liveLattice
-
-type LiveReloadFix a = FuelMonad (BackwardFixedPoint Middle Last DualLive a)
-
-dualLivenessWithInsertion :: BlockSet -> (LGraph Middle Last) -> FuelMonad (LGraph Middle Last)
-dualLivenessWithInsertion procPoints g@(LGraph entry _ _) =
- liftM zdfFpContents $ (res :: LiveReloadFix (LGraph Middle Last))
- where res = zdfBRewriteFromL RewriteDeep emptyBlockEnv "dual liveness with insertion"
- dualLiveLattice (dualLiveTransfers entry procPoints)
- (insertSpillAndReloadRewrites entry procPoints) empty g
- empty = fact_bot dualLiveLattice
-
-dualLiveness :: BlockSet -> LGraph Middle Last -> FuelMonad (BlockEnv DualLive)
-dualLiveness procPoints g@(LGraph entry _ _) =
- liftM zdfFpFacts $ (res :: LiveReloadFix ())
- where res = zdfSolveFromL emptyBlockEnv "dual liveness" dualLiveLattice
- (dualLiveTransfers entry procPoints) empty g
- empty = fact_bot dualLiveLattice
-
-dualLiveTransfers :: BlockId -> BlockSet -> BackwardTransfers Middle Last DualLive
-dualLiveTransfers entry procPoints = BackwardTransfers first middle last
- where last = lastDualLiveness
- middle = middleDualLiveness
- first live id = check live id $ -- live at procPoint => spill
- if id /= entry && elemBlockSet id procPoints then
- DualLive { on_stack = on_stack live `plusRegSet` in_regs live
- , in_regs = emptyRegSet }
- else live
- check live id x = if id == entry then noLiveOnEntry id (in_regs live) x else x
-
-middleDualLiveness :: DualLive -> Middle -> DualLive
-middleDualLiveness live m =
- changeStack updSlots $ changeRegs (middleLiveness m) (changeRegs regs_in live)
- where regs_in live = case m of MidForeignCall {} -> emptyRegSet
- _ -> live
- updSlots live = foldSlotsUsed reload (foldSlotsDefd spill live m) m
- spill live s@(RegSlot r, _, _) = check s $ deleteFromRegSet live r
- spill live _ = live
- reload live s@(RegSlot r, _, _) = check s $ extendRegSet live r
- reload live _ = live
- check (RegSlot (LocalReg _ ty), o, w) x
- | o == w && w == widthInBytes (typeWidth ty) = x
- check _ _ = panic "middleDualLiveness unsupported: slices"
-
-lastDualLiveness :: (BlockId -> DualLive) -> Last -> DualLive
-lastDualLiveness env l = last l
- where last (LastBranch id) = env id
- last l@(LastCall _ Nothing _ _) = changeRegs (gen l . kill l) empty
- last l@(LastCall _ (Just k) _ _) =
- -- nothing can be live in registers at this point, unless safe foreign call
- let live = env k
- live_in = DualLive (on_stack live) (gen l emptyRegSet)
- in if isEmptyUniqSet (in_regs live) then live_in
- else pprTrace "Offending party:" (ppr k <+> ppr live) $
- panic "live values in registers at call continuation"
- last l@(LastCondBranch _ t f) =
- changeRegs (gen l . kill l) $ dualUnion (env t) (env f)
- last l@(LastSwitch _ tbl) = changeRegs (gen l . kill l) $ dualUnionList $
- map env (catMaybes tbl)
- empty = fact_bot dualLiveLattice
-
+ add _ (OldFact old) (NewFact new) = (changeIf $ change1 || change2, DualLive stack regs)
+ where (change1, stack) = add1 (on_stack old) (on_stack new)
+ (change2, regs) = add1 (in_regs old) (in_regs new)
+ add1 old new = if sizeUniqSet join > sizeUniqSet old then (True, join) else (False, old)
+ where join = unionUniqSets old new
+
+dualLivenessWithInsertion :: BlockSet -> CmmGraph -> FuelUniqSM CmmGraph
+dualLivenessWithInsertion procPoints g =
+ liftM fst $ dataflowPassBwd g [] $ analRewBwd dualLiveLattice
+ (dualLiveTransfers (g_entry g) procPoints)
+ (insertSpillAndReloadRewrites g procPoints)
+
+dualLiveness :: BlockSet -> CmmGraph -> FuelUniqSM (BlockEnv DualLive)
+dualLiveness procPoints g =
+ liftM snd $ dataflowPassBwd g [] $ analBwd dualLiveLattice $ dualLiveTransfers (g_entry g) procPoints
+
+dualLiveTransfers :: BlockId -> BlockSet -> (BwdTransfer CmmNode DualLive)
+dualLiveTransfers entry procPoints = mkBTransfer3 first middle last
+ where first :: CmmNode C O -> DualLive -> DualLive
+ first (CmmEntry id) live = check live id $ -- live at procPoint => spill
+ if id /= entry && setMember id procPoints
+ then DualLive { on_stack = on_stack live `plusRegSet` in_regs live
+ , in_regs = emptyRegSet }
+ else live
+ where check live id x = if id == entry then noLiveOnEntry id (in_regs live) x else x
+
+ middle :: CmmNode O O -> DualLive -> DualLive
+ middle m = changeStack updSlots
+ . changeRegs updRegs
+ where -- Reuse middle of liveness analysis from CmmLive
+ updRegs = case getBTransfer3 xferLive of (_, middle, _) -> middle m
+
+ updSlots live = foldSlotsUsed reload (foldSlotsDefd spill live m) m
+ spill live s@(RegSlot r, _, _) = check s $ deleteFromRegSet live r
+ spill live _ = live
+ reload live s@(RegSlot r, _, _) = check s $ extendRegSet live r
+ reload live _ = live
+ check (RegSlot (LocalReg _ ty), o, w) x
+ | o == w && w == widthInBytes (typeWidth ty) = x
+ check _ _ = panic "middleDualLiveness unsupported: slices"
+ last :: CmmNode O C -> FactBase DualLive -> DualLive
+ last l fb = case l of
+ CmmBranch id -> lkp id
+ l@(CmmCall {cml_cont=Nothing}) -> changeRegs (gen l . kill l) empty
+ l@(CmmCall {cml_cont=Just k}) -> call l k
+ l@(CmmForeignCall {succ=k}) -> call l k
+ l@(CmmCondBranch _ t f) -> changeRegs (gen l . kill l) $ dualUnion (lkp t) (lkp f)
+ l@(CmmSwitch _ tbl) -> changeRegs (gen l . kill l) $ dualUnionList $ map lkp (catMaybes tbl)
+ where empty = fact_bot dualLiveLattice
+ lkp id = empty `fromMaybe` lookupFact id fb
+ call l k = DualLive (on_stack (lkp k)) (gen l emptyRegSet)
+
gen :: UserOfLocalRegs a => a -> RegSet -> RegSet
gen a live = foldRegsUsed extendRegSet live a
kill :: DefinerOfLocalRegs a => a -> RegSet -> RegSet
kill a live = foldRegsDefd deleteFromRegSet live a
-insertSpillAndReloadRewrites ::
- BlockId -> BlockSet -> BackwardRewrites Middle Last DualLive
-insertSpillAndReloadRewrites entry procPoints =
- BackwardRewrites first middle last exit
- where middle = middleInsertSpillsAndReloads
- last _ _ = Nothing
- exit = Nothing
- first live id =
- if id /= entry && elemBlockSet id procPoints then
- case map reload (uniqSetToList (in_regs live)) of
+insertSpillAndReloadRewrites :: CmmGraph -> BlockSet -> CmmBwdRewrite DualLive
+insertSpillAndReloadRewrites graph procPoints = deepBwdRw3 first middle nothing
+ -- Beware: deepBwdRw with one polymorphic function seems more reasonable here,
+ -- but GHC miscompiles it, see bug #4044.
+ where first :: CmmNode C O -> Fact O DualLive -> CmmReplGraph C O
+ first e@(CmmEntry id) live = return $
+ if id /= (g_entry graph) && setMember id procPoints then
+ case map reload (uniqSetToList spill_regs) of
[] -> Nothing
- is -> Just (mkMiddles is)
+ is -> Just $ mkFirst e <*> mkMiddles is
else Nothing
+ where
+ -- If we are splitting procedures, we need the LastForeignCall
+ -- to spill its results to the stack because they will only
+ -- be used by a separate procedure (so they can't stay in LocalRegs).
+ splitting = True
+ spill_regs = if splitting then in_regs live
+ else in_regs live `minusRegSet` defs
+ defs = case mapLookup id firstDefs of
+ Just defs -> defs
+ Nothing -> emptyRegSet
+ -- A LastForeignCall may contain some definitions, which take place
+ -- on return from the function call. Therefore, we build a map (firstDefs)
+ -- from BlockId to the set of variables defined on return to the BlockId.
+ firstDefs = mapFold addLive emptyBlockMap (toBlockMap graph)
+ addLive :: CmmBlock -> BlockEnv RegSet -> BlockEnv RegSet
+ addLive b env = case lastNode b of
+ CmmForeignCall {succ=k, res=defs} -> add k (mkRegSet defs) env
+ _ -> env
+ add bid defs env = mapInsert bid defs'' env
+ where defs'' = case mapLookup bid env of
+ Just defs' -> timesRegSet defs defs'
+ Nothing -> defs
+
+ middle :: CmmNode O O -> Fact O DualLive -> CmmReplGraph O O
+ middle (CmmAssign (CmmLocal reg) (CmmLoad (CmmStackSlot (RegSlot reg') _) _)) _ | reg == reg' = return Nothing
+ middle m@(CmmAssign (CmmLocal reg) _) live = return $
+ if reg `elemRegSet` on_stack live then -- must spill
+ my_trace "Spilling" (f4sep [text "spill" <+> ppr reg,
+ text "after"{-, ppr m-}]) $
+ Just $ mkMiddles $ [m, spill reg]
+ else Nothing
+ middle m@(CmmUnsafeForeignCall _ fs _) live = return $
+ case map spill (filter (flip elemRegSet (on_stack live)) fs) ++
+ map reload (uniqSetToList (kill fs (in_regs live))) of
+ [] -> Nothing
+ reloads -> Just $ mkMiddles (m : reloads)
+ middle _ _ = return Nothing
+
+ nothing _ _ = return Nothing
-middleInsertSpillsAndReloads :: DualLive -> Middle -> Maybe (AGraph Middle Last)
-middleInsertSpillsAndReloads live m = middle m
- where middle (MidAssign (CmmLocal reg) (CmmLoad (CmmStackSlot (RegSlot reg') _) _))
- | reg == reg' = Nothing
- middle (MidAssign (CmmLocal reg) _) =
- if reg `elemRegSet` on_stack live then -- must spill
- my_trace "Spilling" (f4sep [text "spill" <+> ppr reg,
- text "after", ppr m]) $
- Just $ mkMiddles $ [m, spill reg]
- else Nothing
- middle (MidForeignCall _ _ fs _) =
- case map spill (filter (flip elemRegSet (on_stack live)) fs) ++
- map reload (uniqSetToList (kill fs (in_regs live))) of
- [] -> Nothing
- reloads -> Just (mkMiddles (m : reloads))
- middle _ = Nothing
-
--- Generating spill and reload code
regSlot :: LocalReg -> CmmExpr
regSlot r = CmmStackSlot (RegSlot r) (widthInBytes $ typeWidth $ localRegType r)
-spill, reload :: LocalReg -> Middle
-spill r = MidStore (regSlot r) (CmmReg $ CmmLocal r)
-reload r = MidAssign (CmmLocal r) (CmmLoad (regSlot r) $ localRegType r)
-
-reloadTail :: RegSet -> ZTail Middle Last -> ZTail Middle Last
-reloadTail regset t = foldl rel t $ uniqSetToList regset
- where rel t r = ZTail (reload r) t
+spill, reload :: LocalReg -> CmmNode O O
+spill r = CmmStore (regSlot r) (CmmReg $ CmmLocal r)
+reload r = CmmAssign (CmmLocal r) (CmmLoad (regSlot r) $ localRegType r)
----------------------------------------------------------------
--- sinking reloads
availRegsLattice :: DataflowLattice AvailRegs
-availRegsLattice = DataflowLattice "register gotten from reloads" empty add False
- -- last True <==> debugging on
+availRegsLattice = DataflowLattice "register gotten from reloads" empty add
where empty = UniverseMinus emptyRegSet
-- | compute in the Tx monad to track whether anything has changed
- add new old =
- let join = interAvail new old in
- if join `smallerAvail` old then aTx join else noTx join
+ add _ (OldFact old) (NewFact new) =
+ if join `smallerAvail` old then (SomeChange, join) else (NoChange, old)
+ where join = interAvail new old
interAvail :: AvailRegs -> AvailRegs -> AvailRegs
smallerAvail (AvailRegs s) (AvailRegs s') = sizeUniqSet s < sizeUniqSet s'
smallerAvail (UniverseMinus s) (UniverseMinus s') = sizeUniqSet s > sizeUniqSet s'
---extendAvail :: AvailRegs -> LocalReg -> AvailRegs
---extendAvail (UniverseMinus s) r = UniverseMinus (deleteFromRegSet s r)
---extendAvail (AvailRegs s) r = AvailRegs (extendRegSet s r)
+extendAvail :: AvailRegs -> LocalReg -> AvailRegs
+extendAvail (UniverseMinus s) r = UniverseMinus (deleteFromRegSet s r)
+extendAvail (AvailRegs s) r = AvailRegs (extendRegSet s r)
-deleteFromAvail :: AvailRegs -> LocalReg -> AvailRegs
-deleteFromAvail (UniverseMinus s) r = UniverseMinus (extendRegSet s r)
-deleteFromAvail (AvailRegs s) r = AvailRegs (deleteFromRegSet s r)
+delFromAvail :: AvailRegs -> LocalReg -> AvailRegs
+delFromAvail (UniverseMinus s) r = UniverseMinus (extendRegSet s r)
+delFromAvail (AvailRegs s) r = AvailRegs (deleteFromRegSet s r)
elemAvail :: AvailRegs -> LocalReg -> Bool
elemAvail (UniverseMinus s) r = not $ elemRegSet r s
elemAvail (AvailRegs s) r = elemRegSet r s
-type CmmAvail = BlockEnv AvailRegs
-type AvailFix = FuelMonad (ForwardFixedPoint Middle Last AvailRegs ())
-
-cmmAvailableReloads :: LGraph Middle Last -> FuelMonad CmmAvail
-cmmAvailableReloads g = liftM zdfFpFacts $ (res :: AvailFix)
- where res = zdfSolveFromL emptyBlockEnv "available reloads" availRegsLattice
- avail_reloads_transfer empty g
- empty = fact_bot availRegsLattice
-
-avail_reloads_transfer :: ForwardTransfers Middle Last AvailRegs
-avail_reloads_transfer = ForwardTransfers first middle last id
- where first avail _ = avail
- middle = flip middleAvail
- last = lastAvail
-
--- | The transfer equations use the traditional 'gen' and 'kill'
--- notations, which should be familiar from the dragon book.
---agen,
-akill :: UserOfLocalRegs a => a -> AvailRegs -> AvailRegs
---agen a live = foldRegsUsed extendAvail live a
-akill a live = foldRegsUsed deleteFromAvail live a
-
--- Note: you can't sink the reload past a use.
-middleAvail :: Middle -> AvailRegs -> AvailRegs
-middleAvail m = middle m
- where middle m live = middle' m $ foldRegsUsed deleteFromAvail live m
- middle' (MidComment {}) live = live
- middle' (MidAssign lhs _expr) live = akill lhs live
- middle' (MidStore {}) live = live
- middle' (MidForeignCall {}) _ = AvailRegs emptyRegSet
-
-lastAvail :: AvailRegs -> Last -> LastOutFacts AvailRegs
-lastAvail _ (LastCall _ (Just k) _ _) = LastOutFacts [(k, AvailRegs emptyRegSet)]
-lastAvail avail l = LastOutFacts $ map (\id -> (id, avail)) $ succs l
-
-type LateReloadFix = FuelMonad (ForwardFixedPoint Middle Last AvailRegs CmmGraph)
-
-insertLateReloads :: (LGraph Middle Last) -> FuelMonad (LGraph Middle Last)
-insertLateReloads g = liftM zdfFpContents $ (res :: LateReloadFix)
- where res = zdfFRewriteFromL RewriteShallow emptyBlockEnv "insert late reloads"
- availRegsLattice avail_reloads_transfer rewrites bot g
- bot = fact_bot availRegsLattice
- rewrites = ForwardRewrites first middle last exit
- first _ _ = Nothing
- middle :: AvailRegs -> Middle -> Maybe (AGraph Middle Last)
- last :: AvailRegs -> Last -> Maybe (AGraph Middle Last)
- middle avail m = maybe_reload_before avail m (ZTail m (ZLast LastExit))
- last avail l = maybe_reload_before avail l (ZLast (LastOther l))
- exit _ = Nothing
- maybe_reload_before avail node tail =
- let used = filterRegsUsed (elemAvail avail) node
- in if isEmptyUniqSet used then Nothing
- else Just $ mkZTail $ reloadTail used tail
-
-removeDeadAssignmentsAndReloads :: BlockSet -> (LGraph Middle Last) -> FuelMonad (LGraph Middle Last)
-removeDeadAssignmentsAndReloads procPoints g@(LGraph entry _ _) =
- liftM zdfFpContents $ (res :: LiveReloadFix (LGraph Middle Last))
- where res = zdfBRewriteFromL RewriteDeep emptyBlockEnv "dead-assignment & -reload elim"
- dualLiveLattice (dualLiveTransfers entry procPoints)
- rewrites (fact_bot dualLiveLattice) g
- rewrites = BackwardRewrites first middle last exit
- exit = Nothing
- last = \_ _ -> Nothing
- middle = middleRemoveDeads
- first _ _ = Nothing
-
-middleRemoveDeads :: DualLive -> Middle -> Maybe (AGraph Middle Last)
-middleRemoveDeads live m = middle m
- where middle (MidAssign (CmmLocal reg') _)
- | not (reg' `elemRegSet` in_regs live) = Just emptyAGraph
- middle _ = Nothing
-
+cmmAvailableReloads :: CmmGraph -> FuelUniqSM (BlockEnv AvailRegs)
+cmmAvailableReloads g =
+ liftM snd $ dataflowPassFwd g [(g_entry g, fact_bot availRegsLattice)] $
+ analFwd availRegsLattice availReloadsTransfer
+
+availReloadsTransfer :: FwdTransfer CmmNode AvailRegs
+availReloadsTransfer = mkFTransfer3 (flip const) middleAvail ((mkFactBase availRegsLattice .) . lastAvail)
+
+middleAvail :: CmmNode O O -> AvailRegs -> AvailRegs
+middleAvail (CmmAssign (CmmLocal r) (CmmLoad l _)) avail
+ | l `isStackSlotOf` r = extendAvail avail r
+middleAvail (CmmAssign lhs _) avail = foldRegsDefd delFromAvail avail lhs
+middleAvail (CmmStore l (CmmReg (CmmLocal r))) avail
+ | l `isStackSlotOf` r = avail
+middleAvail (CmmStore (CmmStackSlot (RegSlot r) _) _) avail = delFromAvail avail r
+middleAvail (CmmStore {}) avail = avail
+middleAvail (CmmUnsafeForeignCall {}) _ = AvailRegs emptyRegSet
+middleAvail (CmmComment {}) avail = avail
+
+lastAvail :: CmmNode O C -> AvailRegs -> [(Label, AvailRegs)]
+lastAvail (CmmCall _ (Just k) _ _ _) _ = [(k, AvailRegs emptyRegSet)]
+lastAvail (CmmForeignCall {succ=k}) _ = [(k, AvailRegs emptyRegSet)]
+lastAvail l avail = map (\id -> (id, avail)) $ successors l
+
+insertLateReloads :: CmmGraph -> FuelUniqSM CmmGraph
+insertLateReloads g =
+ liftM fst $ dataflowPassFwd g [(g_entry g, fact_bot availRegsLattice)] $
+ analRewFwd availRegsLattice availReloadsTransfer rewrites
+ where rewrites = mkFRewrite3 first middle last
+ first _ _ = return Nothing
+ middle m avail = return $ maybe_reload_before avail m (mkMiddle m)
+ last l avail = return $ maybe_reload_before avail l (mkLast l)
+ maybe_reload_before avail node tail =
+ let used = filterRegsUsed (elemAvail avail) node
+ in if isEmptyUniqSet used then Nothing
+ else Just $ reloadTail used tail
+ reloadTail regset t = foldl rel t $ uniqSetToList regset
+ where rel t r = mkMiddle (reload r) <*> t
+
+removeDeadAssignmentsAndReloads :: BlockSet -> CmmGraph -> FuelUniqSM CmmGraph
+removeDeadAssignmentsAndReloads procPoints g =
+ liftM fst $ dataflowPassBwd g [] $ analRewBwd dualLiveLattice
+ (dualLiveTransfers (g_entry g) procPoints)
+ rewrites
+ where rewrites = deepBwdRw3 nothing middle nothing
+ -- Beware: deepBwdRw with one polymorphic function seems more reasonable here,
+ -- but GHC panics while compiling, see bug #4045.
+ middle :: CmmNode O O -> Fact O DualLive -> CmmReplGraph O O
+ middle (CmmAssign (CmmLocal reg') _) live | not (reg' `elemRegSet` in_regs live) = return $ Just emptyGraph
+ middle _ _ = return Nothing
+
+ nothing _ _ = return Nothing
---------------------