%
% (c) Kevin Hammond, Parade/AQUA Projects, Glasgow University, February 15th. 1995
%
%     This is for GUM only.
%
%************************************************************************
%*                                                                      *
\section[Parallel.lh]{Definitions for parallel machines}
%*									*
%************************************************************************

Multi-slurp protection:
\begin{code}
#ifndef Parallel_H
#define Parallel_H
\end{code}

This section contains definitions applicable only to programs compiled
to run on a parallel machine.  Some of these things can probably be
ripped out now that we don't store GAs in closures, but beware that
this {\em might} break GranSim, so check first!  KH

These basic definitions need to be around, one way or the other:
\begin{code}
#define PAR_FIXED_HDR 			(GA_HDR_SIZE)
#define PAR_HDR_POSN			AFTER_INFO_HDR
#define AFTER_PAR_HDR			(PAR_HDR_POSN+PAR_FIXED_HDR)

#define SET_PAR_HDR(closure,ga)		/* nothing */
#define SET_STATIC_PAR_HDR(closure)	/* nothing */
\end{code}

\begin{code}
# ifdef PAR
#  define MAX_PES	128		/* Maximum number of processors */

extern I_ do_gr_profile;
extern I_ do_sp_profile;
extern I_ do_gr_binary;

extern P_ PendingFetches;
extern GLOBAL_TASK_ID *PEs;

extern rtsBool IAmMainThread, GlobalStopPending;
extern rtsBool fishing;
extern GLOBAL_TASK_ID SysManTask;
extern int seed;			/*pseudo-random-number generator seed:*/
					/*Initialised in ParInit*/
extern I_ threadId;                     /*Number of Threads that have existed on a PE*/
extern GLOBAL_TASK_ID mytid;

extern int  nPEs;

extern rtsBool InGlobalGC;  	/* Are we in the midst of performing global GC */

extern HashTable *pGAtoGALAtable;
extern HashTable *LAtoGALAtable;
extern GALA *freeIndirections;
extern GALA *liveIndirections;
extern GALA *freeGALAList;
extern GALA *liveRemoteGAs;
extern int thisPE;

void RunParallelSystem PROTO((StgPtr program_closure));
void initParallelSystem(STG_NO_ARGS);
void SynchroniseSystem(STG_NO_ARGS);

void registerTask PROTO((GLOBAL_TASK_ID gtid));
globalAddr *LAGAlookup PROTO((P_ addr));
P_ GALAlookup PROTO((globalAddr *ga));
globalAddr *MakeGlobal PROTO((P_ addr, rtsBool preferred));
globalAddr *setRemoteGA PROTO((P_ addr, globalAddr *ga, rtsBool preferred));
void splitWeight PROTO((globalAddr *to, globalAddr *from));
globalAddr *addWeight PROTO((globalAddr *ga));
void initGAtables(STG_NO_ARGS);
W_ taskIDtoPE PROTO((GLOBAL_TASK_ID gtid));
void RebuildLAGAtable(STG_NO_ARGS);

void *lookupHashTable PROTO((HashTable *table, StgWord key));
void insertHashTable PROTO((HashTable *table, StgWord key, void *data));
void freeHashTable PROTO((HashTable *table, void (*freeDataFun) PROTO((void *data))));
HashTable *allocHashTable(STG_NO_ARGS);
void *removeHashTable PROTO((HashTable *table, StgWord key, void *data));

extern void myexit PROTO((I_));
#  define EXIT myexit
#  define exit : error : Wrong exit!
# else
#  define EXIT exit
# endif

\end{code}

%************************************************************************
%*									*
\subsection[anti-parallel-SM]{But if we're {\em not} compiling for a parallel system...}
%*									*
%************************************************************************

Get this out of the way.  These are all null definitions.

\begin{code}
# if !(defined(GRAN) || defined(PAR))

#  define GA_HDR_SIZE			0
#  define	GA(closure)	        	/*nothing*/
  
#  define SET_GA(closure,ga)		/* nothing */
#  define SET_STATIC_GA(closure)		/* nothing */
#  define SET_GRAN_HDR(closure,pe)        /* nothing */
#  define SET_STATIC_PROCS(closure)	/* nothing */
  
#  define SET_TASK_ACTIVITY(act)	    	/* nothing */

# else
# ifdef GRAN

#  define GA_HDR_SIZE			1

#  define PROCS_HDR_POSN		PAR_HDR_POSN
#  define PROCS_HDR_SIZE		1

/* Accessing components of the field */
#  define	PROCS(closure)	        (*((P_)(closure)+PROCS_HDR_POSN))

#  define SET_PROCS(closure, procs) \
	PROCS(closure) = (W_)(procs)   	/* Set closure's location */
#  define SET_GRAN_HDR(closure,pe)	SET_PROCS(closure,pe)

#  if defined(GRAN_TNG)
#   define SET_STATIC_PROCS(closure)	, (W_) (Everywhere)
#  else
#   define SET_STATIC_PROCS(closure)	, (W_) (MainPE)
#  endif   /* GRAN_TNG */

#  define SET_TASK_ACTIVITY(act)    	/* nothing */
\end{code}

%************************************************************************
%*									*
\subsection[parallel-GAs]{Parallel-only part of fixed headers (global addresses)}
%*									*
%************************************************************************

Definitions relating to the entire parallel-only fixed-header field.

On GUM, the global addresses for each local closure are stored in a separate
hash table, rather then with the closure in the heap.  We call @getGA@ to
look up the global address associated with a local closure (0 is returned
for local closures that have no global address), and @setGA@ to store a new
global address for a local closure which did not previously have one.

\begin{code}
# else /* it must be PARallel (to end of file) */

#  define GA_HDR_SIZE			0
  
#  define GA(closure)		        getGA(closure)
  
#  define SET_GA(closure, ga)             setGA(closure,ga)
#  define SET_STATIC_GA(closure)
#  define SET_GRAN_HDR(closure,pe)
#  define SET_STATIC_PROCS(closure)
  
#  define MAX_GA_WEIGHT			0	/* Treat as 2^n */
  
#  define PACK_GA(pe,slot)    ((((W_)(pe)) << (BITS_IN(W_)/2)) | ((W_)(slot)))

\end{code}

At the moment, there is no activity profiling for GUM.  This may change.

\begin{code}

#  define SET_TASK_ACTIVITY(act)

\end{code}

%************************************************************************
%*									*
\subsection[parallel-heap-objs]{Special parallel-only heap objects (`closures')}
%*									*
%************************************************************************

% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% NB: The following definitons are BOTH for GUM and GrAnSim -- HWL
% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The rest of  this  file contains definitions  for  {\it  GUM  and GrAnSim}.
Although  we don't  create FetchMe   nodes in  GrAnSim  (we  simulate it by
bitmask  twiddling)  we use FetchMe_info   when converting  nodes into RBHs
(mainly  to keep the code as  close to GUM as  possible). So, we define all
the FetchMe related stuff in GrAnSim, too. % -- HWL

%************************************************************************
%*									*
\subsubsection[FETCHME-closures]{@FETCHME@ heap objects (`closures')}
%*									*
%************************************************************************

FetchMes are pointers into the global heap.  When evaluated, the value
they point to is read from the global heap.

A FetchMe closure has the form:

\begin{onlylatex}
\begin{center}
\end{onlylatex}
\begin{tabular}{||l|l||}\hline
\tr{FETCHME_info} & junk \\ \hline
\end{tabular}
\begin{onlylatex}
\end{center}
\end{onlylatex}

The argument word is a pointer (outside of the heap) to a globalAddr structure...
in particular, the one corresponding to the object to be fetched.  Note that
we can't just used the LAGA table, because weight-splitting may force us to
reassign a local GA to the @FetchMe@ so that we can give out new references.

A @FetchMe@ must have a valid @MUT_LINK@ field, because it may act as
a transition between an RBH on the OldMutables list and a BQ still on
the OldMutables list.


\begin{code}
#  define FETCHME_VHS				IND_VHS
#  define FETCHME_HS				IND_HS
  
#  define FETCHME_GA_LOCN                       FETCHME_HS
  
#  define FETCHME_CLOSURE_SIZE(closure)		IND_CLOSURE_SIZE(closure)
#  define FETCHME_CLOSURE_NoPTRS(closure)		0L
#  define FETCHME_CLOSURE_NoNONPTRS(closure)	(IND_CLOSURE_SIZE(closure)-IND_VHS)
  
#  define SET_FETCHME_HDR(closure,infolbl,cc,size,ptrs)	\
{ SET_FIXED_HDR(closure,FetchMe_info,<bogus CC>);	\
  SET_MUT_RESERVED_WORDS(closure);			\
}

#  define FETCHME_GA(closure)		(((globalAddr **)(closure))[FETCHME_GA_LOCN])

EXTFUN(FetchMe_entry);
EXTDATA_RO(FetchMe_info);

\end{code}

%************************************************************************
%*									*
\subsubsection[BlockedFetch-closures]{@BlockedFetch@ heap objects (`closures')}
%*									*
%************************************************************************

@BlockedFetch@s are inbound fetch messages blocked on local closures.
They arise as entries in a local blocking queue when a fetch has been
received for a local black hole.  When awakened, we look at their
contents to figure out where to send a resume.

A @BlockedFetch@ closure has the form:

\begin{onlylatex}
\begin{center}
\end{onlylatex}
\begin{tabular}{||l|l|l||}\hline
\tr{BF_info} & link & node & gtid & slot & weight \\ \hline
\end{tabular}
\begin{onlylatex}
\end{center}
\end{onlylatex}

\begin{code}
#  define BF_VHS			    (GC_MUT_RESERVED_WORDS)
#  define BF_HS			    (FIXED_HS + BF_VHS)
  
#  define BF_LINK_LOCN		    (BF_HS)
#  define BF_NODE_LOCN		    (BF_HS + 1)
#  define BF_GTID_LOCN		    (BF_HS + 2)
#  define BF_SLOT_LOCN		    (BF_HS + 3)
#  define BF_WEIGHT_LOCN		    (BF_HS + 4)
  
#  define BF_CLOSURE_NoPTRS(closure)    2
#  define BF_CLOSURE_NoNONPTRS(closure) 3
  
#  define BF_CLOSURE_SIZE(closure)    (BF_VHS + 5)
  
#  define BF_LINK(closure)	    (((PP_)closure)[BF_LINK_LOCN])
#  define BF_NODE(closure)	    (((PP_)closure)[BF_NODE_LOCN])
#  define BF_GTID(closure)	    (((P_)closure)[BF_GTID_LOCN])
#  define BF_SLOT(closure)	    (((P_)closure)[BF_SLOT_LOCN])
#  define BF_WEIGHT(closure)	    (((P_)closure)[BF_WEIGHT_LOCN])
  
#  define BF_CLOSURE_PTR(closure, no) (((P_)(closure))[BF_HS + (no) - 1])

/* std start-filling-in macro: */
#  define SET_BF_HDR(closure,infolbl,cc)   	\
{ SET_FIXED_HDR(closure,infolbl,cc);		\
  SET_MUT_RESERVED_WORDS(closure); 	    	\
}

EXTFUN(BF_entry);
EXTDATA_RO(BF_info);

\end{code}

%************************************************************************
%*									*
\subsubsection[FMBQ-closures]{@FMBQ@ (FetchMe with blocking queue) heap objects (`closures')}
%*									*
%************************************************************************

FetchMe's with blocking queues are @Fetchme@ nodes which have been entered
(and therefore a fetch has been sent), but for which we have not yet received 
a @Resume@ message.  They look just like normal blocking queues, but have
a distinguished info pointer.

\begin{code}
#  define FMBQ_VHS			BQ_VHS
#  define FMBQ_HS				BQ_HS
  
#  define FMBQ_CLOSURE_SIZE(closure)	BQ_CLOSURE_SIZE(closure)
#  define FMBQ_CLOSURE_NoPTRS(closure)	BQ_CLOSURE_NoPTRS(closure)
#  define FMBQ_CLOSURE_NoNONPTRS(closure)	BQ_CLOSURE_NoNONPTRS(closure)
#  define FMBQ_CLOSURE_PTR(closure, no)	BQ_CLOSURE_PTR(closure, no)
  
#  define FMBQ_ENTRIES(closure)		BQ_ENTRIES(closure)
#  define FMBQ_LINK(closure)		BQ_LINK(closure)

EXTFUN(FMBQ_entry);
EXTDATA_RO(FMBQ_info);
\end{code}

%************************************************************************
%*									*
\subsection[parallel-info-tables]{Special parallel-only info-table stuff}
%*									*
%************************************************************************

%************************************************************************
%*									*
\subsubsection[FETCHME_ITBL]{@FETCHME_ITBL@}
%*									*
%************************************************************************

ToDo: delete FetchMe CAT (because we don't profile and parallelize at
the same time...)  Even better...set things up so that we can profile
and parallelize at the same time!

\begin{code}

#  define FETCHME_ITBL(itbl_name,entry_code)			\
    CAT_DECLARE(FetchMe,INTERNAL_KIND,"FetchMe","<FetchMe>")	\
    EXTFUN(entry_code);						\
    EXTDATA_RO(MK_REP_LBL(FetchMe,,));				\
    const W_ itbl_name[] = {					\
        (W_) entry_code					    	\
	,(W_) INFO_OTHER_TAG					\
	,(W_) MK_REP_REF(FetchMe,,)				\
	INCLUDE_PROFILING_INFO(FetchMe)				\
    }

#  define FETCHME_RTBL() 					\
    const W_ MK_REP_LBL(FetchMe,,)[] = { 			\
	INCLUDE_TYPE_INFO(FETCHME)				\
	INCLUDE_SIZE_INFO(MIN_UPD_SIZE, 0L)			\
	INCLUDE_PAR_INFO				 	\
	INCLUDE_COPYING_INFO(_Evacuate_FetchMe,_Scavenge_FetchMe) \
	INCLUDE_COMPACTING_INFO(_ScanLink_FetchMe,_PRStart_FetchMe,_ScanMove_FetchMe,_PRIn_Error) \
    }

\end{code}

%************************************************************************
%*									*
\subsubsection[BF_ITBL]{@BF_ITBL@}
%*									*
%************************************************************************

The special info table used for thread state objects (BlockedFetchs).

\begin{code}

#  define BF_ITBL()				    \
    CAT_DECLARE(BF,INTERNAL_KIND,"BlockedFetch","<BlockedFetch>")    \
    EXTFUN(BF_entry);			    \
    EXTDATA_RO(MK_REP_LBL(BF,,));		    \
    const W_ BF_info[] = {		    \
        (W_) BF_entry			    \
	,(W_) INFO_OTHER_TAG			    \
	,(W_) MK_REP_REF(BF,,)		    \
	INCLUDE_PROFILING_INFO(BF)	    \
	}

#  define BF_RTBL() \
    const W_ MK_REP_LBL(BF,,)[] = { \
	INCLUDE_TYPE_INFO(BF)				\
	INCLUDE_SIZE_INFO(BF_CLOSURE_SIZE(dummy),BF_CLOSURE_NoPTRS(dummy))	\
	INCLUDE_PAR_INFO					\
	INCLUDE_COPYING_INFO(_Evacuate_BF,_Scavenge_BF) 	\
	INCLUDE_COMPACTING_INFO(_ScanLink_BF,_PRStart_BF,_ScanMove_BF,_PRIn_BF) \
	}

\end{code}

%************************************************************************
%*									*
\subsubsection[FMBQ_ITBL]{@FMBQ_ITBL@}
%*									*
%************************************************************************

Special info-table for local blocking queues.

\begin{code}
#  define FMBQ_ITBL()				\
    CAT_DECLARE(FMBQ,INTERNAL_KIND,"FMBQ","<FMBQ>") 	\
    EXTFUN(FMBQ_entry);				\
    EXTDATA_RO(MK_REP_LBL(FMBQ,,));		\
    const W_ FMBQ_info[] = {			\
        (W_) FMBQ_entry				\
	,(W_) INFO_OTHER_TAG			\
	,(W_) MK_REP_REF(FMBQ,,)		\
	INCLUDE_PROFILING_INFO(FMBQ)		\
    }

#  define FMBQ_RTBL() \
    const W_ MK_REP_LBL(FMBQ,,)[] = {				\
	INCLUDE_TYPE_INFO(FMBQ)		    			\
	INCLUDE_SIZE_INFO(MIN_UPD_SIZE,INFO_UNUSED)		\
	INCLUDE_PAR_INFO			    	    	\
	INCLUDE_COPYING_INFO(_Evacuate_BQ,_Scavenge_BQ)     	\
	SPEC_COMPACTING_INFO(_ScanLink_BQ,_PRStart_BQ,_ScanMove_BQ,_PRIn_BQ) \
    }

\end{code}

%************************************************************************
%*									*
\subsection[parallel-spark-pool-defs]{Parallel-only Spark pool definitions}
%*									*
%************************************************************************

\begin{code}
#  ifdef GRAN
#   define HAVE_SPARK ((PendingSparksHd[REQUIRED_POOL] != Nil_closure) || 	\
		    (PendingSparksHd[ADVISORY_POOL] != Nil_closure))
#  else
#   define HAVE_SPARK ((PendingSparksHd[REQUIRED_POOL] < PendingSparksTl[REQUIRED_POOL]) \
		||  (PendingSparksHd[ADVISORY_POOL] < PendingSparksTl[ADVISORY_POOL]))
#  endif

#  define HAVE_WORK  	(RUNNING_THREAD || HAVE_SPARK)
#  define RUNNING_THREAD  (CurrentTSO != NULL)
\end{code}

%************************************************************************
%*									*
\subsection[parallel-pack-defs]{Parallel-only Packing definitions}
%*									*
%************************************************************************


Symbolic constants for the packing code.

This constant defines how many words of data we can pack into a single
packet in the parallel (GUM) system.

\begin{code}
#  ifdef GUM
P_      PackNearbyGraph PROTO((P_ closure,W_ *size));
P_      PackTSO PROTO((P_ tso, W_ *size));
P_      PackStkO PROTO((P_ stko, W_ *size));
P_	AllocateHeap PROTO((W_ size)); /* Doesn't belong */

P_      get_closure_info PROTO((P_ closure, W_ *size, W_ *ptrs, W_ *nonptrs, W_ *vhs));

rtsBool isOffset PROTO((globalAddr *ga)),
	isFixed PROTO((globalAddr *ga));

void    InitClosureQueue (STG_NO_ARGS);
P_      DeQueueClosure(STG_NO_ARGS);
void    QueueClosure PROTO((P_ closure));
rtsBool QueueEmpty(STG_NO_ARGS);
void    PrintPacket PROTO((P_ buffer));
void    doGlobalGC(STG_NO_ARGS);

P_ UnpackGraph PROTO((W_ *buffer, globalAddr **gamap, W_ *nGAs));
#  endif

#  define PACK_BUFFER_SIZE	1024
#  define PACK_HEAP_REQUIRED  \
    ((PACK_BUFFER_SIZE - PACK_HDR_SIZE) / (PACK_GA_SIZE + _FHS) * (SPEC_HS + 2))

\end{code}

\begin{code}
#  define PACK_GA_SIZE	3	/* Size of a packed GA in words */
			        /* Size of a packed fetch-me in words */
#  define PACK_FETCHME_SIZE (PACK_GA_SIZE + FIXED_HS)

#  define PACK_HDR_SIZE	1	/* Words of header in a packet */

#  define PACK_PLC_SIZE	2	/* Size of a packed PLC in words */
  
#  define MAX_GAS 	(PACK_BUFFER_SIZE / PACK_GA_SIZE)

\end{code}
End multi-slurp protection:
\begin{code}
# endif /* yes, it is PARallel */
#endif /* it was GRAN or PARallel */

#endif /* Parallel_H */
\end{code}