1 /* ----------------------------------------------------------------------------
2 Time-stamp: <Wed Mar 21 2001 17:16:28 Stardate: [-30]6363.59 hwloidl>
3 $Id: SysMan.c,v 1.5 2001/08/14 13:40:10 sewardj Exp $
5 GUM System Manager Program
6 Handles startup, shutdown and global synchronisation of the parallel system.
8 The Parade/AQUA Projects, Glasgow University, 1994-1995.
9 GdH/APART Projects, Heriot-Watt University, Edinburgh, 1997-2000.
11 ------------------------------------------------------------------------- */
13 //@node GUM System Manager Program, , ,
14 //@section GUM System Manager Program
22 //* Aux startup and shutdown fcts::
24 //* Message handlers::
29 //@node General docu, Includes, GUM System Manager Program, GUM System Manager Program
30 //@subsection General docu
33 The Sysman task currently controls initiation, termination, of a
34 parallel Haskell program running under GUM. In the future it may
35 control global GC synchronisation and statistics gathering. Based on
36 K. Hammond's SysMan.lc in Graph for PVM. SysMan is unusual in that it
37 is not part of the executable produced by ghc: it is a free-standing
38 program that spawns PVM tasks (logical PEs) to evaluate the
39 program. After initialisation it runs in parallel with the PE tasks,
42 OK children, buckle down for some serious weirdness, it works like this ...
44 o The argument vector (argv) for SysMan has one the following 2 shapes:
46 -------------------------------------------------------------------------------
47 | SysMan path | debug flag | pvm-executable path | Num. PEs | Program Args ...|
48 -------------------------------------------------------------------------------
50 -------------------------------------------------------------------
51 | SysMan path | pvm-executable path | Num. PEs | Program Args ... |
52 -------------------------------------------------------------------
54 The "pvm-executable path" is an absolute path of where PVM stashes the
55 code for each PE. The arguments passed on to each PE-executable
58 -------------------------------
59 | Num. PEs | Program Args ... |
60 -------------------------------
62 The arguments passed to the Main-thread PE-executable are
64 -------------------------------------------------------------------
65 | main flag | pvm-executable path | Num. PEs | Program Args ... |
66 -------------------------------------------------------------------
68 o SysMan's algorithm is as follows.
70 o use PVM to spawn (nPE-1) PVM tasks
71 o fork SysMan to create the main-thread PE. This permits the main-thread to
72 read and write to stdin and stdout.
73 o Wait for all the PE-tasks to reply back saying they are ready and if they were the
75 o Broadcast an array of the PE task-ids out to all of the PE-tasks.
76 o Enter a loop awaiting incoming messages, e.g. failure, Garbage-collection,
79 The forked Main-thread algorithm, in SysMan, is as follows.
81 o disconnects from PVM.
82 o sets a flag in argv to indicate that it is the main thread.
83 o `exec's a copy of the pvm-executable (i.e. the program being run)
86 The pvm-executable run by each PE-task, is initialised as follows.
88 o Registers with PVM, obtaining a task-id.
89 o If it was main it gets SysMan's task-id from argv otherwise it can use pvm_parent.
90 oSends a ready message to SysMan together with a flag indicating if it was main or not.
91 o Receives from SysMan the array of task-ids of the other PEs.
92 o If the number of task-ids sent was larger than expected then it must have been a task
93 generated after the rest of the program had started, so it sends its own task-id message
94 to all the tasks it was told about.
99 //@node Includes, Macros etc, General docu, GUM System Manager Program
100 //@subsection Includes
102 /* Evidently not Posix */
103 /* #include "PosixSource.h" */
106 #include "ParTypes.h"
108 #include "Parallel.h"
109 #include "ParallelRts.h" // stats only
111 //@node Macros etc, Variables, Includes, GUM System Manager Program
112 //@subsection Macros etc
114 /* SysMan is put on top of the GHC routine that does the RtsFlags handling.
115 So, we cannot use the standard macros. For the time being we use a macro
116 that is fixed at compile time.
123 /* debugging enabled */
124 //#define IF_PAR_DEBUG(c,s) { s; }
125 /* debugging disabled */
126 #define IF_PAR_DEBUG(c,s) /* nothing */
128 void *stgMallocBytes (int n, char *msg);
130 //@node Variables, Prototypes, Macros etc, GUM System Manager Program
131 //@subsection Variables
134 The following definitions included so that SysMan can be linked with Low
135 Level Communications module (LLComms). They are not used in SysMan.
139 static unsigned PEsArrived = 0;
140 static GlobalTaskId gtids[MAX_PES];
141 static GlobalTaskId sysman_id, sender_id;
142 static unsigned PEsTerminated = 0;
143 static rtsBool Finishing = rtsFalse;
144 static long PEbuffer[MAX_PES];
145 nat nSpawn = 0; // current no. of spawned tasks (see gtids)
146 nat nPEs = 0; // number of PEs specified on startup
148 /* PVM-ish variables */
149 char *petask, *pvmExecutable;
151 int cc, spawn_flag = PvmTaskDefault;
153 #if 0 && defined(PAR_TICKY)
154 /* ToDo: use allGlobalParStats to collect stats of all PEs */
155 GlobalParStats *allGlobalParStats[MAX_PES];
158 //@node Prototypes, Aux startup and shutdown fcts, Variables, GUM System Manager Program
159 //@subsection Prototypes
161 /* prototypes for message handlers called from the main loop of SysMan */
162 void newPE(int nbytes, int opcode, int sender_id);
163 void readyPE(int nbytes, int opcode, int sender_id);
164 void finishPE(int nbytes, int opcode, int sender_id, int exit_code);
166 //@node Aux startup and shutdown fcts, Main fct, Prototypes, GUM System Manager Program
167 //@subsection Aux startup and shutdown fcts
170 Create the PE Tasks. We spawn (nPEs-1) pvm threads: the Main Thread
171 (which starts execution and performs IO) is created by forking SysMan
174 createPEs(int total_nPEs) {
175 int i, spawn_nPEs, iSpawn = 0, nArch, nHost;
176 struct pvmhostinfo *hostp;
179 spawn_nPEs = total_nPEs-1;
180 if (spawn_nPEs > 0) {
181 IF_PAR_DEBUG(verbose,
182 fprintf(stderr, "==== [%x] Spawning %d PEs(%s) ...\n",
183 sysman_id, spawn_nPEs, petask);
184 fprintf(stderr, " args: ");
185 for (i = 0; pargv[i]; ++i)
186 fprintf(stderr, "%s, ", pargv[i]);
187 fprintf(stderr, "\n"));
189 pvm_config(&nHost,&nArch,&hostp);
190 sysman_host=pvm_tidtohost(sysman_id);
192 /* create PEs on the specific machines in the specified order! */
193 for (i=0; (iSpawn<spawn_nPEs) && (i<nHost); i++)
194 if (hostp[i].hi_tid != sysman_host) {
195 checkComms(pvm_spawn(petask, pargv, spawn_flag+PvmTaskHost,
196 hostp[i].hi_name, 1, gtids+iSpawn),
198 IF_PAR_DEBUG(verbose,
199 fprintf(stderr, "==== [%x] Spawned PE %d onto %s\n",
200 sysman_id, i, hostp[i].hi_name));
204 /* create additional PEs anywhere you like */
205 if (iSpawn<spawn_nPEs) {
206 checkComms(pvm_spawn(petask, pargv, spawn_flag, "",
207 spawn_nPEs-iSpawn, gtids+iSpawn),
209 IF_PAR_DEBUG(verbose,
210 fprintf(stderr,"==== [%x] Spawned %d additional PEs anywhere\n",
211 sysman_id, spawn_nPEs-iSpawn));
216 /* old code with random placement of PEs; make that a variant? */
217 # error "Broken startup in SysMan"
218 { /* let pvm place the PEs anywhere; not used anymore */
219 checkComms(pvm_spawn(petask, pargv, spawn_flag, "", spawn_nPEs, gtids),"SysMan startup");
220 IF_PAR_DEBUG(verbose,
221 fprintf(stderr,"==== [%x] Spawned\n", sysman_id));
226 // iSpawn=spawn_nPEs;
232 Check if this pvm task is in the list of tasks we spawned and are waiting
233 on, if so then remove it.
237 alreadySpawned (GlobalTaskId g) {
240 for (i=0; i<nSpawn; i++)
243 gtids[i] = gtids[nSpawn]; //the last takes its place
250 broadcastFinish(void) {
252 int tids[MAX_PES]; /* local buffer of all surviving PEs */
254 for (i=0, j=0; i<nPEs; i++)
256 tids[j++]=PEbuffer[i]; //extract valid tids
258 IF_PAR_DEBUG(verbose,
259 fprintf(stderr,"==== [%x] Broadcasting Finish to %d PEs; initiating shutdown\n",
262 /* ToDo: move into LLComms.c */
263 pvm_initsend(PvmDataDefault);
264 pvm_mcast(tids,j,PP_FINISH);
268 broadcastPEtids (void) {
271 IF_PAR_DEBUG(verbose,
272 fprintf(stderr,"==== [%x] SysMan sending PE table to all PEs\n", sysman_id);
274 fprintf(stderr,"++++ [%x] PE table as seen by SysMan:\n", mytid);
275 for (i = 0; i < nPEs; i++) {
276 fprintf(stderr,"++++ PEbuffer[%d] = %x\n", i, PEbuffer[i]);
280 broadcastOpN(PP_PETIDS, PEGROUP, nPEs, &PEbuffer);
283 //@node Main fct, Message handlers, Aux startup and shutdown fcts, GUM System Manager Program
284 //@subsection Main fct
288 main (int argc, char **argv) {
290 int opcode, nbytes, nSpawn;
293 setbuf(stdout, NULL); // disable buffering of stdout
294 setbuf(stderr, NULL); // disable buffering of stderr
296 IF_PAR_DEBUG(verbose,
298 "==== RFP: GdH enabled SysMan reporting for duty ($Revision: 1.5 $)\n"));
301 if (*argv[1] == '-') {
302 spawn_flag = PvmTaskDebug;
306 sysman_id = pvm_mytid(); /* This must be the first PVM call */
309 fprintf(stderr, "==== PVM initialisation failure\n");
314 Get the full path and filename of the pvm executable (stashed in some
315 PVM directory), and the number of PEs from the command line.
317 pvmExecutable = argv[1];
318 nPEs = atoi(argv[2]);
321 /* as usual 0 means infinity: use all PEs specified in PVM config */
323 struct pvmhostinfo *hostp;
325 /* get info on PVM config */
326 pvm_config(&nHost,&nArch,&hostp);
328 sprintf(argv[2],"%d",nPEs); /* ToCheck: does this work on all archs */
331 /* get the name of the binary to execute */
332 if ((petask = getenv(PETASK)) == NULL) // PETASK set by driver
335 IF_PAR_DEBUG(verbose,
336 fprintf(stderr,"==== [%x] nPEs: %d; executable: |%s|\n",
337 sysman_id, nPEs, petask));
339 /* Check that we can create the number of PE and IMU tasks requested.
341 This comment is most entertaining since we haven't been using IMUs
342 for the last 10 years or so -- HWL */
343 if ((nPEs > MAX_PES) || (nPEs<1)) {
344 fprintf(stderr,"==** SysMan: No more than %d PEs allowed (%d requested)\n Reconfigure GUM setting MAX_PE in ghc/includes/Parallel.h to a higher value\n",
349 IF_PAR_DEBUG(verbose,
350 fprintf(stderr,"==== [%x] is SysMan Task\n", sysman_id));
352 /* Initialise the PE task arguments from Sysman's arguments */
355 /* Initialise list of all PE identifiers */
358 for (i=0; i<nPEs; i++)
361 /* start up the required number of PEs */
362 nSpawn = createPEs(nPEs);
365 Create the MainThread PE by forking SysMan. This arcane coding
366 is required to allow MainThread to read stdin and write to stdout.
369 //nPEs++; /* Record that the number of PEs is increasing */
371 checkComms(cc,"SysMan fork"); /* Parent continues as SysMan */
373 PEbuffer[0]=0; /* we accept the first main and assume its valid. */
374 PEsArrived=1; /* assume you've got main */
376 IF_PAR_DEBUG(verbose,
377 fprintf(stderr,"==== [%x] Sysman successfully initialized!\n",
380 //@cindex message handling loop
381 /* ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ */
382 /* Main message handling loop */
383 /* ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ */
384 /* Process incoming messages */
386 if ((rbufid = pvm_recv(ANY_TASK, ANY_OPCODE)) < 0) {
387 pvm_perror("==** Sysman: Receiving Message (pvm_recv)");
391 pvm_bufinfo(rbufid, &nbytes, &opcode, &sender_id);
393 /* very low level debugging
394 IF_PAR_DEBUG(verbose,
395 fprintf(stderr,"== [%x] SysMan: Message received by SysMan: rbufid=%x, nbytes = %d, opcode = %x, sender_id = %x\n",
396 sysman_id, rbufid, nbytes, opcode, sender_id));
401 case PP_NEWPE: /* a new PE is registering for work */
402 newPE(nbytes, opcode, sender_id);
405 case PP_READY: /* startup complete; let PEs start working */
406 readyPE(nbytes, opcode, sender_id);
410 case PP_GC_INIT: /* start global GC */
411 /* This Function not yet implemented for GUM */
412 fprintf(stderr,"==** Global GC requested by PE %x. Not yet implemented for GUM!\n",
416 case PP_STATS_ON: /* enable statistics gathering */
417 fprintf(stderr,"==** PP_STATS_ON requested by %x. Not yet implemented for GUM!\n",
421 case PP_STATS_OFF: /* disable statistics gathering */
422 fprintf(stderr,"==** PP_STATS_OFF requested by %x. Not yet implemented for GUM!\n",
428 int exit_code = getExitCode(nbytes, &sender_id);
429 finishPE(nbytes, opcode, sender_id, exit_code);
435 char *opname = GetOpName(opcode);
436 fprintf(stderr,"Sysman: Unrecognised opcode %s (%x)\n",
438 fprintf(stderr,"==** Qagh: Sysman: Unrecognised opcode (%x)\n",
445 /* end of SysMan!! */
447 /* forked main thread begins here */
448 IF_PAR_DEBUG(verbose,
449 fprintf(stderr, "==== Main Thread PE has been forked; doing an execv(%s,...)\n",
451 pvmendtask(); // Disconnect from PVM to avoid confusion:
452 // executable reconnects
454 // RFP: assumes that length(arvv[0])>=9 !!!
455 sprintf(argv[0],"-%08X",sysman_id); /*flag that its the Main Thread PE and include sysman's id*/
456 execv(pvmExecutable,argv); /* Parent task becomes Main Thread PE */
461 //@node Message handlers, Auxiliary fcts, Main fct, GUM System Manager Program
462 //@subsection Message handlers
466 A new PE has been added to the configuration.
469 newPE(int nbytes, int opcode, int sender_id) {
470 IF_PAR_DEBUG(verbose,
471 fprintf(stderr,"==== [%x] SysMan detected a new host\n",
474 /* Determine the new machine... assume its the last on the config list? */
475 if (nSpawn < MAX_PES) {
477 struct pvmhostinfo *hostp;
479 /* get conmfiguration of PVM machine */
480 pvm_config(&nHost,&nArch,&hostp);
482 checkComms(pvm_spawn(petask, pargv, spawn_flag+PvmTaskHost,
483 hostp[nHost].hi_name, 1, gtids+nSpawn),
486 IF_PAR_DEBUG(verbose,
487 fprintf(stderr, "==== [%x] Spawned onto %s\n",
488 sysman_id, hostp[nHost].hi_name));
494 Let it be known that PE @sender_id@ participates in the computation.
497 readyPE(int nbytes, int opcode, int sender_id) {
501 struct pvmhostinfo *hostp;
503 //ASSERT(opcode==PP_READY);
505 IF_PAR_DEBUG(verbose,
506 fprintf(stderr,"==== [%x] SysMan received PP_READY message from %x\n",
507 sysman_id, sender_id));
509 pvm_config(&nHost,&nArch,&hostp);
513 //if ((isMain && (PEbuffer[0]==0)) || alreadySpawned(sender_id)) {
514 if (nPEs >= MAX_PES) {
515 fprintf(stderr,"==== [%x] SysMan doesn't need PE %d (max %d PEs allowed)\n",
516 sysman_id, sender_id, MAX_PES);
520 IF_PAR_DEBUG(verbose,
521 fprintf(stderr,"==== [%x] SysMan found Main PE %x\n",
522 sysman_id, sender_id));
523 PEbuffer[0]=sender_id;
525 /* search for PE in list of PEs */
526 for(i=1; i<nPEs; i++)
527 if (PEbuffer[i]==sender_id) {
531 /* it's a new PE: add it to the list of PEs */
533 PEbuffer[nextPE++] = sender_id;
535 IF_PAR_DEBUG(verbose,
536 fprintf(stderr,"==== [%x] SysMan: found PE %d as [%x] on host %s\n",
537 sysman_id, PEsArrived, sender_id, hostp[PEsArrived].hi_name));
539 PEbuffer[PEsArrived++] = sender_id;
543 /* enable better handling of unexpected terminations */
544 checkComms( pvm_notify(PvmTaskExit, PP_FINISH, 1, &sender_id),
547 /* finished registration of all PEs => enable notification */
548 if ((PEsArrived==nPEs) && PEbuffer[0]) {
549 checkComms( pvm_notify(PvmHostAdd, PP_NEWPE, -1, 0),
551 IF_PAR_DEBUG(verbose,
552 fprintf(stderr,"==== [%x] SysMan initialising notificaton for new hosts\n", sysman_id));
555 /* finished notification => send off the PE ids */
556 if ((PEsArrived>=nPEs) && PEbuffer[0]) {
557 if (PEsArrived>nPEs) {
558 IF_PAR_DEBUG(verbose,
559 fprintf(stderr,"==== [%x] Weird: %d PEs registered, but we only asked for %d\n", sysman_id, PEsArrived, nPEs));
569 Shut down the corresponding PE. Check whether it is a regular shutdown
570 or an uncontrolled termination.
573 finishPE(int nbytes, int opcode, int sender_id, int exitCode) {
576 IF_PAR_DEBUG(verbose,
577 fprintf(stderr,"==== [%x] SysMan received PP_FINISH message from %x (exit code: %d)\n",
578 sysman_id, sender_id, exitCode));
580 /* Is it relevant to us? Count the first message */
581 for (i=0; i<nPEs; i++)
582 if (PEbuffer[i] == sender_id) {
586 /* handle exit code */
587 if (exitCode<0) { /* a task exit before a controlled finish? */
588 fprintf(stderr,"==== [%x] Termination at %x with exit(%d)\n",
589 sysman_id, sender_id, exitCode);
590 } else if (exitCode>0) { /* an abnormal exit code? */
591 fprintf(stderr,"==== [%x] Uncontrolled termination at %x with exit(%d)\n",
592 sysman_id, sender_id, exitCode);
593 } else if (!Finishing) { /* exitCode==0 which is good news */
594 if (i!=0) { /* someone other than main PE terminated first? */
595 fprintf(stderr,"==== [%x] Unexpected early termination at %x\n",
596 sysman_id, sender_id);
598 /* start shutdown by broadcasting FINISH to other PEs */
599 IF_PAR_DEBUG(verbose,
600 fprintf(stderr,"==== [%x] Initiating shutdown (requested by [%x] RIP) (exit code: %d)\n", sysman_id, sender_id, exitCode));
605 /* we are in a shutdown already */
606 IF_PAR_DEBUG(verbose,
607 fprintf(stderr,"==== [%x] Finish from %x during shutdown (%d PEs terminated so far; %d total)\n",
608 sysman_id, sender_id, PEsTerminated, nPEs));
611 if (PEsTerminated >= nPEs) {
612 IF_PAR_DEBUG(verbose,
613 fprintf(stderr,"==== [%x] Global Shutdown, Goodbye!! (SysMan has received FINISHes from all PEs)\n", sysman_id));
615 /* received finish from everybody; now, we can exit, too */
616 exit(EXIT_SUCCESS); /* Qapla'! */
621 //@node Auxiliary fcts, Index, Message handlers, GUM System Manager Program
622 //@subsection Auxiliary fcts
624 /* Needed here because its used in loads of places like LLComms etc */
629 * called from STG-land to exit the program
635 fprintf(stderr, "==// [%x] %s in SysMan code; sending PP_FINISH to all PEs ...\n",
636 mytid,(n!=0)?"FAILURE":"FINISH");
643 //@node Index, , Auxiliary fcts, GUM System Manager Program
647 //* main:: @cindex\s-+main
648 //* message handling loop:: @cindex\s-+message handling loop
649 //* stgMallocBytes:: @cindex\s-+stgMallocBytes
650 //* stg_exit:: @cindex\s-+stg_exit