1 /* -----------------------------------------------------------------------------
2 * $Id: PrimOps.hc,v 1.102 2002/10/22 11:01:19 simonmar Exp $
4 * (c) The GHC Team, 1998-2000
6 * Primitive functions / data
8 * ---------------------------------------------------------------------------*/
14 #include "StgStartup.h"
19 #include "BlockAlloc.h" /* tmp */
20 #include "StablePriv.h"
25 #ifdef HAVE_SYS_TYPES_H
26 # include <sys/types.h>
33 classes CCallable and CReturnable don't really exist, but the
34 compiler insists on generating dictionaries containing references
35 to GHC_ZcCCallable_static_info etc., so we provide dummy symbols
36 for these. Some C compilers can't cope with zero-length static arrays,
37 so we have to make these one element long.
40 StgWord GHC_ZCCCallable_static_info[1];
41 StgWord GHC_ZCCReturnable_static_info[1];
43 /* -----------------------------------------------------------------------------
44 Macros for Hand-written primitives.
45 -------------------------------------------------------------------------- */
48 * Horrible macros for returning unboxed tuples.
50 * How an unboxed tuple is returned depends on two factors:
51 * - the number of real registers we have available
52 * - the boxedness of the returned fields.
54 * To return an unboxed tuple from a primitive operation, we have macros
55 * RET_<layout> where <layout> describes the boxedness of each field of the
56 * unboxed tuple: N indicates a non-pointer field, and P indicates a pointer.
58 * We only define the cases actually used, to avoid having too much
59 * garbage in this section. Warning: any bugs in here will be hard to
63 /*------ All Regs available */
65 # define RET_P(a) R1.w = (W_)(a); JMP_(ENTRY_CODE(Sp[0]));
66 # define RET_N(a) RET_P(a)
68 # define RET_PP(a,b) R1.w = (W_)(a); R2.w = (W_)(b); JMP_(ENTRY_CODE(Sp[0]));
69 # define RET_NN(a,b) RET_PP(a,b)
70 # define RET_NP(a,b) RET_PP(a,b)
72 # define RET_PPP(a,b,c) \
73 R1.w = (W_)(a); R2.w = (W_)(b); R3.w = (W_)(c); JMP_(ENTRY_CODE(Sp[0]));
74 # define RET_NNP(a,b,c) RET_PPP(a,b,c)
76 # define RET_NNNP(a,b,c,d) \
77 R1.w = (W_)(a); R2.w = (W_)(b); R3.w = (W_)(c); R4.w = (W_)d; \
78 JMP_(ENTRY_CODE(Sp[0]));
80 # define RET_NPNP(a,b,c,d) \
81 R1.w = (W_)(a); R2.w = (W_)(b); R3.w = (W_)(c); R4.w = (W_)(d); \
82 JMP_(ENTRY_CODE(Sp[0]));
84 # define RET_NNPNNP(a,b,c,d,e,f) \
85 R1.w = (W_)(a); R2.w = (W_)(b); R3.w = (W_)(c); \
86 R4.w = (W_)(d); R5.w = (W_)(e); R6.w = (W_)(f); \
87 JMP_(ENTRY_CODE(Sp[0]));
89 #elif defined(REG_R7) || defined(REG_R6) || defined(REG_R5) || \
90 defined(REG_R4) || defined(REG_R3)
91 # error RET_n macros not defined for this setup.
93 /*------ 2 Registers available */
96 # define RET_P(a) R1.w = (W_)(a); JMP_(ENTRY_CODE(Sp[0]));
97 # define RET_N(a) RET_P(a)
99 # define RET_PP(a,b) R1.w = (W_)(a); R2.w = (W_)(b); \
100 JMP_(ENTRY_CODE(Sp[0]));
101 # define RET_NN(a,b) RET_PP(a,b)
102 # define RET_NP(a,b) RET_PP(a,b)
104 # define RET_PPP(a,b,c) \
105 R1.w = (W_)(a); R2.w = (W_)(b); Sp[-1] = (W_)(c); Sp -= 1; \
106 JMP_(ENTRY_CODE(Sp[1]));
107 # define RET_NNP(a,b,c) \
108 R1.w = (W_)(a); R2.w = (W_)(b); Sp[-1] = (W_)(c); Sp -= 1; \
109 JMP_(ENTRY_CODE(Sp[1]));
111 # define RET_NNNP(a,b,c,d) \
114 /* Sp[-3] = ARGTAG(1); */ \
118 JMP_(ENTRY_CODE(Sp[3]));
120 # define RET_NPNP(a,b,c,d) \
123 /* Sp[-3] = ARGTAG(1); */ \
127 JMP_(ENTRY_CODE(Sp[3]));
129 # define RET_NNPNNP(a,b,c,d,e,f) \
133 /* Sp[-5] = ARGTAG(1); */ \
135 /* Sp[-3] = ARGTAG(1); */ \
139 JMP_(ENTRY_CODE(Sp[6]));
141 /*------ 1 Register available */
142 #elif defined(REG_R1)
143 # define RET_P(a) R1.w = (W_)(a); JMP_(ENTRY_CODE(Sp[0]));
144 # define RET_N(a) RET_P(a)
146 # define RET_PP(a,b) R1.w = (W_)(a); Sp[-1] = (W_)(b); Sp -= 1; \
147 JMP_(ENTRY_CODE(Sp[1]));
148 # define RET_NN(a,b) R1.w = (W_)(a); Sp[-1] = (W_)(b); Sp -= 2; \
149 JMP_(ENTRY_CODE(Sp[2]));
150 # define RET_NP(a,b) RET_PP(a,b)
152 # define RET_PPP(a,b,c) \
153 R1.w = (W_)(a); Sp[-2] = (W_)(b); Sp[-1] = (W_)(c); Sp -= 2; \
154 JMP_(ENTRY_CODE(Sp[2]));
155 # define RET_NNP(a,b,c) \
156 R1.w = (W_)(a); Sp[-2] = (W_)(b); Sp[-1] = (W_)(c); Sp -= 3; \
157 JMP_(ENTRY_CODE(Sp[3]));
159 # define RET_NNNP(a,b,c,d) \
161 /* Sp[-5] = ARGTAG(1); */ \
163 /* Sp[-3] = ARGTAG(1); */ \
167 JMP_(ENTRY_CODE(Sp[5]));
169 # define RET_NPNP(a,b,c,d) \
172 /* Sp[-3] = ARGTAG(1); */ \
176 JMP_(ENTRY_CODE(Sp[4]));
178 # define RET_NNPNNP(a,b,c,d,e,f) \
182 /* Sp[-3] = ARGTAG(1); */ \
184 /* Sp[-5] = ARGTAG(1); */ \
187 /* Sp[-8] = ARGTAG(1); */ \
189 JMP_(ENTRY_CODE(Sp[8]));
191 #else /* 0 Regs available */
193 #define PUSH_P(o,x) Sp[-o] = (W_)(x)
196 #define PUSH_N(o,x) Sp[1-o] = (W_)(x); Sp[-o] = ARG_TAG(1);
198 #define PUSH_N(o,x) Sp[1-o] = (W_)(x);
201 #define PUSHED(m) Sp -= (m); JMP_(ENTRY_CODE(Sp[m]));
203 /* Here's how to construct these macros:
205 * N = number of N's in the name;
206 * P = number of P's in the name;
208 * while (nonNull(name)) {
209 * if (nextChar == 'P') {
220 # define RET_P(a) PUSH_P(1,a); PUSHED(1)
221 # define RET_N(a) PUSH_N(2,a); PUSHED(2)
223 # define RET_PP(a,b) PUSH_P(2,a); PUSH_P(1,b); PUSHED(2)
224 # define RET_NN(a,b) PUSH_N(4,a); PUSH_N(2,b); PUSHED(4)
225 # define RET_NP(a,b) PUSH_N(3,a); PUSH_P(1,b); PUSHED(3)
227 # define RET_PPP(a,b,c) PUSH_P(3,a); PUSH_P(2,b); PUSH_P(1,c); PUSHED(3)
228 # define RET_NNP(a,b,c) PUSH_N(5,a); PUSH_N(3,b); PUSH_P(1,c); PUSHED(5)
230 # define RET_NNNP(a,b,c,d) PUSH_N(7,a); PUSH_N(5,b); PUSH_N(3,c); PUSH_P(1,d); PUSHED(7)
231 # define RET_NPNP(a,b,c,d) PUSH_N(6,a); PUSH_P(4,b); PUSH_N(3,c); PUSH_P(1,d); PUSHED(6)
232 # define RET_NNPNNP(a,b,c,d,e,f) PUSH_N(10,a); PUSH_N(8,b); PUSH_P(6,c); PUSH_N(5,d); PUSH_N(3,e); PUSH_P(1,f); PUSHED(10)
236 /*-----------------------------------------------------------------------------
239 Basically just new*Array - the others are all inline macros.
241 The size arg is always passed in R1, and the result returned in R1.
243 The slow entry point is for returning from a heap check, the saved
244 size argument must be re-loaded from the stack.
245 -------------------------------------------------------------------------- */
247 /* for objects that are *less* than the size of a word, make sure we
248 * round up to the nearest word for the size of the array.
251 #define BYTES_TO_STGWORDS(n) ((n) + sizeof(W_) - 1)/sizeof(W_)
253 FN_(newByteArrayzh_fast)
255 W_ size, stuff_size, n;
258 MAYBE_GC(NO_PTRS,newByteArrayzh_fast);
260 stuff_size = BYTES_TO_STGWORDS(n);
261 size = sizeofW(StgArrWords)+ stuff_size;
262 p = (StgArrWords *)RET_STGCALL1(P_,allocate,size);
263 TICK_ALLOC_PRIM(sizeofW(StgArrWords),stuff_size,0);
264 SET_HDR(p, &stg_ARR_WORDS_info, CCCS);
265 p->words = stuff_size;
266 TICK_RET_UNBOXED_TUP(1)
271 FN_(newPinnedByteArrayzh_fast)
273 W_ size, stuff_size, n;
276 MAYBE_GC(NO_PTRS,newPinnedByteArrayzh_fast);
278 stuff_size = BYTES_TO_STGWORDS(n);
280 // We want an 8-byte aligned array. allocatePinned() gives us
281 // 8-byte aligned memory by default, but we want to align the
282 // *goods* inside the ArrWords object, so we have to check the
283 // size of the ArrWords header and adjust our size accordingly.
284 size = sizeofW(StgArrWords)+ stuff_size;
285 if ((sizeof(StgArrWords) & 7) != 0) {
289 p = (StgArrWords *)RET_STGCALL1(P_,allocatePinned,size);
290 TICK_ALLOC_PRIM(sizeofW(StgArrWords),stuff_size,0);
292 // Again, if the ArrWords header isn't a multiple of 8 bytes, we
293 // have to push the object forward one word so that the goods
294 // fall on an 8-byte boundary.
295 if ((sizeof(StgArrWords) & 7) != 0) {
299 SET_HDR(p, &stg_ARR_WORDS_info, CCCS);
300 p->words = stuff_size;
301 TICK_RET_UNBOXED_TUP(1)
314 MAYBE_GC(R2_PTR,newArrayzh_fast);
316 size = sizeofW(StgMutArrPtrs) + n;
317 arr = (StgMutArrPtrs *)RET_STGCALL1(P_, allocate, size);
318 TICK_ALLOC_PRIM(sizeofW(StgMutArrPtrs), n, 0);
320 SET_HDR(arr,&stg_MUT_ARR_PTRS_info,CCCS);
324 for (p = (P_)arr + sizeofW(StgMutArrPtrs);
325 p < (P_)arr + size; p++) {
329 TICK_RET_UNBOXED_TUP(1);
334 FN_(newMutVarzh_fast)
337 /* Args: R1.p = initialisation value */
340 HP_CHK_GEN_TICKY(sizeofW(StgMutVar), R1_PTR, newMutVarzh_fast,);
341 TICK_ALLOC_PRIM(sizeofW(StgHeader)+1,1, 0); /* hack, dependent on rep. */
342 CCS_ALLOC(CCCS,sizeofW(StgMutVar));
344 mv = (StgMutVar *)(Hp-sizeofW(StgMutVar)+1);
345 SET_HDR(mv,&stg_MUT_VAR_info,CCCS);
348 TICK_RET_UNBOXED_TUP(1);
353 FN_(atomicModifyMutVarzh_fast)
356 StgClosure *z, *x, *y, *r;
358 /* Args: R1.p :: MutVar#, R2.p :: a -> (a,b) */
360 /* If x is the current contents of the MutVar#, then
361 We want to make the new contents point to
365 and the return value is
369 obviously we can share (f x).
371 z = [stg_ap_2 f x] (max (HS + 2) MIN_UPD_SIZE)
372 y = [stg_sel_0 z] (max (HS + 1) MIN_UPD_SIZE)
373 r = [stg_sel_1 z] (max (HS + 1) MIN_UPD_SIZE)
376 #define THUNK_SIZE(n) (sizeofW(StgHeader) + stg_max((n), MIN_UPD_SIZE))
377 #define SIZE (THUNK_SIZE(2) + THUNK_SIZE(1) + THUNK_SIZE(1))
379 HP_CHK_GEN_TICKY(SIZE, R1_PTR|R2_PTR, atomicModifyMutVarzh_fast,);
380 CCS_ALLOC(CCCS,SIZE);
382 x = ((StgMutVar *)R1.cl)->var;
384 TICK_ALLOC_UP_THK(2,0); // XXX
385 z = (StgClosure *) Hp - THUNK_SIZE(2) + 1;
386 SET_HDR(z, &stg_ap_2_upd_info, CCCS);
387 z->payload[0] = R2.cl;
390 TICK_ALLOC_UP_THK(1,1); // XXX
391 y = (StgClosure *) (StgPtr)z - THUNK_SIZE(1);
392 SET_HDR(y, &stg_sel_0_upd_info, CCCS);
395 ((StgMutVar *)R1.cl)->var = y;
397 TICK_ALLOC_UP_THK(1,1); // XXX
398 r = (StgClosure *) (StgPtr)y - THUNK_SIZE(1);
399 SET_HDR(r, &stg_sel_1_upd_info, CCCS);
403 JMP_(ENTRY_CODE(Sp[0]));
407 /* -----------------------------------------------------------------------------
408 Foreign Object Primitives
409 -------------------------------------------------------------------------- */
411 FN_(mkForeignObjzh_fast)
413 /* R1.p = ptr to foreign object,
415 StgForeignObj *result;
418 HP_CHK_GEN_TICKY(sizeofW(StgForeignObj), NO_PTRS, mkForeignObjzh_fast,);
419 TICK_ALLOC_PRIM(sizeofW(StgHeader),
420 sizeofW(StgForeignObj)-sizeofW(StgHeader), 0);
421 CCS_ALLOC(CCCS,sizeofW(StgForeignObj)); /* ccs prof */
423 result = (StgForeignObj *) (Hp + 1 - sizeofW(StgForeignObj));
424 SET_HDR(result,&stg_FOREIGN_info,CCCS);
427 /* returns (# s#, ForeignObj# #) */
428 TICK_RET_UNBOXED_TUP(1);
433 /* These two are out-of-line for the benefit of the NCG */
434 FN_(unsafeThawArrayzh_fast)
437 SET_INFO((StgClosure *)R1.cl,&stg_MUT_ARR_PTRS_info);
438 recordMutable((StgMutClosure*)R1.cl);
440 TICK_RET_UNBOXED_TUP(1);
445 /* -----------------------------------------------------------------------------
446 Weak Pointer Primitives
447 -------------------------------------------------------------------------- */
453 R3.p = finalizer (or NULL)
459 R3.cl = &stg_NO_FINALIZER_closure;
462 HP_CHK_GEN_TICKY(sizeofW(StgWeak),R1_PTR|R2_PTR|R3_PTR, mkWeakzh_fast,);
463 TICK_ALLOC_PRIM(sizeofW(StgHeader)+1, // +1 is for the link field
464 sizeofW(StgWeak)-sizeofW(StgHeader)-1, 0);
465 CCS_ALLOC(CCCS,sizeofW(StgWeak)); /* ccs prof */
467 w = (StgWeak *) (Hp + 1 - sizeofW(StgWeak));
468 SET_HDR(w, &stg_WEAK_info, CCCS);
472 w->finalizer = R3.cl;
474 w->link = weak_ptr_list;
476 IF_DEBUG(weak, fprintf(stderr,"New weak pointer at %p\n",w));
478 TICK_RET_UNBOXED_TUP(1);
483 FN_(finalizzeWeakzh_fast)
490 TICK_RET_UNBOXED_TUP(0);
491 w = (StgDeadWeak *)R1.p;
494 if (w->header.info == &stg_DEAD_WEAK_info) {
495 RET_NP(0,&stg_NO_FINALIZER_closure);
501 // A weak pointer is inherently used, so we do not need to call
502 // LDV_recordDead_FILL_SLOP_DYNAMIC():
503 // LDV_recordDead_FILL_SLOP_DYNAMIC((StgClosure *)w);
504 // or, LDV_recordDead():
505 // LDV_recordDead((StgClosure *)w, sizeofW(StgWeak) - sizeofW(StgProfHeader));
506 // Furthermore, when PROFILING is turned on, dead weak pointers are exactly as
507 // large as weak pointers, so there is no need to fill the slop, either.
508 // See stg_DEAD_WEAK_info in StgMiscClosures.hc.
511 // Todo: maybe use SET_HDR() and remove LDV_recordCreate()?
513 w->header.info = &stg_DEAD_WEAK_info;
516 LDV_recordCreate((StgClosure *)w);
518 f = ((StgWeak *)w)->finalizer;
519 w->link = ((StgWeak *)w)->link;
521 /* return the finalizer */
522 if (f == &stg_NO_FINALIZER_closure) {
523 RET_NP(0,&stg_NO_FINALIZER_closure);
530 FN_(deRefWeakzh_fast)
532 /* R1.p = weak ptr */
538 if (w->header.info == &stg_WEAK_info) {
540 val = (P_)((StgWeak *)w)->value;
549 /* -----------------------------------------------------------------------------
550 Arbitrary-precision Integer operations.
551 -------------------------------------------------------------------------- */
553 FN_(int2Integerzh_fast)
555 /* arguments: R1 = Int# */
557 I_ val, s; /* to avoid aliasing */
558 StgArrWords* p; /* address of array result */
562 HP_CHK_GEN_TICKY(sizeofW(StgArrWords)+1, NO_PTRS, int2Integerzh_fast,);
563 TICK_ALLOC_PRIM(sizeofW(StgArrWords),1,0);
564 CCS_ALLOC(CCCS,sizeofW(StgArrWords)+1); /* ccs prof */
566 p = (StgArrWords *)Hp - 1;
567 SET_ARR_HDR(p, &stg_ARR_WORDS_info, CCCS, 1);
569 /* mpz_set_si is inlined here, makes things simpler */
573 } else if (val > 0) {
580 /* returns (# size :: Int#,
584 TICK_RET_UNBOXED_TUP(2);
589 FN_(word2Integerzh_fast)
591 /* arguments: R1 = Word# */
593 W_ val; /* to avoid aliasing */
595 StgArrWords* p; /* address of array result */
599 HP_CHK_GEN_TICKY(sizeofW(StgArrWords)+1, NO_PTRS, word2Integerzh_fast,)
600 TICK_ALLOC_PRIM(sizeofW(StgArrWords),1,0);
601 CCS_ALLOC(CCCS,sizeofW(StgArrWords)+1); /* ccs prof */
603 p = (StgArrWords *)Hp - 1;
604 SET_ARR_HDR(p, &stg_ARR_WORDS_info, CCCS, 1);
613 /* returns (# size :: Int#,
617 TICK_RET_UNBOXED_TUP(2);
624 * 'long long' primops for converting to/from Integers.
627 #ifdef SUPPORT_LONG_LONGS
629 FN_(int64ToIntegerzh_fast)
631 /* arguments: L1 = Int64# */
633 StgInt64 val; /* to avoid aliasing */
635 I_ s, neg, words_needed;
636 StgArrWords* p; /* address of array result */
642 if ( val >= 0x100000000LL || val <= -0x100000000LL ) {
645 /* minimum is one word */
648 HP_CHK_GEN_TICKY(sizeofW(StgArrWords)+words_needed, NO_PTRS, int64ToIntegerzh_fast,)
649 TICK_ALLOC_PRIM(sizeofW(StgArrWords),words_needed,0);
650 CCS_ALLOC(CCCS,sizeofW(StgArrWords)+words_needed); /* ccs prof */
652 p = (StgArrWords *)(Hp-words_needed+1) - 1;
653 SET_ARR_HDR(p, &stg_ARR_WORDS_info, CCCS, words_needed);
660 hi = (W_)((LW_)val / 0x100000000ULL);
662 if ( words_needed == 2 ) {
666 } else if ( val != 0 ) {
669 } else /* val==0 */ {
672 s = ( neg ? -s : s );
674 /* returns (# size :: Int#,
678 TICK_RET_UNBOXED_TUP(2);
683 FN_(word64ToIntegerzh_fast)
685 /* arguments: L1 = Word64# */
687 StgWord64 val; /* to avoid aliasing */
690 StgArrWords* p; /* address of array result */
694 if ( val >= 0x100000000ULL ) {
699 HP_CHK_GEN_TICKY(sizeofW(StgArrWords)+words_needed, NO_PTRS, word64ToIntegerzh_fast,)
700 TICK_ALLOC_PRIM(sizeofW(StgArrWords),words_needed,0);
701 CCS_ALLOC(CCCS,sizeofW(StgArrWords)+words_needed); /* ccs prof */
703 p = (StgArrWords *)(Hp-words_needed+1) - 1;
704 SET_ARR_HDR(p, &stg_ARR_WORDS_info, CCCS, words_needed);
706 hi = (W_)((LW_)val / 0x100000000ULL);
707 if ( val >= 0x100000000ULL ) {
711 } else if ( val != 0 ) {
714 } else /* val==0 */ {
718 /* returns (# size :: Int#,
722 TICK_RET_UNBOXED_TUP(2);
728 #endif /* SUPPORT_LONG_LONGS */
730 /* ToDo: this is shockingly inefficient */
732 #define GMP_TAKE2_RET1(name,mp_fun) \
735 MP_INT arg1, arg2, result; \
741 /* call doYouWantToGC() */ \
742 MAYBE_GC(R2_PTR | R4_PTR, name); \
744 d1 = (StgArrWords *)R2.p; \
746 d2 = (StgArrWords *)R4.p; \
749 arg1._mp_alloc = d1->words; \
750 arg1._mp_size = (s1); \
751 arg1._mp_d = (unsigned long int *) (BYTE_ARR_CTS(d1)); \
752 arg2._mp_alloc = d2->words; \
753 arg2._mp_size = (s2); \
754 arg2._mp_d = (unsigned long int *) (BYTE_ARR_CTS(d2)); \
756 STGCALL1(mpz_init,&result); \
758 /* Perform the operation */ \
759 STGCALL3(mp_fun,&result,&arg1,&arg2); \
761 TICK_RET_UNBOXED_TUP(2); \
762 RET_NP(result._mp_size, \
763 result._mp_d-sizeofW(StgArrWords)); \
767 #define GMP_TAKE1_RET1(name,mp_fun) \
770 MP_INT arg1, result; \
775 /* call doYouWantToGC() */ \
776 MAYBE_GC(R2_PTR, name); \
778 d1 = (StgArrWords *)R2.p; \
781 arg1._mp_alloc = d1->words; \
782 arg1._mp_size = (s1); \
783 arg1._mp_d = (unsigned long int *) (BYTE_ARR_CTS(d1)); \
785 STGCALL1(mpz_init,&result); \
787 /* Perform the operation */ \
788 STGCALL2(mp_fun,&result,&arg1); \
790 TICK_RET_UNBOXED_TUP(2); \
791 RET_NP(result._mp_size, \
792 result._mp_d-sizeofW(StgArrWords)); \
796 #define GMP_TAKE2_RET2(name,mp_fun) \
799 MP_INT arg1, arg2, result1, result2; \
805 /* call doYouWantToGC() */ \
806 MAYBE_GC(R2_PTR | R4_PTR, name); \
808 d1 = (StgArrWords *)R2.p; \
810 d2 = (StgArrWords *)R4.p; \
813 arg1._mp_alloc = d1->words; \
814 arg1._mp_size = (s1); \
815 arg1._mp_d = (unsigned long int *) (BYTE_ARR_CTS(d1)); \
816 arg2._mp_alloc = d2->words; \
817 arg2._mp_size = (s2); \
818 arg2._mp_d = (unsigned long int *) (BYTE_ARR_CTS(d2)); \
820 STGCALL1(mpz_init,&result1); \
821 STGCALL1(mpz_init,&result2); \
823 /* Perform the operation */ \
824 STGCALL4(mp_fun,&result1,&result2,&arg1,&arg2); \
826 TICK_RET_UNBOXED_TUP(4); \
827 RET_NPNP(result1._mp_size, \
828 result1._mp_d-sizeofW(StgArrWords), \
830 result2._mp_d-sizeofW(StgArrWords)); \
834 GMP_TAKE2_RET1(plusIntegerzh_fast, mpz_add);
835 GMP_TAKE2_RET1(minusIntegerzh_fast, mpz_sub);
836 GMP_TAKE2_RET1(timesIntegerzh_fast, mpz_mul);
837 GMP_TAKE2_RET1(gcdIntegerzh_fast, mpz_gcd);
838 GMP_TAKE2_RET1(quotIntegerzh_fast, mpz_tdiv_q);
839 GMP_TAKE2_RET1(remIntegerzh_fast, mpz_tdiv_r);
840 GMP_TAKE2_RET1(divExactIntegerzh_fast, mpz_divexact);
841 GMP_TAKE2_RET1(andIntegerzh_fast, mpz_and);
842 GMP_TAKE2_RET1(orIntegerzh_fast, mpz_ior);
843 GMP_TAKE2_RET1(xorIntegerzh_fast, mpz_xor);
844 GMP_TAKE1_RET1(complementIntegerzh_fast, mpz_com);
846 GMP_TAKE2_RET2(quotRemIntegerzh_fast, mpz_tdiv_qr);
847 GMP_TAKE2_RET2(divModIntegerzh_fast, mpz_fdiv_qr);
852 /* R1 = the first Int#; R2 = the second Int# */
856 aa = (mp_limb_t)(R1.i);
857 r = RET_STGCALL3(StgInt, mpn_gcd_1, (mp_limb_t *)(&aa), 1, (mp_limb_t)(R2.i));
860 /* Result parked in R1, return via info-pointer at TOS */
861 JMP_(ENTRY_CODE(Sp[0]));
865 FN_(gcdIntegerIntzh_fast)
867 /* R1 = s1; R2 = d1; R3 = the int */
870 r = RET_STGCALL3(StgInt,mpn_gcd_1,(mp_limb_t *)(BYTE_ARR_CTS(R2.p)), R1.i, R3.i);
873 /* Result parked in R1, return via info-pointer at TOS */
874 JMP_(ENTRY_CODE(Sp[0]));
878 FN_(cmpIntegerIntzh_fast)
880 /* R1 = s1; R2 = d1; R3 = the int */
891 // paraphrased from mpz_cmp_si() in the GMP sources
894 } else if (v_digit < 0) {
899 if (usize != vsize) {
900 R1.i = usize - vsize; JMP_(ENTRY_CODE(Sp[0]));
904 R1.i = 0; JMP_(ENTRY_CODE(Sp[0]));
907 u_digit = *(mp_limb_t *)(BYTE_ARR_CTS(R2.p));
909 if (u_digit == (mp_limb_t) (unsigned long) v_digit) {
910 R1.i = 0; JMP_(ENTRY_CODE(Sp[0]));
913 if (u_digit > (mp_limb_t) (unsigned long) v_digit) {
919 JMP_(ENTRY_CODE(Sp[0]));
923 FN_(cmpIntegerzh_fast)
925 /* R1 = s1; R2 = d1; R3 = s2; R4 = d2 */
933 // paraphrased from mpz_cmp() in the GMP sources
937 if (usize != vsize) {
938 R1.i = usize - vsize; JMP_(ENTRY_CODE(Sp[0]));
942 R1.i = 0; JMP_(ENTRY_CODE(Sp[0]));
947 up = BYTE_ARR_CTS(R2.p);
948 vp = BYTE_ARR_CTS(R4.p);
950 cmp = RET_STGCALL3(I_, mpn_cmp, (mp_limb_t *)up, (mp_limb_t *)vp, size);
953 R1.i = 0; JMP_(ENTRY_CODE(Sp[0]));
956 if ((cmp < 0) == (usize < 0)) {
961 /* Result parked in R1, return via info-pointer at TOS */
962 JMP_(ENTRY_CODE(Sp[0]));
966 FN_(integer2Intzh_fast)
975 r = ((mp_limb_t *) (BYTE_ARR_CTS(R2.p)))[0];
978 /* Result parked in R1, return via info-pointer at TOS */
980 JMP_(ENTRY_CODE(Sp[0]));
984 FN_(integer2Wordzh_fast)
994 r = ((mp_limb_t *) (BYTE_ARR_CTS(R2.p)))[0];
997 /* Result parked in R1, return via info-pointer at TOS */
999 JMP_(ENTRY_CODE(Sp[0]));
1004 FN_(decodeFloatzh_fast)
1012 /* arguments: F1 = Float# */
1015 HP_CHK_GEN_TICKY(sizeofW(StgArrWords)+1, NO_PTRS, decodeFloatzh_fast,);
1016 TICK_ALLOC_PRIM(sizeofW(StgArrWords),1,0);
1017 CCS_ALLOC(CCCS,sizeofW(StgArrWords)+1); /* ccs prof */
1019 /* Be prepared to tell Lennart-coded __decodeFloat */
1020 /* where mantissa._mp_d can be put (it does not care about the rest) */
1021 p = (StgArrWords *)Hp - 1;
1022 SET_ARR_HDR(p,&stg_ARR_WORDS_info,CCCS,1)
1023 mantissa._mp_d = (void *)BYTE_ARR_CTS(p);
1025 /* Perform the operation */
1026 STGCALL3(__decodeFloat,&mantissa,&exponent,arg);
1028 /* returns: (Int# (expn), Int#, ByteArray#) */
1029 TICK_RET_UNBOXED_TUP(3);
1030 RET_NNP(exponent,mantissa._mp_size,p);
1034 #define DOUBLE_MANTISSA_SIZE (sizeofW(StgDouble))
1035 #define ARR_SIZE (sizeofW(StgArrWords) + DOUBLE_MANTISSA_SIZE)
1037 FN_(decodeDoublezh_fast)
1044 /* arguments: D1 = Double# */
1047 HP_CHK_GEN_TICKY(ARR_SIZE, NO_PTRS, decodeDoublezh_fast,);
1048 TICK_ALLOC_PRIM(sizeofW(StgArrWords),DOUBLE_MANTISSA_SIZE,0);
1049 CCS_ALLOC(CCCS,ARR_SIZE); /* ccs prof */
1051 /* Be prepared to tell Lennart-coded __decodeDouble */
1052 /* where mantissa.d can be put (it does not care about the rest) */
1053 p = (StgArrWords *)(Hp-ARR_SIZE+1);
1054 SET_ARR_HDR(p, &stg_ARR_WORDS_info, CCCS, DOUBLE_MANTISSA_SIZE);
1055 mantissa._mp_d = (void *)BYTE_ARR_CTS(p);
1057 /* Perform the operation */
1058 STGCALL3(__decodeDouble,&mantissa,&exponent,arg);
1060 /* returns: (Int# (expn), Int#, ByteArray#) */
1061 TICK_RET_UNBOXED_TUP(3);
1062 RET_NNP(exponent,mantissa._mp_size,p);
1066 /* -----------------------------------------------------------------------------
1067 * Concurrency primitives
1068 * -------------------------------------------------------------------------- */
1073 /* args: R1 = closure to spark */
1075 MAYBE_GC(R1_PTR, forkzh_fast);
1077 /* create it right now, return ThreadID in R1 */
1078 R1.t = RET_STGCALL2(StgTSO *, createIOThread,
1079 RtsFlags.GcFlags.initialStkSize, R1.cl);
1080 STGCALL1(scheduleThread, R1.t);
1082 /* switch at the earliest opportunity */
1089 FN_(forkProcesszh_fast)
1097 R1.i = RET_STGCALL1(StgInt, forkProcess, CurrentTSO);
1099 JMP_(ENTRY_CODE(Sp[0]));
1107 JMP_(stg_yield_noregs);
1111 FN_(myThreadIdzh_fast)
1115 RET_P((P_)CurrentTSO);
1119 FN_(labelThreadzh_fast)
1126 STGCALL2(labelThread,R1.p,(char *)R2.p);
1128 JMP_(ENTRY_CODE(Sp[0]));
1133 /* -----------------------------------------------------------------------------
1136 * take & putMVar work as follows. Firstly, an important invariant:
1138 * If the MVar is full, then the blocking queue contains only
1139 * threads blocked on putMVar, and if the MVar is empty then the
1140 * blocking queue contains only threads blocked on takeMVar.
1143 * MVar empty : then add ourselves to the blocking queue
1144 * MVar full : remove the value from the MVar, and
1145 * blocking queue empty : return
1146 * blocking queue non-empty : perform the first blocked putMVar
1147 * from the queue, and wake up the
1148 * thread (MVar is now full again)
1150 * putMVar is just the dual of the above algorithm.
1152 * How do we "perform a putMVar"? Well, we have to fiddle around with
1153 * the stack of the thread waiting to do the putMVar. See
1154 * stg_block_putmvar and stg_block_takemvar in HeapStackCheck.c for
1155 * the stack layout, and the PerformPut and PerformTake macros below.
1157 * It is important that a blocked take or put is woken up with the
1158 * take/put already performed, because otherwise there would be a
1159 * small window of vulnerability where the thread could receive an
1160 * exception and never perform its take or put, and we'd end up with a
1163 * -------------------------------------------------------------------------- */
1165 FN_(isEmptyMVarzh_fast)
1167 /* args: R1 = MVar closure */
1170 r = (I_)((GET_INFO((StgMVar*)(R1.p))) == &stg_EMPTY_MVAR_info);
1183 HP_CHK_GEN_TICKY(sizeofW(StgMVar), NO_PTRS, newMVarzh_fast,);
1184 TICK_ALLOC_PRIM(sizeofW(StgMutVar)-1, // consider head,tail,link as admin wds
1186 CCS_ALLOC(CCCS,sizeofW(StgMVar)); /* ccs prof */
1188 mvar = (StgMVar *) (Hp - sizeofW(StgMVar) + 1);
1189 SET_HDR(mvar,&stg_EMPTY_MVAR_info,CCCS);
1190 mvar->head = mvar->tail = (StgTSO *)&stg_END_TSO_QUEUE_closure;
1191 mvar->value = (StgClosure *)&stg_END_TSO_QUEUE_closure;
1193 TICK_RET_UNBOXED_TUP(1);
1198 /* If R1 isn't available, pass it on the stack */
1200 #define PerformTake(tso, value) ({ \
1201 (tso)->sp[1] = (W_)value; \
1202 (tso)->sp[0] = (W_)&stg_gc_unpt_r1_info; \
1205 #define PerformTake(tso, value) ({ \
1206 (tso)->sp[1] = (W_)value; \
1207 (tso)->sp[0] = (W_)&stg_ut_1_0_unreg_info; \
1212 #define PerformPut(tso) ({ \
1213 StgClosure *val = (StgClosure *)(tso)->sp[2]; \
1214 (tso)->sp[2] = (W_)&stg_gc_noregs_info; \
1219 FN_(takeMVarzh_fast)
1223 const StgInfoTable *info;
1226 /* args: R1 = MVar closure */
1228 mvar = (StgMVar *)R1.p;
1231 info = LOCK_CLOSURE(mvar);
1233 info = GET_INFO(mvar);
1236 /* If the MVar is empty, put ourselves on its blocking queue,
1237 * and wait until we're woken up.
1239 if (info == &stg_EMPTY_MVAR_info) {
1240 if (mvar->head == (StgTSO *)&stg_END_TSO_QUEUE_closure) {
1241 mvar->head = CurrentTSO;
1243 mvar->tail->link = CurrentTSO;
1245 CurrentTSO->link = (StgTSO *)&stg_END_TSO_QUEUE_closure;
1246 CurrentTSO->why_blocked = BlockedOnMVar;
1247 CurrentTSO->block_info.closure = (StgClosure *)mvar;
1248 mvar->tail = CurrentTSO;
1251 /* unlock the MVar */
1252 mvar->header.info = &stg_EMPTY_MVAR_info;
1254 JMP_(stg_block_takemvar);
1257 /* we got the value... */
1260 if (mvar->head != (StgTSO *)&stg_END_TSO_QUEUE_closure) {
1261 /* There are putMVar(s) waiting...
1262 * wake up the first thread on the queue
1264 ASSERT(mvar->head->why_blocked == BlockedOnMVar);
1266 /* actually perform the putMVar for the thread that we just woke up */
1267 mvar->value = PerformPut(mvar->head);
1269 #if defined(GRAN) || defined(PAR)
1270 /* ToDo: check 2nd arg (mvar) is right */
1271 mvar->head = RET_STGCALL2(StgTSO *,unblockOne,mvar->head,mvar);
1273 mvar->head = RET_STGCALL1(StgTSO *,unblockOne,mvar->head);
1275 if (mvar->head == (StgTSO *)&stg_END_TSO_QUEUE_closure) {
1276 mvar->tail = (StgTSO *)&stg_END_TSO_QUEUE_closure;
1279 /* unlock in the SMP case */
1280 SET_INFO(mvar,&stg_FULL_MVAR_info);
1282 TICK_RET_UNBOXED_TUP(1);
1285 /* No further putMVars, MVar is now empty */
1287 /* do this last... we might have locked the MVar in the SMP case,
1288 * and writing the info pointer will unlock it.
1290 SET_INFO(mvar,&stg_EMPTY_MVAR_info);
1291 mvar->value = (StgClosure *)&stg_END_TSO_QUEUE_closure;
1292 TICK_RET_UNBOXED_TUP(1);
1298 FN_(tryTakeMVarzh_fast)
1302 const StgInfoTable *info;
1305 /* args: R1 = MVar closure */
1307 mvar = (StgMVar *)R1.p;
1310 info = LOCK_CLOSURE(mvar);
1312 info = GET_INFO(mvar);
1315 if (info == &stg_EMPTY_MVAR_info) {
1318 /* unlock the MVar */
1319 SET_INFO(mvar,&stg_EMPTY_MVAR_info);
1322 /* HACK: we need a pointer to pass back,
1323 * so we abuse NO_FINALIZER_closure
1325 RET_NP(0, &stg_NO_FINALIZER_closure);
1328 /* we got the value... */
1331 if (mvar->head != (StgTSO *)&stg_END_TSO_QUEUE_closure) {
1332 /* There are putMVar(s) waiting...
1333 * wake up the first thread on the queue
1335 ASSERT(mvar->head->why_blocked == BlockedOnMVar);
1337 /* actually perform the putMVar for the thread that we just woke up */
1338 mvar->value = PerformPut(mvar->head);
1340 #if defined(GRAN) || defined(PAR)
1341 /* ToDo: check 2nd arg (mvar) is right */
1342 mvar->head = RET_STGCALL2(StgTSO *,unblockOne,mvar->head,mvar);
1344 mvar->head = RET_STGCALL1(StgTSO *,unblockOne,mvar->head);
1346 if (mvar->head == (StgTSO *)&stg_END_TSO_QUEUE_closure) {
1347 mvar->tail = (StgTSO *)&stg_END_TSO_QUEUE_closure;
1350 /* unlock in the SMP case */
1351 SET_INFO(mvar,&stg_FULL_MVAR_info);
1354 /* No further putMVars, MVar is now empty */
1355 mvar->value = (StgClosure *)&stg_END_TSO_QUEUE_closure;
1357 /* do this last... we might have locked the MVar in the SMP case,
1358 * and writing the info pointer will unlock it.
1360 SET_INFO(mvar,&stg_EMPTY_MVAR_info);
1363 TICK_RET_UNBOXED_TUP(1);
1371 const StgInfoTable *info;
1374 /* args: R1 = MVar, R2 = value */
1376 mvar = (StgMVar *)R1.p;
1379 info = LOCK_CLOSURE(mvar);
1381 info = GET_INFO(mvar);
1384 if (info == &stg_FULL_MVAR_info) {
1385 if (mvar->head == (StgTSO *)&stg_END_TSO_QUEUE_closure) {
1386 mvar->head = CurrentTSO;
1388 mvar->tail->link = CurrentTSO;
1390 CurrentTSO->link = (StgTSO *)&stg_END_TSO_QUEUE_closure;
1391 CurrentTSO->why_blocked = BlockedOnMVar;
1392 CurrentTSO->block_info.closure = (StgClosure *)mvar;
1393 mvar->tail = CurrentTSO;
1396 /* unlock the MVar */
1397 SET_INFO(mvar,&stg_FULL_MVAR_info);
1399 JMP_(stg_block_putmvar);
1402 if (mvar->head != (StgTSO *)&stg_END_TSO_QUEUE_closure) {
1403 /* There are takeMVar(s) waiting: wake up the first one
1405 ASSERT(mvar->head->why_blocked == BlockedOnMVar);
1407 /* actually perform the takeMVar */
1408 PerformTake(mvar->head, R2.cl);
1410 #if defined(GRAN) || defined(PAR)
1411 /* ToDo: check 2nd arg (mvar) is right */
1412 mvar->head = RET_STGCALL2(StgTSO *,unblockOne,mvar->head,mvar);
1414 mvar->head = RET_STGCALL1(StgTSO *,unblockOne,mvar->head);
1416 if (mvar->head == (StgTSO *)&stg_END_TSO_QUEUE_closure) {
1417 mvar->tail = (StgTSO *)&stg_END_TSO_QUEUE_closure;
1420 /* unlocks the MVar in the SMP case */
1421 SET_INFO(mvar,&stg_EMPTY_MVAR_info);
1423 JMP_(ENTRY_CODE(Sp[0]));
1425 /* No further takes, the MVar is now full. */
1426 mvar->value = R2.cl;
1427 /* unlocks the MVar in the SMP case */
1428 SET_INFO(mvar,&stg_FULL_MVAR_info);
1429 JMP_(ENTRY_CODE(Sp[0]));
1432 /* ToDo: yield afterward for better communication performance? */
1436 FN_(tryPutMVarzh_fast)
1439 const StgInfoTable *info;
1442 /* args: R1 = MVar, R2 = value */
1444 mvar = (StgMVar *)R1.p;
1447 info = LOCK_CLOSURE(mvar);
1449 info = GET_INFO(mvar);
1452 if (info == &stg_FULL_MVAR_info) {
1455 /* unlock the MVar */
1456 mvar->header.info = &stg_FULL_MVAR_info;
1462 if (mvar->head != (StgTSO *)&stg_END_TSO_QUEUE_closure) {
1463 /* There are takeMVar(s) waiting: wake up the first one
1465 ASSERT(mvar->head->why_blocked == BlockedOnMVar);
1467 /* actually perform the takeMVar */
1468 PerformTake(mvar->head, R2.cl);
1470 #if defined(GRAN) || defined(PAR)
1471 /* ToDo: check 2nd arg (mvar) is right */
1472 mvar->head = RET_STGCALL2(StgTSO *,unblockOne,mvar->head,mvar);
1474 mvar->head = RET_STGCALL1(StgTSO *,unblockOne,mvar->head);
1476 if (mvar->head == (StgTSO *)&stg_END_TSO_QUEUE_closure) {
1477 mvar->tail = (StgTSO *)&stg_END_TSO_QUEUE_closure;
1480 /* unlocks the MVar in the SMP case */
1481 SET_INFO(mvar,&stg_EMPTY_MVAR_info);
1483 JMP_(ENTRY_CODE(Sp[0]));
1485 /* No further takes, the MVar is now full. */
1486 mvar->value = R2.cl;
1487 /* unlocks the MVar in the SMP case */
1488 SET_INFO(mvar,&stg_FULL_MVAR_info);
1489 JMP_(ENTRY_CODE(Sp[0]));
1492 /* ToDo: yield afterward for better communication performance? */
1496 /* -----------------------------------------------------------------------------
1497 Stable pointer primitives
1498 ------------------------------------------------------------------------- */
1500 FN_(makeStableNamezh_fast)
1503 StgStableName *sn_obj;
1506 HP_CHK_GEN_TICKY(sizeofW(StgStableName), R1_PTR, makeStableNamezh_fast,);
1507 TICK_ALLOC_PRIM(sizeofW(StgHeader),
1508 sizeofW(StgStableName)-sizeofW(StgHeader), 0);
1509 CCS_ALLOC(CCCS,sizeofW(StgStableName)); /* ccs prof */
1511 index = RET_STGCALL1(StgWord,lookupStableName,R1.p);
1513 /* Is there already a StableName for this heap object? */
1514 if (stable_ptr_table[index].sn_obj == NULL) {
1515 sn_obj = (StgStableName *) (Hp - sizeofW(StgStableName) + 1);
1516 SET_HDR(sn_obj,&stg_STABLE_NAME_info,CCCS);
1518 stable_ptr_table[index].sn_obj = (StgClosure *)sn_obj;
1520 (StgClosure *)sn_obj = stable_ptr_table[index].sn_obj;
1523 TICK_RET_UNBOXED_TUP(1);
1528 FN_(makeStablePtrzh_fast)
1533 MAYBE_GC(R1_PTR, makeStablePtrzh_fast);
1534 sp = RET_STGCALL1(StgStablePtr,getStablePtr,R1.p);
1539 FN_(deRefStablePtrzh_fast)
1541 /* Args: R1 = the stable ptr */
1545 sp = (StgStablePtr)R1.w;
1546 r = stable_ptr_table[(StgWord)sp].addr;
1551 /* -----------------------------------------------------------------------------
1552 Bytecode object primitives
1553 ------------------------------------------------------------------------- */
1565 HP_CHK_GEN_TICKY(sizeofW(StgBCO),R1_PTR|R2_PTR|R3_PTR|R4_PTR, newBCOzh_fast,);
1566 TICK_ALLOC_PRIM(sizeofW(StgHeader), sizeofW(StgBCO)-sizeofW(StgHeader), 0);
1567 CCS_ALLOC(CCCS,sizeofW(StgBCO)); /* ccs prof */
1568 bco = (StgBCO *) (Hp + 1 - sizeofW(StgBCO));
1569 SET_HDR(bco, &stg_BCO_info, CCCS);
1571 bco->instrs = (StgArrWords*)R1.cl;
1572 bco->literals = (StgArrWords*)R2.cl;
1573 bco->ptrs = (StgMutArrPtrs*)R3.cl;
1574 bco->itbls = (StgArrWords*)R4.cl;
1576 TICK_RET_UNBOXED_TUP(1);
1581 FN_(mkApUpd0zh_fast)
1583 /* R1.p = the fn for the AP_UPD
1587 HP_CHK_GEN_TICKY(AP_sizeW(0), R1_PTR, mkApUpd0zh_fast,);
1588 TICK_ALLOC_PRIM(sizeofW(StgHeader), AP_sizeW(0)-sizeofW(StgHeader), 0);
1589 CCS_ALLOC(CCCS,AP_sizeW(0)); /* ccs prof */
1590 ap = (StgAP_UPD *) (Hp + 1 - AP_sizeW(0));
1591 SET_HDR(ap, &stg_AP_UPD_info, CCCS);
1596 TICK_RET_UNBOXED_TUP(1);
1601 /* -----------------------------------------------------------------------------
1602 Thread I/O blocking primitives
1603 -------------------------------------------------------------------------- */
1605 FN_(waitReadzh_fast)
1609 ASSERT(CurrentTSO->why_blocked == NotBlocked);
1610 CurrentTSO->why_blocked = BlockedOnRead;
1611 CurrentTSO->block_info.fd = R1.i;
1612 ACQUIRE_LOCK(&sched_mutex);
1613 APPEND_TO_BLOCKED_QUEUE(CurrentTSO);
1614 RELEASE_LOCK(&sched_mutex);
1615 JMP_(stg_block_noregs);
1619 FN_(waitWritezh_fast)
1623 ASSERT(CurrentTSO->why_blocked == NotBlocked);
1624 CurrentTSO->why_blocked = BlockedOnWrite;
1625 CurrentTSO->block_info.fd = R1.i;
1626 ACQUIRE_LOCK(&sched_mutex);
1627 APPEND_TO_BLOCKED_QUEUE(CurrentTSO);
1628 RELEASE_LOCK(&sched_mutex);
1629 JMP_(stg_block_noregs);
1639 ASSERT(CurrentTSO->why_blocked == NotBlocked);
1640 CurrentTSO->why_blocked = BlockedOnDelay;
1642 ACQUIRE_LOCK(&sched_mutex);
1644 target = (R1.i / (TICK_MILLISECS*1000)) + getourtimeofday();
1645 CurrentTSO->block_info.target = target;
1647 /* Insert the new thread in the sleeping queue. */
1650 while (t != END_TSO_QUEUE && t->block_info.target < target) {
1655 CurrentTSO->link = t;
1657 sleeping_queue = CurrentTSO;
1659 prev->link = CurrentTSO;
1662 RELEASE_LOCK(&sched_mutex);
1663 JMP_(stg_block_noregs);