+++ /dev/null
-dnl Alpha ev6 mpn_addmul_1 -- Multiply a limb vector with a limb and add
-dnl the result to a second limb vector.
-
-dnl Copyright (C) 2000 Free Software Foundation, Inc.
-
-dnl This file is part of the GNU MP Library.
-
-dnl The GNU MP Library is free software; you can redistribute it and/or modify
-dnl it under the terms of the GNU Lesser General Public License as published
-dnl by the Free Software Foundation; either version 2.1 of the License, or (at
-dnl your option) any later version.
-
-dnl The GNU MP Library is distributed in the hope that it will be useful, but
-dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
-dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
-dnl License for more details.
-
-dnl You should have received a copy of the GNU Lesser General Public License
-dnl along with the GNU MP Library; see the file COPYING.LIB. If not, write to
-dnl the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
-dnl MA 02111-1307, USA.
-
-include(`../config.m4')
-
-dnl INPUT PARAMETERS
-dnl res_ptr r16
-dnl s1_ptr r17
-dnl size r18
-dnl s2_limb r19
-
-dnl This code runs at 42 cycles/limb on EV4, 18 cycles/limb on EV5, and
-dnl exactly 3.625 cycles/limb on EV6...
-
-dnl This code was written in close cooperation with ev6 pipeline expert
-dnl Steve Root (root@toober.hlo.dec.com). Any errors are tege's fault, though.
-dnl
-dnl Register usages for unrolled loop:
-dnl 0-3 mul's
-dnl 4-7 acc's
-dnl 8-15 mul results
-dnl 20,21 carry's
-dnl 22,23 save for stores
-
-dnl Sustains 8 mul-adds in 29 cycles in the unrolled inner loop.
-
-dnl The stores can issue a cycle late so we have paired no-op's to 'catch'
-dnl them, so that further disturbance to the schedule is damped.
-
-dnl We couldn't pair the loads, because the entangled schedule of the
-dnl carry's has to happen on one side {0} of the machine. Note, the total
-dnl use of U0, and the total use of L0 (after attending to the stores).
-dnl which is part of the reason why....
-
-dnl This is a great schedule for the d_cache, a poor schedule for the
-dnl b_cache. The lockup on U0 means that any stall can't be recovered
-dnl from. Consider a ldq in L1. say that load gets stalled because it
-dnl collides with a fill from the b_Cache. On the next cycle, this load
-dnl gets priority. If first looks at L0, and goes there. The instruction
-dnl we intended for L0 gets to look at L1, which is NOT where we want
-dnl it. It either stalls 1, because it can't go in L0, or goes there, and
-dnl causes a further instruction to stall.
-
-dnl So for b_cache, we're likely going to want to put one or more cycles
-dnl back into the code! And, of course, put in prefetches. For the
-dnl accumulator, lds, intent to modify. For the multiplier, you might
-dnl want ldq, evict next, if you're not wanting to use it again soon. Use
-dnl 256 ahead of present pointer value. At a place where we have an mt
-dnl followed by a bookkeeping, put the bookkeeping in upper, and the
-dnl prefetch into lower.
-
-dnl Note, the usage of physical registers per cycle is smoothed off, as
-dnl much as possible.
-
-dnl Note, the ldq's and stq's are at the end of the quadpacks. note, we'd
-dnl like not to have a ldq or stq to preceded a conditional branch in a
-dnl quadpack. The conditional branch moves the retire pointer one cycle
-dnl later.
-
-dnl Optimization notes:
-dnl Callee-saves regs: r9 r10 r11 r12 r13 r14 r15 r26 ?r27?
-dnl Reserved regs: r29 r30 r31
-dnl Free caller-saves regs in unrolled code: r24 r25 r28
-dnl We should swap some of the callee-saves regs for some of the free
-dnl caller-saves regs, saving some overhead cycles.
-dnl Most importantly, we should write fast code for the 0-7 case.
-dnl The code we use there are for the 21164, and runs at 7 cycles/limb
-dnl on the 21264. Should not be hard, if we write specialized code for
-dnl 1-7 limbs (the one for 0 limbs should be straightforward). We then just
-dnl need a jump table indexed by the low 3 bits of the count argument.
-
-
-ASM_START()
-PROLOGUE(mpn_addmul_1)
- cmpult r18, 8, r1
- beq r1, $Large
-
- ldq r2, 0(r17) C r2 = s1_limb
- addq r17, 8, r17 C s1_ptr++
- subq r18, 1, r18 C size--
- mulq r2, r19, r3 C r3 = prod_low
- ldq r5, 0(r16) C r5 = *res_ptr
- umulh r2, r19, r0 C r0 = prod_high
- beq r18, $Lend0b C jump if size was == 1
- ldq r2, 0(r17) C r2 = s1_limb
- addq r17, 8, r17 C s1_ptr++
- subq r18, 1, r18 C size--
- addq r5, r3, r3
- cmpult r3, r5, r4
- stq r3, 0(r16)
- addq r16, 8, r16 C res_ptr++
- beq r18, $Lend0a C jump if size was == 2
-
- ALIGN(8)
-$Loop0: mulq r2, r19, r3 C r3 = prod_low
- ldq r5, 0(r16) C r5 = *res_ptr
- addq r4, r0, r0 C cy_limb = cy_limb + 'cy'
- subq r18, 1, r18 C size--
- umulh r2, r19, r4 C r4 = cy_limb
- ldq r2, 0(r17) C r2 = s1_limb
- addq r17, 8, r17 C s1_ptr++
- addq r3, r0, r3 C r3 = cy_limb + prod_low
- cmpult r3, r0, r0 C r0 = carry from (cy_limb + prod_low)
- addq r5, r3, r3
- cmpult r3, r5, r5
- stq r3, 0(r16)
- addq r16, 8, r16 C res_ptr++
- addq r5, r0, r0 C combine carries
- bne r18, $Loop0
-$Lend0a:
- mulq r2, r19, r3 C r3 = prod_low
- ldq r5, 0(r16) C r5 = *res_ptr
- addq r4, r0, r0 C cy_limb = cy_limb + 'cy'
- umulh r2, r19, r4 C r4 = cy_limb
- addq r3, r0, r3 C r3 = cy_limb + prod_low
- cmpult r3, r0, r0 C r0 = carry from (cy_limb + prod_low)
- addq r5, r3, r3
- cmpult r3, r5, r5
- stq r3, 0(r16)
- addq r5, r0, r0 C combine carries
- addq r4, r0, r0 C cy_limb = prod_high + cy
- ret r31, (r26), 1
-$Lend0b:
- addq r5, r3, r3
- cmpult r3, r5, r5
- stq r3, 0(r16)
- addq r0, r5, r0
- ret r31, (r26), 1
-
-$Large:
- lda $30, -240($30)
- stq $9, 8($30)
- stq $10, 16($30)
- stq $11, 24($30)
- stq $12, 32($30)
- stq $13, 40($30)
- stq $14, 48($30)
- stq $15, 56($30)
-
- and r18, 7, r20 C count for the first loop, 0-7
- srl r18, 3, r18 C count for unrolled loop
- bis r31, r31, r0
- beq r20, $Lunroll
- ldq r2, 0(r17) C r2 = s1_limb
- addq r17, 8, r17 C s1_ptr++
- subq r20, 1, r20 C size--
- mulq r2, r19, r3 C r3 = prod_low
- ldq r5, 0(r16) C r5 = *res_ptr
- umulh r2, r19, r0 C r0 = prod_high
- beq r20, $Lend1b C jump if size was == 1
- ldq r2, 0(r17) C r2 = s1_limb
- addq r17, 8, r17 C s1_ptr++
- subq r20, 1, r20 C size--
- addq r5, r3, r3
- cmpult r3, r5, r4
- stq r3, 0(r16)
- addq r16, 8, r16 C res_ptr++
- beq r20, $Lend1a C jump if size was == 2
-
- ALIGN(8)
-$Loop1: mulq r2, r19, r3 C r3 = prod_low
- ldq r5, 0(r16) C r5 = *res_ptr
- addq r4, r0, r0 C cy_limb = cy_limb + 'cy'
- subq r20, 1, r20 C size--
- umulh r2, r19, r4 C r4 = cy_limb
- ldq r2, 0(r17) C r2 = s1_limb
- addq r17, 8, r17 C s1_ptr++
- addq r3, r0, r3 C r3 = cy_limb + prod_low
- cmpult r3, r0, r0 C r0 = carry from (cy_limb + prod_low)
- addq r5, r3, r3
- cmpult r3, r5, r5
- stq r3, 0(r16)
- addq r16, 8, r16 C res_ptr++
- addq r5, r0, r0 C combine carries
- bne r20, $Loop1
-
-$Lend1a:
- mulq r2, r19, r3 C r3 = prod_low
- ldq r5, 0(r16) C r5 = *res_ptr
- addq r4, r0, r0 C cy_limb = cy_limb + 'cy'
- umulh r2, r19, r4 C r4 = cy_limb
- addq r3, r0, r3 C r3 = cy_limb + prod_low
- cmpult r3, r0, r0 C r0 = carry from (cy_limb + prod_low)
- addq r5, r3, r3
- cmpult r3, r5, r5
- stq r3, 0(r16)
- addq r16, 8, r16 C res_ptr++
- addq r5, r0, r0 C combine carries
- addq r4, r0, r0 C cy_limb = prod_high + cy
- br r31, $Lunroll
-$Lend1b:
- addq r5, r3, r3
- cmpult r3, r5, r5
- stq r3, 0(r16)
- addq r16, 8, r16 C res_ptr++
- addq r0, r5, r0
-
-$Lunroll:
- lda r17, -16(r17) C L1 bookkeeping
- lda r16, -16(r16) C L1 bookkeeping
- bis r0, r31, r12
-
-C ____ UNROLLED LOOP SOFTWARE PIPELINE STARTUP ____
-
- ldq r2, 16(r17) C L1
- ldq r3, 24(r17) C L1
- lda r18, -1(r18) C L1 bookkeeping
- ldq r6, 16(r16) C L1
- ldq r7, 24(r16) C L1
- ldq r0, 32(r17) C L1
- mulq r19, r2, r13 C U1
- ldq r1, 40(r17) C L1
- umulh r19, r2, r14 C U1
- mulq r19, r3, r15 C U1
- lda r17, 64(r17) C L1 bookkeeping
- ldq r4, 32(r16) C L1
- ldq r5, 40(r16) C L1
- umulh r19, r3, r8 C U1
- ldq r2, -16(r17) C L1
- mulq r19, r0, r9 C U1
- ldq r3, -8(r17) C L1
- umulh r19, r0, r10 C U1
- addq r6, r13, r6 C L0 lo + acc
- mulq r19, r1, r11 C U1
- cmpult r6, r13, r20 C L0 lo add => carry
- lda r16, 64(r16) C L1 bookkeeping
- addq r6, r12, r22 C U0 hi add => answer
- cmpult r22, r12, r21 C L0 hi add => carry
- addq r14, r20, r14 C U0 hi mul + carry
- ldq r6, -16(r16) C L1
- addq r7, r15, r23 C L0 lo + acc
- addq r14, r21, r14 C U0 hi mul + carry
- ldq r7, -8(r16) C L1
- umulh r19, r1, r12 C U1
- cmpult r23, r15, r20 C L0 lo add => carry
- addq r23, r14, r23 C U0 hi add => answer
- ldq r0, 0(r17) C L1
- mulq r19, r2, r13 C U1
- cmpult r23, r14, r21 C L0 hi add => carry
- addq r8, r20, r8 C U0 hi mul + carry
- ldq r1, 8(r17) C L1
- umulh r19, r2, r14 C U1
- addq r4, r9, r4 C L0 lo + acc
- stq r22, -48(r16) C L0
- stq r23, -40(r16) C L1
- mulq r19, r3, r15 C U1
- addq r8, r21, r8 C U0 hi mul + carry
- cmpult r4, r9, r20 C L0 lo add => carry
- addq r4, r8, r22 C U0 hi add => answer
- ble r18, $Lend C U1 bookkeeping
-
-C ____ MAIN UNROLLED LOOP ____
- ALIGN(16)
-$Loop:
- bis r31, r31, r31 C U1 mt
- cmpult r22, r8, r21 C L0 hi add => carry
- addq r10, r20, r10 C U0 hi mul + carry
- ldq r4, 0(r16) C L1
-
- bis r31, r31, r31 C U1 mt
- addq r5, r11, r23 C L0 lo + acc
- addq r10, r21, r10 C L0 hi mul + carry
- ldq r5, 8(r16) C L1
-
- umulh r19, r3, r8 C U1
- cmpult r23, r11, r20 C L0 lo add => carry
- addq r23, r10, r23 C U0 hi add => answer
- ldq r2, 16(r17) C L1
-
- mulq r19, r0, r9 C U1
- cmpult r23, r10, r21 C L0 hi add => carry
- addq r12, r20, r12 C U0 hi mul + carry
- ldq r3, 24(r17) C L1
-
- umulh r19, r0, r10 C U1
- addq r6, r13, r6 C L0 lo + acc
- stq r22, -32(r16) C L0
- stq r23, -24(r16) C L1
-
- bis r31, r31, r31 C L0 st slosh
- mulq r19, r1, r11 C U1
- bis r31, r31, r31 C L1 st slosh
- addq r12, r21, r12 C U0 hi mul + carry
-
- cmpult r6, r13, r20 C L0 lo add => carry
- bis r31, r31, r31 C U1 mt
- lda r18, -1(r18) C L1 bookkeeping
- addq r6, r12, r22 C U0 hi add => answer
-
- bis r31, r31, r31 C U1 mt
- cmpult r22, r12, r21 C L0 hi add => carry
- addq r14, r20, r14 C U0 hi mul + carry
- ldq r6, 16(r16) C L1
-
- bis r31, r31, r31 C U1 mt
- addq r7, r15, r23 C L0 lo + acc
- addq r14, r21, r14 C U0 hi mul + carry
- ldq r7, 24(r16) C L1
-
- umulh r19, r1, r12 C U1
- cmpult r23, r15, r20 C L0 lo add => carry
- addq r23, r14, r23 C U0 hi add => answer
- ldq r0, 32(r17) C L1
-
- mulq r19, r2, r13 C U1
- cmpult r23, r14, r21 C L0 hi add => carry
- addq r8, r20, r8 C U0 hi mul + carry
- ldq r1, 40(r17) C L1
-
- umulh r19, r2, r14 C U1
- addq r4, r9, r4 C U0 lo + acc
- stq r22, -16(r16) C L0
- stq r23, -8(r16) C L1
-
- bis r31, r31, r31 C L0 st slosh
- mulq r19, r3, r15 C U1
- bis r31, r31, r31 C L1 st slosh
- addq r8, r21, r8 C L0 hi mul + carry
-
- cmpult r4, r9, r20 C L0 lo add => carry
- bis r31, r31, r31 C U1 mt
- lda r17, 64(r17) C L1 bookkeeping
- addq r4, r8, r22 C U0 hi add => answer
-
- bis r31, r31, r31 C U1 mt
- cmpult r22, r8, r21 C L0 hi add => carry
- addq r10, r20, r10 C U0 hi mul + carry
- ldq r4, 32(r16) C L1
-
- bis r31, r31, r31 C U1 mt
- addq r5, r11, r23 C L0 lo + acc
- addq r10, r21, r10 C L0 hi mul + carry
- ldq r5, 40(r16) C L1
-
- umulh r19, r3, r8 C U1
- cmpult r23, r11, r20 C L0 lo add => carry
- addq r23, r10, r23 C U0 hi add => answer
- ldq r2, -16(r17) C L1
-
- mulq r19, r0, r9 C U1
- cmpult r23, r10, r21 C L0 hi add => carry
- addq r12, r20, r12 C U0 hi mul + carry
- ldq r3, -8(r17) C L1
-
- umulh r19, r0, r10 C U1
- addq r6, r13, r6 C L0 lo + acc
- stq r22, 0(r16) C L0
- stq r23, 8(r16) C L1
-
- bis r31, r31, r31 C L0 st slosh
- mulq r19, r1, r11 C U1
- bis r31, r31, r31 C L1 st slosh
- addq r12, r21, r12 C U0 hi mul + carry
-
- cmpult r6, r13, r20 C L0 lo add => carry
- bis r31, r31, r31 C U1 mt
- lda r16, 64(r16) C L1 bookkeeping
- addq r6, r12, r22 C U0 hi add => answer
-
- bis r31, r31, r31 C U1 mt
- cmpult r22, r12, r21 C L0 hi add => carry
- addq r14, r20, r14 C U0 hi mul + carry
- ldq r6, -16(r16) C L1
-
- bis r31, r31, r31 C U1 mt
- addq r7, r15, r23 C L0 lo + acc
- addq r14, r21, r14 C U0 hi mul + carry
- ldq r7, -8(r16) C L1
-
- umulh r19, r1, r12 C U1
- cmpult r23, r15, r20 C L0 lo add => carry
- addq r23, r14, r23 C U0 hi add => answer
- ldq r0, 0(r17) C L1
-
- mulq r19, r2, r13 C U1
- cmpult r23, r14, r21 C L0 hi add => carry
- addq r8, r20, r8 C U0 hi mul + carry
- ldq r1, 8(r17) C L1
-
- umulh r19, r2, r14 C U1
- addq r4, r9, r4 C L0 lo + acc
- stq r22, -48(r16) C L0
- stq r23, -40(r16) C L1
-
- bis r31, r31, r31 C L0 st slosh
- mulq r19, r3, r15 C U1
- bis r31, r31, r31 C L1 st slosh
- addq r8, r21, r8 C U0 hi mul + carry
-
- cmpult r4, r9, r20 C L0 lo add => carry
- addq r4, r8, r22 C U0 hi add => answer
- bis r31, r31, r31 C L1 mt
- bgt r18, $Loop C U1 bookkeeping
-
-C ____ UNROLLED LOOP SOFTWARE PIPELINE FINISH ____
-$Lend:
- cmpult r22, r8, r21 C L0 hi add => carry
- addq r10, r20, r10 C U0 hi mul + carry
- ldq r4, 0(r16) C L1
- addq r5, r11, r23 C L0 lo + acc
- addq r10, r21, r10 C L0 hi mul + carry
- ldq r5, 8(r16) C L1
- umulh r19, r3, r8 C U1
- cmpult r23, r11, r20 C L0 lo add => carry
- addq r23, r10, r23 C U0 hi add => answer
- mulq r19, r0, r9 C U1
- cmpult r23, r10, r21 C L0 hi add => carry
- addq r12, r20, r12 C U0 hi mul + carry
- umulh r19, r0, r10 C U1
- addq r6, r13, r6 C L0 lo + acc
- stq r22, -32(r16) C L0
- stq r23, -24(r16) C L1
- mulq r19, r1, r11 C U1
- addq r12, r21, r12 C U0 hi mul + carry
- cmpult r6, r13, r20 C L0 lo add => carry
- addq r6, r12, r22 C U0 hi add => answer
- cmpult r22, r12, r21 C L0 hi add => carry
- addq r14, r20, r14 C U0 hi mul + carry
- addq r7, r15, r23 C L0 lo + acc
- addq r14, r21, r14 C U0 hi mul + carry
- umulh r19, r1, r12 C U1
- cmpult r23, r15, r20 C L0 lo add => carry
- addq r23, r14, r23 C U0 hi add => answer
- cmpult r23, r14, r21 C L0 hi add => carry
- addq r8, r20, r8 C U0 hi mul + carry
- addq r4, r9, r4 C U0 lo + acc
- stq r22, -16(r16) C L0
- stq r23, -8(r16) C L1
- bis r31, r31, r31 C L0 st slosh
- addq r8, r21, r8 C L0 hi mul + carry
- cmpult r4, r9, r20 C L0 lo add => carry
- addq r4, r8, r22 C U0 hi add => answer
- cmpult r22, r8, r21 C L0 hi add => carry
- addq r10, r20, r10 C U0 hi mul + carry
- addq r5, r11, r23 C L0 lo + acc
- addq r10, r21, r10 C L0 hi mul + carry
- cmpult r23, r11, r20 C L0 lo add => carry
- addq r23, r10, r23 C U0 hi add => answer
- cmpult r23, r10, r21 C L0 hi add => carry
- addq r12, r20, r12 C U0 hi mul + carry
- stq r22, 0(r16) C L0
- stq r23, 8(r16) C L1
- addq r12, r21, r0 C U0 hi mul + carry
-
- ldq $9, 8($30)
- ldq $10, 16($30)
- ldq $11, 24($30)
- ldq $12, 32($30)
- ldq $13, 40($30)
- ldq $14, 48($30)
- ldq $15, 56($30)
- lda $30, 240($30)
- ret r31, (r26), 1
-EPILOGUE(mpn_addmul_1)
-ASM_END()
projects / ghc-hetmet.git / blobdiff