9 #ifndef BOTAN_MP_ASM_INTERNAL_H__
10 #define BOTAN_MP_ASM_INTERNAL_H__
12 #include <botan/internal/mp_madd.h>
16 #if defined(BOTAN_MP_USE_X86_32_ASM)
18 #define ADDSUB2_OP(OPERATION, INDEX) \
19 ASM("movl 4*" #INDEX "(%[y]), %[carry]") \
20 ASM(OPERATION " %[carry], 4*" #INDEX "(%[x])") \
22 #define ADDSUB3_OP(OPERATION, INDEX) \
23 ASM("movl 4*" #INDEX "(%[x]), %[carry]") \
24 ASM(OPERATION " 4*" #INDEX "(%[y]), %[carry]") \
25 ASM("movl %[carry], 4*" #INDEX "(%[z])") \
27 #define LINMUL_OP(WRITE_TO, INDEX) \
28 ASM("movl 4*" #INDEX "(%[x]),%%eax") \
30 ASM("addl %[carry],%%eax") \
31 ASM("adcl $0,%%edx") \
32 ASM("movl %%edx,%[carry]") \
33 ASM("movl %%eax, 4*" #INDEX "(%[" WRITE_TO "])")
35 #define MULADD_OP(IGNORED, INDEX) \
36 ASM("movl 4*" #INDEX "(%[x]),%%eax") \
38 ASM("addl %[carry],%%eax") \
39 ASM("adcl $0,%%edx") \
40 ASM("addl 4*" #INDEX "(%[z]),%%eax") \
41 ASM("adcl $0,%%edx") \
42 ASM("movl %%edx,%[carry]") \
43 ASM("movl %%eax, 4*" #INDEX " (%[z])")
45 #define ADD_OR_SUBTRACT(CORE_CODE) \
46 ASM("rorl %[carry]") \
48 ASM("sbbl %[carry],%[carry]") \
51 #elif defined(BOTAN_MP_USE_X86_64_ASM)
53 #define ADDSUB2_OP(OPERATION, INDEX) \
54 ASM("movq 8*" #INDEX "(%[y]), %[carry]") \
55 ASM(OPERATION " %[carry], 8*" #INDEX "(%[x])") \
57 #define ADDSUB3_OP(OPERATION, INDEX) \
58 ASM("movq 8*" #INDEX "(%[x]), %[carry]") \
59 ASM(OPERATION " 8*" #INDEX "(%[y]), %[carry]") \
60 ASM("movq %[carry], 8*" #INDEX "(%[z])") \
62 #define LINMUL_OP(WRITE_TO, INDEX) \
63 ASM("movq 8*" #INDEX "(%[x]),%%rax") \
65 ASM("addq %[carry],%%rax") \
66 ASM("adcq $0,%%rdx") \
67 ASM("movq %%rdx,%[carry]") \
68 ASM("movq %%rax, 8*" #INDEX "(%[" WRITE_TO "])")
70 #define MULADD_OP(IGNORED, INDEX) \
71 ASM("movq 8*" #INDEX "(%[x]),%%rax") \
73 ASM("addq %[carry],%%rax") \
74 ASM("adcq $0,%%rdx") \
75 ASM("addq 8*" #INDEX "(%[z]),%%rax") \
76 ASM("adcq $0,%%rdx") \
77 ASM("movq %%rdx,%[carry]") \
78 ASM("movq %%rax, 8*" #INDEX " (%[z])")
80 #define ADD_OR_SUBTRACT(CORE_CODE) \
81 ASM("rorq %[carry]") \
83 ASM("sbbq %[carry],%[carry]") \
88 #if defined(ADD_OR_SUBTRACT)
90 #define ASM(x) x "\n\t"
92 #define DO_8_TIMES(MACRO, ARG) \
109 #if defined(BOTAN_MP_USE_X86_32_ASM)
111 ADD_OR_SUBTRACT(ASM(
"adcl %[y],%[x]"))
112 : [x]
"=r"(x), [carry]
"=r"(*carry)
113 :
"0"(x), [y]
"rm"(y),
"1"(*carry)
117 #elif defined(BOTAN_MP_USE_X86_64_ASM)
120 ADD_OR_SUBTRACT(ASM(
"adcq %[y],%[x]"))
121 : [x]
"=r"(x), [carry]
"=r"(*carry)
122 :
"0"(x), [y]
"rm"(y),
"1"(*carry)
130 *carry = c1 | (z < *carry);
138 inline word
word8_add2(word x[8],
const word y[8], word carry)
140 #if defined(BOTAN_MP_USE_X86_32_ASM)
142 ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP,
"adcl"))
144 : [x]
"r"(x), [y]
"r"(y),
"0"(carry)
148 #elif defined(BOTAN_MP_USE_X86_64_ASM)
151 ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP,
"adcq"))
153 : [x]
"r"(x), [y]
"r"(y),
"0"(carry)
157 #elif defined(BOTAN_MP_USE_X86_32_MSVC_ASM)
185 x[0] =
word_add(x[0], y[0], &carry);
186 x[1] =
word_add(x[1], y[1], &carry);
187 x[2] =
word_add(x[2], y[2], &carry);
188 x[3] =
word_add(x[3], y[3], &carry);
189 x[4] =
word_add(x[4], y[4], &carry);
190 x[5] =
word_add(x[5], y[5], &carry);
191 x[6] =
word_add(x[6], y[6], &carry);
192 x[7] =
word_add(x[7], y[7], &carry);
201 const word y[8], word carry)
203 #if defined(BOTAN_MP_USE_X86_32_ASM)
205 ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP,
"adcl"))
207 : [x]
"r"(x), [y]
"r"(y), [z]
"r"(z),
"0"(carry)
211 #elif defined(BOTAN_MP_USE_X86_64_ASM)
214 ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP,
"adcq"))
216 : [x]
"r"(x), [y]
"r"(y), [z]
"r"(z),
"0"(carry)
220 #elif defined(BOTAN_MP_USE_X86_32_MSVC_ASM)
265 z[0] =
word_add(x[0], y[0], &carry);
266 z[1] =
word_add(x[1], y[1], &carry);
267 z[2] =
word_add(x[2], y[2], &carry);
268 z[3] =
word_add(x[3], y[3], &carry);
269 z[4] =
word_add(x[4], y[4], &carry);
270 z[5] =
word_add(x[5], y[5], &carry);
271 z[6] =
word_add(x[6], y[6], &carry);
272 z[7] =
word_add(x[7], y[7], &carry);
282 #if defined(BOTAN_MP_USE_X86_32_ASM)
284 ADD_OR_SUBTRACT(ASM(
"sbbl %[y],%[x]"))
285 : [x]
"=r"(x), [carry]
"=r"(*carry)
286 :
"0"(x), [y]
"rm"(y),
"1"(*carry)
290 #elif defined(BOTAN_MP_USE_X86_64_ASM)
293 ADD_OR_SUBTRACT(ASM(
"sbbq %[y],%[x]"))
294 : [x]
"=r"(x), [carry]
"=r"(*carry)
295 :
"0"(x), [y]
"rm"(y),
"1"(*carry)
302 word z = t0 - *carry;
303 *carry = c1 | (z > t0);
311 inline word
word8_sub2(word x[8],
const word y[8], word carry)
313 #if defined(BOTAN_MP_USE_X86_32_ASM)
315 ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP,
"sbbl"))
317 : [x]
"r"(x), [y]
"r"(y),
"0"(carry)
321 #elif defined(BOTAN_MP_USE_X86_64_ASM)
324 ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP,
"sbbq"))
326 : [x]
"r"(x), [y]
"r"(y),
"0"(carry)
330 #elif defined(BOTAN_MP_USE_X86_32_MSVC_ASM)
366 x[0] =
word_sub(x[0], y[0], &carry);
367 x[1] =
word_sub(x[1], y[1], &carry);
368 x[2] =
word_sub(x[2], y[2], &carry);
369 x[3] =
word_sub(x[3], y[3], &carry);
370 x[4] =
word_sub(x[4], y[4], &carry);
371 x[5] =
word_sub(x[5], y[5], &carry);
372 x[6] =
word_sub(x[6], y[6], &carry);
373 x[7] =
word_sub(x[7], y[7], &carry);
383 #if defined(BOTAN_MP_USE_X86_32_ASM)
385 ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP,
"sbbl"))
387 : [x]
"r"(y), [y]
"r"(x), [z]
"r"(x),
"0"(carry)
391 #elif defined(BOTAN_MP_USE_X86_64_ASM)
394 ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP,
"sbbq"))
396 : [x]
"r"(y), [y]
"r"(x), [z]
"r"(x),
"0"(carry)
401 x[0] =
word_sub(y[0], x[0], &carry);
402 x[1] =
word_sub(y[1], x[1], &carry);
403 x[2] =
word_sub(y[2], x[2], &carry);
404 x[3] =
word_sub(y[3], x[3], &carry);
405 x[4] =
word_sub(y[4], x[4], &carry);
406 x[5] =
word_sub(y[5], x[5], &carry);
407 x[6] =
word_sub(y[6], x[6], &carry);
408 x[7] =
word_sub(y[7], x[7], &carry);
417 const word y[8], word carry)
419 #if defined(BOTAN_MP_USE_X86_32_ASM)
421 ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP,
"sbbl"))
423 : [x]
"r"(x), [y]
"r"(y), [z]
"r"(z),
"0"(carry)
427 #elif defined(BOTAN_MP_USE_X86_64_ASM)
430 ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP,
"sbbq"))
432 : [x]
"r"(x), [y]
"r"(y), [z]
"r"(z),
"0"(carry)
436 #elif defined(BOTAN_MP_USE_X86_32_MSVC_ASM)
473 z[0] =
word_sub(x[0], y[0], &carry);
474 z[1] =
word_sub(x[1], y[1], &carry);
475 z[2] =
word_sub(x[2], y[2], &carry);
476 z[3] =
word_sub(x[3], y[3], &carry);
477 z[4] =
word_sub(x[4], y[4], &carry);
478 z[5] =
word_sub(x[5], y[5], &carry);
479 z[6] =
word_sub(x[6], y[6], &carry);
480 z[7] =
word_sub(x[7], y[7], &carry);
490 #if defined(BOTAN_MP_USE_X86_32_ASM)
492 DO_8_TIMES(LINMUL_OP,
"x")
494 : [x]
"r"(x), [y]
"rm"(y),
"0"(carry)
495 :
"cc",
"%eax",
"%edx");
498 #elif defined(BOTAN_MP_USE_X86_64_ASM)
501 DO_8_TIMES(LINMUL_OP,
"x")
503 : [x]
"r"(x), [y]
"rm"(y),
"0"(carry)
504 :
"cc",
"%rax",
"%rdx");
507 #elif defined(BOTAN_MP_USE_X86_32_MSVC_ASM)
587 #if defined(BOTAN_MP_USE_X86_32_ASM)
589 DO_8_TIMES(LINMUL_OP,
"z")
591 : [z]
"r"(z), [x]
"r"(x), [y]
"rm"(y),
"0"(carry)
592 :
"cc",
"%eax",
"%edx");
595 #elif defined(BOTAN_MP_USE_X86_64_ASM)
597 DO_8_TIMES(LINMUL_OP,
"z")
599 : [z]
"r"(z), [x]
"r"(x), [y]
"rm"(y),
"0"(carry)
600 :
"cc",
"%rax",
"%rdx");
603 #elif defined(BOTAN_MP_USE_X86_32_MSVC_ASM)
681 inline word
word8_madd3(word z[8],
const word x[8], word y, word carry)
683 #if defined(BOTAN_MP_USE_X86_32_ASM)
685 DO_8_TIMES(MULADD_OP,
"")
687 : [z]
"r"(z), [x]
"r"(x), [y]
"rm"(y),
"0"(carry)
688 :
"cc",
"%eax",
"%edx");
691 #elif defined(BOTAN_MP_USE_X86_64_ASM)
694 DO_8_TIMES(MULADD_OP,
"")
696 : [z]
"r"(z), [x]
"r"(x), [y]
"rm"(y),
"0"(carry)
697 :
"cc",
"%rax",
"%rdx");
719 #if defined(BOTAN_MP_USE_X86_32_ASM)
723 ASM(
"addl %[x],%[w0]")
724 ASM(
"adcl %[y],%[w1]")
727 : [w0]
"=r"(*w0), [w1]
"=r"(*w1), [w2]
"=r"(*w2)
728 : [x]
"a"(x), [y]
"d"(y),
"0"(*w0),
"1"(*w1),
"2"(*w2)
731 #elif defined(BOTAN_MP_USE_X86_64_ASM)
736 ASM(
"addq %[x],%[w0]")
737 ASM(
"adcq %[y],%[w1]")
740 : [w0]
"=r"(*w0), [w1]
"=r"(*w1), [w2]
"=r"(*w2)
741 : [x]
"a"(x), [y]
"d"(y),
"0"(*w0),
"1"(*w1),
"2"(*w2)
748 *w2 += (*w1 < carry) ? 1 : 0;
758 #if defined(BOTAN_MP_USE_X86_32_ASM)
762 ASM(
"addl %[x],%[w0]")
763 ASM(
"adcl %[y],%[w1]")
766 ASM(
"addl %[x],%[w0]")
767 ASM(
"adcl %[y],%[w1]")
770 : [w0]
"=r"(*w0), [w1]
"=r"(*w1), [w2]
"=r"(*w2)
771 : [x]
"a"(x), [y]
"d"(y),
"0"(*w0),
"1"(*w1),
"2"(*w2)
774 #elif defined(BOTAN_MP_USE_X86_64_ASM)
779 ASM(
"addq %[x],%[w0]")
780 ASM(
"adcq %[y],%[w1]")
783 ASM(
"addq %[x],%[w0]")
784 ASM(
"adcq %[y],%[w1]")
787 : [w0]
"=r"(*w0), [w1]
"=r"(*w1), [w2]
"=r"(*w2)
788 : [x]
"a"(x), [y]
"d"(y),
"0"(*w0),
"1"(*w1),
"2"(*w2)
796 word top = (y >> (BOTAN_MP_WORD_BITS-1));
798 y |= (x >> (BOTAN_MP_WORD_BITS-1));
811 #undef ADD_OR_SUBTRACT
word word8_sub2_rev(word x[8], const word y[8], word carry)
void word3_muladd(word *w2, word *w1, word *w0, word x, word y)
word word8_add2(word x[8], const word y[8], word carry)
word word8_linmul3(word z[8], const word x[8], word y, word carry)
word word_madd3(word a, word b, word c, word *d)
word word8_sub2(word x[8], const word y[8], word carry)
word word_madd2(word a, word b, word *c)
word word8_madd3(word z[8], const word x[8], word y, word carry)
word word8_linmul2(word x[8], word y, word carry)
word word8_add3(word z[8], const word x[8], const word y[8], word carry)
void word3_muladd_2(word *w2, word *w1, word *w0, word x, word y)
word word_sub(word x, word y, word *carry)
word word_add(word x, word y, word *carry)
word word8_sub3(word z[8], const word x[8], const word y[8], word carry)