| /* SPDX-License-Identifier: GPL-2.0-or-later */ | |
| /* | |
| * Fast SHA-256 implementation for SPE instruction set (PPC) | |
| * | |
| * This code makes use of the SPE SIMD instruction set as defined in | |
| * http://cache.freescale.com/files/32bit/doc/ref_manual/SPEPIM.pdf | |
| * Implementation is based on optimization guide notes from | |
| * http://cache.freescale.com/files/32bit/doc/app_note/AN2665.pdf | |
| * | |
| * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> | |
| */ | |
| #include <asm/ppc_asm.h> | |
| #include <asm/asm-offsets.h> | |
| #define rHP r3 /* pointer to hash values in memory */ | |
| #define rKP r24 /* pointer to round constants */ | |
| #define rWP r4 /* pointer to input data */ | |
| #define rH0 r5 /* 8 32 bit hash values in 8 registers */ | |
| #define rH1 r6 | |
| #define rH2 r7 | |
| #define rH3 r8 | |
| #define rH4 r9 | |
| #define rH5 r10 | |
| #define rH6 r11 | |
| #define rH7 r12 | |
| #define rW0 r14 /* 64 bit registers. 16 words in 8 registers */ | |
| #define rW1 r15 | |
| #define rW2 r16 | |
| #define rW3 r17 | |
| #define rW4 r18 | |
| #define rW5 r19 | |
| #define rW6 r20 | |
| #define rW7 r21 | |
| #define rT0 r22 /* 64 bit temporaries */ | |
| #define rT1 r23 | |
| #define rT2 r0 /* 32 bit temporaries */ | |
| #define rT3 r25 | |
| #define CMP_KN_LOOP | |
| #define CMP_KC_LOOP \ | |
| cmpwi rT1,0; | |
| #define INITIALIZE \ | |
| stwu r1,-128(r1); /* create stack frame */ \ | |
| evstdw r14,8(r1); /* We must save non volatile */ \ | |
| evstdw r15,16(r1); /* registers. Take the chance */ \ | |
| evstdw r16,24(r1); /* and save the SPE part too */ \ | |
| evstdw r17,32(r1); \ | |
| evstdw r18,40(r1); \ | |
| evstdw r19,48(r1); \ | |
| evstdw r20,56(r1); \ | |
| evstdw r21,64(r1); \ | |
| evstdw r22,72(r1); \ | |
| evstdw r23,80(r1); \ | |
| stw r24,88(r1); /* save normal registers */ \ | |
| stw r25,92(r1); | |
| #define FINALIZE \ | |
| evldw r14,8(r1); /* restore SPE registers */ \ | |
| evldw r15,16(r1); \ | |
| evldw r16,24(r1); \ | |
| evldw r17,32(r1); \ | |
| evldw r18,40(r1); \ | |
| evldw r19,48(r1); \ | |
| evldw r20,56(r1); \ | |
| evldw r21,64(r1); \ | |
| evldw r22,72(r1); \ | |
| evldw r23,80(r1); \ | |
| lwz r24,88(r1); /* restore normal registers */ \ | |
| lwz r25,92(r1); \ | |
| xor r0,r0,r0; \ | |
| stw r0,8(r1); /* Delete sensitive data */ \ | |
| stw r0,16(r1); /* that we might have pushed */ \ | |
| stw r0,24(r1); /* from other context that runs */ \ | |
| stw r0,32(r1); /* the same code. Assume that */ \ | |
| stw r0,40(r1); /* the lower part of the GPRs */ \ | |
| stw r0,48(r1); /* was already overwritten on */ \ | |
| stw r0,56(r1); /* the way down to here */ \ | |
| stw r0,64(r1); \ | |
| stw r0,72(r1); \ | |
| stw r0,80(r1); \ | |
| addi r1,r1,128; /* cleanup stack frame */ | |
| #ifdef __BIG_ENDIAN__ | |
| #define LOAD_DATA(reg, off) \ | |
| lwz reg,off(rWP); /* load data */ | |
| #define NEXT_BLOCK \ | |
| addi rWP,rWP,64; /* increment per block */ | |
| #else | |
| #define LOAD_DATA(reg, off) \ | |
| lwbrx reg,0,rWP; /* load data */ \ | |
| addi rWP,rWP,4; /* increment per word */ | |
| #define NEXT_BLOCK /* nothing to do */ | |
| #endif | |
| #define R_LOAD_W(a, b, c, d, e, f, g, h, w, off) \ | |
| LOAD_DATA(w, off) /* 1: W */ \ | |
| rotrwi rT0,e,6; /* 1: S1 = e rotr 6 */ \ | |
| rotrwi rT1,e,11; /* 1: S1' = e rotr 11 */ \ | |
| rotrwi rT2,e,25; /* 1: S1" = e rotr 25 */ \ | |
| xor rT0,rT0,rT1; /* 1: S1 = S1 xor S1' */ \ | |
| and rT3,e,f; /* 1: ch = e and f */ \ | |
| xor rT0,rT0,rT2; /* 1: S1 = S1 xor S1" */ \ | |
| andc rT1,g,e; /* 1: ch' = ~e and g */ \ | |
| lwz rT2,off(rKP); /* 1: K */ \ | |
| xor rT3,rT3,rT1; /* 1: ch = ch xor ch' */ \ | |
| add h,h,rT0; /* 1: temp1 = h + S1 */ \ | |
| add rT3,rT3,w; /* 1: temp1' = ch + w */ \ | |
| rotrwi rT0,a,2; /* 1: S0 = a rotr 2 */ \ | |
| add h,h,rT3; /* 1: temp1 = temp1 + temp1' */ \ | |
| rotrwi rT1,a,13; /* 1: S0' = a rotr 13 */ \ | |
| add h,h,rT2; /* 1: temp1 = temp1 + K */ \ | |
| rotrwi rT3,a,22; /* 1: S0" = a rotr 22 */ \ | |
| xor rT0,rT0,rT1; /* 1: S0 = S0 xor S0' */ \ | |
| add d,d,h; /* 1: d = d + temp1 */ \ | |
| xor rT3,rT0,rT3; /* 1: S0 = S0 xor S0" */ \ | |
| evmergelo w,w,w; /* shift W */ \ | |
| or rT2,a,b; /* 1: maj = a or b */ \ | |
| and rT1,a,b; /* 1: maj' = a and b */ \ | |
| and rT2,rT2,c; /* 1: maj = maj and c */ \ | |
| LOAD_DATA(w, off+4) /* 2: W */ \ | |
| or rT2,rT1,rT2; /* 1: maj = maj or maj' */ \ | |
| rotrwi rT0,d,6; /* 2: S1 = e rotr 6 */ \ | |
| add rT3,rT3,rT2; /* 1: temp2 = S0 + maj */ \ | |
| rotrwi rT1,d,11; /* 2: S1' = e rotr 11 */ \ | |
| add h,h,rT3; /* 1: h = temp1 + temp2 */ \ | |
| rotrwi rT2,d,25; /* 2: S1" = e rotr 25 */ \ | |
| xor rT0,rT0,rT1; /* 2: S1 = S1 xor S1' */ \ | |
| and rT3,d,e; /* 2: ch = e and f */ \ | |
| xor rT0,rT0,rT2; /* 2: S1 = S1 xor S1" */ \ | |
| andc rT1,f,d; /* 2: ch' = ~e and g */ \ | |
| lwz rT2,off+4(rKP); /* 2: K */ \ | |
| xor rT3,rT3,rT1; /* 2: ch = ch xor ch' */ \ | |
| add g,g,rT0; /* 2: temp1 = h + S1 */ \ | |
| add rT3,rT3,w; /* 2: temp1' = ch + w */ \ | |
| rotrwi rT0,h,2; /* 2: S0 = a rotr 2 */ \ | |
| add g,g,rT3; /* 2: temp1 = temp1 + temp1' */ \ | |
| rotrwi rT1,h,13; /* 2: S0' = a rotr 13 */ \ | |
| add g,g,rT2; /* 2: temp1 = temp1 + K */ \ | |
| rotrwi rT3,h,22; /* 2: S0" = a rotr 22 */ \ | |
| xor rT0,rT0,rT1; /* 2: S0 = S0 xor S0' */ \ | |
| or rT2,h,a; /* 2: maj = a or b */ \ | |
| xor rT3,rT0,rT3; /* 2: S0 = S0 xor S0" */ \ | |
| and rT1,h,a; /* 2: maj' = a and b */ \ | |
| and rT2,rT2,b; /* 2: maj = maj and c */ \ | |
| add c,c,g; /* 2: d = d + temp1 */ \ | |
| or rT2,rT1,rT2; /* 2: maj = maj or maj' */ \ | |
| add rT3,rT3,rT2; /* 2: temp2 = S0 + maj */ \ | |
| add g,g,rT3 /* 2: h = temp1 + temp2 */ | |
| #define R_CALC_W(a, b, c, d, e, f, g, h, w0, w1, w4, w5, w7, k, off) \ | |
| rotrwi rT2,e,6; /* 1: S1 = e rotr 6 */ \ | |
| evmergelohi rT0,w0,w1; /* w[-15] */ \ | |
| rotrwi rT3,e,11; /* 1: S1' = e rotr 11 */ \ | |
| evsrwiu rT1,rT0,3; /* s0 = w[-15] >> 3 */ \ | |
| xor rT2,rT2,rT3; /* 1: S1 = S1 xor S1' */ \ | |
| evrlwi rT0,rT0,25; /* s0' = w[-15] rotr 7 */ \ | |
| rotrwi rT3,e,25; /* 1: S1' = e rotr 25 */ \ | |
| evxor rT1,rT1,rT0; /* s0 = s0 xor s0' */ \ | |
| xor rT2,rT2,rT3; /* 1: S1 = S1 xor S1' */ \ | |
| evrlwi rT0,rT0,21; /* s0' = w[-15] rotr 18 */ \ | |
| add h,h,rT2; /* 1: temp1 = h + S1 */ \ | |
| evxor rT0,rT0,rT1; /* s0 = s0 xor s0' */ \ | |
| and rT2,e,f; /* 1: ch = e and f */ \ | |
| evaddw w0,w0,rT0; /* w = w[-16] + s0 */ \ | |
| andc rT3,g,e; /* 1: ch' = ~e and g */ \ | |
| evsrwiu rT0,w7,10; /* s1 = w[-2] >> 10 */ \ | |
| xor rT2,rT2,rT3; /* 1: ch = ch xor ch' */ \ | |
| evrlwi rT1,w7,15; /* s1' = w[-2] rotr 17 */ \ | |
| add h,h,rT2; /* 1: temp1 = temp1 + ch */ \ | |
| evxor rT0,rT0,rT1; /* s1 = s1 xor s1' */ \ | |
| rotrwi rT2,a,2; /* 1: S0 = a rotr 2 */ \ | |
| evrlwi rT1,w7,13; /* s1' = w[-2] rotr 19 */ \ | |
| rotrwi rT3,a,13; /* 1: S0' = a rotr 13 */ \ | |
| evxor rT0,rT0,rT1; /* s1 = s1 xor s1' */ \ | |
| xor rT2,rT2,rT3; /* 1: S0 = S0 xor S0' */ \ | |
| evldw rT1,off(rKP); /* k */ \ | |
| rotrwi rT3,a,22; /* 1: S0' = a rotr 22 */ \ | |
| evaddw w0,w0,rT0; /* w = w + s1 */ \ | |
| xor rT2,rT2,rT3; /* 1: S0 = S0 xor S0' */ \ | |
| evmergelohi rT0,w4,w5; /* w[-7] */ \ | |
| and rT3,a,b; /* 1: maj = a and b */ \ | |
| evaddw w0,w0,rT0; /* w = w + w[-7] */ \ | |
| CMP_K##k##_LOOP \ | |
| add rT2,rT2,rT3; /* 1: temp2 = S0 + maj */ \ | |
| evaddw rT1,rT1,w0; /* wk = w + k */ \ | |
| xor rT3,a,b; /* 1: maj = a xor b */ \ | |
| evmergehi rT0,rT1,rT1; /* wk1/wk2 */ \ | |
| and rT3,rT3,c; /* 1: maj = maj and c */ \ | |
| add h,h,rT0; /* 1: temp1 = temp1 + wk */ \ | |
| add rT2,rT2,rT3; /* 1: temp2 = temp2 + maj */ \ | |
| add g,g,rT1; /* 2: temp1 = temp1 + wk */ \ | |
| add d,d,h; /* 1: d = d + temp1 */ \ | |
| rotrwi rT0,d,6; /* 2: S1 = e rotr 6 */ \ | |
| add h,h,rT2; /* 1: h = temp1 + temp2 */ \ | |
| rotrwi rT1,d,11; /* 2: S1' = e rotr 11 */ \ | |
| rotrwi rT2,d,25; /* 2: S" = e rotr 25 */ \ | |
| xor rT0,rT0,rT1; /* 2: S1 = S1 xor S1' */ \ | |
| and rT3,d,e; /* 2: ch = e and f */ \ | |
| xor rT0,rT0,rT2; /* 2: S1 = S1 xor S1" */ \ | |
| andc rT1,f,d; /* 2: ch' = ~e and g */ \ | |
| add g,g,rT0; /* 2: temp1 = h + S1 */ \ | |
| xor rT3,rT3,rT1; /* 2: ch = ch xor ch' */ \ | |
| rotrwi rT0,h,2; /* 2: S0 = a rotr 2 */ \ | |
| add g,g,rT3; /* 2: temp1 = temp1 + ch */ \ | |
| rotrwi rT1,h,13; /* 2: S0' = a rotr 13 */ \ | |
| rotrwi rT3,h,22; /* 2: S0" = a rotr 22 */ \ | |
| xor rT0,rT0,rT1; /* 2: S0 = S0 xor S0' */ \ | |
| or rT2,h,a; /* 2: maj = a or b */ \ | |
| and rT1,h,a; /* 2: maj' = a and b */ \ | |
| and rT2,rT2,b; /* 2: maj = maj and c */ \ | |
| xor rT3,rT0,rT3; /* 2: S0 = S0 xor S0" */ \ | |
| or rT2,rT1,rT2; /* 2: maj = maj or maj' */ \ | |
| add c,c,g; /* 2: d = d + temp1 */ \ | |
| add rT3,rT3,rT2; /* 2: temp2 = S0 + maj */ \ | |
| add g,g,rT3 /* 2: h = temp1 + temp2 */ | |
| _GLOBAL(ppc_spe_sha256_transform) | |
| INITIALIZE | |
| mtctr r5 | |
| lwz rH0,0(rHP) | |
| lwz rH1,4(rHP) | |
| lwz rH2,8(rHP) | |
| lwz rH3,12(rHP) | |
| lwz rH4,16(rHP) | |
| lwz rH5,20(rHP) | |
| lwz rH6,24(rHP) | |
| lwz rH7,28(rHP) | |
| ppc_spe_sha256_main: | |
| lis rKP,PPC_SPE_SHA256_K@ha | |
| addi rKP,rKP,PPC_SPE_SHA256_K@l | |
| R_LOAD_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, rW0, 0) | |
| R_LOAD_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, rW1, 8) | |
| R_LOAD_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, rW2, 16) | |
| R_LOAD_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, rW3, 24) | |
| R_LOAD_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, rW4, 32) | |
| R_LOAD_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, rW5, 40) | |
| R_LOAD_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, rW6, 48) | |
| R_LOAD_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, rW7, 56) | |
| ppc_spe_sha256_16_rounds: | |
| addi rKP,rKP,64 | |
| R_CALC_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, | |
| rW0, rW1, rW4, rW5, rW7, N, 0) | |
| R_CALC_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, | |
| rW1, rW2, rW5, rW6, rW0, N, 8) | |
| R_CALC_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, | |
| rW2, rW3, rW6, rW7, rW1, N, 16) | |
| R_CALC_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, | |
| rW3, rW4, rW7, rW0, rW2, N, 24) | |
| R_CALC_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, | |
| rW4, rW5, rW0, rW1, rW3, N, 32) | |
| R_CALC_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, | |
| rW5, rW6, rW1, rW2, rW4, N, 40) | |
| R_CALC_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, | |
| rW6, rW7, rW2, rW3, rW5, N, 48) | |
| R_CALC_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, | |
| rW7, rW0, rW3, rW4, rW6, C, 56) | |
| bt gt,ppc_spe_sha256_16_rounds | |
| lwz rW0,0(rHP) | |
| NEXT_BLOCK | |
| lwz rW1,4(rHP) | |
| lwz rW2,8(rHP) | |
| lwz rW3,12(rHP) | |
| lwz rW4,16(rHP) | |
| lwz rW5,20(rHP) | |
| lwz rW6,24(rHP) | |
| lwz rW7,28(rHP) | |
| add rH0,rH0,rW0 | |
| stw rH0,0(rHP) | |
| add rH1,rH1,rW1 | |
| stw rH1,4(rHP) | |
| add rH2,rH2,rW2 | |
| stw rH2,8(rHP) | |
| add rH3,rH3,rW3 | |
| stw rH3,12(rHP) | |
| add rH4,rH4,rW4 | |
| stw rH4,16(rHP) | |
| add rH5,rH5,rW5 | |
| stw rH5,20(rHP) | |
| add rH6,rH6,rW6 | |
| stw rH6,24(rHP) | |
| add rH7,rH7,rW7 | |
| stw rH7,28(rHP) | |
| bdnz ppc_spe_sha256_main | |
| FINALIZE | |
| blr | |
| .data | |
| .align 5 | |
| PPC_SPE_SHA256_K: | |
| .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 | |
| .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 | |
| .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 | |
| .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 | |
| .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc | |
| .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da | |
| .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 | |
| .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 | |
| .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 | |
| .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 | |
| .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 | |
| .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 | |
| .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 | |
| .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 | |
| .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 | |
| .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 |