/* * This source file is licensed under the Apache License 2.0 *and* the MIT * License. Please agree to *both* of the licensing terms! * * * `transformH` function is a derivative work of OpenSSL. The original work * is covered by the following license: * * Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy * in the file LICENSE in the source distribution or at * https://www.openssl.org/source/license.html * * * All other work, including modifications to the `transformH` function is * covered by the following MIT license: * * Copyright (c) 2020-2022 Fastly, Kazuho Oku * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to * deal in the Software without restriction, including without limitation the * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ #include #include #include #include #include #include #include #include "picotls.h" #include "picotls/fusion.h" #if defined(__clang__) #if __has_feature(address_sanitizer) #define NO_SANITIZE_ADDRESS __attribute__((no_sanitize("address"))) #endif #elif __SANITIZE_ADDRESS__ /* gcc */ #define NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) #endif #ifndef NO_SANITIZE_ADDRESS #define NO_SANITIZE_ADDRESS #endif #ifdef _WINDOWS #define aligned_alloc(a, s) _aligned_malloc((s), (a)) #define aligned_free(p) _aligned_free(p) #else #define aligned_free(p) free(p) #endif struct ptls_fusion_aesgcm_context { ptls_fusion_aesecb_context_t ecb; size_t capacity; size_t ghash_cnt; }; struct ptls_fusion_aesgcm_context128 { struct ptls_fusion_aesgcm_context super; struct ptls_fusion_aesgcm_ghash_precompute128 { __m128i H; __m128i r; } ghash[0]; }; struct ptls_fusion_aesgcm_context256 { struct ptls_fusion_aesgcm_context super; union ptls_fusion_aesgcm_ghash_precompute256 { struct { __m128i H[2]; __m128i r[2]; }; struct { __m256i Hx2; __m256i rx2; }; } ghash[0]; }; struct ctr_context { ptls_cipher_context_t super; ptls_fusion_aesecb_context_t fusion; __m128i bits; uint8_t is_ready; }; struct aesgcm_context { ptls_aead_context_t super; ptls_fusion_aesgcm_context_t *aesgcm; /** * retains the static IV in the upper 96 bits (in little endian) */ __m128i static_iv; }; static const uint64_t poly_[2] __attribute__((aligned(16))) = {1, 0xc200000000000000}; #define poly (*(__m128i *)poly_) static const uint8_t byteswap_[32] __attribute__((aligned(32))) = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; #define byteswap128 (*(__m128i *)byteswap_) #define byteswap256 (*(__m256i *)byteswap_) static const uint8_t one_[16] __attribute__((aligned(16))) = {1}; #define one8 (*(__m128i *)one_) static const uint8_t incr128x2_[32] __attribute__((aligned(32))) = {2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}; #define incr128x2 (*(__m256i *)incr128x2_) /* This function is covered by the Apache License and the MIT License. The origin is crypto/modes/asm/ghash-x86_64.pl of openssl * at commit 33388b4. */ static __m128i transformH(__m128i H) { // # <<1 twist // pshufd \$0b11111111,$Hkey,$T2 # broadcast uppermost dword __m128i t2 = _mm_shuffle_epi32(H, 0xff); // movdqa $Hkey,$T1 __m128i t1 = H; // psllq \$1,$Hkey H = _mm_slli_epi64(H, 1); // pxor $T3,$T3 # __m128i t3 = _mm_setzero_si128(); // psrlq \$63,$T1 t1 = _mm_srli_epi64(t1, 63); // pcmpgtd $T2,$T3 # broadcast carry bit t3 = _mm_cmplt_epi32(t2, t3); // pslldq \$8,$T1 t1 = _mm_slli_si128(t1, 8); // por $T1,$Hkey # H<<=1 H = _mm_or_si128(t1, H); // # magic reduction // pand .L0x1c2_polynomial(%rip),$T3 t3 = _mm_and_si128(t3, poly); // pxor $T3,$Hkey # if(carry) H^=0x1c2_polynomial H = _mm_xor_si128(t3, H); return H; } // end of Apache License code static __m128i gfmul(__m128i x, __m128i y) { __m128i lo = _mm_clmulepi64_si128(x, y, 0x00); __m128i hi = _mm_clmulepi64_si128(x, y, 0x11); __m128i a = _mm_shuffle_epi32(x, 78); __m128i b = _mm_shuffle_epi32(y, 78); a = _mm_xor_si128(a, x); b = _mm_xor_si128(b, y); a = _mm_clmulepi64_si128(a, b, 0x00); a = _mm_xor_si128(a, lo); a = _mm_xor_si128(a, hi); b = _mm_slli_si128(a, 8); a = _mm_srli_si128(a, 8); lo = _mm_xor_si128(lo, b); hi = _mm_xor_si128(hi, a); // from https://crypto.stanford.edu/RealWorldCrypto/slides/gueron.pdf __m128i t = _mm_clmulepi64_si128(lo, poly, 0x10); lo = _mm_shuffle_epi32(lo, 78); lo = _mm_xor_si128(lo, t); t = _mm_clmulepi64_si128(lo, poly, 0x10); lo = _mm_shuffle_epi32(lo, 78); lo = _mm_xor_si128(lo, t); return _mm_xor_si128(hi, lo); } static inline __m128i gfmul_do_reduce(__m128i hi, __m128i lo, __m128i mid) { mid = _mm_xor_si128(mid, hi); mid = _mm_xor_si128(mid, lo); lo = _mm_xor_si128(lo, _mm_slli_si128(mid, 8)); hi = _mm_xor_si128(hi, _mm_srli_si128(mid, 8)); /* fast reduction, using https://crypto.stanford.edu/RealWorldCrypto/slides/gueron.pdf */ __m128i r = _mm_clmulepi64_si128(lo, poly, 0x10); lo = _mm_shuffle_epi32(lo, 78); lo = _mm_xor_si128(lo, r); r = _mm_clmulepi64_si128(lo, poly, 0x10); lo = _mm_shuffle_epi32(lo, 78); lo = _mm_xor_si128(lo, r); lo = _mm_xor_si128(hi, lo); return lo; } struct ptls_fusion_gfmul_state128 { __m128i hi, lo, mid; }; #if defined(__GNUC__) && !defined(__clang__) static inline __m128i xor128(__m128i x, __m128i y) { __m128i ret; __asm__("vpxor %2, %1, %0" : "=x"(ret) : "x"(x), "xm"(y)); return ret; } #else #define xor128 _mm_xor_si128 #endif static inline void gfmul_do_step128(struct ptls_fusion_gfmul_state128 *gstate, __m128i X, struct ptls_fusion_aesgcm_ghash_precompute128 *precompute) { __m128i t1 = _mm_clmulepi64_si128(precompute->H, X, 0x00); __m128i t2 = _mm_clmulepi64_si128(precompute->H, X, 0x11); __m128i t3 = _mm_shuffle_epi32(X, 78); t3 = _mm_xor_si128(t3, X); t3 = _mm_clmulepi64_si128(precompute->r, t3, 0x00); gstate->lo = xor128(gstate->lo, t1); gstate->hi = xor128(gstate->hi, t2); gstate->mid = xor128(gstate->mid, t3); } #undef xor128 static inline void gfmul_firststep128(struct ptls_fusion_gfmul_state128 *gstate, __m128i X, struct ptls_fusion_aesgcm_ghash_precompute128 *precompute) { X = _mm_shuffle_epi8(X, byteswap128); X = _mm_xor_si128(gstate->lo, X); gstate->lo = _mm_setzero_si128(); gstate->hi = _mm_setzero_si128(); gstate->mid = _mm_setzero_si128(); gfmul_do_step128(gstate, X, precompute); } static inline void gfmul_nextstep128(struct ptls_fusion_gfmul_state128 *gstate, __m128i X, struct ptls_fusion_aesgcm_ghash_precompute128 *precompute) { X = _mm_shuffle_epi8(X, byteswap128); gfmul_do_step128(gstate, X, precompute); } static inline void gfmul_reduce128(struct ptls_fusion_gfmul_state128 *gstate) { gstate->lo = gfmul_do_reduce(gstate->hi, gstate->lo, gstate->mid); } static inline __m128i gfmul_get_tag128(struct ptls_fusion_gfmul_state128 *gstate, __m128i ek0) { __m128i tag = _mm_shuffle_epi8(gstate->lo, byteswap128); tag = _mm_xor_si128(tag, ek0); return tag; } struct ptls_fusion_gfmul_state256 { __m256i hi, lo, mid; }; static inline void gfmul_do_step256(struct ptls_fusion_gfmul_state256 *gstate, __m256i X, union ptls_fusion_aesgcm_ghash_precompute256 *precompute) { __m256i t = _mm256_clmulepi64_epi128(precompute->Hx2, X, 0x00); gstate->lo = _mm256_xor_si256(gstate->lo, t); t = _mm256_clmulepi64_epi128(precompute->Hx2, X, 0x11); gstate->hi = _mm256_xor_si256(gstate->hi, t); t = _mm256_shuffle_epi32(X, 78); t = _mm256_xor_si256(t, X); t = _mm256_clmulepi64_epi128(precompute->rx2, t, 0x00); gstate->mid = _mm256_xor_si256(gstate->mid, t); } static inline void gfmul_firststep256(struct ptls_fusion_gfmul_state256 *gstate, __m256i X, int half, union ptls_fusion_aesgcm_ghash_precompute256 *precompute) { X = _mm256_shuffle_epi8(X, byteswap256); X = _mm256_xor_si256(gstate->lo, X); if (half) X = _mm256_permute2f128_si256(X, X, 0x08); gstate->lo = _mm256_setzero_si256(); gstate->hi = _mm256_setzero_si256(); gstate->mid = _mm256_setzero_si256(); gfmul_do_step256(gstate, X, precompute); } static inline void gfmul_nextstep256(struct ptls_fusion_gfmul_state256 *gstate, __m256i X, union ptls_fusion_aesgcm_ghash_precompute256 *precompute) { X = _mm256_shuffle_epi8(X, byteswap256); gfmul_do_step256(gstate, X, precompute); } static inline void gfmul_reduce256(struct ptls_fusion_gfmul_state256 *gstate) { #define XOR_256TO128(y) _mm_xor_si128(_mm256_castsi256_si128(y), _mm256_extractf128_si256((y), 1)) __m128i hi = XOR_256TO128(gstate->hi); __m128i lo = XOR_256TO128(gstate->lo); __m128i mid = XOR_256TO128(gstate->mid); #undef XOR_256TO128 lo = gfmul_do_reduce(hi, lo, mid); gstate->lo = _mm256_castsi128_si256(lo); } static inline __m128i gfmul_get_tag256(struct ptls_fusion_gfmul_state256 *gstate, __m128i ek0) { __m128i tag = _mm_shuffle_epi8(_mm256_castsi256_si128(gstate->lo), byteswap128); tag = _mm_xor_si128(tag, ek0); return tag; } static inline __m128i aesecb_encrypt(ptls_fusion_aesecb_context_t *ctx, __m128i v) { #define ROUNDKEY(i) (ctx->aesni256 ? _mm256_castsi256_si128(ctx->keys.m256[i]) : ctx->keys.m128[i]) v = _mm_xor_si128(v, ROUNDKEY(0)); for (size_t i = 1; i < ctx->rounds; ++i) v = _mm_aesenc_si128(v, ROUNDKEY(i)); v = _mm_aesenclast_si128(v, ROUNDKEY(ctx->rounds)); return v; #undef ROUNDKEY } // 32-bytes of 0xff followed by 31-bytes of 0x00 static const uint8_t loadn_mask[63] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; static const uint8_t loadn_shuffle[31] = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, // first 16 bytes map to byte offsets 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; // latter 15 bytes map to zero NO_SANITIZE_ADDRESS static inline __m128i loadn_end_of_page(const void *p, size_t l) { uintptr_t shift = (uintptr_t)p & 15; __m128i pattern = _mm_loadu_si128((const __m128i *)(loadn_shuffle + shift)); return _mm_shuffle_epi8(_mm_load_si128((const __m128i *)((uintptr_t)p - shift)), pattern); } NO_SANITIZE_ADDRESS static inline __m128i loadn128(const void *p, size_t l) { __m128i v, mask = _mm_loadu_si128((__m128i *)(loadn_mask + 32 - l)); uintptr_t mod4k = (uintptr_t)p % 4096; if (PTLS_LIKELY(mod4k <= 4096 - 16) || mod4k + l > 4096) { v = _mm_loadu_si128(p); } else { v = loadn_end_of_page(p, l); } v = _mm_and_si128(v, mask); return v; } NO_SANITIZE_ADDRESS static inline __m256i loadn256(const void *p, size_t l) { __m256i v, mask = _mm256_loadu_si256((__m256i *)(loadn_mask + 32 - l)); uintptr_t mod4k = (uintptr_t)p % 4096; if (PTLS_LIKELY(mod4k < 4096 - 32) || mod4k + l > 4096) { v = _mm256_loadu_si256(p); } else if (l > 16) { __m128i first16 = _mm_loadu_si128(p), second16 = loadn128((uint8_t *)p + 16, l - 16); v = _mm256_permute2f128_si256(_mm256_castsi128_si256(first16), _mm256_castsi128_si256(second16), 0x20); } else if (l == 16) { v = _mm256_castsi128_si256(_mm_loadu_si128(p)); } else { v = _mm256_castsi128_si256(loadn_end_of_page(p, l)); } v = _mm256_and_si256(v, mask); return v; } static inline void storen128(void *_p, size_t l, __m128i v) { uint8_t buf[16], *p = _p; *(__m128i *)buf = v; for (size_t i = 0; i != l; ++i) p[i] = buf[i]; } void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *_ctx, void *output, const void *input, size_t inlen, __m128i ctr, const void *_aad, size_t aadlen, ptls_aead_supplementary_encryption_t *supp) { /* init the bits (we can always run in full), but use the last slot for calculating ek0, if possible */ #define AESECB6_INIT() \ do { \ ctr = _mm_add_epi64(ctr, one8); \ bits0 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ bits1 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ bits2 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ bits3 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ bits4 = _mm_shuffle_epi8(ctr, byteswap128); \ if (PTLS_LIKELY(srclen > 16 * 5)) { \ ctr = _mm_add_epi64(ctr, one8); \ bits5 = _mm_shuffle_epi8(ctr, byteswap128); \ } else { \ if ((state & STATE_EK0_BEEN_FED) == 0) { \ bits5 = ek0; \ state |= STATE_EK0_BEEN_FED; \ } \ if ((state & STATE_SUPP_USED) != 0 && srclen <= 16 * 4 && (const __m128i *)supp->input + 1 <= dst_ghash) { \ bits4 = _mm_loadu_si128(supp->input); \ bits4keys = ((struct ctr_context *)supp->ctx)->fusion.keys.m128; \ state |= STATE_SUPP_IN_PROCESS; \ } \ } \ __m128i k = ctx->super.ecb.keys.m128[0]; \ bits0 = _mm_xor_si128(bits0, k); \ bits1 = _mm_xor_si128(bits1, k); \ bits2 = _mm_xor_si128(bits2, k); \ bits3 = _mm_xor_si128(bits3, k); \ bits4 = _mm_xor_si128(bits4, bits4keys[0]); \ bits5 = _mm_xor_si128(bits5, k); \ } while (0) /* aes block update */ #define AESECB6_UPDATE(i) \ do { \ __m128i k = ctx->super.ecb.keys.m128[i]; \ bits0 = _mm_aesenc_si128(bits0, k); \ bits1 = _mm_aesenc_si128(bits1, k); \ bits2 = _mm_aesenc_si128(bits2, k); \ bits3 = _mm_aesenc_si128(bits3, k); \ bits4 = _mm_aesenc_si128(bits4, bits4keys[i]); \ bits5 = _mm_aesenc_si128(bits5, k); \ } while (0) /* aesenclast */ #define AESECB6_FINAL(i) \ do { \ __m128i k = ctx->super.ecb.keys.m128[i]; \ bits0 = _mm_aesenclast_si128(bits0, k); \ bits1 = _mm_aesenclast_si128(bits1, k); \ bits2 = _mm_aesenclast_si128(bits2, k); \ bits3 = _mm_aesenclast_si128(bits3, k); \ bits4 = _mm_aesenclast_si128(bits4, bits4keys[i]); \ bits5 = _mm_aesenclast_si128(bits5, k); \ } while (0) struct ptls_fusion_aesgcm_context128 *ctx = (void *)_ctx; __m128i ek0, bits0, bits1, bits2, bits3, bits4, bits5 = _mm_setzero_si128(); const __m128i *bits4keys = ctx->super.ecb.keys.m128; /* is changed to supp->ctx.keys when calcurating suppout */ struct ptls_fusion_gfmul_state128 gstate = {0}; __m128i gdatabuf[6]; __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)inlen * 8), byteswap128); // src and dst are updated after the chunk is processed const __m128i *src = input; __m128i *dst = output; size_t srclen = inlen; // aad and src_ghash are updated before the chunk is processed (i.e., when the pointers are fed indo the processor) const __m128i *aad = _aad, *dst_ghash = dst; size_t dst_ghashlen = srclen; struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash + (aadlen + 15) / 16 + (srclen + 15) / 16 + 1; #define STATE_EK0_BEEN_FED 0x3 #define STATE_EK0_INCOMPLETE 0x2 #define STATE_EK0_READY() ((state & STATE_EK0_BEEN_FED) == 0x1) #define STATE_SUPP_USED 0x4 #define STATE_SUPP_IN_PROCESS 0x8 int32_t state = supp != NULL ? STATE_SUPP_USED : 0; /* build counter */ ctr = _mm_insert_epi32(ctr, 1, 0); ek0 = _mm_shuffle_epi8(ctr, byteswap128); /* start preparing AES */ AESECB6_INIT(); AESECB6_UPDATE(1); /* build first ghash data (only AAD can be fed at this point, as this would be calculated alongside the first AES block) */ const __m128i *gdata = gdatabuf; // points to the elements fed into GHASH size_t gdata_cnt = 0; if (PTLS_LIKELY(aadlen != 0)) { while (gdata_cnt < 6) { if (PTLS_LIKELY(aadlen < 16)) { if (aadlen != 0) { gdatabuf[gdata_cnt++] = loadn128(aad, aadlen); aadlen = 0; } goto MainLoop; } gdatabuf[gdata_cnt++] = _mm_loadu_si128(aad++); aadlen -= 16; } } /* the main loop */ MainLoop: while (1) { /* run AES and multiplication in parallel */ size_t i; for (i = 2; i < gdata_cnt + 2; ++i) { AESECB6_UPDATE(i); gfmul_nextstep128(&gstate, _mm_loadu_si128(gdata++), --ghash_precompute); } for (; i < ctx->super.ecb.rounds; ++i) AESECB6_UPDATE(i); AESECB6_FINAL(i); /* apply the bit stream to src and write to dest */ if (PTLS_LIKELY(srclen >= 6 * 16)) { #define APPLY(i) _mm_storeu_si128(dst + i, _mm_xor_si128(_mm_loadu_si128(src + i), bits##i)) APPLY(0); APPLY(1); APPLY(2); APPLY(3); APPLY(4); APPLY(5); #undef APPLY dst += 6; src += 6; srclen -= 6 * 16; } else { if ((state & STATE_EK0_BEEN_FED) == STATE_EK0_BEEN_FED) { ek0 = bits5; state &= ~STATE_EK0_INCOMPLETE; } if ((state & STATE_SUPP_IN_PROCESS) != 0) { _mm_storeu_si128((__m128i *)supp->output, bits4); state &= ~(STATE_SUPP_USED | STATE_SUPP_IN_PROCESS); } if (srclen != 0) { #define APPLY(i) \ do { \ if (PTLS_LIKELY(srclen >= 16)) { \ _mm_storeu_si128(dst++, _mm_xor_si128(_mm_loadu_si128(src++), bits##i)); \ srclen -= 16; \ } else if (PTLS_LIKELY(srclen != 0)) { \ bits0 = bits##i; \ goto ApplyRemainder; \ } else { \ goto ApplyEnd; \ } \ } while (0) APPLY(0); APPLY(1); APPLY(2); APPLY(3); APPLY(4); APPLY(5); #undef APPLY goto ApplyEnd; ApplyRemainder: storen128(dst, srclen, _mm_xor_si128(loadn128(src, srclen), bits0)); dst = (__m128i *)((uint8_t *)dst + srclen); srclen = 0; ApplyEnd:; } } /* next block AES starts here */ AESECB6_INIT(); AESECB6_UPDATE(1); /* setup gdata */ if (PTLS_UNLIKELY(aadlen != 0)) { gdata_cnt = 0; while (gdata_cnt < 6) { if (aadlen < 16) { if (aadlen != 0) { gdatabuf[gdata_cnt++] = loadn128(aad, aadlen); aadlen = 0; } goto GdataFillDST; } gdatabuf[gdata_cnt++] = _mm_loadu_si128(aad++); aadlen -= 16; } gdata = gdatabuf; } else if (PTLS_LIKELY(dst_ghashlen >= 6 * 16)) { gdata = dst_ghash; gdata_cnt = 6; dst_ghash += 6; dst_ghashlen -= 96; } else { gdata_cnt = 0; GdataFillDST: while (gdata_cnt < 6) { if (dst_ghashlen < 16) { if (dst_ghashlen != 0) { gdatabuf[gdata_cnt++] = loadn128(dst_ghash, dst_ghashlen); dst_ghashlen = 0; } if (gdata_cnt < 6) goto Finish; break; } gdatabuf[gdata_cnt++] = _mm_loadu_si128(dst_ghash++); dst_ghashlen -= 16; } gdata = gdatabuf; } } Finish: gdatabuf[gdata_cnt++] = ac; /* We have complete set of data to be fed into GHASH. Let's finish the remaining calculation. * Note that by now, all AES operations for payload encryption and ek0 are complete. This is is because it is necessary for GCM * to process at least the same amount of data (i.e. payload-blocks + AC), and because AES is at least one 96-byte block ahead. */ assert(STATE_EK0_READY()); for (size_t i = 0; i < gdata_cnt; ++i) gfmul_nextstep128(&gstate, gdatabuf[i], --ghash_precompute); gfmul_reduce128(&gstate); _mm_storeu_si128(dst, gfmul_get_tag128(&gstate, ek0)); /* Finish the calculation of supplemental vector. Done at the very last, because the sample might cover the GCM tag. */ if ((state & STATE_SUPP_USED) != 0) { size_t i; if ((state & STATE_SUPP_IN_PROCESS) == 0) { bits4keys = ((struct ctr_context *)supp->ctx)->fusion.keys.m128; bits4 = _mm_xor_si128(_mm_loadu_si128(supp->input), bits4keys[0]); i = 1; } else { i = 2; } do { bits4 = _mm_aesenc_si128(bits4, bits4keys[i++]); } while (i != ctx->super.ecb.rounds); bits4 = _mm_aesenclast_si128(bits4, bits4keys[i]); _mm_storeu_si128((__m128i *)supp->output, bits4); } #undef AESECB6_INIT #undef AESECB6_UPDATE #undef AESECB6_FINAL #undef STATE_EK0_BEEN_FOUND #undef STATE_EK0_READY #undef STATE_SUPP_IN_PROCESS } int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *_ctx, void *output, const void *input, size_t inlen, __m128i ctr, const void *_aad, size_t aadlen, const void *tag) { struct ptls_fusion_aesgcm_context128 *ctx = (void *)_ctx; __m128i ek0 = _mm_setzero_si128(), bits0, bits1 = _mm_setzero_si128(), bits2 = _mm_setzero_si128(), bits3 = _mm_setzero_si128(), bits4 = _mm_setzero_si128(), bits5 = _mm_setzero_si128(); struct ptls_fusion_gfmul_state128 gstate = {0}; __m128i gdatabuf[6]; __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)inlen * 8), byteswap128); struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash + (aadlen + 15) / 16 + (inlen + 15) / 16 + 1; const __m128i *gdata; // points to the elements fed into GHASH size_t gdata_cnt; const __m128i *src_ghash = input, *src_aes = input, *aad = _aad; __m128i *dst = output; size_t nondata_aes_cnt = 0, src_ghashlen = inlen, src_aeslen = inlen; /* schedule ek0 and suppkey */ ctr = _mm_add_epi64(ctr, one8); bits0 = _mm_xor_si128(_mm_shuffle_epi8(ctr, byteswap128), ctx->super.ecb.keys.m128[0]); ++nondata_aes_cnt; #define STATE_IS_FIRST_RUN 0x1 #define STATE_GHASH_HAS_MORE 0x2 int state = STATE_IS_FIRST_RUN | STATE_GHASH_HAS_MORE; /* the main loop */ while (1) { /* setup gdata */ if (PTLS_UNLIKELY(aadlen != 0)) { gdata = gdatabuf; gdata_cnt = 0; while (gdata_cnt < 6) { if (aadlen < 16) { if (aadlen != 0) { gdatabuf[gdata_cnt++] = loadn128(aad, aadlen); aadlen = 0; ++nondata_aes_cnt; } goto GdataFillSrc; } gdatabuf[gdata_cnt++] = _mm_loadu_si128(aad++); aadlen -= 16; ++nondata_aes_cnt; } } else if (PTLS_LIKELY(src_ghashlen >= 6 * 16)) { gdata = src_ghash; gdata_cnt = 6; src_ghash += 6; src_ghashlen -= 6 * 16; } else { gdata = gdatabuf; gdata_cnt = 0; GdataFillSrc: while (gdata_cnt < 6) { if (src_ghashlen < 16) { if (src_ghashlen != 0) { gdatabuf[gdata_cnt++] = loadn128(src_ghash, src_ghashlen); src_ghash = (__m128i *)((uint8_t *)src_ghash + src_ghashlen); src_ghashlen = 0; } if (gdata_cnt < 6 && (state & STATE_GHASH_HAS_MORE) != 0) { gdatabuf[gdata_cnt++] = ac; state &= ~STATE_GHASH_HAS_MORE; } break; } gdatabuf[gdata_cnt++] = _mm_loadu_si128(src_ghash++); src_ghashlen -= 16; } } /* setup aes bits */ if (PTLS_LIKELY(nondata_aes_cnt == 0)) goto InitAllBits; switch (nondata_aes_cnt) { #define INIT_BITS(n, keys) \ case n: \ ctr = _mm_add_epi64(ctr, one8); \ bits##n = _mm_xor_si128(_mm_shuffle_epi8(ctr, byteswap128), keys.m128[0]); InitAllBits: INIT_BITS(0, ctx->super.ecb.keys); INIT_BITS(1, ctx->super.ecb.keys); INIT_BITS(2, ctx->super.ecb.keys); INIT_BITS(3, ctx->super.ecb.keys); INIT_BITS(4, ctx->super.ecb.keys); INIT_BITS(5, ctx->super.ecb.keys); #undef INIT_BITS } { /* run aes and ghash */ #define AESECB6_UPDATE(i) \ do { \ __m128i k = ctx->super.ecb.keys.m128[i]; \ bits0 = _mm_aesenc_si128(bits0, k); \ bits1 = _mm_aesenc_si128(bits1, k); \ bits2 = _mm_aesenc_si128(bits2, k); \ bits3 = _mm_aesenc_si128(bits3, k); \ bits4 = _mm_aesenc_si128(bits4, k); \ bits5 = _mm_aesenc_si128(bits5, k); \ } while (0) size_t aesi; for (aesi = 1; aesi <= gdata_cnt; ++aesi) { AESECB6_UPDATE(aesi); gfmul_nextstep128(&gstate, _mm_loadu_si128(gdata++), --ghash_precompute); } for (; aesi < ctx->super.ecb.rounds; ++aesi) AESECB6_UPDATE(aesi); __m128i k = ctx->super.ecb.keys.m128[aesi]; bits0 = _mm_aesenclast_si128(bits0, k); bits1 = _mm_aesenclast_si128(bits1, k); bits2 = _mm_aesenclast_si128(bits2, k); bits3 = _mm_aesenclast_si128(bits3, k); bits4 = _mm_aesenclast_si128(bits4, k); bits5 = _mm_aesenclast_si128(bits5, k); #undef AESECB6_UPDATE } /* apply aes bits */ if (PTLS_LIKELY(nondata_aes_cnt == 0 && src_aeslen >= 6 * 16)) { #define APPLY(i) _mm_storeu_si128(dst + i, _mm_xor_si128(_mm_loadu_si128(src_aes + i), bits##i)) APPLY(0); APPLY(1); APPLY(2); APPLY(3); APPLY(4); APPLY(5); #undef APPLY dst += 6; src_aes += 6; src_aeslen -= 6 * 16; } else { if ((state & STATE_IS_FIRST_RUN) != 0) { ek0 = bits0; state &= ~STATE_IS_FIRST_RUN; } switch (nondata_aes_cnt) { #define APPLY(i) \ case i: \ if (PTLS_LIKELY(src_aeslen > 16)) { \ _mm_storeu_si128(dst++, _mm_xor_si128(_mm_loadu_si128(src_aes++), bits##i)); \ src_aeslen -= 16; \ } else { \ bits0 = bits##i; \ goto Finish; \ } APPLY(0); APPLY(1); APPLY(2); APPLY(3); APPLY(4); APPLY(5); #undef APPLY } nondata_aes_cnt = 0; } } Finish: if (src_aeslen == 16) { _mm_storeu_si128(dst, _mm_xor_si128(_mm_loadu_si128(src_aes), bits0)); } else if (src_aeslen != 0) { storen128(dst, src_aeslen, _mm_xor_si128(loadn128(src_aes, src_aeslen), bits0)); } assert((state & STATE_IS_FIRST_RUN) == 0); /* the only case where AES operation is complete and GHASH is not is when the application of AC is remaining */ if ((state & STATE_GHASH_HAS_MORE) != 0) { assert(ghash_precompute - 1 == ctx->ghash); gfmul_nextstep128(&gstate, ac, --ghash_precompute); } gfmul_reduce128(&gstate); __m128i calctag = gfmul_get_tag128(&gstate, ek0); return _mm_movemask_epi8(_mm_cmpeq_epi8(calctag, _mm_loadu_si128(tag))) == 0xffff; #undef STATE_IS_FIRST_RUN #undef STATE_GHASH_HAS_MORE } static __m128i expand_key(__m128i key, __m128i temp) { key = _mm_xor_si128(key, _mm_slli_si128(key, 4)); key = _mm_xor_si128(key, _mm_slli_si128(key, 4)); key = _mm_xor_si128(key, _mm_slli_si128(key, 4)); key = _mm_xor_si128(key, temp); return key; } void ptls_fusion_aesecb_init(ptls_fusion_aesecb_context_t *ctx, int is_enc, const void *key, size_t key_size, int aesni256) { assert(is_enc && "decryption is not supported (yet)"); size_t i = 0; switch (key_size) { case 16: /* AES128 */ ctx->rounds = 10; break; case 32: /* AES256 */ ctx->rounds = 14; break; default: assert(!"invalid key size; AES128 / AES256 are supported"); break; } ctx->aesni256 = aesni256; /* load and expand keys using keys.m128 */ ctx->keys.m128[i++] = _mm_loadu_si128((__m128i *)key); if (key_size == 32) ctx->keys.m128[i++] = _mm_loadu_si128((__m128i *)key + 1); while (1) { #define EXPAND(R) \ { \ ctx->keys.m128[i] = \ expand_key(ctx->keys.m128[i - key_size / 16], \ _mm_shuffle_epi32(_mm_aeskeygenassist_si128(ctx->keys.m128[i - 1], R), _MM_SHUFFLE(3, 3, 3, 3))); \ if (i == ctx->rounds) \ break; \ ++i; \ if (key_size > 24) { \ ctx->keys.m128[i] = \ expand_key(ctx->keys.m128[i - key_size / 16], \ _mm_shuffle_epi32(_mm_aeskeygenassist_si128(ctx->keys.m128[i - 1], R), _MM_SHUFFLE(2, 2, 2, 2))); \ ++i; \ } \ } EXPAND(0x1); EXPAND(0x2); EXPAND(0x4); EXPAND(0x8); EXPAND(0x10); EXPAND(0x20); EXPAND(0x40); EXPAND(0x80); EXPAND(0x1b); EXPAND(0x36); #undef EXPAND } /* convert to keys.m256 if aesni256 is used */ if (ctx->aesni256) { size_t i = ctx->rounds; do { ctx->keys.m256[i] = _mm256_broadcastsi128_si256(ctx->keys.m128[i]); } while (i-- != 0); } } void ptls_fusion_aesecb_dispose(ptls_fusion_aesecb_context_t *ctx) { ptls_clear_memory(ctx, sizeof(*ctx)); } void ptls_fusion_aesecb_encrypt(ptls_fusion_aesecb_context_t *ctx, void *dst, const void *src) { __m128i v = _mm_loadu_si128(src); v = aesecb_encrypt(ctx, v); _mm_storeu_si128(dst, v); } /** * returns the number of ghash entries that is required to handle an AEAD block of given size */ static size_t aesgcm_calc_ghash_cnt(size_t capacity) { // round-up by block size, add to handle worst split of the size between AAD and payload, plus context to hash AC return (capacity + 15) / 16 + 2; } static void setup_one_ghash_entry(ptls_fusion_aesgcm_context_t *ctx) { __m128i *H, *r, *Hprev, H0; if (ctx->ecb.aesni256) { struct ptls_fusion_aesgcm_context256 *ctx256 = (void *)ctx; #define GET_SLOT(i, mem) (&ctx256->ghash[(i) / 2].mem[(i) % 2 == 0]) H = GET_SLOT(ctx->ghash_cnt, H); r = GET_SLOT(ctx->ghash_cnt, r); Hprev = ctx->ghash_cnt == 0 ? NULL : GET_SLOT(ctx->ghash_cnt - 1, H); #undef GET_SLOT H0 = ctx256->ghash[0].H[1]; } else { struct ptls_fusion_aesgcm_context128 *ctx128 = (void *)ctx; H = &ctx128->ghash[ctx->ghash_cnt].H; r = &ctx128->ghash[ctx->ghash_cnt].r; Hprev = ctx->ghash_cnt == 0 ? NULL : &ctx128->ghash[ctx->ghash_cnt - 1].H; H0 = ctx128->ghash[0].H; } if (Hprev != NULL) *H = gfmul(*Hprev, H0); *r = _mm_shuffle_epi32(*H, 78); *r = _mm_xor_si128(*r, *H); ++ctx->ghash_cnt; } static size_t calc_aesgcm_context_size(size_t *ghash_cnt, int aesni256) { size_t sz; if (aesni256) { if (*ghash_cnt % 2 != 0) ++*ghash_cnt; sz = offsetof(struct ptls_fusion_aesgcm_context256, ghash) + sizeof(union ptls_fusion_aesgcm_ghash_precompute256) * *ghash_cnt / 2; } else { sz = offsetof(struct ptls_fusion_aesgcm_context128, ghash) + sizeof(struct ptls_fusion_aesgcm_ghash_precompute128) * *ghash_cnt; } return sz; } static ptls_fusion_aesgcm_context_t *new_aesgcm(const void *key, size_t key_size, size_t capacity, int aesni256) { ptls_fusion_aesgcm_context_t *ctx; size_t ghash_cnt = aesgcm_calc_ghash_cnt(capacity), ctx_size = calc_aesgcm_context_size(&ghash_cnt, aesni256); if ((ctx = aligned_alloc(32, ctx_size)) == NULL) return NULL; ptls_fusion_aesecb_init(&ctx->ecb, 1, key, key_size, aesni256); ctx->capacity = capacity; __m128i H0 = aesecb_encrypt(&ctx->ecb, _mm_setzero_si128()); H0 = _mm_shuffle_epi8(H0, byteswap128); H0 = transformH(H0); if (ctx->ecb.aesni256) { ((struct ptls_fusion_aesgcm_context256 *)ctx)->ghash[0].H[1] = H0; } else { ((struct ptls_fusion_aesgcm_context128 *)ctx)->ghash[0].H = H0; } ctx->ghash_cnt = 0; while (ctx->ghash_cnt < ghash_cnt) setup_one_ghash_entry(ctx); return ctx; } ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_new(const void *key, size_t key_size, size_t capacity) { return new_aesgcm(key, key_size, capacity, 0); } ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_set_capacity(ptls_fusion_aesgcm_context_t *ctx, size_t capacity) { size_t new_ghash_cnt = aesgcm_calc_ghash_cnt(capacity); if (new_ghash_cnt <= ctx->ghash_cnt) return ctx; size_t new_ctx_size = calc_aesgcm_context_size(&new_ghash_cnt, ctx->ecb.aesni256), old_ctx_size = calc_aesgcm_context_size(&ctx->ghash_cnt, ctx->ecb.aesni256); ptls_fusion_aesgcm_context_t *newp; if ((newp = aligned_alloc(32, new_ctx_size)) == NULL) return NULL; memcpy(newp, ctx, old_ctx_size); ptls_clear_memory(ctx, old_ctx_size); aligned_free(ctx); ctx = newp; ctx->capacity = capacity; while (ctx->ghash_cnt < new_ghash_cnt) setup_one_ghash_entry(ctx); return ctx; } void ptls_fusion_aesgcm_free(ptls_fusion_aesgcm_context_t *ctx) { ptls_clear_memory(ctx, calc_aesgcm_context_size(&ctx->ghash_cnt, ctx->ecb.aesni256)); /* skip ptls_fusion_aesecb_dispose, based on the knowledge that it does not allocate memory elsewhere */ aligned_free(ctx); } static void ctr_dispose(ptls_cipher_context_t *_ctx) { struct ctr_context *ctx = (struct ctr_context *)_ctx; ptls_fusion_aesecb_dispose(&ctx->fusion); _mm_storeu_si128(&ctx->bits, _mm_setzero_si128()); } static void ctr_init(ptls_cipher_context_t *_ctx, const void *iv) { struct ctr_context *ctx = (struct ctr_context *)_ctx; _mm_storeu_si128(&ctx->bits, aesecb_encrypt(&ctx->fusion, _mm_loadu_si128(iv))); ctx->is_ready = 1; } static void ctr_transform(ptls_cipher_context_t *_ctx, void *output, const void *input, size_t len) { struct ctr_context *ctx = (struct ctr_context *)_ctx; assert((ctx->is_ready && len <= 16) || !"CTR transfomation is supported only once per call to `init` and the maximum size is limited to 16 bytes"); ctx->is_ready = 0; if (len < 16) { storen128(output, len, _mm_xor_si128(_mm_loadu_si128(&ctx->bits), loadn128(input, len))); } else { _mm_storeu_si128(output, _mm_xor_si128(_mm_loadu_si128(&ctx->bits), _mm_loadu_si128(input))); } } static int aesctr_setup(ptls_cipher_context_t *_ctx, int is_enc, const void *key, size_t key_size) { struct ctr_context *ctx = (struct ctr_context *)_ctx; ctx->super.do_dispose = ctr_dispose; ctx->super.do_init = ctr_init; ctx->super.do_transform = ctr_transform; ptls_fusion_aesecb_init(&ctx->fusion, 1, key, key_size, 0 /* probably we do not need aesni256 for CTR? */); ctx->is_ready = 0; return 0; } static int aes128ctr_setup(ptls_cipher_context_t *ctx, int is_enc, const void *key) { return aesctr_setup(ctx, is_enc, key, PTLS_AES128_KEY_SIZE); } static int aes256ctr_setup(ptls_cipher_context_t *ctx, int is_enc, const void *key) { return aesctr_setup(ctx, is_enc, key, PTLS_AES256_KEY_SIZE); } static void aesgcm_dispose_crypto(ptls_aead_context_t *_ctx) { struct aesgcm_context *ctx = (struct aesgcm_context *)_ctx; ptls_fusion_aesgcm_free(ctx->aesgcm); } static void aead_do_encrypt_init(ptls_aead_context_t *_ctx, uint64_t seq, const void *aad, size_t aadlen) { assert(!"FIXME"); } static size_t aead_do_encrypt_update(ptls_aead_context_t *_ctx, void *output, const void *input, size_t inlen) { assert(!"FIXME"); return SIZE_MAX; } static size_t aead_do_encrypt_final(ptls_aead_context_t *_ctx, void *_output) { assert(!"FIXME"); return SIZE_MAX; } static inline __m128i calc_counter(struct aesgcm_context *ctx, uint64_t seq) { __m128i ctr = _mm_setzero_si128(); ctr = _mm_insert_epi64(ctr, seq, 0); ctr = _mm_slli_si128(ctr, 4); ctr = _mm_xor_si128(ctx->static_iv, ctr); return ctr; } void aead_do_encrypt(struct st_ptls_aead_context_t *_ctx, void *output, const void *input, size_t inlen, uint64_t seq, const void *aad, size_t aadlen, ptls_aead_supplementary_encryption_t *supp) { struct aesgcm_context *ctx = (void *)_ctx; if (inlen + aadlen > ctx->aesgcm->capacity) ctx->aesgcm = ptls_fusion_aesgcm_set_capacity(ctx->aesgcm, inlen + aadlen); ptls_fusion_aesgcm_encrypt(ctx->aesgcm, output, input, inlen, calc_counter(ctx, seq), aad, aadlen, supp); } static void aead_do_encrypt_v(struct st_ptls_aead_context_t *ctx, void *output, ptls_iovec_t *input, size_t incnt, uint64_t seq, const void *aad, size_t aadlen) { assert(!"FIXME"); } size_t aead_do_decrypt(ptls_aead_context_t *_ctx, void *output, const void *input, size_t inlen, uint64_t seq, const void *aad, size_t aadlen) { struct aesgcm_context *ctx = (void *)_ctx; if (inlen < 16) return SIZE_MAX; size_t enclen = inlen - 16; if (enclen + aadlen > ctx->aesgcm->capacity) ctx->aesgcm = ptls_fusion_aesgcm_set_capacity(ctx->aesgcm, enclen + aadlen); if (!ptls_fusion_aesgcm_decrypt(ctx->aesgcm, output, input, enclen, calc_counter(ctx, seq), aad, aadlen, (const uint8_t *)input + enclen)) return SIZE_MAX; return enclen; } static inline void aesgcm_get_iv(ptls_aead_context_t *_ctx, void *iv) { struct aesgcm_context *ctx = (struct aesgcm_context *)_ctx; __m128i m128 = _mm_shuffle_epi8(ctx->static_iv, byteswap128); storen128(iv, PTLS_AESGCM_IV_SIZE, m128); } static inline void aesgcm_set_iv(ptls_aead_context_t *_ctx, const void *iv) { struct aesgcm_context *ctx = (struct aesgcm_context *)_ctx; ctx->static_iv = loadn128(iv, PTLS_AESGCM_IV_SIZE); ctx->static_iv = _mm_shuffle_epi8(ctx->static_iv, byteswap128); } static int aesgcm_setup(ptls_aead_context_t *_ctx, int is_enc, const void *key, const void *iv, size_t key_size) { struct aesgcm_context *ctx = (struct aesgcm_context *)_ctx; ctx->static_iv = loadn128(iv, PTLS_AESGCM_IV_SIZE); ctx->static_iv = _mm_shuffle_epi8(ctx->static_iv, byteswap128); if (key == NULL) return 0; ctx->super.dispose_crypto = aesgcm_dispose_crypto; ctx->super.do_get_iv = aesgcm_get_iv; ctx->super.do_set_iv = aesgcm_set_iv; ctx->super.do_encrypt_init = aead_do_encrypt_init; ctx->super.do_encrypt_update = aead_do_encrypt_update; ctx->super.do_encrypt_final = aead_do_encrypt_final; ctx->super.do_encrypt = aead_do_encrypt; ctx->super.do_encrypt_v = aead_do_encrypt_v; ctx->super.do_decrypt = aead_do_decrypt; ctx->aesgcm = new_aesgcm(key, key_size, 1500 /* assume ordinary packet size */, 0 /* no support for aesni256 yet */); return 0; } int aes128gcm_setup(ptls_aead_context_t *ctx, int is_enc, const void *key, const void *iv) { return aesgcm_setup(ctx, is_enc, key, iv, PTLS_AES128_KEY_SIZE); } int aes256gcm_setup(ptls_aead_context_t *ctx, int is_enc, const void *key, const void *iv) { return aesgcm_setup(ctx, is_enc, key, iv, PTLS_AES256_KEY_SIZE); } int ptls_fusion_can_aesni256 = 0; ptls_cipher_algorithm_t ptls_fusion_aes128ctr = {"AES128-CTR", PTLS_AES128_KEY_SIZE, 1, // block size PTLS_AES_IV_SIZE, sizeof(struct ctr_context), aes128ctr_setup}; ptls_cipher_algorithm_t ptls_fusion_aes256ctr = {"AES256-CTR", PTLS_AES256_KEY_SIZE, 1, // block size PTLS_AES_IV_SIZE, sizeof(struct ctr_context), aes256ctr_setup}; ptls_aead_algorithm_t ptls_fusion_aes128gcm = {"AES128-GCM", PTLS_AESGCM_CONFIDENTIALITY_LIMIT, PTLS_AESGCM_INTEGRITY_LIMIT, &ptls_fusion_aes128ctr, NULL, // &ptls_fusion_aes128ecb, PTLS_AES128_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, {0}, // while it may work, no reason to support TLS/1.2 0, 0, sizeof(struct aesgcm_context), aes128gcm_setup}; ptls_aead_algorithm_t ptls_fusion_aes256gcm = {"AES256-GCM", PTLS_AESGCM_CONFIDENTIALITY_LIMIT, PTLS_AESGCM_INTEGRITY_LIMIT, &ptls_fusion_aes256ctr, NULL, // &ptls_fusion_aes256ecb, PTLS_AES256_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, {0}, // while it may work, no reason to support TLS/1.2 0, 0, sizeof(struct aesgcm_context), aes256gcm_setup}; static inline size_t calc_total_length(ptls_iovec_t *input, size_t incnt) { size_t totlen = 0; for (size_t i = 0; i < incnt; ++i) totlen += input[i].len; return totlen; } static inline void reduce_aad128(struct ptls_fusion_gfmul_state128 *gstate, struct ptls_fusion_aesgcm_ghash_precompute128 *ghash, const void *_aad, size_t aadlen) { struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute; const uint8_t *aad = _aad; while (PTLS_UNLIKELY(aadlen >= 6 * 16)) { ghash_precompute = ghash + 6; gfmul_firststep128(gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); aad += 16; aadlen -= 16; for (int i = 1; i < 6; ++i) { gfmul_nextstep128(gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); aad += 16; aadlen -= 16; } gfmul_reduce128(gstate); } if (PTLS_LIKELY(aadlen != 0)) { ghash_precompute = ghash + (aadlen + 15) / 16; if (PTLS_UNLIKELY(aadlen >= 16)) { gfmul_firststep128(gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); aad += 16; aadlen -= 16; while (aadlen >= 16) { gfmul_nextstep128(gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); aad += 16; aadlen -= 16; } if (PTLS_LIKELY(aadlen != 0)) gfmul_nextstep128(gstate, loadn128(aad, aadlen), --ghash_precompute); } else { gfmul_firststep128(gstate, loadn128(aad, aadlen), --ghash_precompute); } assert(ghash == ghash_precompute); gfmul_reduce128(gstate); } } NO_SANITIZE_ADDRESS static inline uint8_t *load_preceding_unaligned(uint8_t *encbuf, uint8_t **output) { uint8_t *encp; if ((encp = encbuf + ((uintptr_t)*output & 63)) != encbuf) { _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)(*output - (encp - encbuf)))); _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256((void *)(*output - (encp - encbuf) + 32))); *output -= encp - encbuf; } return encp; } NO_SANITIZE_ADDRESS static inline void write_remaining_bytes(uint8_t *dst, const uint8_t *src, const uint8_t *end) { /* Write in 64-byte chunks, using NT store instructions. Last partial block, if any, is written to cache, as that cache line * would likely be read when the next TLS record is being built. */ for (; end - src >= 64; dst += 64, src += 64) { _mm256_stream_si256((void *)dst, _mm256_load_si256((void *)src)); _mm256_stream_si256((void *)(dst + 32), _mm256_load_si256((void *)(src + 32))); } _mm_sfence(); /* weakly ordered writes have to be synced before being passed to NIC */ if (src != end) { for (; end - src >= 16; dst += 16, src += 16) _mm_store_si128((void *)dst, _mm_load_si128((void *)src)); if (src != end) storen128((void *)dst, end - src, loadn128((void *)src, end - src)); } } NO_SANITIZE_ADDRESS static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void *_output, ptls_iovec_t *input, size_t incnt, uint64_t seq, const void *aad, size_t aadlen) { /* init the bits (we can always run in full), but use the last slot for calculating ek0, if possible */ #define AESECB6_INIT() \ do { \ ctr = _mm_add_epi64(ctr, one8); \ bits0 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ bits1 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ bits2 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ bits3 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ bits4 = _mm_shuffle_epi8(ctr, byteswap128); \ if (PTLS_LIKELY(srclen > 16 * 5) || src_vecleft != 0) { \ ctr = _mm_add_epi64(ctr, one8); \ bits5 = _mm_shuffle_epi8(ctr, byteswap128); \ } else { \ bits5 = ek0; \ state |= STATE_EK0_READY; \ } \ __m128i k = ctx->super.ecb.keys.m128[0]; \ bits0 = _mm_xor_si128(bits0, k); \ bits1 = _mm_xor_si128(bits1, k); \ bits2 = _mm_xor_si128(bits2, k); \ bits3 = _mm_xor_si128(bits3, k); \ bits4 = _mm_xor_si128(bits4, k); \ bits5 = _mm_xor_si128(bits5, k); \ } while (0) /* aes block update */ #define AESECB6_UPDATE(i) \ do { \ __m128i k = ctx->super.ecb.keys.m128[i]; \ bits0 = _mm_aesenc_si128(bits0, k); \ bits1 = _mm_aesenc_si128(bits1, k); \ bits2 = _mm_aesenc_si128(bits2, k); \ bits3 = _mm_aesenc_si128(bits3, k); \ bits4 = _mm_aesenc_si128(bits4, k); \ bits5 = _mm_aesenc_si128(bits5, k); \ } while (0) /* aesenclast */ #define AESECB6_FINAL(i) \ do { \ __m128i k = ctx->super.ecb.keys.m128[i]; \ bits0 = _mm_aesenclast_si128(bits0, k); \ bits1 = _mm_aesenclast_si128(bits1, k); \ bits2 = _mm_aesenclast_si128(bits2, k); \ bits3 = _mm_aesenclast_si128(bits3, k); \ bits4 = _mm_aesenclast_si128(bits4, k); \ bits5 = _mm_aesenclast_si128(bits5, k); \ } while (0) struct aesgcm_context *agctx = (void *)_ctx; uint8_t *output = _output; #define STATE_EK0_READY 0x1 #define STATE_COPY_128B 0x2 int32_t state = 0; /* Bytes are written here first then written using NT store instructions, 64 bytes at a time. */ uint8_t encbuf[32 * 6] __attribute__((aligned(32))), *encp; /* `encbuf` should be large enough to store up to 63-bytes of unaligned bytes, 6 16-byte AES blocks, plus AEAD tag that is * append to the ciphertext before writing the bytes to main memory using NT store instructions. */ PTLS_BUILD_ASSERT(sizeof(encbuf) >= 64 + 6 * 16 + 16); /* load unaligned data within same cache line preceding `output`, adjusting pointers accordingly */ encp = load_preceding_unaligned(encbuf, &output); /* First write would be 128 bytes (32+6*16), if encbuf contains no less than 32 bytes already. */ if (encp - encbuf >= 32) state |= STATE_COPY_128B; /* setup ctr, retain Ek(0), len(A) | len(C) to be fed into GCM */ __m128i ctr = calc_counter(agctx, seq); ctr = _mm_insert_epi32(ctr, 1, 0); __m128i ek0 = _mm_shuffle_epi8(ctr, byteswap128); __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)calc_total_length(input, incnt) * 8), byteswap128); struct ptls_fusion_aesgcm_context128 *ctx = (void *)agctx->aesgcm; __m128i bits0, bits1, bits2, bits3, bits4, bits5 = _mm_setzero_si128(); struct ptls_fusion_gfmul_state128 gstate = {0}; /* find the first non-empty vec */ const uint8_t *src = NULL; size_t srclen = 0, src_vecleft = incnt; while (srclen == 0 && src_vecleft != 0) { src = (void *)input[0].base; srclen = input[0].len; ++input; --src_vecleft; } /* Prepare first 6 blocks of bit stream, at the same time calculating ghash of AAD. */ AESECB6_INIT(); AESECB6_UPDATE(1); AESECB6_UPDATE(2); reduce_aad128(&gstate, ctx->ghash, aad, aadlen); for (size_t i = 3; i < ctx->super.ecb.rounds; ++i) AESECB6_UPDATE(i); AESECB6_FINAL(ctx->super.ecb.rounds); /* Main loop. This loop: * 1. using current keystream (bits0..bits5), xors a up to 6 * 16 bytes and writes to encbuf, * 2. then if there is no more data to be encrypted, exit the loop, otherwise, * 3. calculate ghash of the blocks being written to encbuf, * 4. calculate next 6 * 16 bytes of keystream, * 5. writes encbuf in 64-byte blocks * When exitting the loop, `remaining_ghash_from` represents the offset within `encbuf` from where ghash remains to be * calculated. */ size_t remaining_ghash_from = encp - encbuf; if (srclen != 0) { while (1) { /* apply the bit stream to input, writing to encbuf */ if (PTLS_LIKELY(srclen >= 6 * 16)) { #define APPLY(i) _mm_storeu_si128((void *)(encp + i * 16), _mm_xor_si128(_mm_loadu_si128((void *)(src + i * 16)), bits##i)) APPLY(0); APPLY(1); APPLY(2); APPLY(3); APPLY(4); APPLY(5); #undef APPLY encp += 6 * 16; src += 6 * 16; srclen -= 6 * 16; if (PTLS_UNLIKELY(srclen == 0)) { if (src_vecleft == 0) { remaining_ghash_from = (encp - encbuf) - 96; break; } src = (void *)input[0].base; srclen = input[0].len; ++input; --src_vecleft; } } else { /* slow path, load at most 6 * 16 bytes to encbuf then encrypt in-place */ size_t bytes_copied = 0; do { if (srclen >= 16 && bytes_copied < 5 * 16) { _mm_storeu_si128((void *)(encp + bytes_copied), _mm_loadu_si128((void *)src)); bytes_copied += 16; src += 16; srclen -= 16; } else { encp[bytes_copied++] = *src++; --srclen; } if (PTLS_UNLIKELY(srclen == 0)) { do { if (src_vecleft == 0) break; src = (void *)input[0].base; srclen = input[0].len; ++input; --src_vecleft; } while (srclen == 0); if (srclen == 0) break; } } while (bytes_copied < 6 * 16); #define APPLY(i) _mm_storeu_si128((void *)(encp + i * 16), _mm_xor_si128(_mm_loadu_si128((void *)(encp + i * 16)), bits##i)) APPLY(0); APPLY(1); APPLY(2); APPLY(3); APPLY(4); APPLY(5); #undef APPLY encp += bytes_copied; if (PTLS_UNLIKELY(srclen == 0)) { /* Calculate amonut of data left to be ghashed, as well as zero-clearing the remainedr of partial block, as it * will be fed into ghash. */ remaining_ghash_from = (encp - encbuf) - bytes_copied; if ((bytes_copied & 15) != 0) _mm_storeu_si128((void *)encp, _mm_setzero_si128()); break; } } /* Next 96-byte block starts here. Run AES and ghash in while writing output using non-temporal stores in 64-byte * blocks. */ AESECB6_INIT(); struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash + 6; gfmul_firststep128(&gstate, _mm_loadu_si128((void *)(encp - 6 * 16)), --ghash_precompute); AESECB6_UPDATE(1); gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encp - 5 * 16)), --ghash_precompute); AESECB6_UPDATE(2); gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encp - 4 * 16)), --ghash_precompute); AESECB6_UPDATE(3); _mm256_stream_si256((void *)output, _mm256_load_si256((void *)encbuf)); _mm256_stream_si256((void *)(output + 32), _mm256_load_si256((void *)(encbuf + 32))); AESECB6_UPDATE(4); gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encp - 3 * 16)), --ghash_precompute); AESECB6_UPDATE(5); gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encp - 2 * 16)), --ghash_precompute); AESECB6_UPDATE(6); gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encp - 1 * 16)), --ghash_precompute); AESECB6_UPDATE(7); if ((state & STATE_COPY_128B) != 0) { _mm256_stream_si256((void *)(output + 64), _mm256_load_si256((void *)(encbuf + 64))); _mm256_stream_si256((void *)(output + 96), _mm256_load_si256((void *)(encbuf + 96))); output += 128; encp -= 128; AESECB6_UPDATE(8); _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)(encbuf + 128))); _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256((void *)(encbuf + 160))); } else { output += 64; encp -= 64; _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)(encbuf + 64))); _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256((void *)(encbuf + 96))); AESECB6_UPDATE(8); } state ^= STATE_COPY_128B; AESECB6_UPDATE(9); if (PTLS_UNLIKELY(ctx->super.ecb.rounds != 10)) { for (size_t i = 10; PTLS_LIKELY(i < ctx->super.ecb.rounds); ++i) AESECB6_UPDATE(i); } assert(ctx->ghash == ghash_precompute); gfmul_reduce128(&gstate); AESECB6_FINAL(ctx->super.ecb.rounds); } } /* Now, All the encrypted bits are built in encbuf. Calculate AEAD tag and append to encbuf. */ { /* Run ghash against the remaining bytes, after appending `ac` (i.e., len(A) | len(C)). At this point, we might be ghashing 7 * blocks at once. */ size_t ac_off = remaining_ghash_from + ((encp - encbuf) - remaining_ghash_from + 15) / 16 * 16; _mm_storeu_si128((void *)(encbuf + ac_off), ac); size_t blocks = ((encp - encbuf) - remaining_ghash_from + 15) / 16 + 1; /* round up, +1 for AC */ assert(blocks <= 7); struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash + blocks; gfmul_firststep128(&gstate, _mm_loadu_si128((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); remaining_ghash_from += 16; while (ghash_precompute != ctx->ghash) { gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); remaining_ghash_from += 16; } gfmul_reduce128(&gstate); } /* Calculate EK0, if in the unlikely case on not been done yet. When encoding in full size (16K), EK0 will be ready. */ if (PTLS_UNLIKELY((state & STATE_EK0_READY) == 0)) { bits5 = _mm_xor_si128(ek0, ctx->super.ecb.keys.m128[0]); for (size_t i = 1; i < ctx->super.ecb.rounds; ++i) bits5 = _mm_aesenc_si128(bits5, ctx->super.ecb.keys.m128[i]); bits5 = _mm_aesenclast_si128(bits5, ctx->super.ecb.keys.m128[ctx->super.ecb.rounds]); } /* append tag to encbuf */ _mm_storeu_si128((void *)encp, gfmul_get_tag128(&gstate, bits5)); encp += 16; /* write remaining bytes */ write_remaining_bytes(output, encbuf, encp); #undef AESECB6_INIT #undef AESECB6_UPDATE #undef AESECB6_FINAL #undef STATE_EK0_READY #undef STATE_COPY_128B } static size_t non_temporal_decrypt128(ptls_aead_context_t *_ctx, void *_output, const void *_input, size_t inlen, uint64_t seq, const void *aad, size_t aadlen) { /* Bail out if the input is too short, or remove tag from range. */ if (inlen < 16) return SIZE_MAX; inlen -= 16; size_t textlen = inlen; /* init the bits (we can always run in full), but use the last slot for calculating ek0, if possible */ #define AESECB6_INIT() \ do { \ ctr = _mm_add_epi64(ctr, one8); \ bits0 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ bits1 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ bits2 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ bits3 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ bits4 = _mm_shuffle_epi8(ctr, byteswap128); \ if (PTLS_LIKELY(inlen > 16 * 5)) { \ ctr = _mm_add_epi64(ctr, one8); \ bits5 = _mm_shuffle_epi8(ctr, byteswap128); \ } else { \ bits5 = ek0; \ state |= STATE_EK0_READY; \ } \ __m128i k = ctx->super.ecb.keys.m128[0]; \ bits0 = _mm_xor_si128(bits0, k); \ bits1 = _mm_xor_si128(bits1, k); \ bits2 = _mm_xor_si128(bits2, k); \ bits3 = _mm_xor_si128(bits3, k); \ bits4 = _mm_xor_si128(bits4, k); \ bits5 = _mm_xor_si128(bits5, k); \ } while (0) /* aes block update */ #define AESECB6_UPDATE(i) \ do { \ __m128i k = ctx->super.ecb.keys.m128[i]; \ bits0 = _mm_aesenc_si128(bits0, k); \ bits1 = _mm_aesenc_si128(bits1, k); \ bits2 = _mm_aesenc_si128(bits2, k); \ bits3 = _mm_aesenc_si128(bits3, k); \ bits4 = _mm_aesenc_si128(bits4, k); \ bits5 = _mm_aesenc_si128(bits5, k); \ } while (0) /* aesenclast */ #define AESECB6_FINAL(i) \ do { \ __m128i k = ctx->super.ecb.keys.m128[i]; \ bits0 = _mm_aesenclast_si128(bits0, k); \ bits1 = _mm_aesenclast_si128(bits1, k); \ bits2 = _mm_aesenclast_si128(bits2, k); \ bits3 = _mm_aesenclast_si128(bits3, k); \ bits4 = _mm_aesenclast_si128(bits4, k); \ bits5 = _mm_aesenclast_si128(bits5, k); \ } while (0) struct aesgcm_context *agctx = (void *)_ctx; uint8_t *output = _output; const uint8_t *input = _input; #define STATE_EK0_READY 0x1 int32_t state = 0; /* setup ctr, retain Ek(0), len(A) | len(C) to be fed into GCM */ __m128i ctr = calc_counter(agctx, seq); ctr = _mm_insert_epi32(ctr, 1, 0); __m128i ek0 = _mm_shuffle_epi8(ctr, byteswap128); __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)inlen * 8), byteswap128); struct ptls_fusion_aesgcm_context128 *ctx = (void *)agctx->aesgcm; __m128i bits0, bits1, bits2, bits3, bits4, bits5 = _mm_setzero_si128(); struct ptls_fusion_gfmul_state128 gstate = {0}; /* Prepare first 6 blocks of bit stream, at the same time calculating ghash of AAD. */ AESECB6_INIT(); AESECB6_UPDATE(1); AESECB6_UPDATE(2); reduce_aad128(&gstate, ctx->ghash, aad, aadlen); for (size_t i = 3; i < ctx->super.ecb.rounds; ++i) AESECB6_UPDATE(i); AESECB6_FINAL(ctx->super.ecb.rounds); /* Main loop. Operate in full blocks (6 * 16 bytes). */ while (PTLS_LIKELY(inlen >= 6 * 16)) { #define DECRYPT(i) _mm_storeu_si128((void *)(output + i * 16), _mm_xor_si128(bits##i, _mm_loadu_si128((void *)(input + i * 16)))) DECRYPT(0); DECRYPT(1); DECRYPT(2); DECRYPT(3); DECRYPT(4); DECRYPT(5); #undef DECRYPT #define GFMUL_NEXT(i) gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(input + i * 16)), ctx->ghash + 5 - i) AESECB6_INIT(); AESECB6_UPDATE(1); AESECB6_UPDATE(2); AESECB6_UPDATE(3); gfmul_firststep128(&gstate, _mm_loadu_si128((void *)input), ctx->ghash + 5); AESECB6_UPDATE(4); GFMUL_NEXT(1); AESECB6_UPDATE(5); GFMUL_NEXT(2); AESECB6_UPDATE(6); GFMUL_NEXT(3); AESECB6_UPDATE(7); GFMUL_NEXT(4); AESECB6_UPDATE(8); GFMUL_NEXT(5); AESECB6_UPDATE(9); gfmul_reduce128(&gstate); if (PTLS_UNLIKELY(ctx->super.ecb.rounds != 10)) { size_t i = 10; do { AESECB6_UPDATE(i); } while (++i < ctx->super.ecb.rounds); } AESECB6_FINAL(ctx->super.ecb.rounds); output += 6 * 16; input += 6 * 16; inlen -= 6 * 16; #undef GFMUL_NEXT } /* Decrypt the remainder as well as finishing GHASH calculation. */ if (inlen != 0) { struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash + (inlen + 15) / 16 + 1; #define ONEBLOCK(i) \ do { \ if (inlen != 0) { \ __m128i b = inlen >= 16 ? _mm_loadu_si128((void *)input) : loadn128(input, inlen); \ if (i == 0) { \ gfmul_firststep128(&gstate, b, --ghash_precompute); \ } else { \ gfmul_nextstep128(&gstate, b, --ghash_precompute); \ } \ b = _mm_xor_si128(b, bits##i); \ if (inlen >= 16) { \ _mm_storeu_si128((void *)output, b); \ output += 16; \ input += 16; \ inlen -= 16; \ } else { \ storen128(output, inlen, b); \ output += inlen; \ input += inlen; \ inlen = 0; \ } \ } \ } while (0) ONEBLOCK(0); ONEBLOCK(1); ONEBLOCK(2); ONEBLOCK(3); ONEBLOCK(4); ONEBLOCK(5); #undef ONEBLOCK gfmul_nextstep128(&gstate, ac, --ghash_precompute); assert(ghash_precompute == ctx->ghash); } else { gfmul_firststep128(&gstate, ac, ctx->ghash); } gfmul_reduce128(&gstate); /* Calculate EK0 if not yet available in bits5. */ if ((state & STATE_EK0_READY) == 0) { bits5 = _mm_xor_si128(ek0, ctx->super.ecb.keys.m128[0]); for (size_t i = 1; i < ctx->super.ecb.rounds; ++i) bits5 = _mm_aesenc_si128(bits5, ctx->super.ecb.keys.m128[i]); bits5 = _mm_aesenclast_si128(bits5, ctx->super.ecb.keys.m128[ctx->super.ecb.rounds]); } /* Calculate GCM tag and compare. */ __m128i calctag = gfmul_get_tag128(&gstate, bits5); __m128i recvtag = _mm_loadu_si128((void *)input); if (_mm_movemask_epi8(_mm_cmpeq_epi8(calctag, recvtag)) != 0xffff) return SIZE_MAX; return textlen; #undef AESECB6_INIT #undef AESECB6_UPDATE #undef AESECB6_FINAL #undef STATE_EK0_READY } NO_SANITIZE_ADDRESS static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void *_output, ptls_iovec_t *input, size_t incnt, uint64_t seq, const void *_aad, size_t aadlen) { /* init the bits (we can always run in full), but use the last slot for calculating ek0, if possible */ #define AESECB6_INIT() \ do { \ ctr = _mm256_add_epi64(ctr, incr128x2); \ bits0 = _mm256_shuffle_epi8(ctr, byteswap256); \ ctr = _mm256_add_epi64(ctr, incr128x2); \ bits1 = _mm256_shuffle_epi8(ctr, byteswap256); \ ctr = _mm256_add_epi64(ctr, incr128x2); \ bits2 = _mm256_shuffle_epi8(ctr, byteswap256); \ ctr = _mm256_add_epi64(ctr, incr128x2); \ bits3 = _mm256_shuffle_epi8(ctr, byteswap256); \ ctr = _mm256_add_epi64(ctr, incr128x2); \ bits4 = _mm256_shuffle_epi8(ctr, byteswap256); \ ctr = _mm256_add_epi64(ctr, incr128x2); \ bits5 = _mm256_shuffle_epi8(ctr, byteswap256); \ if (PTLS_UNLIKELY(srclen <= 32 * 6 - 16) && src_vecleft == 0) { \ bits5 = _mm256_permute2f128_si256(bits5, ac_ek0, 0x30); \ state |= STATE_EK0_READY; \ } \ __m256i k = ctx->super.ecb.keys.m256[0]; \ bits0 = _mm256_xor_si256(bits0, k); \ bits1 = _mm256_xor_si256(bits1, k); \ bits2 = _mm256_xor_si256(bits2, k); \ bits3 = _mm256_xor_si256(bits3, k); \ bits4 = _mm256_xor_si256(bits4, k); \ bits5 = _mm256_xor_si256(bits5, k); \ } while (0) /* aes block update */ #define AESECB6_UPDATE(i) \ do { \ __m256i k = ctx->super.ecb.keys.m256[i]; \ bits0 = _mm256_aesenc_epi128(bits0, k); \ bits1 = _mm256_aesenc_epi128(bits1, k); \ bits2 = _mm256_aesenc_epi128(bits2, k); \ bits3 = _mm256_aesenc_epi128(bits3, k); \ bits4 = _mm256_aesenc_epi128(bits4, k); \ bits5 = _mm256_aesenc_epi128(bits5, k); \ } while (0) /* aesenclast */ #define AESECB6_FINAL(i) \ do { \ __m256i k = ctx->super.ecb.keys.m256[i]; \ bits0 = _mm256_aesenclast_epi128(bits0, k); \ bits1 = _mm256_aesenclast_epi128(bits1, k); \ bits2 = _mm256_aesenclast_epi128(bits2, k); \ bits3 = _mm256_aesenclast_epi128(bits3, k); \ bits4 = _mm256_aesenclast_epi128(bits4, k); \ bits5 = _mm256_aesenclast_epi128(bits5, k); \ } while (0) struct aesgcm_context *agctx = (void *)_ctx; uint8_t *output = _output; const uint8_t *aad = _aad; #define STATE_EK0_READY 0x1 int32_t state = 0; /* Bytes are written here first then written using NT store instructions, 64 bytes at a time. */ uint8_t encbuf[32 * 9] __attribute__((aligned(32))), *encp; /* `encbuf` should be large enough to store up to 63-bytes of unaligned bytes, 6 16-byte AES blocks, plus AEAD tag that is * append to the ciphertext before writing the bytes to main memory using NT store instructions. */ PTLS_BUILD_ASSERT(sizeof(encbuf) >= 64 + 6 * 32 + 16); /* load unaligned data within same cache line preceding `output`, adjusting pointers accordingly */ encp = load_preceding_unaligned(encbuf, &output); /* setup ctr, retaining Ek(0), len(A) | len(C) to be fed into GCM */ __m256i ctr = _mm256_broadcastsi128_si256(calc_counter(agctx, seq)); ctr = _mm256_insert_epi32(ctr, 1, 4); __m256i ac_ek0 = _mm256_permute2f128_si256( /* first half: ac */ _mm256_castsi128_si256( _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)calc_total_length(input, incnt) * 8), byteswap128)), /* second half: ek0 */ _mm256_shuffle_epi8(ctr, byteswap256), 0x30); struct ptls_fusion_aesgcm_context256 *ctx = (void *)agctx->aesgcm; __m256i bits0, bits1, bits2, bits3, bits4, bits5 = _mm256_setzero_si256(); struct ptls_fusion_gfmul_state256 gstate = {0}; /* find the first non-empty vec */ const uint8_t *src = NULL; size_t srclen = 0, src_vecleft = incnt; while (srclen == 0 && src_vecleft != 0) { src = (void *)input[0].base; srclen = input[0].len; ++input; --src_vecleft; } /* Prepare first 6 blocks of bit stream, at the same time calculating ghash of AAD. */ AESECB6_INIT(); AESECB6_UPDATE(1); AESECB6_UPDATE(2); if (PTLS_LIKELY(aadlen != 0)) { union ptls_fusion_aesgcm_ghash_precompute256 *ghash_precompute; while (PTLS_UNLIKELY(aadlen >= 6 * 32)) { ghash_precompute = ctx->ghash + 6; gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)aad), 0, --ghash_precompute); aad += 32; aadlen -= 32; for (int i = 1; i < 6; ++i) { gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)aad), --ghash_precompute); aad += 32; aadlen -= 32; } gfmul_reduce256(&gstate); } if (PTLS_LIKELY(aadlen != 0)) { ghash_precompute = ctx->ghash + (aadlen + 31) / 32; if (PTLS_UNLIKELY(aadlen >= 32)) { if (aadlen % 32 == 0 || aadlen % 32 > 16) { gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)aad), 0, --ghash_precompute); aad += 32; aadlen -= 32; } else { gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)aad), 1, --ghash_precompute); aad += 16; aadlen -= 16; } while (aadlen >= 32) { gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)aad), --ghash_precompute); aad += 32; aadlen -= 32; } if (PTLS_LIKELY(aadlen != 0)) { assert(aadlen > 16); gfmul_nextstep256(&gstate, loadn256(aad, aadlen), --ghash_precompute); } } else { gfmul_firststep256(&gstate, loadn256(aad, aadlen), aadlen <= 16, --ghash_precompute); } assert(ctx->ghash == ghash_precompute); gfmul_reduce256(&gstate); } } for (size_t i = 3; i < ctx->super.ecb.rounds; ++i) AESECB6_UPDATE(i); AESECB6_FINAL(ctx->super.ecb.rounds); /* Main loop. This loop: * 1. using current keystream (bits0..bits5), xors a up to 6 * 16 bytes and writes to encbuf, * 2. then if there is no more data to be encrypted, exit the loop, otherwise, * 3. calculate ghash of the blocks being written to encbuf, * 4. calculate next 6 * 16 bytes of keystream, * 5. writes encbuf in 64-byte blocks * When exitting the loop, `remaining_ghash_from` represents the offset within `encbuf` from where ghash remains to be * calculated. */ size_t remaining_ghash_from = encp - encbuf; if (srclen != 0) { while (1) { /* apply the bit stream to input, writing to encbuf */ if (PTLS_LIKELY(srclen >= 6 * 32)) { #define APPLY(i) _mm256_storeu_si256((void *)(encp + i * 32), _mm256_xor_si256(_mm256_loadu_si256((void *)(src + i * 32)), bits##i)) APPLY(0); APPLY(1); APPLY(2); APPLY(3); APPLY(4); APPLY(5); #undef APPLY encp += 6 * 32; src += 6 * 32; srclen -= 6 * 32; if (PTLS_UNLIKELY(srclen == 0)) { if (src_vecleft == 0) { remaining_ghash_from = (encp - encbuf) - 6 * 32; break; } src = (void *)input[0].base; srclen = input[0].len; ++input; --src_vecleft; } } else { /* slow path, load at most 6 * 32 bytes to encbuf then encrypt in-place */ size_t bytes_copied = 0; do { if (srclen >= 32 && bytes_copied < 5 * 32) { _mm256_storeu_si256((void *)(encp + bytes_copied), _mm256_loadu_si256((void *)src)); bytes_copied += 32; src += 32; srclen -= 32; } else { encp[bytes_copied++] = *src++; --srclen; } if (PTLS_UNLIKELY(srclen == 0)) { do { if (src_vecleft == 0) break; src = (void *)input[0].base; srclen = input[0].len; ++input; --src_vecleft; } while (srclen == 0); if (srclen == 0) break; } } while (bytes_copied < 6 * 32); #define APPLY(i) \ _mm256_storeu_si256((void *)(encp + i * 32), _mm256_xor_si256(_mm256_loadu_si256((void *)(encp + i * 32)), bits##i)) APPLY(0); APPLY(1); APPLY(2); APPLY(3); APPLY(4); APPLY(5); #undef APPLY encp += bytes_copied; if (PTLS_UNLIKELY(srclen == 0)) { /* Calculate amonut of data left to be ghashed, as well as zero-clearing the remainedr of partial block, as it * will be fed into ghash. */ remaining_ghash_from = (encp - encbuf) - bytes_copied; if ((bytes_copied & 15) != 0) _mm_storeu_si128((void *)encp, _mm_setzero_si128()); break; } } /* Next 96-byte block starts here. Run AES and ghash in parallel while writing output using non-temporal store * instructions. */ AESECB6_INIT(); union ptls_fusion_aesgcm_ghash_precompute256 *ghash_precompute = ctx->ghash + 6; gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)(encp - 6 * 32)), 0, --ghash_precompute); AESECB6_UPDATE(1); gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encp - 5 * 32)), --ghash_precompute); AESECB6_UPDATE(2); gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encp - 4 * 32)), --ghash_precompute); AESECB6_UPDATE(3); gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encp - 3 * 32)), --ghash_precompute); AESECB6_UPDATE(4); _mm256_stream_si256((void *)output, _mm256_load_si256((void *)encbuf)); _mm256_stream_si256((void *)(output + 32), _mm256_load_si256((void *)(encbuf + 32))); _mm256_stream_si256((void *)(output + 64), _mm256_load_si256((void *)(encbuf + 64))); _mm256_stream_si256((void *)(output + 96), _mm256_load_si256((void *)(encbuf + 96))); _mm256_stream_si256((void *)(output + 128), _mm256_load_si256((void *)(encbuf + 128))); _mm256_stream_si256((void *)(output + 160), _mm256_load_si256((void *)(encbuf + 160))); AESECB6_UPDATE(5); gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encp - 2 * 32)), --ghash_precompute); AESECB6_UPDATE(6); gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encp - 1 * 32)), --ghash_precompute); output += 192; encp -= 192; AESECB6_UPDATE(7); _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)(encbuf + 192))); AESECB6_UPDATE(8); _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256((void *)(encbuf + 224))); AESECB6_UPDATE(9); if (PTLS_UNLIKELY(ctx->super.ecb.rounds != 10)) { for (size_t i = 10; PTLS_LIKELY(i < ctx->super.ecb.rounds); ++i) AESECB6_UPDATE(i); } assert(ctx->ghash == ghash_precompute); gfmul_reduce256(&gstate); AESECB6_FINAL(ctx->super.ecb.rounds); } } /* Now, All the encrypted bits are built in encbuf. Calculate AEAD tag and append to encbuf. */ { /* Run ghash against the remaining bytes, after appending `ac` (i.e., len(A) | len(C)). At this point, we might be ghashing 7 * blocks at once. */ size_t ac_off = remaining_ghash_from + ((encp - encbuf) - remaining_ghash_from + 15) / 16 * 16; _mm_storeu_si128((void *)(encbuf + ac_off), _mm256_castsi256_si128(ac_ek0)); size_t blocks = ((encp - encbuf) - remaining_ghash_from + 15) / 16 + 1; /* round up, +1 for AC */ assert(blocks <= 13); union ptls_fusion_aesgcm_ghash_precompute256 *ghash_precompute = ctx->ghash + blocks / 2; if (blocks % 2 != 0) { gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)(encbuf + remaining_ghash_from)), 1, ghash_precompute); remaining_ghash_from += 16; } else { gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)(encbuf + remaining_ghash_from)), 0, --ghash_precompute); remaining_ghash_from += 32; } while (ghash_precompute != ctx->ghash) { gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); remaining_ghash_from += 32; } gfmul_reduce256(&gstate); } /* Calculate EK0, if in the unlikely case on not been done yet. When encoding in full size (16K), EK0 will be ready. */ if (PTLS_UNLIKELY((state & STATE_EK0_READY) == 0)) { bits5 = ac_ek0; bits5 = _mm256_xor_si256(bits5, ctx->super.ecb.keys.m256[0]); for (size_t i = 1; i < ctx->super.ecb.rounds; ++i) bits5 = _mm256_aesenc_epi128(bits5, ctx->super.ecb.keys.m256[i]); bits5 = _mm256_aesenclast_epi128(bits5, ctx->super.ecb.keys.m256[ctx->super.ecb.rounds]); } /* append tag to encbuf */ _mm_storeu_si128((void *)encp, gfmul_get_tag256(&gstate, _mm256_castsi256_si128(_mm256_permute2f128_si256(bits5, bits5, 0x11)))); encp += 16; /* write remaining bytes */ write_remaining_bytes(output, encbuf, encp); } static int non_temporal_setup(ptls_aead_context_t *_ctx, int is_enc, const void *key, const void *iv, size_t key_size) { struct aesgcm_context *ctx = (struct aesgcm_context *)_ctx; int aesni256 = is_enc && ptls_fusion_can_aesni256; ctx->static_iv = loadn128(iv, PTLS_AESGCM_IV_SIZE); ctx->static_iv = _mm_shuffle_epi8(ctx->static_iv, byteswap128); if (key == NULL) return 0; ctx->super.dispose_crypto = aesgcm_dispose_crypto; ctx->super.do_get_iv = aesgcm_get_iv; ctx->super.do_set_iv = aesgcm_set_iv; ctx->super.do_encrypt_init = NULL; ctx->super.do_encrypt_update = NULL; ctx->super.do_encrypt_final = NULL; if (is_enc) { ctx->super.do_encrypt = ptls_aead__do_encrypt; ctx->super.do_encrypt_v = aesni256 ? non_temporal_encrypt_v256 : non_temporal_encrypt_v128; ctx->super.do_decrypt = NULL; } else { assert(!aesni256); ctx->super.do_encrypt = NULL; ctx->super.do_encrypt_v = NULL; ctx->super.do_decrypt = non_temporal_decrypt128; } ctx->aesgcm = new_aesgcm(key, key_size, 7 * (ptls_fusion_can_aesni256 ? 32 : 16), // 6 blocks at once, plus len(A) | len(C) that we might append aesni256); return 0; } static int non_temporal_aes128gcm_setup(ptls_aead_context_t *ctx, int is_enc, const void *key, const void *iv) { return non_temporal_setup(ctx, is_enc, key, iv, PTLS_AES128_KEY_SIZE); } static int non_temporal_aes256gcm_setup(ptls_aead_context_t *ctx, int is_enc, const void *key, const void *iv) { return non_temporal_setup(ctx, is_enc, key, iv, PTLS_AES256_KEY_SIZE); } ptls_aead_algorithm_t ptls_non_temporal_aes128gcm = {"AES128-GCM", PTLS_AESGCM_CONFIDENTIALITY_LIMIT, PTLS_AESGCM_INTEGRITY_LIMIT, &ptls_fusion_aes128ctr, NULL, // &ptls_fusion_aes128ecb, PTLS_AES128_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, {PTLS_TLS12_AESGCM_FIXED_IV_SIZE, PTLS_TLS12_AESGCM_RECORD_IV_SIZE}, 1, PTLS_X86_CACHE_LINE_ALIGN_BITS, sizeof(struct aesgcm_context), non_temporal_aes128gcm_setup}; ptls_aead_algorithm_t ptls_non_temporal_aes256gcm = {"AES256-GCM", PTLS_AESGCM_CONFIDENTIALITY_LIMIT, PTLS_AESGCM_INTEGRITY_LIMIT, &ptls_fusion_aes256ctr, NULL, // &ptls_fusion_aes128ecb, PTLS_AES256_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, {PTLS_TLS12_AESGCM_FIXED_IV_SIZE, PTLS_TLS12_AESGCM_RECORD_IV_SIZE}, 1, PTLS_X86_CACHE_LINE_ALIGN_BITS, sizeof(struct aesgcm_context), non_temporal_aes256gcm_setup}; #ifdef _WINDOWS /** * ptls_fusion_is_supported_by_cpu: * Check that the CPU has extended instructions for PCMUL, AES and AVX2. * This test assumes that the CPU is following the x86/x64 architecture. * A slightly more refined test could check that the cpu_info spells out * "genuineIntel" or "authenticAMD", but would fail in presence of * little known CPU brands or some VM */ int ptls_fusion_is_supported_by_cpu(void) { uint32_t cpu_info[4]; uint32_t nb_ids; int is_supported = 0; __cpuid(cpu_info, 0); nb_ids = cpu_info[0]; if (nb_ids >= 7) { uint32_t leaf1_ecx; __cpuid(cpu_info, 1); leaf1_ecx = cpu_info[2]; if (/* PCLMUL */ (leaf1_ecx & (1 << 5)) != 0 && /* AES */ (leaf1_ecx & (1 << 25)) != 0) { uint32_t leaf7_ebx, leaf7_ecx; __cpuid(cpu_info, 7); leaf7_ebx = cpu_info[1]; leaf7_ecx = cpu_info[2]; is_supported = /* AVX2 */ (leaf7_ebx & (1 << 5)) != 0; /* enable 256-bit mode if possible */ if (is_supported && (leaf7_ecx & 0x600) != 0 && !ptls_fusion_can_aesni256) ptls_fusion_can_aesni256 = 1; } } return is_supported; } #else int ptls_fusion_is_supported_by_cpu(void) { unsigned leaf1_ecx, leaf7_ebx, leaf7_ecx; { /* GCC-specific code to obtain CPU features */ unsigned leaf_cnt; __asm__("cpuid" : "=a"(leaf_cnt) : "a"(0) : "ebx", "ecx", "edx"); if (leaf_cnt < 7) return 0; __asm__("cpuid" : "=c"(leaf1_ecx) : "a"(1) : "ebx", "edx"); __asm__("cpuid" : "=b"(leaf7_ebx), "=c"(leaf7_ecx) : "a"(7), "c"(0) : "edx"); } /* AVX2 */ if ((leaf7_ebx & (1 << 5)) == 0) return 0; /* AES */ if ((leaf1_ecx & (1 << 25)) == 0) return 0; /* PCLMUL */ if ((leaf1_ecx & (1 << 1)) == 0) return 0; /* enable 256-bit mode if possible */ if ((leaf7_ecx & 0x600) != 0 && !ptls_fusion_can_aesni256) ptls_fusion_can_aesni256 = 1; return 1; } #endif /* ---------------------------------------------------------------- */ // struct aesgcm_context { // ptls_aead_context_t super; // ptls_fusion_aesgcm_context_t *aesgcm; <- ptls_fusion_aesgcm_free ptls_aead_context_t *aead_context_new() { ptls_aead_context_t *p = malloc(sizeof(struct aesgcm_context)); return p; } void aead_context_free(ptls_aead_context_t *p) { aesgcm_dispose_crypto(p); free(p); } /* ---------------------------------------------------------------- */ ptls_aead_supplementary_encryption_t *supplement_new(uint8_t *key, unsigned int siz) { ptls_aead_supplementary_encryption_t *supp = malloc(sizeof(ptls_aead_supplementary_encryption_t)); if (siz == PTLS_AES256_KEY_SIZE) { supp->ctx = ptls_cipher_new(&ptls_fusion_aes256ctr, 1, key); } else { supp->ctx = ptls_cipher_new(&ptls_fusion_aes128ctr, 1, key); } return supp; } void supplement_free(ptls_aead_supplementary_encryption_t *supp) { ptls_cipher_free(supp->ctx); free(supp); } void supplement_set_sample(ptls_aead_supplementary_encryption_t *supp, uint8_t *sample) { supp->input = sample; } uint8_t *supplement_get_mask(ptls_aead_supplementary_encryption_t *supp) { return (supp->output); }