-
Notifications
You must be signed in to change notification settings - Fork 734
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
aes_gcm/x86_64: Enable AVX2 VAES-CLMUL implementation.
If using GNU binutils as the assembler, this may require a newer version than what was previously required.
- Loading branch information
1 parent
8ae4f2d
commit 0c128bf
Showing
11 changed files
with
208 additions
and
51 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| // Copyright 2015-2025 Brian Smith. | ||
| // | ||
| // Permission to use, copy, modify, and/or distribute this software for any | ||
| // purpose with or without fee is hereby granted, provided that the above | ||
| // copyright notice and this permission notice appear in all copies. | ||
| // | ||
| // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES | ||
| // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF | ||
| // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY | ||
| // SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES | ||
| // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION | ||
| // OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN | ||
| // CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. | ||
|
|
||
| #![cfg(target_arch = "x86_64")] | ||
|
|
||
| use super::{aes, gcm, Counter, BLOCK_LEN}; | ||
| use crate::{aead::aes::Overlapping, c, polyfill::slice::AsChunksMut}; | ||
| use core::num::NonZeroU32; | ||
|
|
||
| pub(super) fn seal_whole_vaes_clmul_avx2( | ||
| aes_key: &aes::hw::Key, | ||
| auth: &mut gcm::Context<gcm::vclmulavx2::Key>, | ||
| ctr: &mut Counter, | ||
| mut in_out: AsChunksMut<u8, BLOCK_LEN>, | ||
| ) { | ||
| prefixed_extern! { | ||
| fn aes_gcm_enc_update_vaes_avx2( | ||
| input: *const u8, | ||
| output: *mut u8, | ||
| len: c::size_t, | ||
| key: &aes::AES_KEY, | ||
| ivec: &Counter, | ||
| Htable: &gcm::HTable, | ||
| Xi: &mut gcm::Xi); | ||
| } | ||
|
|
||
| let in_out = in_out.as_flattened_mut(); | ||
|
|
||
| // Precondition: Since we have a `gcm::Context` then the number of blocks | ||
| // must fit in `u32`. | ||
| let blocks = u32::try_from(in_out.len() / BLOCK_LEN).unwrap(); | ||
|
|
||
| if let Some(blocks) = NonZeroU32::new(blocks) { | ||
| let aes_key = aes_key.inner_less_safe(); | ||
| let (htable, xi) = auth.inner(); | ||
| let input = in_out.as_ptr(); | ||
| let output = in_out.as_mut_ptr(); | ||
| let len = in_out.len(); | ||
| unsafe { aes_gcm_enc_update_vaes_avx2(input, output, len, aes_key, ctr, htable, xi) }; | ||
| ctr.increment_by_less_safe(blocks); | ||
| } | ||
| } | ||
|
|
||
| pub(super) fn open_whole_vaes_clmul_avx2( | ||
| aes_key: &aes::hw::Key, | ||
| auth: &mut gcm::Context<gcm::vclmulavx2::Key>, | ||
| in_out: Overlapping, | ||
| ctr: &mut Counter, | ||
| ) { | ||
| prefixed_extern! { | ||
| fn aes_gcm_dec_update_vaes_avx2( | ||
| input: *const u8, | ||
| output: *mut u8, | ||
| len: c::size_t, | ||
| key: &aes::AES_KEY, | ||
| ivec: &mut Counter, | ||
| Htable: &gcm::HTable, | ||
| Xi: &mut gcm::Xi); | ||
| } | ||
|
|
||
| // Precondition. TODO: Create an overlapping::AsChunks for this. | ||
| assert_eq!(in_out.len() % BLOCK_LEN, 0); | ||
| // Precondition: Since we have a `gcm::Context` then the number of blocks | ||
| // must fit in `u32`. | ||
| let blocks = u32::try_from(in_out.len() / BLOCK_LEN).unwrap(); | ||
|
|
||
| if let Some(blocks) = NonZeroU32::new(blocks) { | ||
| let aes_key = aes_key.inner_less_safe(); | ||
| let (htable, xi) = auth.inner(); | ||
| in_out.with_input_output_len(|input, output, len| unsafe { | ||
| aes_gcm_dec_update_vaes_avx2(input, output, len, aes_key, ctr, htable, xi) | ||
| }); | ||
| ctr.increment_by_less_safe(blocks); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| // Copyright 2018-2025 Brian Smith. | ||
| // | ||
| // Permission to use, copy, modify, and/or distribute this software for any | ||
| // purpose with or without fee is hereby granted, provided that the above | ||
| // copyright notice and this permission notice appear in all copies. | ||
| // | ||
| // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES | ||
| // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF | ||
| // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY | ||
| // SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES | ||
| // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION | ||
| // OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN | ||
| // CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. | ||
|
|
||
| #![cfg(target_arch = "x86_64")] | ||
|
|
||
| use super::{ffi::KeyValue, HTable, UpdateBlock, Xi}; | ||
| use crate::{ | ||
| aead::gcm::ffi::BLOCK_LEN, | ||
| cpu::intel::{Avx2, VAesClmul}, | ||
| polyfill::slice::AsChunks, | ||
| }; | ||
|
|
||
| #[derive(Clone)] | ||
| pub struct Key { | ||
| h_table: HTable, | ||
| } | ||
|
|
||
| impl Key { | ||
| pub(in super::super) fn new(value: KeyValue, _cpu: (Avx2, VAesClmul)) -> Self { | ||
| Self { | ||
| h_table: unsafe { htable_new!(gcm_init_vpclmulqdq_avx2, value) }, | ||
| } | ||
| } | ||
|
|
||
| pub(super) fn inner(&self) -> &HTable { | ||
| &self.h_table | ||
| } | ||
| } | ||
|
|
||
| impl UpdateBlock for Key { | ||
| fn update_block(&self, xi: &mut Xi, a: [u8; BLOCK_LEN]) { | ||
| let input: AsChunks<u8, BLOCK_LEN> = (&a).into(); | ||
| unsafe { ghash!(gcm_ghash_vpclmulqdq_avx2_1, xi, &self.h_table, input) } | ||
| } | ||
| } |
Oops, something went wrong.