diff --git a/src/crypto.rs b/src/crypto.rs index 38b51d0..a823c13 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -1,9 +1,49 @@ pub use ring; -use std::convert::TryInto; -use tracing::warn; + +macro_rules! fixed_bytes_wrapper { + ($vis:vis struct $name:ident, $n:expr, $doc:literal) => { + #[doc = $doc] + #[derive(Clone, Copy, Debug, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize)] + $vis struct $name(crate::utils::serialize_utils::FixedByteArray<$n>); + + impl $name { + pub fn from_slice(slice: &[u8]) -> Option { + Some(Self(slice.try_into().ok()?)) + } + } + + impl AsRef<[u8]> for $name { + fn as_ref(&self) -> &[u8] { + self.0.as_ref() + } + } + + impl std::fmt::Display for $name { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + std::fmt::Display::fmt(&self.0, f) + } + } + + impl std::str::FromStr for $name { + type Err = as std::str::FromStr>::Err; + + fn from_str(s: &str) -> Result { + as std::str::FromStr>::from_str(s).map(Self) + } + } + + #[cfg(test)] + impl crate::utils::PlaceholderSeed for $name { + fn placeholder_seed_parts<'a>(seed_parts: impl IntoIterator) -> Self { + Self(crate::utils::placeholder_bytes( + [ concat!(stringify!($name), ":").as_bytes() ].iter().copied().chain(seed_parts) + ).into()) + } + } + }; +} pub mod sign_ed25519 { - use super::deserialize_slice; pub use ring::signature::Ed25519KeyPair as SecretKeyBase; use ring::signature::KeyPair; pub use ring::signature::Signature as SignatureBase; @@ -11,9 +51,7 @@ pub mod sign_ed25519 { pub use ring::signature::{ED25519, ED25519_PUBLIC_KEY_LEN}; use serde::{Deserialize, Serialize}; use std::convert::TryInto; - use tracing::warn; - - pub type PublicKeyBase = ::PublicKey; + use crate::crypto::generate_secure_random; // Constants copied from the ring library const SCALAR_LEN: usize = 32; @@ -21,63 +59,60 @@ pub mod sign_ed25519 { const SIGNATURE_LEN: usize = ELEM_LEN + SCALAR_LEN; pub const ED25519_SIGNATURE_LEN: usize = SIGNATURE_LEN; - /// Signature data + pub const ED25519_SEED_LEN: usize = 32; + + fixed_bytes_wrapper!(pub struct Signature, ED25519_SIGNATURE_LEN, "Signature data"); + fixed_bytes_wrapper!(pub struct PublicKey, ED25519_PUBLIC_KEY_LEN, "Public key data"); + + /// PKCS8 encoded secret key pair /// We used sodiumoxide serialization before (treated it as slice with 64 bit length prefix). - #[derive(Clone, Copy, Debug, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize)] - pub struct Signature( - #[serde(serialize_with = "<[_]>::serialize")] - #[serde(deserialize_with = "deserialize_slice")] - [u8; ED25519_SIGNATURE_LEN], - ); + /// Slice and vector are serialized the same. + #[derive(Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize)] + pub struct SecretKey(#[serde(with = "crate::utils::serialize_utils::vec_codec")] Vec); - impl Signature { + impl SecretKey { + /// Constructs a `SecretKey` from the given PKCS8 document. + /// + /// ### Arguments + /// + /// * `slice` - a slice containing the encoded PKCS8 document pub fn from_slice(slice: &[u8]) -> Option { - Some(Self(slice.try_into().ok()?)) - } - } - - impl AsRef<[u8]> for Signature { - fn as_ref(&self) -> &[u8] { - self.0.as_ref() + match SecretKeyBase::from_pkcs8(slice) { + Ok(_) => Some(Self(slice.to_vec())), + Err(_) => None, + } } - } - /// Public key data - /// We used sodiumoxide serialization before (treated it as slice with 64 bit length prefix). - #[derive(Clone, Copy, Debug, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize)] - pub struct PublicKey( - #[serde(serialize_with = "<[_]>::serialize")] - #[serde(deserialize_with = "deserialize_slice")] - [u8; ED25519_PUBLIC_KEY_LEN], - ); + /// Gets the public key corresponding to this secret key. + pub fn get_public_key(&self) -> PublicKey { + let keypair = SecretKeyBase::from_pkcs8(&self.0) + .expect("SecretKey contains invalid PKCS8 document?!?"); - impl PublicKey { - pub fn from_slice(slice: &[u8]) -> Option { - Some(Self(slice.try_into().ok()?)) + PublicKey::from_slice(keypair.public_key().as_ref()) + .expect("Keypair public key length is invalid?!?") } - } - impl AsRef<[u8]> for PublicKey { - fn as_ref(&self) -> &[u8] { - self.0.as_ref() + /// Gets a reference to the PKCS8 document which describes this secret key. + pub fn get_pkcs8(&self) -> &[u8] { + &self.0 } } - /// PKCS8 encoded secret key pair - /// We used sodiumoxide serialization before (treated it as slice with 64 bit length prefix). - /// Slice and vector are serialized the same. - #[derive(Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize)] - pub struct SecretKey(Vec); - - impl SecretKey { - pub fn from_slice(slice: &[u8]) -> Option { - Some(Self(slice.to_vec())) + #[cfg(test)] + impl crate::utils::PlaceholderSeed for SecretKey { + fn placeholder_seed_parts<'a>(seed_parts: impl IntoIterator) -> Self { + gen_keypair_from_seed(&crate::utils::placeholder_bytes( + [ "SecretKey:".as_bytes() ].iter().copied().chain(seed_parts) + )).1 } } - impl AsRef<[u8]> for SecretKey { - fn as_ref(&self) -> &[u8] { - self.0.as_ref() + #[cfg(test)] + impl crate::utils::PlaceholderSeed for (PublicKey, SecretKey) { + fn placeholder_seed_parts<'a>(seed_parts: impl IntoIterator) -> Self { + gen_keypair_from_seed(&crate::utils::placeholder_bytes( + [ "(PublicKey, SecretKey):".as_bytes() ].iter().copied().chain(seed_parts) + )) } } @@ -87,21 +122,11 @@ pub mod sign_ed25519 { } pub fn sign_detached(msg: &[u8], sk: &SecretKey) -> Signature { - let secret = match SecretKeyBase::from_pkcs8(sk.as_ref()) { - Ok(secret) => secret, - Err(_) => { - warn!("Invalid secret key"); - return Signature([0; ED25519_SIGNATURE_LEN]); - } - }; + let keypair = SecretKeyBase::from_pkcs8(sk.get_pkcs8()) + .expect("Invalid PKCS8 secret key?!?"); - let signature = match secret.sign(msg).as_ref().try_into() { - Ok(signature) => signature, - Err(_) => { - warn!("Invalid signature"); - return Signature([0; ED25519_SIGNATURE_LEN]); - } - }; + let signature = keypair.sign(msg).as_ref().try_into() + .expect("Invalid signature?!?"); Signature(signature) } @@ -110,10 +135,7 @@ pub mod sign_ed25519 { let start = sm.len() - ED25519_SIGNATURE_LEN; let sig = Signature(match sm[start..].try_into() { Ok(sig) => sig, - Err(_) => { - warn!("Invalid signature"); - return false; - } + Err(_) => return false, }); let msg = &sm[..start]; verify_detached(&sig, msg, pk) @@ -129,47 +151,39 @@ pub mod sign_ed25519 { sm } + /// Generates a completely random Ed25519 keypair. pub fn gen_keypair() -> (PublicKey, SecretKey) { - let rand = ring::rand::SystemRandom::new(); - let pkcs8 = match SecretKeyBase::generate_pkcs8(&rand) { - Ok(pkcs8) => pkcs8, - Err(_) => { - warn!("Failed to generate secret key base for pkcs8"); - return (PublicKey([0; ED25519_PUBLIC_KEY_LEN]), SecretKey(vec![])); - } - }; + let seed = generate_secure_random(); + gen_keypair_from_seed(&seed) + } - let secret = match SecretKeyBase::from_pkcs8(pkcs8.as_ref()) { - Ok(secret) => secret, - Err(_) => { - warn!("Invalid secret key base"); - return (PublicKey([0; ED25519_PUBLIC_KEY_LEN]), SecretKey(vec![])); - } + /// Generates an Ed25519 keypair based on the given seed. + /// + /// ### Arguments + /// + /// * `seed` - the seed to generate the keypair from + fn gen_keypair_from_seed(seed: &[u8; ED25519_SEED_LEN]) -> (PublicKey, SecretKey) { + let rand = ring::test::rand::FixedSliceSequenceRandom { + bytes: &[ seed ], + current: core::cell::UnsafeCell::new(0), }; - let pub_key_gen = match secret.public_key().as_ref().try_into() { - Ok(pub_key_gen) => pub_key_gen, - Err(_) => { - warn!("Invalid public key generation"); - return (PublicKey([0; ED25519_PUBLIC_KEY_LEN]), SecretKey(vec![])); - } - }; - let public = PublicKey(pub_key_gen); - let secret = match SecretKey::from_slice(pkcs8.as_ref()) { - Some(secret) => secret, - None => { - warn!("Invalid secret key"); - return (PublicKey([0; ED25519_PUBLIC_KEY_LEN]), SecretKey(vec![])); - } - }; + let pkcs8 = SecretKeyBase::generate_pkcs8(&rand) + .expect("Failed to generate secret key base for pkcs8"); + + let keypair = SecretKeyBase::from_pkcs8(pkcs8.as_ref()) + .expect("Generated PKCS8 document is invalid?!?"); - (public, secret) + let public_key = PublicKey(keypair.public_key().as_ref().try_into() + .expect("Generated keypair contains an invalid public key?!?")); + let secret_key = SecretKey(pkcs8.as_ref().to_vec()); + (public_key, secret_key) } } pub mod secretbox_chacha20_poly1305 { // Use key and nonce separately like rust-tls does - use super::{deserialize_slice, generate_random}; + use super::generate_secure_random; pub use ring::aead::LessSafeKey as KeyBase; pub use ring::aead::Nonce as NonceBase; pub use ring::aead::NONCE_LEN; @@ -179,45 +193,8 @@ pub mod secretbox_chacha20_poly1305 { pub const KEY_LEN: usize = 256 / 8; - /// key data - #[derive(Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize)] - pub struct Key( - #[serde(serialize_with = "<[_]>::serialize")] - #[serde(deserialize_with = "deserialize_slice")] - [u8; KEY_LEN], - ); - - impl Key { - pub fn from_slice(slice: &[u8]) -> Option { - Some(Self(slice.try_into().ok()?)) - } - } - - impl AsRef<[u8]> for Key { - fn as_ref(&self) -> &[u8] { - self.0.as_ref() - } - } - - /// Nonce data - #[derive(Clone, Copy, Debug, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize)] - pub struct Nonce( - #[serde(serialize_with = "<[_]>::serialize")] - #[serde(deserialize_with = "deserialize_slice")] - [u8; NONCE_LEN], - ); - - impl Nonce { - pub fn from_slice(slice: &[u8]) -> Option { - Some(Self(slice.try_into().ok()?)) - } - } - - impl AsRef<[u8]> for Nonce { - fn as_ref(&self) -> &[u8] { - self.0.as_ref() - } - } + fixed_bytes_wrapper!(pub struct Key, KEY_LEN, "Key data"); + fixed_bytes_wrapper!(pub struct Nonce, NONCE_LEN, "Nonce data"); pub fn seal(mut plain_text: Vec, nonce: &Nonce, key: &Key) -> Option> { let key = get_keybase(key)?; @@ -249,20 +226,20 @@ pub mod secretbox_chacha20_poly1305 { } fn get_noncebase(nonce: &Nonce) -> NonceBase { - NonceBase::assume_unique_for_key(nonce.0) + NonceBase::assume_unique_for_key(*nonce.0) } pub fn gen_key() -> Key { - Key(generate_random()) + Key(generate_secure_random().into()) } pub fn gen_nonce() -> Nonce { - Nonce(generate_random()) + Nonce(generate_secure_random().into()) } } pub mod pbkdf2 { - use super::{deserialize_slice, generate_random}; + use super::generate_secure_random; use ring::pbkdf2::{derive, PBKDF2_HMAC_SHA256}; use serde::{Deserialize, Serialize}; use std::convert::TryInto; @@ -272,24 +249,7 @@ pub mod pbkdf2 { pub const SALT_LEN: usize = 256 / 8; pub const OPSLIMIT_INTERACTIVE: u32 = 100_000; - #[derive(Clone, Copy, Debug, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize)] - pub struct Salt( - #[serde(serialize_with = "<[_]>::serialize")] - #[serde(deserialize_with = "deserialize_slice")] - [u8; SALT_LEN], - ); - - impl Salt { - pub fn from_slice(slice: &[u8]) -> Option { - Some(Self(slice.try_into().ok()?)) - } - } - - impl AsRef<[u8]> for Salt { - fn as_ref(&self) -> &[u8] { - self.0.as_ref() - } - } + fixed_bytes_wrapper!(pub struct Salt, SALT_LEN, "Salt data"); pub fn derive_key(key: &mut [u8], passwd: &[u8], salt: &Salt, iterations: u32) { let iterations = match NonZeroU32::new(iterations) { @@ -303,44 +263,204 @@ pub mod pbkdf2 { } pub fn gen_salt() -> Salt { - Salt(generate_random()) + Salt(generate_secure_random().into()) } } pub mod sha3_256 { + use std::convert::TryInto; + use std::fmt::{Display, Formatter}; + use std::ops::Deref; + use std::str::FromStr; + pub use sha3::digest::Output; pub use sha3::Digest; pub use sha3::Sha3_256; - pub fn digest(data: &[u8]) -> Output { - Sha3_256::digest(data) + pub const HASH_LEN : usize = 256 / 8; + + /// A SHA3-256 hash. + #[derive(Clone, Copy, Debug, PartialOrd, Ord, PartialEq, Eq)] + pub struct Hash( + [u8; HASH_LEN], + ); + + impl Hash { + pub fn from_slice(slice: &[u8]) -> Option { + Some(Self(slice.try_into().ok()?)) + } + } + + impl Display for Hash { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + f.write_str(&hex::encode(&self.0)) + } + } + + impl FromStr for Hash { + type Err = hex::FromHexError; + + fn from_str(s: &str) -> Result { + let mut buf = [0u8; HASH_LEN]; + match hex::decode_to_slice(s, &mut buf) { + Ok(_) => Ok(Self(buf)), + Err(e) => Err(e), + } + } + } + + impl AsRef<[u8]> for Hash { + fn as_ref(&self) -> &[u8] { + &self.0 + } + } + + impl Deref for Hash { + type Target = [u8; HASH_LEN]; + + fn deref(&self) -> &Self::Target { + &self.0 + } + } + + /// Computes the SHA3-256 hash of the given bytes. + /// + /// ### Arguments + /// + /// * `data` - the bytes to hash + pub fn digest(data: &[u8]) -> Hash { + Hash(Sha3_256::digest(data).try_into().unwrap()) } - pub fn digest_all<'a>(data: impl Iterator) -> Output { + /// Concatenates all the given byte slices and computes the SHA3-256 hash of the result. + /// + /// ### Arguments + /// + /// * `data_parts` - the byte slices to concatenate and hash + pub fn digest_all<'a>(data_parts: impl IntoIterator) -> Hash { let mut hasher = Sha3_256::new(); - data.for_each(|v| hasher.update(v)); - hasher.finalize() + data_parts.into_iter().for_each(|v| hasher.update(v)); + Hash(hasher.finalize().try_into().unwrap()) } -} -fn deserialize_slice<'de, D: serde::Deserializer<'de>, const N: usize>( - deserializer: D, -) -> Result<[u8; N], D::Error> { - let value: &[u8] = serde::Deserialize::deserialize(deserializer)?; - value - .try_into() - .map_err(|_| serde::de::Error::custom("Invalid array in deserialization".to_string())) + /// Serializes the given value using bincode-serde and computes the SHA3-256 hash of the result. + /// + /// ### Arguments + /// + /// * `value` - the value to serialize and hash + pub fn digest_serialize(value: &S) -> bincode::Result { + let mut hasher = Sha3_256::new(); + bincode::serialize_into(&mut hasher, value)?; + Ok(Hash(hasher.finalize().try_into().unwrap())) + } } -pub fn generate_random() -> [u8; N] { - let mut value: [u8; N] = [0; N]; - +fn generate_secure_random() -> [u8; N] { use ring::rand::SecureRandom; let rand = ring::rand::SystemRandom::new(); - match rand.fill(&mut value) { - Ok(_) => (), - Err(_) => warn!("Failed to generate random bytes"), - }; + let mut value = [0u8; N]; + rand.fill(&mut value).expect("Failed to generate random bytes"); value } + +#[cfg(test)] +mod test { + use std::fmt::{Debug, Display}; + use std::str::FromStr; + use serde::Serialize; + use serde::de::DeserializeOwned; + use crate::utils::PlaceholderSeed; + use super::*; + + fn test_placeholders_different_seed() { + let [v0, v1] = W::placeholder_array_seed::<2>([]); + assert_eq!(v0, v0); + assert_eq!(v1, v1); + assert_ne!(v0, v1); + } + + fn test_fixed_bytes_wrapper + PlaceholderSeed + Serialize + DeserializeOwned>( + expected_placeholder_hex: &str, + ) { + let placeholder = W::placeholder_seed([]); + assert_eq!(placeholder.to_string(), expected_placeholder_hex); + assert_eq!(W::from_str(expected_placeholder_hex).unwrap(), placeholder); + + let expected_json = format!("\"{}\"", expected_placeholder_hex); + assert_eq!(serde_json::to_string(&placeholder).unwrap(), expected_json); + assert_eq!(serde_json::from_str::(&expected_json).unwrap(), placeholder); + + test_placeholders_different_seed::(); + } + + #[test] + fn test_ed25519_signature() { + test_fixed_bytes_wrapper::( + "9c4e2259fc9b47b4c4cf672c7436dc16ace2970955a002b69a495ca96d9dfaf026dbee622284a1cf306a1189af8a462d2ea498d10f14b637c848168b0ba698a7", + ); + } + + #[test] + fn test_ed25519_public_key() { + test_fixed_bytes_wrapper::( + "1d67f7de4c59192568f8e0381fcd1eb9ce044568e4670038e0b42e421540b4f6", + ); + } + + #[test] + fn test_ed25519_secret_key() { + assert_eq!(hex::encode(sign_ed25519::SecretKey::placeholder_seed([]).get_pkcs8()), + "3053020101300506032b6570042204203651dccde39be8697d8e0690acd90e3b8ce7f596c5f205fbd0b3b3e3a68629e1a12303210092bc778f74110b3fcbcf8a4df71ed9a33c62faa8d01417d381745ef700ef6b73"); + + test_placeholders_different_seed::(); + } + + #[test] + fn test_ed25519_keypair() { + test_placeholders_different_seed::<(sign_ed25519::PublicKey, sign_ed25519::SecretKey)>(); + } + + #[test] + fn test_chacha20_key() { + test_fixed_bytes_wrapper::( + "d2678ac6abff79fa16d2a8f762a3c33b227a519ac1830aee33a19605b7f9cd35", + ); + } + + #[test] + fn test_chacha20_nonce() { + test_fixed_bytes_wrapper::( + "b0980a0a073d6b828fb48ad5", + ); + } + + #[test] + fn test_pbkdf2_salt() { + test_fixed_bytes_wrapper::( + "81c4a8cde605d6b51857eb6ebaead0de98cf254d4855725db7aec45a98699e9c", + ); + } + + #[test] + fn test_sha3_256_digest_serialize() { + // ensure that sha3_256::digest_serialize gives the same result as serializing the full + // object into a buffer and then hashing that + macro_rules! test_case { + ($t:ty: $($n:literal)*) => { + $( + let arr : [$t; $n] = std::array::from_fn(|n| n as $t); + assert_eq!(sha3_256::digest(&bincode::serialize(&arr).unwrap()), + sha3_256::digest_serialize(&arr).unwrap(), + concat!("[", stringify!($t), "; {}]"), $n); + )* + }; + } + + test_case!(u64: + 00 01 02 03 04 05 06 07 08 09 + 10 11 12 13 14 15 16 17 18 19 + 20 21 22 23 24 25 26 27 28 29 + 30 31 32); + } +} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index adb6987..4a1e85b 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -8,6 +8,7 @@ use crate::primitives::asset::TokenAmount; pub mod druid_utils; pub mod error_utils; pub mod script_utils; +pub mod serialize_utils; pub mod test_utils; pub mod transaction_utils; @@ -51,3 +52,105 @@ pub fn add_btreemap( }); m1 } + +/// A trait which indicates that it is possible to acquire a "placeholder" value +/// of a type, which can be used for test purposes. +#[cfg(test)] +pub trait Placeholder : Sized { + /// Gets a placeholder value of this type which can be used for test purposes. + fn placeholder() -> Self; + + /// Gets an array of placeholder values of this type which can be used for test purposes. + fn placeholder_array() -> [Self; N] { + core::array::from_fn(|_| Self::placeholder()) + } +} + +/// A trait which indicates that it is possible to acquire a "placeholder" value +/// of a type, which can be used for test purposes. These placeholder values are consistent +/// across program runs. +#[cfg(test)] +pub trait PlaceholderSeed: Sized + PartialEq { + /// Gets a dummy valid of this type which can be used for test purposes. + /// + /// This allows acquiring multiple distinct placeholder values which are still consistent + /// between runs. + /// + /// ### Arguments + /// + /// * `seed_parts` - the parts of the seed for the placeholder value to obtain. Two placeholder + /// values generated from the same seed are guaranteed to be equal (even + /// across multiple test runs, so long as the value format doesn't change). + fn placeholder_seed_parts<'a>(seed_parts: impl IntoIterator) -> Self; + + /// Gets a dummy valid of this type which can be used for test purposes. + /// + /// This allows acquiring multiple distinct placeholder values which are still consistent + /// between runs. + /// + /// ### Arguments + /// + /// * `seed` - the seed for the placeholder value to obtain. Two placeholder + /// values generated from the same seed are guaranteed to be equal (even + /// across multiple test runs, so long as the value format doesn't change). + fn placeholder_seed(seed: impl AsRef<[u8]>) -> Self { + Self::placeholder_seed_parts([ seed.as_ref() ]) + } + + /// Gets a dummy valid of this type which can be used for test purposes. + /// + /// This allows acquiring multiple distinct placeholder values which are still consistent + /// between runs. + /// + /// ### Arguments + /// + /// * `index` - the index of the placeholder value to obtain. Two placeholder values generated + /// from the same index are guaranteed to be equal (even across multiple test runs, + /// so long as the value format doesn't change). + fn placeholder_indexed(index: u64) -> Self { + Self::placeholder_seed_parts([ index.to_le_bytes().as_slice() ]) + } + + /// Gets an array of placeholder values of this type which can be used for test purposes. + fn placeholder_array_seed(seed: impl AsRef<[u8]>) -> [Self; N] { + core::array::from_fn(|n| Self::placeholder_seed_parts( + [ seed.as_ref(), &(n as u64).to_le_bytes() ] + )) + } + + /// Gets an array of placeholder values of this type which can be used for test purposes. + fn placeholder_array_indexed(base_index: u64) -> [Self; N] { + Self::placeholder_array_seed(base_index.to_le_bytes()) + } +} + +#[cfg(test)] +impl Placeholder for T { + fn placeholder() -> Self { + ::placeholder_seed_parts([]) + } +} + +/// Generates the given number of pseudorandom bytes based on the given seed. +/// +/// This is intended to be used in tests, where random but reproducible placeholder values are often +/// required. +/// +/// ### Arguments +/// +/// * `seed_parts` - the parts of the seed, which will be concatenated to form the RNG seed +#[cfg(test)] +pub fn placeholder_bytes<'a, const N: usize>( + seed_parts: impl IntoIterator +) -> [u8; N] { + // Use Shake-256 to generate an arbitrarily large number of random bytes based on the given seed. + let mut shake256 = sha3::Shake256::default(); + for slice in seed_parts { + sha3::digest::Update::update(&mut shake256, slice); + } + let mut reader = sha3::digest::ExtendableOutput::finalize_xof(shake256); + + let mut res = [0u8; N]; + sha3::digest::XofReader::read(&mut reader, &mut res); + res +} diff --git a/src/utils/serialize_utils.rs b/src/utils/serialize_utils.rs new file mode 100644 index 0000000..af72618 --- /dev/null +++ b/src/utils/serialize_utils.rs @@ -0,0 +1,495 @@ +use std::any::TypeId; +use std::convert::{TryFrom, TryInto}; +use std::fmt; +use std::marker::PhantomData; +use std::ops::{Deref, DerefMut}; +use std::str::FromStr; + +use bincode::config::*; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde::de::{SeqAccess, Visitor}; +use serde::ser::{SerializeTuple}; + +pub fn bincode_default() -> WithOtherTrailing, RejectTrailing> { + DefaultOptions::new() + .with_fixint_encoding() + .reject_trailing_bytes() +} + +pub fn bincode_compact() -> WithOtherTrailing, RejectTrailing> { + DefaultOptions::new() + .with_varint_encoding() + .reject_trailing_bytes() +} + +/// Simple wrapper around a fixed-length byte array. +/// +/// This can be formatted to and parsed from a hexadecimal string using `Display` and `FromStr`. +/// When serialized as JSON, it is also represented as a hexadecimal string. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct FixedByteArray( + #[serde(with = "fixed_array_codec")] + [u8; N], +); + +impl FixedByteArray { + pub fn new(arr: [u8; N]) -> Self { + Self(arr) + } +} + +impl AsRef<[u8]> for FixedByteArray { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl AsMut<[u8]> for FixedByteArray { + fn as_mut(&mut self) -> &mut [u8] { + &mut self.0 + } +} + +impl Deref for FixedByteArray { + type Target = [u8; N]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for FixedByteArray { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl From<[u8; N]> for FixedByteArray { + fn from(value: [u8; N]) -> Self { + Self(value) + } +} + +impl From<&[u8; N]> for FixedByteArray { + fn from(value: &[u8; N]) -> Self { + Self(*value) + } +} + +impl TryFrom<&[u8]> for FixedByteArray { + type Error = std::array::TryFromSliceError; + + fn try_from(value: &[u8]) -> Result { + value.try_into().map(Self) + } +} + +impl TryFrom<&Vec> for FixedByteArray { + type Error = std::array::TryFromSliceError; + + fn try_from(value: &Vec) -> Result { + value.as_slice().try_into().map(Self) + } +} + +impl fmt::LowerHex for FixedByteArray { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // This is hacky because we can't make an array of type [u8; {N * 2}] due to + // generic parameters not being allowed in constant expressions on stable rust + assert_eq!(std::mem::size_of::<[u16; N]>(), std::mem::size_of::<[u8; N]>() * 2); + let mut buf = [0u16; N]; + let slice = unsafe { std::slice::from_raw_parts_mut(buf.as_mut_ptr() as *mut u8, N * 2) }; + hex::encode_to_slice(&self.0, slice).unwrap(); + f.write_str(std::str::from_utf8(slice).unwrap()) + } +} + +impl fmt::Display for FixedByteArray { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::LowerHex::fmt(self, f) + } +} + +impl fmt::Debug for FixedByteArray { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "FixedByteArray<{N}>({self:x})") + } +} + +impl FromStr for FixedByteArray { + type Err = hex::FromHexError; + + fn from_str(s: &str) -> Result { + let mut buf = [0u8; N]; + hex::decode_to_slice(s, &mut buf)?; + Ok(Self(buf)) + } +} + +/// A codec for fixed-size arrays. +pub mod fixed_array_codec { + use super::*; + + pub fn serialize( + values: &[T; N], + serializer: S, + ) -> Result { + if TypeId::of::() == TypeId::of::() && serializer.is_human_readable() { + // We're serializing a byte array for a human-readable format, make it a hex string + vec_codec::serialize(values, serializer) + } else { + // Serialize the array as a tuple, to avoid adding a length prefix + let mut tuple = serializer.serialize_tuple(N)?; + for e in values { + tuple.serialize_element(e)?; + } + tuple.end() + } + } + + pub fn deserialize<'de, T: Deserialize<'de> + 'static, D: Deserializer<'de>, const N: usize>( + deserializer: D, + ) -> Result<[T; N], D::Error> { + if TypeId::of::() == TypeId::of::() && deserializer.is_human_readable() { + // We're deserializing a byte array for a human-readable format, we'll accept two different + // representations: + // - A hexadecimal string + // - An array of byte literals (this format should never be produced by the serializer + // for human-readable formats, but it was in the past, so we'll still support reading + // it for backwards-compatibility). + vec_to_fixed_array(vec_codec::deserialize(deserializer)?) + } else { + // We're deserializing a binary format, read the array as a tuple + // (to avoid adding a length prefix) + + struct FixedArrayVisitor(PhantomData); + impl<'de, T: Deserialize<'de>, const N: usize> Visitor<'de> for FixedArrayVisitor { + type Value = [T; N]; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "a sequence") + } + + fn visit_seq>(self, mut seq: A) -> Result { + let mut vec = Vec::with_capacity(N); + while let Some(val) = seq.next_element::()? { + vec.push(val) + } + vec_to_fixed_array(vec) + } + } + + deserializer.deserialize_tuple(N, FixedArrayVisitor(Default::default())) + } + } +} + +/// A codec for variable-length `Vec`s. +pub mod vec_codec { + use super::*; + + pub fn serialize( + values: &[T], + serializer: S, + ) -> Result { + if TypeId::of::() == TypeId::of::() && serializer.is_human_readable() { + // We're serializing a byte array for a human-readable format, make it a hex string + let bytes = unsafe { std::slice::from_raw_parts(values.as_ptr() as *const u8, values.len()) }; + serializer.serialize_str(&hex::encode(bytes)) + } else { + // Serialize the array as a length-prefixed sequence + values.serialize(serializer) + } + } + + pub fn deserialize<'de, T: Deserialize<'de> + 'static, D: Deserializer<'de>>( + deserializer: D, + ) -> Result, D::Error> { + if TypeId::of::() == TypeId::of::() && deserializer.is_human_readable() { + // We're deserializing a byte array for a human-readable format, we'll accept two different + // representations: + // - A hexadecimal string + // - An array of byte literals (this format should never be produced by the serializer + // for human-readable formats, but it was in the past, so we'll still support reading + // it for backwards-compatibility). + + struct HexStringOrBytesVisitor(); + impl<'de> Visitor<'de> for HexStringOrBytesVisitor { + type Value = Vec; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("hex string or byte array") + } + + fn visit_str(self, value: &str) -> Result { + hex::decode(value).map_err(E::custom) + } + + fn visit_seq(self, mut seq: A) -> Result where A: SeqAccess<'de> { + let mut vec = Vec::new(); + while let Some(elt) = seq.next_element::()? { + vec.push(elt); + } + Ok(vec) + } + } + + Ok(deserializer.deserialize_any(HexStringOrBytesVisitor())?.into_iter() + // This is a hack to convert the Vec into a Vec, even though we already know + // that T = u8. This could be done in a much nicer way if trait specialization were + // a thing, but unfortunately it's still only available on nightly :( + .map(|b| unsafe { std::mem::transmute_copy::(&b) }) + .collect::>()) + } else { + // Read a length-prefixed sequence as a Vec + >::deserialize(deserializer) + } + } +} + +fn vec_to_fixed_array( + vec: Vec, +) -> Result<[T; N], E> { + <[T; N]>::try_from(vec) + .map_err(|vec| E::custom(format!("expected exactly {} elements, but read {}", N, vec.len()))) +} + +/*---- TESTS ----*/ + +#[cfg(test)] +mod tests { + use std::fmt::{Debug, Display}; + use bincode::Options; + + use serde::{Deserialize, Serialize}; + use serde::de::DeserializeOwned; + use super::*; + + fn repeat(orig: &str, n: usize) -> String { + let mut res = String::with_capacity(orig.len() * n); + for _ in 0..n { + res.push_str(orig) + } + res + } + + fn test_bin_codec( + options: fn() -> O, + obj: T, + expect: &str, + ) { + let bytes = options().serialize(&obj).unwrap(); + assert_eq!(hex::encode(&bytes), expect); + assert_eq!(options().deserialize::(&bytes).unwrap(), obj); + } + + fn test_json_codec( + obj: T, + expect: &str, + ) { + let json = serde_json::to_string(&obj).unwrap(); + assert_eq!(json, expect); + assert_eq!(serde_json::from_str::(&json).unwrap(), obj); + } + + fn test_json_deserialize( + obj: T, + json: &str, + ) { + assert_eq!(serde_json::from_str::(&json).unwrap(), obj); + } + + fn test_display_fromstr( + obj: T, + expect: &str, + ) { + let string = obj.to_string(); + assert_eq!(string, expect); + assert_eq!(::from_str(&string).ok().unwrap(), obj); + } + + macro_rules! test_fixed_array { + ($n:literal) => { + test_bin_codec(bincode_default, [VAL; $n], &repeat(HEX, $n)); + test_json_codec([VAL; $n], &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); + }; + } + + macro_rules! test_fixed_array_wrapper { + ($e:ty, $t:ident, $n:literal) => { + #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] + struct $t([$e; $n]); + test_bin_codec(bincode_default, $t([VAL; $n]), &repeat(HEX, $n)); + test_json_codec($t([VAL; $n]), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); + }; + } + + #[test] + fn test_fixed_u32_arrays() { + const VAL : u32 = 0xDEADBEEF; + const HEX : &str = "efbeadde"; + + test_fixed_array!(0); + test_fixed_array!(1); + test_fixed_array!(32); + + test_fixed_array_wrapper!(u32, FixedArrayWrapper0, 0); + test_fixed_array_wrapper!(u32, FixedArrayWrapper1, 1); + test_fixed_array_wrapper!(u32, FixedArrayWrapper32, 32); + + macro_rules! test_fixed_array_wrapper_codec { + ($t:ident, $n:literal) => { + #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] + struct $t(#[serde(with = "fixed_array_codec")] [u32; $n]); + test_bin_codec(bincode_default, $t([VAL; $n]), &repeat(HEX, $n)); + test_json_codec($t([VAL; $n]), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); + }; + } + + test_fixed_array_wrapper_codec!(CodecFixedArrayWrapper0, 0); + test_fixed_array_wrapper_codec!(CodecFixedArrayWrapper1, 1); + test_fixed_array_wrapper_codec!(CodecFixedArrayWrapper32, 32); + test_fixed_array_wrapper_codec!(CodecFixedArrayWrapper33, 33); + } + + #[test] + fn test_fixed_u8_arrays() { + const VAL : u8 = 123; + const HEX : &str = "7b"; + + test_fixed_array!(0); + test_fixed_array!(1); + test_fixed_array!(32); + + test_fixed_array_wrapper!(u8, FixedArrayWrapper0, 0); + test_fixed_array_wrapper!(u8, FixedArrayWrapper1, 1); + test_fixed_array_wrapper!(u8, FixedArrayWrapper32, 32); + + macro_rules! test_fixed_array_wrapper_codec { + ($t:ident, $n:literal) => { + #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] + struct $t(#[serde(with = "fixed_array_codec")] [u8; $n]); + test_bin_codec(bincode_default, $t([VAL; $n]), &repeat(HEX, $n)); + test_json_codec($t([VAL; $n]), &format!("\"{}\"", hex::encode(&[VAL; $n].to_vec()))); + test_json_deserialize($t([VAL; $n]), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); + }; + } + + test_fixed_array_wrapper_codec!(CodecFixedArrayWrapper0, 0); + test_fixed_array_wrapper_codec!(CodecFixedArrayWrapper1, 1); + test_fixed_array_wrapper_codec!(CodecFixedArrayWrapper32, 32); + test_fixed_array_wrapper_codec!(CodecFixedArrayWrapper33, 33); + } + + fn size_to_hex_default(n: usize) -> String { + hex::encode(bincode_default().serialize(&n).unwrap()) + } + + macro_rules! test_vec { + ($n:literal) => { + test_bin_codec(bincode_default, [VAL; $n].to_vec(), &format!("{}{}", size_to_hex_default($n), repeat(HEX, $n))); + test_json_codec([VAL; $n].to_vec(), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); + }; + } + + macro_rules! test_vec_wrapper { + ($n:literal) => { + test_bin_codec(bincode_default, VecWrapper([VAL; $n].to_vec()), &format!("{}{}", size_to_hex_default($n), repeat(HEX, $n))); + test_json_codec(VecWrapper([VAL; $n].to_vec()), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); + }; + } + + #[test] + fn test_u32_vecs() { + const VAL : u32 = 0xDEADBEEF; + const HEX : &str = "efbeadde"; + + test_vec!(0); + test_vec!(1); + test_vec!(32); + test_vec!(33); + + #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] + struct VecWrapper(Vec); + + test_vec_wrapper!(0); + test_vec_wrapper!(1); + test_vec_wrapper!(32); + test_vec_wrapper!(33); + + #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] + struct CodecVecWrapper(#[serde(with = "vec_codec")] Vec); + macro_rules! test_vec_wrapper_codec { + ($n:literal) => { + test_bin_codec(bincode_default, CodecVecWrapper([VAL; $n].to_vec()), &format!("{}{}", size_to_hex_default($n), repeat(HEX, $n))); + test_json_codec(CodecVecWrapper([VAL; $n].to_vec()), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); + }; + } + + test_vec_wrapper_codec!(0); + test_vec_wrapper_codec!(1); + test_vec_wrapper_codec!(32); + test_vec_wrapper_codec!(33); + } + + #[test] + fn test_u8_vecs() { + const VAL : u8 = 123; + const HEX : &str = "7b"; + + test_vec!(0); + test_vec!(1); + test_vec!(32); + test_vec!(33); + + #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] + struct VecWrapper(Vec); + + test_vec_wrapper!(0); + test_vec_wrapper!(1); + test_vec_wrapper!(32); + test_vec_wrapper!(33); + + #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] + struct CodecVecWrapper(#[serde(with = "vec_codec")] Vec); + macro_rules! test_vec_wrapper_codec { + ($n:literal) => { + test_bin_codec(bincode_default, CodecVecWrapper([VAL; $n].to_vec()), &format!("{}{}", size_to_hex_default($n), repeat(HEX, $n))); + test_json_codec(CodecVecWrapper([VAL; $n].to_vec()), &format!("\"{}\"", hex::encode(&[VAL; $n].to_vec()))); + test_json_deserialize(CodecVecWrapper([VAL; $n].to_vec()), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); + }; + } + + test_vec_wrapper_codec!(0); + test_vec_wrapper_codec!(1); + test_vec_wrapper_codec!(32); + test_vec_wrapper_codec!(33); + } + + #[test] + fn test_fixed_byte_array() { + const VAL : u8 = 123; + const HEX : &str = "7b"; + + macro_rules! test_fixed_byte_array { + ($n:literal) => { + test_bin_codec(bincode_default, FixedByteArray::<$n>([VAL; $n]), &repeat(HEX, $n)); + test_json_codec(FixedByteArray::<$n>([VAL; $n]), &format!("\"{}\"", repeat(HEX, $n))); + test_json_deserialize(FixedByteArray::<$n>([VAL; $n]), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); + test_display_fromstr(FixedByteArray::<$n>([VAL; $n]), &repeat(HEX, $n)); + assert_eq!(format!("{:x}", FixedByteArray::<$n>([VAL; $n])), repeat(HEX, $n)); + assert_eq!( + format!("{:?}", FixedByteArray::<$n>([VAL; $n])), + format!("FixedByteArray<{}>({})", $n, repeat(HEX, $n))); + assert_eq!( + format!("{:x?}", FixedByteArray::<$n>([VAL; $n])), + format!("FixedByteArray<{}>({})", $n, repeat(HEX, $n))); + }; + } + + test_fixed_byte_array!(0); + test_fixed_byte_array!(1); + test_fixed_byte_array!(32); + test_fixed_byte_array!(33); + } +}