//! Efficient and customizable data-encoding functions like base64, base32, and hex //! //! This [crate] provides little-endian ASCII base-conversion encodings for //! bases of size 2, 4, 8, 16, 32, and 64. It supports: //! //! - [padding] for streaming //! - canonical encodings (e.g. [trailing bits] are checked) //! - in-place [encoding] and [decoding] functions //! - partial [decoding] functions (e.g. for error recovery) //! - character [translation] (e.g. for case-insensitivity) //! - most and least significant [bit-order] //! - [ignoring] characters when decoding (e.g. for skipping newlines) //! - [wrapping] the output when encoding //! - no-std environments with `default-features = false, features = ["alloc"]` //! - no-alloc environments with `default-features = false` //! //! You may use the [binary] or the [website] to play around. //! //! # Examples //! //! This crate provides predefined encodings as [constants]. These constants are of type //! [`Encoding`]. This type provides encoding and decoding functions with in-place or allocating //! variants. Here is an example using the allocating encoding function of [`BASE64`]: //! //! ```rust //! use data_encoding::BASE64; //! assert_eq!(BASE64.encode(b"Hello world"), "SGVsbG8gd29ybGQ="); //! ``` //! //! Here is an example using the in-place decoding function of [`BASE32`]: //! //! ```rust //! use data_encoding::BASE32; //! let input = b"JBSWY3DPEB3W64TMMQ======"; //! let mut output = vec![0; BASE32.decode_len(input.len()).unwrap()]; //! let len = BASE32.decode_mut(input, &mut output).unwrap(); //! assert_eq!(&output[0 .. len], b"Hello world"); //! ``` //! //! You are not limited to the predefined encodings. You may define your own encodings (with the //! same correctness and performance properties as the predefined ones) using the [`Specification`] //! type: //! //! ```rust //! use data_encoding::Specification; //! let hex = { //! let mut spec = Specification::new(); //! spec.symbols.push_str("0123456789abcdef"); //! spec.encoding().unwrap() //! }; //! assert_eq!(hex.encode(b"hello"), "68656c6c6f"); //! ``` //! //! You may use the [macro] library to define a compile-time custom encoding: //! //! ```rust,ignore //! use data_encoding::Encoding; //! use data_encoding_macro::new_encoding; //! const HEX: Encoding = new_encoding!{ //! symbols: "0123456789abcdef", //! translate_from: "ABCDEF", //! translate_to: "abcdef", //! }; //! const BASE64: Encoding = new_encoding!{ //! symbols: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/", //! padding: '=', //! }; //! ``` //! //! # Properties //! //! The [`HEXUPPER`], [`BASE32`], [`BASE32HEX`], [`BASE64`], and [`BASE64URL`] predefined encodings //! are conform to [RFC4648]. //! //! In general, the encoding and decoding functions satisfy the following properties: //! //! - They are deterministic: their output only depends on their input //! - They have no side-effects: they do not modify a hidden mutable state //! - They are correct: encoding then decoding gives the initial data //! - They are canonical (unless [`is_canonical`] returns false): decoding then encoding gives the //! initial data //! //! This last property is usually not satisfied by base64 implementations. This is a matter of //! choice and this crate has made the choice to let the user choose. Support for canonical encoding //! as described by the [RFC][canonical] is provided. But it is also possible to disable checking //! trailing bits, to add characters translation, to decode concatenated padded inputs, and to //! ignore some characters. //! //! Since the RFC specifies the encoding function on all inputs and the decoding function on all //! possible encoded outputs, the differences between implementations come from the decoding //! function which may be more or less permissive. In this crate, the decoding function of canonical //! encodings rejects all inputs that are not a possible output of the encoding function. Here are //! some concrete examples of decoding differences between this crate, the `base64` crate, and the //! `base64` GNU program: //! //! | Input | `data-encoding` | `base64` | GNU `base64` | //! | ---------- | --------------- | --------- | ------------- | //! | `AAB=` | `Trailing(2)` | `Last(2)` | `\x00\x00` | //! | `AA\nB=` | `Length(4)` | `Length` | `\x00\x00` | //! | `AAB` | `Length(0)` | `Last(2)` | Invalid input | //! | `AAA` | `Length(0)` | `[0, 0]` | Invalid input | //! | `A\rA\nB=` | `Length(4)` | `Byte(1)` | Invalid input | //! | `-_\r\n` | `Symbol(0)` | `Byte(0)` | Invalid input | //! | `AA==AA==` | `[0, 0]` | `Byte(2)` | `\x00\x00` | //! //! We can summarize these discrepancies as follows: //! //! | Discrepancy | `data-encoding` | `base64` | GNU `base64` | //! | -------------------------- | --------------- | -------- | ------------ | //! | Check trailing bits | Yes | Yes | No | //! | Ignored characters | None | None | `\n` | //! | Translated characters | None | None | None | //! | Check padding | Yes | No | Yes | //! | Support concatenated input | Yes | No | Yes | //! //! This crate permits to disable checking trailing bits. It permits to ignore some characters. It //! permits to translate characters. It permits to use unpadded encodings. However, for padded //! encodings, support for concatenated inputs cannot be disabled. This is simply because it doesn't //! make sense to use padding if it is not to support concatenated inputs. //! //! [RFC4648]: https://tools.ietf.org/html/rfc4648 //! [`BASE32HEX`]: constant.BASE32HEX.html //! [`BASE32`]: constant.BASE32.html //! [`BASE64URL`]: constant.BASE64URL.html //! [`BASE64`]: constant.BASE64.html //! [`Encoding`]: struct.Encoding.html //! [`HEXUPPER`]: constant.HEXUPPER.html //! [`Specification`]: struct.Specification.html //! [`is_canonical`]: struct.Encoding.html#method.is_canonical //! [binary]: https://crates.io/crates/data-encoding-bin //! [bit-order]: struct.Specification.html#structfield.bit_order //! [canonical]: https://tools.ietf.org/html/rfc4648#section-3.5 //! [constants]: index.html#constants //! [crate]: https://crates.io/crates/data-encoding //! [decoding]: struct.Encoding.html#method.decode_mut //! [encoding]: struct.Encoding.html#method.encode_mut //! [ignoring]: struct.Specification.html#structfield.ignore //! [macro]: https://crates.io/crates/data-encoding-macro //! [padding]: struct.Specification.html#structfield.padding //! [trailing bits]: struct.Specification.html#structfield.check_trailing_bits //! [translation]: struct.Specification.html#structfield.translate //! [website]: https://data-encoding.rs //! [wrapping]: struct.Specification.html#structfield.wrap
fn div_ceil(x: usize, m: usize) -> usize {
(x + m - 1) / m
}
fn floor(x: usize, m: usize) -> usize {
x / m * m
}
fn vectorize<F: FnMut(usize)>(n: usize, bs: usize, mut f: F) { for k in0 .. n / bs { for i in k * bs .. (k + 1) * bs {
f(i);
}
} for i in floor(n, bs) .. n {
f(i);
}
}
/// Decoding error #[derive(Debug, Copy, Clone, PartialEq, Eq)] pubstruct DecodeError { /// Error position /// /// This position is always a valid input position and represents the first encountered error. pub position: usize,
/// Error kind pub kind: DecodeKind,
}
#[cfg(feature = "std")] impl std::error::Error for DecodeError {}
/// Decoding error with partial result #[derive(Debug, Copy, Clone, PartialEq, Eq)] pubstruct DecodePartial { /// Number of bytes read from input /// /// This number does not exceed the error position: `read <= error.position`. pub read: usize,
/// Number of bytes written to output /// /// This number does not exceed the decoded length: `written <= decode_len(read)`. pub written: usize,
fn encode_block<B: Static<usize>, M: Static<bool>>(
bit: B, msb: M, symbols: &[u8; 256], input: &[u8], output: &mut [u8],
) {
debug_assert!(input.len() <= enc(bit.val()));
debug_assert_eq!(output.len(), encode_len(bit, input.len())); let bit = bit.val(); let msb = msb.val(); letmut x = 0u64; for (i, input) in input.iter().enumerate() {
x |= u64::from(*input) << (8 * order(msb, enc(bit), i));
} for (i, output) in output.iter_mut().enumerate() { let y = x >> (bit * order(msb, dec(bit), i));
*output = symbols[y as usize % 256];
}
}
fn encode_mut<B: Static<usize>, M: Static<bool>>(
bit: B, msb: M, symbols: &[u8; 256], input: &[u8], output: &mut [u8],
) {
debug_assert_eq!(output.len(), encode_len(bit, input.len())); let enc = enc(bit.val()); let dec = dec(bit.val()); let n = input.len() / enc; let bs = match bit.val() { 5 => 2, 6 => 4,
_ => 1,
};
vectorize(n, bs, |i| { let input = unsafe { chunk_unchecked(input, enc, i) }; let output = unsafe { chunk_mut_unchecked(output, dec, i) };
encode_block(bit, msb, symbols, input, output);
});
encode_block(bit, msb, symbols, &input[enc * n ..], &mut output[dec * n ..]);
}
// Fails if an input character does not translate to a symbol. The error is the // lowest index of such character. The output is not written to. fn decode_block<B: Static<usize>, M: Static<bool>>(
bit: B, msb: M, values: &[u8; 256], input: &[u8], output: &mut [u8],
) -> Result<(), usize> {
debug_assert!(output.len() <= enc(bit.val()));
debug_assert_eq!(input.len(), encode_len(bit, output.len())); let bit = bit.val(); let msb = msb.val(); letmut x = 0u64; for j in0 .. input.len() { let y = values[input[j] as usize];
check!(j, y < 1 << bit);
x |= u64::from(y) << (bit * order(msb, dec(bit), j));
} for (j, output) in output.iter_mut().enumerate() {
*output = (x >> (8 * order(msb, enc(bit), j))) as u8;
}
Ok(())
}
// Fails if an input character does not translate to a symbol. The error `pos` // is the lowest index of such character. The output is valid up to `pos / dec * // enc` excluded. fn decode_mut<B: Static<usize>, M: Static<bool>>(
bit: B, msb: M, values: &[u8; 256], input: &[u8], output: &mut [u8],
) -> Result<(), usize> {
debug_assert_eq!(input.len(), encode_len(bit, output.len())); let enc = enc(bit.val()); let dec = dec(bit.val()); let n = input.len() / dec; for i in0 .. n { let input = unsafe { chunk_unchecked(input, dec, i) }; let output = unsafe { chunk_mut_unchecked(output, enc, i) };
decode_block(bit, msb, values, input, output).map_err(|e| dec * i + e)?;
}
decode_block(bit, msb, values, &input[dec * n ..], &mut output[enc * n ..])
.map_err(|e| dec * n + e)
}
// Fails if the padding length is invalid. The error is the index of the first // padding character. fn check_pad<B: Static<usize>>(bit: B, values: &[u8; 256], input: &[u8]) -> Result<usize, usize> { let bit = bit.val();
debug_assert_eq!(input.len(), dec(bit)); let is_pad = |x: &&u8| values[**x as usize] == PADDING; let count = input.iter().rev().take_while(is_pad).count(); let len = input.len() - count;
check!(len, len > 0 && bit * len % 8 < bit);
Ok(len)
}
fn encode_wrap_mut< 'a,
B: Static<usize>,
M: Static<bool>,
P: Static<Option<u8>>,
W: Static<Option<(usize, &'a [u8])>>,
>(
bit: B, msb: M, symbols: &[u8; 256], pad: P, wrap: W, input: &[u8], output: &mut [u8],
) { let (col, end) = match wrap.val() {
None => return encode_pad(bit, msb, symbols, pad, input, output),
Some((col, end)) => (col, end),
};
debug_assert_eq!(output.len(), encode_wrap_len(bit, pad, wrap, input.len()));
debug_assert_eq!(col % dec(bit.val()), 0); let col = col / dec(bit.val()); let enc = col * enc(bit.val()); let dec = col * dec(bit.val()) + end.len(); let olen = dec - end.len(); let n = input.len() / enc; for i in0 .. n { let input = unsafe { chunk_unchecked(input, enc, i) }; let output = unsafe { chunk_mut_unchecked(output, dec, i) };
encode_base(bit, msb, symbols, input, &mut output[.. olen]);
output[olen ..].copy_from_slice(end);
} if input.len() > enc * n { let olen = dec * n + encode_pad_len(bit, pad, input.len() - enc * n);
encode_pad(bit, msb, symbols, pad, &input[enc * n ..], &mut output[dec * n .. olen]);
output[olen ..].copy_from_slice(end);
}
}
// Returns the longest valid input length and associated output length. fn decode_wrap_len<B: Static<usize>, P: Static<bool>>(
bit: B, pad: P, len: usize,
) -> (usize, usize) { let bit = bit.val(); if pad.val() {
(floor(len, dec(bit)), len / dec(bit) * enc(bit))
} else { let trail = bit * len % 8;
(len - trail / bit, bit * len / 8)
}
}
// Fails with Length if length is invalid. The error is the largest valid // length. fn decode_pad_len<B: Static<usize>, P: Static<bool>>(
bit: B, pad: P, len: usize,
) -> Result<usize, DecodeError> { let (ilen, olen) = decode_wrap_len(bit, pad, len);
check!(DecodeError { position: ilen, kind: DecodeKind::Length }, ilen == len);
Ok(olen)
}
// Fails with Length if length is invalid. The error is the largest valid // length. fn decode_base_len<B: Static<usize>>(bit: B, len: usize) -> Result<usize, DecodeError> {
decode_pad_len(bit, Bf, len)
}
// Fails with Symbol if an input character does not translate to a symbol. The // error is the lowest index of such character. // Fails with Trailing if there are non-zero trailing bits. fn decode_base_mut<B: Static<usize>, M: Static<bool>>(
bit: B, msb: M, ctb: bool, values: &[u8; 256], input: &[u8], output: &mut [u8],
) -> Result<usize, DecodePartial> {
debug_assert_eq!(Ok(output.len()), decode_base_len(bit, input.len())); let fail = |pos, kind| DecodePartial {
read: pos / dec(bit.val()) * dec(bit.val()),
written: pos / dec(bit.val()) * enc(bit.val()),
error: DecodeError { position: pos, kind },
};
decode_mut(bit, msb, values, input, output).map_err(|pos| fail(pos, DecodeKind::Symbol))?;
check_trail(bit, msb, ctb, values, input)
.map_err(|()| fail(input.len() - 1, DecodeKind::Trailing))?;
Ok(output.len())
}
// Fails with Symbol if an input character does not translate to a symbol. The // error is the lowest index of such character. // Fails with Padding if some padding length is invalid. The error is the index // of the first padding character of the invalid padding. // Fails with Trailing if there are non-zero trailing bits. fn decode_pad_mut<B: Static<usize>, M: Static<bool>, P: Static<bool>>(
bit: B, msb: M, ctb: bool, values: &[u8; 256], pad: P, input: &[u8], output: &mut [u8],
) -> Result<usize, DecodePartial> { if !pad.val() { return decode_base_mut(bit, msb, ctb, values, input, output);
}
debug_assert_eq!(Ok(output.len()), decode_pad_len(bit, pad, input.len())); let enc = enc(bit.val()); let dec = dec(bit.val()); letmut inpos = 0; letmut outpos = 0; letmut outend = output.len(); while inpos < input.len() { match decode_base_mut(
bit,
msb,
ctb,
values,
&input[inpos ..],
&mut output[outpos .. outend],
) {
Ok(written) => { if cfg!(debug_assertions) {
inpos = input.len();
}
outpos += written; break;
}
Err(partial) => {
inpos += partial.read;
outpos += partial.written;
}
} let inlen =
check_pad(bit, values, &input[inpos .. inpos + dec]).map_err(|pos| DecodePartial {
read: inpos,
written: outpos,
error: DecodeError { position: inpos + pos, kind: DecodeKind::Padding },
})?; let outlen = decode_base_len(bit, inlen).unwrap(); let written = decode_base_mut(
bit,
msb,
ctb,
values,
&input[inpos .. inpos + inlen],
&mut output[outpos .. outpos + outlen],
)
.map_err(|partial| {
debug_assert_eq!(partial.read, 0);
debug_assert_eq!(partial.written, 0);
DecodePartial {
read: inpos,
written: outpos,
error: DecodeError {
position: inpos + partial.error.position,
kind: partial.error.kind,
},
}
})?;
debug_assert_eq!(written, outlen);
inpos += dec;
outpos += outlen;
outend -= enc - outlen;
}
debug_assert_eq!(inpos, input.len());
debug_assert_eq!(outpos, outend);
Ok(outend)
}
// Returns next input and output position. // Fails with Symbol if an input character does not translate to a symbol. The // error is the lowest index of such character. // Fails with Padding if some padding length is invalid. The error is the index // of the first padding character of the invalid padding. // Fails with Trailing if there are non-zero trailing bits. fn decode_wrap_block<B: Static<usize>, M: Static<bool>, P: Static<bool>>(
bit: B, msb: M, ctb: bool, values: &[u8; 256], pad: P, input: &[u8], output: &mut [u8],
) -> Result<(usize, usize), DecodeError> { let dec = dec(bit.val()); letmut buf = [0u8; 8]; letmut shift = [0usize; 8]; letmut bufpos = 0; letmut inpos = 0; while bufpos < dec {
inpos = skip_ignore(values, input, inpos); if inpos == input.len() { break;
}
shift[bufpos] = inpos;
buf[bufpos] = input[inpos];
bufpos += 1;
inpos += 1;
} let olen = decode_pad_len(bit, pad, bufpos).map_err(|mut e| {
e.position = shift[e.position];
e
})?; let written = decode_pad_mut(bit, msb, ctb, values, pad, &buf[.. bufpos], &mut output[.. olen])
.map_err(|partial| {
debug_assert_eq!(partial.read, 0);
debug_assert_eq!(partial.written, 0);
DecodeError { position: shift[partial.error.position], kind: partial.error.kind }
})?;
Ok((inpos, written))
}
// Fails with Symbol if an input character does not translate to a symbol. The // error is the lowest index of such character. // Fails with Padding if some padding length is invalid. The error is the index // of the first padding character of the invalid padding. // Fails with Trailing if there are non-zero trailing bits. // Fails with Length if input length (without ignored characters) is invalid. #[allow(clippy::too_many_arguments)] fn decode_wrap_mut<B: Static<usize>, M: Static<bool>, P: Static<bool>, I: Static<bool>>(
bit: B, msb: M, ctb: bool, values: &[u8; 256], pad: P, has_ignore: I, input: &[u8],
output: &mut [u8],
) -> Result<usize, DecodePartial> { if !has_ignore.val() { return decode_pad_mut(bit, msb, ctb, values, pad, input, output);
}
debug_assert_eq!(output.len(), decode_wrap_len(bit, pad, input.len()).1); letmut inpos = 0; letmut outpos = 0; while inpos < input.len() { let (inlen, outlen) = decode_wrap_len(bit, pad, input.len() - inpos); match decode_pad_mut(
bit,
msb,
ctb,
values,
pad,
&input[inpos .. inpos + inlen],
&mut output[outpos .. outpos + outlen],
) {
Ok(written) => {
inpos += inlen;
outpos += written; break;
}
Err(partial) => {
inpos += partial.read;
outpos += partial.written;
}
} let (ipos, opos) =
decode_wrap_block(bit, msb, ctb, values, pad, &input[inpos ..], &mut output[outpos ..])
.map_err(|mut error| {
error.position += inpos;
DecodePartial { read: inpos, written: outpos, error }
})?;
inpos += ipos;
outpos += opos;
} let inpos = skip_ignore(values, input, inpos); if inpos == input.len() {
Ok(outpos)
} else {
Err(DecodePartial {
read: inpos,
written: outpos,
error: DecodeError { position: inpos, kind: DecodeKind::Length },
})
}
}
/// Order in which bits are read from a byte /// /// The base-conversion encoding is always little-endian. This means that the least significant /// **byte** is always first. However, we can still choose whether, within a byte, this is the most /// significant or the least significant **bit** that is first. If the terminology is confusing, /// testing on an asymmetrical example should be enough to choose the correct value. /// /// # Examples /// /// In the following example, we can see that a base with the `MostSignificantFirst` bit-order has /// the most significant bit first in the encoded output. In particular, the output is in the same /// order as the bits in the byte. The opposite happens with the `LeastSignificantFirst` bit-order. /// The least significant bit is first and the output is in the reverse order. /// /// ```rust /// use data_encoding::{BitOrder, Specification}; /// let mut spec = Specification::new(); /// spec.symbols.push_str("01"); /// spec.bit_order = BitOrder::MostSignificantFirst; // default /// let msb = spec.encoding().unwrap(); /// spec.bit_order = BitOrder::LeastSignificantFirst; /// let lsb = spec.encoding().unwrap(); /// assert_eq!(msb.encode(&[0b01010011]), "01010011"); /// assert_eq!(lsb.encode(&[0b01010011]), "11001010"); /// ``` /// /// # Features /// /// Requires the `alloc` feature. #[derive(Debug, Copy, Clone, PartialEq, Eq)] #[cfg(feature = "alloc")] pubenum BitOrder { /// Most significant bit first /// /// This is the most common and most intuitive bit-order. In particular, this is the bit-order /// used by [RFC4648] and thus the usual hexadecimal, base64, base32, base64url, and base32hex /// encodings. This is the default bit-order when [specifying](struct.Specification.html) a /// base. /// /// [RFC4648]: https://tools.ietf.org/html/rfc4648
MostSignificantFirst,
/// Least significant bit first /// /// # Examples /// /// DNSCurve [base32] uses least significant bit first: /// /// ```rust /// use data_encoding::BASE32_DNSCURVE; /// assert_eq!(BASE32_DNSCURVE.encode(&[0x64, 0x88]), "4321"); /// assert_eq!(BASE32_DNSCURVE.decode(b"4321").unwrap(), vec![0x64, 0x88]); /// ``` /// /// [base32]: constant.BASE32_DNSCURVE.html
LeastSignificantFirst,
} #[cfg(feature = "alloc")] usecrate::BitOrder::*;
/// Base-conversion encoding /// /// See [Specification](struct.Specification.html) for technical details or how to define a new one. // Required fields: // 0 - 256 (256) symbols // 256 - 512 (256) values // 512 - 513 ( 1) padding // 513 - 514 ( 1) reserved(3),ctb(1),msb(1),bit(3) // Optional fields: // 514 - 515 ( 1) width // 515 - * ( N) separator // Invariants: // - symbols is 2^bit unique characters repeated 2^(8-bit) times // - values[128 ..] are INVALID // - values[0 .. 128] are either INVALID, IGNORE, PADDING, or < 2^bit // - padding is either < 128 or INVALID // - values[padding] is PADDING if padding < 128 // - values and symbols are inverse // - ctb is true if 8 % bit == 0 // - width is present if there is x such that values[x] is IGNORE // - width % dec(bit) == 0 // - for all x in separator values[x] is IGNORE #[derive(Debug, Clone, PartialEq, Eq)] pubstruct Encoding(pub InternalEncoding);
/// How to translate characters when decoding /// /// The order matters. The first character of the `from` field is translated to the first character /// of the `to` field. The second to the second. Etc. /// /// See [Specification](struct.Specification.html) for more information. /// /// # Features /// /// Requires the `alloc` feature. #[derive(Debug, Clone)] #[cfg(feature = "alloc")] pubstruct Translate { /// Characters to translate from pub from: String,
/// Characters to translate to pub to: String,
}
/// How to wrap the output when encoding /// /// See [Specification](struct.Specification.html) for more information. /// /// # Features /// /// Requires the `alloc` feature. #[derive(Debug, Clone)] #[cfg(feature = "alloc")] pubstruct Wrap { /// Wrapping width /// /// Must be a multiple of: /// /// - 8 for a bit-width of 1 (binary), 3 (octal), and 5 (base32) /// - 4 for a bit-width of 2 (base4) and 6 (base64) /// - 2 for a bit-width of 4 (hexadecimal) /// /// Wrapping is disabled if null. pub width: usize,
/// Wrapping characters /// /// Wrapping is disabled if empty. pub separator: String,
}
/// Base-conversion specification /// /// It is possible to define custom encodings given a specification. To do so, it is important to /// understand the theory first. /// /// # Theory /// /// Each subsection has an equivalent subsection in the [Practice](#practice) section. /// /// ## Basics /// /// The main idea of a [base-conversion] encoding is to see `[u8]` as numbers written in /// little-endian base256 and convert them in another little-endian base. For performance reasons, /// this crate restricts this other base to be of size 2 (binary), 4 (base4), 8 (octal), 16 /// (hexadecimal), 32 (base32), or 64 (base64). The converted number is written as `[u8]` although /// it doesn't use all the 256 possible values of `u8`. This crate encodes to ASCII, so only values /// smaller than 128 are allowed. /// /// More precisely, we need the following elements: /// /// - The bit-width N: 1 for binary, 2 for base4, 3 for octal, 4 for hexadecimal, 5 for base32, and /// 6 for base64 /// - The [bit-order](enum.BitOrder.html): most or least significant bit first /// - The symbols function S from [0, 2<sup>N</sup>) (called values and written `uN`) to symbols /// (represented as `u8` although only ASCII symbols are allowed, i.e. smaller than 128) /// - The values partial function V from ASCII to [0, 2<sup>N</sup>), i.e. from `u8` to `uN` /// - Whether trailing bits are checked: trailing bits are leading zeros in theory, but since /// numbers are little-endian they come last /// /// For the encoding to be correct (i.e. encoding then decoding gives back the initial input), /// V(S(i)) must be defined and equal to i for all i in [0, 2<sup>N</sup>). For the encoding to be /// [canonical][canonical] (i.e. different inputs decode to different outputs, or equivalently, /// decoding then encoding gives back the initial input), trailing bits must be checked and if V(i) /// is defined then S(V(i)) is equal to i for all i. /// /// Encoding and decoding are given by the following pipeline: /// /// ```text /// [u8] <--1--> [[bit; 8]] <--2--> [[bit; N]] <--3--> [uN] <--4--> [u8] /// 1: Map bit-order between each u8 and [bit; 8] /// 2: Base conversion between base 2^8 and base 2^N (check trailing bits) /// 3: Map bit-order between each [bit; N] and uN /// 4: Map symbols/values between each uN and u8 (values must be defined) /// ``` /// /// ## Extensions /// /// All these extensions make the encoding not canonical. /// /// ### Padding /// /// Padding is useful if the following conditions are met: /// /// - the bit-width is 3 (octal), 5 (base32), or 6 (base64) /// - the length of the data to encode is not known in advance /// - the data must be sent without buffering /// /// Bases for which the bit-width N does not divide 8 may not concatenate encoded data. This comes /// from the fact that it is not possible to make the difference between trailing bits and encoding /// bits. Padding solves this issue by adding a new character to discriminate between trailing bits /// and encoding bits. The idea is to work by blocks of lcm(8, N) bits, where lcm(8, N) is the least /// common multiple of 8 and N. When such block is not complete, it is padded. /// /// To preserve correctness, the padding character must not be a symbol. /// /// ### Ignore characters when decoding /// /// Ignoring characters when decoding is useful if after encoding some characters are added for /// convenience or any other reason (like wrapping). In that case we want to first ignore thoses /// characters before decoding. /// /// To preserve correctness, ignored characters must not contain symbols or the padding character. /// /// ### Wrap output when encoding /// /// Wrapping output when encoding is useful if the output is meant to be printed in a document where /// width is limited (typically 80-columns documents). In that case, the wrapping width and the /// wrapping separator have to be defined. /// /// To preserve correctness, the wrapping separator characters must be ignored (see previous /// subsection). As such, wrapping separator characters must also not contain symbols or the padding /// character. /// /// ### Translate characters when decoding /// /// Translating characters when decoding is useful when encoded data may be copied by a humain /// instead of a machine. Humans tend to confuse some characters for others. In that case we want to /// translate those characters before decoding. /// /// To preserve correctness, the characters we translate _from_ must not contain symbols or the /// padding character, and the characters we translate _to_ must only contain symbols or the padding /// character. /// /// # Practice /// /// ## Basics /// /// ```rust /// use data_encoding::{Encoding, Specification}; /// fn make_encoding(symbols: &str) -> Encoding { /// let mut spec = Specification::new(); /// spec.symbols.push_str(symbols); /// spec.encoding().unwrap() /// } /// let binary = make_encoding("01"); /// let octal = make_encoding("01234567"); /// let hexadecimal = make_encoding("0123456789abcdef"); /// assert_eq!(binary.encode(b"Bit"), "010000100110100101110100"); /// assert_eq!(octal.encode(b"Bit"), "20464564"); /// assert_eq!(hexadecimal.encode(b"Bit"), "426974"); /// ``` /// /// The `binary` base has 2 symbols `0` and `1` with value 0 and 1 respectively. The `octal` base /// has 8 symbols `0` to `7` with value 0 to 7. The `hexadecimal` base has 16 symbols `0` to `9` and /// `a` to `f` with value 0 to 15. The following diagram gives the idea of how encoding works in the /// previous example (note that we can actually write such diagram only because the bit-order is /// most significant first): /// /// ```text /// [ octal] | 2 : 0 : 4 : 6 : 4 : 5 : 6 : 4 | /// [ binary] |0 1 0 0 0 0 1 0|0 1 1 0 1 0 0 1|0 1 1 1 0 1 0 0| /// [hexadecimal] | 4 : 2 | 6 : 9 | 7 : 4 | /// ^-- LSB ^-- MSB /// ``` /// /// Note that in theory, these little-endian numbers are read from right to left (the most /// significant bit is at the right). Since leading zeros are meaningless (in our usual decimal /// notation 0123 is the same as 123), it explains why trailing bits must be zero. Trailing bits may /// occur when the bit-width of a base does not divide 8. Only binary, base4, and hexadecimal don't /// have trailing bits issues. So let's consider octal and base64, which have trailing bits in /// similar circumstances: /// /// ```rust /// use data_encoding::{Specification, BASE64_NOPAD}; /// let octal = { /// let mut spec = Specification::new(); /// spec.symbols.push_str("01234567"); /// spec.encoding().unwrap() /// }; /// assert_eq!(BASE64_NOPAD.encode(b"B"), "Qg"); /// assert_eq!(octal.encode(b"B"), "204"); /// ``` /// /// We have the following diagram, where the base64 values are written between parentheses: /// /// ```text /// [base64] | Q(16) : g(32) : [has 4 zero trailing bits] /// [ octal] | 2 : 0 : 4 : [has 1 zero trailing bit ] /// |0 1 0 0 0 0 1 0|0 0 0 0 /// [ ascii] | B | /// ^-^-^-^-- leading zeros / trailing bits /// ``` /// /// ## Extensions /// /// ### Padding /// /// For octal and base64, lcm(8, 3) == lcm(8, 6) == 24 bits or 3 bytes. For base32, lcm(8, 5) is 40 /// bits or 5 bytes. Let's consider octal and base64: /// /// ```rust /// use data_encoding::{Specification, BASE64}; /// let octal = { /// let mut spec = Specification::new(); /// spec.symbols.push_str("01234567"); /// spec.padding = Some('='); /// spec.encoding().unwrap() /// }; /// // We start encoding but we only have "B" for now. /// assert_eq!(BASE64.encode(b"B"), "Qg=="); /// assert_eq!(octal.encode(b"B"), "204====="); /// // Now we have "it". /// assert_eq!(BASE64.encode(b"it"), "aXQ="); /// assert_eq!(octal.encode(b"it"), "322720=="); /// // By concatenating everything, we may decode the original data. /// assert_eq!(BASE64.decode(b"Qg==aXQ=").unwrap(), b"Bit"); /// assert_eq!(octal.decode(b"204=====322720==").unwrap(), b"Bit"); /// ``` /// /// We have the following diagrams: /// /// ```text /// [base64] | Q(16) : g(32) : = : = | /// [ octal] | 2 : 0 : 4 : = : = : = : = : = | /// |0 1 0 0 0 0 1 0|. . . . . . . .|. . . . . . . .| /// [ ascii] | B | end of block aligned --^ /// ^-- beginning of block aligned /// /// [base64] | a(26) : X(23) : Q(16) : = | /// [ octal] | 3 : 2 : 2 : 7 : 2 : 0 : = : = | /// |0 1 1 0 1 0 0 1|0 1 1 1 0 1 0 0|. . . . . . . .| /// [ ascii] | i | t | /// ``` /// /// ### Ignore characters when decoding /// /// The typical use-case is to ignore newlines (`\r` and `\n`). But to keep the example small, we /// will ignore spaces. /// /// ```rust /// let mut spec = data_encoding::HEXLOWER.specification(); /// spec.ignore.push_str(" \t"); /// let base = spec.encoding().unwrap(); /// assert_eq!(base.decode(b"42 69 74"), base.decode(b"426974")); /// ``` /// /// ### Wrap output when encoding /// /// The typical use-case is to wrap after 64 or 76 characters with a newline (`\r\n` or `\n`). But /// to keep the example small, we will wrap after 8 characters with a space. /// /// ```rust /// let mut spec = data_encoding::BASE64.specification(); /// spec.wrap.width = 8; /// spec.wrap.separator.push_str(" "); /// let base64 = spec.encoding().unwrap(); /// assert_eq!(base64.encode(b"Hey you"), "SGV5IHlv dQ== "); /// ``` /// /// Note that the output always ends with the separator. /// /// ### Translate characters when decoding /// /// The typical use-case is to translate lowercase to uppercase or reciprocally, but it is also used /// for letters that look alike, like `O0` or `Il1`. Let's illustrate both examples. /// /// ```rust /// let mut spec = data_encoding::HEXLOWER.specification(); /// spec.translate.from.push_str("ABCDEFOIl"); /// spec.translate.to.push_str("abcdef011"); /// let base = spec.encoding().unwrap(); /// assert_eq!(base.decode(b"BOIl"), base.decode(b"b011")); /// ``` /// /// # Features /// /// Requires the `alloc` feature. /// /// [base-conversion]: https://en.wikipedia.org/wiki/Positional_notation#Base_conversion /// [canonical]: https://tools.ietf.org/html/rfc4648#section-3.5 #[derive(Debug, Clone)] #[cfg(feature = "alloc")] pubstruct Specification { /// Symbols /// /// The number of symbols must be 2, 4, 8, 16, 32, or 64. Symbols must be ASCII characters /// (smaller than 128) and they must be unique. pub symbols: String,
/// Bit-order /// /// The default is to use most significant bit first since it is the most common. pub bit_order: BitOrder,
/// Check trailing bits /// /// The default is to check trailing bits. This field is ignored when unnecessary (i.e. for /// base2, base4, and base16). pub check_trailing_bits: bool,
/// Padding /// /// The default is to not use padding. The padding character must be ASCII and must not be a /// symbol. pub padding: Option<char>,
/// Characters to ignore when decoding /// /// The default is to not ignore characters when decoding. The characters to ignore must be /// ASCII and must not be symbols or the padding character. pub ignore: String,
/// How to wrap the output when encoding /// /// The default is to not wrap the output when encoding. The wrapping characters must be ASCII /// and must not be symbols or the padding character. pub wrap: Wrap,
/// How to translate characters when decoding /// /// The default is to not translate characters when decoding. The characters to translate from /// must be ASCII and must not have already been assigned a semantics. The characters to /// translate to must be ASCII and must have been assigned a semantics (symbol, padding /// character, or ignored character). pub translate: Translate,
}
/// Returns the encoded length of an input of length `len` /// /// See [`encode_mut`] for when to use it. /// /// [`encode_mut`]: struct.Encoding.html#method.encode_mut pubfn encode_len(&self, len: usize) -> usize {
dispatch! { let bit: usize = self.bit(); let pad: Option<u8> = self.pad(); let wrap: Option<(usize, &[u8])> = self.wrap();
encode_wrap_len(bit, pad, wrap, len)
}
}
/// Encodes `input` in `output` /// /// # Panics /// /// Panics if the `output` length does not match the result of [`encode_len`] for the `input` /// length. /// /// # Examples /// /// ```rust /// use data_encoding::BASE64; /// # let mut buffer = vec![0; 100]; /// let input = b"Hello world"; /// let output = &mut buffer[0 .. BASE64.encode_len(input.len())]; /// BASE64.encode_mut(input, output); /// assert_eq!(output, b"SGVsbG8gd29ybGQ="); /// ``` /// /// [`encode_len`]: struct.Encoding.html#method.encode_len #[allow(clippy::cognitive_complexity)] pubfn encode_mut(&self, input: &[u8], output: &mut [u8]) {
assert_eq!(output.len(), self.encode_len(input.len()));
dispatch! { let bit: usize = self.bit(); let msb: bool = self.msb(); let pad: Option<u8> = self.pad(); let wrap: Option<(usize, &[u8])> = self.wrap();
encode_wrap_mut(bit, msb, self.sym(), pad, wrap, input, output)
}
}
/// Appends the encoding of `input` to `output` /// /// # Examples /// /// ```rust /// use data_encoding::BASE64; /// # let mut buffer = vec![0; 100]; /// let input = b"Hello world"; /// let mut output = "Result: ".to_string(); /// BASE64.encode_append(input, &mut output); /// assert_eq!(output, "Result: SGVsbG8gd29ybGQ="); /// ``` /// /// # Features /// /// Requires the `alloc` feature. #[cfg(feature = "alloc")] pubfn encode_append(&self, input: &[u8], output: &mut String) { let output = unsafe { output.as_mut_vec() }; let output_len = output.len();
output.resize(output_len + self.encode_len(input.len()), 0u8); self.encode_mut(input, &mut output[output_len ..]);
}
/// Returns the decoded length of an input of length `len` /// /// See [`decode_mut`] for when to use it. /// /// # Errors /// /// Returns an error if `len` is invalid. The error kind is [`Length`] and the [position] is the /// greatest valid input length. /// /// [`decode_mut`]: struct.Encoding.html#method.decode_mut /// [`Length`]: enum.DecodeKind.html#variant.Length /// [position]: struct.DecodeError.html#structfield.position pubfn decode_len(&self, len: usize) -> Result<usize, DecodeError> { let (ilen, olen) = dispatch! { let bit: usize = self.bit(); let pad: bool = self.pad().is_some();
decode_wrap_len(bit, pad, len)
};
check!(
DecodeError { position: ilen, kind: DecodeKind::Length }, self.has_ignore() || len == ilen
);
Ok(olen)
}
/// Decodes `input` in `output` /// /// Returns the length of the decoded output. This length may be smaller than the output length /// if the input contained padding or ignored characters. The output bytes after the returned /// length are not initialized and should not be read. /// /// # Panics /// /// Panics if the `output` length does not match the result of [`decode_len`] for the `input` /// length. Also panics if `decode_len` fails for the `input` length. /// /// # Errors /// /// Returns an error if `input` is invalid. See [`decode`] for more details. The are two /// differences though: /// /// - [`Length`] may be returned only if the encoding allows ignored characters, because /// otherwise this is already checked by [`decode_len`]. /// - The [`read`] first bytes of the input have been successfully decoded to the [`written`] /// first bytes of the output. /// /// # Examples /// /// ```rust /// use data_encoding::BASE64; /// # let mut buffer = vec![0; 100]; /// let input = b"SGVsbA==byB3b3JsZA=="; /// let output = &mut buffer[0 .. BASE64.decode_len(input.len()).unwrap()]; /// let len = BASE64.decode_mut(input, output).unwrap(); /// assert_eq!(&output[0 .. len], b"Hello world"); /// ``` /// /// [`decode_len`]: struct.Encoding.html#method.decode_len /// [`decode`]: struct.Encoding.html#method.decode /// [`Length`]: enum.DecodeKind.html#variant.Length /// [`read`]: struct.DecodePartial.html#structfield.read /// [`written`]: struct.DecodePartial.html#structfield.written #[allow(clippy::cognitive_complexity)] pubfn decode_mut(&self, input: &[u8], output: &mut [u8]) -> Result<usize, DecodePartial> {
assert_eq!(Ok(output.len()), self.decode_len(input.len()));
dispatch! { let bit: usize = self.bit(); let msb: bool = self.msb(); let pad: bool = self.pad().is_some(); let has_ignore: bool = self.has_ignore();
decode_wrap_mut(bit, msb, self.ctb(), self.val(), pad, has_ignore,
input, output)
}
}
/// Returns decoded `input` /// /// # Errors /// /// Returns an error if `input` is invalid. The error kind can be: /// /// - [`Length`] if the input length is invalid. The [position] is the greatest valid input /// length. /// - [`Symbol`] if the input contains an invalid character. The [position] is the first invalid /// character. /// - [`Trailing`] if the input has non-zero trailing bits. This is only possible if the /// encoding checks trailing bits. The [position] is the first character containing non-zero /// trailing bits. /// - [`Padding`] if the input has an invalid padding length. This is only possible if the /// encoding uses padding. The [position] is the first padding character of the first padding /// of invalid length. /// /// # Examples /// /// ```rust /// use data_encoding::BASE64; /// assert_eq!(BASE64.decode(b"SGVsbA==byB3b3JsZA==").unwrap(), b"Hello world"); /// ``` /// /// # Features /// /// Requires the `alloc` feature. /// /// [`Length`]: enum.DecodeKind.html#variant.Length /// [`Symbol`]: enum.DecodeKind.html#variant.Symbol /// [`Trailing`]: enum.DecodeKind.html#variant.Trailing /// [`Padding`]: enum.DecodeKind.html#variant.Padding /// [position]: struct.DecodeError.html#structfield.position #[cfg(feature = "alloc")] pubfn decode(&self, input: &[u8]) -> Result<Vec<u8>, DecodeError> { letmut output = vec![0u8; self.decode_len(input.len())?]; let len = self.decode_mut(input, &mut output).map_err(|partial| partial.error)?;
output.truncate(len);
Ok(output)
}
/// Returns whether the encoding is canonical /// /// An encoding is not canonical if one of the following conditions holds: /// /// - trailing bits are not checked /// - padding is used /// - characters are ignored /// - characters are translated pubfn is_canonical(&self) -> bool { if !self.ctb() { returnfalse;
} let bit = self.bit(); let sym = self.sym(); let val = self.val(); for i in0 .. 256 { if val[i] == INVALID { continue;
} if val[i] >= 1 << bit { returnfalse;
} if sym[val[i] as usize] != i as u8 { returnfalse;
}
} true
}
/// Returns the encoding specification /// /// # Features /// /// Requires the `alloc` feature. #[cfg(feature = "alloc")] pubfn specification(&self) -> Specification { letmut specification = Specification::new();
specification
.symbols
.push_str(core::str::from_utf8(&self.sym()[0 .. 1 << self.bit()]).unwrap());
specification.bit_order = ifself.msb() { MostSignificantFirst } else { LeastSignificantFirst };
specification.check_trailing_bits = self.ctb(); iflet Some(pad) = self.pad() {
specification.padding = Some(pad as char);
} for i in0 .. 128u8 { ifself.val()[i as usize] != IGNORE { continue;
}
specification.ignore.push(i as char);
} iflet Some((col, end)) = self.wrap() {
specification.wrap.width = col;
specification.wrap.separator = core::str::from_utf8(end).unwrap().to_owned();
} for i in0 .. 128u8 { let canonical = ifself.val()[i as usize] < 1 << self.bit() { self.sym()[self.val()[i as usize] as usize]
} elseifself.val()[i as usize] == PADDING { self.pad().unwrap()
} else { continue;
}; if i == canonical { continue;
}
specification.translate.from.push(i as char);
specification.translate.to.push(canonical as char);
}
specification
}
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.