arcana/rsa/
oaep.rs

1// SPDX-License-Identifier: Apache-2.0
2// Copyright 2026 Cédric Mesnil <cslashm@pm.me>
3
4// Index-based loops in this module deliberately mirror the reference
5// algorithm (parallel-array access `a[i]`/`b[i]`, or a position-indexed
6// packing/reduction state) and keep the constant-time data layout explicit;
7// the `iter().enumerate()` rewrite would obscure it. Documented, single-lint
8// allow per CLAUDE.md §6 (scoped to this module, not a blanket suppression).
9#![allow(clippy::needless_range_loop)]
10
11//! RSA-OAEP encryption padding (RFC 8017 / PKCS#1 v2.2 §7.1).
12//!
13//! OAEP is the modern RSA encryption padding and supersedes
14//! PKCS#1 v1.5 in every protocol designed after the
15//! Bleichenbacher era.
16//!
17//! # Hash agility
18//!
19//! Per RFC 8017 §7.1 the scheme is parameterised by two hash
20//! functions: the **label hash** `Hash` (used for `lHash = Hash(L)`
21//! and to size the seed) and the **mask generation function** hash
22//! (MGF1's underlying `Hash`). They may differ. The generic
23//! [`oaep_encrypt_with`] / [`oaep_decrypt_with`] take both as type
24//! parameters `<H, MGF>`; the [`oaep_encrypt`] / [`oaep_decrypt`]
25//! convenience wrappers fix both to SHA-256 (the historical default
26//! and the byte-for-byte-stable public symbols that pre-existed the
27//! hash-agility change).
28//!
29//! # Side-channel posture
30//!
31//! OAEP is structurally **harder** to break with padding-oracle
32//! attacks than PKCS#1 v1.5, because the label-hash check at the
33//! top of decryption is naturally constant-time when implemented
34//! carefully. Items on the audit list (`T2-J` is the workspace
35//! roadmap entry covering both PKCS#1 v1.5 and OAEP):
36//!
37//! - The `H(L)` comparison must use `silentops::ct_eq`, not `==`.
38//! - The `0x01` separator byte search must not branch on its
39//!   position (CT scan + branchless flag accumulation).
40//! - All decrypt errors must produce the same byte length and
41//!   the same elapsed time.
42//!
43//! This module relies on [`super::rsa::rsa_decrypt_raw`], which
44//! is itself **not yet protected against Bellcore** (roadmap
45//! item `T1-C`). A CRT-faulted decrypt produces a malformed
46//! plaintext that OAEP rejects, but the rejection itself can
47//! leak `gcd(N, S - S')` to the attacker. See
48//! `arcana/doc/sca/countermeasures/rsa.rst`.
49
50use super::bigint::BigInt;
51use super::rsa::{RsaPublicKey, RsaSecretKey, representative_in_range, rsa_decrypt_raw, rsa_encrypt_raw};
52use crate::Hasher;
53use crate::hash::sha256::Sha256;
54
55/// MGF1 mask generation function using hash `MGF` (RFC 8017 Appendix B.2.1).
56fn mgf1<MGF: Hasher>(seed: &[u8], len: usize) -> Vec<u8> {
57    let mut output = Vec::with_capacity(len);
58    let mut counter: u32 = 0;
59    while output.len() < len {
60        let mut h = MGF::new();
61        h.update(seed);
62        h.update(&counter.to_be_bytes());
63        let block = h.finalize();
64        let take = (len - output.len()).min(block.len());
65        output.extend_from_slice(&block[..take]);
66        counter += 1;
67    }
68    output.truncate(len);
69    output
70}
71
72/// XOR two byte slices in place: a ^= b.
73fn xor_in_place(a: &mut [u8], b: &[u8]) {
74    for (x, y) in a.iter_mut().zip(b.iter()) {
75        *x ^= y;
76    }
77}
78
79/// OAEP encrypt a message with RSA, generic over the label hash `H`
80/// and the MGF1 hash `MGF` (RFC 8017 §7.1.1).
81///
82/// `label` can be empty (common case). `rng` fills buffers with random
83/// bytes. The seed length equals `H::OUTPUT_LEN` (the label-hash size),
84/// per RFC 8017 §7.1.1 step 2(d).
85///
86/// Panics if the message is longer than `k - 2*hLen - 2` bytes, where
87/// `k` is the modulus byte length and `hLen = H::OUTPUT_LEN`.
88pub fn oaep_encrypt_with<H: Hasher, MGF: Hasher>(
89    pk: &RsaPublicKey,
90    msg: &[u8],
91    label: &[u8],
92    rng: &mut dyn FnMut(&mut [u8]),
93) -> Vec<u8> {
94    let h_len = H::OUTPUT_LEN;
95    let k = pk.modulus_byte_len();
96    let max_msg_len = k - 2 * h_len - 2;
97    assert!(
98        msg.len() <= max_msg_len,
99        "OAEP: message too long (max {} bytes, got {})",
100        max_msg_len,
101        msg.len()
102    );
103
104    let l_hash = H::hash(label);
105
106    // DB = lHash || PS || 0x01 || M
107    let db_len = k - h_len - 1;
108    let mut db = vec![0u8; db_len];
109    db[..h_len].copy_from_slice(&l_hash);
110    // PS is zeros (already zero)
111    let ps_len = db_len - h_len - 1 - msg.len();
112    db[h_len + ps_len] = 0x01;
113    db[h_len + ps_len + 1..].copy_from_slice(msg);
114
115    // Generate random seed of length hLen.
116    let mut seed = vec![0u8; h_len];
117    rng(&mut seed);
118
119    // dbMask = MGF1(seed, db_len)
120    let db_mask = mgf1::<MGF>(&seed, db_len);
121    xor_in_place(&mut db, &db_mask);
122
123    // seedMask = MGF1(maskedDB, hLen)
124    let seed_mask = mgf1::<MGF>(&db, h_len);
125    xor_in_place(&mut seed, &seed_mask);
126
127    // EM = 0x00 || maskedSeed || maskedDB
128    let mut em = vec![0u8; k];
129    em[0] = 0x00;
130    em[1..1 + h_len].copy_from_slice(&seed);
131    em[1 + h_len..].copy_from_slice(&db);
132
133    let m = BigInt::from_be_bytes(&em);
134    let c = rsa_encrypt_raw(pk, &m);
135    c.to_be_bytes(k)
136}
137
138/// OAEP decrypt a ciphertext, generic over the label hash `H` and the
139/// MGF1 hash `MGF` (RFC 8017 §7.1.2).
140///
141/// Returns `None` if decryption or padding verification fails.
142pub fn oaep_decrypt_with<H: Hasher, MGF: Hasher>(sk: &RsaSecretKey, ct: &[u8], label: &[u8]) -> Option<Vec<u8>> {
143    let h_len = H::OUTPUT_LEN;
144    let k = sk.modulus_byte_len();
145    if ct.len() != k || k < 2 * h_len + 2 {
146        return None;
147    }
148
149    let c = BigInt::from_be_bytes(ct);
150    // RSADP step 1 (RFC 8017 §5.1.2): reject a ciphertext representative
151    // c not in [0, n-1] → "decryption error". c and n are public; the
152    // branch is on public data and precedes the private-key operation, so
153    // the OAEP error path below (CT lHash/separator handling) is unaffected.
154    if !representative_in_range(&c, &sk.n) {
155        return None;
156    }
157    let m = rsa_decrypt_raw(sk, &c);
158    let em = m.to_be_bytes(k);
159
160    // EM = Y || maskedSeed || maskedDB
161    if em[0] != 0x00 {
162        return None;
163    }
164
165    let masked_seed = &em[1..1 + h_len];
166    let masked_db = &em[1 + h_len..];
167
168    // Recover seed.
169    let seed_mask = mgf1::<MGF>(masked_db, h_len);
170    let mut seed = vec![0u8; h_len];
171    seed.copy_from_slice(masked_seed);
172    xor_in_place(&mut seed, &seed_mask);
173
174    // Recover DB.
175    let db_len = k - h_len - 1;
176    let db_mask = mgf1::<MGF>(&seed, db_len);
177    let mut db = vec![0u8; db_len];
178    db.copy_from_slice(masked_db);
179    xor_in_place(&mut db, &db_mask);
180
181    // Verify lHash.
182    let l_hash = H::hash(label);
183    let mut valid = true;
184    for i in 0..h_len {
185        if db[i] != l_hash[i] {
186            valid = false;
187        }
188    }
189
190    // Find the 0x01 separator.
191    let mut sep = None;
192    for i in h_len..db.len() {
193        if db[i] == 0x01 {
194            sep = Some(i);
195            break;
196        } else if db[i] != 0x00 {
197            valid = false;
198            break;
199        }
200    }
201
202    if !valid {
203        return None;
204    }
205    let sep = sep?;
206
207    Some(db[sep + 1..].to_vec())
208}
209
210/// OAEP encrypt a message with RSA using SHA-256 for both the label hash
211/// and MGF1 (convenience wrapper over [`oaep_encrypt_with`]).
212///
213/// `label` can be empty (common case). `rng` fills buffers with random bytes.
214pub fn oaep_encrypt(pk: &RsaPublicKey, msg: &[u8], label: &[u8], rng: &mut dyn FnMut(&mut [u8])) -> Vec<u8> {
215    oaep_encrypt_with::<Sha256, Sha256>(pk, msg, label, rng)
216}
217
218/// OAEP decrypt a ciphertext using SHA-256 for both the label hash and
219/// MGF1 (convenience wrapper over [`oaep_decrypt_with`]).
220///
221/// Returns `None` if decryption or padding verification fails.
222pub fn oaep_decrypt(sk: &RsaSecretKey, ct: &[u8], label: &[u8]) -> Option<Vec<u8>> {
223    oaep_decrypt_with::<Sha256, Sha256>(sk, ct, label)
224}
225
226#[cfg(test)]
227mod tests {
228    use super::*;
229    use crate::hash::sha1::Sha1;
230    use crate::hash::sha384::Sha384;
231    use crate::hash::sha512::Sha512;
232
233    fn test_rng() -> impl FnMut(&mut [u8]) {
234        let mut state: u64 = 0xdeadbeefcafebabe;
235        move |buf: &mut [u8]| {
236            for b in buf.iter_mut() {
237                state = state
238                    .wrapping_mul(6364136223846793005)
239                    .wrapping_add(1442695040888963407);
240                *b = (state >> 33) as u8;
241            }
242        }
243    }
244
245    #[test]
246    fn test_mgf1() {
247        // Basic sanity: MGF1 produces deterministic output of correct length.
248        let mask1 = mgf1::<Sha256>(b"seed", 64);
249        let mask2 = mgf1::<Sha256>(b"seed", 64);
250        assert_eq!(mask1.len(), 64);
251        assert_eq!(mask1, mask2);
252        // Different seed => different mask.
253        let mask3 = mgf1::<Sha256>(b"other", 64);
254        assert_ne!(mask1, mask3);
255    }
256
257    #[test]
258    fn test_oaep_encrypt_decrypt_roundtrip() {
259        let mut rng = test_rng();
260        let (pk, sk) = super::super::rsa::rsa_keygen(1024, &mut rng);
261        let msg = b"Hello, OAEP!";
262        let ct = oaep_encrypt(&pk, msg, b"", &mut rng);
263        let pt = oaep_decrypt(&sk, &ct, b"").expect("OAEP decryption failed");
264        assert_eq!(&pt, msg);
265    }
266
267    #[test]
268    fn test_oaep_wrong_label() {
269        let mut rng = test_rng();
270        let (pk, sk) = super::super::rsa::rsa_keygen(1024, &mut rng);
271        let msg = b"test";
272        let ct = oaep_encrypt(&pk, msg, b"label_a", &mut rng);
273        let result = oaep_decrypt(&sk, &ct, b"label_b");
274        assert!(result.is_none(), "Decryption should fail with wrong label");
275    }
276
277    /// Hash-agile round-trips: label hash and MGF1 hash both varied,
278    /// including a mixed (label != MGF) configuration (RFC 8017 §7.1).
279    #[test]
280    fn test_oaep_hash_agility_roundtrip() {
281        let mut rng = test_rng();
282        // 2048-bit key so SHA-512/SHA-512 (2*64+2 = 130 <= 256) fits.
283        let (pk, sk) = super::super::rsa::rsa_keygen(2048, &mut rng);
284        let msg = b"hash-agile OAEP";
285        let label = b"ctx";
286
287        macro_rules! roundtrip {
288            ($h:ty, $mgf:ty) => {{
289                let ct = oaep_encrypt_with::<$h, $mgf>(&pk, msg, label, &mut rng);
290                let pt = oaep_decrypt_with::<$h, $mgf>(&sk, &ct, label).expect("decrypt failed");
291                assert_eq!(&pt, msg);
292            }};
293        }
294
295        roundtrip!(Sha1, Sha1);
296        roundtrip!(Sha256, Sha256);
297        roundtrip!(Sha384, Sha384);
298        roundtrip!(Sha512, Sha512);
299        // Mixed: label hash SHA-256, MGF1 hash SHA-1.
300        roundtrip!(Sha256, Sha1);
301    }
302}
arcana/rsa/oaep.rs

arcana/rsa/
oaep.rs
