confium_test_harness/
result.rs

1//! Outcome of running one test vector.
2//!
3//! A [`TestResult`] records whether the protocol completed, what bytes
4//! it produced, how many rounds and bytes-on-the-wire it took, and how
5//! long it ran for. The reporter ([`crate::report`]) turns a batch of
6//! these into the JSON NIST consumes.
7
8use std::time::Duration;
9
10use crate::vector::TestVector;
11
12/// Whether the run succeeded, aborted cleanly (e.g. detected Byzantine
13/// misbehavior), or errored.
14#[derive(Debug, Clone, Copy, PartialEq, Eq)]
15pub enum Outcome {
16    /// Protocol completed and produced output matching (if specified)
17    /// the vector's expected bytes.
18    Pass,
19    /// Protocol completed but the output didn't match the expected
20    /// bytes (only possible when `expected_signature_hex` was set).
21    Fail,
22    /// Protocol completed but a non-conformance was observed that the
23    /// vector's conformance level downgrades from an error to a warning
24    /// (e.g. a `should_pass` vector whose output mismatched). Recorded
25    /// in the report; does not gate the candidate.
26    Warn,
27    /// Protocol aborted cleanly — the scheme detected the configured
28    /// Byzantine behavior and signaled misbehavior. Counts as a pass
29    /// for Byzantine-detection vectors.
30    Aborted,
31}
32
33impl Outcome {
34    pub fn as_str(self) -> &'static str {
35        match self {
36            Outcome::Pass => "pass",
37            Outcome::Fail => "fail",
38            Outcome::Warn => "warn",
39            Outcome::Aborted => "aborted",
40        }
41    }
42}
43
44/// Result of executing one [`TestVector`] against one scheme.
45#[derive(Debug, Clone)]
46pub struct TestResult {
47    pub scheme_name: String,
48    pub scheme_version: String,
49    pub parties: u32,
50    pub threshold: u32,
51    pub outcome: Outcome,
52    /// Bytes the protocol produced (signature, DKG output, …). Empty
53    /// for aborted runs.
54    pub output: Vec<u8>,
55    /// Total messages exchanged across all rounds, all parties.
56    pub messages_exchanged: u64,
57    /// Total bytes carried by those messages.
58    pub bytes_exchanged: u64,
59    /// Number of rounds the protocol ran before completing or aborting.
60    pub rounds: u8,
61    /// Wall time of the run, captured by the runner.
62    pub elapsed: Duration,
63    /// Human-readable note — e.g. mismatch detail, abort reason.
64    pub note: Option<String>,
65}
66
67impl TestResult {
68    /// Build a result from a vector + the runtime observations. The
69    /// caller supplies the produced output and the observed round
70    /// count; this constructor decides `Pass` / `Fail` / `Warn` by
71    /// comparing against the vector's expected bytes and conformance
72    /// level:
73    ///
74    /// - Output matches (or no expected bytes) and round count matches
75    ///   (or no `expected_round_count` set) → `Pass`.
76    /// - Output mismatches on a `must_pass` vector → `Fail`.
77    /// - Output mismatches on a `should_pass` vector → `Warn`.
78    /// - Output mismatches on an `informational` vector → `Pass` (the
79    ///   mismatch is recorded in `note` but never gates the candidate).
80    /// - Round count differs from `expected_round_count` → `Warn` even
81    ///   on a `must_pass` vector (the implementation produced a valid
82    ///   signature; it just took a different number of rounds). The
83    ///   mismatch is appended to the note.
84    pub fn from_run(
85        vector: &TestVector,
86        output: Vec<u8>,
87        messages_exchanged: u64,
88        bytes_exchanged: u64,
89        rounds: u8,
90        elapsed: Duration,
91    ) -> Self {
92        let output_matches = match vector.test.expected_bytes() {
93            Some(expected) => expected == output,
94            None => true,
95        };
96        let mismatch_note = if output_matches {
97            None
98        } else {
99            Some(format!(
100                "output mismatch: expected {} bytes, got {} bytes",
101                vector.test.expected_bytes().map(|e| e.len()).unwrap_or(0),
102                output.len()
103            ))
104        };
105
106        let round_note = match vector.expected_round_count {
107            Some(expected) if expected != rounds => Some(format!(
108                "round count differs: expected {}, observed {}",
109                expected, rounds
110            )),
111            _ => None,
112        };
113
114        use crate::vector::ConformanceLevel;
115        let outcome = if output_matches {
116            // Output is correct. A round-count divergence on an
117            // otherwise-passing vector is still only a warning — the
118            // signature was produced, the implementation just took a
119            // different number of rounds than the vector expected.
120            match round_note {
121                Some(_) => Outcome::Warn,
122                None => Outcome::Pass,
123            }
124        } else {
125            match vector.conformance_level {
126                ConformanceLevel::MustPass => Outcome::Fail,
127                ConformanceLevel::ShouldPass => Outcome::Warn,
128                // Informational failures never gate the candidate.
129                ConformanceLevel::Informational => Outcome::Pass,
130            }
131        };
132
133        let note = match (mismatch_note, round_note) {
134            (Some(a), Some(b)) => Some(format!("{}; {}", a, b)),
135            (Some(a), None) => Some(a),
136            (None, Some(b)) => Some(b),
137            (None, None) => None,
138        };
139
140        TestResult {
141            scheme_name: vector.scheme.name.clone(),
142            scheme_version: vector.scheme.version.clone(),
143            parties: vector.test.parties,
144            threshold: vector.test.threshold,
145            outcome,
146            output,
147            messages_exchanged,
148            bytes_exchanged,
149            rounds,
150            elapsed,
151            note,
152        }
153    }
154
155    /// Construct an aborted result (no output, scheme signaled
156    /// misbehavior). Used by the runner when the configured Byzantine
157    /// behavior tripped the scheme's abort path.
158    pub fn aborted(
159        vector: &TestVector,
160        reason: impl Into<String>,
161        rounds: u8,
162        elapsed: Duration,
163    ) -> Self {
164        TestResult {
165            scheme_name: vector.scheme.name.clone(),
166            scheme_version: vector.scheme.version.clone(),
167            parties: vector.test.parties,
168            threshold: vector.test.threshold,
169            outcome: Outcome::Aborted,
170            output: Vec::new(),
171            messages_exchanged: 0,
172            bytes_exchanged: 0,
173            rounds,
174            elapsed,
175            note: Some(reason.into()),
176        }
177    }
178}
179
180#[cfg(test)]
181mod tests {
182    use super::*;
183    use crate::vector::ConformanceLevel;
184    use crate::vector::SchemeSpec;
185
186    fn vector(expected: Option<&str>) -> TestVector {
187        TestVector {
188            scheme: SchemeSpec {
189                name: "test".into(),
190                version: "1".into(),
191            },
192            test: crate::vector::TestVectorTest {
193                parties: 3,
194                threshold: 2,
195                message: String::new(),
196                seed: String::new(),
197                expected_signature_hex: expected.unwrap_or("").to_string(),
198            },
199            peer_behavior: Vec::new(),
200            conformance_level: Default::default(),
201            reference: None,
202            expected_round_count: None,
203            share_material: None,
204        }
205    }
206
207    fn vector_with_level(level: ConformanceLevel, expected: Option<&str>) -> TestVector {
208        let mut v = vector(expected);
209        v.conformance_level = level;
210        v
211    }
212
213    #[test]
214    fn pass_when_no_expected_bytes() {
215        let v = vector(None);
216        let r = TestResult::from_run(&v, vec![1, 2, 3], 4, 12, 2, Duration::from_micros(50));
217        assert_eq!(r.outcome, Outcome::Pass);
218        assert!(r.note.is_none());
219    }
220
221    #[test]
222    fn pass_when_output_matches_expected() {
223        let v = vector(Some("0x010203"));
224        let r = TestResult::from_run(&v, vec![1, 2, 3], 4, 12, 2, Duration::from_micros(50));
225        assert_eq!(r.outcome, Outcome::Pass);
226    }
227
228    #[test]
229    fn fail_when_output_mismatches_expected() {
230        let v = vector(Some("0x010203"));
231        let r = TestResult::from_run(&v, vec![9, 9, 9], 4, 12, 2, Duration::from_micros(50));
232        assert_eq!(r.outcome, Outcome::Fail);
233        assert!(r.note.as_ref().unwrap().contains("mismatch"));
234    }
235
236    #[test]
237    fn should_pass_mismatch_is_a_warning_not_a_failure() {
238        let v = vector_with_level(ConformanceLevel::ShouldPass, Some("0x010203"));
239        let r = TestResult::from_run(&v, vec![9, 9, 9], 4, 12, 2, Duration::from_micros(50));
240        assert_eq!(
241            r.outcome,
242            Outcome::Warn,
243            "should_pass mismatch must downgrade to a warning"
244        );
245        assert!(r.note.as_ref().unwrap().contains("mismatch"));
246    }
247
248    #[test]
249    fn informational_mismatch_is_a_pass() {
250        let v = vector_with_level(ConformanceLevel::Informational, Some("0x010203"));
251        let r = TestResult::from_run(&v, vec![9, 9, 9], 4, 12, 2, Duration::from_micros(50));
252        assert_eq!(
253            r.outcome,
254            Outcome::Pass,
255            "informational mismatch must never gate the candidate"
256        );
257        // The mismatch is still recorded in the note for the report.
258        assert!(r.note.as_ref().unwrap().contains("mismatch"));
259    }
260
261    #[test]
262    fn round_count_mismatch_on_passing_vector_is_a_warning() {
263        let mut v = vector(None);
264        v.expected_round_count = Some(3);
265        let r = TestResult::from_run(&v, vec![1, 2, 3], 4, 12, 5, Duration::from_micros(50));
266        assert_eq!(r.outcome, Outcome::Warn);
267        assert!(r.note.as_ref().unwrap().contains("round count"));
268    }
269
270    #[test]
271    fn outcome_warn_serializes_as_warn_string() {
272        assert_eq!(Outcome::Warn.as_str(), "warn");
273    }
274
275    #[test]
276    fn aborted_records_reason() {
277        let v = vector(None);
278        let r = TestResult::aborted(&v, "byzantine-drop detected", 1, Duration::from_micros(10));
279        assert_eq!(r.outcome, Outcome::Aborted);
280        assert_eq!(r.output.len(), 0);
281        assert_eq!(r.note.as_deref(), Some("byzantine-drop detected"));
282    }
283}
confium_test_harness/result.rs

confium_test_harness/
result.rs
