expectation_story/
report.rs

1//! Results of running plans, and what they say about each expectation.
2
3use crate::plan::{Because, Check, Plan};
4use expectation_model::{ExpId, Suite, Value};
5use std::fmt::Write;
6
7#[derive(Clone, Copy, Debug, PartialEq, Eq)]
8pub enum Status {
9    Passed,
10    Failed,
11    /// Not run, because an earlier step failed.
12    Skipped,
13    /// Not run, because the driver can't do something it needs yet.
14    Unsupported,
15    /// No result: it hasn't been run.
16    NotRun,
17}
18
19impl Status {
20    fn name(self) -> &'static str {
21        match self {
22            Status::Passed => "passed",
23            Status::Failed => "failed",
24            Status::Skipped => "skipped",
25            Status::Unsupported => "unsupported",
26            Status::NotRun => "not-run",
27        }
28    }
29
30    fn parse(s: &str) -> Option<Status> {
31        [Status::Passed, Status::Failed, Status::Skipped, Status::Unsupported, Status::NotRun]
32            .into_iter()
33            .find(|x| x.name() == s)
34    }
35
36    /// Did it run (pass or fail), as opposed to not running at all?
37    pub fn ran(self) -> bool {
38        matches!(self, Status::Passed | Status::Failed)
39    }
40}
41
42#[derive(Clone, Debug)]
43pub struct CheckResult {
44    pub observed: Option<Value>,
45    pub passed: bool,
46    pub error: Option<String>,
47}
48
49#[derive(Clone, Debug)]
50pub struct StepResult {
51    pub status: Status,
52    /// Same order as the plan step's checks; empty if skipped.
53    pub checks: Vec<CheckResult>,
54    /// Relative to the docs directory.
55    pub screenshot: Option<String>,
56    pub error: Option<String>,
57    /// How many times the checks were observed before they passed or it gave up.
58    pub polls: u32,
59}
60
61impl StepResult {
62    pub fn skipped() -> Self {
63        Self::with(Status::Skipped)
64    }
65
66    pub fn with(status: Status) -> Self {
67        StepResult { status, checks: vec![], screenshot: None, error: None, polls: 0 }
68    }
69}
70
71#[derive(Clone, Debug)]
72pub struct PlanResult {
73    pub status: Status,
74    pub start: StepResult,
75    pub steps: Vec<StepResult>,
76}
77
78impl PlanResult {
79    /// The same status throughout, with nothing observed.
80    pub fn with(status: Status, plan: &Plan) -> Self {
81        PlanResult {
82            status,
83            start: StepResult::with(status),
84            steps: plan.steps.iter().map(|_| StepResult::with(status)).collect(),
85        }
86    }
87}
88
89/// One result per plan, in plan order.
90#[derive(Clone, Debug, Default)]
91pub struct Report {
92    pub results: Vec<PlanResult>,
93}
94
95#[derive(Clone, Copy, Debug, PartialEq, Eq)]
96pub enum Verdict {
97    /// A test checked its promises and they held.
98    Verified,
99    /// A test checked its promises and one didn't hold.
100    Failed,
101    /// No test that ran checked it.
102    Untested,
103    /// The tests that would check it can't run yet: the driver doesn't support them.
104    Unsupported,
105    /// The tests that would check it haven't run.
106    NotRun,
107}
108
109impl Report {
110    /// Nothing failed. Tests that aren't supported yet, or weren't run, don't count against it.
111    pub fn passed(&self) -> bool {
112        !self.results.iter().any(|r| r.status == Status::Failed)
113    }
114
115    pub fn count(&self, status: Status) -> usize {
116        self.results.iter().filter(|r| r.status == status).count()
117    }
118
119    /// For plans that haven't run.
120    pub fn not_run(plans: &[Plan]) -> Report {
121        Report { results: plans.iter().map(|p| PlanResult::with(Status::NotRun, p)).collect() }
122    }
123
124    /// Plan by plan, the more informative of the two: a result that ran over one that's
125    /// unsupported, and either over none. Merges the runs of different drivers.
126    pub fn merge(mut self, other: Report) -> Report {
127        let rank = |s: Status| match s {
128            Status::NotRun => 0,
129            Status::Unsupported => 1,
130            _ => 2,
131        };
132        for (mine, theirs) in self.results.iter_mut().zip(other.results) {
133            if rank(theirs.status) > rank(mine.status) {
134                *mine = theirs;
135            }
136        }
137        self
138    }
139
140    /// Each expectation's verdict, from the checks it contributed.
141    pub fn verdicts(&self, suite: &Suite, plans: &[Plan]) -> Vec<Verdict> {
142        let mut out = vec![Verdict::Untested; suite.expectations.len()];
143        let mut mark = |checks: &[Check], results: &[CheckResult]| {
144            for (c, r) in checks.iter().zip(results) {
145                let Because::Expectation(e) = c.because else { continue };
146                let v = &mut out[e.0 as usize];
147                if !r.passed {
148                    *v = Verdict::Failed;
149                } else if *v == Verdict::Untested {
150                    *v = Verdict::Verified;
151                }
152            }
153        };
154        for (plan, result) in plans.iter().zip(&self.results) {
155            mark(&plan.start_checks, &result.start.checks);
156            for (step, r) in plan.steps.iter().zip(&result.steps) {
157                mark(&step.checks, &r.checks);
158            }
159        }
160        // Untested only because the tests that would check it can't run yet, or haven't.
161        for (status, verdict) in [(Status::Unsupported, Verdict::Unsupported), (Status::NotRun, Verdict::NotRun)] {
162            for (plan, result) in plans.iter().zip(&self.results) {
163                if result.status != status {
164                    continue;
165                }
166                for e in plan.start_invariants.iter().copied().chain(plan.steps.iter().flat_map(|s| s.exercises())) {
167                    if out[e.0 as usize] == Verdict::Untested {
168                        out[e.0 as usize] = verdict;
169                    }
170                }
171            }
172        }
173        out
174    }
175
176    /// The results as text, one line per plan, step and check, keyed by each plan's slug so
177    /// they load back onto a suite that has changed since.
178    pub fn save(&self, plans: &[Plan]) -> String {
179        let field = |s: &str| s.replace(['\t', '\n'], " ");
180        let opt = |s: &Option<String>| s.as_deref().map(field).unwrap_or_else(|| "-".into());
181        let mut out = String::from("expectation-results 1\n");
182        for (plan, r) in plans.iter().zip(&self.results) {
183            if r.status == Status::NotRun {
184                continue;
185            }
186            let _ = writeln!(out, "plan\t{}\t{}\t{}", plan.slug, r.status.name(), fingerprint(plan));
187            for (n, step) in std::iter::once(&r.start).chain(&r.steps).enumerate() {
188                let _ = writeln!(
189                    out,
190                    "step\t{n}\t{}\t{}\t{}\t{}",
191                    step.status.name(),
192                    step.polls,
193                    opt(&step.screenshot),
194                    opt(&step.error)
195                );
196                for c in &step.checks {
197                    let observed = match &c.observed {
198                        Some(Value::Number(v)) => format!("n:{v}"),
199                        Some(Value::Enum(s)) => format!("e:{}", field(s)),
200                        None => "-".into(),
201                    };
202                    let _ = writeln!(out, "check\t{}\t{observed}\t{}", c.passed as u8, opt(&c.error));
203                }
204            }
205        }
206        out
207    }
208
209    /// Results saved by [`Report::save`], lined up with `plans`. Plans it has nothing for, or
210    /// whose steps no longer line up, haven't been run.
211    pub fn load(text: &str, plans: &[Plan]) -> Report {
212        let opt = |s: &str| (s != "-").then(|| s.to_string());
213        let mut found: std::collections::HashMap<String, PlanResult> = Default::default();
214        let mut current: Option<(String, PlanResult)> = None;
215        // What each saved plan was when it ran; a plan that has changed since hasn't run.
216        let mut prints: std::collections::HashMap<String, String> = Default::default();
217        for line in text.lines().skip(1) {
218            let f: Vec<&str> = line.split('\t').collect();
219            match f.as_slice() {
220                ["plan", slug, status, rest @ ..] => {
221                    found.extend(current.take());
222                    if let [print] = rest {
223                        prints.insert(slug.to_string(), print.to_string());
224                    }
225                    let status = Status::parse(status).unwrap_or(Status::NotRun);
226                    current =
227                        Some((slug.to_string(), PlanResult { status, start: StepResult::with(status), steps: vec![] }));
228                }
229                ["step", n, status, polls, shot, error] => {
230                    let Some((_, r)) = current.as_mut() else { continue };
231                    let step = StepResult {
232                        status: Status::parse(status).unwrap_or(Status::NotRun),
233                        checks: vec![],
234                        screenshot: opt(shot),
235                        error: opt(error),
236                        polls: polls.parse().unwrap_or(0),
237                    };
238                    if *n == "0" { r.start = step } else { r.steps.push(step) }
239                }
240                ["check", passed, observed, error] => {
241                    let Some((_, r)) = current.as_mut() else { continue };
242                    let observed = match observed.split_once(':') {
243                        Some(("n", v)) => v.parse().ok().map(Value::Number),
244                        Some(("e", v)) => Some(Value::Enum(v.to_string())),
245                        _ => None,
246                    };
247                    let check = CheckResult { observed, passed: *passed == "1", error: opt(error) };
248                    r.steps.last_mut().unwrap_or(&mut r.start).checks.push(check);
249                }
250                _ => {}
251            }
252        }
253        found.extend(current);
254        let results = plans
255            .iter()
256            .map(|p| match found.remove(&p.slug) {
257                Some(r)
258                    if r.steps.len() == p.steps.len()
259                        && prints.get(&p.slug).is_none_or(|print| *print == fingerprint(p)) =>
260                {
261                    r
262                }
263                _ => PlanResult::with(Status::NotRun, p),
264            })
265            .collect();
266        Report { results }
267    }
268
269    /// What went wrong, for a test failure message.
270    pub fn failures(&self, suite: &Suite, plans: &[Plan]) -> String {
271        let mut s = String::new();
272        for (plan, result) in plans.iter().zip(&self.results) {
273            if result.status != Status::Failed {
274                continue;
275            }
276            let _ = writeln!(s, "✗ {}", plan.title);
277            let steps = std::iter::once((None, &plan.start_checks, &result.start))
278                .chain(plan.steps.iter().zip(&result.steps).map(|(p, r)| (Some(p), &p.checks, r)));
279            for (step, checks, r) in steps {
280                if r.status != Status::Failed {
281                    continue;
282                }
283                let what = match step {
284                    None => "arranging the start".to_string(),
285                    Some(p) => p
286                        .text
287                        .iter()
288                        .map(|t| format!("{} {}", t.keyword.as_str(), t.text))
289                        .collect::<Vec<_>>()
290                        .join(" "),
291                };
292                let _ = writeln!(s, "    at: {what}");
293                if let Some(e) = &r.error {
294                    let _ = writeln!(s, "    error: {e}");
295                }
296                for (c, cr) in checks.iter().zip(&r.checks).filter(|(_, cr)| !cr.passed) {
297                    let var = suite.vocabulary.var(c.var);
298                    let why = match &c.because {
299                        Because::Arranged => "arranged".to_string(),
300                        Because::Expectation(e) => format!("«{}»", suite.exp(*e).name),
301                        Because::Story(span) => format!("the story, {span}"),
302                    };
303                    let seen = match (&cr.observed, &cr.error) {
304                        (_, Some(e)) => format!("couldn't observe it: {e}"),
305                        (Some(v), _) => format!("observed {v}"),
306                        (None, _) => "not observed".into(),
307                    };
308                    let _ = writeln!(s, "    expected {} ({why}); {seen}", var.describe(&c.expect));
309                }
310            }
311        }
312        s
313    }
314}
315
316/// Every `*.tsv` results file in `dir` (one per driver that has run), merged and lined up
317/// with `plans`. Nothing there: nothing has run.
318/// A short hash of what a plan does and checks, so that results saved for an older version
319/// of it aren't shown as its results.
320pub fn fingerprint(plan: &Plan) -> String {
321    let mut h: u64 = 0xcbf2_9ce4_8422_2325;
322    let mut eat = |s: &str| {
323        for b in s.bytes().chain([0]) {
324            h = (h ^ b as u64).wrapping_mul(0x0100_0000_01b3);
325        }
326    };
327    eat(&format!("{:?}", plan.start));
328    for step in &plan.steps {
329        for t in &step.text {
330            eat(&t.text);
331        }
332        for c in &step.checks {
333            eat(&format!("{:?} {:?}", c.var, c.expect));
334        }
335    }
336    format!("{h:016x}")
337}
338
339pub fn load_results(dir: &std::path::Path, plans: &[Plan]) -> Report {
340    load_results_for(dir, plans, None)
341}
342
343/// The results file for `runner` on `platform`: `app@iOS.tsv`. A runner that is the same
344/// on every platform has none: `api.tsv`.
345pub fn results_file(runner: &str, platform: Option<&str>) -> String {
346    match platform {
347        Some(p) => format!("{runner}@{p}.tsv"),
348        None => format!("{runner}.tsv"),
349    }
350}
351
352/// As [`load_results`], for the docs of `platform`: the results of runners that are the same on
353/// every platform (`api.tsv`), and of those that ran on this one (`app@iOS.tsv`), not the
354/// results another platform's run saved.
355pub fn load_results_for(dir: &std::path::Path, plans: &[Plan], platform: Option<&str>) -> Report {
356    let mut files: Vec<_> = std::fs::read_dir(dir)
357        .into_iter()
358        .flatten()
359        .flatten()
360        .map(|e| e.path())
361        .filter(|p| p.extension().is_some_and(|x| x == "tsv"))
362        .filter(|p| match p.file_stem().and_then(|s| s.to_str()).and_then(|s| s.split_once('@')) {
363            Some((_, on)) => Some(on) == platform,
364            None => true,
365        })
366        .collect();
367    files.sort();
368    let mut report = Report::not_run(plans);
369    for f in files {
370        if let Ok(text) = std::fs::read_to_string(&f) {
371            report = report.merge(Report::load(&text, plans));
372        }
373    }
374    report
375}
376
377/// An expectation's verdict is about the expectation; this is the id it's indexed by.
378pub fn verdict_of(verdicts: &[Verdict], e: ExpId) -> Verdict {
379    verdicts.get(e.0 as usize).copied().unwrap_or(Verdict::Untested)
380}
expectation_story/report.rs

expectation_story/
report.rs
