expectation_harness/
run.rs

1//! Running plans against a driver.
2
3use crate::driver::{Driver, Need, Progress};
4use crate::service::Services;
5use expectation_model::{Suite, Trigger};
6use expectation_story::{Check, CheckResult, Plan, PlanResult, Report, Status, StepResult};
7use std::path::PathBuf;
8
9#[derive(Clone, Debug)]
10pub struct RunOptions {
11    /// Screenshots go to `<docs_dir>/screenshots/<plan>/<step>.<ext>`; `None` takes none.
12    pub docs_dir: Option<PathBuf>,
13    /// How many times to [`Driver::advance`] while waiting for a step's checks to hold.
14    pub patience: u32,
15}
16
17impl Default for RunOptions {
18    fn default() -> Self {
19        RunOptions { docs_dir: None, patience: 100 }
20    }
21}
22
23pub fn run(suite: &Suite, plans: &[Plan], driver: &mut dyn Driver, opts: &RunOptions) -> Report {
24    Report { results: plans.iter().map(|p| run_plan(suite, p, driver, opts)).collect() }
25}
26
27/// As [`run`], with what the driver needs from outside started before each plan: a service that
28/// is up stays, one that died is started again, and when one cannot be had the plan is not
29/// supported yet, with what to do about it.
30pub fn run_with(suite: &Suite, plans: &[Plan], driver: &mut dyn Driver, opts: &RunOptions, services: &mut Services) -> Report {
31    let needs = driver.needs();
32    let names: Vec<&str> = needs.iter().map(String::as_str).collect();
33    let fresh = driver.fresh();
34    let fresh: Vec<&str> = fresh.iter().map(String::as_str).collect();
35    let results = plans
36        .iter()
37        .map(|plan| match unsupported(suite, plan, driver).map_or_else(|| services.restart(&fresh).and_then(|()| services.start(&names)), |_| Ok(())) {
38            Ok(()) => run_plan(suite, plan, driver, opts),
39            Err(why) => {
40                let mut r = PlanResult::with(Status::Unsupported, plan);
41                r.start.error = Some(format!("It needs {why}."));
42                r
43            }
44        })
45        .collect();
46    Report { results }
47}
48
49fn needs<'a>(suite: &'a Suite, plan: &Plan) -> Vec<Need<'a>> {
50    let v = &suite.vocabulary;
51    let mut out: Vec<Need> = plan.start.vars().map(|x| Need::Arrange(v.var(x))).collect();
52    let observe = |c: &Check| Need::Observe(v.var(c.var));
53    out.extend(plan.start_checks.iter().map(observe));
54    for step in &plan.steps {
55        out.extend(step.action.changes.vars().map(|x| Need::Set(v.var(x))));
56        out.extend(step.action.event.map(|e| Need::Perform(v.event(e))));
57        out.extend(step.checks.iter().map(observe));
58    }
59    out
60}
61
62/// A plan the driver cannot run yet is not run and not failed: the result says what it lacks.
63fn unsupported(suite: &Suite, plan: &Plan, driver: &dyn Driver) -> Option<PlanResult> {
64    let mut missing: Vec<String> =
65        needs(suite, plan).into_iter().filter(|n| !driver.supports(*n)).map(|n| n.to_string()).collect();
66    for (x, c) in plan.start.iter() {
67        let var = suite.vocabulary.var(x);
68        let value = var.witness(c).expect("plans only arrange satisfiable states");
69        if !driver.can_arrange(var, &value) {
70            missing.push(format!("arrange {} as {value}", var.subject));
71        }
72    }
73    missing.sort();
74    missing.dedup();
75    if !missing.is_empty() {
76        // Not a failure: the system (or its driver) just doesn't do this yet.
77        let mut r = PlanResult::with(Status::Unsupported, plan);
78        // A few of them, so the reason stays readable: another system's tests lack dozens.
79        let mut shown: Vec<String> = missing.iter().take(3).cloned().collect();
80        if missing.len() > 3 {
81            shown.push(format!("{} more things", missing.len() - 3));
82        }
83        r.start.error = Some(format!("The driver can't {} yet.", expectation_model::text::join_and(&shown)));
84        return Some(r);
85    }
86    None
87}
88
89fn run_plan(suite: &Suite, plan: &Plan, driver: &mut dyn Driver, opts: &RunOptions) -> PlanResult {
90    let failed = |error: String| StepResult {
91        status: Status::Failed,
92        checks: vec![],
93        screenshot: None,
94        error: Some(error),
95        polls: 0,
96    };
97    let skip_all = |start: StepResult| PlanResult {
98        status: Status::Failed,
99        start,
100        steps: plan.steps.iter().map(|_| StepResult::skipped()).collect(),
101    };
102
103    if let Some(r) = unsupported(suite, plan, driver) {
104        return r;
105    }
106
107    let shots = opts.docs_dir.as_ref().map(|d| {
108        let dir = d.join("screenshots").join(&plan.slug);
109        let _ = std::fs::remove_dir_all(&dir);
110        dir
111    });
112    let shoot = |driver: &mut dyn Driver, n: usize| -> Option<String> {
113        let dir = shots.as_ref()?;
114        let shot = driver.screenshot().ok()??;
115        let name = format!("{n:02}.{}", shot.extension);
116        std::fs::create_dir_all(dir).ok()?;
117        std::fs::write(dir.join(&name), &shot.bytes).ok()?;
118        Some(format!("screenshots/{}/{name}", plan.slug))
119    };
120
121    if let Err(e) = driver.reset() {
122        return skip_all(failed(format!("couldn't reset: {e}")));
123    }
124    for (x, c) in plan.start.iter() {
125        let var = suite.vocabulary.var(x);
126        let value = var.witness(c).expect("plans only arrange satisfiable states");
127        if let Err(e) = driver.arrange(var, &value) {
128            return skip_all(failed(format!("couldn't arrange {}: {e}", var.describe(c))));
129        }
130    }
131    driver.settle();
132    let mut start = observe(suite, driver, &plan.start_checks, opts.patience);
133    start.screenshot = shoot(driver, 0);
134
135    let mut ok = start.status == Status::Passed;
136    let mut steps = vec![];
137    for (n, step) in plan.steps.iter().enumerate() {
138        if !ok {
139            steps.push(StepResult::skipped());
140            continue;
141        }
142        let mut r = match act(suite, driver, &step.action) {
143            Ok(()) => {
144                driver.settle();
145                observe(suite, driver, &step.checks, opts.patience)
146            }
147            Err(e) => failed(e),
148        };
149        r.screenshot = shoot(driver, n + 1);
150        ok = r.status == Status::Passed;
151        steps.push(r);
152    }
153    PlanResult { status: if ok { Status::Passed } else { Status::Failed }, start, steps }
154}
155
156fn act(suite: &Suite, driver: &mut dyn Driver, action: &Trigger) -> Result<(), String> {
157    for (x, c) in action.changes.iter() {
158        let var = suite.vocabulary.var(x);
159        let value = var.witness(c).ok_or_else(|| format!("no value satisfies {}", var.describe(c)))?;
160        driver.set(var, &value).map_err(|e| format!("couldn't set {}: {e}", var.describe(c)))?;
161    }
162    if let Some(e) = action.event {
163        let event = suite.vocabulary.event(e);
164        driver.perform(event).map_err(|err| format!("couldn't make it happen that {}: {err}", event.text))?;
165    }
166    Ok(())
167}
168
169/// Observe until every check holds at once, or patience runs out.
170fn observe(suite: &Suite, driver: &mut dyn Driver, checks: &[Check], patience: u32) -> StepResult {
171    let mut polls = 0;
172    loop {
173        polls += 1;
174        let results: Vec<CheckResult> = checks
175            .iter()
176            .map(|c| {
177                let var = suite.vocabulary.var(c.var);
178                match driver.observe(var) {
179                    Ok(v) => CheckResult { passed: var.admits(&c.expect, &v), observed: Some(v), error: None },
180                    Err(e) => CheckResult { observed: None, passed: false, error: Some(e) },
181                }
182            })
183            .collect();
184        let passed = results.iter().all(|r| r.passed);
185        if passed || polls > patience || driver.advance() == Progress::Static {
186            let status = if passed { Status::Passed } else { Status::Failed };
187            return StepResult { status, checks: results, screenshot: None, error: None, polls };
188        }
189    }
190}
expectation_harness/run.rs

expectation_harness/
run.rs
