openwrt/flashrom
1
0
Fork 0
mirror of https://review.coreboot.org/flashrom.git synced 2025-07-04 23:35:18 +02:00
Code Issues Releases Wiki Activity

util/flashrom_tester: Upstream E2E testing framework

Browse Source 
The following is a E2E tester for a specific chip/chipset
combo. The tester itself is completely self-contained and
allows the user to specify which tests they wish to preform.
Supported tests include:

 - chip-name
 - read
 - write
 - erase
 - wp-locking

Change-Id: Ic2905a76cad90b1546b9328d668bf8abbf8aed44
Signed-off-by: Edward O'Callaghan <quasisec@google.com>
Reviewed-on: https://review.coreboot.org/c/flashrom/+/38951
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: David Hendricks <david.hendricks@gmail.com>
This commit is contained in:
Edward O'Callaghan
2020-02-18 14:38:08 +11:00
committed by Edward O'Callaghan
parent 7a7fee1695
commit 0f510a7458
16 changed files with 2680 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

636
util/flashrom_tester/src/tester.rs Normal file
View File

@ -0,0 +1,636 @@
//
// Copyright 2019, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Alternatively, this software may be distributed under the terms of the
// GNU General Public License ("GPL") version 2 as published by the Free
// Software Foundation.
//
use super::rand_util;
use super::types;
use super::utils::{self, LayoutSizes};
use flashrom::{FlashChip, Flashrom, FlashromCmd};
use serde_json::json;
use std::mem::MaybeUninit;
use std::sync::Mutex;
// type-signature comes from the return type of lib.rs workers.
type TestError = Box<dyn std::error::Error>;
pub type TestResult = Result<(), TestError>;
pub struct TestEnv<'a> {
chip_type: FlashChip,
/// Flashrom instantiation information.
///
/// Where possible, prefer to use methods on the TestEnv rather than delegating
/// to the raw flashrom functions.
pub cmd: &'a FlashromCmd,
layout: LayoutSizes,
pub wp: WriteProtectState<'a, 'static>,
/// The path to a file containing the flash contents at test start.
// TODO(pmarheine) migrate this to a PathBuf for clarity
original_flash_contents: String,
/// The path to a file containing flash-sized random data
// TODO(pmarheine) make this a PathBuf too
random_data: String,
}
impl<'a> TestEnv<'a> {
pub fn create(chip_type: FlashChip, cmd: &'a FlashromCmd) -> Result<Self, String> {
let rom_sz = cmd.get_size()?;
let out = TestEnv {
chip_type: chip_type,
cmd: cmd,
layout: utils::get_layout_sizes(rom_sz)?,
wp: WriteProtectState::from_hardware(cmd)?,
original_flash_contents: "/tmp/flashrom_tester_golden.bin".into(),
random_data: "/tmp/random_content.bin".into(),
};
info!("Stashing golden image for verification/recovery on completion");
flashrom::read(&out.cmd, &out.original_flash_contents)?;
flashrom::verify(&out.cmd, &out.original_flash_contents)?;
info!("Generating random flash-sized data");
rand_util::gen_rand_testdata(&out.random_data, rom_sz as usize)
.map_err(|io_err| format!("I/O error writing random data file: {:#}", io_err))?;
Ok(out)
}
pub fn run_test<T: TestCase>(&mut self, test: T) -> TestResult {
let use_dut_control = self.chip_type == FlashChip::SERVO;
if use_dut_control && flashrom::dut_ctrl_toggle_wp(false).is_err() {
error!("failed to dispatch dut_ctrl_toggle_wp()!");
}
let name = test.get_name();
info!("Beginning test: {}", name);
let out = test.run(self);
info!("Completed test: {}; result {:?}", name, out);
if use_dut_control && flashrom::dut_ctrl_toggle_wp(true).is_err() {
error!("failed to dispatch dut_ctrl_toggle_wp()!");
}
out
}
pub fn chip_type(&self) -> FlashChip {
// This field is not public because it should be immutable to tests,
// so this getter enforces that it is copied.
self.chip_type
}
/// Return the path to a file that contains random data and is the same size
/// as the flash chip.
pub fn random_data_file(&self) -> &str {
&self.random_data
}
pub fn layout(&self) -> &LayoutSizes {
&self.layout
}
/// Return true if the current Flash contents are the same as the golden image
/// that was present at the start of testing.
pub fn is_golden(&self) -> bool {
flashrom::verify(&self.cmd, &self.original_flash_contents).is_ok()
}
/// Do whatever is necessary to make the current Flash contents the same as they
/// were at the start of testing.
pub fn ensure_golden(&mut self) -> Result<(), String> {
self.wp.set_hw(false)?.set_sw(false)?;
flashrom::write(&self.cmd, &self.original_flash_contents)
}
/// Attempt to erase the flash.
pub fn erase(&self) -> Result<(), String> {
flashrom::erase(self.cmd)
}
/// Verify that the current Flash contents are the same as the file at the given
/// path.
///
/// Returns Err if they are not the same.
pub fn verify(&self, contents_path: &str) -> Result<(), String> {
flashrom::verify(self.cmd, contents_path)
}
}
impl Drop for TestEnv<'_> {
fn drop(&mut self) {
info!("Verifying flash remains unmodified");
if !self.is_golden() {
warn!("ROM seems to be in a different state at finish; restoring original");
if let Err(e) = self.ensure_golden() {
error!("Failed to write back golden image: {:?}", e);
}
}
}
}
/// RAII handle for setting write protect in either hardware or software.
///
/// Given an instance, the state of either write protect can be modified by calling
/// `set` or `push`. When it goes out of scope, the write protects will be returned
/// to the state they had then it was created.
///
/// The lifetime `'p` on this struct is the parent state it derives from; `'static`
/// implies it is derived from hardware, while anything else is part of a stack
/// created by `push`ing states. An initial state is always static, and the stack
/// forms a lifetime chain `'static -> 'p -> 'p1 -> ... -> 'pn`.
pub struct WriteProtectState<'a, 'p> {
/// The parent state this derives from.
///
/// If it's a root (gotten via `from_hardware`), then this is Hardware and the
/// liveness flag will be reset on drop.
initial: InitialState<'p>,
// Tuples are (hardware, software)
current: (bool, bool),
cmd: &'a FlashromCmd,
}
enum InitialState<'p> {
Hardware(bool, bool),
Previous(&'p WriteProtectState<'p, 'p>),
}
impl InitialState<'_> {
fn get_target(&self) -> (bool, bool) {
match self {
InitialState::Hardware(hw, sw) => (*hw, *sw),
InitialState::Previous(s) => s.current,
}
}
}
impl<'a> WriteProtectState<'a, 'static> {
/// Initialize a state from the current state of the hardware.
///
/// Panics if there is already a live state derived from hardware. In such a situation the
/// new state must be derived from the live one, or the live one must be dropped first.
pub fn from_hardware(cmd: &'a FlashromCmd) -> Result<Self, String> {
let mut lock = Self::get_liveness_lock()
.lock()
.expect("Somebody panicked during WriteProtectState init from hardware");
if *lock {
drop(lock); // Don't poison the lock
panic!("Attempted to create a new WriteProtectState when one is already live");
}
let hw = Self::get_hw(cmd)?;
let sw = Self::get_sw(cmd)?;
info!("Initial hardware write protect: HW={} SW={}", hw, sw);
*lock = true;
Ok(WriteProtectState {
initial: InitialState::Hardware(hw, sw),
current: (hw, sw),
cmd,
})
}
/// Get the actual hardware write protect state.
fn get_hw(cmd: &FlashromCmd) -> Result<bool, String> {
if cmd.fc.can_control_hw_wp() {
super::utils::get_hardware_wp()
} else {
Ok(false)
}
}
/// Get the actual software write protect state.
fn get_sw(cmd: &FlashromCmd) -> Result<bool, String> {
flashrom::wp_status(cmd, true)
}
}
impl<'a, 'p> WriteProtectState<'a, 'p> {
/// Return true if the current programmer supports setting the hardware
/// write protect.
///
/// If false, calls to set_hw() will do nothing.
pub fn can_control_hw_wp(&self) -> bool {
self.cmd.fc.can_control_hw_wp()
}
/// Set the software write protect.
pub fn set_sw(&mut self, enable: bool) -> Result<&mut Self, String> {
info!("request={}, current={}", enable, self.current.1);
if self.current.1 != enable {
flashrom::wp_toggle(self.cmd, /* en= */ enable)?;
self.current.1 = enable;
}
Ok(self)
}
/// Set the hardware write protect.
pub fn set_hw(&mut self, enable: bool) -> Result<&mut Self, String> {
if self.current.0 != enable {
if self.can_control_hw_wp() {
super::utils::toggle_hw_wp(/* dis= */ !enable)?;
self.current.0 = enable;
} else if enable {
info!(
"Ignoring attempt to enable hardware WP with {:?} programmer",
self.cmd.fc
);
}
}
Ok(self)
}
/// Stack a new write protect state on top of the current one.
///
/// This is useful if you need to temporarily make a change to write protection:
///
/// ```no_run
/// # fn main() -> Result<(), String> {
/// # let cmd: flashrom::FlashromCmd = unimplemented!();
/// let wp = flashrom_tester::tester::WriteProtectState::from_hardware(&cmd)?;
/// {
/// let mut wp = wp.push();
/// wp.set_sw(false)?;
/// // Do something with software write protect disabled
/// }
/// // Now software write protect returns to its original state, even if
/// // set_sw() failed.
/// # Ok(())
/// # }
/// ```
///
/// This returns a new state which restores the original when it is dropped- the new state
/// refers to the old, so the compiler enforces that states are disposed of in the reverse
/// order of their creation and correctly restore the original state.
pub fn push<'p1>(&'p1 self) -> WriteProtectState<'a, 'p1> {
WriteProtectState {
initial: InitialState::Previous(self),
current: self.current,
cmd: self.cmd,
}
}
fn get_liveness_lock() -> &'static Mutex<bool> {
static INIT: std::sync::Once = std::sync::Once::new();
/// Value becomes true when there is a live WriteProtectState derived `from_hardware`,
/// blocking duplicate initialization.
///
/// This is required because hardware access is not synchronized; it's possible to leave the
/// hardware in an unintended state by creating a state handle from it, modifying the state,
/// creating another handle from the hardware then dropping the first handle- then on drop
/// of the second handle it will restore the state to the modified one rather than the initial.
///
/// This flag ensures that a duplicate root state cannot be created.
///
/// This is a Mutex<bool> rather than AtomicBool because acquiring the flag needs to perform
/// several operations that may themselves fail- acquisitions must be fully synchronized.
static mut LIVE_FROM_HARDWARE: MaybeUninit<Mutex<bool>> = MaybeUninit::uninit();
unsafe {
INIT.call_once(|| {
LIVE_FROM_HARDWARE.as_mut_ptr().write(Mutex::new(false));
});
&*LIVE_FROM_HARDWARE.as_ptr()
}
}
/// Reset the hardware to what it was when this state was created, reporting errors.
///
/// This behaves exactly like allowing a state to go out of scope, but it can return
/// errors from that process rather than panicking.
pub fn close(mut self) -> Result<(), String> {
unsafe {
let out = self.drop_internal();
// We just ran drop, don't do it again
std::mem::forget(self);
out
}
}
/// Internal Drop impl.
///
/// This is unsafe because it effectively consumes self when clearing the
/// liveness lock. Callers must be able to guarantee that self will be forgotten
/// if the state was constructed from hardware in order to uphold the liveness
/// invariant (that only a single state constructed from hardware exists at any
/// time).
unsafe fn drop_internal(&mut self) -> Result<(), String> {
let lock = match self.initial {
InitialState::Hardware(_, _) => Some(
Self::get_liveness_lock()
.lock()
.expect("Somebody panicked during WriteProtectState drop from hardware"),
),
_ => None,
};
let (hw, sw) = self.initial.get_target();
fn enable_str(enable: bool) -> &'static str {
if enable {
"en"
} else {
"dis"
}
}
// Toggle both protects back to their initial states.
// Software first because we can't change it once hardware is enabled.
if sw != self.current.1 {
// Is the hw wp currently enabled?
if self.current.0 {
super::utils::toggle_hw_wp(/* dis= */ true).map_err(|e| {
format!(
"Failed to {}able hardware write protect: {}",
enable_str(false),
e
)
})?;
}
flashrom::wp_toggle(self.cmd, /* en= */ sw).map_err(|e| {
format!(
"Failed to {}able software write protect: {}",
enable_str(sw),
e
)
})?;
}
assert!(
self.cmd.fc.can_control_hw_wp() || (!self.current.0 && !hw),
"HW WP must be disabled if it cannot be controlled"
);
if hw != self.current.0 {
super::utils::toggle_hw_wp(/* dis= */ !hw).map_err(|e| {
format!(
"Failed to {}able hardware write protect: {}",
enable_str(hw),
e
)
})?;
}
if let Some(mut lock) = lock {
// Initial state was constructed via from_hardware, now we can clear the liveness
// lock since reset is complete.
*lock = false;
}
Ok(())
}
}
impl<'a, 'p> Drop for WriteProtectState<'a, 'p> {
/// Sets both write protects to the state they had when this state was created.
///
/// Panics on error because there is no mechanism to report errors in Drop.
fn drop(&mut self) {
unsafe { self.drop_internal() }.expect("Error while dropping WriteProtectState")
}
}
pub trait TestCase {
fn get_name(&self) -> &str;
fn expected_result(&self) -> TestConclusion;
fn run(&self, env: &mut TestEnv) -> TestResult;
}
impl<S: AsRef<str>, F: Fn(&mut TestEnv) -> TestResult> TestCase for (S, F) {
fn get_name(&self) -> &str {
self.0.as_ref()
}
fn expected_result(&self) -> TestConclusion {
TestConclusion::Pass
}
fn run(&self, env: &mut TestEnv) -> TestResult {
(self.1)(env)
}
}
impl<T: TestCase + ?Sized> TestCase for &T {
fn get_name(&self) -> &str {
(*self).get_name()
}
fn expected_result(&self) -> TestConclusion {
(*self).expected_result()
}
fn run(&self, env: &mut TestEnv) -> TestResult {
(*self).run(env)
}
}
#[allow(dead_code)]
#[derive(Copy, Clone, PartialEq, Debug)]
pub enum TestConclusion {
Pass,
Fail,
UnexpectedPass,
UnexpectedFail,
}
pub struct ReportMetaData {
pub chip_name: String,
pub os_release: String,
pub system_info: String,
pub bios_info: String,
}
fn decode_test_result(res: TestResult, con: TestConclusion) -> (TestConclusion, Option<TestError>) {
use TestConclusion::*;
match (res, con) {
(Ok(_), Fail) => (UnexpectedPass, None),
(Err(e), Pass) => (UnexpectedFail, Some(e)),
_ => (Pass, None),
}
}
pub fn run_all_tests<T, TS>(
chip: FlashChip,
cmd: &FlashromCmd,
ts: TS,
) -> Vec<(String, (TestConclusion, Option<TestError>))>
where
T: TestCase + Copy,
TS: IntoIterator<Item = T>,
{
let mut env = TestEnv::create(chip, cmd).expect("Failed to set up test environment");
let mut results = Vec::new();
for t in ts {
let result = decode_test_result(env.run_test(t), t.expected_result());
results.push((t.get_name().into(), result));
}
results
}
#[derive(Debug, PartialEq, Clone, Copy)]
pub enum OutputFormat {
Pretty,
Json,
}
impl std::str::FromStr for OutputFormat {
type Err = ();
fn from_str(s: &str) -> Result<Self, Self::Err> {
use OutputFormat::*;
if s.eq_ignore_ascii_case("pretty") {
Ok(Pretty)
} else if s.eq_ignore_ascii_case("json") {
Ok(Json)
} else {
Err(())
}
}
}
pub fn collate_all_test_runs(
truns: &[(String, (TestConclusion, Option<TestError>))],
meta_data: ReportMetaData,
format: OutputFormat,
) {
match format {
OutputFormat::Pretty => {
println!();
println!(" =============================");
println!(" ===== AVL qual RESULTS ====");
println!(" =============================");
println!();
println!(" %---------------------------%");
println!(" os release: {}", meta_data.os_release);
println!(" chip name: {}", meta_data.chip_name);
println!(" system info: \n{}", meta_data.system_info);
println!(" bios info: \n{}", meta_data.bios_info);
println!(" %---------------------------%");
println!();
for trun in truns.iter() {
let (name, (result, error)) = trun;
if *result != TestConclusion::Pass {
println!(
" {} {}",
style!(format!(" <+> {} test:", name), types::BOLD),
style_dbg!(result, types::RED)
);
match error {
None => {}
Some(e) => info!(" - {} failure details:\n{}", name, e.to_string()),
};
} else {
println!(
" {} {}",
style!(format!(" <+> {} test:", name), types::BOLD),
style_dbg!(result, types::GREEN)
);
}
}
println!();
}
OutputFormat::Json => {
use serde_json::{Map, Value};
let mut all_pass = true;
let mut tests = Map::<String, Value>::new();
for (name, (result, error)) in truns {
let passed = *result == TestConclusion::Pass;
all_pass &= passed;
let error = match error {
Some(e) => Value::String(format!("{:#?}", e)),
None => Value::Null,
};
assert!(
!tests.contains_key(name),
"Found multiple tests named {:?}",
name
);
tests.insert(
name.into(),
json!({
"pass": passed,
"error": error,
}),
);
}
let json = json!({
"pass": all_pass,
"metadata": {
"os_release": meta_data.os_release,
"chip_name": meta_data.chip_name,
"system_info": meta_data.system_info,
"bios_info": meta_data.bios_info,
},
"tests": tests,
});
println!("{:#}", json);
}
}
}
#[cfg(test)]
mod tests {
#[test]
fn decode_test_result() {
use super::decode_test_result;
use super::TestConclusion::*;
let (result, err) = decode_test_result(Ok(()), Pass);
assert_eq!(result, Pass);
assert!(err.is_none());
let (result, err) = decode_test_result(Ok(()), Fail);
assert_eq!(result, UnexpectedPass);
assert!(err.is_none());
let (result, err) = decode_test_result(Err("broken".into()), Pass);
assert_eq!(result, UnexpectedFail);
assert!(err.is_some());
let (result, err) = decode_test_result(Err("broken".into()), Fail);
assert_eq!(result, Pass);
assert!(err.is_none());
}
#[test]
fn output_format_round_trip() {
use super::OutputFormat::{self, *};
assert_eq!(format!("{:?}", Pretty).parse::<OutputFormat>(), Ok(Pretty));
assert_eq!(format!("{:?}", Json).parse::<OutputFormat>(), Ok(Json));
}
}