From a56c92d90733a9cfb3d22056c87b232323aa5d26 Mon Sep 17 00:00:00 2001 From: Ayush Singh <ayush@beagleboard.org> Date: Mon, 17 Mar 2025 18:01:55 +0530 Subject: [PATCH] bb-imager: Move download support to separate crate - Refactor Image abstraction from enum to trait based. Allows easy extending in the future. - Removes a lot of dependencies from CLI since it does not need downloader. - Planning to push bb-downloader to crates.io after some experimenting with the interface. Signed-off-by: Ayush Singh <ayush@beagleboard.org> --- Cargo.lock | 21 +- Cargo.toml | 2 +- bb-downloader/Cargo.toml | 27 ++ bb-downloader/README.md | 11 + bb-downloader/src/lib.rs | 290 +++++++++++++++++++++ bb-imager-cli/src/main.rs | 22 +- bb-imager-gui/Cargo.toml | 1 + bb-imager-gui/src/helpers.rs | 130 ++++++++- bb-imager-gui/src/main.rs | 43 ++- bb-imager-gui/src/pages/board_selection.rs | 9 +- bb-imager-gui/src/pages/configuration.rs | 4 +- bb-imager-gui/src/pages/image_selection.rs | 11 +- bb-imager/Cargo.toml | 8 +- bb-imager/src/common.rs | 122 +-------- bb-imager/src/config/mod.rs | 6 - bb-imager/src/download.rs | 236 ----------------- bb-imager/src/error.rs | 2 - bb-imager/src/img.rs | 58 ++++- bb-imager/src/lib.rs | 2 - bb-imager/src/util.rs | 50 ---- 20 files changed, 591 insertions(+), 464 deletions(-) create mode 100644 bb-downloader/Cargo.toml create mode 100644 bb-downloader/README.md create mode 100644 bb-downloader/src/lib.rs delete mode 100644 bb-imager/src/download.rs delete mode 100644 bb-imager/src/util.rs diff --git a/Cargo.lock b/Cargo.lock index 076300c..b4f6f17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -461,6 +461,22 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bb-downloader" +version = "0.1.0" +dependencies = [ + "const-hex", + "futures", + "reqwest", + "serde", + "sha2", + "tempfile", + "thiserror 2.0.12", + "tokio", + "tracing", + "url", +] + [[package]] name = "bb-drivelist" version = "0.1.0" @@ -526,17 +542,13 @@ dependencies = [ "bin_file", "chrono", "const-hex", - "crc32fast", "directories", "futures", "liblzma", - "reqwest", "semver", "serde", "serde_json", "serde_with", - "sha2", - "tempfile", "thiserror 2.0.12", "tokio", "tracing", @@ -562,6 +574,7 @@ dependencies = [ name = "bb-imager-gui" version = "0.0.4" dependencies = [ + "bb-downloader", "bb-imager", "directories", "embed-resource", diff --git a/Cargo.toml b/Cargo.toml index e301cda..3eb3d60 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ members = [ "bb-imager", "bb-imager-cli", "bb-flasher-pb2-mspm0", "bb-imager-gui", "bb-imager-service", - "xtask", "bb-flasher-bcf", "bb-flasher-sd", "iced-loading", + "xtask", "bb-flasher-bcf", "bb-flasher-sd", "iced-loading", "bb-downloader", ] [profile.release] diff --git a/bb-downloader/Cargo.toml b/bb-downloader/Cargo.toml new file mode 100644 index 0000000..107ef13 --- /dev/null +++ b/bb-downloader/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "bb-downloader" +version = "0.1.0" +description = "A simple async downloader for applications" +edition.workspace = true +authors.workspace = true +repository.workspace = true +license.workspace = true +readme = "README.md" +keywords = ["downloader", "beagle"] +categories = ["Asynchronous", "filesystem", "Caching", "Network programming"] + +[dependencies] +reqwest = { version = "0.12", default-features = false, features = ["stream", "rustls-tls"] } +sha2 = "0.10" +thiserror = "2.0" +futures = "0.3" +tempfile = "3.17" +tracing = "0.1" +url = "2.5" +serde = { version = "1.0", optional = true } +tokio = { version = "1.43", default-features = false, features = ["fs"] } +const-hex = "1.14" + +[features] +default = [] +json = ["reqwest/json", "dep:serde"] diff --git a/bb-downloader/README.md b/bb-downloader/README.md new file mode 100644 index 0000000..774b942 --- /dev/null +++ b/bb-downloader/README.md @@ -0,0 +1,11 @@ +# BB Downloader + +A simple downloader library with support for caching. It is designed to be used with applications requiring the downloaded assets to be cached in file system. + +# Features + +- Async +- Cache downloaded file in a directory in filesystem. +- Check if a file is available in cache. +- Uses SHA256 for verifying cached files. +- Optional support to download files without caching. diff --git a/bb-downloader/src/lib.rs b/bb-downloader/src/lib.rs new file mode 100644 index 0000000..50861f4 --- /dev/null +++ b/bb-downloader/src/lib.rs @@ -0,0 +1,290 @@ +//! A simple downloader library designed to be used in Applications with support to cache +//! downloaded assets. +//! +//! # Features +//! +//! - Async +//! - Cache downloaded file in a directory in filesystem. +//! - Check if a file is available in cache. +//! - Uses SHA256 for verifying cached files. +//! - Optional support to download files without caching. + +use futures::{Stream, StreamExt, channel::mpsc}; +#[cfg(feature = "json")] +use serde::de::DeserializeOwned; +use sha2::{Digest as _, Sha256}; +use std::{ + io, + path::{Path, PathBuf}, + time::Duration, +}; +use thiserror::Error; +use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; + +type Result<T, E = Error> = std::result::Result<T, E>; + +#[derive(Error, Debug)] +/// Errors for this crate +pub enum Error { + /// Incorrect Sha256. File might be corrupted + #[error("Incorrect Sha256. File might be corrupted")] + Sha256Error, + #[error("Reqwest Error: {0}")] + ReqwestError(#[from] reqwest::Error), + #[error("IO Error: {0}")] + IoError(#[from] std::io::Error), +} + +/// Simple downloader that caches files in the provided directory. Uses SHA256 to determine if the +/// file is already downloaded. +/// +/// You do not have to wrap the Client in an Rc or Arc to reuse it, because it already uses an Arc +/// internally. +#[derive(Debug, Clone)] +pub struct Downloader { + client: reqwest::Client, + cache_dir: PathBuf, +} + +impl Downloader { + /// Create a new downloader that uses a directory for storing cached files. + pub fn new(cache_dir: PathBuf) -> Self { + assert!(!cache_dir.is_dir()); + + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .build() + .expect("Unsupported OS"); + + Self { client, cache_dir } + } + + /// Check if a downloaded file with a particular SHA256 is already in cache. + pub fn check_cache_from_sha(self, sha256: [u8; 32]) -> Option<PathBuf> { + let file_path = self.path_from_sha(sha256); + + if file_path.exists() { + Some(file_path) + } else { + None + } + } + + /// Check if a downloaded file is already in cache. + /// + /// [`check_cache_from_sha`](Self::check_cache_from_sha) should be prefered in cases when SHA256 + /// of the file to download is already known. + pub fn check_cache_from_url(self, url: &url::Url) -> Option<PathBuf> { + // Use hash of url for file name + let file_path = self.path_from_url(url); + if file_path.exists() { + Some(file_path) + } else { + None + } + } + + /// Download a JSON file without caching the contents. Should be used when there is no point in + /// caching the file. + #[cfg(feature = "json")] + pub async fn download_json_no_cache<T, U>(self, url: U) -> Result<T> + where + T: DeserializeOwned, + U: reqwest::IntoUrl, + { + self.client + .get(url) + .send() + .await + .map_err(Error::from)? + .json() + .await + .map_err(Error::from) + } + + /// Checks if the file is present in cache. If the file is present, returns path to it. Else + /// downloads the file. + /// + /// [`download_with_sha`](Self::download_with_sha) should be prefered when the SHA256 of the + /// file is known in advance. + /// + /// # Progress + /// + /// Download progress can be optionally tracked using a [`futures::channel::mpsc`]. + pub async fn download( + self, + url: url::Url, + mut chan: Option<mpsc::Sender<f32>>, + ) -> Result<PathBuf> { + // Use hash of url for file name + let file_path = self.path_from_url(&url); + + if file_path.exists() { + return Ok(file_path); + } + chan_send(chan.as_mut(), 0.0); + + let mut cur_pos = 0; + let mut tmp_file = AsyncTempFile::new()?; + { + let mut tmp_file = tokio::io::BufWriter::new(tmp_file.as_mut()); + + let response = self.client.get(url).send().await.map_err(Error::from)?; + let response_size = response.content_length(); + let mut response_stream = response.bytes_stream(); + + let response_size = match response_size { + Some(x) => x as usize, + None => response_stream.size_hint().0, + }; + + while let Some(x) = response_stream.next().await { + let mut data = x.map_err(Error::from)?; + cur_pos += data.len(); + tmp_file.write_all_buf(&mut data).await?; + chan_send(chan.as_mut(), (cur_pos as f32) / (response_size as f32)); + } + } + + tmp_file.persist(&file_path).await?; + Ok(file_path) + } + + /// Checks if the file is present in cache. If the file is present, returns path to it. Else + /// downloads the file. + /// + /// Uses SHA256 to verify that the file in cache is valid. + /// + /// # Progress + /// + /// Download progress can be optionally tracked using a [`futures::channel::mpsc`]. + pub async fn download_with_sha( + &self, + url: url::Url, + sha256: [u8; 32], + mut chan: Option<mpsc::Sender<f32>>, + ) -> Result<PathBuf> { + let file_path = self.path_from_sha(sha256); + + if file_path.exists() { + let hash = sha256_from_path(&file_path).await?; + if hash == sha256 { + return Ok(file_path); + } + + // Delete old file + let _ = tokio::fs::remove_file(&file_path).await; + } + chan_send(chan.as_mut(), 0.0); + + let mut tmp_file = AsyncTempFile::new()?; + { + let mut tmp_file = tokio::io::BufWriter::new(tmp_file.as_mut()); + + let response = self.client.get(url).send().await.map_err(Error::from)?; + + let mut cur_pos = 0; + let response_size = response.content_length(); + + let mut response_stream = response.bytes_stream(); + + let response_size = match response_size { + Some(x) => x as usize, + None => response_stream.size_hint().0, + }; + + let mut hasher = Sha256::new(); + + while let Some(x) = response_stream.next().await { + let mut data = x.map_err(Error::from)?; + cur_pos += data.len(); + hasher.update(&data); + tmp_file.write_all_buf(&mut data).await?; + + chan_send(chan.as_mut(), (cur_pos as f32) / (response_size as f32)); + } + + let hash: [u8; 32] = hasher + .finalize() + .as_slice() + .try_into() + .expect("SHA-256 is 32 bytes"); + + if hash != sha256 { + tracing::warn!("{hash:?} != {sha256:?}"); + return Err(Error::Sha256Error.into()); + } + } + + tmp_file.persist(&file_path).await?; + + Ok(file_path) + } + + fn path_from_url(&self, url: &url::Url) -> PathBuf { + let file_name: [u8; 32] = Sha256::new() + .chain_update(url.as_str()) + .finalize() + .as_slice() + .try_into() + .expect("SHA-256 is 32 bytes"); + self.path_from_sha(file_name) + } + + fn path_from_sha(&self, sha256: [u8; 32]) -> PathBuf { + let file_name = const_hex::encode(sha256); + self.cache_dir.join(file_name) + } +} + +struct AsyncTempFile(tokio::fs::File); + +impl AsyncTempFile { + fn new() -> std::io::Result<Self> { + let f = tempfile::tempfile()?; + Ok(Self(tokio::fs::File::from_std(f))) + } + + async fn persist(&mut self, path: &Path) -> std::io::Result<()> { + let mut f = tokio::fs::File::create_new(path).await?; + self.0.seek(io::SeekFrom::Start(0)).await?; + tokio::io::copy(&mut self.0, &mut f).await?; + Ok(()) + } +} + +impl AsMut<tokio::fs::File> for AsyncTempFile { + fn as_mut(&mut self) -> &mut tokio::fs::File { + &mut self.0 + } +} + +async fn sha256_from_path(p: &Path) -> std::io::Result<[u8; 32]> { + let file = tokio::fs::File::open(p).await?; + let mut reader = tokio::io::BufReader::new(file); + let mut hasher = Sha256::new(); + let mut buffer = [0; 512]; + + loop { + let count = reader.read(&mut buffer).await?; + if count == 0 { + break; + } + + hasher.update(&buffer[..count]); + } + + let hash = hasher + .finalize() + .as_slice() + .try_into() + .expect("SHA-256 is 32 bytes"); + + Ok(hash) +} + +fn chan_send(chan: Option<&mut mpsc::Sender<f32>>, msg: f32) { + if let Some(c) = chan { + let _ = c.try_send(msg); + } +} diff --git a/bb-imager-cli/src/main.rs b/bb-imager-cli/src/main.rs index ef87540..0373481 100644 --- a/bb-imager-cli/src/main.rs +++ b/bb-imager-cli/src/main.rs @@ -1,6 +1,6 @@ mod cli; -use bb_imager::{DownloadFlashingStatus, SelectedImage}; +use bb_imager::{DownloadFlashingStatus, img::LocalImage}; use clap::{CommandFactory, Parser}; use cli::{Commands, DestinationsTarget, Opt, TargetCommands}; use std::{ffi::CString, path::PathBuf}; @@ -20,7 +20,6 @@ async fn main() { } async fn flash(target: TargetCommands, quite: bool) { - let downloader = bb_imager::download::Downloader::new(); let (tx, mut rx) = tokio::sync::mpsc::channel(20); if !quite { @@ -107,7 +106,7 @@ async fn flash(target: TargetCommands, quite: bool) { } => { let customization = bb_imager::flasher::FlashingBcfConfig { verify: !no_verify }; bb_imager::FlashingConfig::BeagleConnectFreedom { - img: SelectedImage::Local(img), + img: LocalImage::new(img), port: dst, customization, } @@ -136,38 +135,35 @@ async fn flash(target: TargetCommands, quite: bool) { .update_wifi(wifi); bb_imager::FlashingConfig::LinuxSd { - img: SelectedImage::Local(img), + img: LocalImage::new(img), dst, customization, } } TargetCommands::Msp430 { img, dst } => bb_imager::FlashingConfig::Msp430 { - img: SelectedImage::Local(img), + img: LocalImage::new(img), port: CString::new(dst).expect("Failed to parse destination"), }, #[cfg(feature = "pb2_mspm0")] TargetCommands::Pb2Mspm0 { no_eeprom, img } => bb_imager::FlashingConfig::Pb2Mspm0 { - img: SelectedImage::Local(img), + img: LocalImage::new(img), persist_eeprom: !no_eeprom, }, }; flashing_config - .download_flash_customize(downloader, tx) + .download_flash_customize(tx) .await .expect("Failed to flash"); } async fn format(dst: PathBuf, quite: bool) { - let downloader = bb_imager::download::Downloader::new(); let (tx, _) = tokio::sync::mpsc::channel(20); let term = console::Term::stdout(); - let config = bb_imager::FlashingConfig::LinuxSdFormat { dst }; - config - .download_flash_customize(downloader, tx) - .await - .unwrap(); + let config: bb_imager::FlashingConfig<LocalImage> = + bb_imager::FlashingConfig::LinuxSdFormat { dst }; + config.download_flash_customize(tx).await.unwrap(); if !quite { term.write_line("Formatting successful").unwrap(); diff --git a/bb-imager-gui/Cargo.toml b/bb-imager-gui/Cargo.toml index 21fb1dc..75f0e38 100644 --- a/bb-imager-gui/Cargo.toml +++ b/bb-imager-gui/Cargo.toml @@ -26,6 +26,7 @@ iced-loading = { path = "../iced-loading" } serde = { version = "1.0.218", features = ["derive"] } serde_json = { version = "1.0.139" } directories = "6.0.0" +bb-downloader = { path = "../bb-downloader", features = ["json"] } [build-dependencies] embed-resource = "2.5.0" diff --git a/bb-imager-gui/src/helpers.rs b/bb-imager-gui/src/helpers.rs index 9abfbda..ff8cd1e 100644 --- a/bb-imager-gui/src/helpers.rs +++ b/bb-imager-gui/src/helpers.rs @@ -1,8 +1,9 @@ use std::{borrow::Cow, collections::HashSet, io::Read, path::PathBuf, sync::LazyLock}; use bb_imager::{DownloadFlashingStatus, config::OsListItem}; +use futures::StreamExt; use iced::{ - Element, + Element, futures, widget::{self, button, text}, }; use serde::{Deserialize, Serialize}; @@ -400,30 +401,36 @@ pub fn search_bar(cur_search: &str) -> Element<BBImagerMessage> { .into() } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone)] pub enum BoardImage { SdFormat, Image { flasher: bb_imager::Flasher, - img: bb_imager::SelectedImage, + img: SelectedImage, }, } impl BoardImage { - pub const fn local(path: PathBuf, flasher: bb_imager::Flasher) -> Self { + pub fn local(path: PathBuf, flasher: bb_imager::Flasher) -> Self { Self::Image { - img: bb_imager::SelectedImage::local(path), + img: bb_imager::img::LocalImage::new(path).into(), flasher, } } - pub fn remote(image: bb_imager::config::OsImage, flasher: bb_imager::Flasher) -> Self { + pub fn remote( + image: bb_imager::config::OsImage, + flasher: bb_imager::Flasher, + downloader: bb_downloader::Downloader, + ) -> Self { Self::Image { - img: bb_imager::SelectedImage::remote( + img: RemoteImage::new( image.name, image.url, image.image_download_sha256, - ), + downloader, + ) + .into(), flasher, } } @@ -434,6 +441,13 @@ impl BoardImage { BoardImage::Image { flasher, .. } => *flasher, } } + + pub(crate) fn is_sd_format(&self) -> bool { + match self { + BoardImage::SdFormat => true, + _ => false, + } + } } impl std::fmt::Display for BoardImage { @@ -690,3 +704,103 @@ impl Default for Pb2Mspm0Customization { } } } + +#[derive(Debug, Clone)] +pub(crate) struct RemoteImage { + name: String, + url: url::Url, + extract_sha256: [u8; 32], + downloader: bb_downloader::Downloader, +} + +impl RemoteImage { + pub(crate) const fn new( + name: String, + url: url::Url, + extract_sha256: [u8; 32], + downloader: bb_downloader::Downloader, + ) -> Self { + Self { + name, + url, + extract_sha256, + downloader, + } + } +} + +impl bb_imager::img::ImageFile for RemoteImage { + async fn resolve( + &self, + chan: Option<tokio::sync::mpsc::Sender<DownloadFlashingStatus>>, + ) -> std::io::Result<PathBuf> { + let (tx, rx) = futures::channel::mpsc::channel(20); + + if let Some(chan) = chan { + tokio::spawn(async move { + let chan_ref = &chan; + rx.map(DownloadFlashingStatus::DownloadingProgress) + .for_each(|m| async move { + let _ = chan_ref.try_send(m); + }) + .await + }); + } + + self.downloader + .download_with_sha(self.url.clone(), self.extract_sha256, Some(tx)) + .await + .map_err(|e| { + if let bb_downloader::Error::IoError(x) = e { + x + } else { + std::io::Error::other(format!("Failed to open image: {e}")) + } + }) + } +} + +impl std::fmt::Display for RemoteImage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.name) + } +} + +#[derive(Debug, Clone)] +pub(crate) enum SelectedImage { + LocalImage(bb_imager::img::LocalImage), + RemoteImage(RemoteImage), +} + +impl bb_imager::img::ImageFile for SelectedImage { + async fn resolve( + &self, + chan: Option<tokio::sync::mpsc::Sender<DownloadFlashingStatus>>, + ) -> std::io::Result<PathBuf> { + match self { + SelectedImage::LocalImage(x) => x.resolve(chan).await, + SelectedImage::RemoteImage(x) => x.resolve(chan).await, + } + } +} + +impl std::fmt::Display for SelectedImage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SelectedImage::LocalImage(x) => x.fmt(f), + SelectedImage::RemoteImage(x) => x.fmt(f), + } + } +} + +impl From<RemoteImage> for SelectedImage { + fn from(value: RemoteImage) -> Self { + Self::RemoteImage(value) + } +} + +impl From<bb_imager::img::LocalImage> for SelectedImage { + fn from(value: bb_imager::img::LocalImage) -> Self { + Self::LocalImage(value) + } +} diff --git a/bb-imager-gui/src/main.rs b/bb-imager-gui/src/main.rs index 6418d49..89933be 100644 --- a/bb-imager-gui/src/main.rs +++ b/bb-imager-gui/src/main.rs @@ -2,6 +2,7 @@ use std::{collections::HashSet, time::Duration}; +use constants::PACKAGE_QUALIFIER; use helpers::ProgressBarState; use iced::{Element, Subscription, Task, futures::SinkExt, widget}; use message::BBImagerMessage; @@ -54,11 +55,11 @@ fn main() -> iced::Result { .run_with(move || BBImager::new(boards, app_config)) } -#[derive(Default, Debug)] +#[derive(Debug)] struct BBImager { app_config: helpers::GuiConfiguration, boards: helpers::Boards, - downloader: bb_imager::download::Downloader, + downloader: bb_downloader::Downloader, screen: Vec<Screen>, selected_board: Option<usize>, selected_image: Option<helpers::BoardImage>, @@ -81,7 +82,16 @@ impl BBImager { boards: helpers::Boards, app_config: helpers::GuiConfiguration, ) -> (Self, Task<BBImagerMessage>) { - let downloader = bb_imager::download::Downloader::default(); + let downloader = bb_downloader::Downloader::new( + directories::ProjectDirs::from( + PACKAGE_QUALIFIER.0, + PACKAGE_QUALIFIER.1, + PACKAGE_QUALIFIER.2, + ) + .unwrap() + .cache_dir() + .to_path_buf(), + ); // Fetch old config let client = downloader.clone(); @@ -89,7 +99,7 @@ impl BBImager { let config_task = Task::perform( async move { let data: bb_imager::config::Config = client - .download_json(constants::BB_IMAGER_ORIGINAL_CONFIG) + .download_json_no_cache(constants::BB_IMAGER_ORIGINAL_CONFIG) .await .map_err(|e| format!("Config parsing failed: {e}"))?; @@ -122,7 +132,14 @@ impl BBImager { ), destination_selectable: true, screen: Vec::with_capacity(3), - ..Default::default() + selected_board: Default::default(), + selected_image: Default::default(), + selected_dst: Default::default(), + destinations: Default::default(), + search_bar: Default::default(), + cancel_flashing: Default::default(), + customization: Default::default(), + flashing_state: Default::default(), }; ans.screen.push(Screen::Home); @@ -143,7 +160,7 @@ impl BBImager { let tasks = icons.into_iter().map(|icon| { Task::perform( - self.downloader.clone().download_without_sha(icon.clone()), + self.downloader.clone().download(icon.clone(), None), move |p| match p { Ok(_) => BBImagerMessage::Null, Err(_) => { @@ -183,7 +200,7 @@ impl BBImager { let jobs = icons.into_iter().map(|x| { Task::perform( - self.downloader.clone().download_without_sha(x.clone()), + self.downloader.clone().download(x.clone(), None), move |p| match p { Ok(_path) => BBImagerMessage::Null, Err(e) => { @@ -376,7 +393,7 @@ impl BBImager { let target_clone: Vec<usize> = target.to_vec(); return Task::perform( - self.downloader.clone().download_json(url.clone()), + self.downloader.clone().download_json_no_cache(url.clone()), move |x| match x { Ok(item) => BBImagerMessage::ResolveRemoteSubitemItem { item, @@ -411,7 +428,9 @@ impl BBImager { Task::perform( self.downloader .clone() - .download_json::<Vec<bb_imager::config::OsListItem>, url::Url>(url.clone()), + .download_json_no_cache::<Vec<bb_imager::config::OsListItem>, url::Url>( + url.clone(), + ), move |x| match x { Ok(item) => BBImagerMessage::ResolveRemoteSubitemItem { item, @@ -562,7 +581,7 @@ impl BBImager { fn flashing_config( &self, customization: FlashingCustomization, - ) -> bb_imager::common::FlashingConfig { + ) -> bb_imager::common::FlashingConfig<helpers::SelectedImage> { match ( self.selected_image.clone(), customization, @@ -622,8 +641,6 @@ impl BBImager { docs_url.as_ref().map(|x| x.to_string()).unwrap_or_default(), )); - let downloader = self.downloader.clone(); - let config = self.flashing_config(customization.unwrap_or(self.config())); let s = iced::stream::channel(20, move |mut chan| async move { @@ -633,7 +650,7 @@ impl BBImager { let (tx, mut rx) = tokio::sync::mpsc::channel(19); - let flash_task = tokio::spawn(config.download_flash_customize(downloader, tx)); + let flash_task = tokio::spawn(config.download_flash_customize(tx)); let mut chan_clone = chan.clone(); let progress_task = tokio::spawn(async move { while let Some(progress) = rx.recv().await { diff --git a/bb-imager-gui/src/pages/board_selection.rs b/bb-imager-gui/src/pages/board_selection.rs index 218e7c2..69337ba 100644 --- a/bb-imager-gui/src/pages/board_selection.rs +++ b/bb-imager-gui/src/pages/board_selection.rs @@ -1,25 +1,24 @@ use iced::{ - widget::{self, button, text}, Element, + widget::{self, button, text}, }; use crate::{ - constants, + BBImagerMessage, constants, helpers::{self, img_or_svg}, - BBImagerMessage, }; pub fn view<'a>( boards: &'a helpers::Boards, search_bar: &'a str, - downloader: &'a bb_imager::download::Downloader, + downloader: &'a bb_downloader::Downloader, ) -> Element<'a, BBImagerMessage> { let items = boards .devices() .filter(|(_, x)| x.name.to_lowercase().contains(&search_bar.to_lowercase())) .map(|(id, dev)| { let image: Element<BBImagerMessage> = match &dev.icon { - Some(url) => match downloader.clone().check_image(url) { + Some(url) => match downloader.clone().check_cache_from_url(url) { Some(y) => img_or_svg(y, 100), None => widget::svg(widget::svg::Handle::from_memory( constants::DOWNLOADING_ICON, diff --git a/bb-imager-gui/src/pages/configuration.rs b/bb-imager-gui/src/pages/configuration.rs index 439aca3..a2f43a8 100644 --- a/bb-imager-gui/src/pages/configuration.rs +++ b/bb-imager-gui/src/pages/configuration.rs @@ -300,9 +300,7 @@ impl FlashingCustomization { app_config: &helpers::GuiConfiguration, ) -> Self { match flasher { - bb_imager::Flasher::SdCard if img == &helpers::BoardImage::SdFormat => { - Self::LinuxSdFormat - } + bb_imager::Flasher::SdCard if img.is_sd_format() => Self::LinuxSdFormat, bb_imager::Flasher::SdCard => { Self::LinuxSd(app_config.sd_customization().cloned().unwrap_or_default()) } diff --git a/bb-imager-gui/src/pages/image_selection.rs b/bb-imager-gui/src/pages/image_selection.rs index 821ade5..6fa8d12 100644 --- a/bb-imager-gui/src/pages/image_selection.rs +++ b/bb-imager-gui/src/pages/image_selection.rs @@ -46,7 +46,7 @@ impl ImageSelectionPage { &self, images: Option<Vec<(usize, &'a bb_imager::config::OsListItem)>>, search_bar: &'a str, - downloader: &'a bb_imager::download::Downloader, + downloader: &'a bb_downloader::Downloader, // Allow optional format entry extra_entries: E, ) -> Element<'a, BBImagerMessage> @@ -90,7 +90,7 @@ impl ImageSelectionPage { fn entry_subitem<'a>( &self, image: &'a bb_imager::config::OsImage, - downloader: &'a bb_imager::download::Downloader, + downloader: &'a bb_downloader::Downloader, ) -> widget::Button<'a, BBImagerMessage> { let row3 = widget::row( [ @@ -103,7 +103,7 @@ impl ImageSelectionPage { .align_y(iced::alignment::Vertical::Center) .spacing(5); - let icon = match downloader.clone().check_image(&image.icon) { + let icon = match downloader.clone().check_cache_from_url(&image.icon) { Some(y) => img_or_svg(y, ICON_WIDTH), None => widget::svg(widget::svg::Handle::from_memory( constants::DOWNLOADING_ICON, @@ -128,6 +128,7 @@ impl ImageSelectionPage { .on_press(BBImagerMessage::SelectImage(helpers::BoardImage::remote( image.clone(), self.flasher, + downloader.clone(), ))) .style(widget::button::secondary) } @@ -135,7 +136,7 @@ impl ImageSelectionPage { fn entry<'a>( &self, item: &'a bb_imager::config::OsListItem, - downloader: &'a bb_imager::download::Downloader, + downloader: &'a bb_downloader::Downloader, idx: usize, ) -> widget::Button<'a, BBImagerMessage> { match item { @@ -154,7 +155,7 @@ impl ImageSelectionPage { flasher, .. } => { - let icon = match downloader.clone().check_image(icon) { + let icon = match downloader.clone().check_cache_from_url(icon) { Some(y) => img_or_svg(y, ICON_WIDTH), None => widget::svg(widget::svg::Handle::from_memory( constants::DOWNLOADING_ICON, diff --git a/bb-imager/Cargo.toml b/bb-imager/Cargo.toml index b5f7d3d..3cb8462 100644 --- a/bb-imager/Cargo.toml +++ b/bb-imager/Cargo.toml @@ -11,19 +11,15 @@ license.workspace = true liblzma = { version = "0.3.6", features = ["static", "parallel"] } chrono = { version = "0.4.39", optional = true , default-features = false } const-hex = { version = "1.14.0" } -crc32fast = "1.4.2" directories = "6.0.0" -reqwest = { version = "0.12.12", default-features = false, features = ["stream", "rustls-tls"] } semver = { version = "1.0.25", optional = true } serde = { version = "1.0.218", features = ["derive"], optional = true } serde_json = { version = "1.0.139", optional = true } -sha2 = "0.10.8" thiserror = "2.0.11" -tokio = { version = "1.43.0", default-features = false, features = ["rt-multi-thread", "process", "fs", "time"] } +tokio = { version = "1.43.0", default-features = false, features = ["rt-multi-thread", "sync", "net", "time"] } tracing = "0.1.41" url = "2.5.4" futures = "0.3.31" -tempfile = "3.17.1" bb-flasher-pb2-mspm0 = { path = "../bb-flasher-pb2-mspm0", optional = true } bb-flasher-bcf = { path = "../bb-flasher-bcf", features = ["msp430", "cc1352p7"] } bb-flasher-sd = { path = "../bb-flasher-sd" } @@ -36,5 +32,5 @@ default = [] udev = ["bb-flasher-sd/udev"] pb2_mspm0_raw = ["bb-flasher-pb2-mspm0", "dep:bin_file"] pb2_mspm0_dbus = ["dep:zbus", "dep:serde", "dep:bin_file"] -config = ["dep:serde_json", "dep:serde", "dep:serde_with", "const-hex/serde", "semver/serde", "url/serde", "chrono/serde", "reqwest/json"] +config = ["dep:serde_json", "dep:serde", "dep:serde_with", "const-hex/serde", "semver/serde", "url/serde", "chrono/serde"] macos_authopen = ["bb-flasher-sd/macos_authopen"] diff --git a/bb-imager/src/common.rs b/bb-imager/src/common.rs index 4fafc7f..5b89c7b 100644 --- a/bb-imager/src/common.rs +++ b/bb-imager/src/common.rs @@ -79,105 +79,37 @@ impl std::fmt::Display for Destination { } } -#[derive(Debug, Clone, PartialEq)] -pub enum SelectedImage { - Local(PathBuf), - Remote { - name: String, - url: url::Url, - extract_sha256: [u8; 32], - }, -} - -impl SelectedImage { - pub const fn local(name: PathBuf) -> Self { - Self::Local(name) - } - - pub const fn remote(name: String, url: url::Url, download_sha256: [u8; 32]) -> Self { - Self::Remote { - name, - url, - extract_sha256: download_sha256, - } - } - - /// Download image if not local - pub async fn resolve_img( - self, - downloader: crate::download::Downloader, - chan: &tokio::sync::mpsc::Sender<crate::DownloadFlashingStatus>, - ) -> crate::error::Result<std::path::PathBuf> { - match self { - crate::SelectedImage::Local(x) => Ok(x), - crate::SelectedImage::Remote { - url, - extract_sha256, - .. - } => { - downloader - .download_progress(url, extract_sha256, chan) - .await - } - } - } -} - -impl std::fmt::Display for SelectedImage { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - SelectedImage::Local(p) => write!( - f, - "{}", - p.file_name() - .expect("image cannot be a directory") - .to_string_lossy() - ), - SelectedImage::Remote { name, .. } => write!(f, "{}", name), - } - } -} - -#[cfg(feature = "config")] -impl From<&crate::config::OsImage> for SelectedImage { - fn from(value: &crate::config::OsImage) -> Self { - Self::remote( - value.name.clone(), - value.url.clone(), - value.image_download_sha256, - ) - } -} - -pub enum FlashingConfig { +pub enum FlashingConfig<I: crate::img::ImageFile> { LinuxSdFormat { dst: PathBuf, }, LinuxSd { - img: SelectedImage, + img: I, dst: PathBuf, customization: sd::FlashingSdLinuxConfig, }, BeagleConnectFreedom { - img: SelectedImage, + img: I, port: String, customization: bcf::FlashingBcfConfig, }, Msp430 { - img: SelectedImage, + img: I, port: std::ffi::CString, }, #[cfg(any(feature = "pb2_mspm0_raw", feature = "pb2_mspm0_dbus"))] Pb2Mspm0 { - img: SelectedImage, + img: I, persist_eeprom: bool, }, } -impl FlashingConfig { +impl<I> FlashingConfig<I> +where + I: crate::img::ImageFile + Send + 'static, +{ pub async fn download_flash_customize( self, - downloader: crate::download::Downloader, chan: tokio::sync::mpsc::Sender<DownloadFlashingStatus>, ) -> crate::error::Result<()> { match self { @@ -195,22 +127,8 @@ impl FlashingConfig { .enable_io() .build() .unwrap(); - let img_path = rt - .block_on(async move { img.resolve_img(downloader, &chan).await }) - .map_err(|e| { - if let crate::error::Error::IoError(x) = e { - x - } else { - std::io::Error::other(format!("Failed to download image: {e}")) - } - })?; - - let img = crate::img::OsImage::from_path(&img_path).map_err(|e| { - if let crate::error::Error::IoError(x) = e { - x - } else { - std::io::Error::other(format!("Failed to open image: {e}")) - } + let img = rt.block_on(async move { + crate::img::OsImage::open(img, chan.clone()).await })?; let img_size = img.size(); @@ -228,11 +146,7 @@ impl FlashingConfig { customization, } => { tracing::info!("Port opened"); - let img_path = img.resolve_img(downloader, &chan).await?; - let mut img = - tokio::task::spawn_blocking(move || crate::img::OsImage::from_path(&img_path)) - .await - .unwrap()?; + let mut img = crate::img::OsImage::open(img, chan.clone()).await?; let mut data = Vec::new(); img.read_to_end(&mut data) @@ -241,11 +155,7 @@ impl FlashingConfig { bcf::flash(data, &port, &chan, customization.verify).await } FlashingConfig::Msp430 { img, port } => { - let img_path = img.resolve_img(downloader, &chan).await?; - let mut img = - tokio::task::spawn_blocking(move || crate::img::OsImage::from_path(&img_path)) - .await - .unwrap()?; + let mut img = crate::img::OsImage::open(img, chan.clone()).await?; tracing::info!("Image opened"); let mut data = Vec::new(); @@ -259,11 +169,7 @@ impl FlashingConfig { img, persist_eeprom, } => { - let img_path = img.resolve_img(downloader, &chan).await?; - let mut img = - tokio::task::spawn_blocking(move || crate::img::OsImage::from_path(&img_path)) - .await - .unwrap()?; + let mut img = crate::img::OsImage::open(img, chan.clone()).await?; tracing::info!("Image opened"); let mut data = String::new(); diff --git a/bb-imager/src/config/mod.rs b/bb-imager/src/config/mod.rs index 4d5789c..6c4291c 100644 --- a/bb-imager/src/config/mod.rs +++ b/bb-imager/src/config/mod.rs @@ -98,12 +98,6 @@ impl Config { } } -impl From<OsImage> for crate::SelectedImage { - fn from(value: OsImage) -> Self { - Self::remote(value.name, value.url, value.image_download_sha256) - } -} - #[cfg(test)] mod tests { #[test] diff --git a/bb-imager/src/download.rs b/bb-imager/src/download.rs deleted file mode 100644 index c08b347..0000000 --- a/bb-imager/src/download.rs +++ /dev/null @@ -1,236 +0,0 @@ -//! Module for downloading remote images for flashing - -use directories::ProjectDirs; -use futures::{Stream, StreamExt}; -#[cfg(feature = "config")] -use serde::de::DeserializeOwned; -use sha2::{Digest as _, Sha256}; -use std::{ - io, - path::{Path, PathBuf}, - time::Duration, -}; -use thiserror::Error; -use tokio::io::{AsyncSeekExt, AsyncWriteExt}; - -use crate::{error::Result, util::sha256_file_progress, DownloadFlashingStatus}; - -#[derive(Error, Debug, Clone)] -pub enum Error { - #[error("UnknownError")] - UnknownError, - #[error("Reqwest Error: {0}")] - ReqwestError(String), - #[error("Incorrect Sha256. File might be corrupted")] - Sha256Error, - #[error("Json Parsing Error: {0}")] - JsonError(String), -} - -impl From<reqwest::Error> for Error { - fn from(value: reqwest::Error) -> Self { - Self::ReqwestError(value.to_string()) - } -} - -#[derive(Debug, Clone)] -pub struct Downloader { - client: reqwest::Client, - dirs: ProjectDirs, -} - -impl Default for Downloader { - fn default() -> Self { - Self::new() - } -} - -impl Downloader { - pub fn new() -> Self { - let client = reqwest::Client::builder() - .connect_timeout(Duration::from_secs(10)) - .build() - .expect("Unsupported OS"); - - let dirs = ProjectDirs::from("org", "beagleboard", "imagingutility") - .expect("Failed to find project directories"); - - tracing::info!("Cache Dir: {:?}", dirs.cache_dir()); - if let Err(e) = std::fs::create_dir_all(dirs.cache_dir()) { - if e.kind() != io::ErrorKind::AlreadyExists { - panic!( - "Failed to create cache dir: {:?} due to error {e}", - dirs.cache_dir() - ) - } - } - - Self { client, dirs } - } - - pub fn check_cache(self, sha256: [u8; 32]) -> Option<PathBuf> { - let file_path = self.path_from_sha(sha256); - - if file_path.exists() { - Some(file_path) - } else { - None - } - } - - pub fn check_image(self, url: &url::Url) -> Option<PathBuf> { - // Use hash of url for file name - let file_path = self.path_from_url(url); - if file_path.exists() { - Some(file_path) - } else { - None - } - } - - #[cfg(feature = "config")] - pub async fn download_json<T, U>(self, url: U) -> Result<T> - where - T: DeserializeOwned, - U: reqwest::IntoUrl, - { - self.client - .get(url) - .send() - .await - .map_err(Error::from)? - .json() - .await - .map_err(|e| Error::JsonError(e.to_string()).into()) - } - - pub async fn download_without_sha(self, url: url::Url) -> Result<PathBuf> { - // Use hash of url for file name - let file_path = self.path_from_url(&url); - - if file_path.exists() { - return Ok(file_path); - } - - let mut tmp_file = AsyncTempFile::new()?; - - let response = self.client.get(url).send().await.map_err(Error::from)?; - - let mut response_stream = response.bytes_stream(); - - while let Some(x) = response_stream.next().await { - let mut data = x.map_err(Error::from)?; - tmp_file.as_mut().write_all_buf(&mut data).await?; - } - - tmp_file.persist(&file_path).await?; - Ok(file_path) - } - - pub async fn download_progress( - &self, - url: url::Url, - sha256: [u8; 32], - chan: &tokio::sync::mpsc::Sender<DownloadFlashingStatus>, - ) -> Result<PathBuf> { - let file_path = self.path_from_sha(sha256); - - if file_path.exists() { - let _ = chan.try_send(DownloadFlashingStatus::VerifyingProgress(0.0)); - - let hash = sha256_file_progress(&file_path, chan).await?; - if hash == sha256 { - return Ok(file_path); - } - - // Delete old file - let _ = tokio::fs::remove_file(&file_path).await; - } - let _ = chan.try_send(DownloadFlashingStatus::DownloadingProgress(0.0)); - - let mut tmp_file = AsyncTempFile::new()?; - let response = self.client.get(url).send().await.map_err(Error::from)?; - - let mut cur_pos = 0; - let response_size = response.content_length(); - - let mut response_stream = response.bytes_stream(); - - let response_size = match response_size { - Some(x) => x as usize, - None => response_stream.size_hint().0, - }; - - let mut hasher = Sha256::new(); - - while let Some(x) = response_stream.next().await { - let mut data = x.map_err(Error::from)?; - cur_pos += data.len(); - hasher.update(&data); - tmp_file.as_mut().write_all_buf(&mut data).await?; - - let _ = chan.try_send(DownloadFlashingStatus::DownloadingProgress( - (cur_pos as f32) / (response_size as f32), - )); - } - - let hash: [u8; 32] = hasher - .finalize() - .as_slice() - .try_into() - .expect("SHA-256 is 32 bytes"); - - let _ = chan.try_send(DownloadFlashingStatus::Verifying); - - if hash != sha256 { - tracing::warn!("{hash:?} != {sha256:?}"); - return Err(Error::Sha256Error.into()); - } - - tmp_file.persist(&file_path).await?; - - Ok(file_path) - } - - pub async fn download(self, url: url::Url, sha256: [u8; 32]) -> Result<PathBuf> { - let (tx, _) = tokio::sync::mpsc::channel(1); - self.download_progress(url, sha256, &tx).await - } - - fn path_from_url(&self, url: &url::Url) -> PathBuf { - let file_name: [u8; 32] = Sha256::new() - .chain_update(url.as_str()) - .finalize() - .as_slice() - .try_into() - .expect("SHA-256 is 32 bytes"); - self.path_from_sha(file_name) - } - - fn path_from_sha(&self, sha256: [u8; 32]) -> PathBuf { - let file_name = const_hex::encode(sha256); - self.dirs.cache_dir().join(file_name) - } -} - -struct AsyncTempFile(tokio::fs::File); - -impl AsyncTempFile { - fn new() -> Result<Self> { - let f = tempfile::tempfile()?; - Ok(Self(tokio::fs::File::from_std(f))) - } - - async fn persist(&mut self, path: &Path) -> Result<()> { - let mut f = tokio::fs::File::create_new(path).await?; - self.0.seek(io::SeekFrom::Start(0)).await?; - tokio::io::copy(&mut self.0, &mut f).await?; - Ok(()) - } -} - -impl AsMut<tokio::fs::File> for AsyncTempFile { - fn as_mut(&mut self) -> &mut tokio::fs::File { - &mut self.0 - } -} diff --git a/bb-imager/src/error.rs b/bb-imager/src/error.rs index f450086..ef0e34d 100644 --- a/bb-imager/src/error.rs +++ b/bb-imager/src/error.rs @@ -12,8 +12,6 @@ pub enum Error { BeagleConnectFreedomError(#[from] bcf::Error), #[error("MSP430 Error: {0}")] MSP430Error(#[from] msp430::Error), - #[error("Download Error: {0}")] - DownloadError(#[from] crate::download::Error), #[error("Io Error: {0}")] IoError(#[from] std::io::Error), #[error("Image Error: {0}")] diff --git a/bb-imager/src/img.rs b/bb-imager/src/img.rs index c661429..f52ab6c 100644 --- a/bb-imager/src/img.rs +++ b/bb-imager/src/img.rs @@ -1,11 +1,12 @@ //! Module to handle extraction of compressed firmware, auto detection of type of extraction, etc -use crate::error::Result; +use crate::{DownloadFlashingStatus, error::Result}; use std::{ io::{Read, Seek}, - path::Path, + path::{Path, PathBuf}, }; use thiserror::Error; +use tokio::sync::mpsc; #[derive(Error, Debug)] pub enum Error { @@ -25,6 +26,21 @@ pub enum OsImageReader { } impl OsImage { + pub(crate) async fn open( + img: impl ImageFile, + chan: mpsc::Sender<DownloadFlashingStatus>, + ) -> std::io::Result<Self> { + let img_path = img.resolve(Some(chan)).await?; + + Self::from_path(&img_path).map_err(|e| { + if let crate::error::Error::IoError(x) = e { + x + } else { + std::io::Error::other(format!("Failed to open image: {e}")) + } + }) + } + pub fn from_path(path: &Path) -> Result<Self> { let mut file = std::fs::File::open(path)?; @@ -82,3 +98,41 @@ fn size(file: &std::fs::Metadata) -> u64 { use std::os::windows::fs::MetadataExt; file.file_size() } + +pub trait ImageFile { + fn resolve( + &self, + chan: Option<mpsc::Sender<DownloadFlashingStatus>>, + ) -> impl Future<Output = std::io::Result<PathBuf>>; +} + +#[derive(Debug, Clone)] +pub struct LocalImage(PathBuf); + +impl LocalImage { + pub const fn new(path: PathBuf) -> Self { + Self(path) + } +} + +impl ImageFile for LocalImage { + fn resolve( + &self, + _: Option<mpsc::Sender<DownloadFlashingStatus>>, + ) -> impl Future<Output = std::io::Result<PathBuf>> { + std::future::ready(Ok(self.0.clone())) + } +} + +impl std::fmt::Display for LocalImage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + self.0 + .file_name() + .expect("image cannot be a directory") + .to_string_lossy() + ) + } +} diff --git a/bb-imager/src/lib.rs b/bb-imager/src/lib.rs index 61027bd..96f53b1 100644 --- a/bb-imager/src/lib.rs +++ b/bb-imager/src/lib.rs @@ -1,10 +1,8 @@ pub mod common; #[cfg(feature = "config")] pub mod config; -pub mod download; pub mod error; pub mod flasher; pub mod img; -pub(crate) mod util; pub use common::*; diff --git a/bb-imager/src/util.rs b/bb-imager/src/util.rs deleted file mode 100644 index 7165883..0000000 --- a/bb-imager/src/util.rs +++ /dev/null @@ -1,50 +0,0 @@ -//! Helper functions - -use crate::error::Result; -use std::path::Path; - -use sha2::{Digest, Sha256}; - -const BUF_SIZE: usize = 4 * 1024; - -pub(crate) async fn sha256_file_progress( - path: &Path, - chan: &tokio::sync::mpsc::Sender<crate::DownloadFlashingStatus>, -) -> Result<[u8; 32]> { - let file = tokio::fs::File::open(path).await?; - let file_len = file.metadata().await?.len(); - - sha256_reader_progress(file, file_len, chan).await -} - -pub(crate) async fn sha256_reader_progress<R: tokio::io::AsyncReadExt + Unpin>( - mut reader: R, - size: u64, - chan: &tokio::sync::mpsc::Sender<crate::DownloadFlashingStatus>, -) -> Result<[u8; 32]> { - let mut hasher = Sha256::new(); - let mut buffer = [0; BUF_SIZE]; - let mut pos = 0; - - loop { - let count = reader.read(&mut buffer).await?; - if count == 0 { - break; - } - - hasher.update(&buffer[..count]); - - pos += count; - let _ = chan.try_send(crate::DownloadFlashingStatus::VerifyingProgress( - pos as f32 / size as f32, - )); - } - - let hash = hasher - .finalize() - .as_slice() - .try_into() - .expect("SHA-256 is 32 bytes"); - - Ok(hash) -} -- GitLab