From: Ian Jackson Date: Sat, 1 May 2021 19:02:14 +0000 (+0100) Subject: bundles: Initial implementation of framework X-Git-Tag: otter-0.6.0~483 X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=70b8894b563611131ad47df16e08e2a87b932430;p=otter.git bundles: Initial implementation of framework See todos in bundles.rs. Signed-off-by: Ian Jackson --- diff --git a/daemon/cmdlistener.rs b/daemon/cmdlistener.rs index 3b5b5581..6b0d481d 100644 --- a/daemon/cmdlistener.rs +++ b/daemon/cmdlistener.rs @@ -68,7 +68,7 @@ type PCH = PermissionCheckHow; //#[throws(CSE)] fn execute_and_respond(cs: &mut CommandStreamData, cmd: MgmtCommand, - _bulk_upload: ReadFrame, + mut bulk_upload: ReadFrame, mut for_response: WriteFrame) -> Result<(), CSE> where R: Read, W: Write @@ -214,6 +214,28 @@ fn execute_and_respond(cs: &mut CommandStreamData, cmd: MgmtCommand, resp } + MC::UploadBundle { game, hash, kind } => { + let (mut upload, auth) = { + let ag = AccountsGuard::lock(); + let gref = Instance::lookup_by_name_unauth(&game)?; + let bundles = gref.lock_bundles(); + let mut igu = gref.lock()?; + let (ig, auth) = cs.check_acl(&ag, &mut igu, PCH::Instance, + &[TP::UploadBundles])?; + let mut bundles = bundles.by(auth); + let upload = bundles.start_upload(ig, kind)?; + (upload, auth) + }; + upload.bulk(&mut bulk_upload)?; + { + let gref = Instance::lookup_by_name(&game, auth)?; + let mut bundles = gref.lock_bundles(); + let mut ig = gref.lock()?; + bundles.finish_upload(&mut ig, upload, &hash)?; + }; + Fine + } + MC::ListGames { all } => { let ag = AccountsGuard::lock(); let names = Instance::list_names( diff --git a/src/bin/otter.rs b/src/bin/otter.rs index e10ee880..548055da 100644 --- a/src/bin/otter.rs +++ b/src/bin/otter.rs @@ -513,6 +513,7 @@ const PLAYER_ALWAYS_PERMS: &[TablePermission] = &[ const PLAYER_DEFAULT_PERMS: &[TablePermission] = &[ TP::ChangePieces, + TP::UploadBundles, ]; #[throws(AE)] diff --git a/src/bundles.rs b/src/bundles.rs new file mode 100644 index 00000000..154c409d --- /dev/null +++ b/src/bundles.rs @@ -0,0 +1,231 @@ +// Copyright 2020-2021 Ian Jackson and contributors to Otter +// SPDX-License-Identifier: AGPL-3.0-or-later +// There is NO WARRANTY. + +#![allow(dead_code)] // todo + +use crate::prelude::*; + +pub use crate::prelude::Sha512Trunc256 as Digester; + +#[derive(Debug,Copy,Clone,Hash,Eq,PartialEq,Serialize,Deserialize)] +pub struct Hash(pub [u8; 32]); + +#[derive(Debug,Copy,Clone,Hash,Eq,PartialEq,Serialize,Deserialize)] +#[derive(EnumString,Display,Ord,PartialOrd)] +pub enum Kind { + #[strum(to_string="zip")] Zip, +// #[strum(to_string="game.toml")] GameSpec, // identification problems +} + +pub type Index = u16; +const BUNDLES_MAX: Index = 64; + +#[derive(Copy,Clone,Debug,Hash,PartialEq,Eq,Ord,PartialOrd)] +#[derive(Serialize,Deserialize)] +pub struct Id { index: Index, kind: Kind, } + +#[derive(Debug,Clone)] +pub struct InstanceBundles { + // todo: this vec is needed during loading only! + bundles: Vec>, +} + +#[derive(Debug,Clone)] +struct Note { + kind: Kind, + state: State, +} + +type BadBundle = String; // todo: make this a newtype + +#[derive(Debug,Clone)] +enum State { + Uploading, + BadBundle(BadBundle), + Loaded(Loaded), +} + +#[derive(Debug,Clone)] +struct Loaded { + meta: BundleMeta, +} + +pub fn b_dir(instance: &InstanceName) -> String { + savefilename(instance, "b-", "") +} +fn b_file(instance: &InstanceName, index: Index, suffix: S) -> String +where S: Display +{ + format!("{}/{:05}.{}", + savefilename(instance, "b-", ""), + index, suffix) +} + +impl Id { + fn path_tmp(&self, instance: &InstanceName) -> String { + b_file(instance, self.index, "tmp") + } + + fn path(&self, instance: &InstanceName) -> String { + b_file(instance, self.index, self.kind) + } +} + +#[derive(Error,Debug)] +enum IncorporateError { + NotBundle(&'static str), + IE(#[from] IE), +} +display_as_debug!{IncorporateError} +impl From<&'static str> for IncorporateError { + fn from(why: &'static str) -> Self { Self::NotBundle(why) } +} + +pub struct Uploading { + instance: Arc, + id: Id, + file: DigestWrite>, +} + +#[throws(IE)] +fn load_bundle(ib: &mut InstanceBundles, ig: &mut Instance, + id: Id, path: &str) { + #[derive(Error,Debug)] + enum LoadError { + BadBundle(BadBundle), + IE(#[from] IE), + } + display_as_debug!{LoadError} + + let iu: usize = id.index.into(); + + match ib.bundles.get(iu) { + None => ib.bundles.resize_with(iu+1, default), + Some(None) => { }, + Some(Some(Note { kind:_, state: State::Uploading })) => { }, + Some(Some(ref note)) => throw!(IE::DuplicateBundle { + index: id.index, + kinds: [note.kind, id.kind], + }) + }; + let slot = &mut ib.bundles[iu]; + + #[throws(LoadError)] + fn inner(_ig: &mut Instance, + _id: Id, _path: &str) -> Loaded { + Loaded { meta: BundleMeta { title: "title!".to_owned() } } + // todo: find zipfile, read metdata toml + // todo:: show in UI for download + // todo: do actual things, eg libraries and specs + } + + let state = match inner(ig,id,path) { + Ok(loaded) => State::Loaded(loaded), + Err(LoadError::BadBundle(bad)) => State::BadBundle(bad), + Err(LoadError::IE(ie)) => throw!(ie), + }; + + *slot = Some(Note { kind: id.kind, state }) +} + +#[throws(IncorporateError)] +fn incorporate_bundle(ib: &mut InstanceBundles, ig: &mut Instance, + fpath: &str) { + let fleaf = fpath.rsplitn(2, '/').next().unwrap(); + + let [lhs, rhs] = fleaf.splitn(2, '.') + .collect::>() + .into_inner().map_err(|_| "no dot")?; + + let index = lhs.parse().map_err(|_| "bad index")?; + let kind = rhs.parse().map_err(|_| "bad extension")?; + + load_bundle(ib, ig, Id { index, kind }, fpath)?; +} + +impl InstanceBundles { + pub fn new() -> Self { InstanceBundles{ bundles: default() } } + + #[throws(IE)] + pub fn load_game_bundles(ig: &mut Instance) -> Self { + let bd = b_dir(&ig.name); + let mo = glob::MatchOptions { + require_literal_leading_dot: true, + ..default() + }; + let mut ib = InstanceBundles::new(); + for fpath in + glob::glob_with(&format!("{}/*", bd), mo) + .context("pattern for bundle glob")? + { + let fpath = fpath.context("bundle glob")?; + let fpath = fpath + .to_str().ok_or_else(|| anyhow!("glob unicode conversion"))?; + match incorporate_bundle(&mut ib, ig, fpath) { + Ok(()) => { }, + Err(IncorporateError::NotBundle(why)) => { + debug!("bundle file {:?} skippping {}", &fpath, why); + } + Err(IncorporateError::IE(ie)) => throw!(ie), + } + } + ib + } + + #[throws(MgmtError)] + pub fn start_upload(&mut self, ig: &mut Instance, kind: Kind) + -> Uploading { + let state = State::Uploading; + let slot = Some(Note { kind, state }); + let index = self.bundles.len(); + if index >= usize::from(BUNDLES_MAX) { throw!(ME::TooManyBundles) } + let index = index.try_into().unwrap(); + self.bundles.push(slot); + let id = Id { kind, index }; + let tmp = id.path_tmp(&ig.name); + let file = fs::File::create(&tmp) + .with_context(|| tmp.clone()).context("create").map_err(IE::from)?; + let file = BufWriter::new(file); + let file = DigestWrite::new(file); + let instance = ig.name.clone(); + Uploading { file, instance, id } + } +} + +impl Uploading { + #[throws(MgmtError)] + pub fn bulk(&mut self, data: &mut R) + where R: Read + { + io::copy(data, &mut self.file) + .with_context(|| self.id.path_tmp(&*self.instance)) + .context("copy").map_err(IE::from)? + } +} + +impl InstanceBundles { + #[throws(MgmtError)] + pub fn finish_upload(&mut self, ig: &mut Instance, + Uploading { instance:_, id, file }: Uploading, + expected: &Hash) { + let (hash, mut file) = file.finish(); + let tmp = id.path_tmp(&ig.name); + let install = id.path(&ig.name); + file.flush() + .with_context(|| tmp.clone()).context("flush").map_err(IE::from)?; + if hash.as_slice() != &expected.0[..] { throw!(ME::UploadCorrupted) } + load_bundle(self, ig, id, &tmp)?; + match self.bundles.get(usize::from(id.index)) { + Some(Some(Note { state: State::Loaded(..), .. })) => { + fs::rename(&tmp, &install) + .with_context(||install.clone()) + .context("install").map_err(IE::from)?; + } + Some(Some(Note { state: State::BadBundle(ref bad), .. })) => { + throw!(ME::BadBundle(bad.clone())) + } + ref x => panic!("unexpected {:?}", x), + }; + } +} diff --git a/src/commands.rs b/src/commands.rs index 1618b27d..2d06892e 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -37,6 +37,26 @@ pub enum MgmtCommand { game: InstanceName, }, + /* + ClearBundles { + game: InstanceName, + },*/ + UploadBundle { + game: InstanceName, + hash: bundles::Hash, + kind: bundles::Kind, + }, + /* + ListBundles { + game: InstanceName, + }, + DownloadBundle { + game: InstanceName, + index: bundles::Index, + kind: bundles::Kind, + }, + */ + LibraryListByGlob { glob: shapelib::ItemSpec, }, @@ -208,6 +228,9 @@ pub enum MgmtError { TomlSyntaxError(String), TomlStructureError(String), RngIsReal, + UploadCorrupted, + TooManyBundles, + BadBundle(String), } impl Display for MgmtError { #[throws(fmt::Error)] diff --git a/src/error.rs b/src/error.rs index 8b2eac03..12942a8a 100644 --- a/src/error.rs +++ b/src/error.rs @@ -42,6 +42,8 @@ pub enum InternalError { MessagePackEncodeFail(#[from] rmp_serde::encode::Error), #[error("Server MessagePack decoding error (game load failed) {0}")] MessagePackDecodeFail(#[from] rmp_serde::decode::Error), + #[error("Duplicate bundle: {index} {kinds:?}")] + DuplicateBundle { index: bundles::Index, kinds: [bundles::Kind;2] }, #[error("Server internal logic error {0}")] InternalLogicError(InternalLogicError), #[error("SVG processing/generation error {0:?}")] diff --git a/src/global.rs b/src/global.rs index 065d3207..ba8980c4 100644 --- a/src/global.rs +++ b/src/global.rs @@ -36,6 +36,7 @@ pub struct InstanceWeakRef(std::sync::Weak); #[derive(Debug)] pub struct InstanceOuter { c: Mutex, + b: Mutex, } #[derive(Debug,Clone,Serialize,Deserialize,Default)] @@ -288,6 +289,10 @@ impl InstanceRef { pub fn downgrade_to_weak(&self) -> InstanceWeakRef { InstanceWeakRef(Arc::downgrade(&self.0)) } + + pub fn lock_bundles(&self) -> MutexGuard<'_, InstanceBundles> { + self.0.b.lock() + } } impl InstanceWeakRef { @@ -310,6 +315,11 @@ impl Unauthorised { gref: must_not_escape.clone(), }) } + + pub fn lock_bundles<'r>(&'r self) -> Unauthorised, A> { + let must_not_escape = self.by_ref(Authorisation::authorise_any()); + Unauthorised::of(must_not_escape.lock_bundles()) + } } impl Instance { @@ -343,7 +353,8 @@ impl Instance { }; let c = Mutex::new(c); - let gref = InstanceRef(Arc::new(InstanceOuter { c })); + let b = Mutex::new(InstanceBundles::new()); + let gref = InstanceRef(Arc::new(InstanceOuter { c, b })); let mut ig = gref.lock()?; let entry = games.entry(name); @@ -405,6 +416,7 @@ impl Instance { mut g: MutexGuard, _: Authorisation) { let a_savefile = savefilename(&g.g.name, "a-", ""); + let b_dir = bundles::b_dir(&g.g.name); let g_file = savefilename(&g.g.name, "g-", ""); fs::remove_file(&g_file).context("remove").context(g_file)?; @@ -427,6 +439,7 @@ impl Instance { ); } best_effort(|f| fs::remove_file(f), &a_savefile, "auth file"); + best_effort(|f| fs::remove_dir_all(f), &b_dir, "bundles dir"); })(); // <- No ?, ensures that IEFE is infallible (barring panics) } @@ -931,7 +944,7 @@ impl<'ig> InstanceGuard<'ig> { // ---------- save/load ---------- #[derive(Copy,Clone,Debug)] -enum SaveFileOrDir { File, #[allow(dead_code)]/*xxx*/ Dir } +enum SaveFileOrDir { File, Dir } impl SaveFileOrDir { #[throws(io::Error)] @@ -968,6 +981,7 @@ fn savefilename_parse(leaf: &[u8]) -> SavefilenameParseResult { use SavefilenameParseResult::*; if leaf.starts_with(b"a-") { return Auxiliary(SaveFileOrDir::File) } + if leaf.starts_with(b"b-") { return Auxiliary(SaveFileOrDir::Dir ) } let rhs = match leaf.strip_prefix(b"g-") { Some(rhs) => rhs, None => return NotGameFile, @@ -979,7 +993,7 @@ fn savefilename_parse(leaf: &[u8]) -> SavefilenameParseResult { let name = InstanceName::from_str(&rhs)?; - let aux_leaves = [ b"a-" ].iter().map(|prefix| { + let aux_leaves = [ b"a-", b"b-" ].iter().map(|prefix| { let mut s: Vec<_> = (prefix[..]).into(); s.extend(after_ftype_prefix); s }).collect(); GameFile { name, aux_leaves } @@ -1130,7 +1144,7 @@ impl InstanceGuard<'_> { (tokens, accounts.bulk_check(&acctids)) }; - let g = Instance { + let mut g = Instance { gs, iplayers, links, acl: acl.into(), ipieces: IPieces(ipieces), @@ -1141,6 +1155,10 @@ impl InstanceGuard<'_> { tokens_clients: default(), tokens_players: default(), }; + + let b = InstanceBundles::load_game_bundles(&mut g)?; + let b = Mutex::new(b); + let c = InstanceContainer { live: true, game_dirty: false, @@ -1148,7 +1166,7 @@ impl InstanceGuard<'_> { g, }; let c = Mutex::new(c); - let gref = InstanceRef(Arc::new(InstanceOuter { c })); + let gref = InstanceRef(Arc::new(InstanceOuter { c, b })); let mut g = gref.lock().unwrap(); let ig = &mut *g; diff --git a/src/lib.rs b/src/lib.rs index 215c069c..6369a5c6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,7 @@ pub mod prelude; pub mod accounts; pub mod authproofs; +pub mod bundles; pub mod clock; pub mod commands; pub mod config; diff --git a/src/prelude.rs b/src/prelude.rs index a423c066..77683e20 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -122,6 +122,7 @@ pub use crate::accounts::loaded_acl::{self, EffectiveACL, LoadedAcl, PermSet}; pub use crate::accounts::*; pub use crate::authproofs::{self, Authorisation, Unauthorised}; pub use crate::authproofs::AuthorisationSuperuser; +pub use crate::bundles::{self, InstanceBundles}; pub use crate::commands::{AccessTokenInfo, AccessTokenReport, MgmtError}; pub use crate::commands::{MgmtCommand, MgmtResponse}; pub use crate::commands::{MgmtGameInstruction, MgmtGameResponse}; diff --git a/src/spec.rs b/src/spec.rs index 1c50cb9d..be423347 100644 --- a/src/spec.rs +++ b/src/spec.rs @@ -85,6 +85,13 @@ pub enum SpecError { } display_as_debug!{SpecError} +//---------- Bundle "otter.toml" file ---------- + +#[derive(Debug,Clone,Serialize,Deserialize)] +pub struct BundleMeta { + pub title: String, +} + //---------- Table TOML file ---------- #[derive(Debug,Serialize,Deserialize)] @@ -128,6 +135,7 @@ pub enum TablePermission { ViewNotSecret, Play, ChangePieces, + UploadBundles, SetLinks, ResetOthersAccess, RedeliverOthersAccess,