diff --git a/Cargo.lock b/Cargo.lock index 554ee1afe2..fe03fb8845 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4682,14 +4682,32 @@ dependencies = [ "tokio", ] +[[package]] +name = "spk-proto" +version = "0.44.0" +dependencies = [ + "data-encoding", + "flatbuffers", + "flatc-rust", + "futures", + "miette", + "ring", + "rstest", + "serde", + "thiserror 1.0.69", +] + [[package]] name = "spk-schema" version = "0.44.0" dependencies = [ + "arc-swap", + "bytes", "config", "data-encoding", "dunce", "enum_dispatch", + "flatbuffers", "format_serde_error", "ignore", "indexmap 2.11.0", @@ -4709,6 +4727,7 @@ dependencies = [ "shellexpand", "spfs", "spk-config", + "spk-proto", "spk-schema-foundation", "spk-schema-tera", "strum", @@ -4725,10 +4744,12 @@ version = "0.44.0" dependencies = [ "arc-swap", "async-trait", + "bytes", "colored", "data-encoding", "derive-where", "enum_dispatch", + "flatbuffers", "format_serde_error", "ignore", "indexmap 2.11.0", @@ -4748,6 +4769,7 @@ dependencies = [ "serde_yaml 0.9.34+deprecated", "serial_test", "spfs", + "spk-proto", "strum", "sys-info", "tap", diff --git a/Cargo.toml b/Cargo.toml index 49ca0b41d5..c69e74278e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "crates/spk-config", "crates/spk-exec", "crates/spk-launcher", + "crates/spk-proto", "crates/spk-schema", "crates/spk-schema/crates/*", "crates/spk-solve", @@ -65,6 +66,7 @@ dyn-clone = "1.0" enum_dispatch = "0.3.13" featurecomb = "0.1.3" flatbuffers = "25.2" +flatc-rust = "0.2" format_serde_error = { version = "0.3", default-features = false } fuser = "0.16.0" futures = "0.3.28" @@ -140,6 +142,7 @@ spk-cmd-repo = { path = "crates/spk-cli/cmd-repo" } spk-cmd-test = { path = "crates/spk-cli/cmd-test" } spk-config = { path = "crates/spk-config" } spk-exec = { path = "crates/spk-exec" } +spk-proto = { path = "crates/spk-proto" } spk-schema = { path = "crates/spk-schema" } spk-schema-foundation = { path = "crates/spk-schema/crates/foundation" } spk-schema-tera = { path = "crates/spk-schema/crates/tera" } diff --git a/crates/spfs-proto/Cargo.toml b/crates/spfs-proto/Cargo.toml index 9340f22f93..6c0c3bbcb1 100644 --- a/crates/spfs-proto/Cargo.toml +++ b/crates/spfs-proto/Cargo.toml @@ -24,7 +24,7 @@ serde = { workspace = true, optional = true } thiserror = { workspace = true } [build-dependencies] -flatc-rust = "0.2" +flatc-rust = { workspace = true } [dev-dependencies] ring = { workspace = true } diff --git a/crates/spk-proto/Cargo.toml b/crates/spk-proto/Cargo.toml new file mode 100644 index 0000000000..c24d6ec7e2 --- /dev/null +++ b/crates/spk-proto/Cargo.toml @@ -0,0 +1,31 @@ +[package] +authors = { workspace = true } +edition = { workspace = true } +name = "spk-proto" +version = { workspace = true } +license-file = { workspace = true } +homepage = { workspace = true } +repository = { workspace = true } +readme = { workspace = true } +description = { workspace = true } + +[lints] +workspace = true + +[features] +serde = ["dep:serde"] + +[dependencies] +data-encoding = { workspace = true } +flatbuffers = { workspace = true } +futures = { workspace = true } +miette = { workspace = true } +serde = { workspace = true, optional = true } +thiserror = { workspace = true } + +[build-dependencies] +flatc-rust = { workspace = true } + +[dev-dependencies] +ring = { workspace = true } +rstest = { workspace = true } diff --git a/crates/spk-proto/build.rs b/crates/spk-proto/build.rs new file mode 100644 index 0000000000..8787f603f5 --- /dev/null +++ b/crates/spk-proto/build.rs @@ -0,0 +1,31 @@ +// Copyright (c) Contributors to the SPK project. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/spkenv/spk + +use std::env; +use std::path::{Path, PathBuf}; + +fn main() { + println!("cargo:rerun-if-changed=schema/spk.fbs"); + + let cmd = match std::env::var_os("FLATC") { + Some(exe) => flatc_rust::Flatc::from_path(exe), + None => flatc_rust::Flatc::from_env_path(), + }; + + let out_dir = env::var("OUT_DIR").unwrap(); + + cmd.run(flatc_rust::Args { + lang: "rust", + inputs: &[Path::new("schema/spk.fbs")], + out_dir: &PathBuf::from(&out_dir), + ..Default::default() + }) + .expect("schema compiler command"); + + let generated_file = PathBuf::from(out_dir).join("spk_generated.rs"); + println!( + "cargo:rerun-if-changed=schema/spk.fbs generated file: {}", + generated_file.display() + ); +} diff --git a/crates/spk-proto/schema/spk.fbs b/crates/spk-proto/schema/spk.fbs new file mode 100644 index 0000000000..8478344331 --- /dev/null +++ b/crates/spk-proto/schema/spk.fbs @@ -0,0 +1,331 @@ +// Copyright (c) Contributors to the SPK project. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/spkenv/spk + +// When adding or modifying this file, take care to read +// the compatibility and evolution notes here: +// https://flatbuffers.dev/flatbuffers_guide_writing_schema.html +// In general, only add fields to the end of tables. Never add +// new fields to struct types. + +// The kinds of component names. +enum ComponentEnum: uint8 { + All = 0, + Build, + Run, + Source, + // The Named value indicates the name field in the Component table + // should be used for the component's name. + Named, +} + +// Cannot use a union, because vectors of unions are not allowed. So +// the name field is optional and only used when the kind value is +// ComponentEnum::Named. +table Component { + kind: ComponentEnum; + name: string; +} + +// For a Version's epsilon values +enum Epsilon: uint8 { + Minus = 0, + None = 1, + Plus = 2, +} + +// For a Version's pre and post tags +table TagSetItem { + name: string (required); + number: uint32; +} + + +// The pieces of a pkg/version/build Ident, but not combined as a +// BuildIdent. This is used for embedded packages. +table IdentPartsBuf { + repository_name: string; + pkg_name: string (required); + version_str: string; + build_str: string; +} + +// The package identifier data for an embedded package in a BuildIdent +table EmbeddedSourcePackage { + ident: IdentPartsBuf (required); + components: [Component]; + // ignored fields: unparsed +} + +// A embedded package identifier in a BuildIdent +table EmbeddedSource { + source: EmbeddedSourcePackage; +} + +// A build digest part in BuildIdent +table BuildId { + id: string (required); +} + +// A placeholder for the source enum variant used in the Build +// union. That the type is source is enough, but it might also be +// adding to the number of tables issue. +table Source { +} + +// The union of possible build id parts (digest or label) of a BuildIdent +union Build { + Source, + EmbeddedSource, + BuildId, +} + +// Build option inheritance kinds +enum Inheritance: uint8 { + Weak = 0, + StrongForBuildOnly, + Strong, +} + +// Prerelease policy - makes None vs Some(Ex or Incl) clearer +enum PreReleasePolicy: uint8 { + None = 0, + ExcludeAll, + IncludeAll, + +} + +// A lone CompatRule, either an API or Binary, not a None. +enum LoneCompatRule: uint8 { + // Here only to indicate it is not set, i.e. would be None on the rust side + None = 0, + // These indicate Some(value) + API, + Binary, +} + +// A package option +table PkgOpt { + name: string (required); + components: [Component]; + prerelease_policy: PreReleasePolicy; + required_compat: LoneCompatRule; + value: string; + // ignored fields: default +} + +// A var option +table VarOpt { + name: string (required); + inheritance: Inheritance; + // compat - compat objects are stored as a string for now, however most + // build compat objects are the default value, which is not stored. + compat: string; + required: bool; + value: string; + // ignored fields: default, choices, description +} + +// Union of the kinds of options +union OptEnum { + PkgOpt, + VarOpt, +} + +// A helper for an option because vectors of unions are not allowed +table Opt { + opt: OptEnum; +} + +// The inclusion policy for a pkg request +enum InclusionPolicy: uint8 { + Always=0, + IfAlreadyPresent, +} + +// The pin policy for pkg request in install requirements +enum PinPolicy: uint8 { + Required=0, + IfPresentInBuildEnv, +} + +// An option value attached to a pkg request with options +table PkgRequestOptionValue { + name: string (required); + value: string; + // PkgRequestOptionValue has two enum variants: Complete and + // Partial. Both variants contain a single string. This uses a flag + // to decide between them rather than introducing another table, + // union, enum combination to record them. + is_complete: bool; +} + +// A package request with options +table PkgRequestWithOptions { + // from pkg_request: PkgRequest. In spk, a PkgRequest contains a + // 'pkg: RangeIdent' field, but the RangeIdent pieces have been + // flattened and included directly as: repo_name, name, + // components, version_filter, and build fields + name: string (required); + repo_name: string; + components: [Component]; + // The VersionFilters field is a set in spk, but it can contain + // lots of details and variations. So for now this is kept as a + // string, e.g. '>=1.2.3,<4,5,6' and parsed when needed. + // TODO: this will likely change in a future version because + // processing this is showing up in current profiling. + version_filter: string; + build: Build; + prerelease_policy: PreReleasePolicy; + inclusion_policy: InclusionPolicy; + pin: string; + pin_policy: PinPolicy; + // This is not a full compat, just a single CompatRule. Not sure + // why not it is not the same as the other required_compat objects in spk. + required_compat: LoneCompatRule; + options: [PkgRequestOptionValue]; + // requested_by - ignored +} + +// A var request with pinned value +table VarRequestPinnedValue { + name: string (required); + value: string; + // ignored fields: description +} + +// Union of the kinds of requests with options +union RequestWithOptions { + PkgRequestWithOptions, + VarRequestPinnedValue, +} + +// A helper for a requirement because list of unions are not allowed +table RequirementWithOptions { + request: RequestWithOptions; +} + +// A package embedded in a component of another package +table ComponentEmbeddedPackage { + // pkg - OptVersionIdent - split into name and version fields + name: string (required); + version: Version; + components: [Component]; +} + +// A cut down ComponentSpec with containing what is needed when solving +table SolverComponentSpec { + name: Component (required); + uses: [Component]; + requirements_with_options: [RequirementWithOptions]; + // embedded - only its components subfield is stored, the + // (embedded) fabricated field is ignored + embedded_components: [ComponentEmbeddedPackage]; + // ignored fields: files, requirements, file_match_mode +} + +// A cut down Spec for embedded packages containing what is needed when solving +table SolverEmbeddedPackageSpec { + // The embedded package's BuildIdent as a string + ident: string (required); + // meta - ignored + // compat - compat objects are stored as a string for now, however most + // build compat objects are the default value, which is not stored. + compat: string; + deprecated: bool; + // sources: Vec - ignored + // build: EmbeddedBuildSpec - mostly ignored, only build_options are kept + build_options: [Opt]; + // test: Vec - ignored + // install: EmbeddedInstallSpec - mostly ignored, flattened into only this field, + component_specs: [SolverComponentSpec]; + // requirements - field ignored in favour of install requirements with options, + // which are stored in the requirements field below + requirements: [RequirementWithOptions]; +} + +// A Package/Version/Build containing what is needed when solving +table BuildIndex { + // Packages may have been published with non-normalized version + // numbers, e.g. '1.0' instead of '1.0.0'. The index needs to + // store original published version number, from the recipe spec, + // in the build to make sure the exact build version numbers are + // available. + published_version: Version; + // The build digest/identifier only. Must be combined with a field + // from a PackageIndex and the published_version field above to + // make a complete BuildIdent value. + build: Build; + // If the build is deprecated the remaining fields will be empty, + // and this build index will not be any use for solving operations + // that need any of the remaining fields. + is_deprecated: bool; + // The spec is split up into multiple fields based on the parts the + // solver needs. There are many ignored fields. + // + // pkg - not stored, but its part are in the combo of PackageIndex, + // VersionIndex, and BuildIndex tables. + // meta - ignored + // compat - stored as a string for now, however most build compat objects + // are the default value, which is not stored. + compat: string; + // sources: SourceSpec - ignored, only needed when building from source + // build: BuildSpec - ignored, only build_options are stored, the + // other data is only needed if building. + build_options: [Opt]; + // tests: TestSpec - ignored, only needed if testing. + // install: InstallSpec - partially stored by splitting into parts, + // not the environment, not old requirements, + // and only parts of the various other sub-specs + embedded: [SolverEmbeddedPackageSpec]; + component_specs: [SolverComponentSpec]; + runtime_requirements: [RequirementWithOptions]; + // A list of component names of components actually published in + // the repo at time of indexing. This can be a subset of the names + // in the component_specs field (which come from the recipe and may + // not all be present in the repo at index time). + published_components: [Component]; +} + +// A version number +table Version { + parts: [uint32]; + epsilon: Epsilon; + pre: [TagSetItem]; + post: [TagSetItem]; +} + +// A Package/Version +table VersionIndex { + // The version number only + version: Version (required); + // Unsorted + builds: [BuildIndex]; +} + +// A Package +table PackageIndex { + // A PkgNameBuf, + name: string (required, key); + // Sorted highest to lowest + versions: [VersionIndex]; +} + +// For global var values +table GlobalVar { + // An OptNameBuf + name: string (required); + // The set of known values for this global variable + values: [string]; +} + +// The index for a repository +table RepositoryIndex { + // For spk to check it supports this schema + index_schema_version: uint32 = 1; + // Sorted by name + packages: [PackageIndex]; + global_vars: [GlobalVar]; +} + +root_type RepositoryIndex; diff --git a/crates/spk-proto/src/.gitignore b/crates/spk-proto/src/.gitignore new file mode 100644 index 0000000000..814620dd36 --- /dev/null +++ b/crates/spk-proto/src/.gitignore @@ -0,0 +1 @@ +spk_generated.rs diff --git a/crates/spk-proto/src/lib.rs b/crates/spk-proto/src/lib.rs new file mode 100644 index 0000000000..aeb6803b2b --- /dev/null +++ b/crates/spk-proto/src/lib.rs @@ -0,0 +1,10 @@ +// Copyright (c) Contributors to the SPK project. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/spkenv/spk + +#![allow(unused_imports)] +#![allow(unsafe_op_in_unsafe_fn)] +#![allow(mismatched_lifetime_syntaxes)] +#![allow(clippy::all)] + +include!(concat!(env!("OUT_DIR"), "/spk_generated.rs")); diff --git a/crates/spk-schema/Cargo.toml b/crates/spk-schema/Cargo.toml index 4f4b40e2a2..5a457193a1 100644 --- a/crates/spk-schema/Cargo.toml +++ b/crates/spk-schema/Cargo.toml @@ -15,10 +15,13 @@ workspace = true [features] [dependencies] +arc-swap = { workspace = true } +bytes = { workspace = true } config = { workspace = true } data-encoding = "2.3" dunce = { workspace = true } enum_dispatch = { workspace = true } +flatbuffers = { workspace = true } format_serde_error = { workspace = true, default-features = false, features = [ "serde_yaml", "colored", @@ -39,6 +42,7 @@ serde_yaml = { workspace = true } shellexpand = "3.1.0" spfs = { workspace = true } spk-config = { workspace = true } +spk-proto = { workspace = true } spk-schema-foundation = { workspace = true } spk-schema-tera = { workspace = true } strum = { workspace = true } diff --git a/crates/spk-schema/crates/foundation/Cargo.toml b/crates/spk-schema/crates/foundation/Cargo.toml index 7c444c1e21..8819c43aa9 100644 --- a/crates/spk-schema/crates/foundation/Cargo.toml +++ b/crates/spk-schema/crates/foundation/Cargo.toml @@ -20,10 +20,12 @@ parsedbuf-serde = [] [dependencies] arc-swap = { workspace = true } async-trait = { workspace = true } +bytes = { workspace = true } colored = { workspace = true } data-encoding = "2.3" derive-where = { workspace = true } enum_dispatch = { workspace = true } +flatbuffers = { workspace = true } format_serde_error = { workspace = true, default-features = false, features = [ "serde_yaml", "colored", @@ -43,6 +45,7 @@ rstest = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_yaml = { workspace = true } serial_test = { workspace = true } +spk-proto = { workspace = true } spfs = { workspace = true } strum = { workspace = true, features = ["derive"] } sys-info = "0.9.0" diff --git a/crates/spk-schema/src/error.rs b/crates/spk-schema/src/error.rs index b9b6d57e95..40a205bf1a 100644 --- a/crates/spk-schema/src/error.rs +++ b/crates/spk-schema/src/error.rs @@ -71,6 +71,8 @@ pub enum Error { #[error(transparent)] #[diagnostic(forward(0))] SpkConfigError(#[from] spk_config::Error), + #[error("{0}'s {1}() method not implemented for IndexedPackage")] + SpkIndexedPackageDoesNotImplement(String, String), } impl Error { diff --git a/crates/spk-schema/src/fb_converter.rs b/crates/spk-schema/src/fb_converter.rs new file mode 100644 index 0000000000..2a65aecf63 --- /dev/null +++ b/crates/spk-schema/src/fb_converter.rs @@ -0,0 +1,1415 @@ +// Copyright (c) Contributors to the SPK project. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/spkenv/spk + +use std::collections::{BTreeMap, BTreeSet}; +use std::str::FromStr; +use std::sync::Arc; + +use spk_schema_foundation::IsDefault; +use spk_schema_foundation::ident::{ + BuildIdent, + InclusionPolicy, + OptVersionIdent, + PinPolicy, + PkgRequest, + PkgRequestOptionValue, + PkgRequestOptions, + PkgRequestWithOptions, + PreReleasePolicy, + RangeIdent, + RequestWithOptions, +}; +use spk_schema_foundation::ident_build::{Build, EmbeddedSource, EmbeddedSourcePackage}; +use spk_schema_foundation::ident_component::{Component, ComponentBTreeSetBuf}; +use spk_schema_foundation::ident_ops::parsing::NormalizedVersionString; +use spk_schema_foundation::name::{OptNameBuf, PkgNameBuf, RepositoryNameBuf}; +use spk_schema_foundation::option_map::OptionMap; +use spk_schema_foundation::spec_ops::FileMatcher; +use spk_schema_foundation::version_range::VersionFilter; + +use crate::component_embedded_packages::ComponentEmbeddedPackage; +use crate::deprecate::Deprecate; +use crate::ident_build::BuildId; +use crate::ident_ops::parsing::IdentPartsBuf; +use crate::option::{PkgOpt, VarOpt}; +use crate::package::{BuildOptions, Components}; +use crate::v0::{EmbeddedBuildSpec, EmbeddedInstallSpec, EmbeddedPackageSpec}; +use crate::version::{Compat, CompatRule, Epsilon, TagSet, Version, VersionParts}; +use crate::{ + ComponentEmbeddedPackagesList, + ComponentSpec, + ComponentSpecList, + EmbeddedPackagesList, + Inheritance, + Opt, + RequirementsList, +}; + +/// Helper for turning a Vec into to flatbuffer Some(list) or None, if +/// the list is empty. +#[macro_export] +macro_rules! flatbuffer_vector { + ($builder:ident, $list:expr) => { + if $list.is_empty() { + None + } else { + Some($builder.create_vector(&$list)) + } + }; +} + +// Flatbuffer conversions functions. Kept together for now. In future, +// they may be split up across objects and traits. + +#[inline] +pub fn fb_prerelease_policy_to_prerelease_policy( + fb_prerelease_policy: spk_proto::PreReleasePolicy, +) -> Option { + match fb_prerelease_policy { + spk_proto::PreReleasePolicy::None => None, + spk_proto::PreReleasePolicy::ExcludeAll => Some(PreReleasePolicy::ExcludeAll), + spk_proto::PreReleasePolicy::IncludeAll => Some(PreReleasePolicy::IncludeAll), + _ => { + // Covering up to ::MAX for the compiler, but this should not happen + debug_assert!( + false, + "Unhandled spk_proto::PreReleasePolicy enum number encountered" + ); + None + } + } +} + +#[inline] +pub fn fb_inclusion_policy_to_inclusion_policy( + fb_inclusion_policy: spk_proto::InclusionPolicy, +) -> InclusionPolicy { + match fb_inclusion_policy { + spk_proto::InclusionPolicy::Always => InclusionPolicy::Always, + spk_proto::InclusionPolicy::IfAlreadyPresent => InclusionPolicy::IfAlreadyPresent, + _ => { + // Covering up to ::MAX for the compiler, but this should not happen + debug_assert!( + false, + "Unhandled spk_proto::InclusionPolicy enum number encountered" + ); + InclusionPolicy::default() + } + } +} + +#[inline] +pub fn fb_lone_compat_rule_to_lone_compat_rule( + fb_compat_rule: spk_proto::LoneCompatRule, +) -> Option { + match fb_compat_rule { + spk_proto::LoneCompatRule::API => Some(CompatRule::API), + spk_proto::LoneCompatRule::Binary => Some(CompatRule::Binary), + // Includes spk_proto::LoneCompatRule::None, which means no + // CompatRule was present. + _ => None, + } +} + +#[inline] +pub fn fb_component_to_component(fb_component: &spk_proto::Component) -> Component { + match fb_component.kind() { + spk_proto::ComponentEnum::All => Component::All, + spk_proto::ComponentEnum::Build => Component::Build, + spk_proto::ComponentEnum::Run => Component::Run, + spk_proto::ComponentEnum::Source => Component::Source, + spk_proto::ComponentEnum::Named => { + // Named enum variant means there will be a value in the + // component's name field. + Component::Named( + fb_component + .name() + .expect("A ComponentEnum::Named in flatbuffer data should not have None in its name field") + .to_string(), + ) + } + _ => { + // Covering up to ::MAX for the compiler, but this should not happen + debug_assert!( + false, + "Unhandled spk_proto::ComponentEnum enum number encountered" + ); + Component::Run + } + } +} + +#[inline] +pub fn fb_component_names_to_component_names( + fb_component_names: &flatbuffers::Vector< + '_, + flatbuffers::ForwardsUOffset>, + >, +) -> Vec { + fb_component_names + .iter() + .map(|c| fb_component_to_component(&c)) + .collect() +} + +#[inline] +pub fn fb_component_names_to_component_names_set( + fb_component_names: &Option< + flatbuffers::Vector<'_, flatbuffers::ForwardsUOffset>>, + >, +) -> BTreeSet { + if let Some(cs) = fb_component_names { + cs.iter().map(|c| fb_component_to_component(&c)).collect() + } else { + Default::default() + } +} + +#[inline] +pub fn fb_component_specs_to_component_names( + fb_components: &flatbuffers::Vector< + '_, + flatbuffers::ForwardsUOffset>, + >, +) -> Vec { + fb_components + .iter() + .map(|c| fb_component_to_component(&c.name())) + .collect() +} + +#[inline] +pub fn fb_component_specs_to_component_name_set( + fb_components: &flatbuffers::Vector< + '_, + flatbuffers::ForwardsUOffset>, + >, +) -> BTreeSet { + fb_components + .iter() + .map(|c| fb_component_to_component(&c.name())) + .collect() +} + +pub fn fb_requirements_to_requirements( + requirements_with_options: Option< + flatbuffers::Vector< + '_, + flatbuffers::ForwardsUOffset>, + >, + >, +) -> Vec { + let mut requirements = Vec::new(); + + if let Some(fb_reqs) = requirements_with_options { + for fb_req in fb_reqs.iter() { + match fb_req.request_type() { + spk_proto::RequestWithOptions::VarRequestPinnedValue => { + if let Some(fb_var_req) = fb_req.request_as_var_request_pinned_value() + && let Some(value) = fb_var_req.value() + { + let name = + unsafe { OptNameBuf::from_string(fb_var_req.name().to_string()) }; + let var_req = + spk_schema_foundation::ident::VarRequest::new_with_value(name, value); + + requirements.push(RequestWithOptions::Var(var_req)) + } + } + spk_proto::RequestWithOptions::PkgRequestWithOptions => { + if let Some(fb_pkg_req) = fb_req.request_as_pkg_request_with_options() { + let repo_name = if let Some(n) = fb_pkg_req.repo_name() { + let name = unsafe { RepositoryNameBuf::from_string(n.to_string()) }; + Some(name) + } else { + None + }; + + let ident_name = + unsafe { PkgNameBuf::from_string(fb_pkg_req.name().to_string()) }; + + let components = + fb_component_names_to_component_names_set(&fb_pkg_req.components()); + + // TODO: this will be the next thing to get a + // proper flatbuffer representation because it + // is now showing up as the next significant + // chunk on the large solve flamegraphs + let version_filter = + fb_version_filter_to_version_filter(fb_pkg_req.version_filter()); + + let build = get_build_from_fb_pkg_request_with_options(&fb_pkg_req); + + let prerelease_policy = fb_prerelease_policy_to_prerelease_policy( + fb_pkg_req.prerelease_policy(), + ); + let inclusion_policy = + fb_inclusion_policy_to_inclusion_policy(fb_pkg_req.inclusion_policy()); + + let pin = fb_pin_to_pin(&fb_pkg_req.pin()); + let pin_policy = fb_pin_policy_to_pin_policy(fb_pkg_req.pin_policy()); + + let required_compat = + fb_lone_compat_rule_to_lone_compat_rule(fb_pkg_req.required_compat()); + + let range_ident = RangeIdent { + repository_name: repo_name, + name: ident_name, + components, + version: version_filter, + build, + }; + + let pkg_request = PkgRequest { + pkg: range_ident, + prerelease_policy, + inclusion_policy, + pin, + pin_policy, + required_compat, + requested_by: Default::default(), + }; + + let options = fb_pkg_request_option_values_to_pkg_request_options( + fb_pkg_req.options(), + ); + + let pkg_req = PkgRequestWithOptions { + pkg_request, + options, + }; + + requirements.push(RequestWithOptions::Pkg(pkg_req)); + } + } + _ => { + // Covering up to ::MAX for the compiler, but this should not happen + debug_assert!( + false, + "Unhandled spk_proto::RequestWithOptions enum number encountered" + ); + } + }; + } + } + + requirements +} + +pub fn fb_component_specs_to_component_specs( + fb_c_specs: &flatbuffers::Vector< + '_, + flatbuffers::ForwardsUOffset>, + >, + build_options: &OptionMap, +) -> ComponentSpecList { + let mut component_specs = Vec::new(); + + for c_spec in fb_c_specs.iter() { + let fb_component = c_spec.name(); + let component_name = fb_component_to_component(&fb_component); + + let uses = if let Some(fb_uses) = c_spec.uses() { + fb_component_names_to_component_names(&fb_uses) + } else { + Vec::new() + }; + + let component_requirements = + fb_requirements_to_requirements(c_spec.requirements_with_options()); + let component_reqs_list = + unsafe { RequirementsList::::new_checked(component_requirements) } + .into(); + + let embedded = fb_component_emb_pkgs_to_component_emb_pkgs(c_spec.embedded_components()); + let component_embedded_packages: ComponentEmbeddedPackagesList = + embedded.into_iter().into(); + + // This value of this doesn't matter for solving but + // helps avoid false mismatches in tests + let file_matcher = if component_name == Component::Build || component_name == Component::Run + { + // A single '*' rule + FileMatcher::all() + } else { + // Empty of rules + FileMatcher::default() + }; + + let component_spec = unsafe { + ComponentSpec::new_unchecked( + component_name, + uses, + file_matcher, + build_options, + component_reqs_list, + component_embedded_packages, + ) + }; + + component_specs.push(component_spec); + } + + ComponentSpecList::::new(component_specs) +} + +pub fn fb_embedded_package_specs_to_embedded_package_specs( + fb_embedded: &flatbuffers::Vector< + '_, + flatbuffers::ForwardsUOffset>, + >, +) -> EmbeddedPackagesList { + let mut embedded_package_specs = EmbeddedPackagesList::default(); + + for fb_emb_spec in fb_embedded.iter() { + let build_ident = BuildIdent::from_str(fb_emb_spec.ident()).unwrap_or_else(|_| unreachable!("An Embedded package spec in flatbuffer data should have a valid ident: '{}' is not valid", fb_emb_spec.ident())); + + let build_options = fb_opts_to_opts(fb_emb_spec.build_options()); + let options: OptionMap = build_options + .iter() + .map(|opt| (opt.full_name().to_owned(), opt.get_value(None))) + .collect(); + + let component_specs = if let Some(fb_c_specs) = fb_emb_spec.component_specs() { + fb_component_specs_to_component_specs(&fb_c_specs, &options) + } else { + Default::default() + }; + + let requirements = fb_requirements_to_requirements(fb_emb_spec.requirements()); + + let embedded_spec = unsafe { + EmbeddedPackageSpec::new_unchecked( + build_ident, + EmbeddedBuildSpec { + options: build_options, + }, + RequirementsList::::new_checked(requirements), + EmbeddedInstallSpec { + // TODO: does this need to be kept, they're the old data format?? + requirements: RequirementsList::default(), + components: component_specs, + }, + ) + }; + + embedded_package_specs.push(embedded_spec) + } + embedded_package_specs +} + +// Note: fb_compat objects in packages are not optional in the rust +// structs, but fb_compat's stored in var opts are optional in the +// rust struct. +#[inline] +fn var_opt_fb_compat_to_var_opt_compat(fb_compat: Option<&str>) -> Option { + // None stored as an fb_compat represents no compat specified at + // all for a var opt. Some compat stored means a compat was + // specified, and it may even be the same as the default compat. + fb_compat.map(|fc| { + Compat::new_from_compat_str(fc) + .expect("A Compat in flatbuffer data should be a valid Compat when parsed") + }) +} + +#[inline] +pub fn fb_compat_to_compat(fb_compat: Option<&str>) -> Compat { + if let Some(compat) = fb_compat { + Compat::new_from_compat_str(compat) + .expect("A Compat in flatbuffer data should be a valid Compat when parsed") + } else { + // In this case, None, so nothing, stored as an fb_compat + // represents the default compat. This is different to the var + // opt compat method above. + Compat::default() + } +} + +#[inline] +pub fn fb_pin_to_pin(pin: &Option<&str>) -> Option { + pin.as_ref().map(|p| p.to_string()) +} + +#[inline] +pub fn fb_pin_policy_to_pin_policy(pin_policy: spk_proto::PinPolicy) -> PinPolicy { + match pin_policy { + spk_proto::PinPolicy::Required => PinPolicy::Required, + spk_proto::PinPolicy::IfPresentInBuildEnv => PinPolicy::IfPresentInBuildEnv, + _ => { + // Covering up to ::MAX for the compiler, but this should not happen + debug_assert!( + false, + "Unhandled spk_proto::PinPolicy enum number encountered" + ); + PinPolicy::Required + } + } +} + +#[inline] +fn fb_value_to_value(fb_value: Option<&str>) -> Option { + fb_value.map(|v| v.to_string()) +} + +pub fn fb_pkg_opt_to_opt(fb_pkg_opt: spk_proto::PkgOpt) -> Opt { + let name = unsafe { PkgNameBuf::from_string(fb_pkg_opt.name().to_string()) }; + + let components = if let Some(fb_components) = fb_pkg_opt.components() { + ComponentBTreeSetBuf::from(fb_components.iter().map(|c| fb_component_to_component(&c))) + } else { + Default::default() + }; + + let prerelease_policy = + fb_prerelease_policy_to_prerelease_policy(fb_pkg_opt.prerelease_policy()); + + let required_compat = fb_lone_compat_rule_to_lone_compat_rule(fb_pkg_opt.required_compat()); + + let value = fb_value_to_value(fb_pkg_opt.value()); + + let po = unsafe { + PkgOpt::new_unchecked(name, components, prerelease_policy, value, required_compat) + }; + + Opt::Pkg(po) +} + +pub fn fb_var_opt_to_opt(fb_var_opt: spk_proto::VarOpt) -> Opt { + let inheritance = match fb_var_opt.inheritance() { + spk_proto::Inheritance::Weak => Inheritance::Weak, + spk_proto::Inheritance::StrongForBuildOnly => Inheritance::StrongForBuildOnly, + spk_proto::Inheritance::Strong => Inheritance::Strong, + _ => { + // Covering up to ::MAX for the compiler, but this should not happen + debug_assert!( + false, + "Unhandled spk_proto::Inheritance enum number encountered" + ); + Inheritance::default() + } + }; + + let compat = var_opt_fb_compat_to_var_opt_compat(fb_var_opt.compat()); + + let required = fb_var_opt.required(); + + let value = fb_value_to_value(fb_var_opt.value()); + + let vo = + unsafe { VarOpt::new_unchecked(fb_var_opt.name(), inheritance, compat, required, value) }; + + Opt::Var(vo) +} + +#[inline] +pub fn fb_opt_to_opt(fb_opt: &spk_proto::Opt<'_>) -> Opt { + match fb_opt.opt_type() { + spk_proto::OptEnum::PkgOpt => { + if let Some(fb_pkg_opt) = fb_opt.opt_as_pkg_opt() { + fb_pkg_opt_to_opt(fb_pkg_opt) + } else { + unreachable!( + "The Pkg Opt flatbuffer data should not be None when OptEnum::PkgOpt is set" + ); + } + } + spk_proto::OptEnum::VarOpt => { + if let Some(fb_var_opt) = fb_opt.opt_as_var_opt() { + fb_var_opt_to_opt(fb_var_opt) + } else { + unreachable!( + "The Var Opt flatbuffer data should not be None when OptEnum::VarOpt is set" + ); + } + } + _ => { + // Covering up to ::MAX for the compiler, but this should not happen + debug_assert!( + false, + "Unhandled spk_proto::OptEnum enum number encountered" + ); + unreachable!( + "The opt_type flatbuffer data should be either OptEnum::PkgOpt or OptEnum::VarOpt" + ); + } + } +} + +#[inline] +pub fn fb_opts_to_opts( + fb_opts: Option>>>, +) -> Vec { + if let Some(opts) = fb_opts { + opts.iter().map(|fb_opt| fb_opt_to_opt(&fb_opt)).collect() + } else { + Default::default() + } +} + +fn fb_build_as_embedded_source_to_build(build: spk_proto::EmbeddedSource) -> Build { + let es = if let Some(esp) = build.source() { + // This a known embedded source package + let id = esp.ident(); + let ident = IdentPartsBuf { + repository_name: id.repository_name().map(String::from), + pkg_name: String::from(id.pkg_name()), + version_str: id.version_str().map(|vs| { + // Should be safe as the data was a normalized + // version string before it was stored. + unsafe { NormalizedVersionString::new_unchecked(vs.to_string()) } + }), + build_str: id.build_str().map(String::from), + }; + + let components = fb_component_names_to_component_names_set(&esp.components()); + let unparsed = None; + + EmbeddedSource::Package(Box::new(EmbeddedSourcePackage { + ident, + components, + unparsed, + })) + } else { + EmbeddedSource::Unknown + }; + + Build::Embedded(es) +} + +// Almost the same as the next method, but the build result can be +// optional here +#[inline] +pub fn get_build_from_fb_pkg_request_with_options( + pkg_req: &spk_proto::PkgRequestWithOptions, +) -> Option { + // First, handle the case when there is no build in the request + pkg_req.build()?; + + // and then the case where there is a build in the request + let b = match pkg_req.build_type() { + spk_proto::Build::Source => Build::Source, + spk_proto::Build::EmbeddedSource => { + let build = pkg_req.build_as_embedded_source().expect("PkgRequestWithOptions in flatbuffer data should contain an embedded source build when build_type is set to EmbeddedSource"); + fb_build_as_embedded_source_to_build(build) + } + spk_proto::Build::BuildId => { + let build = pkg_req.build_as_build_id().expect( + "PkgRequestWithOptions in flatbuffer data should contain a build id when build_type is set to BuildId", + ); + Build::BuildId(unsafe { + BuildId::new_unchecked(build.id().chars().collect::>()) + }) + } + _ => { + // Covering up to ::MAX for the compiler, but this should not happen + debug_assert!( + false, + "Unhandled spk_proto::BuildId enum number encountered" + ); + Build::Source + } + }; + + Some(b) +} + +// Almost the same as the previous method, but the build is not optional +#[inline] +pub fn get_build_from_fb_build_index(build_index: spk_proto::BuildIndex) -> Build { + // A build index will always have a build value + match build_index.build_type() { + spk_proto::Build::Source => Build::Source, + spk_proto::Build::EmbeddedSource => { + let build = build_index.build_as_embedded_source().expect("A BuildIndex in flatbuffer data should contain an embedded source build when build_type is set to EmbeddedSource"); + fb_build_as_embedded_source_to_build(build) + } + spk_proto::Build::BuildId => { + let build = build_index.build_as_build_id().expect( + "A BuildIndex in flatbuffer data should contain a build id when build_type is set to BuildId", + ); + Build::BuildId(unsafe { + BuildId::new_unchecked(build.id().chars().collect::>()) + }) + } + _ => { + // Covering up to ::MAX for the compiler, but this should not happen + debug_assert!( + false, + "Unhandled spk_proto::BuildId enum number encountered" + ); + Build::Source + } + } +} + +#[inline] +pub fn fb_version_to_version(ver: spk_proto::Version) -> Version { + let parts = VersionParts { + parts: match ver.parts() { + Some(numbers) => numbers.iter().collect(), + None => Vec::new(), + }, + epsilon: match ver.epsilon() { + spk_proto::Epsilon::Minus => Epsilon::Minus, + spk_proto::Epsilon::None => Epsilon::None, + spk_proto::Epsilon::Plus => Epsilon::Plus, + _ => { + // Covering up to ::MAX for the compiler, but this should not happen + debug_assert!( + false, + "Unhandled spk_proto::Epsilon enum number encountered" + ); + Epsilon::None + } + }, + }; + let mut pre = BTreeMap::new(); + if let Some(tags) = ver.pre() { + for tag_set_item in tags { + pre.insert(tag_set_item.name().to_string(), tag_set_item.number()); + } + } + let mut post = BTreeMap::new(); + if let Some(tags) = ver.post() { + for tag_set_item in tags { + post.insert(tag_set_item.name().to_string(), tag_set_item.number()); + } + } + + Version { + parts, + pre: TagSet { tags: pre }, + post: TagSet { tags: post }, + } +} + +pub fn fb_component_emb_pkgs_to_component_emb_pkgs( + component_embedded_packages: Option< + flatbuffers::Vector<'_, flatbuffers::ForwardsUOffset>, + >, +) -> Vec { + if let Some(fb_comp_embedded_pkgs) = component_embedded_packages { + fb_comp_embedded_pkgs + .iter() + .map(|component_embedded_pkg| { + // Recombine name and version from this flatbuffer object make an ident + let name = + unsafe { PkgNameBuf::from_string(component_embedded_pkg.name().to_string()) }; + let version = component_embedded_pkg + .version() + .map(|v| fb_version_to_version(v)); + let ident = OptVersionIdent::new(name, version); + + let components = + fb_component_names_to_component_names_set(&component_embedded_pkg.components()); + + unsafe { ComponentEmbeddedPackage::new_unchecked(ident, components) } + }) + .collect() + // TODO: This does not set fabricated, not sure if that is a problem or not? + } else { + Vec::new() + } +} + +pub fn fb_pkg_request_option_values_to_pkg_request_options( + pr_options: Option< + flatbuffers::Vector>, + >, +) -> PkgRequestOptions { + let mut options = PkgRequestOptions::new(); + + if let Some(fb_options) = pr_options { + for fb_opt in fb_options.iter() { + let name = unsafe { OptNameBuf::from_string(fb_opt.name().to_string()) }; + + let opt_value = if let Some(v) = fb_opt.value() { + v.to_string() + } else { + // This should not happen because the option should not + // have been saved without a value. + "".to_string() + }; + + let value = if fb_opt.is_complete() { + PkgRequestOptionValue::Complete(opt_value) + } else { + PkgRequestOptionValue::Partial(opt_value) + }; + + options.insert(name, value); + } + } + + options +} + +// TODO: this will change if the version filter gets a proper +// flatbuffer representation to move it away from a string. This look +// like it is advisable to do in future, based on current profiling. +#[inline] +pub fn fb_version_filter_to_version_filter(version_filter: Option<&str>) -> VersionFilter { + if let Some(filter_string) = version_filter { + unsafe { VersionFilter::new_unchecked(filter_string) } + } else { + Default::default() + } +} + +#[inline] +fn component_to_fb_component<'a>( + builder: &mut flatbuffers::FlatBufferBuilder<'a>, + c: &Component, +) -> flatbuffers::WIPOffset> { + let args = match c { + Component::All => spk_proto::ComponentArgs { + kind: spk_proto::ComponentEnum::All, + name: None, + }, + Component::Build => spk_proto::ComponentArgs { + kind: spk_proto::ComponentEnum::Build, + name: None, + }, + Component::Run => spk_proto::ComponentArgs { + kind: spk_proto::ComponentEnum::Run, + name: None, + }, + Component::Source => spk_proto::ComponentArgs { + kind: spk_proto::ComponentEnum::Source, + name: None, + }, + Component::Named(name) => { + let fb_name = builder.create_string(name); + spk_proto::ComponentArgs { + kind: spk_proto::ComponentEnum::Named, + name: Some(fb_name), + } + } + }; + spk_proto::Component::create(builder, &args) +} + +// TODO: change to Iterator generic 1 of 3 +#[inline] +pub fn components_to_fb_components<'a>( + builder: &mut flatbuffers::FlatBufferBuilder<'a>, + components: &[Component], +) -> Option< + flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, +> { + let comps: Vec<_> = components + .iter() + .map(|c| component_to_fb_component(builder, c)) + .collect(); + + flatbuffer_vector!(builder, comps) +} + +// TODO: change to Iterator generic 2 of 3 +#[inline] +fn components_setbuf_to_fb_components<'a>( + builder: &mut flatbuffers::FlatBufferBuilder<'a>, + components: &ComponentBTreeSetBuf, +) -> Option< + flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, +> { + let comps: Vec<_> = components + .iter() + .map(|c| component_to_fb_component(builder, c)) + .collect(); + + flatbuffer_vector!(builder, comps) +} + +// TODO: change to Iterator generic 3 of 3 +#[inline] +fn components_set_to_fb_components<'a>( + builder: &mut flatbuffers::FlatBufferBuilder<'a>, + components: &BTreeSet, +) -> Option< + flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, +> { + let comps: Vec<_> = components + .iter() + .map(|c| component_to_fb_component(builder, c)) + .collect(); + + flatbuffer_vector!(builder, comps) +} + +#[inline] +fn lone_compat_rule_to_fb_lone_compat_rule( + optional_compat_rule: Option, +) -> spk_proto::LoneCompatRule { + match optional_compat_rule { + Some(compat_rule) => match compat_rule { + CompatRule::API => spk_proto::LoneCompatRule::API, + CompatRule::Binary => spk_proto::LoneCompatRule::Binary, + // CompatRule::None is not allowed in a lone compat rules + // used inside spk, but it still needs to be recorded in + // the index's enum. + CompatRule::None => spk_proto::LoneCompatRule::None, + }, + None => spk_proto::LoneCompatRule::None, + } +} + +#[inline] +fn prerelease_policy_to_fb_prerelease_policy( + optional_prerelease_policy: Option, +) -> spk_proto::PreReleasePolicy { + match optional_prerelease_policy { + Some(policy) => match policy { + PreReleasePolicy::ExcludeAll => spk_proto::PreReleasePolicy::ExcludeAll, + PreReleasePolicy::IncludeAll => spk_proto::PreReleasePolicy::IncludeAll, + }, + None => spk_proto::PreReleasePolicy::None, + } +} + +#[inline] +fn inclusion_policy_to_fb_inclusion_policy( + inclusion_policy: InclusionPolicy, +) -> spk_proto::InclusionPolicy { + match inclusion_policy { + InclusionPolicy::Always => spk_proto::InclusionPolicy::Always, + InclusionPolicy::IfAlreadyPresent => spk_proto::InclusionPolicy::IfAlreadyPresent, + } +} + +#[inline] +fn pin_to_fb_pin<'a>( + builder: &mut flatbuffers::FlatBufferBuilder<'a>, + pin: &Option, +) -> Option> { + if let Some(p) = pin { + let fb_pin = builder.create_string(p); + Some(fb_pin) + } else { + None + } +} + +#[inline] +fn pin_policy_to_fb_pin_policy(pin_policy: PinPolicy) -> spk_proto::PinPolicy { + match pin_policy { + PinPolicy::Required => spk_proto::PinPolicy::Required, + PinPolicy::IfPresentInBuildEnv => spk_proto::PinPolicy::IfPresentInBuildEnv, + } +} + +fn opts_to_fb_pkg_request_option_values<'a>( + builder: &mut flatbuffers::FlatBufferBuilder<'a>, + pr_options: &PkgRequestOptions, +) -> Option< + flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, +> { + let mut fb_options = Vec::new(); + + for (name, value) in pr_options.iter() { + let fb_name = builder.create_string(name); + let fb_value = match value { + PkgRequestOptionValue::Complete(val) => builder.create_string(val), + PkgRequestOptionValue::Partial(val) => builder.create_string(val), + }; + let is_complete = matches!(*value, PkgRequestOptionValue::Complete(_)); + + let fb_option_value = spk_proto::PkgRequestOptionValue::create( + builder, + &spk_proto::PkgRequestOptionValueArgs { + name: Some(fb_name), + value: Some(fb_value), + is_complete, + }, + ); + + fb_options.push(fb_option_value) + } + + flatbuffer_vector!(builder, fb_options) +} + +pub fn opts_to_fb_opts<'a>( + builder: &mut flatbuffers::FlatBufferBuilder<'a>, + options: &[Opt], +) -> Option< + flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, +> { + let mut fb_options = Vec::new(); + for opt in options { + let n = opt.full_name(); + let v = opt.get_value(None); + + let fb_name = builder.create_string(n); + let fb_value = builder.create_string(&v); + + let (fb_opt, fb_opt_type) = match opt { + Opt::Pkg(pkg_opt) => { + let fb_components = + components_setbuf_to_fb_components(builder, &pkg_opt.components); + // The pkg_opt.default field is ignored and not stored + // in the flatbuffer data. + let fb_prerelease_policy = + prerelease_policy_to_fb_prerelease_policy(pkg_opt.prerelease_policy); + let required_compat = + lone_compat_rule_to_fb_lone_compat_rule(pkg_opt.required_compat); + + let fb_opt = spk_proto::PkgOpt::create( + builder, + &spk_proto::PkgOptArgs { + name: Some(fb_name), + components: fb_components, + prerelease_policy: fb_prerelease_policy, + required_compat, + value: Some(fb_value), + }, + ) + .as_union_value(); + + (fb_opt, spk_proto::OptEnum::PkgOpt) + } + Opt::Var(var_opt) => { + let inheritance = match var_opt.inheritance() { + Inheritance::Weak => spk_proto::Inheritance::Weak, + Inheritance::StrongForBuildOnly => spk_proto::Inheritance::StrongForBuildOnly, + Inheritance::Strong => spk_proto::Inheritance::Strong, + }; + + let fb_compat = var_opt_compat_to_var_opt_fb_compat(builder, &var_opt.compat); + let required = var_opt.required; + + let fb_opt = spk_proto::VarOpt::create( + builder, + &spk_proto::VarOptArgs { + name: Some(fb_name), + inheritance, + compat: fb_compat, + required, + value: Some(fb_value), + }, + ) + .as_union_value(); + (fb_opt, spk_proto::OptEnum::VarOpt) + } + }; + + let fb_opt = spk_proto::Opt::create( + builder, + &spk_proto::OptArgs { + opt: Some(fb_opt), + opt_type: fb_opt_type, + }, + ); + + fb_options.push(fb_opt); + } + + flatbuffer_vector!(builder, fb_options) +} + +fn component_emb_pkgs_to_fb_component_emb_pkgs<'a>( + builder: &mut flatbuffers::FlatBufferBuilder<'a>, + component_emb_pkgs: &ComponentEmbeddedPackagesList, +) -> Option< + flatbuffers::WIPOffset< + flatbuffers::Vector< + 'a, + flatbuffers::ForwardsUOffset>, + >, + >, +> { + let mut comp_emb_pkgs = Vec::new(); + + for emb_comp in component_emb_pkgs.iter() { + let fb_name = builder.create_string(emb_comp.pkg.name()); + + let fb_version = emb_comp + .pkg + .target() + .as_ref() + .map(|ver| version_to_fb_version(builder, ver)); + + let fb_components = components_set_to_fb_components(builder, emb_comp.components()); + + let fb_emb_comp = spk_proto::ComponentEmbeddedPackage::create( + builder, + &spk_proto::ComponentEmbeddedPackageArgs { + name: Some(fb_name), + version: fb_version, + components: fb_components, + }, + ); + + comp_emb_pkgs.push(fb_emb_comp); + } + + flatbuffer_vector!(builder, comp_emb_pkgs) +} + +pub fn component_specs_to_fb_component_specs<'a>( + builder: &mut flatbuffers::FlatBufferBuilder<'a>, + component_specs: &ComponentSpecList, +) -> Option< + flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, +> { + let mut fb_component_specs = Vec::new(); + + for cs in component_specs.iter() { + let fb_component_name = component_to_fb_component(builder, &cs.name); + + let fb_uses = components_to_fb_components(builder, &cs.uses); + + let fb_requirements = requirements_with_options_to_fb_requirements_with_options( + builder, + cs.requirements_with_options(), + ); + + let fb_comp_emb_pkgs = component_emb_pkgs_to_fb_component_emb_pkgs(builder, &cs.embedded); + + let fb_comp_spec = spk_proto::SolverComponentSpec::create( + builder, + &spk_proto::SolverComponentSpecArgs { + name: Some(fb_component_name), + uses: fb_uses, + requirements_with_options: fb_requirements, + embedded_components: fb_comp_emb_pkgs, + }, + ); + fb_component_specs.push(fb_comp_spec); + } + + flatbuffer_vector!(builder, fb_component_specs) +} + +pub fn embedded_pkg_specs_to_fb_embedded_package_specs<'a>( + builder: &mut flatbuffers::FlatBufferBuilder<'a>, + embedded: Arc>, +) -> Option< + flatbuffers::WIPOffset< + flatbuffers::Vector< + 'a, + flatbuffers::ForwardsUOffset>, + >, + >, +> { + let mut fb_embedded_specs = Vec::new(); + + for emb_spec in embedded.iter() { + let fb_emb_ident = builder.create_string(&format!("{:#}", emb_spec.ident())); + + let fb_emb_compat = compat_to_fb_compat(builder, &emb_spec.compat); + let fb_emb_build_options = opts_to_fb_opts(builder, &emb_spec.build_options()); + + let fb_emb_requirements = requirements_with_options_to_fb_requirements_with_options( + builder, + emb_spec.install_requirements_with_options(), + ); + + let fb_emb_component_specs = + component_specs_to_fb_component_specs(builder, &emb_spec.components()); + + let fb_emb_pkg = spk_proto::SolverEmbeddedPackageSpec::create( + builder, + &spk_proto::SolverEmbeddedPackageSpecArgs { + ident: Some(fb_emb_ident), + compat: fb_emb_compat, + deprecated: emb_spec.is_deprecated(), + build_options: fb_emb_build_options, + requirements: fb_emb_requirements, + component_specs: fb_emb_component_specs, + }, + ); + + fb_embedded_specs.push(fb_emb_pkg); + } + + flatbuffer_vector!(builder, fb_embedded_specs) +} + +pub fn requirements_with_options_to_fb_requirements_with_options<'a>( + builder: &mut flatbuffers::FlatBufferBuilder<'a>, + requirements: &RequirementsList, +) -> Option< + flatbuffers::WIPOffset< + flatbuffers::Vector< + 'a, + flatbuffers::ForwardsUOffset>, + >, + >, +> { + let mut reqs = Vec::new(); + + for req in requirements.iter() { + match req { + RequestWithOptions::Var(vr) => { + let fb_name = builder.create_string(&vr.var); + let fb_value = builder.create_string(&vr.value); + let fb_var_req = spk_proto::VarRequestPinnedValue::create( + builder, + &spk_proto::VarRequestPinnedValueArgs { + name: Some(fb_name), + value: Some(fb_value), + }, + ) + .as_union_value(); + + let fb_req = spk_proto::RequirementWithOptions::create( + builder, + &spk_proto::RequirementWithOptionsArgs { + request: Some(fb_var_req), + request_type: spk_proto::RequestWithOptions::VarRequestPinnedValue, + }, + ); + + reqs.push(fb_req); + } + RequestWithOptions::Pkg(pr) => { + let fb_repo_name = pr + .pkg + .repository_name + .as_ref() + .map(|name| builder.create_string(name.as_ref())); + let fb_name = builder.create_string(pr.pkg.name()); + + let fb_components = components_set_to_fb_components(builder, &pr.pkg.components); + + // TODO: for now version filters strings, but in future a proper + // version filter will breakdown into pieces in the flatbuffer + let fb_version_filter = if pr.pkg.version.is_empty() { + None + } else { + // A version filter is a bunch of version ranges, e.g. + // [kg/3.10 + // pkg/3.10.0 + // so need to use the alternate format here for version + // filters to ensure extra .0's don't get baked into requests. + Some(builder.create_string(&format!("{:#}", pr.pkg.version))) + }; + + let (fb_build, fb_build_type) = if let Some(build) = &pr.pkg.build { + let (fb, fbt) = build_to_fb_build(builder, build); + (Some(fb), fbt) + } else { + (None, spk_proto::Build::NONE) + }; + + let prerelease_policy = + prerelease_policy_to_fb_prerelease_policy(pr.pkg_request.prerelease_policy); + + let inclusion_policy = + inclusion_policy_to_fb_inclusion_policy(pr.pkg_request.inclusion_policy); + + let fb_pin = pin_to_fb_pin(builder, &pr.pkg_request.pin); + + let fb_pin_policy = pin_policy_to_fb_pin_policy(pr.pkg_request.pin_policy); + + let fb_required_compat = + lone_compat_rule_to_fb_lone_compat_rule(pr.pkg_request.required_compat); + + let fb_options = opts_to_fb_pkg_request_option_values(builder, &pr.options); + + let fb_pkg_req = spk_proto::PkgRequestWithOptions::create( + builder, + &spk_proto::PkgRequestWithOptionsArgs { + repo_name: fb_repo_name, + name: Some(fb_name), + components: fb_components, + version_filter: fb_version_filter, + build: fb_build, + build_type: fb_build_type, + prerelease_policy, + inclusion_policy, + pin: fb_pin, + pin_policy: fb_pin_policy, + required_compat: fb_required_compat, + options: fb_options, + }, + ) + .as_union_value(); + + let fb_req = spk_proto::RequirementWithOptions::create( + builder, + &spk_proto::RequirementWithOptionsArgs { + request: Some(fb_pkg_req), + request_type: spk_proto::RequestWithOptions::PkgRequestWithOptions, + }, + ); + + reqs.push(fb_req); + } + } + } + + flatbuffer_vector!(builder, reqs) +} + +fn version_tags_to_fb_version_tags<'a>( + builder: &mut flatbuffers::FlatBufferBuilder<'a>, + version_tags: &BTreeMap, +) -> Option< + flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, +> { + let mut tags = Vec::new(); + for (name, number) in version_tags { + let tag_name = builder.create_string(name); + let fb_tag = spk_proto::TagSetItem::create( + builder, + &spk_proto::TagSetItemArgs { + name: Some(tag_name), + number: *number, + }, + ); + tags.push(fb_tag); + } + flatbuffer_vector!(builder, tags) +} + +#[inline] +pub fn version_to_fb_version<'a>( + builder: &mut flatbuffers::FlatBufferBuilder<'a>, + version: &Version, +) -> flatbuffers::WIPOffset> { + let fb_parts = flatbuffer_vector!(builder, version.parts.parts); + + let fb_epsilon = match version.parts.epsilon { + Epsilon::Minus => spk_proto::Epsilon::Minus, + Epsilon::None => spk_proto::Epsilon::None, + Epsilon::Plus => spk_proto::Epsilon::Plus, + }; + + let fb_pre_tags = version_tags_to_fb_version_tags(builder, &version.pre.tags); + let fb_post_tags = version_tags_to_fb_version_tags(builder, &version.post.tags); + + spk_proto::Version::create( + builder, + &spk_proto::VersionArgs { + parts: fb_parts, + epsilon: fb_epsilon, + pre: fb_pre_tags, + post: fb_post_tags, + }, + ) +} + +pub fn build_to_fb_build<'a>( + builder: &mut flatbuffers::FlatBufferBuilder<'a>, + build: &Build, +) -> ( + flatbuffers::WIPOffset, + spk_proto::Build, +) { + let fb_build_id = match build { + Build::Source => { + spk_proto::Source::create(builder, &spk_proto::SourceArgs {}).as_union_value() + } + Build::Embedded(es) => { + let source = match es { + EmbeddedSource::Package(esp) => { + let fb_repository_name = esp + .ident + .repository_name + .as_ref() + .map(|s| builder.create_string(s)); + let fb_pkg_name = builder.create_string(&esp.ident.pkg_name); + let fb_version_str = esp + .ident + .version_str + .as_ref() + .map(|s| builder.create_string(s.as_str())); + let fb_build_str = esp + .ident + .build_str + .as_ref() + .map(|s| builder.create_string(s)); + + let fb_ident_parts_buf = spk_proto::IdentPartsBuf::create( + builder, + &spk_proto::IdentPartsBufArgs { + repository_name: fb_repository_name, + pkg_name: Some(fb_pkg_name), + version_str: fb_version_str, + build_str: fb_build_str, + }, + ); + + let fb_components = components_set_to_fb_components(builder, &esp.components); + + let fb_esp = spk_proto::EmbeddedSourcePackage::create( + builder, + &spk_proto::EmbeddedSourcePackageArgs { + ident: Some(fb_ident_parts_buf), + components: fb_components, + }, + ); + Some(fb_esp) + } + + EmbeddedSource::Unknown => None, + }; + spk_proto::EmbeddedSource::create(builder, &spk_proto::EmbeddedSourceArgs { source }) + .as_union_value() + } + Build::BuildId(id) => { + let fb_id = builder.create_string(&id.to_string()); + spk_proto::BuildId::create(builder, &spk_proto::BuildIdArgs { id: Some(fb_id) }) + .as_union_value() + } + }; + + let fb_build_type = match build { + Build::Source => spk_proto::Build::Source, + Build::Embedded(_es) => spk_proto::Build::EmbeddedSource, + Build::BuildId(_id) => spk_proto::Build::BuildId, + }; + + (fb_build_id, fb_build_type) +} + +// Note fb_compat objects in packages are not optional in the rust +// structs, but fb_compat's stored in var opts are optional in the +// rust struct. +#[inline] +pub fn var_opt_compat_to_var_opt_fb_compat<'a>( + builder: &mut flatbuffers::FlatBufferBuilder<'a>, + var_opt_compat: &Option, +) -> Option> { + if let Some(compat) = var_opt_compat { + // A non-empty compat specified for the var opt. This might + // even be the default compat, but need to store it anyway + // because the schema doesn't distinguish this yet. + let compat_string = compat.to_string(); + Some(builder.create_string(&compat_string)) + } else { + // No compat specified for the var opt + None + } +} + +#[inline] +pub fn compat_to_fb_compat<'a>( + builder: &mut flatbuffers::FlatBufferBuilder<'a>, + compat: &Compat, +) -> Option> { + // Package compat objects that are the default are treated as None + if compat.is_default() { + None + } else { + let compat_string = compat.to_string(); + Some(builder.create_string(&compat_string)) + } +} diff --git a/crates/spk-schema/src/lib.rs b/crates/spk-schema/src/lib.rs index 82cf94e419..b7808e43d5 100644 --- a/crates/spk-schema/src/lib.rs +++ b/crates/spk-schema/src/lib.rs @@ -10,6 +10,7 @@ mod deprecate; mod embedded_packages_list; mod environ; mod error; +mod fb_converter; mod input_variant; mod install_spec; mod metadata; @@ -45,15 +46,55 @@ pub use environ::{ SetEnv, }; pub use error::{Error, Result}; +pub use fb_converter::{ + build_to_fb_build, + compat_to_fb_compat, + component_specs_to_fb_component_specs, + components_to_fb_components, + embedded_pkg_specs_to_fb_embedded_package_specs, + fb_compat_to_compat, + fb_component_emb_pkgs_to_component_emb_pkgs, + fb_component_names_to_component_names, + fb_component_names_to_component_names_set, + fb_component_specs_to_component_name_set, + fb_component_specs_to_component_names, + fb_component_specs_to_component_specs, + fb_component_to_component, + fb_embedded_package_specs_to_embedded_package_specs, + fb_inclusion_policy_to_inclusion_policy, + fb_lone_compat_rule_to_lone_compat_rule, + fb_opt_to_opt, + fb_opts_to_opts, + fb_pin_policy_to_pin_policy, + fb_pin_to_pin, + fb_pkg_opt_to_opt, + fb_pkg_request_option_values_to_pkg_request_options, + fb_prerelease_policy_to_prerelease_policy, + fb_var_opt_to_opt, + fb_version_filter_to_version_filter, + fb_version_to_version, + get_build_from_fb_build_index, + get_build_from_fb_pkg_request_with_options, + opts_to_fb_opts, + requirements_with_options_to_fb_requirements_with_options, + version_to_fb_version, +}; pub use input_variant::InputVariant; pub use install_spec::InstallSpec; pub use option::{Inheritance, Opt}; -pub use package::{Components, DownstreamRequirements, OptionValues, Package, PackageMut}; +pub use package::{ + BuildOptions, + Components, + DownstreamRequirements, + OptionValues, + Package, + PackageMut, +}; pub use recipe::{BuildEnv, Recipe}; pub use requirements_list::{RequirementsList, convert_requests_to_requests_with_options}; pub use serde_json; pub use source_spec::{GitSource, LocalSource, ScriptSource, SourceSpec, TarSource}; -pub use spec::{ApiVersion, Spec, SpecFileData, SpecRecipe, SpecTemplate, SpecVariant}; +pub use spec::{ApiVersion, Spec, SpecFileData, SpecRecipe, SpecTemplate, SpecTest, SpecVariant}; pub use spk_schema_foundation::ident::{ self as ident, AnyIdent, @@ -79,7 +120,7 @@ pub use spk_schema_foundation::{ }; pub use template::{Template, TemplateData, TemplateExt}; pub use test::{Test, TestStage}; -pub use v0::{AutoHostVars, RecipeComponentSpec, Script}; +pub use v0::{AutoHostVars, IndexedPackage, RecipeComponentSpec, Script}; pub use validation::{ValidationRule, ValidationSpec}; pub use variant::{Variant, VariantExt}; diff --git a/crates/spk-schema/src/spec.rs b/crates/spk-schema/src/spec.rs index 5d78195f78..e84c445f1d 100644 --- a/crates/spk-schema/src/spec.rs +++ b/crates/spk-schema/src/spec.rs @@ -388,7 +388,12 @@ impl Recipe for SpecRecipe { } fn generate_source_build(&self, root: &Path) -> Result { - each_variant!(self, r, r.generate_source_build(root).map(Spec::V0Package)) + each_variant!( + self, + r, + r.generate_source_build(root) + .map(|p| Spec::V0Package(Box::new(p))) + ) } fn generate_binary_build(self, variant: &V, build_env: &E) -> Result @@ -401,7 +406,7 @@ impl Recipe for SpecRecipe { self, r, r.generate_binary_build(variant, build_env) - .map(Spec::V0Package) + .map(|p| Spec::V0Package(Box::new(p))) ) } @@ -632,7 +637,11 @@ impl Test for SpecTest { #[enum_dispatch(Deprecate, DeprecateMut)] pub enum Spec { #[serde(rename = "v0/package")] - V0Package(super::v0::PackageSpec), + V0Package(Box), + /// Package spec from an index, only usable in solves + #[serde(skip)] + #[serde(rename = "v0/indexedpackage")] + V0IndexedPackage(Box), } impl Components for Spec { @@ -641,6 +650,7 @@ impl Components for Spec { fn components(&self) -> Cow<'_, super::ComponentSpecList> { match self { Spec::V0Package(spec) => spec.components(), + Spec::V0IndexedPackage(spec) => spec.components(), } } } @@ -649,6 +659,7 @@ impl HasBuildIdent for Spec { fn build_ident(&self) -> &BuildIdent { match self { Spec::V0Package(r) => r.build_ident(), + Spec::V0IndexedPackage(spec) => spec.build_ident(), } } } @@ -657,6 +668,7 @@ impl OptionValues for Spec { fn option_values(&self) -> OptionMap { match self { Spec::V0Package(r) => r.option_values(), + Spec::V0IndexedPackage(spec) => spec.option_values(), } } } @@ -665,6 +677,7 @@ impl Satisfy for Spec { fn check_satisfies_request(&self, request: &PkgRequestWithOptions) -> Compatibility { match self { Spec::V0Package(r) => r.check_satisfies_request(request), + Spec::V0IndexedPackage(spec) => spec.check_satisfies_request(request), } } } @@ -673,6 +686,7 @@ impl Satisfy> for Spec { fn check_satisfies_request(&self, request: &VarRequest) -> Compatibility { match self { Spec::V0Package(r) => r.check_satisfies_request(request), + Spec::V0IndexedPackage(spec) => spec.check_satisfies_request(request), } } } @@ -681,6 +695,7 @@ impl HasVersion for Spec { fn version(&self) -> &Version { match self { Spec::V0Package(r) => r.version(), + Spec::V0IndexedPackage(spec) => spec.version(), } } } @@ -689,6 +704,7 @@ impl Named for Spec { fn name(&self) -> &PkgName { match self { Spec::V0Package(r) => r.name(), + Spec::V0IndexedPackage(spec) => spec.name(), } } } @@ -697,6 +713,7 @@ impl RuntimeEnvironment for Spec { fn runtime_environment(&self) -> &[crate::EnvOp] { match self { Spec::V0Package(r) => r.runtime_environment(), + Spec::V0IndexedPackage(spec) => spec.runtime_environment(), } } } @@ -705,6 +722,7 @@ impl Versioned for Spec { fn compat(&self) -> Cow<'_, Compat> { match self { Spec::V0Package(spec) => spec.compat(), + Spec::V0IndexedPackage(spec) => spec.compat(), } } } @@ -717,30 +735,35 @@ impl Package for Spec { fn ident(&self) -> &BuildIdent { match self { Spec::V0Package(spec) => Package::ident(spec), + Spec::V0IndexedPackage(spec) => super::v0::IndexedPackage::ident(spec), } } fn metadata(&self) -> &crate::metadata::Meta { match self { Spec::V0Package(spec) => spec.metadata(), + Spec::V0IndexedPackage(spec) => spec.metadata(), } } fn matches_all_filters(&self, filter_by: &Option>) -> bool { match self { Spec::V0Package(spec) => spec.matches_all_filters(filter_by), + Spec::V0IndexedPackage(spec) => spec.matches_all_filters(filter_by), } } fn sources(&self) -> &Vec { match self { Spec::V0Package(spec) => spec.sources(), + Spec::V0IndexedPackage(spec) => spec.sources(), } } fn embedded(&self) -> Cow<'_, super::EmbeddedPackagesList> { match self { Spec::V0Package(spec) => spec.embedded(), + Spec::V0IndexedPackage(spec) => spec.embedded(), } } @@ -751,30 +774,37 @@ impl Package for Spec { Spec::V0Package(spec) => spec .embedded_as_packages() .map(|vec| vec.into_iter().map(|(r, c)| (r.into(), c)).collect()), + Spec::V0IndexedPackage(spec) => spec + .embedded_as_packages() + .map(|vec| vec.into_iter().map(|(r, c)| (r.into(), c)).collect()), } } fn get_build_options(&self) -> Cow<'_, [Opt]> { match self { Spec::V0Package(spec) => spec.get_build_options(), + Spec::V0IndexedPackage(spec) => spec.get_build_options(), } } fn get_build_requirements(&self) -> crate::Result>> { match self { Spec::V0Package(spec) => spec.get_build_requirements(), + Spec::V0IndexedPackage(spec) => spec.get_build_requirements(), } } fn get_all_tests(&self) -> Vec { match self { Spec::V0Package(spec) => spec.get_all_tests(), + Spec::V0IndexedPackage(spec) => spec.get_all_tests(), } } fn runtime_requirements(&self) -> Cow<'_, crate::RequirementsList> { match self { Spec::V0Package(spec) => spec.runtime_requirements(), + Spec::V0IndexedPackage(spec) => spec.runtime_requirements(), } } } @@ -786,6 +816,7 @@ impl DownstreamRequirements for Spec { ) -> Cow<'_, crate::RequirementsList> { match self { Spec::V0Package(spec) => spec.downstream_build_requirements(components), + Spec::V0IndexedPackage(spec) => spec.downstream_build_requirements(components), } } @@ -795,6 +826,7 @@ impl DownstreamRequirements for Spec { ) -> Cow<'_, crate::RequirementsList> { match self { Spec::V0Package(spec) => spec.downstream_runtime_requirements(components), + Spec::V0IndexedPackage(spec) => spec.downstream_runtime_requirements(components), } } } @@ -803,12 +835,14 @@ impl PackageMut for Spec { fn set_build(&mut self, build: Build) { match self { Spec::V0Package(spec) => spec.set_build(build), + Spec::V0IndexedPackage(spec) => spec.set_build(build), } } fn insert_or_merge_install_requirement(&mut self, req: PinnedRequest) -> Result<()> { match self { Spec::V0Package(spec) => spec.insert_or_merge_install_requirement(req), + Spec::V0IndexedPackage(spec) => spec.insert_or_merge_install_requirement(req), } } } @@ -862,7 +896,19 @@ impl FromYaml for Spec { impl From for Spec { fn from(value: v0::EmbeddedPackageSpec) -> Self { - Spec::V0Package(value.into()) + Spec::V0Package(Box::new(value.into())) + } +} + +impl From for Spec { + fn from(value: v0::PackageSpec) -> Self { + Spec::V0Package(Box::new(value)) + } +} + +impl From for Spec { + fn from(value: v0::IndexedPackage) -> Self { + Spec::V0IndexedPackage(Box::new(value)) } } diff --git a/crates/spk-schema/src/v0/embedded_recipe_spec_test.rs b/crates/spk-schema/src/v0/embedded_recipe_spec_test.rs index e1d99fe59b..40afa4cb1e 100644 --- a/crates/spk-schema/src/v0/embedded_recipe_spec_test.rs +++ b/crates/spk-schema/src/v0/embedded_recipe_spec_test.rs @@ -44,15 +44,19 @@ fn test_sources_relative_to_spec_file(tmpdir: tempfile::TempDir) { .unwrap() .render(&OptionMap::default()) .unwrap(); - let crate::Spec::V0Package(recipe) = spec + if let crate::Spec::V0Package(recipe) = spec .into_recipe() .unwrap() .generate_source_build(&spec_dir) - .unwrap(); - if let Some(SourceSpec::Local(local)) = recipe.sources.first() { - assert_eq!(local.path, spec_dir); + .unwrap() + { + if let Some(SourceSpec::Local(local)) = recipe.sources.first() { + assert_eq!(local.path, spec_dir); + } else { + panic!("expected spec to have one local source spec"); + } } else { - panic!("expected spec to have one local source spec"); + panic!("expected spec to be a Spec::V0Package"); } } diff --git a/crates/spk-schema/src/v0/indexed_package.rs b/crates/spk-schema/src/v0/indexed_package.rs new file mode 100644 index 0000000000..40a60f05e8 --- /dev/null +++ b/crates/spk-schema/src/v0/indexed_package.rs @@ -0,0 +1,549 @@ +// Copyright (c) Contributors to the SPK project. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/spkenv/spk + +use std::borrow::Cow; +use std::cmp::Ordering; +use std::str::FromStr; +use std::sync::Arc; + +use arc_swap::{ArcSwap, ArcSwapOption}; +use serde::Serialize; +use spk_schema_foundation::IsDefault; +use spk_schema_foundation::ident::{ + BuildIdent, + PinnedRequest, + PinnedValue, + PkgRequestWithOptions, + RequestWithOptions, +}; +use spk_schema_foundation::ident_build::Build; +use spk_schema_foundation::ident_component::Component; +use spk_schema_foundation::name::OptNameBuf; +use spk_schema_foundation::option_map::{OptFilter, OptionMap}; +use spk_schema_foundation::spec_ops::HasBuildIdent; +use spk_schema_foundation::version::{IncompatibleReason, VarOptionProblem}; + +use super::check_package_spec_satisfies_pkg_request; +use crate::fb_converter::fb_requirements_to_requirements; +use crate::foundation::name::PkgName; +use crate::foundation::spec_ops::prelude::*; +use crate::foundation::version::{Compat, Compatibility, Version}; +use crate::ident::{Satisfy, VarRequest}; +use crate::package::OptionValues; +use crate::spec::SpecTest; +use crate::v0::EmbeddedPackageSpec; +use crate::{ + BuildOptions, + ComponentSpec, + ComponentSpecList, + Components, + Deprecate, + DeprecateMut, + DownstreamRequirements, + EmbeddedPackagesList, + Error, + Opt, + Package, + PackageMut, + RequirementsList, + Result, + RuntimeEnvironment, + SourceSpec, + fb_compat_to_compat, + fb_component_specs_to_component_specs, + fb_embedded_package_specs_to_embedded_package_specs, + fb_opt_to_opt, + fb_opts_to_opts, +}; + +// A package extracted from an index +#[derive(Debug, Serialize)] +pub struct IndexedPackage { + build_ident: BuildIdent, + #[serde(skip)] + buf: bytes::Bytes, + offset: usize, + + // Internal caches - used to save parsing/construction time, for + // now, because there are not flatbuffer equivalents of all the + // spk objects inside and returned by Spec and the associated traits. + // The caches can go away if full flatbuffer replacements are added. + #[serde(skip)] + cached_compat: ArcSwapOption, + #[serde(skip)] + cached_embedded: ArcSwap>, + #[serde(skip)] + cached_component_specs: ArcSwap>, + #[serde(skip)] + cached_requirements: ArcSwap>, + #[serde(skip)] + cached_build_opts: ArcSwap>, +} + +impl Clone for IndexedPackage { + fn clone(&self) -> Self { + Self { + build_ident: self.build_ident.clone(), + buf: self.buf.clone(), + offset: self.offset, + cached_compat: ArcSwapOption::new(self.cached_compat.load_full()), + cached_embedded: ArcSwap::new(self.cached_embedded.load_full()), + cached_component_specs: ArcSwap::new(self.cached_component_specs.load_full()), + cached_requirements: ArcSwap::new(self.cached_requirements.load_full()), + cached_build_opts: ArcSwap::new(self.cached_build_opts.load_full()), + } + } +} + +impl std::hash::Hash for IndexedPackage { + fn hash(&self, state: &mut H) { + // Different packages from the same index will have different + // offsets. + self.offset.hash(state); + self.build_ident.hash(state); + // The index bytes buf and cached fields are skipped + } +} + +impl std::cmp::PartialEq for IndexedPackage { + fn eq(&self, other: &Self) -> bool { + // This deliberately ignores the buf field and the caches. + if self.offset == other.offset { + true + } else { + self.build_ident == other.build_ident + } + } +} + +impl std::cmp::Eq for IndexedPackage {} + +impl PartialOrd for IndexedPackage { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for IndexedPackage { + fn cmp(&self, other: &Self) -> Ordering { + self.build_ident.cmp(&other.build_ident) + } +} + +impl IndexedPackage { + pub fn new(build_ident: BuildIdent, buf: bytes::Bytes, offset: usize) -> IndexedPackage { + Self { + build_ident, + buf, + offset, + cached_compat: ArcSwapOption::from(None), + cached_embedded: ArcSwap::new(Arc::new(EmbeddedPackagesList::default())), + // Not using default because that would be a list with + // build and a run components. + cached_component_specs: ArcSwap::new(Arc::new(ComponentSpecList::new(Vec::new()))), + cached_requirements: ArcSwap::new(Arc::new(RequirementsList::default())), + cached_build_opts: ArcSwap::new(Arc::new(Vec::new())), + } + } + + #[inline] + pub fn build_index(&self) -> spk_proto::BuildIndex<'_> { + // Safety: we trust that the buffer and offset have been + // validated, or come from a trusted source. + unsafe { + ::follow(&self.buf[..], self.offset) + } + } +} + +impl BuildOptions for IndexedPackage { + fn build_options(&self) -> Cow<'_, [Opt]> { + self.get_build_options() + } +} + +impl Deprecate for IndexedPackage { + fn is_deprecated(&self) -> bool { + self.build_index().is_deprecated() + } +} + +impl DeprecateMut for IndexedPackage { + fn deprecate(&mut self) -> Result<()> { + Err(Error::SpkIndexedPackageDoesNotImplement( + "DeprecateMut".to_string(), + "deprecate".to_string(), + )) + } + + fn set_deprecated(&mut self, deprecated: bool) -> Result<()> { + if deprecated { + self.deprecate() + } else { + self.undeprecate() + } + } + + fn undeprecate(&mut self) -> Result<()> { + Err(Error::SpkIndexedPackageDoesNotImplement( + "DeprecateMut".to_string(), + "undeprecate".to_string(), + )) + } +} + +impl Satisfy for IndexedPackage { + fn check_satisfies_request(&self, pkg_request: &PkgRequestWithOptions) -> Compatibility { + check_package_spec_satisfies_pkg_request(self, pkg_request) + } +} + +impl Satisfy> for IndexedPackage { + fn check_satisfies_request(&self, var_request: &VarRequest) -> Compatibility { + // Copied from V0 Spec and slightly adjusted for + // the flatbuffer data. + let opt_required = var_request.var.namespace() == Some(self.name()); + let mut opt: Option<&Opt> = None; + let request_name = &var_request.var; + let mut option: Opt; + + if let Some(build_opts) = self.build_index().build_options() { + for fb_opt in build_opts.iter() { + option = fb_opt_to_opt(&fb_opt); + let o_name = unsafe { OptNameBuf::from_string(option.base_name().to_string()) }; + if request_name == &o_name { + opt = Some(&option); + break; + } + if *request_name == o_name.with_namespace(self.name()) { + opt = Some(&option); + break; + } + } + } + + match opt { + None => { + if opt_required { + return Compatibility::Incompatible(IncompatibleReason::VarOptionMissing( + var_request.var.clone(), + )); + } + Compatibility::Compatible + } + Some(Opt::Pkg(opt)) => opt.validate(Some(&*var_request.value)), + Some(Opt::Var(opt)) => { + let request_value = &*var_request.value; + let exact = opt.get_value(Some(request_value)); + if exact.as_deref() == Some(request_value) { + return Compatibility::Compatible; + } + + // For values that aren't exact matches, if the option specifies + // a compat rule, try treating the values as version numbers + // and see if they satisfy the rule. + if let Some(compat) = &opt.compat { + let base_version = exact.clone(); + let Ok(base_version) = Version::from_str(&base_version.unwrap_or_default()) + else { + return Compatibility::Incompatible(IncompatibleReason::VarOptionMismatch( + VarOptionProblem::IncompatibleBuildOptionInvalidVersion { + var_request: var_request.var.clone(), + base: exact.unwrap_or_default(), + request_value: request_value.to_string(), + }, + )); + }; + + let Ok(request_version) = Version::from_str(request_value) else { + return Compatibility::Incompatible(IncompatibleReason::VarOptionMismatch( + VarOptionProblem::IncompatibleBuildOptionInvalidVersion { + var_request: var_request.var.clone(), + base: exact.unwrap_or_default(), + request_value: request_value.to_string(), + }, + )); + }; + + let result = compat.is_binary_compatible(&base_version, &request_version); + if let Compatibility::Incompatible(incompatible) = result { + return Compatibility::Incompatible(IncompatibleReason::VarOptionMismatch( + VarOptionProblem::IncompatibleBuildOptionWithContext { + var_request: var_request.var.clone(), + exact: exact.unwrap_or_else(|| "None".to_string()), + request_value: request_value.to_string(), + context: Box::new(incompatible), + }, + )); + } + return result; + } + + Compatibility::Incompatible(IncompatibleReason::VarOptionMismatch( + VarOptionProblem::IncompatibleBuildOption { + var_request: var_request.var.clone(), + exact: exact.unwrap_or_else(|| "None".to_string()), + request_value: request_value.to_string(), + }, + )) + } + } + } +} + +impl HasVersion for IndexedPackage { + fn version(&self) -> &Version { + self.build_ident.version() + } +} + +impl HasBuild for IndexedPackage { + fn build(&self) -> &Build { + self.build_ident.build() + } +} + +impl HasBuildIdent for IndexedPackage { + fn build_ident(&self) -> &BuildIdent { + &self.build_ident + } +} + +impl Named for IndexedPackage { + fn name(&self) -> &PkgName { + self.build_ident.name() + } +} + +impl RuntimeEnvironment for IndexedPackage { + fn runtime_environment(&self) -> &[crate::EnvOp] { + let err = Error::SpkIndexedPackageDoesNotImplement( + "RuntimeEnvironment".to_string(), + "runtime_environment".to_string(), + ); + // TODO: should this change the return value, e.g. to + // Result<&[crate::EnvOp]>, update all the caller's handling, + // and return an error for this implementation? + unreachable!("{err}"); + } +} + +impl Versioned for IndexedPackage { + fn compat(&self) -> Cow<'_, Compat> { + if self.cached_compat.load().is_none() { + // The compat hasn't been read and decoded yet + let compat = fb_compat_to_compat(self.build_index().compat()); + self.cached_compat.store(Some(Arc::new(compat))) + } + // The unwrap should be safe because is_none() was checked + // above and the value updated to some compat value. + let c = (**self.cached_compat.load().as_ref().unwrap()).clone(); + Cow::Owned(c) + } +} + +impl Components for IndexedPackage { + type ComponentSpecT = ComponentSpec; + + fn components(&self) -> Cow<'_, ComponentSpecList> { + if let Some(fb_c_specs) = self.build_index().component_specs() + && self.cached_component_specs.load().is_empty() + { + let build_options = self.option_values(); + let component_specs = + fb_component_specs_to_component_specs(&fb_c_specs, &build_options); + + self.cached_component_specs.store(Arc::new(component_specs)); + } + + Cow::Owned((**self.cached_component_specs.load()).clone()) + } +} + +impl Package for IndexedPackage { + // Only used for embedded_as_packages() method, which can't easily + // return a IndexedPackage for an embedded package without + // generating a flatbuffer bytes representation of each decoded + // EmbeddedPackageSpec pieces. So this uses a full package spec + // for the return values of that method. + type Package = crate::v0::PackageSpec; + type EmbeddedPackage = EmbeddedPackageSpec; + + #[inline] + fn ident(&self) -> &BuildIdent { + &self.build_ident + } + + fn metadata(&self) -> &crate::metadata::Meta { + let err = + Error::SpkIndexedPackageDoesNotImplement("Package".to_string(), "metadata".to_string()); + // TODO: should this change the return value, update all the + // caller's handling, and return an error for this implementation? + unreachable!("{err}"); + } + + fn matches_all_filters(&self, filter_by: &Option>) -> bool { + // A duplicate of the code in V0 PackageSpec's matches_all_filters() + // method. It relies on self.check_satisfies_request() method + // which is not in the Package trait, or else this could be + // the default implementation for the Package trait. + if let Some(filters) = filter_by { + let settings = self.option_values(); + + for filter in filters { + if !settings.contains_key(&filter.name) { + // Not having an option with the filter's name is + // considered a match. + continue; + } + + let var_request = + VarRequest::new_with_value(filter.name.clone(), filter.value.clone()); + + let compat = self.check_satisfies_request(&var_request); + if !compat.is_ok() { + return false; + } + } + } + // All the filters match, or there were no filters + true + } + + fn sources(&self) -> &Vec { + let err = + Error::SpkIndexedPackageDoesNotImplement("Package".to_string(), "sources".to_string()); + // TODO: should this change the return value, update all the + // caller's handling, and return an error for this implementation? + unreachable!("{err}"); + } + + fn embedded(&self) -> Cow<'_, EmbeddedPackagesList> { + if let Some(fb_embedded) = self.build_index().embedded() + && self.cached_embedded.load().is_default() + { + let embedded_package_specs = + fb_embedded_package_specs_to_embedded_package_specs(&fb_embedded); + + self.cached_embedded.store(Arc::new(embedded_package_specs)); + } + + Cow::Owned((**self.cached_embedded.load()).clone()) + } + + fn embedded_as_packages( + &self, + ) -> std::result::Result)>, &str> { + Ok(self + .embedded() + .iter() + .map(|embed| (embed.clone().into(), None)) + .collect()) + } + + fn get_build_options(&self) -> Cow<'_, [Opt]> { + if self.build_index().build_options().is_some() && self.cached_build_opts.load().is_empty() + { + let build_options = fb_opts_to_opts(self.build_index().build_options()); + self.cached_build_opts.store(Arc::new(build_options)) + } + + Cow::Owned((**self.cached_build_opts.load()).clone()) + } + + fn get_build_requirements(&self) -> crate::Result>> { + Err(Error::SpkIndexedPackageDoesNotImplement( + "Package".to_string(), + "get_build_requirements".to_string(), + )) + } + + fn runtime_requirements(&self) -> Cow<'_, RequirementsList> { + if self.build_index().runtime_requirements().is_some() + && self.cached_requirements.load().is_default() + { + let reqs = fb_requirements_to_requirements(self.build_index().runtime_requirements()); + let requirements = unsafe { RequirementsList::::new_checked(reqs) }; + + self.cached_requirements.store(Arc::new(requirements)); + } + + Cow::Owned((**self.cached_requirements.load()).clone()) + } + + fn get_all_tests(&self) -> Vec { + let err = Error::SpkIndexedPackageDoesNotImplement( + "Package".to_string(), + "get_all_tests".to_string(), + ); + // TODO: should this change the return value, update all the + // caller's handling, and return an error for this implementation? + unreachable!("{err}"); + } +} + +impl DownstreamRequirements for IndexedPackage { + fn downstream_build_requirements<'a>( + &self, + _components: impl IntoIterator, + ) -> Cow<'_, RequirementsList> { + // This is for build var requirements and inheritance used in + // building. This kinds of package has no build data stored. + let err = Error::SpkIndexedPackageDoesNotImplement( + "DownstreamRequirements".to_string(), + "downstream_build_requirements".to_string(), + ); + // TODO: should this change the return value, update all the + // caller's handling, and return an error for this implementation? + unreachable!("{err}"); + } + + fn downstream_runtime_requirements<'a>( + &self, + _components: impl IntoIterator, + ) -> Cow<'_, RequirementsList> { + // This is also for build var requirements and inheritance + // used in building. This package has no build data stored. + let err = Error::SpkIndexedPackageDoesNotImplement( + "DownstreamRequirements".to_string(), + "downstream_runtime_requirements".to_string(), + ); + // TODO: should this change the return value, update all the + // caller's handling, and return an error for this implementation? + unreachable!("{err}"); + } +} + +impl OptionValues for IndexedPackage { + fn option_values(&self) -> OptionMap { + let mut option_map = OptionMap::default(); + + for opt in fb_opts_to_opts(self.build_index().build_options()).iter() { + let value = opt.get_value(None); + let name = opt.full_name(); + option_map.insert(name.into(), value.to_string()); + } + + option_map + } +} + +impl PackageMut for IndexedPackage { + fn set_build(&mut self, _build: Build) { + let err = Error::SpkIndexedPackageDoesNotImplement( + "PackageMut".to_string(), + "set_build".to_string(), + ); + // TODO: should this change the return value, update all the + // caller's handling, and return an error for this implementation? + unreachable!("{err}"); + } + + fn insert_or_merge_install_requirement(&mut self, _req: PinnedRequest) -> Result<()> { + Err(Error::SpkIndexedPackageDoesNotImplement( + "PackageMut".to_string(), + "insert_or_merge_install_requirement".to_string(), + )) + } +} diff --git a/crates/spk-schema/src/v0/mod.rs b/crates/spk-schema/src/v0/mod.rs index 4e687fb50c..115cd1fc46 100644 --- a/crates/spk-schema/src/v0/mod.rs +++ b/crates/spk-schema/src/v0/mod.rs @@ -7,6 +7,7 @@ mod embedded_install_spec; mod embedded_package_spec; mod embedded_recipe_install_spec; mod embedded_recipe_spec; +mod indexed_package; mod package_spec; mod platform; mod recipe_build_spec; @@ -23,6 +24,7 @@ pub use embedded_install_spec::EmbeddedInstallSpec; pub use embedded_package_spec::EmbeddedPackageSpec; pub use embedded_recipe_install_spec::EmbeddedRecipeInstallSpec; pub use embedded_recipe_spec::EmbeddedRecipeSpec; +pub use indexed_package::IndexedPackage; pub use package_spec::PackageSpec; pub(crate) use package_spec::check_package_spec_satisfies_pkg_request; pub use platform::Platform; diff --git a/crates/spk-schema/src/v0/package_spec_test.rs b/crates/spk-schema/src/v0/package_spec_test.rs index a98236670f..31e9147c2c 100644 --- a/crates/spk-schema/src/v0/package_spec_test.rs +++ b/crates/spk-schema/src/v0/package_spec_test.rs @@ -38,15 +38,20 @@ fn test_sources_relative_to_spec_file(tmpdir: tempfile::TempDir) { .unwrap() .render(&OptionMap::default()) .unwrap(); - let crate::Spec::V0Package(recipe) = spec + + if let crate::Spec::V0Package(recipe) = spec .into_recipe() .unwrap() .generate_source_build(&spec_dir) - .unwrap(); - if let Some(SourceSpec::Local(local)) = recipe.sources.first() { - assert_eq!(local.path, spec_dir); + .unwrap() + { + if let Some(SourceSpec::Local(local)) = recipe.sources.first() { + assert_eq!(local.path, spec_dir); + } else { + panic!("expected spec to have one local source spec"); + } } else { - panic!("expected spec to have one local source spec"); + panic!("expected spec to be a Spec::V0Package"); } } diff --git a/crates/spk-schema/src/v0/recipe_spec_test.rs b/crates/spk-schema/src/v0/recipe_spec_test.rs index 791f987340..350579c3fe 100644 --- a/crates/spk-schema/src/v0/recipe_spec_test.rs +++ b/crates/spk-schema/src/v0/recipe_spec_test.rs @@ -44,16 +44,20 @@ fn test_sources_relative_to_spec_file(tmpdir: tempfile::TempDir) { .unwrap() .render(&OptionMap::default()) .unwrap(); - let crate::Spec::V0Package(recipe) = spec + if let crate::Spec::V0Package(recipe) = spec .into_recipe() .unwrap() .generate_source_build(&spec_dir) - .unwrap(); - if let Some(SourceSpec::Local(local)) = recipe.sources.first() { - assert_eq!(local.path, spec_dir); + .unwrap() + { + if let Some(SourceSpec::Local(local)) = recipe.sources.first() { + assert_eq!(local.path, spec_dir); + } else { + panic!("expected spec to have one local source spec"); + } } else { - panic!("expected spec to have one local source spec"); - } + panic!("expected spec to be a V0Package"); + }; } #[rstest] diff --git a/crates/spk-solve/crates/macros/src/lib.rs b/crates/spk-solve/crates/macros/src/lib.rs index a85837d913..56edbc9a37 100644 --- a/crates/spk-solve/crates/macros/src/lib.rs +++ b/crates/spk-solve/crates/macros/src/lib.rs @@ -167,7 +167,7 @@ macro_rules! make_build_and_components { let name = spk_schema::foundation::ident_component::Component::parse(name).expect("invalid component name"); components.insert(name, $crate::spfs::encoding::EMPTY_DIGEST.into()); } - (spk_schema::Spec::V0Package(build), components) + (spk_schema::Spec::V0Package(Box::new(build)), components) }}; (package = $package:ident, [$($dep:expr),*], $opts:expr, [$($component:expr),*]) => {{ let mut components = std::collections::HashMap::::new(); @@ -187,7 +187,7 @@ macro_rules! make_build_and_components { } } } - (spk_schema::Spec::V0Package($package), components) + (spk_schema::Spec::V0Package(Box::new($package)), components) }}; ($spec:tt, [$($dep:expr),*], $opts:expr, [$($component:expr),*]) => {{ let json = $crate::serde_json::json!($spec); diff --git a/crates/spk-storage/src/storage/spfs.rs b/crates/spk-storage/src/storage/spfs.rs index de2a823f8e..2b565fc281 100644 --- a/crates/spk-storage/src/storage/spfs.rs +++ b/crates/spk-storage/src/storage/spfs.rs @@ -540,6 +540,9 @@ impl Storage for SpfsRepository { }); Spec::V0Package(spec) } + Spec::V0IndexedPackage(_) => { + unreachable!("Can't read Spec::V0IndexedPackage spec from an Spk SPFS repo. Those come from an SPK IndexedRepo.",) + } }) .map_err(|err| { Error::InvalidPackageSpec(Box::new(InvalidPackageSpec( @@ -804,6 +807,9 @@ impl crate::Repository for SpfsRepository { }); Spec::V0Package(spec) } + Spec::V0IndexedPackage(_) => { + unreachable!("Can't read Spec::V0IndexedPackage spec from an Spk SPFS repo. Those come from an SPK IndexedRepo.",) + } }) .map_err(|err| { Error::InvalidPackageSpec(Box::new(InvalidPackageSpec( diff --git a/crates/spk-storage/src/walker_test.rs b/crates/spk-storage/src/walker_test.rs index 50faec24e9..f025a88b90 100644 --- a/crates/spk-storage/src/walker_test.rs +++ b/crates/spk-storage/src/walker_test.rs @@ -125,9 +125,9 @@ async fn test_walker_builder_walker_walk() { }), RepoWalkerItem::Build(WalkedBuild { repo_name, - spec: Arc::new(Spec::V0Package(spk_schema::v0::PackageSpec::new( + spec: Arc::new(Spec::V0Package(Box::new(spk_schema::v0::PackageSpec::new( build_ident.clone(), - ))), + )))), }), RepoWalkerItem::Component(WalkedComponent { repo_name, @@ -138,9 +138,9 @@ async fn test_walker_builder_walker_walk() { // Note: this doesn't check files RepoWalkerItem::EndOfBuild(WalkedBuild { repo_name, - spec: Arc::new(Spec::V0Package(spk_schema::v0::PackageSpec::new( + spec: Arc::new(Spec::V0Package(Box::new(spk_schema::v0::PackageSpec::new( build_ident, - ))), + )))), }), RepoWalkerItem::EndOfVersion(WalkedVersion { repo_name, diff --git a/cspell.json b/cspell.json index 6389e2171e..b81cb15c5e 100644 --- a/cspell.json +++ b/cspell.json @@ -97,6 +97,7 @@ "codegen", "colour", "compat", + "compats", "Compat", "compn", "coreutils", @@ -237,6 +238,8 @@ "filesystems", "filesytem", "FIVZIKA", + "flamegraph", + "flamegraphs", "flatbuf", "flatbuffer", "flatbuffers", @@ -316,6 +319,7 @@ "inary", "incdir", "incompat", + "indexedpackage", "indexmap", "indicatif", "Initialise", @@ -657,6 +661,7 @@ "seekable", "serde", "setattr", + "setbuf", "setcap", "setdefault", "setegid", @@ -677,6 +682,7 @@ "SIMD", "SNAPPROCESS", "solvables", + "solverpackage", "somedata", "SOMEDIGEST", "somedir",