Indexes 4: Adds Indexes for SPK repositories

Cargo.toml

@@ -80,6 +80,7 @@ indicatif = "0.17.8"
 is_default_derive_macro = { path = "crates/is_default_derive_macro" }
 itertools = "0.14"
 libc = "0.2.172"
+memmap2 = "0.9.10"
 miette = "7.0"
 nix = { version = "0.29", features = ["mount", "sched", "user"] }
 nom = "7.1"

crates/spk-build/src/archive_test.rs

@@ -42,7 +42,9 @@ async fn test_archive_create_parents(#[case] solver: SolverImpl) {
     let filename = rt.tmpdir.path().join("deep/nested/path/archive.spk");
     let repo = match &*rt.tmprepo {
         spk_solve::RepositoryHandle::SPFS(repo) => repo,
-        spk_solve::RepositoryHandle::Mem(_) | spk_solve::RepositoryHandle::Runtime(_) => {
+        spk_solve::RepositoryHandle::Mem(_)
+        | spk_solve::RepositoryHandle::Runtime(_)
+        | spk_solve::RepositoryHandle::Indexed(_) => {
             panic!("only spfs repositories are supported")
         }
     };

crates/spk-cli/group3/src/cmd_export.rs

@@ -52,7 +52,9 @@ impl Run for Export {
             .iter()
             .map(|repo| match &**repo {
                 storage::RepositoryHandle::SPFS(repo) => Ok(repo),
-                storage::RepositoryHandle::Mem(_) | storage::RepositoryHandle::Runtime(_) => {
+                storage::RepositoryHandle::Mem(_)
+                | storage::RepositoryHandle::Runtime(_)
+                | storage::RepositoryHandle::Indexed(_) => {
                     bail!("Only spfs repositories are supported")
                 }
             })

crates/spk-cli/group3/src/cmd_import_test.rs

@@ -44,7 +44,9 @@ async fn test_archive_io(#[case] solver: SolverImpl) {
     filename.ensure();
     let repo = match &*rt.tmprepo {
         spk_solve::RepositoryHandle::SPFS(repo) => repo,
-        spk_solve::RepositoryHandle::Mem(_) | spk_solve::RepositoryHandle::Runtime(_) => {
+        spk_solve::RepositoryHandle::Mem(_)
+        | spk_solve::RepositoryHandle::Runtime(_)
+        | spk_solve::RepositoryHandle::Indexed(_) => {
             panic!("only spfs repositories are supported")
         }
     };

crates/spk-config/src/config.rs

@@ -87,6 +87,41 @@ pub struct Solver {
 
     /// Name of the solver whose output to show  when multiple solvers are being run.
     pub solver_to_show: String,
+
+    /// Whether to get the solver to use repository indexes, if
+    /// available, instead of the repository directly.
+    pub use_indexes: bool,
+
+    /// Default setting for indexes, if using indexes is enabled for
+    /// the solver.
+    pub indexes: Index,
+}
+
+/// The settings for one or more indexes
+#[derive(Clone, Default, Debug, Deserialize, Serialize)]
+#[serde(default)]
+pub struct Index {
+    /// Whether to validate the index data before using it.
+    /// Validating is safer but can add overhead at the start of a
+    /// solve that uses indexes.
+    pub verify_before_use: bool,
+
+    /// What kind of index to use. Only applies if there is more than
+    /// one kind of index available for the repository. The default is
+    /// 'flatb', a flatbuffers file based index.
+    pub kind: String,
+}
+
+/// The settings for a single repository
+#[derive(Clone, Default, Debug, Deserialize, Serialize)]
+#[serde(default)]
+pub struct Repository {
+    /// Whether to use an index with this repository, if one is
+    /// available.
+    pub use_index: bool,
+
+    /// Setting for the repositories index, if an index is enabled.
+    pub index: Index,
 }
 
 #[derive(Clone, Default, Debug, Deserialize, Serialize)]
@@ -142,6 +177,7 @@ pub struct Config {
     // with environment variables.
     pub sentry: Sentry,
     pub solver: Solver,
+    pub repositories: HashMap<String, Repository>,
     pub statsd: Statsd,
     pub metadata: Metadata,
     pub cli: Cli,

crates/spk-schema/src/fb_converter.rs

@@ -4,7 +4,6 @@
 
 use std::collections::{BTreeMap, BTreeSet};
 use std::str::FromStr;
-use std::sync::Arc;
 
 use spk_schema_foundation::IsDefault;
 use spk_schema_foundation::ident::{
@@ -1096,7 +1095,7 @@ pub fn component_specs_to_fb_component_specs<'a>(
 
 pub fn embedded_pkg_specs_to_fb_embedded_package_specs<'a>(
     builder: &mut flatbuffers::FlatBufferBuilder<'a>,
-    embedded: Arc<EmbeddedPackagesList<EmbeddedPackageSpec>>,
+    embedded: &EmbeddedPackagesList<EmbeddedPackageSpec>,
 ) -> Option<
     flatbuffers::WIPOffset<
         flatbuffers::Vector<

crates/spk-schema/src/v0/indexed_package.rs

@@ -43,6 +43,7 @@ use crate::{
     DownstreamRequirements,
     EmbeddedPackagesList,
     Error,
+    Inheritance,
     Opt,
     Package,
     PackageMut,
@@ -503,15 +504,42 @@ impl DownstreamRequirements for IndexedPackage {
         &self,
         _components: impl IntoIterator<Item = &'a Component>,
     ) -> Cow<'_, RequirementsList<RequestWithOptions>> {
-        // This is also for build var requirements and inheritance
-        // used in building. This package has no build data stored.
-        let err = Error::SpkIndexedPackageDoesNotImplement(
-            "DownstreamRequirements".to_string(),
-            "downstream_runtime_requirements".to_string(),
-        );
-        // TODO: should this change the return value, update all the
-        // caller's handling, and return an error for this implementation?
-        unreachable!("{err}");
+        // This is used when deprecating an embedded stub/package
+        // and this is exercised during the automated repository tests.
+
+        // This is a version of downstream_runtime_requirements() and
+        // downstream_requirements() from v0/package_spec.rs modified
+        // for this kind of flatbuffer backed package.
+        let build_options = self.build_options();
+        let embedded = self.embedded();
+
+        let requests = build_options
+            .iter()
+            .filter_map(|opt| match opt {
+                Opt::Var(v) => Some(v.with_default_namespace(self.name())),
+                Opt::Pkg(_) => None,
+            })
+            .chain(embedded.iter().flat_map(|embed| {
+                embed.build().options.iter().filter_map(|opt| match opt {
+                    Opt::Var(v) => Some(v.with_default_namespace(embed.name())),
+                    Opt::Pkg(_) => None,
+                })
+            }))
+            .filter(|o| o.inheritance() == Inheritance::Strong || o.required)
+            .map(|o| {
+                VarRequest {
+                    // we are assuming that the var here will have a value because
+                    // this is a built binary package
+                    value: o.get_value(None).unwrap_or_default().into(),
+                    var: o.var,
+                    // Index doesn't store the an option's description
+                    description: None,
+                }
+            })
+            .map(RequestWithOptions::Var);
+        RequirementsList::<RequestWithOptions>::try_from_iter(requests)
+            .map(Cow::Owned)
+            .expect("build opts (from a RepoIndex) do not contain duplicates")
     }
 }
 

crates/spk-solve/src/solvers/resolvo/spk_provider.rs

@@ -138,7 +138,21 @@ impl ResolvoPackageName {
                 // "requires_build_from_source" being true.
                 let mut located_builds = HashSet::new();
 
+                // This gets the current request with the same pkg_name, so can
+                // get build from it. Then if there is a digest in this request,
+                // and if the pkg build ident matching this one's build ident, then
+                // this is a direct request for a build, which has implications
+                // later for deprecated builds processing.
                 let root_pkg_request = provider.global_pkg_requests.get(&pkg_name.name);
+                let direct_build_request = if let Some(pkg_request) = root_pkg_request {
+                    if pkg_request.pkg.build.is_some() {
+                        pkg_request.pkg.build.clone()
+                    } else {
+                        None
+                    }
+                } else {
+                    None
+                };
 
                 for repo in &provider.repos {
                     let versions = repo
@@ -319,6 +333,41 @@ impl ResolvoPackageName {
                         continue;
                     }
 
+                    // TODO: These checks, and the earlier ones for direct_build_requests,
+                    // may be rendered obsolete with indexes storing the normal data for
+                    // deprecated packages, and indexes being fast.
+                    //
+                    // TODO: if going forward with this, test it with command line added
+                    // package build requests.
+                    //
+                    // A short circuit for avoiding some processing if this build
+                    // is deprecated. Works for an IndexedRepo, all other SPK repos
+                    // return an error. By asking the IndexedRepo directly if the
+                    // build is deprecated, we can avoid reading the full build
+                    // to check. The two cases are:
+                    //
+                    // if build is deprecated and not used in any known direct build reqs,
+                    //      then add to excludes and don't read it.
+                    // if build is deprecated and used in a known direct build reqs,
+                    //      then keep it around just in case that build appears in the resolve.
+                    //
+                    if let Ok(true) = repo.is_build_deprecated(ident.as_build()).await {
+                        if let Some(ref build_ident) = direct_build_request
+                            && build_ident == ident.build()
+                        {
+                            // Allow it to continue because, even though it is deprecated,
+                            // the current request is for a specific build and this
+                            // build matches the request.
+                        } else {
+                            // Exclude this deprecated build.
+                            let reason = provider
+                                .pool
+                                .intern_string(format!("{} is deprecated", ident));
+                            candidates.excluded.push((solvable_id, reason));
+                            continue;
+                        }
+                    }
+
                     match repo.read_package(ident.target()).await {
                         Ok(package) => {
                             // Filter builds that don't satisfy global var requests