Indexes 5: Adds spk repo index subcommand for index generation and updates

crates/spfs/src/error.rs

@@ -104,6 +104,8 @@ pub enum Error {
         #[source]
         source: storage::OpenRepositoryError,
     },
+    #[error("Repository does not support index storage location: {0:?}")]
+    NoIndexStorageLocation(url::Url),
 
     #[error("No remote named '{0}' configured")]
     #[diagnostic(

crates/spfs/src/storage/handle.rs

@@ -3,6 +3,7 @@
 // https://github.com/spkenv/spk
 
 use std::borrow::Cow;
+use std::path::PathBuf;
 use std::pin::Pin;
 use std::sync::Arc;
 
@@ -15,9 +16,13 @@ use super::prelude::*;
 use super::tag::TagSpecAndTagStream;
 use super::{TagNamespace, TagNamespaceBuf, TagStorageMut};
 use crate::graph::ObjectProto;
+use crate::storage::IndexPath;
 use crate::tracking::{self, BlobRead};
 use crate::{Error, Result, graph};
 
+// Index sub-directory inside a repository
+const INDEX_SUB_DIR: &str = "index";
+
 #[derive(Debug)]
 #[allow(clippy::large_enum_variant)]
 pub enum RepositoryHandle {
@@ -150,6 +155,47 @@ impl Address for RepositoryHandle {
     }
 }
 
+#[async_trait::async_trait]
+impl IndexPath for RepositoryHandle {
+    async fn index_path(&self) -> Result<PathBuf> {
+        // Only FS repositories have a location for indexes at this time.
+        match self {
+            RepositoryHandle::FS(repo) => {
+                // Makes the spfs fs repository specific index
+                // sub-directory, if it does not exist, and returns
+                // the path to it.
+                let mut index_path = PathBuf::new();
+                index_path.push(repo.root());
+                index_path.push(INDEX_SUB_DIR);
+
+                crate::runtime::makedirs_with_perms(&index_path, 0o777).map_err(|source| {
+                    Error::String(format!(
+                        "Unable to make '{INDEX_SUB_DIR}' sub-directory in spfs filesystem repo: {source}"
+                    ))
+                })?;
+
+                Ok(index_path)
+            }
+
+            RepositoryHandle::Tar(repo) => {
+                Err(Error::NoIndexStorageLocation(repo.address().into_owned()))
+            }
+            RepositoryHandle::Rpc(repo) => {
+                Err(Error::NoIndexStorageLocation(repo.address().into_owned()))
+            }
+            RepositoryHandle::FallbackProxy(repo) => {
+                Err(Error::NoIndexStorageLocation(repo.address().into_owned()))
+            }
+            RepositoryHandle::Proxy(repo) => {
+                Err(Error::NoIndexStorageLocation(repo.address().into_owned()))
+            }
+            RepositoryHandle::Pinned(repo) => {
+                Err(Error::NoIndexStorageLocation(repo.address().into_owned()))
+            }
+        }
+    }
+}
+
 #[async_trait::async_trait]
 impl TagStorage for RepositoryHandle {
     #[inline]

crates/spfs/src/storage/index_path.rs

@@ -0,0 +1,15 @@
+// Copyright (c) Contributors to the SPK project.
+// SPDX-License-Identifier: Apache-2.0
+// https://github.com/spkenv/spk
+
+use std::path::PathBuf;
+
+use crate::Result;
+
+/// The index location path of a repository.
+#[async_trait::async_trait]
+pub trait IndexPath {
+    /// Get the index location path of this repository, will create it
+    /// if it does not exist.
+    async fn index_path(&self) -> Result<PathBuf>;
+}

crates/spfs/src/storage/mod.rs

@@ -5,6 +5,7 @@
 mod address;
 mod blob;
 mod error;
+mod index_path;
 mod layer;
 mod manifest;
 pub mod payload;
@@ -27,6 +28,7 @@ pub use address::Address;
 pub use blob::{BlobStorage, BlobStorageExt};
 pub use error::OpenRepositoryError;
 pub use handle::RepositoryHandle;
+pub use index_path::IndexPath;
 pub use layer::{LayerStorage, LayerStorageExt};
 pub use manifest::ManifestStorage;
 pub use payload::PayloadStorage;

crates/spk-cli/cmd-repo/Cargo.toml

@@ -16,6 +16,8 @@ workspace = true
 miette = { workspace = true, features = ["fancy"] }
 async-trait = { workspace = true }
 clap = { workspace = true }
+itertools = { workspace = true }
 spk-cli-common = { workspace = true }
+spk-schema = { workspace = true }
 spk-storage = { workspace = true }
 tracing = { workspace = true }

crates/spk-cli/cmd-repo/src/cmd_repo.rs

@@ -2,10 +2,15 @@
 // SPDX-License-Identifier: Apache-2.0
 // https://github.com/spkenv/spk
 
+use std::str::FromStr;
+use std::time::Instant;
+
 use clap::{Args, Subcommand};
+use itertools::Itertools;
 use miette::{Context, Result};
-use spk_cli_common::{CommandArgs, Run};
-use spk_storage as storage;
+use spk_cli_common::{CommandArgs, Run, flags};
+use spk_schema::ident::OptVersionIdent;
+use spk_storage::{self as storage, FlatBufferRepoIndex, RepositoryHandle, RepositoryIndexMut};
 use storage::Repository;
 
 /// Perform repository-level actions and maintenance
@@ -45,19 +50,128 @@ pub enum RepoCommand {
         #[clap(name = "REPO")]
         repo: String,
     },
+    /// Generate an index for a repository
+    Index {
+        /// Repository to generate or update an index from.
+        #[clap(long, short = 'r')]
+        repo: String,
+
+        /// Package or package/version of a published package to
+        /// update in an existing index.
+        ///
+        /// Can be specified multiple times. Other packages in the
+        /// index will not be updated. Without this the full index
+        /// will be constructed from scratch. If the repo does not
+        /// have an index, a full index will be constructed from
+        /// scratch if the repository supports an index.
+        ///
+        /// This option is only supported for flatbuffer indexes.
+        #[clap(long, name = "PACKAGE/VERSION")]
+        update: Vec<String>,
+    },
 }
 
 impl RepoCommand {
     pub async fn run(&mut self) -> Result<i32> {
-        let repo = match &self {
-            Self::Upgrade { repo } => repo,
-        };
-        let repo = match repo.as_str() {
-            "local" => storage::local_repository().await?,
-            _ => storage::remote_repository(repo).await?,
-        };
-        let status = repo.upgrade().await.wrap_err("Upgrade failed")?;
-        tracing::info!("{}", status);
-        Ok(1)
+        match &self {
+            // spk repo upgrade ...
+            Self::Upgrade { repo: repo_name } => {
+                let repo = match repo_name.as_str() {
+                    "local" => storage::local_repository().await?,
+                    _ => storage::remote_repository(repo_name).await?,
+                };
+
+                let status = repo.upgrade().await.wrap_err("Upgrade failed")?;
+                tracing::info!("{}", status);
+                Ok(1)
+            }
+
+            // spk repo index ...
+            Self::Index { repo, update } => {
+                // Generate or update an index a repo. The repo must
+                // be the underlying repo and not an indexed repo. So as
+                // a safety measure, this disables index use for this
+                // command regardless of config or command line flags.
+                flags::disable_index_use();
+
+                // Construct the repo handle to operate on, and repo
+                // list that contains it.
+                let repo_to_index: RepositoryHandle = match repo.as_str() {
+                    "local" => storage::local_repository().await?.into(),
+                    name => storage::remote_repository(name).await?.into(),
+                };
+                let repos = vec![(repo_to_index.name().to_string(), repo_to_index.clone())];
+
+                if !update.is_empty() {
+                    // Update the existing index for the given package/version
+                    let start = Instant::now();
+                    let idents: Vec<OptVersionIdent> = update
+                        .iter()
+                        .filter_map(|pv| match OptVersionIdent::from_str(pv) {
+                            Ok(i) => Some(i),
+                            Err(err) => {
+                                tracing::warn!(
+                                    "Skipping '{pv}': Unable to parse it as a package/version: {err}"
+                                );
+                                None
+                            }
+                        })
+                        .collect();
+
+                    tracing::debug!(
+                        "Command line update option: [{}]",
+                        update.iter().map(ToString::to_string).join(", ")
+                    );
+                    tracing::info!(
+                        "Package/versions to update: [{}]",
+                        idents.iter().map(ToString::to_string).join(", ")
+                    );
+                    if idents.is_empty() {
+                        tracing::error!(
+                            "No valid package/versions given, nothing to update. Stopping."
+                        );
+                        return Ok(2);
+                    }
+
+                    // Load the current index for this repo now
+                    let mut was_full_index = String::from("");
+                    match FlatBufferRepoIndex::from_repo_file(&repo_to_index).await {
+                        Ok(current_index) => {
+                            current_index
+                                .update_packages(&repo_to_index, &idents)
+                                .await?
+                        }
+                        Err(err) => {
+                            // There isn't an existing index, so generate one from scratch that
+                            // will also include the update package version.
+                            tracing::warn!("Failed to load flatbuffer index: {err}");
+                            tracing::warn!("No current index to update. Creating a full index ...");
+                            FlatBufferRepoIndex::index_repo(&repos).await?;
+                            was_full_index =
+                                " [no previous index, so a full index was created]".to_string()
+                        }
+                    };
+
+                    tracing::info!(
+                        "Index update for '{}' in '{}' repo completed in: {} secs{was_full_index}",
+                        idents.iter().map(ToString::to_string).join(", "),
+                        repo_to_index.name(),
+                        start.elapsed().as_secs_f64()
+                    );
+                } else {
+                    // Generate a full index from scratch
+                    let start = Instant::now();
+                    FlatBufferRepoIndex::index_repo(&repos).await?;
+
+                    tracing::info!(
+                        "Index generation for '{}' repo completed in: {} secs",
+                        repo_to_index.name(),
+                        start.elapsed().as_secs_f64()
+                    );
+                }
+
+                Ok(0)
+            }
+        }
     }
 }