Indexes 7: Adds a lock file around index generation and updates

crates/spfs/src/storage/fs/tag.rs

@@ -953,7 +953,7 @@ impl TagLock {
     /// # Safety:
     /// Tag locks are used to ensure that only one process is writing to a tag file at a time.
     /// Removing the lock file without ensuring that the tag file is not being written to may
-    /// cause the data within the file to become corrupt.
+    /// cause the data within the tag file to become corrupt.
     pub unsafe fn remove<P: AsRef<Path>>(tag_file: P) -> Result<()> {
         let mut lock_file = tag_file.as_ref().to_path_buf();
         lock_file.set_extension(Self::LOCK_EXT);

crates/spk-cli/cmd-repo/src/cmd_repo.rs

@@ -135,23 +135,37 @@ impl RepoCommand {
 
                     // Load the current index for this repo now
                     let mut was_full_index = String::from("");
-                    match FlatBufferRepoIndex::from_repo_file(&repo_to_index).await {
+                    let result = match FlatBufferRepoIndex::from_repo_file(&repo_to_index).await {
                         Ok(current_index) => {
-                            current_index
-                                .update_packages(&repo_to_index, &idents)
-                                .await?
+                            current_index.update_packages(&repo_to_index, &idents).await
                         }
                         Err(err) => {
                             // There isn't an existing index, so generate one from scratch that
                             // will also include the update package version.
                             tracing::warn!("Failed to load flatbuffer index: {err}");
                             tracing::warn!("No current index to update. Creating a full index ...");
-                            FlatBufferRepoIndex::index_repo(&repos).await?;
                             was_full_index =
-                                " [no previous index, so a full index was created]".to_string()
+                                " [no previous index, so a full index was created]".to_string();
+                            FlatBufferRepoIndex::index_repo(&repos).await
                         }
                     };
 
+                    if result.is_err() {
+                        // Need to keep these if-statements separate
+                        // to allow for two different error handling cases.
+                        #[allow(clippy::collapsible_if)]
+                        if let Some(err) = result.err() {
+                            if let Some(spk_storage::Error::UnableToGetWriteLockError(..)) =
+                                err.root_cause().downcast_ref::<spk_storage::Error>()
+                            {
+                                tracing::error!("{err}");
+                                // A distinct exit code when unable to lock the index file.
+                                return Ok(3);
+                            }
+                            return Err(err);
+                        }
+                    }
+
                     tracing::info!(
                         "Index update for '{}' in '{}' repo completed in: {} secs{was_full_index}",
                         idents.iter().map(ToString::to_string).join(", "),

crates/spk-config/src/config.rs

@@ -104,6 +104,14 @@ pub struct Index {
     /// one kind of index available for the repository. The default is
     /// 'flatb', a flatbuffers file based index.
     pub kind: String,
+
+    /// Time to sleep between getting a write lock on the index data,
+    /// for index generation and updates.
+    pub lock_sleep_seconds: u64,
+
+    /// Maximum number of times to try to get a write lock on the
+    /// index data, for index generation and updates.
+    pub lock_max_tries: u64,
 }
 
 impl Default for Index {
@@ -113,6 +121,12 @@ impl Default for Index {
             // safer but can add some overhead.
             verify_before_use: true,
             kind: String::from(FLATBUFFER_INDEX_TOKEN),
+            // Sleeping for 6 seconds between write lock attempts and
+            // allowing 5 tries before bailing out, gives a total of
+            // about 30 seconds before timing out of getting a lock
+            // for index writing.
+            lock_sleep_seconds: 6,
+            lock_max_tries: 5,
         }
     }
 }

crates/spk-storage/Cargo.toml

@@ -13,6 +13,10 @@ description = { workspace = true }
 workspace = true
 
 [features]
+sentry = [
+    "dep:sentry",
+    "dep:sentry-miette",
+]
 
 [dependencies]
 arc-swap = { workspace = true }
@@ -33,6 +37,7 @@ glob = { workspace = true }
 ignore = "0.4.18"
 indexmap = { workspace = true }
 itertools = { workspace = true }
+libc = { workspace = true }
 memmap2 = { workspace = true }
 miette = { workspace = true }
 nom = { workspace = true }
@@ -44,6 +49,7 @@ relative-path = { workspace = true }
 ring = { workspace = true }
 rstest = { workspace = true }
 sentry = { workspace = true, optional = true }
+sentry-miette = { workspace = true, optional = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 serde_yaml = { workspace = true }
@@ -61,6 +67,7 @@ tracing-subscriber = { workspace = true }
 ulid = { workspace = true }
 url = "2.2"
 variantly = { workspace = true }
+whoami = { workspace = true }
 
 [dev-dependencies]
 spfstest = { workspace = true }

crates/spk-storage/src/error.rs

@@ -91,6 +91,14 @@ pub enum Error {
     IndexFailedToGenerate(String),
     #[error("Unknown index kind: '{0}', unable to {1}load that kind of index")]
     IndexUnknownKind(String, String),
+    #[error("Unable to open the {0} lock file at all: {1}: {2}")]
+    UnableToOpenLockFileError(String, String, String),
+    #[error(
+        "Unable to lock the {0} file exclusively. {1} tries with {2} seconds between each. Lock is held by {3} for {4}. Giving up. Try again later, or investigate the process that made the lock file."
+    )]
+    UnableToGetWriteLockError(String, u64, u64, String, String),
+    #[error("Failed to remove the {0} lock file: {1}: {2}")]
+    UnableToRemoveWriteLockError(String, String, String),
 
     #[error("{0}")]
     String(String),