diff --git a/CHANGELOG.md b/CHANGELOG.md index ddb6768e8605..1e72a20887b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## 4.0.9 / 2026-03-25 + +### Enhancements: + +* Fix: include owner role in `gem owner`. Pull request [#9403](https://github.com/ruby/rubygems/pull/9403) by gjtorikian +* Installs bundler 4.0.9 as a default gem. + +### Bug fixes: + +* Fix: Ensure trailing slash is added to source URIs added via gem sources. Pull request [#9055](https://github.com/ruby/rubygems/pull/9055) by zirni + +### Documentation: + +* [DOC] Fix link. Pull request [#9409](https://github.com/ruby/rubygems/pull/9409) by BurdetteLamar + ## 4.0.8 / 2026-03-11 ### Enhancements: diff --git a/bundler/CHANGELOG.md b/bundler/CHANGELOG.md index 6ae9e04962ba..cdf2ea026864 100644 --- a/bundler/CHANGELOG.md +++ b/bundler/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 4.0.9 / 2026-03-25 + +### Enhancements: + +* Check the git version only **once** per `bundle install`. Pull request [#9406](https://github.com/ruby/rubygems/pull/9406) by Edouard-chin +* Normalize the number of workers when performing parallel operations. Pull request [#9400](https://github.com/ruby/rubygems/pull/9400) by Edouard-chin +* Add exponential backoff to bundler retries. Pull request [#9163](https://github.com/ruby/rubygems/pull/9163) by ChrisBr +* Introduce a priority queue. Pull request [#9389](https://github.com/ruby/rubygems/pull/9389) by Edouard-chin +* Split the download and install process of a gem. Pull request [#9381](https://github.com/ruby/rubygems/pull/9381) by Edouard-chin + +### Bug fixes: + +* Retry git fetch without --depth for dumb HTTP transport. Pull request [#9405](https://github.com/ruby/rubygems/pull/9405) by hsbt + ## 4.0.8 (2026-03-11) ### Enhancements: diff --git a/bundler/lib/bundler/cli/pristine.rb b/bundler/lib/bundler/cli/pristine.rb index b8545fe4c9c3..f463f0bce824 100644 --- a/bundler/lib/bundler/cli/pristine.rb +++ b/bundler/lib/bundler/cli/pristine.rb @@ -53,7 +53,7 @@ def run true end.map(&:name) - jobs = installer.send(:installation_parallelization) + jobs = Bundler.settings.installation_parallelization pristine_count = definition.specs.count - installed_specs.count # allow a pristining a single gem to skip the parallel worker jobs = [jobs, pristine_count].min diff --git a/bundler/lib/bundler/definition.rb b/bundler/lib/bundler/definition.rb index efc749e9b326..d9abc85d228f 100644 --- a/bundler/lib/bundler/definition.rb +++ b/bundler/lib/bundler/definition.rb @@ -1122,7 +1122,9 @@ def source_requirements end def preload_git_source_worker - @preload_git_source_worker ||= Bundler::Worker.new(5, "Git source preloading", ->(source, _) { source.specs }) + workers = Bundler.settings.installation_parallelization + + @preload_git_source_worker ||= Bundler::Worker.new(workers, "Git source preloading", ->(source, _) { source.specs }) end def preload_git_sources diff --git a/bundler/lib/bundler/fetcher/gem_remote_fetcher.rb b/bundler/lib/bundler/fetcher/gem_remote_fetcher.rb index 3c3c1826a1b1..3159e056880a 100644 --- a/bundler/lib/bundler/fetcher/gem_remote_fetcher.rb +++ b/bundler/lib/bundler/fetcher/gem_remote_fetcher.rb @@ -8,7 +8,7 @@ class GemRemoteFetcher < Gem::RemoteFetcher def initialize(*) super - @pool_size = 5 + @pool_size = Bundler.settings.installation_parallelization end def request(*args) diff --git a/bundler/lib/bundler/installer.rb b/bundler/lib/bundler/installer.rb index c5fd75431f41..3455f72c2143 100644 --- a/bundler/lib/bundler/installer.rb +++ b/bundler/lib/bundler/installer.rb @@ -189,21 +189,13 @@ def install(options) standalone = options[:standalone] force = options[:force] local = options[:local] || options[:"prefer-local"] - jobs = installation_parallelization + jobs = Bundler.settings.installation_parallelization spec_installations = ParallelInstaller.call(self, @definition.specs, jobs, standalone, force, local: local) spec_installations.each do |installation| post_install_messages[installation.name] = installation.post_install_message if installation.has_post_install_message? end end - def installation_parallelization - if jobs = Bundler.settings[:jobs] - return jobs - end - - Bundler.settings.processor_count - end - def load_plugins Gem.load_plugins diff --git a/bundler/lib/bundler/installer/gem_installer.rb b/bundler/lib/bundler/installer/gem_installer.rb index 5c4fa7825325..f3b43c31ee09 100644 --- a/bundler/lib/bundler/installer/gem_installer.rb +++ b/bundler/lib/bundler/installer/gem_installer.rb @@ -25,6 +25,20 @@ def install_from_spec [false, specific_failure_message(e)] end + def download + spec.source.download( + spec, + force: force, + local: local, + build_args: Array(spec_settings), + previous_spec: previous_spec, + ) + + [true, nil] + rescue Bundler::BundlerError => e + [false, specific_failure_message(e)] + end + private def specific_failure_message(e) diff --git a/bundler/lib/bundler/installer/parallel_installer.rb b/bundler/lib/bundler/installer/parallel_installer.rb index d10e5ec92403..020db30b8443 100644 --- a/bundler/lib/bundler/installer/parallel_installer.rb +++ b/bundler/lib/bundler/installer/parallel_installer.rb @@ -24,6 +24,10 @@ def enqueued? state == :enqueued end + def enqueue_with_priority? + state == :installable && spec.extensions.any? + end + def failed? state == :failed end @@ -32,6 +36,12 @@ def ready_to_enqueue? state == :none end + def ready_to_install?(installed_specs) + return false unless state == :downloaded + + spec.extensions.none? || dependencies_installed?(installed_specs) + end + def has_post_install_message? !post_install_message.empty? end @@ -84,6 +94,7 @@ def initialize(installer, all_specs, size, standalone, force, local: false, skip def call if @rake + do_download(@rake, 0) do_install(@rake, 0) Gem::Specification.reset end @@ -107,26 +118,54 @@ def failed_specs end def install_with_worker - enqueue_specs - process_specs until finished_installing? + installed_specs = {} + enqueue_specs(installed_specs) + + process_specs(installed_specs) until finished_installing? end def install_serially until finished_installing? raise "failed to find a spec to enqueue while installing serially" unless spec_install = @specs.find(&:ready_to_enqueue?) spec_install.state = :enqueued + do_download(spec_install, 0) do_install(spec_install, 0) end end def worker_pool @worker_pool ||= Bundler::Worker.new @size, "Parallel Installer", lambda {|spec_install, worker_num| - do_install(spec_install, worker_num) + case spec_install.state + when :enqueued + do_download(spec_install, worker_num) + when :installable + do_install(spec_install, worker_num) + else + spec_install + end } end - def do_install(spec_install, worker_num) + def do_download(spec_install, worker_num) Plugin.hook(Plugin::Events::GEM_BEFORE_INSTALL, spec_install) + + gem_installer = Bundler::GemInstaller.new( + spec_install.spec, @installer, @standalone, worker_num, @force, @local + ) + + success, message = gem_installer.download + + if success + spec_install.state = :downloaded + else + spec_install.error = "#{message}\n\n#{require_tree_for_spec(spec_install.spec)}" + spec_install.state = :failed + end + + spec_install + end + + def do_install(spec_install, worker_num) gem_installer = Bundler::GemInstaller.new( spec_install.spec, @installer, @standalone, worker_num, @force, @local ) @@ -147,9 +186,19 @@ def do_install(spec_install, worker_num) # Some specs might've had to wait til this spec was installed to be # processed so the call to `enqueue_specs` is important after every # dequeue. - def process_specs - worker_pool.deq - enqueue_specs + def process_specs(installed_specs) + spec = worker_pool.deq + + if spec.installed? + installed_specs[spec.name] = true + return + elsif spec.failed? + return + elsif spec.ready_to_install?(installed_specs) + spec.state = :installable + end + + worker_pool.enq(spec, priority: spec.enqueue_with_priority?) end def finished_installing? @@ -185,18 +234,15 @@ def require_tree_for_spec(spec) # Later we call this lambda again to install specs that depended on # previously installed specifications. We continue until all specs # are installed. - def enqueue_specs - installed_specs = {} - @specs.each do |spec| - next unless spec.installed? - installed_specs[spec.name] = true - end - + def enqueue_specs(installed_specs) @specs.each do |spec| - if spec.ready_to_enqueue? && spec.dependencies_installed?(installed_specs) - spec.state = :enqueued - worker_pool.enq spec + if spec.installed? + installed_specs[spec.name] = true + next end + + spec.state = :enqueued + worker_pool.enq spec end end end diff --git a/bundler/lib/bundler/man/bundle-config.1 b/bundler/lib/bundler/man/bundle-config.1 index 000fe664da6c..ed66ba9a4817 100644 --- a/bundler/lib/bundler/man/bundle-config.1 +++ b/bundler/lib/bundler/man/bundle-config.1 @@ -146,7 +146,7 @@ When set, no post install messages will be printed\. To silence a single gem, us Generate a \fBgems\.rb\fR instead of a \fBGemfile\fR when running \fBbundle init\fR\. .TP \fBjobs\fR (\fBBUNDLE_JOBS\fR) -The number of gems Bundler can install in parallel\. Defaults to the number of available processors\. +The number of gems Bundler can download and install in parallel\. Defaults to the number of available processors\. .TP \fBlockfile\fR (\fBBUNDLE_LOCKFILE\fR) The path to the lockfile that bundler should use\. By default, Bundler adds \fB\.lock\fR to the end of the \fBgemfile\fR entry\. Can be set to \fBfalse\fR in the Gemfile to disable lockfile creation entirely (see gemfile(5))\. diff --git a/bundler/lib/bundler/man/bundle-config.1.ronn b/bundler/lib/bundler/man/bundle-config.1.ronn index a8670a36709d..b70293cfedda 100644 --- a/bundler/lib/bundler/man/bundle-config.1.ronn +++ b/bundler/lib/bundler/man/bundle-config.1.ronn @@ -192,8 +192,8 @@ learn more about their operation in [bundle install(1)](bundle-install.1.html). * `init_gems_rb` (`BUNDLE_INIT_GEMS_RB`): Generate a `gems.rb` instead of a `Gemfile` when running `bundle init`. * `jobs` (`BUNDLE_JOBS`): - The number of gems Bundler can install in parallel. Defaults to the number of - available processors. + The number of gems Bundler can download and install in parallel. + Defaults to the number of available processors. * `lockfile` (`BUNDLE_LOCKFILE`): The path to the lockfile that bundler should use. By default, Bundler adds `.lock` to the end of the `gemfile` entry. Can be set to `false` in the diff --git a/bundler/lib/bundler/plugin/api/source.rb b/bundler/lib/bundler/plugin/api/source.rb index 6c888d037350..798326673ad7 100644 --- a/bundler/lib/bundler/plugin/api/source.rb +++ b/bundler/lib/bundler/plugin/api/source.rb @@ -74,6 +74,14 @@ def options_to_lock {} end + # Download the gem specified by the spec at appropriate path. + # + # A source plugin can implement this method to split the download and the + # installation of a gem. + # + # @return [Boolean] Whether the download of the gem succeeded. + def download(spec, opts); end + # Install the gem specified by the spec at appropriate path. # `install_path` provides a sufficient default, if the source can only # satisfy one gem, but is not binding. diff --git a/bundler/lib/bundler/plugin/installer.rb b/bundler/lib/bundler/plugin/installer.rb index 853ad9edca4f..9be8b36843bb 100644 --- a/bundler/lib/bundler/plugin/installer.rb +++ b/bundler/lib/bundler/plugin/installer.rb @@ -110,7 +110,8 @@ def install_from_specs(specs) paths = {} specs.each do |spec| - spec.source.install spec + spec.source.download(spec) + spec.source.install(spec) paths[spec.name] = spec end diff --git a/bundler/lib/bundler/retry.rb b/bundler/lib/bundler/retry.rb index 090cb7e2cae1..49b0f638387d 100644 --- a/bundler/lib/bundler/retry.rb +++ b/bundler/lib/bundler/retry.rb @@ -6,6 +6,8 @@ class Retry attr_accessor :name, :total_runs, :current_run class << self + attr_accessor :default_base_delay + def default_attempts default_retries + 1 end @@ -16,11 +18,17 @@ def default_retries end end - def initialize(name, exceptions = nil, retries = self.class.default_retries) + # Set default base delay for exponential backoff + self.default_base_delay = 1.0 + + def initialize(name, exceptions = nil, retries = self.class.default_retries, opts = {}) @name = name @retries = retries @exceptions = Array(exceptions) || [] @total_runs = @retries + 1 # will run once, then upto attempts.times + @base_delay = opts[:base_delay] || self.class.default_base_delay + @max_delay = opts[:max_delay] || 60.0 + @jitter = opts[:jitter] || 0.5 end def attempt(&block) @@ -48,9 +56,27 @@ def fail_attempt(e) Bundler.ui.info "" unless Bundler.ui.debug? raise e end - return true unless name - Bundler.ui.info "" unless Bundler.ui.debug? # Add new line in case dots preceded this - Bundler.ui.warn "Retrying #{name} due to error (#{current_run.next}/#{total_runs}): #{e.class} #{e.message}", true + if name + Bundler.ui.info "" unless Bundler.ui.debug? # Add new line in case dots preceded this + Bundler.ui.warn "Retrying #{name} due to error (#{current_run.next}/#{total_runs}): #{e.class} #{e.message}", true + end + backoff_sleep if @base_delay > 0 + true + end + + def backoff_sleep + # Exponential backoff: delay = base_delay * 2^(attempt - 1) + # Add jitter to prevent thundering herd: random value between 0 and jitter seconds + delay = @base_delay * (2**(@current_run - 1)) + delay = [@max_delay, delay].min + jitter_amount = rand * @jitter + total_delay = delay + jitter_amount + Bundler.ui.debug "Sleeping for #{total_delay.round(2)} seconds before retry" + sleep(total_delay) + end + + def sleep(duration) + Kernel.sleep(duration) end def keep_trying? diff --git a/bundler/lib/bundler/self_manager.rb b/bundler/lib/bundler/self_manager.rb index 1db77fd46b3f..82efbf56a4fb 100644 --- a/bundler/lib/bundler/self_manager.rb +++ b/bundler/lib/bundler/self_manager.rb @@ -63,6 +63,7 @@ def install_and_restart_with(version) end def install(spec) + spec.source.download(spec) spec.source.install(spec) end diff --git a/bundler/lib/bundler/settings.rb b/bundler/lib/bundler/settings.rb index d00a4bb916f6..cb4877808356 100644 --- a/bundler/lib/bundler/settings.rb +++ b/bundler/lib/bundler/settings.rb @@ -303,6 +303,10 @@ def app_cache_path @app_cache_path ||= self[:cache_path] || "vendor/cache" end + def installation_parallelization + self[:jobs] || processor_count + end + def validate! all.each do |raw_key| [@local_config, @env_config, @global_config].each do |settings| diff --git a/bundler/lib/bundler/source.rb b/bundler/lib/bundler/source.rb index 2b90a0eff1bf..cf71be880125 100644 --- a/bundler/lib/bundler/source.rb +++ b/bundler/lib/bundler/source.rb @@ -31,6 +31,8 @@ def version_message(spec, locked_spec = nil) message end + def download(*); end + def can_lock?(spec) spec.source == self end diff --git a/bundler/lib/bundler/source/git/git_proxy.rb b/bundler/lib/bundler/source/git/git_proxy.rb index fe05e9d57b27..72f7dc771038 100644 --- a/bundler/lib/bundler/source/git/git_proxy.rb +++ b/bundler/lib/bundler/source/git/git_proxy.rb @@ -57,6 +57,29 @@ class GitProxy attr_accessor :path, :uri, :branch, :tag, :ref, :explicit_ref attr_writer :revision + def self.version + @version ||= full_version[/((\.?\d+)+).*/, 1] + end + + def self.full_version + @full_version ||= begin + raise GitNotInstalledError.new unless Bundler.git_present? + + require "open3" + out, err, status = Open3.capture3("git", "--version") + + raise GitCommandError.new("--version", SharedHelpers.pwd, err) unless status.success? + Bundler.ui.warn err unless err.empty? + + out.sub(/git version\s*/, "").strip + end + end + + def self.reset + @version = nil + @full_version = nil + end + def initialize(path, uri, options = {}, revision = nil, git = nil) @path = path @uri = uri @@ -92,11 +115,11 @@ def contains?(commit) end def version - @version ||= full_version.match(/((\.?\d+)+).*/)[1] + self.class.version end def full_version - @full_version ||= git_local("--version").sub(/git version\s*/, "").strip + self.class.full_version end def checkout @@ -156,7 +179,7 @@ def installed_to?(destination) private def git_remote_fetch(args) - command = ["fetch", "--force", "--quiet", "--no-tags", *args, "--", configured_uri, refspec].compact + command = fetch_command(args) command_with_no_credentials = check_allowed(command) Bundler::Retry.new("`#{command_with_no_credentials}` at #{path}", [MissingGitRevisionError]).attempts do @@ -166,6 +189,11 @@ def git_remote_fetch(args) if err.include?("couldn't find remote ref") || err.include?("not our ref") raise MissingGitRevisionError.new(command_with_no_credentials, path, commit || explicit_ref, credential_filtered_uri) else + if shallow? + args -= depth_args + command = fetch_command(args) + command_with_no_credentials = check_allowed(command) + end raise GitCommandError.new(command_with_no_credentials, path, err) end end @@ -178,7 +206,8 @@ def clone_needs_extra_fetch? FileUtils.mkdir_p(p) end - command = ["clone", "--bare", "--no-hardlinks", "--quiet", *extra_clone_args, "--", configured_uri, path.to_s] + clone_args = extra_clone_args + command = clone_command(clone_args) command_with_no_credentials = check_allowed(command) Bundler::Retry.new("`#{command_with_no_credentials}`", [MissingGitRevisionError]).attempts do @@ -189,13 +218,10 @@ def clone_needs_extra_fetch? err.include?("Remote branch #{branch_option} not found") # git 2.49 or higher raise MissingGitRevisionError.new(command_with_no_credentials, nil, explicit_ref, credential_filtered_uri) else - idx = command.index("--depth") - if idx - command.delete_at(idx) - command.delete_at(idx) + if shallow? + clone_args -= depth_args + command = clone_command(clone_args) command_with_no_credentials = check_allowed(command) - - err += "Retrying without --depth argument." end raise GitCommandError.new(command_with_no_credentials, path, err) end @@ -204,14 +230,14 @@ def clone_needs_extra_fetch? def clone_needs_unshallow? return false unless path.join("shallow").exist? - return true if full_clone? + return true unless shallow? @revision && @revision != head_revision end def extra_ref return false if not_pinned? - return true unless full_clone? + return true if shallow? ref.start_with?("refs/") end @@ -427,8 +453,16 @@ def extra_clone_args args end + def fetch_command(args) + ["fetch", "--force", "--quiet", "--no-tags", *args, "--", configured_uri, refspec].compact + end + + def clone_command(args) + ["clone", "--bare", "--no-hardlinks", "--quiet", *args, "--", configured_uri, path.to_s] + end + def depth_args - return [] if full_clone? + return [] unless shallow? ["--depth", depth.to_s] end @@ -443,8 +477,8 @@ def branch_option branch || tag end - def full_clone? - depth.nil? + def shallow? + !depth.nil? end def needs_allow_any_sha1_in_want? diff --git a/bundler/lib/bundler/source/rubygems.rb b/bundler/lib/bundler/source/rubygems.rb index e1e030ffc899..5a77d6448eb0 100644 --- a/bundler/lib/bundler/source/rubygems.rb +++ b/bundler/lib/bundler/source/rubygems.rb @@ -9,6 +9,7 @@ class Rubygems < Source # Ask for X gems per API request API_REQUEST_SIZE = 100 + REQUIRE_MUTEX = Mutex.new attr_accessor :remotes @@ -21,6 +22,8 @@ def initialize(options = {}) @allow_local = options["allow_local"] || false @prefer_local = false @checksum_store = Checksum::Store.new + @gem_installers = {} + @gem_installers_mutex = Mutex.new Array(options["remotes"]).reverse_each {|r| add_remote(r) } @@ -162,49 +165,40 @@ def specs end end - def install(spec, options = {}) + def download(spec, options = {}) if (spec.default_gem? && !cached_built_in_gem(spec, local: options[:local])) || (installed?(spec) && !options[:force]) - print_using_message "Using #{version_message(spec, options[:previous_spec])}" - return nil # no post-install message + return true end - path = fetch_gem_if_possible(spec, options[:previous_spec]) - raise GemNotFound, "Could not find #{spec.file_name} for installation" unless path - - return if Bundler.settings[:no_install] - - install_path = rubygems_dir - bin_path = Bundler.system_bindir - - require_relative "../rubygems_gem_installer" - - installer = Bundler::RubyGemsGemInstaller.at( - path, - security_policy: Bundler.rubygems.security_policies[Bundler.settings["trust-policy"]], - install_dir: install_path.to_s, - bin_dir: bin_path.to_s, - ignore_dependencies: true, - wrappers: true, - env_shebang: true, - build_args: options[:build_args], - bundler_extension_cache_path: extension_cache_path(spec) - ) + installer = rubygems_gem_installer(spec, options) if spec.remote s = begin installer.spec rescue Gem::Package::FormatError - Bundler.rm_rf(path) + Bundler.rm_rf(installer.gem) raise rescue Gem::Security::Exception => e raise SecurityError, - "The gem #{File.basename(path, ".gem")} can't be installed because " \ + "The gem #{installer.gem} can't be installed because " \ "the security policy didn't allow it, with the message: #{e.message}" end spec.__swap__(s) end + spec + end + + def install(spec, options = {}) + if (spec.default_gem? && !cached_built_in_gem(spec, local: options[:local])) || (installed?(spec) && !options[:force]) + print_using_message "Using #{version_message(spec, options[:previous_spec])}" + return nil # no post-install message + end + + return if Bundler.settings[:no_install] + + installer = rubygems_gem_installer(spec, options) spec.source.checksum_store.register(spec, installer.gem_checksum) message = "Installing #{version_message(spec, options[:previous_spec])}" @@ -511,6 +505,34 @@ def extension_cache_slug(spec) return unless remote = spec.remote remote.cache_slug end + + # We are using a mutex to reaed and write from/to the hash. + # The reason this double synchronization was added is for performance + # and lock the mutex for the shortest possible amount of time. Otherwise, + # all threads are fighting over this mutex and when it gets acquired it gets locked + # until a threads finishes downloading a gem, leaving the other threads waiting + # doing nothing. + def rubygems_gem_installer(spec, options) + @gem_installers_mutex.synchronize { @gem_installers[spec.name] } || begin + path = fetch_gem_if_possible(spec, options[:previous_spec]) + raise GemNotFound, "Could not find #{spec.file_name} for installation" unless path + + REQUIRE_MUTEX.synchronize { require_relative "../rubygems_gem_installer" } + + installer = Bundler::RubyGemsGemInstaller.at( + path, + security_policy: Bundler.rubygems.security_policies[Bundler.settings["trust-policy"]], + install_dir: rubygems_dir.to_s, + bin_dir: Bundler.system_bindir.to_s, + ignore_dependencies: true, + wrappers: true, + env_shebang: true, + build_args: options[:build_args], + bundler_extension_cache_path: extension_cache_path(spec) + ) + @gem_installers_mutex.synchronize { @gem_installers[spec.name] ||= installer } + end + end end end end diff --git a/bundler/lib/bundler/version.rb b/bundler/lib/bundler/version.rb index 68a3af1a3af4..94fe327a2979 100644 --- a/bundler/lib/bundler/version.rb +++ b/bundler/lib/bundler/version.rb @@ -1,7 +1,7 @@ # frozen_string_literal: false module Bundler - VERSION = "4.0.8".freeze + VERSION = "4.0.9".freeze def self.bundler_major_version @bundler_major_version ||= gem_version.segments.first diff --git a/bundler/lib/bundler/worker.rb b/bundler/lib/bundler/worker.rb index 7137484cc6d0..77f4f004aa69 100644 --- a/bundler/lib/bundler/worker.rb +++ b/bundler/lib/bundler/worker.rb @@ -22,6 +22,7 @@ def initialize(exn) def initialize(size, name, func) @name = name @request_queue = Thread::Queue.new + @request_queue_with_priority = Thread::Queue.new @response_queue = Thread::Queue.new @func = func @size = size @@ -32,9 +33,10 @@ def initialize(size, name, func) # Enqueue a request to be executed in the worker pool # # @param obj [String] mostly it is name of spec that should be downloaded - def enq(obj) + def enq(obj, priority: false) + queue = priority ? @request_queue_with_priority : @request_queue create_threads unless @threads - @request_queue.enq obj + queue.enq obj end # Retrieves results of job function being executed in worker pool @@ -52,7 +54,13 @@ def stop def process_queue(i) loop do - obj = @request_queue.deq + obj = begin + @request_queue_with_priority.deq(true) + rescue ThreadError + @request_queue.deq(false, timeout: 0.05) + end + + next if obj.nil? break if obj.equal? POISON @response_queue.enq apply_func(obj, i) end diff --git a/bundler/spec/bundler/env_spec.rb b/bundler/spec/bundler/env_spec.rb index e0ab0a45e330..259b4ee9dc2f 100644 --- a/bundler/spec/bundler/env_spec.rb +++ b/bundler/spec/bundler/env_spec.rb @@ -217,8 +217,9 @@ def with_clear_paths(env_var, env_value) context "when the git version is OS specific" do it "includes OS specific information with the version number" do - expect(git_proxy_stub).to receive(:git_local).with("--version"). - and_return("git version 1.2.3 (Apple Git-BS)") + status = double("success?" => true) + expect(Open3).to receive(:capture3).with("git", "--version"). + and_return(["git version 1.2.3 (Apple Git-BS)", "", status]) expect(Bundler::Source::Git::GitProxy).to receive(:new).and_return(git_proxy_stub) expect(described_class.report).to include("Git 1.2.3 (Apple Git-BS)") diff --git a/bundler/spec/bundler/gem_helper_spec.rb b/bundler/spec/bundler/gem_helper_spec.rb index 94f66537d3eb..0e67afa1cf59 100644 --- a/bundler/spec/bundler/gem_helper_spec.rb +++ b/bundler/spec/bundler/gem_helper_spec.rb @@ -222,7 +222,7 @@ def sha512_hexdigest(path) mock_confirm_message "#{app_name} (#{app_version}) installed." subject.install_gem(nil, :local) expect(app_gem_path).to exist - gem_command :list + installed_gems_list expect(out).to include("#{app_name} (#{app_version})") end end diff --git a/bundler/spec/bundler/installer/parallel_installer_spec.rb b/bundler/spec/bundler/installer/parallel_installer_spec.rb new file mode 100644 index 000000000000..49bcb5310ba9 --- /dev/null +++ b/bundler/spec/bundler/installer/parallel_installer_spec.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +require "bundler/installer/parallel_installer" +require "bundler/rubygems_gem_installer" +require "rubygems/remote_fetcher" +require "bundler" + +RSpec.describe Bundler::ParallelInstaller do + describe "priority queue" do + before do + require "support/artifice/compact_index" + + @previous_client = Gem::Request::ConnectionPools.client + Gem::Request::ConnectionPools.client = Gem::Net::HTTP + Gem::RemoteFetcher.fetcher.close_all + + build_repo2 do + build_gem "gem_with_extension", &:add_c_extension + build_gem "gem_without_extension" + end + + gemfile <<~G + source "https://gem.repo2" + + gem "gem_with_extension" + gem "gem_without_extension" + G + lockfile <<~L + GEM + remote: https://gem.repo2/ + specs: + gem_with_extension (1.0) + gem_without_extension (1.0) + + DEPENDENCIES + gem_with_extension + gem_without_extension + L + + @old_ui = Bundler.ui + Bundler.ui = Bundler::UI::Silent.new + end + + after do + Bundler.ui = @old_ui + Gem::Request::ConnectionPools.client = @previous_client + Artifice.deactivate + end + + let(:definition) do + allow(Bundler).to receive(:root) { bundled_app } + + definition = Bundler::Definition.build(bundled_app.join("Gemfile"), bundled_app.join("Gemfile.lock"), false) + definition.tap(&:setup_domain!) + end + let(:installer) { Bundler::Installer.new(bundled_app, definition) } + + it "queues native extensions in priority" do + parallel_installer = Bundler::ParallelInstaller.new(installer, definition.specs, 2, false, true) + worker_pool = parallel_installer.send(:worker_pool) + expected = 6 # Enqueue to download bundler and the 2 gems. Enqueue to install Bundler and the 2 gems. + + expect(worker_pool).to receive(:enq).exactly(expected).times.and_wrap_original do |original_enq, spec, opts| + unless opts.nil? # Enqueued for download, no priority + if spec.name == "gem_with_extension" + expect(opts).to eq({ priority: true }) + else + expect(opts).to eq({ priority: false }) + end + end + + opts ||= {} + original_enq.call(spec, **opts) + end + + parallel_installer.call + end + end +end diff --git a/bundler/spec/bundler/retry_spec.rb b/bundler/spec/bundler/retry_spec.rb index 7481622a967d..5c84d0bea5c8 100644 --- a/bundler/spec/bundler/retry_spec.rb +++ b/bundler/spec/bundler/retry_spec.rb @@ -78,4 +78,113 @@ end end end + + context "exponential backoff" do + it "can be disabled by setting base_delay to 0" do + attempts = 0 + expect do + Bundler::Retry.new("test", [], 2, base_delay: 0).attempt do + attempts += 1 + raise "error" + end + end.to raise_error(StandardError) + + # Verify no sleep was called (implicitly - if sleep was called, timing would be different) + expect(attempts).to eq(3) + end + + it "is enabled by default with 1 second base delay" do + original_base_delay = Bundler::Retry.default_base_delay + Bundler::Retry.default_base_delay = 1.0 + + attempts = 0 + sleep_times = [] + + allow_any_instance_of(Bundler::Retry).to receive(:sleep) do |_instance, delay| + sleep_times << delay + end + + expect do + Bundler::Retry.new("test", [], 2, jitter: 0).attempt do + attempts += 1 + raise "error" + end + end.to raise_error(StandardError) + + expect(attempts).to eq(3) + expect(sleep_times.length).to eq(2) + # First retry: 1.0 * 2^0 = 1.0 + expect(sleep_times[0]).to eq(1.0) + # Second retry: 1.0 * 2^1 = 2.0 + expect(sleep_times[1]).to eq(2.0) + ensure + Bundler::Retry.default_base_delay = original_base_delay + end + + it "sleeps with exponential backoff when base_delay is set" do + attempts = 0 + sleep_times = [] + + allow_any_instance_of(Bundler::Retry).to receive(:sleep) do |_instance, delay| + sleep_times << delay + end + + expect do + Bundler::Retry.new("test", [], 2, base_delay: 1.0, jitter: 0).attempt do + attempts += 1 + raise "error" + end + end.to raise_error(StandardError) + + expect(attempts).to eq(3) + expect(sleep_times.length).to eq(2) + # First retry: 1.0 * 2^0 = 1.0 + expect(sleep_times[0]).to eq(1.0) + # Second retry: 1.0 * 2^1 = 2.0 + expect(sleep_times[1]).to eq(2.0) + end + + it "respects max_delay" do + sleep_times = [] + + allow_any_instance_of(Bundler::Retry).to receive(:sleep) do |_instance, delay| + sleep_times << delay + end + + expect do + Bundler::Retry.new("test", [], 3, base_delay: 10.0, max_delay: 15.0, jitter: 0).attempt do + raise "error" + end + end.to raise_error(StandardError) + + # First retry: 10.0 * 2^0 = 10.0 + expect(sleep_times[0]).to eq(10.0) + # Second retry: 10.0 * 2^1 = 20.0, capped at 15.0 + expect(sleep_times[1]).to eq(15.0) + # Third retry: 10.0 * 2^2 = 40.0, capped at 15.0 + expect(sleep_times[2]).to eq(15.0) + end + + it "adds jitter to delay" do + sleep_times = [] + + allow_any_instance_of(Bundler::Retry).to receive(:sleep) do |_instance, delay| + sleep_times << delay + end + + expect do + Bundler::Retry.new("test", [], 2, base_delay: 1.0, jitter: 0.5).attempt do + raise "error" + end + end.to raise_error(StandardError) + + expect(sleep_times.length).to eq(2) + # First retry should be between 1.0 and 1.5 (base + jitter) + expect(sleep_times[0]).to be >= 1.0 + expect(sleep_times[0]).to be <= 1.5 + # Second retry should be between 2.0 and 2.5 + expect(sleep_times[1]).to be >= 2.0 + expect(sleep_times[1]).to be <= 2.5 + end + end end diff --git a/bundler/spec/bundler/source/git/git_proxy_spec.rb b/bundler/spec/bundler/source/git/git_proxy_spec.rb index b2b7ab5c5400..1f10ca4b0776 100644 --- a/bundler/spec/bundler/source/git/git_proxy_spec.rb +++ b/bundler/spec/bundler/source/git/git_proxy_spec.rb @@ -10,7 +10,9 @@ let(:revision) { nil } let(:git_source) { nil } let(:clone_result) { double(Process::Status, success?: true) } + let(:fail_result) { double(Process::Status, success?: false) } let(:base_clone_args) { ["clone", "--bare", "--no-hardlinks", "--quiet", "--no-tags", "--depth", "1", "--single-branch"] } + let(:base_fetch_args) { ["fetch", "--force", "--quiet", "--no-tags", "--depth", "1"] } subject(:git_proxy) { described_class.new(path, uri, options, revision, git_source) } context "with explicit ref" do @@ -99,7 +101,7 @@ describe "#version" do context "with a normal version number" do before do - expect(git_proxy).to receive(:git_local).with("--version"). + expect(described_class).to receive(:full_version). and_return("git version 1.2.3") end @@ -114,7 +116,7 @@ context "with a OSX version number" do before do - expect(git_proxy).to receive(:git_local).with("--version"). + expect(described_class).to receive(:full_version). and_return("git version 1.2.3 (Apple Git-BS)") end @@ -129,7 +131,7 @@ context "with a msysgit version number" do before do - expect(git_proxy).to receive(:git_local).with("--version"). + expect(described_class).to receive(:full_version). and_return("git version 1.2.3.msysgit.0") end @@ -146,8 +148,9 @@ describe "#full_version" do context "with a normal version number" do before do - expect(git_proxy).to receive(:git_local).with("--version"). - and_return("git version 1.2.3") + status = double("success?" => true) + expect(Open3).to receive(:capture3).with("git", "--version"). + and_return(["git version 1.2.3", "", status]) end it "returns the git version number" do @@ -157,8 +160,9 @@ context "with a OSX version number" do before do - expect(git_proxy).to receive(:git_local).with("--version"). - and_return("git version 1.2.3 (Apple Git-BS)") + status = double("success?" => true) + expect(Open3).to receive(:capture3).with("git", "--version"). + and_return(["git version 1.2.3 (Apple Git-BS)", "", status]) end it "does not strip out OSX specific additions in the version string" do @@ -168,8 +172,9 @@ context "with a msysgit version number" do before do - expect(git_proxy).to receive(:git_local).with("--version"). - and_return("git version 1.2.3.msysgit.0") + status = double("success?" => true) + expect(Open3).to receive(:capture3).with("git", "--version"). + and_return(["git version 1.2.3.msysgit.0", "", status]) end it "does not strip out msysgit specific additions in the version string" do @@ -200,13 +205,12 @@ context "URI is HTTP" do let(:uri) { "http://github.com/ruby/rubygems.git" } - let(:without_depth_arguments) { ["clone", "--bare", "--no-hardlinks", "--quiet", "--no-tags", "--single-branch"] } - let(:fail_clone_result) { double(Process::Status, success?: false) } + let(:clone_args_without_depth) { ["clone", "--bare", "--no-hardlinks", "--quiet", "--no-tags", "--single-branch"] } - it "retries without --depth when git url is http and fails" do + it "retries clone without --depth when dumb http transport fails" do allow(git_proxy).to receive(:git_local).with("--version").and_return("git version 2.14.0") - allow(git_proxy).to receive(:capture).with([*base_clone_args, "--", uri, path.to_s], nil).and_return(["", "dumb http transport does not support shallow capabilities", fail_clone_result]) - expect(git_proxy).to receive(:capture).with([*without_depth_arguments, "--", uri, path.to_s], nil).and_return(["", "", clone_result]) + expect(git_proxy).to receive(:capture).with([*base_clone_args, "--", uri, path.to_s], nil).and_return(["", "dumb http transport does not support shallow capabilities", fail_result]) + expect(git_proxy).to receive(:capture).with([*clone_args_without_depth, "--", uri, path.to_s], nil).and_return(["", "", clone_result]) subject.checkout end @@ -332,6 +336,19 @@ subject.checkout end end + + context "URI is HTTP" do + let(:uri) { "http://github.com/ruby/rubygems.git" } + + it "retries fetch without --depth when dumb http transport fails" do + parsed_revision = Digest::SHA1.hexdigest("ruby") + allow(git_proxy).to receive(:git_local).with("rev-parse", "--abbrev-ref", "HEAD", dir: path).and_return(parsed_revision) + allow(git_proxy).to receive(:git_local).with("--version").and_return("git version 2.14.0") + expect(git_proxy).to receive(:capture).with([*base_fetch_args, "--", uri, "refs/heads/#{parsed_revision}:refs/heads/#{parsed_revision}"], path).and_return(["", "dumb http transport does not support shallow capabilities", fail_result]) + expect(git_proxy).to receive(:capture).with(["fetch", "--force", "--quiet", "--no-tags", "--", uri, "refs/heads/#{parsed_revision}:refs/heads/#{parsed_revision}"], path).and_return(["", "", clone_result]) + subject.checkout + end + end end end end diff --git a/bundler/spec/bundler/worker_spec.rb b/bundler/spec/bundler/worker_spec.rb index e4ebbd2932cf..2ad2845e378c 100644 --- a/bundler/spec/bundler/worker_spec.rb +++ b/bundler/spec/bundler/worker_spec.rb @@ -20,6 +20,26 @@ end end + describe "priority queue" do + it "process elements from the priority queue first" do + processed_elements = [] + + function = proc do |element, _| + processed_elements << element + end + + worker = described_class.new(1, "Spec Worker", function) + worker.instance_variable_set(:@threads, []) # Prevent the enqueueing from starting work. + worker.enq("Normal element") + worker.enq("Priority element", priority: true) + worker.send(:create_threads) + + worker.stop + + expect(processed_elements).to eq(["Priority element", "Normal element"]) + end + end + describe "handling interrupts" do let(:status) do pid = Process.fork do diff --git a/bundler/spec/commands/check_spec.rb b/bundler/spec/commands/check_spec.rb index 72da24fb0b2d..4dce7813a624 100644 --- a/bundler/spec/commands/check_spec.rb +++ b/bundler/spec/commands/check_spec.rb @@ -164,7 +164,7 @@ bundle "config set --local path vendor/bundle" bundle :cache - gem_command "uninstall myrack", env: { "GEM_HOME" => vendored_gems.to_s } + uninstall_gem("myrack", env: { "GEM_HOME" => vendored_gems.to_s }) bundle "check", raise_on_error: false expect(err).to include("* myrack (1.0.0)") diff --git a/bundler/spec/commands/clean_spec.rb b/bundler/spec/commands/clean_spec.rb index 81209388ae97..582bfd5fd1c3 100644 --- a/bundler/spec/commands/clean_spec.rb +++ b/bundler/spec/commands/clean_spec.rb @@ -379,7 +379,7 @@ def should_not_have_gems(*gems) gem "myrack" G - gem_command :list + installed_gems_list expect(out).to include("myrack (1.0.0)").and include("thin (1.0)") end @@ -498,7 +498,7 @@ def should_not_have_gems(*gems) end bundle :update, all: true - gem_command :list + installed_gems_list expect(out).to include("foo (1.0.1, 1.0)") end @@ -522,7 +522,7 @@ def should_not_have_gems(*gems) bundle "clean --force" expect(out).to include("Removing foo (1.0)") - gem_command :list + installed_gems_list expect(out).not_to include("foo (1.0)") expect(out).to include("myrack (1.0.0)") end @@ -556,7 +556,7 @@ def should_not_have_gems(*gems) expect(err).to include(system_gem_path.to_s) expect(err).to include("grant write permissions") - gem_command :list + installed_gems_list expect(out).to include("foo (1.0)") expect(out).to include("myrack (1.0.0)") end diff --git a/bundler/spec/install/gems/compact_index_spec.rb b/bundler/spec/install/gems/compact_index_spec.rb index bb4d4011f5b9..d082b9be72ce 100644 --- a/bundler/spec/install/gems/compact_index_spec.rb +++ b/bundler/spec/install/gems/compact_index_spec.rb @@ -997,7 +997,7 @@ def start gem "activemerchant" end G - gem_command "uninstall activemerchant" + uninstall_gem("activemerchant") bundle "update rails", artifice: "compact_index" count = lockfile.match?("CHECKSUMS") ? 2 : 1 # Once in the specs, and once in CHECKSUMS expect(lockfile.scan(/activemerchant \(/).size).to eq(count) diff --git a/bundler/spec/realworld/fixtures/tapioca/Gemfile.lock b/bundler/spec/realworld/fixtures/tapioca/Gemfile.lock index b0d3d94ad81b..f98b1b01daa8 100644 --- a/bundler/spec/realworld/fixtures/tapioca/Gemfile.lock +++ b/bundler/spec/realworld/fixtures/tapioca/Gemfile.lock @@ -46,4 +46,4 @@ DEPENDENCIES tapioca BUNDLED WITH - 4.0.8 + 4.0.9 diff --git a/bundler/spec/realworld/fixtures/warbler/Gemfile.lock b/bundler/spec/realworld/fixtures/warbler/Gemfile.lock index 44174abdf3f8..9d797c254725 100644 --- a/bundler/spec/realworld/fixtures/warbler/Gemfile.lock +++ b/bundler/spec/realworld/fixtures/warbler/Gemfile.lock @@ -36,4 +36,4 @@ DEPENDENCIES warbler! BUNDLED WITH - 4.0.8 + 4.0.9 diff --git a/bundler/spec/spec_helper.rb b/bundler/spec/spec_helper.rb index ac2d2aeb315c..d94e5b810e44 100644 --- a/bundler/spec/spec_helper.rb +++ b/bundler/spec/spec_helper.rb @@ -93,6 +93,9 @@ def self.ruby=(ruby) require_relative "support/rubygems_ext" Spec::Rubygems.test_setup + # Disable retry delays in tests to speed them up + Bundler::Retry.default_base_delay = 0 + # Simulate bundler has not yet been loaded ENV.replace(ENV.to_hash.delete_if {|k, _v| k.start_with?(Bundler::EnvironmentPreserver::BUNDLER_PREFIX) }) @@ -104,6 +107,14 @@ def self.ruby=(ruby) ENV["XDG_CONFIG_HOME"] = nil ENV["GEMRC"] = nil + # Prevent tests from modifying the user's global git config. + # GIT_CONFIG_GLOBAL and GIT_CONFIG_NOSYSTEM are available since Git 2.32. + git_version = `git --version`[/(\d+\.\d+\.\d+)/, 1] + if Gem::Version.new(git_version) >= Gem::Version.new("2.32") + ENV["GIT_CONFIG_GLOBAL"] = File.join(ENV["HOME"], ".gitconfig") + ENV["GIT_CONFIG_NOSYSTEM"] = "1" + end + # Don't wrap output in tests ENV["THOR_COLUMNS"] = "10000" diff --git a/bundler/spec/support/builders.rb b/bundler/spec/support/builders.rb index a58b575b63b3..43ab7e053dfb 100644 --- a/bundler/spec/support/builders.rb +++ b/bundler/spec/support/builders.rb @@ -664,7 +664,7 @@ def _build(opts) Bundler.rubygems.build(@spec, opts[:skip_validation]) end elsif opts[:skip_validation] - @context.gem_command "build --force #{@spec.name}", dir: lib_path + Dir.chdir(lib_path) { Gem::Package.build(@spec, true) } else Dir.chdir(lib_path) { Gem::Package.build(@spec) } end diff --git a/bundler/spec/support/filters.rb b/bundler/spec/support/filters.rb index c6b60b5d52fe..6c9127cd950e 100644 --- a/bundler/spec/support/filters.rb +++ b/bundler/spec/support/filters.rb @@ -18,10 +18,13 @@ def inspect end end +git_version = Gem::Version.new(`git --version`[/(\d+\.\d+\.\d+)/, 1]) + RSpec.configure do |config| config.filter_run_excluding realworld: true config.filter_run_excluding rubygems: RequirementChecker.against(Gem.rubygems_version) + config.filter_run_excluding git: RequirementChecker.against(git_version) config.filter_run_excluding ruby_repo: !ENV["GEM_COMMAND"].nil? config.filter_run_excluding no_color_tty: Gem.win_platform? || !ENV["GITHUB_ACTION"].nil? config.filter_run_excluding permissions: Gem.win_platform? diff --git a/bundler/spec/support/helpers.rb b/bundler/spec/support/helpers.rb index 52e6ff5d9a31..6a6cfc8b0084 100644 --- a/bundler/spec/support/helpers.rb +++ b/bundler/spec/support/helpers.rb @@ -25,6 +25,7 @@ def reset! FileUtils.mkdir_p(home) FileUtils.mkdir_p(tmpdir) Bundler.reset! + Bundler::Source::Git::GitProxy.reset Gem.clear_paths end @@ -182,19 +183,6 @@ def gembin(cmd, options = {}) sys_exec(cmd.to_s, options) end - def gem_command(command, options = {}) - env = options[:env] || {} - env["RUBYOPT"] = opt_add(opt_add("-r#{hax}", env["RUBYOPT"]), ENV["RUBYOPT"]) - options[:env] = env - - # Sometimes `gem install` commands hang at dns resolution, which has a - # default timeout of 60 seconds. When that happens, the timeout for a - # command is expired too. So give `gem install` commands a bit more time. - options[:timeout] = 120 - - sys_exec("#{Path.gem_bin} #{command}", options) - end - def sys_exec(cmd, options = {}, &block) env = options[:env] || {} env["RUBYOPT"] = opt_add(opt_add("-r#{spec_dir}/support/switch_rubygems.rb", env["RUBYOPT"]), ENV["RUBYOPT"]) @@ -326,9 +314,20 @@ def self.install_dev_bundler def install_gem(path, install_dir, default = false) raise ArgumentError, "`#{path}` does not exist!" unless File.exist?(path) - args = "--no-document --ignore-dependencies --verbose --local --install-dir #{install_dir}" - - gem_command "install #{args} '#{path}'" + require "rubygems/installer" + + with_simulated_platform do + installer = Gem::Installer.at( + path.to_s, + install_dir: install_dir.to_s, + document: [], + ignore_dependencies: true, + wrappers: true, + env_shebang: true, + force: true + ) + installer.install + end if default gem = Pathname.new(path).basename.to_s.match(/(.*)\.gem/)[1] @@ -343,6 +342,57 @@ def install_gem(path, install_dir, default = false) end end + def uninstall_gem(name, options = {}) + require "rubygems/uninstaller" + + gem_home = options.dig(:env, "GEM_HOME") || system_gem_path.to_s + + with_env_vars("GEM_HOME" => gem_home) do + Gem.clear_paths + + uninstaller = Gem::Uninstaller.new( + name, + ignore: true, + executables: true, + all: true + ) + uninstaller.uninstall + ensure + Gem.clear_paths + end + end + + def installed_gems_list(options = {}) + gem_home = options.dig(:env, "GEM_HOME") || system_gem_path.to_s + + # Temporarily set GEM_HOME for the command + old_gem_home = ENV["GEM_HOME"] + ENV["GEM_HOME"] = gem_home + Gem.clear_paths + + begin + require "rubygems/commands/list_command" + + # Capture output from the list command + output_io = StringIO.new + cmd = Gem::Commands::ListCommand.new + cmd.ui = Gem::StreamUI.new(StringIO.new, output_io, StringIO.new, false) + cmd.invoke + output = output_io.string.strip + ensure + ENV["GEM_HOME"] = old_gem_home + Gem.clear_paths + end + + # Create a fake command execution so `out` helper works + command_execution = Spec::CommandExecution.new("gem list", timeout: 60) + command_execution.original_stdout << output + command_execution.exitstatus = 0 + command_executions << command_execution + + output + end + def with_built_bundler(version = nil, opts = {}, &block) require_relative "builders" @@ -374,6 +424,36 @@ def without_env_side_effects ENV.replace(backup) end + # Simulate the platform set by BUNDLER_SPEC_PLATFORM for in-process + # operations, mirroring what hax.rb does for subprocesses. + def with_simulated_platform + spec_platform = ENV["BUNDLER_SPEC_PLATFORM"] + unless spec_platform + return yield + end + + old_arch = RbConfig::CONFIG["arch"] + old_host_os = RbConfig::CONFIG["host_os"] + + if /mingw|mswin/.match?(spec_platform) + Gem.class_variable_set(:@@win_platform, nil) # rubocop:disable Style/ClassVars + RbConfig::CONFIG["host_os"] = spec_platform.gsub(/^[^-]+-/, "").tr("-", "_") + end + + RbConfig::CONFIG["arch"] = spec_platform + Gem::Platform.instance_variable_set(:@local, nil) + Gem.instance_variable_set(:@platforms, []) + + yield + ensure + if spec_platform + RbConfig::CONFIG["arch"] = old_arch + RbConfig::CONFIG["host_os"] = old_host_os + Gem::Platform.instance_variable_set(:@local, nil) + Gem.instance_variable_set(:@platforms, []) + end + end + def with_path_added(path) with_path_as([path.to_s, ENV["PATH"]].join(File::PATH_SEPARATOR)) do yield diff --git a/bundler/spec/support/windows_tag_group.rb b/bundler/spec/support/windows_tag_group.rb index c41c446462c4..bd6acb9d55ca 100644 --- a/bundler/spec/support/windows_tag_group.rb +++ b/bundler/spec/support/windows_tag_group.rb @@ -137,6 +137,7 @@ module WindowsTagGroup "spec/bundler/build_metadata_spec.rb", "spec/bundler/current_ruby_spec.rb", "spec/bundler/installer/gem_installer_spec.rb", + "spec/bundler/installer/parallel_installer_spec.rb", "spec/bundler/cli_common_spec.rb", "spec/bundler/ci_detector_spec.rb", ], diff --git a/lib/rubygems.rb b/lib/rubygems.rb index b200527d7948..3bb9bf0ce968 100644 --- a/lib/rubygems.rb +++ b/lib/rubygems.rb @@ -9,7 +9,7 @@ require "rbconfig" module Gem - VERSION = "4.0.8" + VERSION = "4.0.9" end require_relative "rubygems/defaults" @@ -37,7 +37,7 @@ module Gem # Further RubyGems documentation can be found at: # # * {RubyGems Guides}[https://guides.rubygems.org] -# * {RubyGems API}[https://www.rubydoc.info/github/ruby/rubygems] (also available from +# * {RubyGems API}[https://guides.rubygems.org/rubygems-org-api/] (also available from # gem server) # # == RubyGems Plugins diff --git a/lib/rubygems/commands/owner_command.rb b/lib/rubygems/commands/owner_command.rb index 12bfe3a834b7..ec6b798db90c 100644 --- a/lib/rubygems/commands/owner_command.rb +++ b/lib/rubygems/commands/owner_command.rb @@ -79,7 +79,8 @@ def show_owners(name) say "Owners for gem: #{name}" owners.each do |owner| - say "- #{owner["email"] || owner["handle"] || owner["id"]}" + identifier = owner["email"] || owner["handle"] || owner["id"] + say "- #{identifier} (#{owner["role"]})" end end end diff --git a/lib/rubygems/commands/sources_command.rb b/lib/rubygems/commands/sources_command.rb index 7e5c2a2465e6..b399af2bd3d5 100644 --- a/lib/rubygems/commands/sources_command.rb +++ b/lib/rubygems/commands/sources_command.rb @@ -50,11 +50,8 @@ def initialize end def add_source(source_uri) # :nodoc: - check_rubygems_https source_uri - - source = Gem::Source.new source_uri - - check_typo_squatting(source) + source = build_new_source(source_uri) + source_uri = source.uri.to_s begin if Gem.sources.include? source @@ -76,11 +73,8 @@ def add_source(source_uri) # :nodoc: end def append_source(source_uri) # :nodoc: - check_rubygems_https source_uri - - source = Gem::Source.new source_uri - - check_typo_squatting(source) + source = build_new_source(source_uri) + source_uri = source.uri.to_s begin source.load_specs :released @@ -103,11 +97,8 @@ def append_source(source_uri) # :nodoc: end def prepend_source(source_uri) # :nodoc: - check_rubygems_https source_uri - - source = Gem::Source.new source_uri - - check_typo_squatting(source) + source = build_new_source(source_uri) + source_uri = source.uri.to_s begin source.load_specs :released @@ -141,6 +132,19 @@ def check_typo_squatting(source) end end + def normalize_source_uri(source_uri) # :nodoc: + # Ensure the source URI has a trailing slash for proper RFC 2396 path merging + # Without a trailing slash, the last path segment is treated as a file and removed + # during relative path resolution (e.g., "/blish" + "gems/foo.gem" = "/gems/foo.gem") + # With a trailing slash, it's treated as a directory (e.g., "/blish/" + "gems/foo.gem" = "/blish/gems/foo.gem") + uri = Gem::URI.parse(source_uri) + uri.path = uri.path.gsub(%r{/+\z}, "") + "/" if uri.path && !uri.path.empty? + uri.to_s + rescue Gem::URI::Error + # If parsing fails, return the original URI and let later validation handle it + source_uri + end + def check_rubygems_https(source_uri) # :nodoc: uri = Gem::URI source_uri @@ -273,7 +277,8 @@ def execute end def remove_source(source_uri) # :nodoc: - source = Gem::Source.new source_uri + source = build_source(source_uri) + source_uri = source.uri.to_s if configured_sources&.include? source Gem.sources.delete source @@ -328,4 +333,16 @@ def configured_sources def config_file_name Gem.configuration.config_file_name end + + def build_source(source_uri) + source_uri = normalize_source_uri(source_uri) + Gem::Source.new(source_uri) + end + + def build_new_source(source_uri) + source = build_source(source_uri) + check_rubygems_https(source.uri.to_s) + check_typo_squatting(source) + source + end end diff --git a/test/rubygems/test_gem_commands_owner_command.rb b/test/rubygems/test_gem_commands_owner_command.rb index 80b1497c415a..d8d220243c5d 100644 --- a/test/rubygems/test_gem_commands_owner_command.rb +++ b/test/rubygems/test_gem_commands_owner_command.rb @@ -32,9 +32,12 @@ def test_show_owners - email: user1@example.com id: 1 handle: user1 + role: owner - email: user2@example.com + role: maintainer - id: 3 handle: user3 + role: owner - id: 4 EOF @@ -48,9 +51,9 @@ def test_show_owners assert_equal Gem.configuration.rubygems_api_key, @stub_fetcher.last_request["Authorization"] assert_match(/Owners for gem: freewill/, @stub_ui.output) - assert_match(/- user1@example.com/, @stub_ui.output) - assert_match(/- user2@example.com/, @stub_ui.output) - assert_match(/- user3/, @stub_ui.output) + assert_match(/- user1@example.com \(owner\)/, @stub_ui.output) + assert_match(/- user2@example.com \(maintainer\)/, @stub_ui.output) + assert_match(/- user3 \(owner\)/, @stub_ui.output) assert_match(/- 4/, @stub_ui.output) end diff --git a/test/rubygems/test_gem_commands_sources_command.rb b/test/rubygems/test_gem_commands_sources_command.rb index 00eb9239940e..71c6d5ce1668 100644 --- a/test/rubygems/test_gem_commands_sources_command.rb +++ b/test/rubygems/test_gem_commands_sources_command.rb @@ -60,6 +60,82 @@ def test_execute_add assert_equal "", @ui.error end + def test_execute_add_without_trailing_slash + setup_fake_source("https://rubygems.pkg.github.com/my-org") + + @cmd.handle_options %W[--add https://rubygems.pkg.github.com/my-org] + + use_ui @ui do + @cmd.execute + end + + assert_equal [@gem_repo, "https://rubygems.pkg.github.com/my-org/"], Gem.sources + + expected = <<-EOF +https://rubygems.pkg.github.com/my-org/ added to sources + EOF + + assert_equal expected, @ui.output + assert_equal "", @ui.error + end + + def test_execute_add_multiple_trailing_slash + setup_fake_source("https://rubygems.pkg.github.com/my-org/") + + @cmd.handle_options %W[--add https://rubygems.pkg.github.com/my-org///] + + use_ui @ui do + @cmd.execute + end + + assert_equal [@gem_repo, "https://rubygems.pkg.github.com/my-org/"], Gem.sources + + expected = <<-EOF +https://rubygems.pkg.github.com/my-org/ added to sources + EOF + + assert_equal expected, @ui.output + assert_equal "", @ui.error + end + + def test_execute_append_without_trailing_slash + setup_fake_source("https://rubygems.pkg.github.com/my-org") + + @cmd.handle_options %W[--append https://rubygems.pkg.github.com/my-org] + + use_ui @ui do + @cmd.execute + end + + assert_equal [@gem_repo, "https://rubygems.pkg.github.com/my-org/"], Gem.sources + + expected = <<-EOF +https://rubygems.pkg.github.com/my-org/ added to sources + EOF + + assert_equal expected, @ui.output + assert_equal "", @ui.error + end + + def test_execute_prepend_without_trailing_slash + setup_fake_source("https://rubygems.pkg.github.com/my-org") + + @cmd.handle_options %W[--prepend https://rubygems.pkg.github.com/my-org] + + use_ui @ui do + @cmd.execute + end + + assert_equal ["https://rubygems.pkg.github.com/my-org/", @gem_repo], Gem.sources + + expected = <<-EOF +https://rubygems.pkg.github.com/my-org/ added to sources + EOF + + assert_equal expected, @ui.output + assert_equal "", @ui.error + end + def test_execute_append setup_fake_source(@new_repo) @@ -530,17 +606,14 @@ def test_execute_add_https_rubygems_org @cmd.handle_options %W[--add #{https_rubygems_org}] - ui = Gem::MockGemUi.new "n" - - use_ui ui do - assert_raise Gem::MockGemUi::TermError do - @cmd.execute - end + use_ui @ui do + @cmd.execute end - assert_equal [@gem_repo], Gem.sources + assert_equal [@gem_repo, https_rubygems_org], Gem.sources expected = <<-EXPECTED +#{https_rubygems_org} added to sources EXPECTED assert_equal expected, @ui.output @@ -554,17 +627,14 @@ def test_execute_append_https_rubygems_org @cmd.handle_options %W[--append #{https_rubygems_org}] - ui = Gem::MockGemUi.new "n" - - use_ui ui do - assert_raise Gem::MockGemUi::TermError do - @cmd.execute - end + use_ui @ui do + @cmd.execute end - assert_equal [@gem_repo], Gem.sources + assert_equal [@gem_repo, https_rubygems_org], Gem.sources expected = <<-EXPECTED +#{https_rubygems_org} added to sources EXPECTED assert_equal expected, @ui.output @@ -583,7 +653,7 @@ def test_execute_add_bad_uri assert_equal [@gem_repo], Gem.sources expected = <<-EOF -beta-gems.example.com is not a URI +beta-gems.example.com/ is not a URI EOF assert_equal expected, @ui.output @@ -602,7 +672,26 @@ def test_execute_append_bad_uri assert_equal [@gem_repo], Gem.sources expected = <<-EOF -beta-gems.example.com is not a URI +beta-gems.example.com/ is not a URI + EOF + + assert_equal expected, @ui.output + assert_equal "", @ui.error + end + + def test_execute_prepend_bad_uri + @cmd.handle_options %w[--prepend beta-gems.example.com] + + use_ui @ui do + assert_raise Gem::MockGemUi::TermError do + @cmd.execute + end + end + + assert_equal [@gem_repo], Gem.sources + + expected = <<-EOF +beta-gems.example.com/ is not a URI EOF assert_equal expected, @ui.output @@ -778,6 +867,31 @@ def test_execute_remove_redundant_source_trailing_slash Gem.configuration.sources = nil end + def test_execute_remove_without_trailing_slash + source_uri = "https://rubygems.pkg.github.com/my-org/" + + Gem.configuration.sources = [source_uri] + + setup_fake_source(source_uri) + + @cmd.handle_options %W[--remove https://rubygems.pkg.github.com/my-org] + + use_ui @ui do + @cmd.execute + end + + assert_equal [], Gem.sources + + expected = <<-EOF +#{source_uri} removed from sources + EOF + + assert_equal expected, @ui.output + assert_equal "", @ui.error + ensure + Gem.configuration.sources = nil + end + def test_execute_update @cmd.handle_options %w[--update] @@ -888,6 +1002,6 @@ def setup_fake_source(uri) Marshal.dump specs, io end - @fetcher.data["#{uri}/specs.#{@marshal_version}.gz"] = specs_dump_gz.string + @fetcher.data["#{uri.chomp("/")}/specs.#{@marshal_version}.gz"] = specs_dump_gz.string end end diff --git a/test/rubygems/test_gem_commands_which_command.rb b/test/rubygems/test_gem_commands_which_command.rb index cbd5b5ef14ba..e114d6e689b2 100644 --- a/test/rubygems/test_gem_commands_which_command.rb +++ b/test/rubygems/test_gem_commands_which_command.rb @@ -38,8 +38,6 @@ def test_execute_directory end def test_execute_one_missing - # TODO: this test fails in isolation - util_foo_bar @cmd.handle_options %w[foo_bar missinglib] diff --git a/tool/bundler/dev_gems.rb.lock b/tool/bundler/dev_gems.rb.lock index eaaf0e5192cf..7212b75fde15 100644 --- a/tool/bundler/dev_gems.rb.lock +++ b/tool/bundler/dev_gems.rb.lock @@ -129,4 +129,4 @@ CHECKSUMS turbo_tests (2.2.5) sha256=3fa31497d12976d11ccc298add29107b92bda94a90d8a0a5783f06f05102509f BUNDLED WITH - 4.0.8 + 4.0.9 diff --git a/tool/bundler/lint_gems.rb.lock b/tool/bundler/lint_gems.rb.lock index 351d0c1e2db4..e9401bcd3b03 100644 --- a/tool/bundler/lint_gems.rb.lock +++ b/tool/bundler/lint_gems.rb.lock @@ -119,4 +119,4 @@ CHECKSUMS wmi-lite (1.0.7) sha256=116ef5bb470dbe60f58c2db9047af3064c16245d6562c646bc0d90877e27ddda BUNDLED WITH - 4.0.8 + 4.0.9 diff --git a/tool/bundler/release_gems.rb.lock b/tool/bundler/release_gems.rb.lock index 5b89ee95eaa9..7f0fad0e7eee 100644 --- a/tool/bundler/release_gems.rb.lock +++ b/tool/bundler/release_gems.rb.lock @@ -87,4 +87,4 @@ CHECKSUMS uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6 BUNDLED WITH - 4.0.8 + 4.0.9 diff --git a/tool/bundler/rubocop_gems.rb.lock b/tool/bundler/rubocop_gems.rb.lock index 181457fb8b5f..8d5a6f25cd3f 100644 --- a/tool/bundler/rubocop_gems.rb.lock +++ b/tool/bundler/rubocop_gems.rb.lock @@ -156,4 +156,4 @@ CHECKSUMS unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f BUNDLED WITH - 4.0.8 + 4.0.9 diff --git a/tool/bundler/standard_gems.rb.lock b/tool/bundler/standard_gems.rb.lock index 9fb818ae5000..b6f7aa209e0c 100644 --- a/tool/bundler/standard_gems.rb.lock +++ b/tool/bundler/standard_gems.rb.lock @@ -176,4 +176,4 @@ CHECKSUMS unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f BUNDLED WITH - 4.0.8 + 4.0.9 diff --git a/tool/bundler/test_gems.rb.lock b/tool/bundler/test_gems.rb.lock index b7f017c673d8..8a257ffd78a7 100644 --- a/tool/bundler/test_gems.rb.lock +++ b/tool/bundler/test_gems.rb.lock @@ -103,4 +103,4 @@ CHECKSUMS tilt (2.6.1) sha256=35a99bba2adf7c1e362f5b48f9b581cce4edfba98117e34696dde6d308d84770 BUNDLED WITH - 4.0.8 + 4.0.9 diff --git a/tool/bundler/vendor_gems.rb.lock b/tool/bundler/vendor_gems.rb.lock index d27bd3c51bfa..e13d334dac6d 100644 --- a/tool/bundler/vendor_gems.rb.lock +++ b/tool/bundler/vendor_gems.rb.lock @@ -72,4 +72,4 @@ CHECKSUMS uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6 BUNDLED WITH - 4.0.8 + 4.0.9