diff --git a/NEWS.md b/NEWS.md index f13c16f8ffcc06..749e28cf9d423a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -68,7 +68,9 @@ releases. ### The following default gems are updated. * RubyGems 4.1.0.dev + * 4.0.3 to [v4.0.4][RubyGems-v4.0.4], [v4.0.5][RubyGems-v4.0.5], [v4.0.6][RubyGems-v4.0.6], [v4.0.7][RubyGems-v4.0.7], [v4.0.8][RubyGems-v4.0.8], [v4.0.9][RubyGems-v4.0.9], [v4.0.10][RubyGems-v4.0.10] * bundler 4.1.0.dev + * 4.0.3 to [v4.0.4][bundler-v4.0.4], [v4.0.5][bundler-v4.0.5], [v4.0.6][bundler-v4.0.6], [v4.0.7][bundler-v4.0.7], [v4.0.8][bundler-v4.0.8], [v4.0.9][bundler-v4.0.9], [v4.0.10][bundler-v4.0.10] * erb 6.0.3 * 6.0.1 to [v6.0.2][erb-v6.0.2], [v6.0.3][erb-v6.0.3] * json 2.19.3 @@ -76,12 +78,12 @@ releases. * openssl 4.0.1 * 4.0.0 to [v4.0.1][openssl-v4.0.1] * prism 1.9.0 - * 1.8.1 to [v1.9.0][prism-v1.9.0] + * 1.7.0 to [v1.8.0][prism-v1.8.0], [v1.8.1][prism-v1.8.1], [v1.9.0][prism-v1.9.0] * resolv 0.7.1 * 0.7.0 to [v0.7.1][resolv-v0.7.1] * stringio 3.2.1.dev -* strscan 3.1.7.dev - * 3.1.6 to [v3.1.7][strscan-v3.1.7] +* strscan 3.1.9.dev + * 3.1.6 to [v3.1.7][strscan-v3.1.7], [v3.1.8][strscan-v3.1.8] * syntax_suggest 3.0.0 * timeout 0.6.1 * 0.6.0 to [v0.6.1][timeout-v0.6.1] @@ -98,7 +100,7 @@ releases. * net-imap 0.6.3 * 0.6.2 to [v0.6.3][net-imap-v0.6.3] * rbs 4.0.2 - * 3.10.0 to [v3.10.1][rbs-v3.10.1], [v3.10.2][rbs-v3.10.2], [v3.10.3][rbs-v3.10.3], [v3.10.4][rbs-v3.10.4], [v4.0.0.dev.5][rbs-v4.0.0.dev.5], [v4.0.0][rbs-v4.0.0], [v4.0.2][rbs-v4.0.2] + * 3.10.0 to [v3.10.1][rbs-v3.10.1], [v3.10.2][rbs-v3.10.2], [v3.10.3][rbs-v3.10.3], [v3.10.4][rbs-v3.10.4], [v4.0.0.dev.1][rbs-v4.0.0.dev.1], [v4.0.0.dev.2][rbs-v4.0.0.dev.2], [v4.0.0.dev.3][rbs-v4.0.0.dev.3], [v4.0.0.dev.4][rbs-v4.0.0.dev.4], [v4.0.0.dev.5][rbs-v4.0.0.dev.5], [v4.0.0][rbs-v4.0.0], [v4.0.1.dev.1][rbs-v4.0.1.dev.1], [v4.0.1.dev.2][rbs-v4.0.1.dev.2], [v4.0.1][rbs-v4.0.1], [v4.0.2][rbs-v4.0.2] * mutex_m 0.3.0 * bigdecimal 4.1.1 * 4.0.1 to [v4.1.0][bigdecimal-v4.1.0], [v4.1.1][bigdecimal-v4.1.1] @@ -164,18 +166,20 @@ A lot of work has gone into making Ractors more stable, performant, and usable. [Feature #21853]: https://bugs.ruby-lang.org/issues/21853 [Feature #21861]: https://bugs.ruby-lang.org/issues/21861 [Feature #21932]: https://bugs.ruby-lang.org/issues/21932 -[test-unit-3.7.4]: https://github.com/test-unit/test-unit/releases/tag/3.7.4 -[test-unit-3.7.5]: https://github.com/test-unit/test-unit/releases/tag/3.7.5 -[rss-0.3.2]: https://github.com/ruby/rss/releases/tag/0.3.2 -[net-imap-v0.6.2]: https://github.com/ruby/net-imap/releases/tag/v0.6.2 -[debug-v1.11.1]: https://github.com/ruby/debug/releases/tag/v1.11.1 -[rdoc-v7.0.0]: https://github.com/ruby/rdoc/releases/tag/v7.0.0 -[rdoc-v7.0.1]: https://github.com/ruby/rdoc/releases/tag/v7.0.1 -[rdoc-v7.0.2]: https://github.com/ruby/rdoc/releases/tag/v7.0.2 -[rdoc-v7.0.3]: https://github.com/ruby/rdoc/releases/tag/v7.0.3 -[prism-v1.8.1]: https://github.com/ruby/prism/releases/tag/v1.8.1 -[zlib-v3.2.3]: https://github.com/ruby/zlib/releases/tag/v3.2.3 -[pstore-v0.2.1]: https://github.com/ruby/pstore/releases/tag/v0.2.1 +[RubyGems-v4.0.4]: https://github.com/rubygems/rubygems/releases/tag/v4.0.4 +[RubyGems-v4.0.5]: https://github.com/rubygems/rubygems/releases/tag/v4.0.5 +[RubyGems-v4.0.6]: https://github.com/rubygems/rubygems/releases/tag/v4.0.6 +[RubyGems-v4.0.7]: https://github.com/rubygems/rubygems/releases/tag/v4.0.7 +[RubyGems-v4.0.8]: https://github.com/rubygems/rubygems/releases/tag/v4.0.8 +[RubyGems-v4.0.9]: https://github.com/rubygems/rubygems/releases/tag/v4.0.9 +[RubyGems-v4.0.10]: https://github.com/rubygems/rubygems/releases/tag/v4.0.10 +[bundler-v4.0.4]: https://github.com/rubygems/rubygems/releases/tag/bundler-v4.0.4 +[bundler-v4.0.5]: https://github.com/rubygems/rubygems/releases/tag/bundler-v4.0.5 +[bundler-v4.0.6]: https://github.com/rubygems/rubygems/releases/tag/bundler-v4.0.6 +[bundler-v4.0.7]: https://github.com/rubygems/rubygems/releases/tag/bundler-v4.0.7 +[bundler-v4.0.8]: https://github.com/rubygems/rubygems/releases/tag/bundler-v4.0.8 +[bundler-v4.0.9]: https://github.com/rubygems/rubygems/releases/tag/bundler-v4.0.9 +[bundler-v4.0.10]: https://github.com/rubygems/rubygems/releases/tag/bundler-v4.0.10 [erb-v6.0.2]: https://github.com/ruby/erb/releases/tag/v6.0.2 [erb-v6.0.3]: https://github.com/ruby/erb/releases/tag/v6.0.3 [json-v2.18.1]: https://github.com/ruby/json/releases/tag/v2.18.1 @@ -184,10 +188,14 @@ A lot of work has gone into making Ractors more stable, performant, and usable. [json-v2.19.2]: https://github.com/ruby/json/releases/tag/v2.19.2 [json-v2.19.3]: https://github.com/ruby/json/releases/tag/v2.19.3 [openssl-v4.0.1]: https://github.com/ruby/openssl/releases/tag/v4.0.1 +[prism-v1.8.0]: https://github.com/ruby/prism/releases/tag/v1.8.0 +[prism-v1.8.1]: https://github.com/ruby/prism/releases/tag/v1.8.1 [prism-v1.9.0]: https://github.com/ruby/prism/releases/tag/v1.9.0 [resolv-v0.7.1]: https://github.com/ruby/resolv/releases/tag/v0.7.1 [strscan-v3.1.7]: https://github.com/ruby/strscan/releases/tag/v3.1.7 +[strscan-v3.1.8]: https://github.com/ruby/strscan/releases/tag/v3.1.8 [timeout-v0.6.1]: https://github.com/ruby/timeout/releases/tag/v0.6.1 +[zlib-v3.2.3]: https://github.com/ruby/zlib/releases/tag/v3.2.3 [rake-v13.4.0]: https://github.com/ruby/rake/releases/tag/v13.4.0 [rake-v13.4.1]: https://github.com/ruby/rake/releases/tag/v13.4.1 [rake-v13.4.2]: https://github.com/ruby/rake/releases/tag/v13.4.2 @@ -198,8 +206,15 @@ A lot of work has gone into making Ractors more stable, performant, and usable. [rbs-v3.10.2]: https://github.com/ruby/rbs/releases/tag/v3.10.2 [rbs-v3.10.3]: https://github.com/ruby/rbs/releases/tag/v3.10.3 [rbs-v3.10.4]: https://github.com/ruby/rbs/releases/tag/v3.10.4 +[rbs-v4.0.0.dev.1]: https://github.com/ruby/rbs/releases/tag/v4.0.0.dev.1 +[rbs-v4.0.0.dev.2]: https://github.com/ruby/rbs/releases/tag/v4.0.0.dev.2 +[rbs-v4.0.0.dev.3]: https://github.com/ruby/rbs/releases/tag/v4.0.0.dev.3 +[rbs-v4.0.0.dev.4]: https://github.com/ruby/rbs/releases/tag/v4.0.0.dev.4 [rbs-v4.0.0.dev.5]: https://github.com/ruby/rbs/releases/tag/v4.0.0.dev.5 [rbs-v4.0.0]: https://github.com/ruby/rbs/releases/tag/v4.0.0 +[rbs-v4.0.1.dev.1]: https://github.com/ruby/rbs/releases/tag/v4.0.1.dev.1 +[rbs-v4.0.1.dev.2]: https://github.com/ruby/rbs/releases/tag/v4.0.1.dev.2 +[rbs-v4.0.1]: https://github.com/ruby/rbs/releases/tag/v4.0.1 [rbs-v4.0.2]: https://github.com/ruby/rbs/releases/tag/v4.0.2 [bigdecimal-v4.1.0]: https://github.com/ruby/bigdecimal/releases/tag/v4.1.0 [bigdecimal-v4.1.1]: https://github.com/ruby/bigdecimal/releases/tag/v4.1.1 @@ -208,6 +223,7 @@ A lot of work has gone into making Ractors more stable, performant, and usable. [repl_type_completor-v0.1.13]: https://github.com/ruby/repl_type_completor/releases/tag/v0.1.13 [repl_type_completor-v0.1.14]: https://github.com/ruby/repl_type_completor/releases/tag/v0.1.14 [repl_type_completor-v0.1.15]: https://github.com/ruby/repl_type_completor/releases/tag/v0.1.15 +[pstore-v0.2.1]: https://github.com/ruby/pstore/releases/tag/v0.2.1 [rdoc-v7.1.0]: https://github.com/ruby/rdoc/releases/tag/v7.1.0 [rdoc-v7.2.0]: https://github.com/ruby/rdoc/releases/tag/v7.2.0 [win32ole-v1.9.3]: https://github.com/ruby/win32ole/releases/tag/v1.9.3 diff --git a/ext/strscan/strscan.c b/ext/strscan/strscan.c index 7efee0b5db4d1e..0f20889dc326a1 100644 --- a/ext/strscan/strscan.c +++ b/ext/strscan/strscan.c @@ -22,7 +22,7 @@ extern size_t onig_region_memsize(const struct re_registers *regs); #include -#define STRSCAN_VERSION "3.1.7.dev" +#define STRSCAN_VERSION "3.1.9.dev" #ifdef HAVE_RB_DEPRECATE_CONSTANT diff --git a/lib/bundler/installer/parallel_installer.rb b/lib/bundler/installer/parallel_installer.rb index 020db30b8443b5..fef326ed0a9c8a 100644 --- a/lib/bundler/installer/parallel_installer.rb +++ b/lib/bundler/installer/parallel_installer.rb @@ -6,7 +6,7 @@ module Bundler class ParallelInstaller class SpecInstallation - attr_accessor :spec, :name, :full_name, :post_install_message, :state, :error + attr_accessor :spec, :name, :full_name, :post_install_message, :state, :error, :dependencies def initialize(spec) @spec = spec @name = spec.name @@ -46,25 +46,11 @@ def has_post_install_message? !post_install_message.empty? end - def ignorable_dependency?(dep) - dep.type == :development || dep.name == @name - end - - # Checks installed dependencies against spec's dependencies to make - # sure needed dependencies have been installed. + # Recursively checks that all dependencies (direct and transitive) have been installed. def dependencies_installed?(installed_specs) - dependencies.all? {|d| installed_specs.include? d.name } - end - - # Represents only the non-development dependencies, the ones that are - # itself and are in the total list. - def dependencies - @dependencies ||= all_dependencies.reject {|dep| ignorable_dependency? dep } - end - - # Represents all dependencies - def all_dependencies - @spec.dependencies + dependencies.all? do |dep| + installed_specs.include?(dep.name) && dep.dependencies_installed?(installed_specs) + end end def to_s @@ -85,6 +71,12 @@ def initialize(installer, all_specs, size, standalone, force, local: false, skip @force = force @local = local @specs = all_specs.map {|s| SpecInstallation.new(s) } + specs_by_name = @specs.to_h {|s| [s.name, s] } + @specs.each do |spec_install| + spec_install.dependencies = spec_install.spec.dependencies.filter_map do |dep| + specs_by_name[dep.name] unless dep.type == :development || dep.name == spec_install.name + end + end @specs.each do |spec_install| spec_install.state = :installed if skip.include?(spec_install.name) end if skip diff --git a/spec/bundler/bundler/installer/spec_installation_spec.rb b/spec/bundler/bundler/installer/spec_installation_spec.rb index cbe2589b99b740..57868766d9b37a 100644 --- a/spec/bundler/bundler/installer/spec_installation_spec.rb +++ b/spec/bundler/bundler/installer/spec_installation_spec.rb @@ -3,18 +3,17 @@ require "bundler/installer/parallel_installer" RSpec.describe Bundler::ParallelInstaller::SpecInstallation do - let!(:dep) do - a_spec = Object.new - def a_spec.name - "I like tests" - end - - def a_spec.full_name - "I really like tests" - end - a_spec + def build_spec(name, extensions: []) + spec = Object.new + spec.define_singleton_method(:name) { name } + spec.define_singleton_method(:full_name) { "#{name}-1.0" } + spec.define_singleton_method(:extensions) { extensions } + spec.define_singleton_method(:dependencies) { [] } + spec end + let!(:dep) { build_spec("I like tests") } + describe "#ready_to_enqueue?" do context "when in enqueued state" do it "is falsey" do @@ -39,29 +38,51 @@ def a_spec.full_name end describe "#dependencies_installed?" do - context "when all dependencies are installed" do - it "returns true" do - dependencies = [] - dependencies << instance_double("SpecInstallation", spec: "alpha", name: "alpha", installed?: true, all_dependencies: [], type: :production) - dependencies << instance_double("SpecInstallation", spec: "beta", name: "beta", installed?: true, all_dependencies: [], type: :production) - all_specs = dependencies + [instance_double("SpecInstallation", spec: "gamma", name: "gamma", installed?: false, all_dependencies: [], type: :production)] + it "returns true when all dependencies are installed" do + alpha = described_class.new(build_spec("alpha")) + alpha.dependencies = [] + + beta = described_class.new(build_spec("beta")) + beta.dependencies = [alpha] + + gamma = described_class.new(build_spec("gamma")) + gamma.dependencies = [beta] + + expect(gamma.dependencies_installed?({})).to be_falsey + expect(gamma.dependencies_installed?({ "beta" => true })).to be_falsey + expect(gamma.dependencies_installed?({ "alpha" => true, "beta" => true })).to be_truthy + end + end + + describe "#ready_to_install?" do + context "when spec has no extensions" do + it "returns true regardless of dependencies" do + beta = described_class.new(build_spec("beta")) + beta.dependencies = [] + spec = described_class.new(dep) - allow(spec).to receive(:all_dependencies).and_return(dependencies) - installed_specs = all_specs.select(&:installed?).map {|s| [s.name, true] }.to_h - expect(spec.dependencies_installed?(installed_specs)).to be_truthy + spec.state = :downloaded + spec.dependencies = [beta] + + expect(spec.ready_to_install?({})).to be_truthy end end - context "when all dependencies are not installed" do - it "returns false" do - dependencies = [] - dependencies << instance_double("SpecInstallation", spec: "alpha", name: "alpha", installed?: false, all_dependencies: [], type: :production) - dependencies << instance_double("SpecInstallation", spec: "beta", name: "beta", installed?: true, all_dependencies: [], type: :production) - all_specs = dependencies + [instance_double("SpecInstallation", spec: "gamma", name: "gamma", installed?: false, all_dependencies: [], type: :production)] - spec = described_class.new(dep) - allow(spec).to receive(:all_dependencies).and_return(dependencies) - installed_specs = all_specs.select(&:installed?).map {|s| [s.name, true] }.to_h - expect(spec.dependencies_installed?(installed_specs)).to be_falsey + context "when spec has extensions" do + it "returns true when all dependencies are installed" do + alpha = described_class.new(build_spec("alpha")) + alpha.dependencies = [] + + beta = described_class.new(build_spec("beta")) + beta.dependencies = [alpha] + + gamma = described_class.new(build_spec("gamma", extensions: ["ext/Rakefile"])) + gamma.state = :downloaded + gamma.dependencies = [beta] + + expect(gamma.ready_to_install?({})).to be_falsey + expect(gamma.ready_to_install?({ "beta" => true })).to be_falsey + expect(gamma.ready_to_install?({ "alpha" => true, "beta" => true })).to be_truthy end end end diff --git a/spec/bundler/commands/install_spec.rb b/spec/bundler/commands/install_spec.rb index bb9b2b1f98a500..8dedeb7d0776aa 100644 --- a/spec/bundler/commands/install_spec.rb +++ b/spec/bundler/commands/install_spec.rb @@ -1379,6 +1379,48 @@ def run end end + describe "when a native extension requires a transitive dependency at build time" do + before do + build_repo4 do + build_gem "alpha", "1.0.0" do |s| + extension = "ext/alpha/extconf.rb" + s.extensions = extension + s.write(extension, <<~CODE) + require "mkmf" + sleep 1 + create_makefile("alpha") + CODE + s.write "lib/alpha.rb", "ALPHA = '1.0.0'" + end + + build_gem "beta", "1.0.0" do |s| + s.add_dependency "alpha" + s.write "lib/beta.rb", "require 'alpha'\nBETA = '1.0.0'" + end + + build_gem "gamma", "1.0.0" do |s| + s.add_dependency "beta" + extension = "ext/gamma/extconf.rb" + s.extensions = extension + s.write(extension, <<~EXTCONF) + require "beta" + require "mkmf" + create_makefile("gamma") + EXTCONF + end + end + end + + it "installs successfully" do + install_gemfile <<~G + source "https://gem.repo4" + gem "gamma" + G + + expect(the_bundle).to include_gems "alpha 1.0.0", "beta 1.0.0", "gamma 1.0.0" + end + end + describe "when configured path is UTF-8 and a file inside a gem package too" do let(:app_path) do path = tmp("♥") diff --git a/tool/lib/envutil.rb b/tool/lib/envutil.rb index dcc9148a8407f0..b4c7d1d03564fd 100644 --- a/tool/lib/envutil.rb +++ b/tool/lib/envutil.rb @@ -48,14 +48,6 @@ class << self attr_reader :original_internal_encoding, :original_external_encoding, :original_verbose, :original_warning - if RUBY_ENGINE == "truffleruby" - # Tests relying on timeout have high variance on TruffleRuby due to the highly-optimizing JIT, deoptimization, profiling interpreter, different GC, etc. - # Setting a default timeout scale helps avoid transient failures for tests relying on timeouts. - # We choose 10 because it is the same number used in CRuby CI on macOS: - # https://github.com/ruby/ruby/blob/9d46b0c735877f152a0b4b16b8153c6f395dee28/.github/workflows/macos.yml#L133 - self.timeout_scale = 10 - end - def capture_global_values @original_internal_encoding = Encoding.default_internal @original_external_encoding = Encoding.default_external @@ -71,6 +63,14 @@ def capture_global_values end end + if RUBY_ENGINE == "truffleruby" + # Tests relying on timeout have high variance on TruffleRuby due to the highly-optimizing JIT, deoptimization, profiling interpreter, different GC, etc. + # Setting a default timeout scale helps avoid transient failures for tests relying on timeouts. + # We choose 10 because it is the same number used in CRuby CI on macOS: + # https://github.com/ruby/ruby/blob/9d46b0c735877f152a0b4b16b8153c6f395dee28/.github/workflows/macos.yml#L133 + self.timeout_scale = 10 + end + def apply_timeout_scale(t) if scale = EnvUtil.timeout_scale t * scale diff --git a/tool/update-NEWS-github-release.rb b/tool/update-NEWS-github-release.rb index 679e7101a76be8..0ef6fcfe377145 100755 --- a/tool/update-NEWS-github-release.rb +++ b/tool/update-NEWS-github-release.rb @@ -3,7 +3,6 @@ require "bundler/inline" require "json" require "net/http" -require "set" require "uri" gemfile do @@ -21,9 +20,15 @@ # Build a gem=>version map from stdgems.org stdgems.json for a given Ruby version (e.g., "3.4") def fetch_default_gems_versions(ruby_version) uri = URI.parse("https://stdgems.org/stdgems.json") - json = JSON.parse(Net::HTTP.get(uri)) + body = http_get(uri) + json = JSON.parse(body) gems = json["gems"] || [] + # Prefer the initial release key (e.g. "4.0.0") over the rolling + # major.minor key (e.g. "4.0") so the diff baseline reflects the original + # X.Y.0 release rather than the latest patch level. + initial_release_key = (ruby_version =~ /\A\d+\.\d+\z/) ? "#{ruby_version}.0" : nil + map = {} gems.each do |g| # Only include default gems (skip ones marked removed) @@ -38,7 +43,9 @@ def fetch_default_gems_versions(ruby_version) category_versions = versions[category] || {} next if selected_version - if category_versions.key?(ruby_version) + if initial_release_key && category_versions.key?(initial_release_key) + selected_version = category_versions[initial_release_key] + elsif category_versions.key?(ruby_version) selected_version = category_versions[ruby_version] else # Fall back to the highest patch version matching the given major.minor @@ -81,24 +88,35 @@ def load_versions(arg) elsif arg.match?(/^\d+\.\d+(?:\.\d+)?$/) fetch_default_gems_versions(arg) elsif arg.downcase == "news" || arg =~ %r{https?://.*/NEWS\.md} - fetch_versions_to_from_news(arg) + fetch_versions_from_news(arg) else abort "Invalid argument: #{arg}. Provide a file path or a Ruby version (e.g., 3.4)." end end # Build a gem=>version map by parsing the "## Stdlib updates" section from Ruby's NEWS.md -def fetch_versions_to_from_news(arg) +def fetch_versions_from_news(arg) if arg.downcase == "news" body = read_local_news_md else - uri = URI.parse(arg) - body = Net::HTTP.get(uri) + body = http_get(URI.parse(arg)) end parse_stdlib_versions_from_news(body) end +# Fetch a URL with a clear abort message on network or HTTP failures. +# Used for sources whose absence makes the rest of the script meaningless. +def http_get(uri) + res = Net::HTTP.get_response(uri) + unless res.is_a?(Net::HTTPSuccess) + abort "error: #{uri} returned HTTP #{res.code} #{res.message}" + end + res.body +rescue SystemCallError, SocketError, IOError, Net::HTTPError => e + abort "error: failed to fetch #{uri}: #{e.class}: #{e.message}" +end + def read_local_news_md news_path = File.join(__dir__, "..", "NEWS.md") unless File.exist?(news_path) @@ -107,6 +125,44 @@ def read_local_news_md File.read(news_path) end +# Build a gem=>version map from the current repository state. Default gems +# come from {ext,lib}/**/*.gemspec (mirroring default_gems_list.yml) and +# bundled gems come from gems/bundled_gems. This avoids reading NEWS.md as +# the source of "current versions", which would create a circular dependency +# with update-NEWS-gemlist.rb. +def load_current_versions + require "rubygems" + root = File.expand_path("..", __dir__) + map = {} + + rg_path = File.join(root, "lib", "rubygems.rb") + if File.exist?(rg_path) + File.foreach(rg_path) do |line| + if /^\s*VERSION\s*=\s*"([^"]+)"/ =~ line + map["RubyGems"] = $1 + break + end + end + end + + Dir.glob(File.join(root, "{ext,lib}/**/*.gemspec")).each do |path| + spec = Gem::Specification.load(path) + next unless spec + map[spec.name] = spec.version.to_s + end + + bundled_path = File.join(root, "gems", "bundled_gems") + if File.exist?(bundled_path) + File.foreach(bundled_path) do |line| + next if line.start_with?("#") + name, version = line.split(" ", 3) + map[name] = version if name && version + end + end + + map +end + def parse_stdlib_versions_from_news(body) # Extract the Stdlib updates section start_idx = body.index(/^## Stdlib updates$/) @@ -150,25 +206,38 @@ def resolve_repo(name) { repo: name, org: "minitest" } when "test-unit" { repo: name, org: "test-unit" } + when "RubyGems" + { repo: "rubygems", org: "rubygems" } when "bundler" - { repo: "rubygems", org: "ruby" } + { repo: "rubygems", org: "rubygems", tag_prefix: "bundler-" } else { repo: name, org: "ruby" } end end -def fetch_release_range(name, from_version, to_version, org, repo) +def fetch_release_range(name, from_version, to_version, org, repo, tag_prefix: "") releases = [] - Octokit.releases("#{org}/#{repo}").each do |release| - releases << release.tag_name + begin + Octokit.releases("#{org}/#{repo}").each do |release| + releases << release.tag_name + end + rescue Octokit::Error, Faraday::Error => e + warn "warning: skipping #{name} (#{org}/#{repo}): #{e.class}: #{e.message}" + return nil end - # Keep only version-like tags and sort ascending by semantic version - releases = releases.select { |t| t =~ /^v\d/ || t =~ /^\d/ || t =~ /^bundler-\d/ } - releases = releases.sort_by { |t| Gem::Version.new(t.sub(/^bundler-/, "").sub(/^v/, "").tr("_", ".")) } + # Keep only this gem's version-like tags and sort ascending by semantic version + prefix = Regexp.escape(tag_prefix) + releases = releases.select { |t| t =~ /\A#{prefix}v?\d/ } + releases = releases.sort_by { |t| Gem::Version.new(t.sub(/\A#{prefix}/, "").sub(/^v/, "").tr("_", ".")) } + + start_index = releases.index("#{tag_prefix}v#{from_version}") || releases.index("#{tag_prefix}#{from_version}") + end_index = releases.index("#{tag_prefix}v#{to_version}") || releases.index("#{tag_prefix}#{to_version}") + + # If the "to" version is unreleased (e.g. 4.1.0.dev), include every released + # tag after the baseline up to the latest one available. + end_index ||= releases.length - 1 if to_version =~ /(?:\.|-)(?:dev|beta|alpha|rc|pre)/i - start_index = releases.index("v#{from_version}") || releases.index(from_version) || releases.index("bundler-v#{from_version}") - end_index = releases.index("v#{to_version}") || releases.index(to_version) || releases.index("bundler-v#{to_version}") return nil unless start_index && end_index range = releases[start_index + 1..end_index] @@ -183,21 +252,20 @@ def collect_gem_updates(versions_from, versions_to) versions_to.each do |name, version| # Skip items which do not exist in the FROM map to reduce API calls next unless versions_from.key?(name) - next if name == "RubyGems" || name == "bundler" info = resolve_repo(name) org = info[:org] repo = info[:repo] + tag_prefix = info[:tag_prefix] || "" - release_range = fetch_release_range(name, versions_from[name], version, org, repo) + release_range = fetch_release_range(name, versions_from[name], version, org, repo, tag_prefix: tag_prefix) next unless release_range - footnote_links = [] - release_range.each do |rel| - footnote_links << { - ref: "#{name}-#{rel.sub(/^bundler-/, '')}", + footnote_links = release_range.map do |rel| + tag = rel.sub(/\A#{Regexp.escape(tag_prefix)}/, "") + { + ref: "#{name}-#{tag}", url: "https://github.com/#{org}/#{repo}/releases/tag/#{rel}", - tag: rel.sub(/^bundler-/, ''), } end @@ -207,21 +275,28 @@ def collect_gem_updates(versions_from, versions_to) from_version: versions_from[name], release_range: release_range, footnote_links: footnote_links, + tag_prefix: tag_prefix, } end results end +def format_release_diff(result) + prefix = Regexp.escape(result[:tag_prefix] || "") + links = result[:release_range].map do |rel| + tag = rel.sub(/\A#{prefix}/, "") + "[#{tag}][#{result[:name]}-#{tag}]" + end + " * #{result[:from_version]} to #{links.join(', ')}" +end + def print_results(results) footnote_lines = [] results.each do |r| puts "* #{r[:name]} #{r[:version]}" - links = r[:release_range].map { |rel| - "[#{rel.sub(/^bundler-/, '')}][#{r[:name]}-#{rel.sub(/^bundler-/, '')}]" - } - puts " * #{r[:from_version]} to #{links.join(', ')}" + puts format_release_diff(r) r[:footnote_links].each do |fl| footnote_lines << "[#{fl[:ref]}]: #{fl[:url]}" end @@ -238,36 +313,25 @@ def update_news_md(results) content = File.read(news_path) lines = content.lines - # Build a lookup: gem name => result - result_by_name = {} - results.each { |r| result_by_name[r[:name]] = r } + result_by_name = results.to_h { |r| [r[:name], r] } new_lines = [] i = 0 while i < lines.length line = lines[i] - # Check if this line is a gem bullet like "* gemname x.y.z" if line =~ /^\* ([A-Za-z0-9_\-]+)\s+(\d+(?:\.\d+){0,3})\b/ gem_name = $1 - gem_name_normalized = gem_name == "RubyGems" ? "RubyGems" : gem_name new_lines << line - if result_by_name.key?(gem_name_normalized) - r = result_by_name[gem_name_normalized] - - # Skip any existing sub-bullet lines that follow (lines starting with spaces + *) + if (r = result_by_name[gem_name]) + # Skip any existing sub-bullet lines that follow while i + 1 < lines.length && lines[i + 1] =~ /^\s+\*/ i += 1 end - # Insert the version diff sub-bullet - links = r[:release_range].map { |rel| - "[#{rel.sub(/^bundler-/, '')}][#{r[:name]}-#{rel.sub(/^bundler-/, '')}]" - } - sub_bullet = " * #{r[:from_version]} to #{links.join(', ')}\n" - new_lines << sub_bullet + new_lines << "#{format_release_diff(r)}\n" end else new_lines << line @@ -275,36 +339,44 @@ def update_news_md(results) i += 1 end - # Collect all new footnote links - all_footnotes = [] + # All footnote definitions we can emit from this run, indexed by ref name + available_footnotes = {} results.each do |r| r[:footnote_links].each do |fl| - all_footnotes << "[#{fl[:ref]}]: #{fl[:url]}" + available_footnotes[fl[:ref]] = "[#{fl[:ref]}]: #{fl[:url]}" end end - # Remove any existing footnote links that we are about to add (avoid duplicates) - existing_refs = Set.new(all_footnotes.map { |f| f[/^\[([^\]]+)\]:/, 1] }) - new_lines = new_lines.reject do |line| - if line =~ /^\[([^\]]+)\]:\s+https:\/\/github\.com\// - existing_refs.include?($1) - else - false + # Refs the regenerated body actually uses (e.g. `][gem-vX.Y.Z]`) + used_refs = new_lines.join.scan(/\]\[([^\]]+)\]/).flatten.uniq + used_set = used_refs.to_set + + # Drop existing GitHub release-tag link defs that are either orphaned (no + # body reference) or about to be re-emitted from this run's results. Defs + # still referenced by gems we couldn't refresh are preserved in place. + release_ref_pattern = %r{^\[([^\]]+)\]:\s+https://github\.com/[^/]+/[^/]+/releases/tag/} + new_lines.reject! do |line| + if (m = line.match(release_ref_pattern)) + ref = m[1] + !used_set.include?(ref) || available_footnotes.key?(ref) end end - # Ensure the file ends with a newline before adding footnotes - unless new_lines.last&.end_with?("\n") - new_lines << "\n" - end + # Trim trailing blank lines so the appended footer block is clean + new_lines.pop while new_lines.last == "\n" + new_lines << "\n" unless new_lines.last&.end_with?("\n") - # Append footnote links at the end of the file - all_footnotes.each do |footnote| - new_lines << "#{footnote}\n" + # Append footnote defs only for refs the body still references + emitted = 0 + used_refs.each do |ref| + if (footnote = available_footnotes[ref]) + new_lines << "#{footnote}\n" + emitted += 1 + end end File.write(news_path, new_lines.join) - puts "Updated #{news_path} with #{results.length} gem update entries and #{all_footnotes.length} footnote links." + puts "Updated #{news_path} with #{results.length} gem update entries and #{emitted} footnote links." end # --- Main --- @@ -312,7 +384,7 @@ def update_news_md(results) update_mode = ARGV.delete("--update") versions_from = load_versions(ARGV[0]) -versions_to = load_versions("news") +versions_to = load_current_versions results = collect_gem_updates(versions_from, versions_to)