Skip to content

Commit

Permalink
plugins: improve remove command to support multiple plugins (#17030)
Browse files Browse the repository at this point in the history
Removal works in a single pass by finding plugins that would have unmet
dependencies if all of the specified plugins were to be removed, and
proceeding with the removal only if no conflicts were created.

> ~~~
> ╭─{ rye@perhaps:~/src/elastic/logstash@main (pluginmanager-remove-multiple ✘) }
> ╰─● bin/logstash-plugin remove logstash-input-syslog logstash-filter-grok
> Using system java: /Users/rye/.jenv/shims/java
> Resolving dependencies......
> Successfully removed logstash-input-syslog
> Successfully removed logstash-filter-grok
> [success (00:00:05)]
~~~
  • Loading branch information
yaauie authored Feb 19, 2025
1 parent 9abad66 commit 0895588
Show file tree
Hide file tree
Showing 11 changed files with 349 additions and 82 deletions.
94 changes: 51 additions & 43 deletions lib/pluginmanager/bundler/logstash_uninstall.rb
Original file line number Diff line number Diff line change
Expand Up @@ -34,54 +34,50 @@ def initialize(gemfile_path, lockfile_path)
@lockfile_path = lockfile_path
end

# To be uninstalled the candidate gems need to be standalone.
def dependants_gems(gem_name)
builder = Dsl.new
builder.eval_gemfile(::File.join(::File.dirname(gemfile_path), "original gemfile"), File.read(gemfile_path))
definition = builder.to_definition(lockfile_path, {})

definition.specs
.select { |spec| spec.dependencies.collect(&:name).include?(gem_name) }
.collect(&:name).sort.uniq
end

def uninstall!(gem_name)
unfreeze_gemfile do
dependencies_from = dependants_gems(gem_name)

if dependencies_from.size > 0
display_cant_remove_message(gem_name, dependencies_from)
false
else
remove_gem(gem_name)
true
def uninstall!(gems_to_remove)
gems_to_remove = Array(gems_to_remove)

unsatisfied_dependency_mapping = Dsl.evaluate(gemfile_path, lockfile_path, {}).specs.each_with_object({}) do |spec, memo|
next if gems_to_remove.include?(spec.name)
deps = spec.runtime_dependencies.collect(&:name)
deps.intersection(gems_to_remove).each do |missing_dependency|
memo[missing_dependency] ||= []
memo[missing_dependency] << spec.name
end
end
end
if unsatisfied_dependency_mapping.any?
unsatisfied_dependency_mapping.each { |gem_to_remove, gems_depending_on_removed| display_cant_remove_message(gem_to_remove, gems_depending_on_removed) }
LogStash::PluginManager.ui.info("No plugins were removed.")
return false
end

with_mutable_gemfile do |gemfile|
gems_to_remove.each { |gem_name| gemfile.remove(gem_name) }

def remove_gem(gem_name)
builder = Dsl.new
file = File.new(gemfile_path, "r+")
builder = Dsl.new
builder.eval_gemfile(::File.join(::File.dirname(gemfile_path), "gemfile to changes"), gemfile.generate)

gemfile = LogStash::Gemfile.new(file).load
gemfile.remove(gem_name)
builder.eval_gemfile(::File.join(::File.dirname(gemfile_path), "gemfile to changes"), gemfile.generate)
# build a definition, providing an intentionally-empty "unlock" mapping
# to ensure that all gem versions remain locked
definition = builder.to_definition(lockfile_path, {})

definition = builder.to_definition(lockfile_path, {})
definition.lock(lockfile_path)
gemfile.save
# lock the definition and save our modified gemfile
definition.lock(lockfile_path)
gemfile.save

gems_to_remove.each do |gem_name|
LogStash::PluginManager.ui.info("Successfully removed #{gem_name}")
end

LogStash::PluginManager.ui.info("Successfully removed #{gem_name}")
ensure
file.close if file
return true
end
end

def display_cant_remove_message(gem_name, dependencies_from)
message = <<-eos
Failed to remove \"#{gem_name}\" because the following plugins or libraries depend on it:
* #{dependencies_from.join("\n* ")}
eos
message = <<~EOS
Failed to remove \"#{gem_name}\" because the following plugins or libraries depend on it:
* #{dependencies_from.join("\n* ")}
EOS
LogStash::PluginManager.ui.info(message)
end

Expand All @@ -93,10 +89,22 @@ def unfreeze_gemfile
end
end

def self.uninstall!(gem_name, options = { :gemfile => LogStash::Environment::GEMFILE, :lockfile => LogStash::Environment::LOCKFILE })
gemfile_path = options[:gemfile]
lockfile_path = options[:lockfile]
LogstashUninstall.new(gemfile_path, lockfile_path).uninstall!(gem_name)
##
# Yields a mutable gemfile backed by an open, writable file handle.
# It is the responsibility of the caller to send `LogStash::Gemfile#save` to persist the result.
# @yieldparam [LogStash::Gemfile]
def with_mutable_gemfile
unfreeze_gemfile do
File.open(gemfile_path, 'r+') do |file|
yield LogStash::Gemfile.new(file).load
end
end
end

def self.uninstall!(gem_names, options={})
gemfile_path = options[:gemfile] || LogStash::Environment::GEMFILE
lockfile_path = options[:lockfile] || LogStash::Environment::LOCKFILE
LogstashUninstall.new(gemfile_path, lockfile_path).uninstall!(Array(gem_names))
end
end
end
61 changes: 32 additions & 29 deletions lib/pluginmanager/remove.rb
Original file line number Diff line number Diff line change
Expand Up @@ -20,48 +20,51 @@
require "pluginmanager/command"

class LogStash::PluginManager::Remove < LogStash::PluginManager::Command
parameter "PLUGIN", "plugin name"
parameter "PLUGIN ...", "plugin name(s)"

def execute
signal_error("One or more plugins must be specified") if plugin_list.empty?
signal_error("File #{LogStash::Environment::GEMFILE_PATH} does not exist or is not writable, aborting") unless File.writable?(LogStash::Environment::GEMFILE_PATH)

LogStash::Bundler.prepare({:without => [:build, :development]})

if LogStash::PluginManager::ALIASES.has_key?(plugin)
unless LogStash::PluginManager.installed_plugin?(plugin, gemfile)
aliased_plugin = LogStash::PluginManager::ALIASES[plugin]
puts "Cannot remove the alias #{plugin}, which is an alias for #{aliased_plugin}; if you wish to remove it, you must remove the aliased plugin instead."
return
plugin_list.each do |plugin|
if LogStash::PluginManager::ALIASES.has_key?(plugin)
unless LogStash::PluginManager.installed_plugin?(plugin, gemfile)
aliased_plugin = LogStash::PluginManager::ALIASES[plugin]
puts "Cannot remove the alias #{plugin}, which is an alias for #{aliased_plugin}; if you wish to remove it, you must remove the aliased plugin instead."
return
end
end
end

# If a user is attempting to uninstall X-Pack, present helpful output to guide
# them toward the OSS-only distribution of Logstash
LogStash::PluginManager::XPackInterceptor::Remove.intercept!(plugin)
# If a user is attempting to uninstall X-Pack, present helpful output to guide
# them toward the OSS-only distribution of Logstash
LogStash::PluginManager::XPackInterceptor::Remove.intercept!(plugin)

# if the plugin is provided by an integration plugin. abort.
if integration_plugin = LogStash::PluginManager.which_integration_plugin_provides(plugin, gemfile)
signal_error("This plugin is already provided by '#{integration_plugin.name}' so it can't be removed individually")
end
# if the plugin is provided by an integration plugin. abort.
if integration_plugin = LogStash::PluginManager.which_integration_plugin_provides(plugin, gemfile)
signal_error("The plugin `#{plugin}` is provided by '#{integration_plugin.name}' so it can't be removed individually")
end

not_installed_message = "This plugin has not been previously installed"
plugin_gem_spec = LogStash::PluginManager.find_plugins_gem_specs(plugin).any?
if plugin_gem_spec
# make sure this is an installed plugin and present in Gemfile.
# it is not possible to uninstall a dependency not listed in the Gemfile, for example a dependent codec
signal_error(not_installed_message) unless LogStash::PluginManager.installed_plugin?(plugin, gemfile)
else
# locally installed plugins are not recoginized by ::Gem::Specification
# we may ::Bundler.setup to reload but it resets all dependencies so we get error message for future bundler operations
# error message: `Bundler::GemNotFound: Could not find rubocop-1.60.2... in locally installed gems`
# instead we make sure Gemfile has a definition and ::Gem::Specification recognizes local gem
signal_error(not_installed_message) unless !!gemfile.find(plugin)
not_installed_message = "The plugin `#{plugin}` has not been previously installed"
plugin_gem_spec = LogStash::PluginManager.find_plugins_gem_specs(plugin).any?
if plugin_gem_spec
# make sure this is an installed plugin and present in Gemfile.
# it is not possible to uninstall a dependency not listed in the Gemfile, for example a dependent codec
signal_error(not_installed_message) unless LogStash::PluginManager.installed_plugin?(plugin, gemfile)
else
# locally installed plugins are not recoginized by ::Gem::Specification
# we may ::Bundler.setup to reload but it resets all dependencies so we get error message for future bundler operations
# error message: `Bundler::GemNotFound: Could not find rubocop-1.60.2... in locally installed gems`
# instead we make sure Gemfile has a definition and ::Gem::Specification recognizes local gem
signal_error(not_installed_message) unless !!gemfile.find(plugin)

local_gem = gemfile.locally_installed_gems.detect { |local_gem| local_gem.name == plugin }
signal_error(not_installed_message) unless local_gem
local_gem = gemfile.locally_installed_gems.detect { |local_gem| local_gem.name == plugin }
signal_error(not_installed_message) unless local_gem
end
end

exit(1) unless ::Bundler::LogstashUninstall.uninstall!(plugin)
exit(1) unless ::Bundler::LogstashUninstall.uninstall!(plugin_list)
LogStash::Bundler.genericize_platform
remove_unused_locally_installed_gems!
rescue => exception
Expand Down
4 changes: 4 additions & 0 deletions qa/integration/fixtures/plugins/generate-gems.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
#!/usr/bin/env sh

cd "$( dirname "$0" )"
find . -name '*.gemspec' | xargs -n1 gem build
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
Gem::Specification.new do |s|
s.name = File.basename(__FILE__, ".gemspec")
s.version = '0.1.1'
s.licenses = ['Apache-2.0']
s.summary = "A dummy plugin with two plugin dependencies"
s.description = "This plugin is only used in the acceptance test"
s.authors = ["Elasticsearch"]
s.email = '[email protected]'
s.homepage = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
s.require_paths = ["lib"]

# Files
s.files = [__FILE__]

# Tests
s.test_files = []

# Special flag to let us know this is actually a logstash plugin
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }

# Gem dependencies
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"

s.add_runtime_dependency "logstash-filter-one_no_dependencies", "~> 0.1"
s.add_runtime_dependency "logstash-filter-three_no_dependencies", "~> 0.1"
end
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
Gem::Specification.new do |s|
s.name = File.basename(__FILE__, ".gemspec")
s.version = '0.1.1'
s.licenses = ['Apache-2.0']
s.summary = "A dummy plugin with no dependencies"
s.description = "This plugin is only used in the acceptance test"
s.authors = ["Elasticsearch"]
s.email = '[email protected]'
s.homepage = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
s.require_paths = ["lib"]

# Files
s.files = [__FILE__]

# Tests
s.test_files = []

# Special flag to let us know this is actually a logstash plugin
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }

# Gem dependencies
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
end
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
Gem::Specification.new do |s|
s.name = File.basename(__FILE__, ".gemspec")
s.version = '0.1.1'
s.licenses = ['Apache-2.0']
s.summary = "A dummy plugin with no dependencies"
s.description = "This plugin is only used in the acceptance test"
s.authors = ["Elasticsearch"]
s.email = '[email protected]'
s.homepage = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
s.require_paths = ["lib"]

# Files
s.files = [__FILE__]

# Tests
s.test_files = []

# Special flag to let us know this is actually a logstash plugin
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }

# Gem dependencies
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
end
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
Gem::Specification.new do |s|
s.name = File.basename(__FILE__, ".gemspec")
s.version = '0.1.1'
s.licenses = ['Apache-2.0']
s.summary = "A dummy plugin with one plugin dependency"
s.description = "This plugin is only used in the acceptance test"
s.authors = ["Elasticsearch"]
s.email = '[email protected]'
s.homepage = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
s.require_paths = ["lib"]

# Files
s.files = [__FILE__]

# Tests
s.test_files = []

# Special flag to let us know this is actually a logstash plugin
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }

# Gem dependencies
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"

s.add_runtime_dependency "logstash-filter-one_no_dependencies", "~> 0.1"
end
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
Gem::Specification.new do |s|
s.name = File.basename(__FILE__, ".gemspec")
s.version = '0.1.1'
s.licenses = ['Apache-2.0']
s.summary = "A dummy plugin with no dependencies"
s.description = "This plugin is only used in the acceptance test"
s.authors = ["Elasticsearch"]
s.email = '[email protected]'
s.homepage = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
s.require_paths = ["lib"]

# Files
s.files = [__FILE__]

# Tests
s.test_files = []

# Special flag to let us know this is actually a logstash plugin
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }

# Gem dependencies
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
end
20 changes: 13 additions & 7 deletions qa/integration/services/logstash_service.rb
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
require "childprocess"
require "bundler"
require "socket"
require "shellwords"
require "tempfile"
require 'yaml'

Expand Down Expand Up @@ -337,8 +338,9 @@ def initialize(logstash_service)
@logstash_plugin = File.join(@logstash.logstash_home, LOGSTASH_PLUGIN)
end

def remove(plugin_name)
run("remove #{plugin_name}")
def remove(plugin_name, *additional_plugins)
plugin_list = Shellwords.shelljoin([plugin_name]+additional_plugins)
run("remove #{plugin_list}")
end

def prepare_offline_pack(plugins, output_zip = nil)
Expand All @@ -351,20 +353,24 @@ def prepare_offline_pack(plugins, output_zip = nil)
end
end

def list(plugin_name, verbose = false)
run("list #{plugin_name} #{verbose ? "--verbose" : ""}")
def list(*plugins, verbose: false)
command = "list"
command << " --verbose" if verbose
command << " #{Shellwords.shelljoin(plugins)}" if plugins.any?
run(command)
end

def install(plugin_name)
run("install #{plugin_name}")
def install(plugin_name, *additional_plugins)
plugin_list = ([plugin_name]+additional_plugins).flatten
run("install #{Shellwords.shelljoin(plugin_list)}")
end

def run(command)
run_raw("#{logstash_plugin} #{command}")
end

def run_raw(cmd, change_dir = true, environment = {})
@logstash.run_cmd(cmd.split(' '), change_dir, environment)
@logstash.run_cmd(Shellwords.shellsplit(cmd), change_dir, environment)
end
end
end
Loading

0 comments on commit 0895588

Please sign in to comment.