From b6cddb1f04a43147195f29c87a71f81b8ae3fb19 Mon Sep 17 00:00:00 2001 From: Fatih Mehmet ARSLAN Date: Mon, 12 Apr 2021 09:37:25 +0000 Subject: [PATCH 1/3] parametric solr url, document id field name made customizable --- nohup.out | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 nohup.out diff --git a/nohup.out b/nohup.out new file mode 100644 index 0000000..6f3c3a4 --- /dev/null +++ b/nohup.out @@ -0,0 +1,13 @@ +Using JAVA_HOME defined java: /home/gitpod/.sdkman/candidates/java/current +WARNING, using JAVA_HOME while Logstash distribution comes with a bundled JDK +warning: ignoring JAVA_TOOL_OPTIONS=-Xmx1879m +OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release. +WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults +Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console +[INFO ] 2021-04-12 06:15:23.860 [main] runner - Starting Logstash {"logstash.version"=>"7.12.0", "jruby.version"=>"jruby 9.2.13.0 (2.5.7) 2020-08-03 9a89c94bcc OpenJDK 64-Bit Server VM 11.0.10+9-LTS on 11.0.10+9-LTS +indy +jit [linux-x86_64]"} +[FATAL] 2021-04-12 06:15:23.883 [main] runner - An unexpected error occurred! {:error=>#, :backtrace=>["/usr/share/logstash/logstash-core/lib/logstash/settings.rb:530:in `validate'", "/usr/share/logstash/logstash-core/lib/logstash/settings.rb:290:in `validate_value'", "/usr/share/logstash/logstash-core/lib/logstash/settings.rb:201:in `block in validate_all'", "org/jruby/RubyHash.java:1415:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/settings.rb:200:in `validate_all'", "/usr/share/logstash/logstash-core/lib/logstash/runner.rb:326:in `execute'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/clamp-0.6.5/lib/clamp/command.rb:67:in `run'", "/usr/share/logstash/logstash-core/lib/logstash/runner.rb:274:in `run'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/clamp-0.6.5/lib/clamp/command.rb:132:in `run'", "/usr/share/logstash/lib/bootstrap/environment.rb:88:in `
'"]} +[FATAL] 2021-04-12 06:15:23.887 [main] Logstash - Logstash stopped processing because of an error: (SystemExit) exit +org.jruby.exceptions.SystemExit: (SystemExit) exit + at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:747) ~[jruby-complete-9.2.13.0.jar:?] + at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:710) ~[jruby-complete-9.2.13.0.jar:?] + at usr.share.logstash.lib.bootstrap.environment.
(/usr/share/logstash/lib/bootstrap/environment.rb:89) ~[?:?] From eb3059351759dc239604c1bc7ff0726aa226c967 Mon Sep 17 00:00:00 2001 From: Fatih Mehmet ARSLAN Date: Mon, 12 Apr 2021 10:57:25 +0000 Subject: [PATCH 2/3] parametric sorl url, customizable document id field name --- lib/logstash/outputs/solr_http.rb | 33 +++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/lib/logstash/outputs/solr_http.rb b/lib/logstash/outputs/solr_http.rb index 7a0f106..c02159a 100644 --- a/lib/logstash/outputs/solr_http.rb +++ b/lib/logstash/outputs/solr_http.rb @@ -39,10 +39,12 @@ class LogStash::Outputs::SolrHTTP < LogStash::Outputs::Base # '%{foo}' so you can assign your own IDs config :document_id, :validate => :string, :default => nil + # Solr field name of document ID field name. + config :document_id_field, :validate => :string, :default => "id" + public def register require "rsolr" - @solr = RSolr.connect :url => @solr_url buffer_initialize( :max_items => @flush_size, :max_interval => @idle_flush_time, @@ -52,26 +54,37 @@ def register public def receive(event) - buffer_receive(event) end #def receive public def flush(events, close=false) - documents = [] #this is the array of hashes that we push to Solr as documents - + documents = Hash.new #this is the map of hashes that we push to Solr as documents events.each do |event| - document = event.to_hash() - document["@timestamp"] = document["@timestamp"].iso8601 #make the timestamp ISO + url = event.sprintf(@solr_url) # solr url sprintf + + if documents[url].nil? + documents[url]=[] # create a new array to url + @logger.debug("new url created [#{url}]") + end + document = event.to_hash() if @document_id.nil? - document ["id"] = UUIDTools::UUID.random_create #add a unique ID + document [@document_id_field] = UUIDTools::UUID.random_create #add a unique ID else - document ["id"] = event.sprintf(@document_id) #or use the one provided + document [@document_id_field] = event.sprintf(@document_id) #or use the one provided end - documents.push(document) + documents[url].push(document) + end + @logger.debug("#{documents.keys.length()} url detected") + documents.keys.each do |url| + solr = RSolr.connect :url => url + @logger.debug("solr connected [#{url}]") + @logger.debug("#{documents[url].length()} documents indexing...") + solr.add(documents[url]) + @logger.debug("#{documents[url].length()} documents indexed.") + solr.commit :commit_attributes => {} end - @solr.add(documents) rescue Exception => e @logger.warn("An error occurred while indexing: #{e.message}") end #def flush From a9e43795087b25d2b6a8bc8ff243d8776ea0dcbe Mon Sep 17 00:00:00 2001 From: Fatih Mehmet ARSLAN Date: Mon, 12 Apr 2021 10:57:46 +0000 Subject: [PATCH 3/3] package name chane --- logstash-output-solr_http.gemspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/logstash-output-solr_http.gemspec b/logstash-output-solr_http.gemspec index bc1c7cd..5893d78 100644 --- a/logstash-output-solr_http.gemspec +++ b/logstash-output-solr_http.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| - s.name = 'logstash-output-solr_http' - s.version = '3.0.5' + s.name = 'mantis-logstash-output-solr_http' + s.version = '3.0.5-m10' s.licenses = ['Apache License (2.0)'] s.summary = "Stores and indexes logs in Solr" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"