Merge pull request #1 from fmarslan/v3.0.5-m7

V3.0.5 m7

lib/logstash/outputs/solr_http.rb

@@ -39,10 +39,12 @@ class LogStash::Outputs::SolrHTTP < LogStash::Outputs::Base
   # '%{foo}' so you can assign your own IDs
   config :document_id, :validate => :string, :default => nil
 
+  # Solr field name of document ID field name.
+  config :document_id_field, :validate => :string, :default => "id"
+
   public
   def register
     require "rsolr"
-    @solr = RSolr.connect :url => @solr_url
     buffer_initialize(
       :max_items => @flush_size,
       :max_interval => @idle_flush_time,
@@ -52,26 +54,37 @@ def register
 
   public
   def receive(event)
-
     buffer_receive(event)
   end #def receive
 
   public
   def flush(events, close=false)
-    documents = []  #this is the array of hashes that we push to Solr as documents
-
+    documents = Hash.new   #this is the map of hashes that we push to Solr as documents
     events.each do |event|
-        document = event.to_hash()
-        document["@timestamp"] = document["@timestamp"].iso8601 #make the timestamp ISO
+        url = event.sprintf(@solr_url)  # solr url sprintf
+
+        if documents[url].nil? 
+          documents[url]=[]  # create a new array to url
+          @logger.debug("new url created [#{url}]")
+        end        
+        document = event.to_hash()        
         if @document_id.nil?
-          document ["id"] = UUIDTools::UUID.random_create    #add a unique ID
+          document [@document_id_field] = UUIDTools::UUID.random_create    #add a unique ID
         else
-          document ["id"] = event.sprintf(@document_id)      #or use the one provided
+          document [@document_id_field] = event.sprintf(@document_id)      #or use the one provided
         end
-        documents.push(document)
+        documents[url].push(document)
+    end
+    @logger.debug("#{documents.keys.length()} url detected")
+    documents.keys.each do |url|
+      solr = RSolr.connect :url => url
+      @logger.debug("solr connected [#{url}]")
+      @logger.debug("#{documents[url].length()} documents indexing...")
+      solr.add(documents[url])
+      @logger.debug("#{documents[url].length()} documents indexed.")
+      solr.commit :commit_attributes => {}
     end
 
-    @solr.add(documents)
     rescue Exception => e
       @logger.warn("An error occurred while indexing: #{e.message}")
   end #def flush

logstash-output-solr_http.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
-  s.name            = 'logstash-output-solr_http'
-  s.version         = '3.0.5'
+  s.name            = 'mantis-logstash-output-solr_http'
+  s.version         = '3.0.5-m10'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Stores and indexes logs in Solr"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

nohup.out

@@ -0,0 +1,13 @@
+Using JAVA_HOME defined java: /home/gitpod/.sdkman/candidates/java/current
+WARNING, using JAVA_HOME while Logstash distribution comes with a bundled JDK
+warning: ignoring JAVA_TOOL_OPTIONS=-Xmx1879m
+OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
+WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
+Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
+[INFO ] 2021-04-12 06:15:23.860 [main] runner - Starting Logstash {"logstash.version"=>"7.12.0", "jruby.version"=>"jruby 9.2.13.0 (2.5.7) 2020-08-03 9a89c94bcc OpenJDK 64-Bit Server VM 11.0.10+9-LTS on 11.0.10+9-LTS +indy +jit [linux-x86_64]"}
+[FATAL] 2021-04-12 06:15:23.883 [main] runner - An unexpected error occurred! {:error=>#<ArgumentError: Path "/usr/share/logstash/data" must be a writable directory. It is not writable.>, :backtrace=>["/usr/share/logstash/logstash-core/lib/logstash/settings.rb:530:in `validate'", "/usr/share/logstash/logstash-core/lib/logstash/settings.rb:290:in `validate_value'", "/usr/share/logstash/logstash-core/lib/logstash/settings.rb:201:in `block in validate_all'", "org/jruby/RubyHash.java:1415:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/settings.rb:200:in `validate_all'", "/usr/share/logstash/logstash-core/lib/logstash/runner.rb:326:in `execute'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/clamp-0.6.5/lib/clamp/command.rb:67:in `run'", "/usr/share/logstash/logstash-core/lib/logstash/runner.rb:274:in `run'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/clamp-0.6.5/lib/clamp/command.rb:132:in `run'", "/usr/share/logstash/lib/bootstrap/environment.rb:88:in `<main>'"]}
+[FATAL] 2021-04-12 06:15:23.887 [main] Logstash - Logstash stopped processing because of an error: (SystemExit) exit
+org.jruby.exceptions.SystemExit: (SystemExit) exit
+	at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:747) ~[jruby-complete-9.2.13.0.jar:?]
+	at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:710) ~[jruby-complete-9.2.13.0.jar:?]
+	at usr.share.logstash.lib.bootstrap.environment.<main>(/usr/share/logstash/lib/bootstrap/environment.rb:89) ~[?:?]