article file import via plugin

Gemfile

@@ -64,6 +64,8 @@ gem "terser", "~> 1.1"
 
 # we use the git version of acts_as_versioned, and need to include it in this Gemfile
 gem 'acts_as_versioned', git: 'https://github.com/technoweenie/acts_as_versioned.git'
+
+gem 'foodsoft_article_import', path: 'plugins/article_import'
 gem 'foodsoft_discourse', path: 'plugins/discourse'
 gem 'foodsoft_documents', path: 'plugins/documents'
 gem 'foodsoft_links', path: 'plugins/links'

Gemfile.lock

@@ -16,6 +16,14 @@ GIT
     acts_as_versioned (0.6.0)
       activerecord (>= 3.0.9)
 
+PATH
+  remote: plugins/article_import
+  specs:
+    foodsoft_article_import (0.0.1)
+      deface (~> 1.0)
+      rails
+      roo (~> 2.9.0)
+
 PATH
   remote: plugins/discourse
   specs:
@@ -624,6 +632,7 @@ DEPENDENCIES
   exception_notification
   factory_bot_rails
   faker
+  foodsoft_article_import!
   foodsoft_discourse!
   foodsoft_documents!
   foodsoft_links!

app/models/concerns/localize_input.rb

@@ -5,10 +5,10 @@ def self.parse(input)
     return input unless input.is_a? String
 
     Rails.logger.debug { "Input: #{input.inspect}" }
-    separator = I18n.t('separator', scope: 'number.format')
-    delimiter = I18n.t('delimiter', scope: 'number.format')
-    input.gsub!(delimiter, '') if input.match(/\d+#{Regexp.escape(delimiter)}+\d+#{Regexp.escape(separator)}+\d+/) # Remove delimiter
-    input.gsub!(separator, '.') # Replace separator with db compatible character
+    separator = I18n.t("separator", scope: "number.format")
+    delimiter = I18n.t("delimiter", scope: "number.format")
+    input.gsub!(delimiter, "") if input.match(/\d+#{Regexp.escape(delimiter)}+\d+#{Regexp.escape(separator)}+\d+/) # Remove delimiter
+    input.gsub!(separator, ".") or input.gsub!(",", ".") # Replace separator with db compatible character
     input
   rescue StandardError
     Rails.logger.warn "Can't localize input: #{input}"

app/views/articles/_sync_table.html.haml

@@ -32,8 +32,8 @@
             = form.text_field 'name', :size => 0
             - hidden_fields.each do |field|
               = form.hidden_field field
-          %td{:style => highlight_new(attrs, :note)}= form.text_field 'note', class: 'input-small'
-          %td{:style => highlight_new(attrs, :manufacturer)}= form.text_field 'manufacturer', class: 'input-small'
+          %td{:style => highlight_new(attrs, :note)}= form.text_field 'note', class: 'input-medium'
+          %td{:style => highlight_new(attrs, :manufacturer)}= form.text_field 'manufacturer', class: 'input-medium'
           %td{:style => highlight_new(attrs, :origin)}= form.text_field 'origin', class: 'input-mini'
           %td{:style => highlight_new(attrs, :unit)}= form.text_field 'unit', class: 'input-mini'
           %td{:style => highlight_new(attrs, :unit_quantity)}= form.text_field 'unit_quantity', class: 'input-mini'
@@ -49,8 +49,8 @@
             .input-prepend
               %span.add-on= t 'number.currency.format.unit'
               = form.text_field 'deposit', class: 'input-mini', style: 'width: 45px'
-          %td= form.select :article_category_id, ArticleCategory.all.map {|a| [ a.name, a.id ] },
-            {include_blank: true}, class: 'input-small'
+          %td{:style => highlight_new(attrs, :article_category)}
+            = form.select :article_category_id, ArticleCategory.all.map {|a| [ a.name, a.id ] }, {include_blank: true}
       - unless changed_article.errors.empty?
         %tr.alert
           %td(colspan=11)= changed_article.errors.full_messages.join(', ')

plugins/article_import/app/overrides/articles/upload/replace_label_with_file_format_option.html.haml.deface

@@ -0,0 +1,10 @@
+/ insert_after 'erb:contains("file_field")'
+- if FoodsoftArticleImport.enabled?
+  %label(for="articles_file")
+    %strong="select the file type you are about to upload"
+  =f.collection_select :type, FoodsoftArticleImport::FORMATS , :to_s, :to_s
+/ insert_before 'erb:contains("articles_outlist_absent")'
+-if FoodsoftArticleImport.enabled?
+  %label(for="articles_update_category")
+    = f.check_box "update_category"
+    = t 'Kategorien aus der Datei übernehmen und erstellen.'

plugins/article_import/app/overrides/controllers/articles_controller_override.rb

@@ -0,0 +1,20 @@
+if FoodsoftArticleImport.enabled?
+  ArticlesController.class_eval do
+    def parse_upload
+      uploaded_file = params[:articles]['file'] or raise I18n.t('articles.controller.parse_upload.no_file')
+      type = params[:articles]['type']
+      options = { filename: uploaded_file.original_filename }
+      options[:outlist_absent] = (params[:articles]['outlist_absent'] == '1')
+      options[:convert_units] = (params[:articles]['convert_units'] == '1')
+      options[:update_category] = (params[:articles]['update_category'] == '1')
+
+      @updated_article_pairs, @outlisted_articles, @new_articles = @supplier.sync_from_file uploaded_file.tempfile, type, options
+      if @updated_article_pairs.empty? && @outlisted_articles.empty? && @new_articles.empty?
+        redirect_to supplier_articles_path(@supplier), :notice => I18n.t('articles.controller.parse_upload.notice')
+      end
+      @ignored_article_count = 0
+    rescue => error
+      redirect_to upload_supplier_articles_path(@supplier), :alert => I18n.t('errors.general_msg', :msg => error.message)
+    end
+  end
+end

plugins/article_import/app/overrides/models/article_override.rb

@@ -0,0 +1,39 @@
+if FoodsoftArticleImport.enabled?
+  Article.class_eval do
+    def unequal_attributes(new_article, options = {})
+      # try to convert different units when desired
+      if options[:convert_units] == false
+        new_price = nil
+        new_unit_quantity = nil
+      else
+        new_price, new_unit_quantity = convert_units(new_article)
+      end
+      if new_price && new_unit_quantity
+        new_unit = self.unit
+      else
+        new_price = new_article.price
+        new_unit_quantity = new_article.unit_quantity
+        new_unit = new_article.unit
+      end
+
+      attribute_hash = {
+        :name => [self.name, new_article.name],
+        :manufacturer => [self.manufacturer, new_article.manufacturer.to_s],
+        :origin => [self.origin, new_article.origin],
+        :unit => [self.unit, new_unit],
+        :price => [self.price.to_f.round(2), new_price.to_f.round(2)],
+        :tax => [self.tax, new_article.tax],
+        :deposit => [self.deposit.to_f.round(2), new_article.deposit.to_f.round(2)],
+        # take care of different num-objects.
+        :unit_quantity => [self.unit_quantity.to_s.to_f, new_unit_quantity.to_s.to_f],
+        :note => [self.note.to_s, new_article.note.to_s]
+      }
+      if options[:update_category] == true
+        new_article_category = new_article.article_category
+        attribute_hash[:article_category] = [self.article_category, new_article_category] unless new_article_category.blank?
+      end
+
+      Article.compare_attributes(attribute_hash)
+    end
+  end
+end

plugins/article_import/app/overrides/models/supplier_override.rb

@@ -0,0 +1,53 @@
+if FoodsoftArticleImport.enabled?
+  Supplier.class_eval do
+      # Synchronise articles with spreadsheet.
+    #
+    # @param file [File] Spreadsheet file to parse
+    # @param options [Hash] Options passed to {FoodsoftArticleImport#parse} except when listed here.
+    # @option options [Boolean] :outlist_absent Set to +true+ to remove articles not in spreadsheet.
+    # @option options [Boolean] :convert_units Omit or set to +true+ to keep current units, recomputing unit quantity and price.
+    def sync_from_file(file, type, options = {})
+      all_order_numbers = []
+      updated_article_pairs, outlisted_articles, new_articles = [], [], []
+      custom_codes_path = File.join(Rails.root, "config", "custom_codes.yml")
+      opts = options.except(:convert_units, :outlist_absent)
+      custom_codes_file_path = custom_codes_path if File.exist?(custom_codes_path)
+      FoodsoftArticleImport.parse(file, custom_file_path: custom_codes_file_path, type: type, **opts) do |new_attrs, status, line|
+        article = articles.undeleted.where(order_number: new_attrs[:order_number]).first
+
+        if new_attrs[:article_category].present? && options[:update_category]
+          new_attrs[:article_category] = ArticleCategory.find_match(new_attrs[:article_category]) || ArticleCategory.create_or_find_by!(name: new_attrs[:article_category])
+        else
+          new_attrs[:article_category] = nil
+        end
+
+        new_attrs[:tax] ||= FoodsoftConfig[:tax_default]
+        new_article = articles.build(new_attrs)
+        if status.nil?
+          if article.nil?
+            new_articles << new_article
+          else
+            unequal_attributes = article.unequal_attributes(new_article, options.slice(:convert_units, :update_category))
+            unless unequal_attributes.empty?
+              article.attributes = unequal_attributes
+              updated_article_pairs << [article, unequal_attributes]
+            end
+          end
+        elsif status == :outlisted && article.present?
+          outlisted_articles << article
+
+        # stop when there is a parsing error
+        elsif status.is_a? String
+          # @todo move I18n key to model
+          raise I18n.t('articles.model.error_parse', :msg => status, :line => line.to_s)
+        end
+
+        all_order_numbers << article.order_number if article
+      end
+      if options[:outlist_absent]
+        outlisted_articles += articles.undeleted.where.not(order_number: all_order_numbers + [nil])
+      end
+      [updated_article_pairs, outlisted_articles, new_articles]
+    end
+  end
+end

plugins/article_import/foodsoft_article_import.gemspec

@@ -0,0 +1,20 @@
+$:.push File.expand_path("../lib", __FILE__)
+
+# Maintain your gem's version:
+require "foodsoft_article_import/version"
+
+# Describe your gem and declare its dependencies:
+Gem::Specification.new do |s|
+  s.name        = "foodsoft_article_import"
+  s.version     = FoodsoftArticleImport::VERSION
+  s.authors     = ["viehlieb"]
+  s.email       = ["[email protected]"]
+  s.summary     = "Manages manual article import from file. File Formats supported are: foodsoft file(csv), bnn files (.bnn) and odin files (xml)"
+
+  s.files = Dir["{app,config,db,lib}/**/*"] + ["Rakefile", "README.md"]
+
+  s.add_dependency "rails"
+  s.add_dependency "deface", "~> 1.0"
+  s.add_dependency 'roo', '~> 2.9.0'
+  s.add_development_dependency 'simplecov'
+end

plugins/article_import/lib/foodsoft_article_import.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+require "deface"
+require 'foodsoft_article_import/engine'
+require 'digest/sha1'
+require 'tempfile'
+require 'csv'
+require 'yaml'
+require 'active_support/core_ext/hash/keys'
+require_relative 'foodsoft_article_import/bnn'
+require_relative 'foodsoft_article_import/odin'
+require_relative 'foodsoft_article_import/foodsoft'
+module FoodsoftArticleImport
+  class ConversionFailedException < StandardError; end
+
+  FORMATS = %w(bnn foodsoft odin).freeze
+  def self.enabled?
+    FoodsoftConfig[:use_article_import]
+  end
+
+  def self.file_formats
+    @@file_formats ||= {
+      'bnn' => FoodsoftArticleImport::Bnn,
+      'foodsoft' => FoodsoftArticleImport::Foodsoft,
+      'odin' => FoodsoftArticleImport::Odin,
+    }.freeze
+  end
+
+  # Parse file by type (one of {.file_formats})
+  #
+  # @param file [File, Tempfile]
+  # @option opts [String] type file format (required) (see {.file_formats})
+  # @return [File, Roo::Spreadsheet] file with encoding set if needed
+  def self.parse(file, custom_file_path: nil, type: nil, **opts, &blk)
+    @@filename = opts[:filename] if opts[:filename]
+    custom_file_path ||= nil
+    type ||= 'bnn'
+    parser = file_formats[type]
+    if block_given?
+      parser.parse(file, custom_file_path: custom_file_path, &blk)
+    else
+      data = []
+      parser.parse(file, custom_file_path: custom_file_path) { |a| data << a }
+      data
+    end
+  end
+
+  # Helper method to generate an article number for suppliers that do not have one
+  def self.generate_number(article)
+    # something unique, but not too unique
+    s = "#{article[:name]}-#{article[:unit_quantity]}x#{article[:unit]}"
+    s = s.downcase.gsub(/[^a-z0-9.]/, '')
+    # prefix abbreviated sha1-hash with colon to indicate that it's a generated number
+    article[:order_number] = ":#{Digest::SHA1.hexdigest(s)[-7..]}"
+    article
+  end
+
+  # Helper method for opening a spreadsheet file
+  #
+  # @param file [File] file to open
+  # @param filename [String, NilClass] optional filename for guessing the file format
+  # @param encoding [String, NilClass] optional CSV encoding
+  # @param col_sep [String, NilClass] optional column separator
+  # @return [Roo::Spreadsheet]
+  def self.open_spreadsheet(file, encoding: nil, col_sep: nil, liberal_parsing: nil)
+    opts = { csv_options: {} }
+    opts[:csv_options][:encoding] = encoding if encoding
+    opts[:csv_options][:col_sep] = col_sep if col_sep
+    opts[:csv_options][:liberal_parsing] = true if liberal_parsing
+    opts[:extension] = File.extname(File.basename(file)) if file
+    begin
+      Roo::Spreadsheet.open(file, **opts)
+    rescue StandardError => e
+      raise "Failed to parse foodsoft file. make sure file format is correct: #{e.message}"
+    end
+  end
+end

plugins/article_import/lib/foodsoft_article_import/bnn.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+# Module for translation and parsing of BNN-files (www.n-bnn.de)
+#
+module FoodsoftArticleImport
+  module Bnn
+    @@codes = {}
+    @@midgard = {}
+    # Loads the codes_file config/bnn_codes.yml into the class variable @@codes
+    def self.load_codes(custom_file_path = nil)
+      @gem_lib = File.expand_path '..', __dir__
+      dir = File.join @gem_lib, 'foodsoft_article_import'
+      begin
+        @@codes = YAML.safe_load(File.open(File.join(dir, 'bnn_codes.yml'))).symbolize_keys
+        if custom_file_path
+          custom_codes = YAML.safe_load(File.open(custom_file_path)).symbolize_keys
+          custom_codes.each_key do |key|
+            custom_codes[key] = custom_codes[key].merge @@codes[key] if @@codes.keys.include?(key)
+            @@codes = @@codes.merge custom_codes
+          end
+        end
+        @@midgard = YAML.safe_load(File.open(File.join(dir, 'midgard_codes.yml'))).symbolize_keys
+      rescue StandardError => e
+        raise "Failed to load bnn_codes: #{dir}/{bnn,midgard}_codes.yml: #{e.message}"
+      end
+    end
+
+    $missing_bnn_codes = []
+
+    # translates codes from BNN to foodsoft-code
+    def self.translate(key, value)
+      if @@codes[key][value]
+        @@codes[key][value]
+      elsif @@midgard[key]
+        @@midgard[key][value]
+      elsif !value.nil?
+        $missing_bnn_codes << value
+        nil
+      end
+    end
+
+    NAME = 'BNN (CSV)'
+    OUTLIST = false
+    OPTIONS = {
+      encoding: 'IBM850',
+      col_sep: ';'
+    }.freeze
+
+    # parses a bnn-file
+    def self.parse(file, custom_file_path: nil, **opts)
+      custom_file_path ||= nil
+      encoding = opts[:encoding] || OPTIONS[:encoding]
+      col_sep = opts[:col_sep] || OPTIONS[:col_sep]
+      load_codes(custom_file_path)
+      CSV.foreach(file, { col_sep: col_sep, encoding: encoding, headers: true }).with_index(1) do |row, i|
+        # check if the line is empty
+        unless row[0] == '' || row[0].nil?
+          article = {
+            name: row[6],
+            order_number: row[0],
+            note: row[7],
+            manufacturer: translate(:manufacturer, row[10]),
+            origin: row[12],
+            article_category: translate(:category, row[16]),
+            unit: row[23],
+            price: row[37],
+            tax: translate(:tax, row[33]),
+            unit_quantity: row[22]
+          }
+          # TODO: Complete deposit list....
+          article.merge!(deposit: translate(:deposit, row[26])) if translate(:deposit, row[26])
+
+          if !row[62].nil?
+            # consider special prices
+            article[:note] = "Sonderpreis: #{article[:price]} von #{row[62]} bis #{row[63]}"
+            yield article, :special, i
+
+            # Check now for article status, we only consider outlisted articles right now
+            # N=neu, A=Änderung, X=ausgelistet, R=Restbestand,
+            # V=vorübergehend ausgelistet, W=wiedergelistet
+          elsif row[1] == 'X' || row[1] == 'V'
+            yield article, :outlisted, i
+          else
+            yield article, nil, i
+          end
+        end
+      end
+    end
+  end
+end