From ce23597657758f4924f97345f894d88e97702123 Mon Sep 17 00:00:00 2001 From: Du Peng Date: Thu, 2 Feb 2023 12:52:56 +0100 Subject: [PATCH 1/4] (quality) check statistics_module is active? before run build_stats task --- lib/tasks/fablab/es.rake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/tasks/fablab/es.rake b/lib/tasks/fablab/es.rake index 6adcde5bf..74942eef9 100644 --- a/lib/tasks/fablab/es.rake +++ b/lib/tasks/fablab/es.rake @@ -5,6 +5,8 @@ namespace :fablab do namespace :es do desc '(re)Build ElasticSearch fablab base for stats' task build_stats: :environment do + exit unless Setting.get('statistics_module') + delete_stats_index create_stats_index create_stats_mappings From 4d942b41615bf41a1da8ca230f8ab8ce33dd317a Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 2 Feb 2023 15:43:05 +0100 Subject: [PATCH 2/4] (quality) Optimized statistics fetcher service --- CHANGELOG.md | 2 + .../builders/members_builder_service.rb | 2 +- .../builders/projects_builder_service.rb | 2 +- .../builders/reservations_builder_service.rb | 2 +- .../builders/store_orders_builder_service.rb | 2 +- app/services/statistics/fetcher_service.rb | 157 ++++++++++-------- 6 files changed, 92 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8240708ba..0f0897900 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog Fab-manager +- Optimized memory consumption in statistics fetcher service + ## v5.6.9 2023 February 02 - Updated shakapaker to 6.5.5 diff --git a/app/services/statistics/builders/members_builder_service.rb b/app/services/statistics/builders/members_builder_service.rb index 0d792c9af..a7be86cf4 100644 --- a/app/services/statistics/builders/members_builder_service.rb +++ b/app/services/statistics/builders/members_builder_service.rb @@ -7,7 +7,7 @@ class Statistics::Builders::MembersBuilderService class << self def build(options = default_options) # account list - Statistics::FetcherService.members_list(options).each do |m| + Statistics::FetcherService.each_member(options) do |m| Stats::Account.create({ date: format_date(m[:date]), type: 'member', subType: 'created', diff --git a/app/services/statistics/builders/projects_builder_service.rb b/app/services/statistics/builders/projects_builder_service.rb index 0030fb66d..8e47e7c14 100644 --- a/app/services/statistics/builders/projects_builder_service.rb +++ b/app/services/statistics/builders/projects_builder_service.rb @@ -8,7 +8,7 @@ class Statistics::Builders::ProjectsBuilderService class << self def build(options = default_options) # project list - Statistics::FetcherService.projects_list(options).each do |p| + Statistics::FetcherService.each_project(options) do |p| Stats::Project.create({ date: format_date(p[:date]), type: 'project', subType: 'published', diff --git a/app/services/statistics/builders/reservations_builder_service.rb b/app/services/statistics/builders/reservations_builder_service.rb index c14ae8ea4..28bdae03d 100644 --- a/app/services/statistics/builders/reservations_builder_service.rb +++ b/app/services/statistics/builders/reservations_builder_service.rb @@ -8,7 +8,7 @@ class << self def build(options = default_options) # machine/space/training list %w[machine space training event].each do |category| - Statistics::FetcherService.send("reservations_#{category}_list", options).each do |r| + Statistics::FetcherService.send("each_#{category}_reservation", options) do |r| %w[booking hour].each do |type| stat = "Stats::#{category.capitalize}" .constantize diff --git a/app/services/statistics/builders/store_orders_builder_service.rb b/app/services/statistics/builders/store_orders_builder_service.rb index 640418591..302d24959 100644 --- a/app/services/statistics/builders/store_orders_builder_service.rb +++ b/app/services/statistics/builders/store_orders_builder_service.rb @@ -13,7 +13,7 @@ def build(options = default_options) } # orders list states.each do |sub_type, order_states| - Statistics::FetcherService.store_orders_list(order_states, options).each do |o| + Statistics::FetcherService.each_store_order(order_states, options) do |o| Stats::Order.create({ date: format_date(o[:date]), type: 'store', subType: sub_type, diff --git a/app/services/statistics/fetcher_service.rb b/app/services/statistics/fetcher_service.rb index 26c1fbbc5..42eb79817 100644 --- a/app/services/statistics/fetcher_service.rb +++ b/app/services/statistics/fetcher_service.rb @@ -13,7 +13,7 @@ def subscriptions_list(options = default_options) result = [] InvoiceItem.where("object_type = '#{Subscription.name}' AND invoice_items.created_at >= :start_date " \ 'AND invoice_items.created_at <= :end_date', options) - .eager_load(invoice: [:coupon]).each do |i| + .eager_load(invoice: [:coupon]).find_each do |i| next if i.invoice.is_a?(Avoir) sub = i.object @@ -39,97 +39,105 @@ def subscriptions_list(options = default_options) result end - def reservations_machine_list(options = default_options) - result = [] + # @param options [Hash] + # @yield [reservation] + # @yieldparam [Hash] + def each_machine_reservation(options = default_options) Reservation .where("reservable_type = 'Machine' AND slots_reservations.canceled_at IS NULL AND " \ 'reservations.created_at >= :start_date AND reservations.created_at <= :end_date', options) .eager_load(:slots, :slots_reservations, :invoice_items, statistic_profile: [:group]) - .each do |r| + .find_each do |r| next unless r.reservable profile = r.statistic_profile - result.push({ date: r.created_at.to_date, - reservation_id: r.id, - machine_id: r.reservable.id, - machine_type: r.reservable.friendly_id, - machine_name: r.reservable.name, - slot_dates: r.slots.map(&:start_at).map(&:to_date), - nb_hours: (r.slots.map(&:duration).map(&:to_i).reduce(:+) / 3600.0).to_i, - ca: calcul_ca(r.original_invoice) }.merge(user_info(profile))) + result = { date: r.created_at.to_date, + reservation_id: r.id, + machine_id: r.reservable.id, + machine_type: r.reservable.friendly_id, + machine_name: r.reservable.name, + slot_dates: r.slots.map(&:start_at).map(&:to_date), + nb_hours: (r.slots.map(&:duration).map(&:to_i).reduce(:+) / 3600.0).to_i, + ca: calcul_ca(r.original_invoice) }.merge(user_info(profile)) + yield result end - result end - def reservations_space_list(options = default_options) - result = [] + # @param options [Hash] + # @yield [reservation] + # @yieldparam [Hash] + def each_space_reservation(options = default_options) Reservation .where("reservable_type = 'Space' AND slots_reservations.canceled_at IS NULL AND " \ 'reservations.created_at >= :start_date AND reservations.created_at <= :end_date', options) .eager_load(:slots, :slots_reservations, :invoice_items, statistic_profile: [:group]) - .each do |r| + .find_each do |r| next unless r.reservable profile = r.statistic_profile - result.push({ date: r.created_at.to_date, - reservation_id: r.id, - space_id: r.reservable.id, - space_name: r.reservable.name, - space_type: r.reservable.slug, - slot_dates: r.slots.map(&:start_at).map(&:to_date), - nb_hours: (r.slots.map(&:duration).map(&:to_i).reduce(:+) / 3600.0).to_i, - ca: calcul_ca(r.original_invoice) }.merge(user_info(profile))) + result = { date: r.created_at.to_date, + reservation_id: r.id, + space_id: r.reservable.id, + space_name: r.reservable.name, + space_type: r.reservable.slug, + slot_dates: r.slots.map(&:start_at).map(&:to_date), + nb_hours: (r.slots.map(&:duration).map(&:to_i).reduce(:+) / 3600.0).to_i, + ca: calcul_ca(r.original_invoice) }.merge(user_info(profile)) + yield result end - result end - def reservations_training_list(options = default_options) - result = [] + # @param options [Hash] + # @yield [reservation] + # @yieldparam [Hash] + def each_training_reservation(options = default_options) Reservation .where("reservable_type = 'Training' AND slots_reservations.canceled_at IS NULL AND " \ 'reservations.created_at >= :start_date AND reservations.created_at <= :end_date', options) .eager_load(:slots, :slots_reservations, :invoice_items, statistic_profile: [:group]) - .each do |r| + .find_each do |r| next unless r.reservable profile = r.statistic_profile slot = r.slots.first - result.push({ date: r.created_at.to_date, - reservation_id: r.id, - training_id: r.reservable.id, - training_type: r.reservable.friendly_id, - training_name: r.reservable.name, - training_date: slot.start_at.to_date, - nb_hours: difference_in_hours(slot.start_at, slot.end_at), - ca: calcul_ca(r.original_invoice) }.merge(user_info(profile))) + result = { date: r.created_at.to_date, + reservation_id: r.id, + training_id: r.reservable.id, + training_type: r.reservable.friendly_id, + training_name: r.reservable.name, + training_date: slot.start_at.to_date, + nb_hours: difference_in_hours(slot.start_at, slot.end_at), + ca: calcul_ca(r.original_invoice) }.merge(user_info(profile)) + yield result end - result end - def reservations_event_list(options = default_options) - result = [] + # @param options [Hash] + # @yield [reservation] + # @yieldparam [Hash] + def each_event_reservation(options = default_options) Reservation .where("reservable_type = 'Event' AND slots_reservations.canceled_at IS NULL AND " \ 'reservations.created_at >= :start_date AND reservations.created_at <= :end_date', options) .eager_load(:slots, :slots_reservations, :invoice_items, statistic_profile: [:group]) - .each do |r| + .find_each do |r| next unless r.reservable profile = r.statistic_profile slot = r.slots.first - result.push({ date: r.created_at.to_date, - reservation_id: r.id, - event_id: r.reservable.id, - event_type: r.reservable.category.slug, - event_name: r.reservable.name, - event_date: slot.start_at.to_date, - event_theme: (r.reservable.event_themes.first ? r.reservable.event_themes.first.name : ''), - age_range: (r.reservable.age_range_id ? r.reservable.age_range.name : ''), - nb_places: r.total_booked_seats, - nb_hours: difference_in_hours(slot.start_at, slot.end_at), - ca: calcul_ca(r.original_invoice) }.merge(user_info(profile))) + result = { date: r.created_at.to_date, + reservation_id: r.id, + event_id: r.reservable.id, + event_type: r.reservable.category.slug, + event_name: r.reservable.name, + event_date: slot.start_at.to_date, + event_theme: (r.reservable.event_themes.first ? r.reservable.event_themes.first.name : ''), + age_range: (r.reservable.age_range_id ? r.reservable.age_range.name : ''), + nb_places: r.total_booked_seats, + nb_hours: difference_in_hours(slot.start_at, slot.end_at), + ca: calcul_ca(r.original_invoice) }.merge(user_info(profile)) + yield result end - result end def members_ca_list(options = default_options) @@ -139,7 +147,7 @@ def members_ca_list(options = default_options) users_list = [] Reservation.where('reservations.created_at >= :start_date AND reservations.created_at <= :end_date', options) .eager_load(:slots, :invoice_items, statistic_profile: [:group]) - .each do |r| + .find_each do |r| next unless r.reservable reservations_ca_list.push( @@ -148,7 +156,7 @@ def members_ca_list(options = default_options) end Avoir.where('invoices.created_at >= :start_date AND invoices.created_at <= :end_date', options) .eager_load(:invoice_items, statistic_profile: [:group]) - .each do |i| + .find_each do |i| # the following line is a workaround for issue #196 profile = i.statistic_profile || i.main_item.object&.wallet&.user&.statistic_profile avoirs_ca_list.push({ date: i.created_at.to_date, ca: calcul_avoir_ca(i) || 0 }.merge(user_info(profile))) @@ -162,43 +170,50 @@ def members_ca_list(options = default_options) users_list end - def members_list(options = default_options) - result = [] + # @param options [Hash] + # @yield [reservation] + # @yieldparam [Hash] + def each_member(options = default_options) member = Role.find_by(name: 'member') StatisticProfile.where('role_id = :member AND created_at >= :start_date AND created_at <= :end_date', options.merge(member: member.id)) - .each do |sp| + .find_each do |sp| next if sp.user&.need_completion? - result.push({ date: sp.created_at.to_date }.merge(user_info(sp))) + result = { date: sp.created_at.to_date }.merge(user_info(sp)) + yield result end - result end - def projects_list(options = default_options) - result = [] + # @param options [Hash] + # @yield [reservation] + # @yieldparam [Hash] + def each_project(options = default_options) Project.where('projects.published_at >= :start_date AND projects.published_at <= :end_date', options) .eager_load(:licence, :themes, :components, :machines, :project_users, author: [:group]) - .each do |p| - result.push({ date: p.created_at.to_date }.merge(user_info(p.author)).merge(project_info(p))) + .find_each do |p| + result = { date: p.created_at.to_date }.merge(user_info(p.author)).merge(project_info(p)) + yield result end - result end - def store_orders_list(states, options = default_options) - result = [] + # @param states [Array] + # @param options [Hash] + # @yield [reservation] + # @yieldparam [Hash] + def each_store_order(states, options = default_options) Order.includes(order_items: [:orderable]) .joins(:order_items, :order_activities) .where(order_items: { orderable_type: 'Product' }) .where(orders: { state: states }) .where('order_activities.created_at >= :start_date AND order_activities.created_at <= :end_date', options) .group('orders.id') - .each do |o| - result.push({ date: o.created_at.to_date, ca: calcul_ca(o.invoice) } - .merge(user_info(o.statistic_profile)) - .merge(store_order_info(o))) + .find_each do |o| + result = { date: o.created_at.to_date, ca: calcul_ca(o.invoice) } + .merge(user_info(o.statistic_profile)) + .merge(store_order_info(o)) + yield result end - result end private From aee2c684855dda3e668c4d3e358fc7cad29fc868 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 2 Feb 2023 15:48:58 +0100 Subject: [PATCH 3/4] (bug) private method create_statistic_subtype --- CHANGELOG.md | 2 ++ config/initializers/sentry.rb | 2 +- db/seeds/statistics.rb | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f0897900..81d5a5b6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog Fab-manager - Optimized memory consumption in statistics fetcher service +- Fix a bug: private method `create_statistic_subtype' +- [TODO DEPLOY] `rails db:seed` ## v5.6.9 2023 February 02 diff --git a/config/initializers/sentry.rb b/config/initializers/sentry.rb index c2ee3c56a..885fba101 100644 --- a/config/initializers/sentry.rb +++ b/config/initializers/sentry.rb @@ -22,7 +22,7 @@ # Set traces_sample_rate to 1.0 to capture 100% # of transactions for performance monitoring. # We recommend adjusting this value in production. - config.traces_sample_rate = 0.1 + config.traces_sample_rate = 0.01 config.environment = Rails.env config.release = Version.current end diff --git a/db/seeds/statistics.rb b/db/seeds/statistics.rb index b31b9a824..bd29a34ea 100644 --- a/db/seeds/statistics.rb +++ b/db/seeds/statistics.rb @@ -145,7 +145,7 @@ Plan.find_each do |plan| type = plan.find_statistic_type subtype = if StatisticSubType.find_by(key: plan.slug).nil? - plan.create_statistic_subtype + StatisticSubType.create!(key: plan.slug, label: plan.name) else StatisticSubType.find_by(key: plan.slug) end From ed5670df07cbe9136c0b0ce2bf71f8556e6cb5da Mon Sep 17 00:00:00 2001 From: Sylvain Date: Thu, 2 Feb 2023 15:54:34 +0100 Subject: [PATCH 4/4] Version 5.6.10 --- CHANGELOG.md | 2 ++ package.json | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81d5a5b6c..0866a053c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog Fab-manager +## v5.6.10 2023 February 02 + - Optimized memory consumption in statistics fetcher service - Fix a bug: private method `create_statistic_subtype' - [TODO DEPLOY] `rails db:seed` diff --git a/package.json b/package.json index 6a1cba88d..67094675b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "fab-manager", - "version": "5.6.9", + "version": "5.6.10", "description": "Fab-manager is the FabLab management solution. It provides a comprehensive, web-based, open-source tool to simplify your administrative tasks and your marker's projects.", "keywords": [ "fablab",