From daa5e0050ec98a425fd66081ccd117480c12dc46 Mon Sep 17 00:00:00 2001 From: "epugh@opensourceconnections.com" Date: Fri, 26 Aug 2022 07:34:31 -0400 Subject: [PATCH 1/5] basic setup --- .../javascripts/factories/ScorerFactory.js | 1 + app/assets/templates/views/queriesLayout.html | 2 +- app/models/scorer.rb | 2 ++ .../admin/communal_scorers/_form.html.erb | 13 ++++++++++ .../admin/communal_scorers/index.html.erb | 4 ++++ .../v1/scorers/_communal_scorer.json.jbuilder | 1 + .../api/v1/scorers/_scorer.json.jbuilder | 1 + ...0824161522_add_richer_fields_to_scorers.rb | 7 ++++++ db/schema.rb | 5 +++- db/seeds.rb | 24 +++++++++++++++---- 10 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 db/migrate/20220824161522_add_richer_fields_to_scorers.rb diff --git a/app/assets/javascripts/factories/ScorerFactory.js b/app/assets/javascripts/factories/ScorerFactory.js index 05f508942..f6b15ed61 100644 --- a/app/assets/javascripts/factories/ScorerFactory.js +++ b/app/assets/javascripts/factories/ScorerFactory.js @@ -28,6 +28,7 @@ self.manualMaxScore = data.manualMaxScore || false; self.manualMaxScoreValue = data.manualMaxScoreValue || 100; self.name = data.name; + self.tooltip = data.tooltip; self.owned = data.owned; self.ownerId = data.owner_id; self.ownerName = data.owner_name; diff --git a/app/assets/templates/views/queriesLayout.html b/app/assets/templates/views/queriesLayout.html index ee7eab894..e75f53cf4 100644 --- a/app/assets/templates/views/queriesLayout.html +++ b/app/assets/templates/views/queriesLayout.html @@ -54,7 +54,7 @@

— - {{ getScorer().name }} + {{ getScorer().name }} | {{ getScorer().tooltip }}

diff --git a/app/models/scorer.rb b/app/models/scorer.rb index a103c499e..90be9bc26 100644 --- a/app/models/scorer.rb +++ b/app/models/scorer.rb @@ -56,6 +56,8 @@ class Scorer < ApplicationRecord scope :communal, -> { where(communal: true) } + enum rollup_method: [:average_of_scores, :sum_of_scores] + # the default scorer for users who don't have one specified. def self.system_default_scorer find_by(name: Rails.application.config.quepid_default_scorer) diff --git a/app/views/admin/communal_scorers/_form.html.erb b/app/views/admin/communal_scorers/_form.html.erb index 33d428cf2..a7f97615c 100644 --- a/app/views/admin/communal_scorers/_form.html.erb +++ b/app/views/admin/communal_scorers/_form.html.erb @@ -64,6 +64,19 @@ +
+ <%= f.label :tooltip, class: 'form-label' %> + <%= f.text_field :tooltip, class: 'form-control' %> +
A short "tooltip" style description.
+
+ +
+ <%= f.label :description, class: 'form-label' %> +
The gory details about this Scorer!
+ <%= f.text_area :description %> +
+ + <%= f.submit class: 'btn btn-primary' %> <% end %> diff --git a/app/views/admin/communal_scorers/index.html.erb b/app/views/admin/communal_scorers/index.html.erb index 839887bd3..a9b668dc4 100644 --- a/app/views/admin/communal_scorers/index.html.erb +++ b/app/views/admin/communal_scorers/index.html.erb @@ -7,12 +7,14 @@ ID Name + Tooltip Code Scale Manual Max Score? Manual Max Score Value Show Scale Labels? Scale with Labels + Description Updated At @@ -23,6 +25,7 @@ <%= scorer.id %> <%= scorer.name %> + <%= scorer.tooltip %>
<%= scorer.manual_max_score_value %> <%= scorer.show_scale_labels? %> <%= scorer.scale_with_labels %> + <%= scorer.description %> <%= scorer.updated_at %> <% if scorer.updated_at %> diff --git a/app/views/api/v1/scorers/_communal_scorer.json.jbuilder b/app/views/api/v1/scorers/_communal_scorer.json.jbuilder index eccfc5dec..4b118875b 100644 --- a/app/views/api/v1/scorers/_communal_scorer.json.jbuilder +++ b/app/views/api/v1/scorers/_communal_scorer.json.jbuilder @@ -4,6 +4,7 @@ json.scorerId scorer.id json.communal scorer.communal json.code scorer.code json.name scorer.name +json.tooltip scorer.tooltip json.scale scorer.scale json.manualMaxScore scorer.manual_max_score json.manualMaxScoreValue scorer.manual_max_score_value diff --git a/app/views/api/v1/scorers/_scorer.json.jbuilder b/app/views/api/v1/scorers/_scorer.json.jbuilder index 0ff18748a..483d35128 100644 --- a/app/views/api/v1/scorers/_scorer.json.jbuilder +++ b/app/views/api/v1/scorers/_scorer.json.jbuilder @@ -6,6 +6,7 @@ json.scorerId scorer.id json.communal scorer.communal json.code scorer.code json.name scorer.name +json.tooltip scorer.tooltip json.scale scorer.scale json.owner_id scorer.owner_id json.owned scorer.owner_id == current_user.id diff --git a/db/migrate/20220824161522_add_richer_fields_to_scorers.rb b/db/migrate/20220824161522_add_richer_fields_to_scorers.rb new file mode 100644 index 000000000..c34dd2cbd --- /dev/null +++ b/db/migrate/20220824161522_add_richer_fields_to_scorers.rb @@ -0,0 +1,7 @@ +class AddRicherFieldsToScorers < ActiveRecord::Migration[6.1] + def change + add_column :scorers, :tooltip, :string + add_column :scorers, :description, :text + add_column :scorers, :rollup_method, :integer, default: 0 + end +end diff --git a/db/schema.rb b/db/schema.rb index 3b8d0a6ca..af4df701f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2022_08_21_103132) do +ActiveRecord::Schema.define(version: 2022_08_24_161522) do create_table "annotations", id: :integer, charset: "utf8", force: :cascade do |t| t.text "message" @@ -110,6 +110,9 @@ t.datetime "created_at", null: false t.datetime "updated_at", null: false t.boolean "communal", default: false + t.string "tooltip" + t.text "description" + t.integer "rollup_method", default: 0 end create_table "snapshot_docs", id: :integer, charset: "latin1", force: :cascade do |t| diff --git a/db/seeds.rb b/db/seeds.rb index d977ff70e..149991f43 100755 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -14,7 +14,10 @@ name: 'nDCG@10', communal: true, manual_max_score: true, - manual_max_score_value: 1 + manual_max_score_value: 1, + tooltip: 'tool tip here', + description: 'Normalized discounted sum of the gain values. The ratio of the DCG to the Ideal DCG that results from the optimal ordering of the rated relevant documents. Useful for comparing across queries. Useful for averaging over multiple queries.', + rollup_method: 'average_of_scores' ) Scorer.where(name: 'DCG@10').first_or_create( @@ -23,7 +26,10 @@ show_scale_labels: true, code: File.readlines('./db/scorers/dcg@10.js','\n').join('\n'), name: 'DCG@10', - communal: true + communal: true, + tooltip: 'tool tip here', + description: 'Discounted sum of the gain values. Not useful for comparing across queries. Not useful for averaging over multiple queries.', + rollup_method: 'sum_of_scores' ) Scorer.where(name: 'CG@10').first_or_create( @@ -32,7 +38,9 @@ show_scale_labels: true, code: File.readlines('./db/scorers/cg@10.js','\n').join('\n'), name: 'CG@10', - communal: true + communal: true, + description: 'Simple sum of the gain values. Not useful for comparing across queries. Not useful for averaging over multiple queries.', + rollup_method: 'sum_of_scores' ) Scorer.where(name: 'P@10').first_or_create( @@ -41,7 +49,10 @@ show_scale_labels: true, code: File.readlines('./db/scorers/p@10.js','\n').join('\n'), name: 'P@10', - communal: true + communal: true, + tooltip: 'tool tip here', + description: 'The effective accuracy, of the top k (10) retrieved, how many of them were in the set of documents rated relevant. Useful for comparing across queries. Useful for averaging across queries.', + rollup_method: 'average_of_scores' ) Scorer.where(name: 'AP@10').first_or_create( @@ -50,7 +61,10 @@ show_scale_labels: true, code: File.readlines('./db/scorers/ap@10.js','\n').join('\n'), name: 'AP@10', - communal: true + communal: true, + tooltip: 'tool tip here', + description: '(while still called @10, AP is computed on |R| the set of all relevant documents) The average of the precision points (accuracy) at each rank at which a relevant document appears. This average is across the full set of relevant documents. Useful for comparing across queries. Useful for averaging over multiple queries. This metric is the most common summary metric for capturing overall performance of a retrieval system independent of the retrieval task.', + rollup_method: 'sum_of_scores' ) if ENV['SEED_SAMPLE_DATA'] From 7d41f3c17dd5019610ae8c362c23fab598ee17a8 Mon Sep 17 00:00:00 2001 From: "epugh@opensourceconnections.com" Date: Fri, 26 Aug 2022 10:37:45 -0400 Subject: [PATCH 2/5] better docs! --- app/models/scorer.rb | 2 +- test/fixtures/scorers.yml | 2 +- test/models/scorer_test.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/scorer.rb b/app/models/scorer.rb index 964fce8b0..a4bdfa899 100644 --- a/app/models/scorer.rb +++ b/app/models/scorer.rb @@ -11,7 +11,7 @@ # manual_max_score :boolean default(FALSE) # manual_max_score_value :integer default(100) # name :string(255) -# rollup_method :integer default(0) +# rollup_method :integer default("average_of_scores") # scale :string(255) # scale_with_labels :text(65535) # show_scale_labels :boolean default(FALSE) diff --git a/test/fixtures/scorers.yml b/test/fixtures/scorers.yml index 11860b967..930672b5f 100644 --- a/test/fixtures/scorers.yml +++ b/test/fixtures/scorers.yml @@ -9,7 +9,7 @@ # manual_max_score :boolean default(FALSE) # manual_max_score_value :integer default(100) # name :string(255) -# rollup_method :integer default(0) +# rollup_method :integer default("average_of_scores") # scale :string(255) # scale_with_labels :text(65535) # show_scale_labels :boolean default(FALSE) diff --git a/test/models/scorer_test.rb b/test/models/scorer_test.rb index d82f5ab59..d3effe797 100644 --- a/test/models/scorer_test.rb +++ b/test/models/scorer_test.rb @@ -11,7 +11,7 @@ # manual_max_score :boolean default(FALSE) # manual_max_score_value :integer default(100) # name :string(255) -# rollup_method :integer default(0) +# rollup_method :integer default("average_of_scores") # scale :string(255) # scale_with_labels :text(65535) # show_scale_labels :boolean default(FALSE) From 4fece80f4f3efe201b1251d1423d9a0d4bf2488c Mon Sep 17 00:00:00 2001 From: "epugh@opensourceconnections.com" Date: Sat, 27 Aug 2022 08:57:35 -0400 Subject: [PATCH 3/5] show a tool tip for the various scorers! --- .../clone_scorer_modal_instance_controller.js | 6 ------ .../edit_scorer_modal_instance_controller.js | 4 ---- .../new_scorer_modal_instance_controller.js | 4 ---- .../components/scorer_form/scorer_form.html | 13 +++++++++++-- .../scorer_listing/scorer_listing.html | 2 ++ .../javascripts/factories/ScorerFactory.js | 1 + app/assets/javascripts/services/scorerSvc.js | 4 ++++ app/assets/templates/views/pick_scorer.html | 2 +- app/assets/templates/views/queriesLayout.html | 2 +- .../admin/communal_scorers_controller.rb | 4 +++- app/controllers/api/v1/scorers_controller.rb | 2 ++ .../admin/communal_scorers/_form.html.erb | 2 +- .../admin/communal_scorers/show.html.erb | 19 +++++++++++++++++++ .../api/v1/scorers/_scorer.json.jbuilder | 3 ++- 14 files changed, 47 insertions(+), 21 deletions(-) diff --git a/app/assets/javascripts/components/clone_scorer/clone_scorer_modal_instance_controller.js b/app/assets/javascripts/components/clone_scorer/clone_scorer_modal_instance_controller.js index 3eb6f96ce..2ce1d2f3b 100644 --- a/app/assets/javascripts/components/clone_scorer/clone_scorer_modal_instance_controller.js +++ b/app/assets/javascripts/components/clone_scorer/clone_scorer_modal_instance_controller.js @@ -22,12 +22,6 @@ angular.module('QuepidApp') scorerControllerActionsSvc.figureOutScaleChoice(ctrl); - - - ctrl.scorerOptions = { - showName: true - }; - ctrl.ok = function () { $uibModalInstance.close(ctrl.scorer); }; diff --git a/app/assets/javascripts/components/edit_scorer/edit_scorer_modal_instance_controller.js b/app/assets/javascripts/components/edit_scorer/edit_scorer_modal_instance_controller.js index 635788833..215c05ba9 100644 --- a/app/assets/javascripts/components/edit_scorer/edit_scorer_modal_instance_controller.js +++ b/app/assets/javascripts/components/edit_scorer/edit_scorer_modal_instance_controller.js @@ -20,10 +20,6 @@ angular.module('QuepidApp') scorerControllerActionsSvc.figureOutScaleChoice(ctrl); - ctrl.scorerOptions = { - showName: true - }; - ctrl.ok = function () { $uibModalInstance.close(ctrl.scorer); }; diff --git a/app/assets/javascripts/components/new_scorer/new_scorer_modal_instance_controller.js b/app/assets/javascripts/components/new_scorer/new_scorer_modal_instance_controller.js index e0bdac3df..fa7729d90 100644 --- a/app/assets/javascripts/components/new_scorer/new_scorer_modal_instance_controller.js +++ b/app/assets/javascripts/components/new_scorer/new_scorer_modal_instance_controller.js @@ -20,10 +20,6 @@ angular.module('QuepidApp') ctrl.scaleChoice = 'binaryScale'; ctrl.scorer = defaultScorer; - ctrl.scorerOptions = { - showName: true - }; - ctrl.ok = function () { $uibModalInstance.close(ctrl.scorer); }; diff --git a/app/assets/javascripts/components/scorer_form/scorer_form.html b/app/assets/javascripts/components/scorer_form/scorer_form.html index 2b1873e46..1a16937b7 100644 --- a/app/assets/javascripts/components/scorer_form/scorer_form.html +++ b/app/assets/javascripts/components/scorer_form/scorer_form.html @@ -1,4 +1,4 @@ -
+
@@ -17,8 +17,17 @@ ng-model="ctrl.scorer.code">
+
+ + +
+
+ + +
+
-
+
+
+ + Tooltip: + + +
+ <%= @scorer.tooltip %> +
+
+ +
+ + Description: + + +
+ <%= @scorer.description %> +
+
diff --git a/app/views/api/v1/scorers/_scorer.json.jbuilder b/app/views/api/v1/scorers/_scorer.json.jbuilder index 483d35128..49526c5e3 100644 --- a/app/views/api/v1/scorers/_scorer.json.jbuilder +++ b/app/views/api/v1/scorers/_scorer.json.jbuilder @@ -6,7 +6,8 @@ json.scorerId scorer.id json.communal scorer.communal json.code scorer.code json.name scorer.name -json.tooltip scorer.tooltip +json.tooltip scorer.tooltip +json.description scorer.description json.scale scorer.scale json.owner_id scorer.owner_id json.owned scorer.owner_id == current_user.id From 3387df3f21779289b50dad3216dfc29ba3b61474 Mon Sep 17 00:00:00 2001 From: "epugh@opensourceconnections.com" Date: Sat, 27 Aug 2022 10:06:37 -0400 Subject: [PATCH 4/5] plumbing to convey rollupMethod/rollup_method --- .../javascripts/components/scorer_form/scorer_form.html | 5 +++++ app/assets/javascripts/factories/ScorerFactory.js | 1 + app/assets/javascripts/services/scorerSvc.js | 2 ++ app/controllers/admin/communal_scorers_controller.rb | 1 + app/controllers/api/v1/scorers_controller.rb | 1 + app/views/admin/communal_scorers/show.html.erb | 9 +++++++++ app/views/api/v1/scorers/_communal_scorer.json.jbuilder | 1 + app/views/api/v1/scorers/_scorer.json.jbuilder | 1 + 8 files changed, 21 insertions(+) diff --git a/app/assets/javascripts/components/scorer_form/scorer_form.html b/app/assets/javascripts/components/scorer_form/scorer_form.html index 1a16937b7..14edd9c12 100644 --- a/app/assets/javascripts/components/scorer_form/scorer_form.html +++ b/app/assets/javascripts/components/scorer_form/scorer_form.html @@ -26,6 +26,11 @@
+
+ + +
+

+
+ + Rollup Method: + + +
+ <%= @scorer.rollup_method %> +
+
diff --git a/app/views/api/v1/scorers/_communal_scorer.json.jbuilder b/app/views/api/v1/scorers/_communal_scorer.json.jbuilder index 4b118875b..40ff1da25 100644 --- a/app/views/api/v1/scorers/_communal_scorer.json.jbuilder +++ b/app/views/api/v1/scorers/_communal_scorer.json.jbuilder @@ -5,6 +5,7 @@ json.communal scorer.communal json.code scorer.code json.name scorer.name json.tooltip scorer.tooltip +json.rollup_method scorer.rollup_method json.scale scorer.scale json.manualMaxScore scorer.manual_max_score json.manualMaxScoreValue scorer.manual_max_score_value diff --git a/app/views/api/v1/scorers/_scorer.json.jbuilder b/app/views/api/v1/scorers/_scorer.json.jbuilder index 49526c5e3..b2efeffca 100644 --- a/app/views/api/v1/scorers/_scorer.json.jbuilder +++ b/app/views/api/v1/scorers/_scorer.json.jbuilder @@ -9,6 +9,7 @@ json.name scorer.name json.tooltip scorer.tooltip json.description scorer.description json.scale scorer.scale +json.rollup_method scorer.rollup_method json.owner_id scorer.owner_id json.owned scorer.owner_id == current_user.id json.owner_name scorer.owner.name unless scorer.owner.nil? From caaf2b4a949d413b4b8fd9733ddc66c2f317d60f Mon Sep 17 00:00:00 2001 From: "epugh@opensourceconnections.com" Date: Sat, 27 Aug 2022 10:07:21 -0400 Subject: [PATCH 5/5] starting to figure out how to pass in avg versus sum... feels like this should be on the scorer.... not the queries..... --- .../components/qscore/qscore_controller.js | 2 +- .../javascripts/controllers/queriesCtrl.js | 59 ++++++++++++++++++- app/assets/templates/views/queriesLayout.html | 5 +- 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/app/assets/javascripts/components/qscore/qscore_controller.js b/app/assets/javascripts/components/qscore/qscore_controller.js index 7c29ad372..4cc6c0a8c 100644 --- a/app/assets/javascripts/components/qscore/qscore_controller.js +++ b/app/assets/javascripts/components/qscore/qscore_controller.js @@ -11,7 +11,7 @@ angular.module('QuepidApp') ctrl.diffScore = '?'; ctrl.diffStyle = {}; ctrl.score = '?'; - ctrl.scoreType = ctrl.scoreType || 'normal'; + ctrl.scoreType = ctrl.scoreType; // either header or query ctrl.style = { 'background-color': qscoreSvc.scoreToColor(ctrl.score, ctrl.maxScore) }; $scope.$watch('ctrl.scorable.currentScore', function() { diff --git a/app/assets/javascripts/controllers/queriesCtrl.js b/app/assets/javascripts/controllers/queriesCtrl.js index 9432b5aa2..75b22aa5c 100644 --- a/app/assets/javascripts/controllers/queriesCtrl.js +++ b/app/assets/javascripts/controllers/queriesCtrl.js @@ -102,6 +102,8 @@ angular.module('QuepidApp') $scope.getScorer = getScorer; + $scope.pickScorable = pickScorable; + $scope.reverse = $location.search().reverse; $scope.sortBy($location.search().sort || 'default', !$scope.reverse); @@ -167,11 +169,49 @@ angular.module('QuepidApp') } }; - // a simulated "query" that the results view uses for display var lastVersion = -1; + + // a simulated "query" that the results view uses for display + // This method is used by scorers whose rollupMethod is averaging var avgQuery = { lastScore: -1, calcScore: function() { + console.log("In avgQuery calcScore"); + // rescore only if + // - there are no unscored queries + // - we seem to have a new version of the query service + if (!queriesSvc.hasUnscoredQueries() && + (lastVersion !== queriesSvc.version())) { + + runScore(this); + saveScoring(); + } + }, + score: () => { + console.log("In avgQuery score"); + var deferred = $q.defer(); + deferred.resolve($scope.queries.avgQuery.diff.currentScore); + return deferred.promise; + }, + diff: { + score: function() { + console.log("In avgQuery diff score"); + return queriesSvc.scoreAllDiffs().then( (scoreInfo) => { + $scope.queries.avgQuery.diff.currentScore = scoreInfo; + return scoreInfo; + }); + } + } + //var diff: null, // TODO fill out + }; + + // a simulated "query" that the results view uses for display + // This method is used by scorers whose rollupMethod is sum + // note: this hasn't actually been converted to a SUM! + var sumQuery = { + lastScore: -1, + calcScore: function() { + console.log("In sumQuery calcScore"); // rescore only if // - there are no unscored queries // - we seem to have a new version of the query service @@ -183,12 +223,14 @@ angular.module('QuepidApp') } }, score: () => { + console.log("In sumQuery score"); var deferred = $q.defer(); deferred.resolve($scope.queries.avgQuery.diff.currentScore); return deferred.promise; }, diff: { score: function() { + console.log("In sumQuery diff score"); return queriesSvc.scoreAllDiffs().then( (scoreInfo) => { $scope.queries.avgQuery.diff.currentScore = scoreInfo; return scoreInfo; @@ -268,7 +310,9 @@ angular.module('QuepidApp') } } + // Do we need to define them here, or can, based on the type of scorer, we define it later??? $scope.queries.avgQuery = avgQuery; + $scope.queries.sumQuery = sumQuery; // get all the queries for this case for the query service $scope.queriesList = []; @@ -316,6 +360,19 @@ angular.module('QuepidApp') return scorerSvc.defaultScorer; } + function pickScorable() { + var scorable; + switch (getScorer().rollupMethod) { + case 'sum_of_scores': + scorable = $scope.queries.sumQuery; + break; + case 'average_of_scores': + scorable = $scope.queries.avgQuery; + break; + } + return scorable; + } + /*jslint latedef:false*/ function pickCaseScorer() { var modalInstance = $uibModal.open({ diff --git a/app/assets/templates/views/queriesLayout.html b/app/assets/templates/views/queriesLayout.html index 4e96f8ad1..924467bd4 100644 --- a/app/assets/templates/views/queriesLayout.html +++ b/app/assets/templates/views/queriesLayout.html @@ -4,12 +4,13 @@
+ {{ getScorer().rollupMethod }} — - +