From 8989cca25550a96fe41009d67a94e9eb68e362fe Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Mon, 18 Mar 2024 17:41:22 +0100 Subject: [PATCH 01/77] initi package (wip) for importer extension --- src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st | 5 +++++ src/GitLabProjectHealth-Model-Importer/GLPHApiFile.class.st | 5 +++++ .../GLPHModelImporter.class.st | 5 +++++ src/GitLabProjectHealth-Model-Importer/package.st | 1 + 4 files changed, 16 insertions(+) create mode 100644 src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st create mode 100644 src/GitLabProjectHealth-Model-Importer/GLPHApiFile.class.st create mode 100644 src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st create mode 100644 src/GitLabProjectHealth-Model-Importer/package.st diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st new file mode 100644 index 0000000..d70ac14 --- /dev/null +++ b/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st @@ -0,0 +1,5 @@ +Class { + #name : #GLPHApi, + #superclass : #GLHApi, + #category : #'GitLabProjectHealth-Model-Importer' +} diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHApiFile.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHApiFile.class.st new file mode 100644 index 0000000..977ea04 --- /dev/null +++ b/src/GitLabProjectHealth-Model-Importer/GLPHApiFile.class.st @@ -0,0 +1,5 @@ +Class { + #name : #GLPHApiFile, + #superclass : #GLHApiFile, + #category : #'GitLabProjectHealth-Model-Importer' +} diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st new file mode 100644 index 0000000..e6d70a1 --- /dev/null +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -0,0 +1,5 @@ +Class { + #name : #GLPHModelImporter, + #superclass : #GLHModelImporter, + #category : #'GitLabProjectHealth-Model-Importer' +} diff --git a/src/GitLabProjectHealth-Model-Importer/package.st b/src/GitLabProjectHealth-Model-Importer/package.st new file mode 100644 index 0000000..2420566 --- /dev/null +++ b/src/GitLabProjectHealth-Model-Importer/package.st @@ -0,0 +1 @@ +Package { #name : #'GitLabProjectHealth-Model-Importer' } From e3250bac685611a4d83885580c0864be21089038 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Mon, 18 Mar 2024 18:04:12 +0100 Subject: [PATCH 02/77] add merge_resquest first api + private token accessor --- .../GLPHApi.class.st | 26 +++++++++++++++++++ .../GLPHModelImporter.class.st | 6 +++++ 2 files changed, 32 insertions(+) diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st index d70ac14..0892534 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st @@ -3,3 +3,29 @@ Class { #superclass : #GLHApi, #category : #'GitLabProjectHealth-Model-Importer' } + +{ #category : #initialization } +GLPHApi >> initialize [ + + super initialize +] + +{ #category : #'as yet unclassified' } +GLPHApi >> mergeRequestOfProject: aProjectID [ + + ^ self client get: + self baseAPIUrl , '/projects/' , aProjectID printString + , '/merge_requests/' +] + +{ #category : #accessing } +GLPHApi >> privateToken [ + + ^ self client request headers at: #'PRIVATE-TOKEN' +] + +{ #category : #accessing } +GLPHApi >> privateToken: anObject [ + + self client headerAt: #'PRIVATE-TOKEN' add: anObject +] diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index e6d70a1..a549a9d 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -3,3 +3,9 @@ Class { #superclass : #GLHModelImporter, #category : #'GitLabProjectHealth-Model-Importer' } + +{ #category : #initialization } +GLPHModelImporter >> initialize [ + + super initialize. +] From 1414ac74e6c5d2a5847bfd8f550cb8196848b3cd Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Mon, 18 Mar 2024 18:27:02 +0100 Subject: [PATCH 03/77] start importer --- .../GLPHApi.class.st | 21 ++++++++++++++----- .../GLPHApiFile.class.st | 5 ----- .../GLPHModelImporter.class.st | 12 +++++++++++ 3 files changed, 28 insertions(+), 10 deletions(-) delete mode 100644 src/GitLabProjectHealth-Model-Importer/GLPHApiFile.class.st diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st index 0892534..3ba3576 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st @@ -4,6 +4,14 @@ Class { #category : #'GitLabProjectHealth-Model-Importer' } +{ #category : #'as yet unclassified' } +GLPHApi >> diffsMergeRequestOfProject: aProjectID withId: anMRiid [ + + ^ self client get: + self baseAPIUrl , '/projects/' , aProjectID printString + , '/merge_requests/' , anMRiid printString , '/diffs/' +] + { #category : #initialization } GLPHApi >> initialize [ @@ -18,14 +26,17 @@ GLPHApi >> mergeRequestOfProject: aProjectID [ , '/merge_requests/' ] -{ #category : #accessing } -GLPHApi >> privateToken [ +{ #category : #'as yet unclassified' } +GLPHApi >> mergeRequestOfProject: aProjectID withId: anMRiid [ - ^ self client request headers at: #'PRIVATE-TOKEN' + ^ self client get: + self baseAPIUrl , '/projects/' , aProjectID printString + , '/merge_requests/' , anMRiid printString ] { #category : #accessing } -GLPHApi >> privateToken: anObject [ +GLPHApi >> project: aProjectID [ - self client headerAt: #'PRIVATE-TOKEN' add: anObject + ^ self client get: + self baseAPIUrl , '/projects/' , aProjectID printString ] diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHApiFile.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHApiFile.class.st deleted file mode 100644 index 977ea04..0000000 --- a/src/GitLabProjectHealth-Model-Importer/GLPHApiFile.class.st +++ /dev/null @@ -1,5 +0,0 @@ -Class { - #name : #GLPHApiFile, - #superclass : #GLHApiFile, - #category : #'GitLabProjectHealth-Model-Importer' -} diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index a549a9d..29622fa 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -4,8 +4,20 @@ Class { #category : #'GitLabProjectHealth-Model-Importer' } +{ #category : #'private - api' } +GLPHModelImporter >> completeImportProject: aGLHProject [ + + super completeImportProject: aGLHProject. + self importMergeRequests: aGLHProject +] + +{ #category : #'as yet unclassified' } +GLPHModelImporter >> importProject: aProjectID [ +] + { #category : #initialization } GLPHModelImporter >> initialize [ super initialize. + withCommitDiffs := true ] From f7207e2a742010b1012c1b89099180b9fad530d6 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Wed, 20 Mar 2024 16:24:14 +0100 Subject: [PATCH 04/77] add merge request parser --- .../GLHProject.extension.st | 19 ++++ .../GLPHEMergeRequest.class.st | 29 ++++++ .../GLHModelImporter.class.st | 10 +- .../GLPHMetamodelGenerator.class.st | 10 +- .../GLPHModelImporter.class.st | 98 +++++++++++++++++++ 5 files changed, 159 insertions(+), 7 deletions(-) create mode 100644 src/GLPHExtended-Model/GLHProject.extension.st diff --git a/src/GLPHExtended-Model/GLHProject.extension.st b/src/GLPHExtended-Model/GLHProject.extension.st new file mode 100644 index 0000000..17b1368 --- /dev/null +++ b/src/GLPHExtended-Model/GLHProject.extension.st @@ -0,0 +1,19 @@ +Extension { #name : #GLHProject } + +{ #category : #'*GLPHExtended-Model-accessing' } +GLHProject >> mergeRequests [ + "Relation named: #mergeRequests type: #GLPHEMergeRequest opposite: #project" + + + + + + ^ self attributeAt: #mergeRequests ifAbsentPut: [ FMMultivalueLink on: self opposite: #project: ] +] + +{ #category : #'*GLPHExtended-Model-accessing' } +GLHProject >> mergeRequests: anObject [ + + + self mergeRequests value: anObject +] diff --git a/src/GLPHExtended-Model/GLPHEMergeRequest.class.st b/src/GLPHExtended-Model/GLPHEMergeRequest.class.st index fc069d0..d46fce9 100644 --- a/src/GLPHExtended-Model/GLPHEMergeRequest.class.st +++ b/src/GLPHExtended-Model/GLPHEMergeRequest.class.st @@ -4,6 +4,11 @@ a gitlab merge request ## Relations ====================== +### Parents +| Relation | Origin | Opposite | Type | Comment | +|---| +| `project` | `GLPHEMergeRequest` | `mergeRequests` | `GLHProject` | | + ### Other | Relation | Origin | Opposite | Type | Comment | |---| @@ -630,6 +635,30 @@ GLPHEMergeRequest >> pipeline: anObject [ anObject mergeResquest: self ] ] +{ #category : #accessing } +GLPHEMergeRequest >> project [ + "Relation named: #project type: #GLHProject opposite: #mergeRequests" + + + + + ^ self attributeAt: #project ifAbsent: [ nil ] +] + +{ #category : #accessing } +GLPHEMergeRequest >> project: anObject [ + + + self attributeAt: #project put: (FMMultivalueLink on: self update: #mergeRequests from: self project to: anObject). +] + +{ #category : #navigation } +GLPHEMergeRequest >> projectGroup [ + + + ^ MooseSpecializedGroup with: self project +] + { #category : #accessing } GLPHEMergeRequest >> project_id [ diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index ce20ac8..a0d13cf 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -206,12 +206,13 @@ GLHModelImporter >> importGroup: aGroupID [ | result groupResult | ('Import group: ' , aGroupID printString) recordInfo. + result := self glhApi group: aGroupID. groupResult := self parseGroupResult: result. - groupResult projects do: [ :project | + groupResult projects do: [ :project | self completeImportProject: project ]. self addGroupResultToModel: groupResult. - (self subGroupsOf: aGroupID) do: [ :subGroup | + (self subGroupsOf: aGroupID) do: [ :subGroup | groupResult addSubGroup: (self importGroup: subGroup id) ]. ^ groupResult ] @@ -326,14 +327,15 @@ GLHModelImporter >> parseFileTreeResult: aResult [ GLHModelImporter >> parseGroupResult: aResult [ | reader | + reader := NeoJSONReader on: aResult readStream. - reader for: GLHGroup do: [ :mapping | + reader for: GLHGroup do: [ :mapping | mapping mapInstVars. (mapping mapInstVar: #projects) valueSchema: #ArrayOfProjects ]. reader mapInstVarsFor: GLHProject. reader for: #ArrayOfProjects - customDo: [ :customMappting | + customDo: [ :customMappting | customMappting listOfElementSchema: GLHProject ]. ^ reader nextAs: GLHGroup ] diff --git a/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st b/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st index 2593761..4bf5cbb 100644 --- a/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st +++ b/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st @@ -11,7 +11,8 @@ Class { 'diff', 'user', 'pipeline', - 'commit' + 'commit', + 'project' ], #category : #'GitLabProjectHealth-ExtendModel-Generator' } @@ -76,7 +77,8 @@ GLPHMetamodelGenerator >> defineClasses [ diff := self remoteEntity: #Diff withPrefix: #GLH. user := self remoteEntity: #User withPrefix: #GLH. pipeline := self remoteEntity: #Pipeline withPrefix: #GLH. - commit := self remoteEntity: #Commit withPrefix: #GLH + commit := self remoteEntity: #Commit withPrefix: #GLH. + project := self remoteEntity: #Project withPrefix: #GLH ] { #category : #definition } @@ -106,7 +108,7 @@ GLPHMetamodelGenerator >> defineRelations [ "MERGE REQUEST RELATIONS" (mergeRequest property: #author) *- (user property: #createdMergeRequests). - + (mergeRequest property: #merge_user) *- (user property: #usedMerges). (mergeRequest property: #closed_by) @@ -135,6 +137,8 @@ GLPHMetamodelGenerator >> defineRelations [ (mergeRequest property: #diff) - (diff property: #mergeRequest). + (project property: #mergeRequests) <>-*(mergeRequest property: #project). + "DIFF" (diff property: #changes) <>-* (change property: #diff) ] diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 29622fa..4a8f577 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -11,6 +11,20 @@ GLPHModelImporter >> completeImportProject: aGLHProject [ self importMergeRequests: aGLHProject ] +{ #category : #import } +GLPHModelImporter >> importMergeRequests: aGLHProject [ + + | results parsedResults | + results := self glhApi mergeRequestOfProject: aGLHProject id . + parsedResults := self parseMergeRequestResult: results. + 1 halt. + aGLHProject mergeRequests addAll: parsedResults. + + self withCommitDiffs ifTrue: [ + aGLHProject repository commits do: [ :commit | + self importDiffOfCommit: commit ] ] +] + { #category : #'as yet unclassified' } GLPHModelImporter >> importProject: aProjectID [ ] @@ -21,3 +35,87 @@ GLPHModelImporter >> initialize [ super initialize. withCommitDiffs := true ] + +{ #category : #parsing } +GLPHModelImporter >> parseMergeRequestResult: result [ + | reader | + reader := NeoJSONReader on: result readStream. + + "declare quil y a un array a mapper" + reader + for: #ArrayOfMergeRequest + customDo: [ :customMappting | + customMappting listOfElementSchema: GLPHEMergeRequest ]. + + "declare la liste des pro" + reader for: GLPHEMergeRequest do: [ :mapping | + + mapping mapInstVars: + #( blocking_discussions_resolved +changes_count +description +detailed_merge_status +discussion_locked +downvotes +draft +first_deployed_to_production_at +force_remove_source_branch +has_conflicts +id +iid +labels +latest_build_finished_at +latest_build_started_at +merge_commit_sha +merge_status +merge_when_pipeline_succeeds +merged_at +milestone +project_id +reference +references_full +references_relative +references_short +sha +should_remove_source_branch +source_branch +source_project_id +squash +squash_commit_sha +squash_on_merge +state +subscribed +target_branch +target_project_id +task_completion_status_completed_count +task_completion_status_count +time_stats_human_time_estimate +time_stats_human_total_time_spent +time_stats_time_estimate +time_stats_total_time_spent +title +updated_at +upvotes +user_notes_count +web_url +work_in_progress). +" (mapping mapInstVar: #closed_at) valueSchema: DateAndTime. + (mapping mapInstVar: #created_at) valueSchema: DateAndTime. + (mapping mapInstVar: #assignee) valueSchema: GLHUser. + (mapping mapInstVar: #author) valueSchema: GLHUser. + (mapping mapInstVar: #closed_by) valueSchema: GLHUser. + (mapping mapInstVar: #mergeCommit) valueSchema: GLHCommit." + + " mapping + mapProperty: 'stats' + getter: [ :el | ""Not used"" ] + setter: [ :commit :value | + commit deletions: (value at: #deletions). + commit additions: (value at: #additions) ] +" +]. +" reader for: DateAndTime customDo: [ :mapping | + mapping decoder: [ :string | DateAndTime fromString: string ] ]." + + ^ reader nextAs: #ArrayOfMergeRequest +] From 87ad490a62100a0f39a95197f76fa4cd0f373a45 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Wed, 20 Mar 2024 16:25:06 +0100 Subject: [PATCH 05/77] remove halt --- .../GLPHModelImporter.class.st | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 4a8f577..7875339 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -17,7 +17,7 @@ GLPHModelImporter >> importMergeRequests: aGLHProject [ | results parsedResults | results := self glhApi mergeRequestOfProject: aGLHProject id . parsedResults := self parseMergeRequestResult: results. - 1 halt. + aGLHProject mergeRequests addAll: parsedResults. self withCommitDiffs ifTrue: [ From ba5e104b580a1105e2ee7d6a12dbbfd02bdb4129 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Wed, 3 Apr 2024 10:22:09 +0200 Subject: [PATCH 06/77] importing changes for diffs --- src/GLPHExtended-Model/GLHDiff.extension.st | 14 +--- .../GLPHEMergeRequest.class.st | 28 +++---- .../GLHModelImporter.class.st | 6 +- .../GLPHMetamodelGenerator.class.st | 7 +- .../GLPHModelImporter.class.st | 79 ++++++++++++++++++- 5 files changed, 99 insertions(+), 35 deletions(-) diff --git a/src/GLPHExtended-Model/GLHDiff.extension.st b/src/GLPHExtended-Model/GLHDiff.extension.st index 6d58216..269360f 100644 --- a/src/GLPHExtended-Model/GLHDiff.extension.st +++ b/src/GLPHExtended-Model/GLHDiff.extension.st @@ -56,11 +56,10 @@ GLHDiff >> deletion: anObject [ { #category : #'*GLPHExtended-Model-accessing' } GLHDiff >> mergeRequest [ - "Relation named: #mergeRequest type: #GLPHEMergeRequest opposite: #diff" + "Relation named: #mergeRequest type: #GLPHEMergeRequest opposite: #diffs" - - + ^ self attributeAt: #mergeRequest ifAbsent: [ nil ] ] @@ -69,12 +68,5 @@ GLHDiff >> mergeRequest [ GLHDiff >> mergeRequest: anObject [ - (self attributeAt: #mergeRequest ifAbsentPut: [nil]) == anObject ifTrue: [ ^ anObject ]. - anObject ifNil: [ | otherSide | - otherSide := self mergeRequest. - self attributeAt: #mergeRequest put: anObject. - otherSide diff: nil ] - ifNotNil: [ - self attributeAt: #mergeRequest put: anObject. - anObject diff: self ] + self attributeAt: #mergeRequest put: (FMMultivalueLink on: self update: #diffs from: self mergeRequest to: anObject). ] diff --git a/src/GLPHExtended-Model/GLPHEMergeRequest.class.st b/src/GLPHExtended-Model/GLPHEMergeRequest.class.st index d46fce9..2ea40f4 100644 --- a/src/GLPHExtended-Model/GLPHEMergeRequest.class.st +++ b/src/GLPHExtended-Model/GLPHEMergeRequest.class.st @@ -16,7 +16,7 @@ a gitlab merge request | `assignees` | `GLPHEMergeRequest` | `assignedMergeRequests` | `GLHUser` | | | `author` | `GLPHEMergeRequest` | `createdMergeRequests` | `GLHUser` | | | `closed_by` | `GLPHEMergeRequest` | `closedMergeRequests` | `GLHUser` | | -| `diff` | `GLPHEMergeRequest` | `mergeRequest` | `GLHDiff` | | +| `diffs` | `GLPHEMergeRequest` | `mergeRequest` | `GLHDiff` | | | `mergeCommit` | `GLPHEMergeRequest` | `commitedMergeRequest` | `GLHCommit` | | | `merge_user` | `GLPHEMergeRequest` | `usedMerges` | `GLHUser` | | | `merged_by` | `GLPHEMergeRequest` | `mergerOfMergeRequests` | `GLHUser` | | @@ -155,6 +155,12 @@ GLPHEMergeRequest >> addAssignee: anObject [ ^ self assignees add: anObject ] +{ #category : #adding } +GLPHEMergeRequest >> addDiff: anObject [ + + ^ self diffs add: anObject +] + { #category : #adding } GLPHEMergeRequest >> addReviewer: anObject [ @@ -311,26 +317,20 @@ GLPHEMergeRequest >> detailed_merge_status: anObject [ ] { #category : #accessing } -GLPHEMergeRequest >> diff [ - "Relation named: #diff type: #GLHDiff opposite: #mergeRequest" +GLPHEMergeRequest >> diffs [ + "Relation named: #diffs type: #GLHDiff opposite: #mergeRequest" - - ^ self attributeAt: #diff ifAbsent: [ nil ] + + + ^ self attributeAt: #diffs ifAbsentPut: [ FMMultivalueLink on: self opposite: #mergeRequest: ] ] { #category : #accessing } -GLPHEMergeRequest >> diff: anObject [ +GLPHEMergeRequest >> diffs: anObject [ - (self attributeAt: #diff ifAbsentPut: [nil]) == anObject ifTrue: [ ^ anObject ]. - anObject ifNil: [ | otherSide | - otherSide := self diff. - self attributeAt: #diff put: anObject. - otherSide mergeRequest: nil ] - ifNotNil: [ - self attributeAt: #diff put: anObject. - anObject mergeRequest: self ] + self diffs value: anObject ] { #category : #accessing } diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index a0d13cf..7c22401 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -36,6 +36,7 @@ GLHModelImporter >> completeImportProject: aGLHProject [ aGLHProject addPipeline: pipeline ]. aGLHProject creator: (self importUser: aGLHProject creator_id). aGLHProject repository: GLHRepository new. + self glhModel add: aGLHProject . self glhModel add: aGLHProject repository. self importRepository: aGLHProject repository. self importCommits: aGLHProject @@ -146,11 +147,10 @@ GLHModelImporter >> importDiffOfCommit: aCommit [ | result diffsResult | aCommit diffs ifNotEmpty: [ - 'Diff of already importer: ' + 'Diff already importer: ' , aCommit short_id printString recordInfo. ^ aCommit diffs ]. - ('Import diff of commit: ' , aCommit short_id printString) - recordInfo. + ('Import diff of commit: ' , aCommit short_id printString) recordInfo. result := self glhApi commitDiff: aCommit id ofProject: aCommit repository project id diff --git a/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st b/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st index 4bf5cbb..9755404 100644 --- a/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st +++ b/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st @@ -135,10 +135,11 @@ GLPHMetamodelGenerator >> defineRelations [ (mergeRequest property: #squashCommit) - (commit property: #squashedMergeRequest). - (mergeRequest property: #diff) - (diff property: #mergeRequest). + (mergeRequest property: #diffs) -* (diff property: #mergeRequest). + + (project property: #mergeRequests) + <>-* (mergeRequest property: #project). - (project property: #mergeRequests) <>-*(mergeRequest property: #project). - "DIFF" (diff property: #changes) <>-* (change property: #diff) ] diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 7875339..32ce46d 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -8,21 +8,62 @@ Class { GLPHModelImporter >> completeImportProject: aGLHProject [ super completeImportProject: aGLHProject. + self importMergeRequests: aGLHProject + +] + +{ #category : #import } +GLPHModelImporter >> importChangesForDiff: aGLHDiff [ + |result| + ('Import changes of diff ' , aGLHDiff printString) recordInfo. + result := self parseDiffString: aGLHDiff. + result do: [ :change | + aGLHDiff changes add: change ]. + aGLHDiff mooseModel addAll: aGLHDiff changes. + + ^ aGLHDiff changes. + +] + +{ #category : #'as yet unclassified' } +GLPHModelImporter >> importDiffOfMergeRequest: aMergeRequest [ + + | result diffsResult | + + aMergeRequest diffs ifNotEmpty: [ + 'Diff of already importer: ' + , aMergeRequest iid printString recordInfo. + ^ aMergeRequest diffs ]. + ('Import diff of mergeRequest ' , aMergeRequest iid printString) recordInfo. + result := self glhApi + diffsMergeRequestOfProject: (aMergeRequest project_id) + withId: aMergeRequest iid. + + diffsResult := self parseDiffResult: result. + aMergeRequest diffs addAll: diffsResult. + aMergeRequest mooseModel addAll: aMergeRequest diffs. + + aMergeRequest diffs do: [ :diff | + self importChangesForDiff: diff. + ]. + + ^ diffsResult ] { #category : #import } GLPHModelImporter >> importMergeRequests: aGLHProject [ | results parsedResults | - results := self glhApi mergeRequestOfProject: aGLHProject id . + results := self glhApi mergeRequestOfProject: aGLHProject id. parsedResults := self parseMergeRequestResult: results. aGLHProject mergeRequests addAll: parsedResults. - + aGLHProject mooseModel addAll: aGLHProject mergeRequests. + self withCommitDiffs ifTrue: [ - aGLHProject repository commits do: [ :commit | - self importDiffOfCommit: commit ] ] + aGLHProject mergeRequests do: [ :mr | + self importDiffOfMergeRequest: mr ] ] ] { #category : #'as yet unclassified' } @@ -36,6 +77,36 @@ GLPHModelImporter >> initialize [ withCommitDiffs := true ] +{ #category : #parsing } +GLPHModelImporter >> parseDiffString: aDiff [ + + | lines ranges changes lineRangesInfo addedLines deletedLines | + lines := aDiff diffString lines. + ranges := lines select: [ :line | line beginsWith: #'@@' ]. + + changes := (lines collect: [ :line | + | aChange range | + aChange := nil. + + (line beginsWith: #'@@') + ifTrue: [ lineRangesInfo := line splitOn: ' ' ] + ifFalse: [ + (line trim beginsWith: #+) ifTrue: [ + aChange := GLPHEAddition new. + aChange linesOfCode: line. + aChange originalLineRange: (lineRangesInfo at: 2). + aChange newLineRange: (lineRangesInfo at: 3) ]. + (line trim beginsWith: #-) ifTrue: [ + aChange := GLPHEDeletion new. + aChange linesOfCode: line. + aChange originalLineRange: (lineRangesInfo at: 2). + aChange newLineRange: (lineRangesInfo at: 3) ] ]. + + aChange ]) reject: #isNil. +1 halt. + ^ changes +] + { #category : #parsing } GLPHModelImporter >> parseMergeRequestResult: result [ | reader | From 87efb28fdb7f150b43e3e8802b98cd7a00212919 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Wed, 3 Apr 2024 15:15:13 +0200 Subject: [PATCH 07/77] rework model for changes and ranges of diff strings --- .../GLPHEChange.extension.st | 35 +++++ .../GLPHEDiffRange.extension.st | 13 ++ .../GLPHELineOfCode.extension.st | 22 +++ src/GLPHExtended-Model-Extension/package.st | 1 + src/GLPHExtended-Model/GLHDiff.extension.st | 18 +++ src/GLPHExtended-Model/GLPHEChange.class.st | 66 ++++----- .../GLPHEDiffRange.class.st | 135 ++++++++++++++++++ .../GLPHELineOfCode.class.st | 19 +++ .../GLPHETEntityCreator.trait.st | 14 ++ .../GLPHMetamodelGenerator.class.st | 53 ++++--- .../GLPHModelImporter.class.st | 48 +++---- 11 files changed, 342 insertions(+), 82 deletions(-) create mode 100644 src/GLPHExtended-Model-Extension/GLPHEChange.extension.st create mode 100644 src/GLPHExtended-Model-Extension/GLPHEDiffRange.extension.st create mode 100644 src/GLPHExtended-Model-Extension/GLPHELineOfCode.extension.st create mode 100644 src/GLPHExtended-Model-Extension/package.st create mode 100644 src/GLPHExtended-Model/GLPHEDiffRange.class.st create mode 100644 src/GLPHExtended-Model/GLPHELineOfCode.class.st diff --git a/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st b/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st new file mode 100644 index 0000000..61fcbf3 --- /dev/null +++ b/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st @@ -0,0 +1,35 @@ +Extension { #name : #GLPHEChange } + +{ #category : #'*GLPHExtended-Model-Extension' } +GLPHEChange class >> createFrom: aDiffLine [ + "Factory a Change from a loc" + |aChange| + aChange := nil. + (aDiffLine beginsWith: #'@@') ifTrue: [ + |infos| + aChange := GLPHELineOfCode newFromLoCRange: aDiffLine. + 1 halt. + infos := (aDiffLine splitOn: '@@') copyWithoutFirst. + infos := aDiffLine splitOn: ' ' . + ]. + ^ aChange. +] + +{ #category : #'*GLPHExtended-Model-Extension' } +GLPHEChange class >> newFrom: aDiffLine [ + "Factory a Change from a loc" + + | aChange code| + aChange := GLPHELineOfCode new. + + code := aDiffLine. + (aDiffLine beginsWith: #'@@') ifTrue: [ + code := (aDiffLine splitOn: '@@') copyWithoutFirst second. + ]. + (aDiffLine trim beginsWith: #+) ifTrue: [ aChange := GLPHEAddition new. ]. + (aDiffLine trim beginsWith: #-) ifTrue: [ aChange := GLPHEDeletion new. ]. + + aChange sourceCode: code. + + ^ aChange +] diff --git a/src/GLPHExtended-Model-Extension/GLPHEDiffRange.extension.st b/src/GLPHExtended-Model-Extension/GLPHEDiffRange.extension.st new file mode 100644 index 0000000..40ee7e9 --- /dev/null +++ b/src/GLPHExtended-Model-Extension/GLPHEDiffRange.extension.st @@ -0,0 +1,13 @@ +Extension { #name : #GLPHEDiffRange } + +{ #category : #'*GLPHExtended-Model-Extension' } +GLPHEDiffRange class >> newFrom: aLine [ + |range infos rangesInfo | + range := GLPHEDiffRange new. + infos := (aLine splitOn: '@@') copyWithoutFirst. + rangesInfo := infos first trim splitOn: ' '. + range originalLineRange: rangesInfo first. + range newLineRange: rangesInfo second . + +^ range. +] diff --git a/src/GLPHExtended-Model-Extension/GLPHELineOfCode.extension.st b/src/GLPHExtended-Model-Extension/GLPHELineOfCode.extension.st new file mode 100644 index 0000000..bb0aaa3 --- /dev/null +++ b/src/GLPHExtended-Model-Extension/GLPHELineOfCode.extension.st @@ -0,0 +1,22 @@ +Extension { #name : #GLPHELineOfCode } + +{ #category : #'*GLPHExtended-Model-Extension' } +GLPHELineOfCode class >> newFrom: aDiffLine [ + + | aLoC infos ranges | + infos := (aDiffLine splitOn: '@@') copyWithoutFirst. + aLoC := GLPHELineOfCode new. + aLoC sourceCode: infos second. + ^ aLoC. +] + +{ #category : #'*GLPHExtended-Model-Extension' } +GLPHELineOfCode class >> newFromLoCRange: aDiffLine [ + |aLoC infos ranges| + infos := (aDiffLine splitOn: '@@') copyWithoutFirst. + ranges := infos first splitOn: ','. + aLoC := GLPHELineOfCode new. + aLoC originalLineRange: ranges first. + aLoC newLineRange: 1. + aLoC lineOfCode: infos second. +] diff --git a/src/GLPHExtended-Model-Extension/package.st b/src/GLPHExtended-Model-Extension/package.st new file mode 100644 index 0000000..b6d34a8 --- /dev/null +++ b/src/GLPHExtended-Model-Extension/package.st @@ -0,0 +1 @@ +Package { #name : #'GLPHExtended-Model-Extension' } diff --git a/src/GLPHExtended-Model/GLHDiff.extension.st b/src/GLPHExtended-Model/GLHDiff.extension.st index 269360f..ac49683 100644 --- a/src/GLPHExtended-Model/GLHDiff.extension.st +++ b/src/GLPHExtended-Model/GLHDiff.extension.st @@ -54,6 +54,24 @@ GLHDiff >> deletion: anObject [ self deletion value: anObject ] +{ #category : #'*GLPHExtended-Model-accessing' } +GLHDiff >> diffRanges [ + "Relation named: #diffRanges type: #GLPHEDiffRange opposite: #diff" + + + + + + ^ self attributeAt: #diffRanges ifAbsentPut: [ FMMultivalueLink on: self opposite: #diff: ] +] + +{ #category : #'*GLPHExtended-Model-accessing' } +GLHDiff >> diffRanges: anObject [ + + + self diffRanges value: anObject +] + { #category : #'*GLPHExtended-Model-accessing' } GLHDiff >> mergeRequest [ "Relation named: #mergeRequest type: #GLPHEMergeRequest opposite: #diffs" diff --git a/src/GLPHExtended-Model/GLPHEChange.class.st b/src/GLPHExtended-Model/GLPHEChange.class.st index 21a79f3..df2bd7f 100644 --- a/src/GLPHExtended-Model/GLPHEChange.class.st +++ b/src/GLPHExtended-Model/GLPHEChange.class.st @@ -4,10 +4,10 @@ super class for addition and deletion ## Relations ====================== -### Parents +### Other | Relation | Origin | Opposite | Type | Comment | |---| -| `diff` | `GLPHEChange` | `changes` | `GLHDiff` | | +| `diffRange` | `GLPHEChange` | `changes` | `GLPHEDiffRange` | | ## Properties @@ -15,18 +15,19 @@ super class for addition and deletion | Name | Type | Default value | Comment | |---| -| `linesOfCode` | `String` | nil | | -| `newLineRange` | `String` | nil | | -| `originalLineRange` | `String` | nil | | +| `index` | `Number` | nil | | +| `relativeIndex` | `Number` | nil | | +| `sourceCode` | `String` | nil | | " Class { #name : #GLPHEChange, #superclass : #GLPHEEntity, #instVars : [ - '#linesOfCode => FMProperty', - '#newLineRange => FMProperty', - '#originalLineRange => FMProperty' + '#diffRange => FMOne type: #GLPHEDiffRange opposite: #changes', + '#index => FMProperty', + '#relativeIndex => FMProperty', + '#sourceCode => FMProperty' ], #category : #'GLPHExtended-Model-Entities' } @@ -41,67 +42,58 @@ GLPHEChange class >> annotation [ ] { #category : #accessing } -GLPHEChange >> diff [ - "Relation named: #diff type: #GLHDiff opposite: #changes" +GLPHEChange >> diffRange [ + "Relation named: #diffRange type: #GLPHEDiffRange opposite: #changes" - - - ^ self attributeAt: #diff ifAbsent: [ nil ] + ^ diffRange ] { #category : #accessing } -GLPHEChange >> diff: anObject [ +GLPHEChange >> diffRange: anObject [ - self attributeAt: #diff put: (FMMultivalueLink on: self update: #changes from: self diff to: anObject). -] - -{ #category : #navigation } -GLPHEChange >> diffGroup [ - - - ^ MooseSpecializedGroup with: self diff + diffRange := anObject ] { #category : #accessing } -GLPHEChange >> linesOfCode [ +GLPHEChange >> index [ - + - ^ linesOfCode + ^ index ] { #category : #accessing } -GLPHEChange >> linesOfCode: anObject [ +GLPHEChange >> index: anObject [ - linesOfCode := anObject + index := anObject ] { #category : #accessing } -GLPHEChange >> newLineRange [ +GLPHEChange >> relativeIndex [ - + - ^ newLineRange + ^ relativeIndex ] { #category : #accessing } -GLPHEChange >> newLineRange: anObject [ +GLPHEChange >> relativeIndex: anObject [ - newLineRange := anObject + relativeIndex := anObject ] { #category : #accessing } -GLPHEChange >> originalLineRange [ +GLPHEChange >> sourceCode [ - + - ^ originalLineRange + ^ sourceCode ] { #category : #accessing } -GLPHEChange >> originalLineRange: anObject [ +GLPHEChange >> sourceCode: anObject [ - originalLineRange := anObject + sourceCode := anObject ] diff --git a/src/GLPHExtended-Model/GLPHEDiffRange.class.st b/src/GLPHExtended-Model/GLPHEDiffRange.class.st new file mode 100644 index 0000000..c5db1ed --- /dev/null +++ b/src/GLPHExtended-Model/GLPHEDiffRange.class.st @@ -0,0 +1,135 @@ +" +a indicate on which line of code are concerned by a diff (usually declared with @@ oldRange, newRange @@) + +## Relations +====================== + +### Parents +| Relation | Origin | Opposite | Type | Comment | +|---| +| `diff` | `GLPHEDiffRange` | `diffRanges` | `GLHDiff` | | + +### Other +| Relation | Origin | Opposite | Type | Comment | +|---| +| `changes` | `GLPHEDiffRange` | `diffRange` | `GLPHEChange` | | + + +## Properties +====================== + +| Name | Type | Default value | Comment | +|---| +| `lineOfCode` | `String` | nil | | +| `newLineRange` | `String` | nil | | +| `originalLineRange` | `String` | nil | | + +" +Class { + #name : #GLPHEDiffRange, + #superclass : #GLPHEEntity, + #instVars : [ + '#originalLineRange => FMProperty', + '#newLineRange => FMProperty', + '#lineOfCode => FMProperty', + '#changes => FMMany type: #GLPHEChange opposite: #diffRange' + ], + #category : #'GLPHExtended-Model-Entities' +} + +{ #category : #meta } +GLPHEDiffRange class >> annotation [ + + + + + ^ self +] + +{ #category : #adding } +GLPHEDiffRange >> addChange: anObject [ + + ^ self changes add: anObject +] + +{ #category : #accessing } +GLPHEDiffRange >> changes [ + "Relation named: #changes type: #GLPHEChange opposite: #diffRange" + + + + ^ changes +] + +{ #category : #accessing } +GLPHEDiffRange >> changes: anObject [ + + + changes value: anObject +] + +{ #category : #accessing } +GLPHEDiffRange >> diff [ + "Relation named: #diff type: #GLHDiff opposite: #diffRanges" + + + + + ^ self attributeAt: #diff ifAbsent: [ nil ] +] + +{ #category : #accessing } +GLPHEDiffRange >> diff: anObject [ + + + self attributeAt: #diff put: (FMMultivalueLink on: self update: #diffRanges from: self diff to: anObject). +] + +{ #category : #navigation } +GLPHEDiffRange >> diffGroup [ + + + ^ MooseSpecializedGroup with: self diff +] + +{ #category : #accessing } +GLPHEDiffRange >> lineOfCode [ + + + + ^ lineOfCode +] + +{ #category : #accessing } +GLPHEDiffRange >> lineOfCode: anObject [ + + lineOfCode := anObject +] + +{ #category : #accessing } +GLPHEDiffRange >> newLineRange [ + + + + ^ newLineRange +] + +{ #category : #accessing } +GLPHEDiffRange >> newLineRange: anObject [ + + newLineRange := anObject +] + +{ #category : #accessing } +GLPHEDiffRange >> originalLineRange [ + + + + ^ originalLineRange +] + +{ #category : #accessing } +GLPHEDiffRange >> originalLineRange: anObject [ + + originalLineRange := anObject +] diff --git a/src/GLPHExtended-Model/GLPHELineOfCode.class.st b/src/GLPHExtended-Model/GLPHELineOfCode.class.st new file mode 100644 index 0000000..72bd3ff --- /dev/null +++ b/src/GLPHExtended-Model/GLPHELineOfCode.class.st @@ -0,0 +1,19 @@ +" +a line of code in a diff comparaison + + +" +Class { + #name : #GLPHELineOfCode, + #superclass : #GLPHEChange, + #category : #'GLPHExtended-Model-Entities' +} + +{ #category : #meta } +GLPHELineOfCode class >> annotation [ + + + + + ^ self +] diff --git a/src/GLPHExtended-Model/GLPHETEntityCreator.trait.st b/src/GLPHExtended-Model/GLPHETEntityCreator.trait.st index 6196e96..e47e98b 100644 --- a/src/GLPHExtended-Model/GLPHETEntityCreator.trait.st +++ b/src/GLPHExtended-Model/GLPHETEntityCreator.trait.st @@ -39,6 +39,20 @@ GLPHETEntityCreator >> newDeletion [ ^ self add: GLPHEDeletion new ] +{ #category : #'entity creation' } +GLPHETEntityCreator >> newDiffRange [ + + + ^ self add: GLPHEDiffRange new +] + +{ #category : #'entity creation' } +GLPHETEntityCreator >> newLineOfCode [ + + + ^ self add: GLPHELineOfCode new +] + { #category : #'entity creation' } GLPHETEntityCreator >> newMergeRequest [ diff --git a/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st b/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st index 9755404..99395c8 100644 --- a/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st +++ b/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st @@ -12,7 +12,9 @@ Class { 'user', 'pipeline', 'commit', - 'project' + 'project', + 'lineOfCode', + 'diffRange' ], #category : #'GitLabProjectHealth-ExtendModel-Generator' } @@ -34,19 +36,11 @@ GLPHMetamodelGenerator class >> submetamodels [ { #category : #'as yet unclassified' } GLPHMetamodelGenerator >> changeProperties [ - "must parse the range. - originalLineRange = '-120,5' --> change starts at line #120 - 5 is the number of lines in the original file affected by the change" - - change property: #originalLineRange type: #String. - change property: #newLineRange type: #String. - change property: #linesOfCode type: #String. - - addition - property: #delimiter - type: #String - defaultValue: '#+'. + change property: #index type: #Number. + change property: #relativeIndex type: #Number. + change property: #sourceCode type: #String. + addition property: #delimiter type: #String defaultValue: '#+'. deletion property: #delimiter type: #String defaultValue: '#-' ] @@ -58,7 +52,6 @@ GLPHMetamodelGenerator >> defineClasses [ newClassNamed: #MergeRequest comment: 'a gitlab merge request'. - change := builder newClassNamed: #Change comment: 'super class for addition and deletion '. @@ -72,6 +65,17 @@ GLPHMetamodelGenerator >> defineClasses [ comment: 'a detected deletion of a line, seen within a diff '. + lineOfCode := builder + newClassNamed: #LineOfCode + comment: 'a line of code in a diff comparaison '. + + diffRange := builder + newClassNamed: #DiffRange + comment: + 'a indicate on which line of code are concerned by a diff (usually declared with @@ oldRange, newRange @@) '. + + + repository := self remoteEntity: #Repository withPrefix: #GLH. file := self remoteEntity: #File withPrefix: #GLH. diff := self remoteEntity: #Diff withPrefix: #GLH. @@ -87,17 +91,18 @@ GLPHMetamodelGenerator >> defineHierarchy [ super defineHierarchy. (builder ensureClassNamed: #Entity) --|> #TEntityMetaLevelDependency. change <|-- addition. - change <|-- deletion - + change <|-- deletion. + change <|-- lineOfCode. ] { #category : #definition } GLPHMetamodelGenerator >> defineProperties [ super defineProperties. - + self changeProperties. self mergeRequestProperties. + self diffRangeProperties. ] { #category : #definition } @@ -141,7 +146,19 @@ GLPHMetamodelGenerator >> defineRelations [ <>-* (mergeRequest property: #project). "DIFF" - (diff property: #changes) <>-* (change property: #diff) + (diff property: #diffRanges) <>-* (diffRange property: #diff). + (diffRange property: #changes) -* (change property: #diffRange) + +] + +{ #category : #'as yet unclassified' } +GLPHMetamodelGenerator >> diffRangeProperties [ +"must parse the range. + originalLineRange = '-120,5' --> change starts at line #120 + 5 is the number of lines in the original file affected by the change" + diffRange property: #originalLineRange type: #String. + diffRange property: #newLineRange type: #String. + diffRange property: #lineOfCode type: #String ] { #category : #'as yet unclassified' } diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 32ce46d..16a0071 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -17,11 +17,8 @@ GLPHModelImporter >> completeImportProject: aGLHProject [ GLPHModelImporter >> importChangesForDiff: aGLHDiff [ |result| ('Import changes of diff ' , aGLHDiff printString) recordInfo. - result := self parseDiffString: aGLHDiff. - result do: [ :change | - aGLHDiff changes add: change ]. - aGLHDiff mooseModel addAll: aGLHDiff changes. - + self parseDiffString: aGLHDiff. + ^ aGLHDiff changes. ] @@ -80,30 +77,27 @@ GLPHModelImporter >> initialize [ { #category : #parsing } GLPHModelImporter >> parseDiffString: aDiff [ - | lines ranges changes lineRangesInfo addedLines deletedLines | + | lines range changes index | lines := aDiff diffString lines. - ranges := lines select: [ :line | line beginsWith: #'@@' ]. - + index := 0. changes := (lines collect: [ :line | - | aChange range | - aChange := nil. - - (line beginsWith: #'@@') - ifTrue: [ lineRangesInfo := line splitOn: ' ' ] - ifFalse: [ - (line trim beginsWith: #+) ifTrue: [ - aChange := GLPHEAddition new. - aChange linesOfCode: line. - aChange originalLineRange: (lineRangesInfo at: 2). - aChange newLineRange: (lineRangesInfo at: 3) ]. - (line trim beginsWith: #-) ifTrue: [ - aChange := GLPHEDeletion new. - aChange linesOfCode: line. - aChange originalLineRange: (lineRangesInfo at: 2). - aChange newLineRange: (lineRangesInfo at: 3) ] ]. - - aChange ]) reject: #isNil. -1 halt. + | aChange | + + aChange := GLPHEChange newFrom:line. + (line trim beginsWith: #'@@') ifTrue: [ + range := GLPHEDiffRange newFrom: line. + range diff: aDiff. + aDiff mooseModel add: range. + ]. + + aChange diffRange: range. + aChange relativeIndex: (index). + index := index + 1. + aDiff mooseModel add: aChange. + aChange + ]) reject: #isNil. + + ^ changes ] From 5fc27385cfcea30e73c7f47bb9e3e8ff6cd8a302 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 4 Apr 2024 17:39:06 +0200 Subject: [PATCH 08/77] add loading commit change on branches --- .../GLHApi.class.st | 18 ++ .../GLHModelImporter.class.st | 8 +- .../GLPHApi.class.st | 77 +++++++++ .../GLPHModelImporter.class.st | 157 +++++++++++++++--- 4 files changed, 235 insertions(+), 25 deletions(-) diff --git a/src/GitLabHealth-Model-Importer/GLHApi.class.st b/src/GitLabHealth-Model-Importer/GLHApi.class.st index 2d8f80b..b47e0cc 100644 --- a/src/GitLabHealth-Model-Importer/GLHApi.class.st +++ b/src/GitLabHealth-Model-Importer/GLHApi.class.st @@ -83,6 +83,24 @@ GLHApi >> commitsOfProject: aProjectId [ page: nil ] +{ #category : #commits } +GLHApi >> commitsOfProject: anInteger forRefName: refName since: since [ + + ^ self commitsOfProject: anInteger + forRefName: refName + since: since + until: nil + path: nil + author: nil + all: nil + with_stats: nil + firstParent: nil + order: nil + trailers: nil + perPage: nil + page: nil. +] + { #category : #'api - commits' } GLHApi >> commitsOfProject: aProjectId forRefName: refName since: since until: until path: aPath author: author all: aAllBoolean with_stats: withStateBoolean firstParent: firstParent order: orderString trailers: trailers perPage: aPerPageNumber page: aPageNumber [ "https://docs.gitlab.com/ee/api/commits.html#list-repository-commits" diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index 7c22401..1865c98 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -221,15 +221,15 @@ GLHModelImporter >> importGroup: aGroupID [ GLHModelImporter >> importRepository: aGLHRepository [ | resultBranches branches | - [ + [ resultBranches := self glhApi branchesOfRepository: aGLHRepository project id. branches := self parseBranchesResult: resultBranches. - branches do: [ :branch | aGLHRepository addBranch: branch ]. - self withFiles ifTrue: [ + branches do: [ :branch | aGLHRepository addBranch: branch. aGLHRepository mooseModel add: branch. ]. + self withFiles ifTrue: [ branches do: [ :branch | self importFilesOfBranch: branch ] ] ] on: NeoJSONParseError - do: [ + do: [ self inform: aGLHRepository project name , ' has no repository' ] ] diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st index 3ba3576..2d9135e 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st @@ -4,6 +4,83 @@ Class { #category : #'GitLabProjectHealth-Model-Importer' } +{ #category : #commit } +GLPHApi >> commitsOfProject: anInteger forRefName: refName [ + + ^ self commitsOfProject: anInteger + forRefName: refName + since: nil + until: nil + path: nil + author: nil + all: nil + with_stats: nil + firstParent: nil + order: nil + trailers: nil + perPage: nil + page: nil. +] + +{ #category : #commits } +GLPHApi >> commitsOfProject: anInteger forRefName: refName since: since [ + + ^ self commitsOfProject: anInteger + forRefName: refName + since: since + until: nil + path: nil + author: nil + all: nil + with_stats: nil + firstParent: nil + order: nil + trailers: nil + perPage: nil + page: nil. +] + +{ #category : #commit } +GLPHApi >> commitsOfProject: aGLHProject forRefName: refName until: toDate [ + + | newlyFoundCommit page | + page := 0. + newlyFoundCommit := { true }. + [ newlyFoundCommit isNotEmpty ] whileTrue: [ + | results parsedResults existingCommits | + page := page + 1. + ('import commit page ' , page printString) recordInfo. + results := self + commitsOfProject: aGLHProject id + forRefName: nil + since: nil + until: nil + path: nil + author: nil + all: nil + with_stats: nil + firstParent: nil + order: nil + trailers: nil + perPage: 100 + page: page. + parsedResults := self parseCommitsResult: results. + existingCommits := aGLHProject mooseModel allWithType: GLHCommit. + + newlyFoundCommit := parsedResults reject: [ :commitParsed | + (toDate isNil or: [ + commitParsed committed_date + < toDate asDateAndTime ]) or: [ + existingCommits anySatisfy: [ :existingCommit | + existingCommit id = commitParsed id ] ] ]. + aGLHProject mooseModel addAll: newlyFoundCommit. + aGLHProject repository commits addAll: newlyFoundCommit ]. + self withCommitDiffs ifTrue: [ + aGLHProject repository commits do: [ :commit | + self importDiffOfCommit: commit ] ] + +] + { #category : #'as yet unclassified' } GLPHApi >> diffsMergeRequestOfProject: aProjectID withId: anMRiid [ diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 16a0071..60b1678 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -4,13 +4,54 @@ Class { #category : #'GitLabProjectHealth-Model-Importer' } +{ #category : #commit } +GLPHModelImporter >> commitsOfProject: aGLHProject forRefName: refName until: toDate [ + + | newlyFoundCommit page | + page := 0. + newlyFoundCommit := { true }. + [ newlyFoundCommit isNotEmpty ] whileTrue: [ + | results parsedResults existingCommits | + page := page + 1. + ('import commit page ' , page printString) recordInfo. + results := self glhApi + commitsOfProject: aGLHProject id + forRefName: refName + since: nil + until: nil + path: nil + author: nil + all: nil + with_stats: nil + firstParent: nil + order: nil + trailers: nil + perPage: 100 + page: page. + parsedResults := self parseCommitsResult: results. + existingCommits := aGLHProject mooseModel allWithType: GLHCommit. + + newlyFoundCommit := parsedResults reject: [ :commitParsed | + (toDate isNil or: [ + commitParsed committed_date + < toDate asDateAndTime ]) or: [ + existingCommits anySatisfy: [ :existingCommit | + existingCommit id = commitParsed id ] ] ]. + aGLHProject mooseModel addAll: newlyFoundCommit. + aGLHProject repository commits addAll: newlyFoundCommit ]. + self withCommitDiffs ifTrue: [ + aGLHProject repository commits do: [ :commit | + self importDiffOfCommit: commit ] ] +] + { #category : #'private - api' } GLPHModelImporter >> completeImportProject: aGLHProject [ super completeImportProject: aGLHProject. + + self importMergeRequests: aGLHProject. - self importMergeRequests: aGLHProject - + ] { #category : #import } @@ -23,6 +64,77 @@ GLPHModelImporter >> importChangesForDiff: aGLHDiff [ ] +{ #category : #import } +GLPHModelImporter >> importCommitsOfBranch: aGLHBranch [ + + | commits | + " result := self glhApi + commitsOfProject: aGLHBranch repository project id + forRefName: aGLHBranch name." + commits := self + importCommitsOfBranch: aGLHBranch + forRefName: aGLHBranch name + until: Date today. + + commits do: [ :aCommit | + | diffs | + ('Import diff of commit: ' , aCommit short_id printString) + recordInfo. + diffs := self importDiffOfCommit: aCommit. + aCommit diffs addAll: diffs. + aCommit mooseModel addAll: diffs. + + diffs do: [ :diff | + | changes | + "changes are added into the model during the import" + self importChangesForDiff: diff ] ] +] + +{ #category : #commit } +GLPHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName until: toDate [ + + | newlyFoundCommit page | + page := 0. + newlyFoundCommit := { true }. + [ newlyFoundCommit isNotEmpty ] whileTrue: [ + | results parsedResults existingCommits | + page := page + 1. + ('import commit page ' , page printString) recordInfo. + results := self glhApi + commitsOfProject: aGLHBranch repository project id + forRefName: aGLHBranch name + since: nil + until: toDate asDateAndTime asString + path: nil + author: nil + all: nil + with_stats: nil + firstParent: nil + order: nil + trailers: nil + perPage: 100 + page: page. + + + parsedResults := self parseCommitsResult: results. + existingCommits := aGLHBranch commits. + + + newlyFoundCommit := parsedResults reject: [ :commitParsed | + existingCommits anySatisfy: [ :existingCommit | + existingCommit id = commitParsed id ] ]. + + "newlyFoundCommit := self parseCommitsResult: results." + + aGLHBranch mooseModel addAll: newlyFoundCommit. + aGLHBranch commits addAll: newlyFoundCommit ]. + + aGLHBranch commits do: [ :commit | + commit repository: aGLHBranch repository. + commit repository project: aGLHBranch repository project ]. + ^ aGLHBranch commits +] + { #category : #'as yet unclassified' } GLPHModelImporter >> importDiffOfMergeRequest: aMergeRequest [ @@ -67,6 +179,15 @@ GLPHModelImporter >> importMergeRequests: aGLHProject [ GLPHModelImporter >> importProject: aProjectID [ ] +{ #category : #'private - api' } +GLPHModelImporter >> importRepository: aGLHRepository [ + + super importRepository: aGLHRepository. + + aGLHRepository branches do: [ :branch | + self importCommitsOfBranch: branch ]. +] + { #category : #initialization } GLPHModelImporter >> initialize [ @@ -77,28 +198,22 @@ GLPHModelImporter >> initialize [ { #category : #parsing } GLPHModelImporter >> parseDiffString: aDiff [ - | lines range changes index | + | lines range index | lines := aDiff diffString lines. index := 0. - changes := (lines collect: [ :line | - | aChange | - - aChange := GLPHEChange newFrom:line. - (line trim beginsWith: #'@@') ifTrue: [ - range := GLPHEDiffRange newFrom: line. - range diff: aDiff. - aDiff mooseModel add: range. - ]. - - aChange diffRange: range. - aChange relativeIndex: (index). - index := index + 1. - aDiff mooseModel add: aChange. - aChange - ]) reject: #isNil. - + lines do: [ :line | + | aChange | + aChange := GLPHEChange newFrom: line. + (line trim beginsWith: #'@@') ifTrue: [ + range := GLPHEDiffRange newFrom: line. + range diff: aDiff. + aDiff mooseModel add: range ]. - ^ changes + aChange diffRange: range. + aChange relativeIndex: index. + index := index + 1. + aDiff mooseModel add: aChange. + aChange ]. ] { #category : #parsing } From ec3402a1e7f6ad07c366fe2115221a2066ade5e1 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Fri, 5 Apr 2024 11:11:14 +0200 Subject: [PATCH 09/77] add project importer --- .../GLPHModelImporter.class.st | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 60b1678..88c897d 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -177,6 +177,13 @@ GLPHModelImporter >> importMergeRequests: aGLHProject [ { #category : #'as yet unclassified' } GLPHModelImporter >> importProject: aProjectID [ + | result projectResult | + ('Import group: ' , aProjectID printString) recordInfo. + + result := self glhApi project: aProjectID. + projectResult := self parseProjectResult: result. + + ^ self completeImportProject: projectResult ] { #category : #'private - api' } @@ -299,3 +306,14 @@ work_in_progress). ^ reader nextAs: #ArrayOfMergeRequest ] + +{ #category : #parsing } +GLPHModelImporter >> parseProjectResult: aResult [ + | reader | + reader := NeoJSONReader on: aResult readStream. + reader for: GLHProject do: [ :mapping | + mapping mapInstVars. ]. +" reader mapInstVarsFor: GLHProject." + + ^ reader nextAs: GLHProject +] From a381b1017d30d8c44896a65f641a3db8512a7489 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Fri, 5 Apr 2024 11:20:30 +0200 Subject: [PATCH 10/77] add commit import since a certain date --- .../GLHModelImporter.class.st | 5 +- .../GLPHModelImporter.class.st | 51 +++++++++++++++++-- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index 1865c98..7c91d2c 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -31,15 +31,14 @@ GLHModelImporter >> beWithouFiles [ GLHModelImporter >> completeImportProject: aGLHProject [ ('Complete import project: ' , aGLHProject id printString) recordInfo. - (self pipelinesOf: aGLHProject id) do: [ :pipeline | + (self pipelinesOf: aGLHProject id) do: [ :pipeline | self glhModel add: pipeline. aGLHProject addPipeline: pipeline ]. aGLHProject creator: (self importUser: aGLHProject creator_id). aGLHProject repository: GLHRepository new. - self glhModel add: aGLHProject . + self glhModel add: aGLHProject. self glhModel add: aGLHProject repository. self importRepository: aGLHProject repository. - self importCommits: aGLHProject ] { #category : #private } diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 88c897d..cf853ad 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -72,9 +72,9 @@ GLPHModelImporter >> importCommitsOfBranch: aGLHBranch [ commitsOfProject: aGLHBranch repository project id forRefName: aGLHBranch name." commits := self - importCommitsOfBranch: aGLHBranch - forRefName: aGLHBranch name - until: Date today. + importCommitsOfBranch: aGLHBranch + forRefName: aGLHBranch name + since: Date today - 1 week. commits do: [ :aCommit | | diffs | @@ -90,6 +90,51 @@ GLPHModelImporter >> importCommitsOfBranch: aGLHBranch [ self importChangesForDiff: diff ] ] ] +{ #category : #commit } +GLPHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName since: fromDate [ + + | newlyFoundCommit page | + page := 0. + newlyFoundCommit := { true }. + [ newlyFoundCommit isNotEmpty ] whileTrue: [ + | results parsedResults existingCommits | + page := page + 1. + ('import commit page ' , page printString) recordInfo. + results := self glhApi + commitsOfProject: aGLHBranch repository project id + forRefName: aGLHBranch name + since: fromDate asDateAndTime asString + until: nil + path: nil + author: nil + all: nil + with_stats: nil + firstParent: nil + order: nil + trailers: nil + perPage: 100 + page: page. + + + parsedResults := self parseCommitsResult: results. + existingCommits := aGLHBranch commits. + + + newlyFoundCommit := parsedResults reject: [ :commitParsed | + existingCommits anySatisfy: [ :existingCommit | + existingCommit id = commitParsed id ] ]. + + "newlyFoundCommit := self parseCommitsResult: results." + + aGLHBranch mooseModel addAll: newlyFoundCommit. + aGLHBranch commits addAll: newlyFoundCommit ]. + + aGLHBranch commits do: [ :commit | + commit repository: aGLHBranch repository. + commit repository project: aGLHBranch repository project ]. + ^ aGLHBranch commits +] + { #category : #commit } GLPHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName until: toDate [ From a454e29faa8449e03b608c5c83b1f6449d103aa3 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Fri, 5 Apr 2024 15:39:51 +0200 Subject: [PATCH 11/77] update OG model --- src/GLPHExtended-Model/GLHDiff.extension.st | 54 ----------- .../GLHMetamodelGenerator.class.st | 6 +- src/GitLabHealth-Model/GLHBranch.class.st | 23 +++++ src/GitLabHealth-Model/GLHCommit.class.st | 97 ++++++++++++++++--- src/GitLabHealth-Model/GLHDiff.class.st | 22 +++++ src/GitLabHealth-Model/GLHFile.class.st | 18 ++++ src/GitLabHealth-Model/GLHFileBlob.class.st | 2 + .../GLHFileDirectory.class.st | 11 +++ src/GitLabHealth-Model/GLHGroup.class.st | 28 ++++++ src/GitLabHealth-Model/GLHPipeline.class.st | 18 ++++ src/GitLabHealth-Model/GLHProject.class.st | 37 +++++++ src/GitLabHealth-Model/GLHRepository.class.st | 17 ++++ .../GLHTEntityCreator.trait.st | 2 + src/GitLabHealth-Model/GLHUser.class.st | 37 +++++++ .../GLPHModelImporter.class.st | 19 ++-- 15 files changed, 314 insertions(+), 77 deletions(-) diff --git a/src/GLPHExtended-Model/GLHDiff.extension.st b/src/GLPHExtended-Model/GLHDiff.extension.st index ac49683..d9daf32 100644 --- a/src/GLPHExtended-Model/GLHDiff.extension.st +++ b/src/GLPHExtended-Model/GLHDiff.extension.st @@ -1,59 +1,5 @@ Extension { #name : #GLHDiff } -{ #category : #'*GLPHExtended-Model-accessing' } -GLHDiff >> addition [ - "Relation named: #addition type: #GLPHEAddition opposite: #diffs" - - - - - - ^ self attributeAt: #addition ifAbsentPut: [ FMMultivalueLink on: self opposite: #diffs: ] -] - -{ #category : #'*GLPHExtended-Model-accessing' } -GLHDiff >> addition: anObject [ - - - self addition value: anObject -] - -{ #category : #'*GLPHExtended-Model-accessing' } -GLHDiff >> changes [ - "Relation named: #changes type: #GLPHEChange opposite: #diff" - - - - - - ^ self attributeAt: #changes ifAbsentPut: [ FMMultivalueLink on: self opposite: #diff: ] -] - -{ #category : #'*GLPHExtended-Model-accessing' } -GLHDiff >> changes: anObject [ - - - self changes value: anObject -] - -{ #category : #'*GLPHExtended-Model-accessing' } -GLHDiff >> deletion [ - "Relation named: #deletion type: #GLPHEDeletion opposite: #diffs" - - - - - - ^ self attributeAt: #deletion ifAbsentPut: [ FMMultivalueLink on: self opposite: #diffs: ] -] - -{ #category : #'*GLPHExtended-Model-accessing' } -GLHDiff >> deletion: anObject [ - - - self deletion value: anObject -] - { #category : #'*GLPHExtended-Model-accessing' } GLHDiff >> diffRanges [ "Relation named: #diffRanges type: #GLPHEDiffRange opposite: #diff" diff --git a/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st b/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st index 5392ba0..a3ec614 100644 --- a/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st +++ b/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st @@ -145,10 +145,12 @@ GLHMetamodelGenerator >> defineRelations [ (project property: #repository) <>- (repository property: #project). (repository property: #branches) <>-* (branch property: #repository). (branch property: #files) <>-* (file property: #branch). - (fileDirectory property: #files) <>-* (file property: #directoryOwner). + (fileDirectory property: #files) + <>-* (file property: #directoryOwner). (repository property: #commits) <>-* (commit property: #repository). (branch property: #commits) <>-* (commit property: #branch). - (commit property: #diffs) <>-* (diff property: #commit) + (commit property: #diffs) <>-* (diff property: #commit). + (commit property: #parentCommit) - (commit property: #childCommit) ] { #category : #definition } diff --git a/src/GitLabHealth-Model/GLHBranch.class.st b/src/GitLabHealth-Model/GLHBranch.class.st index e50f03d..5373566 100644 --- a/src/GitLabHealth-Model/GLHBranch.class.st +++ b/src/GitLabHealth-Model/GLHBranch.class.st @@ -1,5 +1,28 @@ " A git branch + +## Relations +====================== + +### Parents +| Relation | Origin | Opposite | Type | Comment | +|---| +| `repository` | `GLHBranch` | `branches` | `GLHRepository` | | + +### Children +| Relation | Origin | Opposite | Type | Comment | +|---| +| `commits` | `GLHBranch` | `branch` | `GLHCommit` | | +| `files` | `GLHBranch` | `branch` | `GLHFile` | | + + +## Properties +====================== + +| Name | Type | Default value | Comment | +|---| +| `name` | `String` | nil | | + " Class { #name : #GLHBranch, diff --git a/src/GitLabHealth-Model/GLHCommit.class.st b/src/GitLabHealth-Model/GLHCommit.class.st index f11639a..28b262f 100644 --- a/src/GitLabHealth-Model/GLHCommit.class.st +++ b/src/GitLabHealth-Model/GLHCommit.class.st @@ -1,27 +1,71 @@ " a commit attached to a repository + +## Relations +====================== + +### Parents +| Relation | Origin | Opposite | Type | Comment | +|---| +| `branch` | `GLHCommit` | `commits` | `GLHBranch` | | +| `repository` | `GLHCommit` | `commits` | `GLHRepository` | | + +### Children +| Relation | Origin | Opposite | Type | Comment | +|---| +| `diffs` | `GLHCommit` | `commit` | `GLHDiff` | | + +### Other +| Relation | Origin | Opposite | Type | Comment | +|---| +| `childCommit` | `GLHCommit` | `parentCommit` | `GLHCommit` | | +| `parentCommit` | `GLHCommit` | `childCommit` | `GLHCommit` | | + + +## Properties +====================== + +| Name | Type | Default value | Comment | +|---| +| `additions` | `Number` | nil | The number of line that git consider added| +| `author_email` | `String` | nil | | +| `author_name` | `String` | nil | | +| `authored_date` | `Object` | nil | | +| `committed_date` | `Object` | nil | | +| `committer_email` | `String` | nil | | +| `committer_name` | `String` | nil | | +| `created_at` | `Object` | nil | | +| `deletions` | `Number` | nil | The number of line that git consider deleted| +| `id` | `String` | nil | | +| `message` | `String` | nil | | +| `short_id` | `String` | nil | | +| `title` | `String` | nil | | +| `web_url` | `String` | nil | | + " Class { #name : #GLHCommit, #superclass : #GLHEntity, #instVars : [ - '#id => FMProperty', - '#short_id => FMProperty', - '#title => FMProperty', - '#author_name => FMProperty', + '#additions => FMProperty', '#author_email => FMProperty', + '#author_name => FMProperty', '#authored_date => FMProperty', - '#committer_name => FMProperty', - '#committer_email => FMProperty', + '#branch => FMOne type: #GLHBranch opposite: #commits', + '#childCommit => FMOne type: #GLHCommit opposite: #parentCommit', '#committed_date => FMProperty', + '#committer_email => FMProperty', + '#committer_name => FMProperty', '#created_at => FMProperty', - '#message => FMProperty', - '#web_url => FMProperty', - '#additions => FMProperty', '#deletions => FMProperty', + '#diffs => FMMany type: #GLHDiff opposite: #commit', + '#id => FMProperty', + '#message => FMProperty', + '#parentCommit => FMOne type: #GLHCommit opposite: #childCommit', '#repository => FMOne type: #GLHRepository opposite: #commits', - '#branch => FMOne type: #GLHBranch opposite: #commits', - '#diffs => FMMany type: #GLHDiff opposite: #commit' + '#short_id => FMProperty', + '#title => FMProperty', + '#web_url => FMProperty' ], #category : #'GitLabHealth-Model-Entities' } @@ -121,6 +165,22 @@ GLHCommit >> branchGroup [ ^ MooseSpecializedGroup with: self branch ] +{ #category : #accessing } +GLHCommit >> childCommit [ + "Relation named: #childCommit type: #GLHCommit opposite: #parentCommit" + + + + ^ childCommit +] + +{ #category : #accessing } +GLHCommit >> childCommit: anObject [ + + + childCommit := anObject +] + { #category : #accessing } GLHCommit >> committed_date [ @@ -243,6 +303,21 @@ GLHCommit >> message: anObject [ message := anObject ] +{ #category : #accessing } +GLHCommit >> parentCommit [ + "Relation named: #parentCommit type: #GLHCommit opposite: #childCommit" + + + ^ parentCommit +] + +{ #category : #accessing } +GLHCommit >> parentCommit: anObject [ + + + parentCommit := anObject +] + { #category : #accessing } GLHCommit >> repository [ "Relation named: #repository type: #GLHRepository opposite: #commits" diff --git a/src/GitLabHealth-Model/GLHDiff.class.st b/src/GitLabHealth-Model/GLHDiff.class.st index ca234a0..8c9be42 100644 --- a/src/GitLabHealth-Model/GLHDiff.class.st +++ b/src/GitLabHealth-Model/GLHDiff.class.st @@ -1,5 +1,27 @@ " The diff of a commit + +## Relations +====================== + +### Parents +| Relation | Origin | Opposite | Type | Comment | +|---| +| `commit` | `GLHDiff` | `diffs` | `GLHCommit` | | + + +## Properties +====================== + +| Name | Type | Default value | Comment | +|---| +| `deleted_file` | `Boolean` | false | Is the file of the diff has been removed| +| `diffString` | `String` | nil | The string presenting the diff. It should follow the [Detailed-Unified](https://www.gnu.org/software/diffutils/manual/html_node/Detailed-Unified.html) format| +| `new_file` | `Boolean` | false | Is the file of the diff is a new file| +| `new_path` | `String` | nil | The path to the file in the resulted repository commitish| +| `old_path` | `String` | nil | The path to the file in the origin repository commitish| +| `renamed_file` | `Boolean` | false | Is the file of the diff has been renamed (see old_path/new_path) file| + " Class { #name : #GLHDiff, diff --git a/src/GitLabHealth-Model/GLHFile.class.st b/src/GitLabHealth-Model/GLHFile.class.st index f01af30..7f34602 100644 --- a/src/GitLabHealth-Model/GLHFile.class.st +++ b/src/GitLabHealth-Model/GLHFile.class.st @@ -1,5 +1,23 @@ " A file + +## Relations +====================== + +### Parents +| Relation | Origin | Opposite | Type | Comment | +|---| +| `branch` | `GLHFile` | `files` | `GLHBranch` | | +| `directoryOwner` | `GLHFile` | `files` | `GLHFileDirectory` | | + + +## Properties +====================== + +| Name | Type | Default value | Comment | +|---| +| `name` | `String` | nil | | + " Class { #name : #GLHFile, diff --git a/src/GitLabHealth-Model/GLHFileBlob.class.st b/src/GitLabHealth-Model/GLHFileBlob.class.st index f8099bb..d646f8a 100644 --- a/src/GitLabHealth-Model/GLHFileBlob.class.st +++ b/src/GitLabHealth-Model/GLHFileBlob.class.st @@ -1,5 +1,7 @@ " A file blob + + " Class { #name : #GLHFileBlob, diff --git a/src/GitLabHealth-Model/GLHFileDirectory.class.st b/src/GitLabHealth-Model/GLHFileDirectory.class.st index 0254002..9838cb5 100644 --- a/src/GitLabHealth-Model/GLHFileDirectory.class.st +++ b/src/GitLabHealth-Model/GLHFileDirectory.class.st @@ -1,5 +1,16 @@ " A file directory + +## Relations +====================== + +### Children +| Relation | Origin | Opposite | Type | Comment | +|---| +| `files` | `GLHFileDirectory` | `directoryOwner` | `GLHFile` | | + + + " Class { #name : #GLHFileDirectory, diff --git a/src/GitLabHealth-Model/GLHGroup.class.st b/src/GitLabHealth-Model/GLHGroup.class.st index 60309df..b6d3126 100644 --- a/src/GitLabHealth-Model/GLHGroup.class.st +++ b/src/GitLabHealth-Model/GLHGroup.class.st @@ -1,5 +1,33 @@ " A GitLab Group + +## Relations +====================== + +### Parents +| Relation | Origin | Opposite | Type | Comment | +|---| +| `group` | `GLHGroup` | `subGroups` | `GLHGroup` | | + +### Children +| Relation | Origin | Opposite | Type | Comment | +|---| +| `projects` | `GLHGroup` | `group` | `GLHProject` | | +| `subGroups` | `GLHGroup` | `group` | `GLHGroup` | | + + +## Properties +====================== + +| Name | Type | Default value | Comment | +|---| +| `avatar_url` | `String` | nil | | +| `description` | `String` | nil | | +| `id` | `Number` | nil | | +| `name` | `String` | nil | Basic name of the entity, not full reference.| +| `visibility` | `String` | nil | | +| `web_url` | `String` | nil | | + " Class { #name : #GLHGroup, diff --git a/src/GitLabHealth-Model/GLHPipeline.class.st b/src/GitLabHealth-Model/GLHPipeline.class.st index 06b3002..b11dc22 100644 --- a/src/GitLabHealth-Model/GLHPipeline.class.st +++ b/src/GitLabHealth-Model/GLHPipeline.class.st @@ -1,5 +1,23 @@ " A GitLab Pipeline execution + +## Relations +====================== + +### Parents +| Relation | Origin | Opposite | Type | Comment | +|---| +| `project` | `GLHPipeline` | `pipelines` | `GLHProject` | | + + +## Properties +====================== + +| Name | Type | Default value | Comment | +|---| +| `runDate` | `Object` | nil | Date this pipeline was run| +| `status` | `String` | nil | #success or #failure| + " Class { #name : #GLHPipeline, diff --git a/src/GitLabHealth-Model/GLHProject.class.st b/src/GitLabHealth-Model/GLHProject.class.st index 167489d..f74e08f 100644 --- a/src/GitLabHealth-Model/GLHProject.class.st +++ b/src/GitLabHealth-Model/GLHProject.class.st @@ -1,5 +1,42 @@ " A GitLab Project + +## Relations +====================== + +### Parents +| Relation | Origin | Opposite | Type | Comment | +|---| +| `group` | `GLHProject` | `projects` | `GLHGroup` | | + +### Children +| Relation | Origin | Opposite | Type | Comment | +|---| +| `pipelines` | `GLHProject` | `project` | `GLHPipeline` | | +| `repository` | `GLHProject` | `project` | `GLHRepository` | | + +### Other +| Relation | Origin | Opposite | Type | Comment | +|---| +| `creator` | `GLHProject` | `createdProjects` | `GLHUser` | | + + +## Properties +====================== + +| Name | Type | Default value | Comment | +|---| +| `archived` | `Boolean` | nil | | +| `archived` | `Boolean` | nil | | +| `avatar_url` | `String` | nil | | +| `creator_id` | `Number` | nil | | +| `description` | `String` | nil | | +| `id` | `Number` | nil | | +| `name` | `String` | nil | Basic name of the entity, not full reference.| +| `readme_url` | `String` | nil | | +| `topics` | `Object` | nil | | +| `web_url` | `String` | nil | | + " Class { #name : #GLHProject, diff --git a/src/GitLabHealth-Model/GLHRepository.class.st b/src/GitLabHealth-Model/GLHRepository.class.st index e2023a0..e761250 100644 --- a/src/GitLabHealth-Model/GLHRepository.class.st +++ b/src/GitLabHealth-Model/GLHRepository.class.st @@ -1,5 +1,22 @@ " A git repository + +## Relations +====================== + +### Parents +| Relation | Origin | Opposite | Type | Comment | +|---| +| `project` | `GLHRepository` | `repository` | `GLHProject` | | + +### Children +| Relation | Origin | Opposite | Type | Comment | +|---| +| `branches` | `GLHRepository` | `repository` | `GLHBranch` | | +| `commits` | `GLHRepository` | `repository` | `GLHCommit` | | + + + " Class { #name : #GLHRepository, diff --git a/src/GitLabHealth-Model/GLHTEntityCreator.trait.st b/src/GitLabHealth-Model/GLHTEntityCreator.trait.st index ac070be..84d43f3 100644 --- a/src/GitLabHealth-Model/GLHTEntityCreator.trait.st +++ b/src/GitLabHealth-Model/GLHTEntityCreator.trait.st @@ -1,6 +1,8 @@ " This trait is used by Famix models. It provides an API for creating entities and adding them to the model. + + " Trait { #name : #GLHTEntityCreator, diff --git a/src/GitLabHealth-Model/GLHUser.class.st b/src/GitLabHealth-Model/GLHUser.class.st index 7e2b6b8..3ad7340 100644 --- a/src/GitLabHealth-Model/GLHUser.class.st +++ b/src/GitLabHealth-Model/GLHUser.class.st @@ -1,5 +1,42 @@ " A GitLab User + +## Relations +====================== + +### Other +| Relation | Origin | Opposite | Type | Comment | +|---| +| `createdProjects` | `GLHUser` | `creator` | `GLHProject` | | + + +## Properties +====================== + +| Name | Type | Default value | Comment | +|---| +| `avatar_url` | `String` | nil | | +| `bio` | `String` | nil | | +| `bot` | `String` | nil | | +| `created_at` | `String` | nil | | +| `followers` | `String` | nil | | +| `following` | `String` | nil | | +| `id` | `Number` | nil | | +| `job_title` | `String` | nil | | +| `linkedin` | `String` | nil | | +| `location` | `String` | nil | | +| `name` | `String` | nil | | +| `organization` | `String` | nil | | +| `pronouns` | `String` | nil | | +| `public_email` | `String` | nil | | +| `skype` | `String` | nil | | +| `state` | `String` | nil | | +| `twitter` | `String` | nil | | +| `username` | `String` | nil | | +| `web_url` | `String` | nil | | +| `website_url` | `String` | nil | | +| `work_information` | `String` | nil | | + " Class { #name : #GLHUser, diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index cf853ad..d909449 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -60,7 +60,7 @@ GLPHModelImporter >> importChangesForDiff: aGLHDiff [ ('Import changes of diff ' , aGLHDiff printString) recordInfo. self parseDiffString: aGLHDiff. - ^ aGLHDiff changes. + ^ aGLHDiff diffRanges . ] @@ -74,19 +74,19 @@ GLPHModelImporter >> importCommitsOfBranch: aGLHBranch [ commits := self importCommitsOfBranch: aGLHBranch forRefName: aGLHBranch name - since: Date today - 1 week. - + since: Date today - 3 day. + + commits do: [ :aCommit | | diffs | + 1 halt. ('Import diff of commit: ' , aCommit short_id printString) recordInfo. diffs := self importDiffOfCommit: aCommit. aCommit diffs addAll: diffs. aCommit mooseModel addAll: diffs. - diffs do: [ :diff | - | changes | - "changes are added into the model during the import" + diffs do: [ :diff | "changes are added into the model during the import" self importChangesForDiff: diff ] ] ] @@ -119,19 +119,18 @@ GLPHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName since parsedResults := self parseCommitsResult: results. existingCommits := aGLHBranch commits. - newlyFoundCommit := parsedResults reject: [ :commitParsed | existingCommits anySatisfy: [ :existingCommit | existingCommit id = commitParsed id ] ]. - "newlyFoundCommit := self parseCommitsResult: results." - aGLHBranch mooseModel addAll: newlyFoundCommit. aGLHBranch commits addAll: newlyFoundCommit ]. - aGLHBranch commits do: [ :commit | + " (aGLHBranch commits reject: #isNil) do: [ :commit | + 1 halt. commit repository: aGLHBranch repository. commit repository project: aGLHBranch repository project ]. +" ^ aGLHBranch commits ] From f7dbe737884bcf3a3a86cf8b48ceade1b2e3c681 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Fri, 5 Apr 2024 17:32:50 +0200 Subject: [PATCH 12/77] --- .../GLHMetamodelGenerator.class.st | 11 ++++++++--- .../GLHModelImporter.class.st | 14 ++++++++++++-- src/GitLabHealth-Model/GLHCommit.class.st | 8 ++++++++ .../GLPHModelImporter.class.st | 13 ++++++------- 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st b/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st index a3ec614..6343e10 100644 --- a/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st +++ b/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st @@ -43,6 +43,8 @@ GLHMetamodelGenerator >> branchProperties [ { #category : #definition } GLHMetamodelGenerator >> commitProperties [ + commit property: #parent_ids type: #Object defaultValue: 'OrderedCollection new'. + commit property: #id type: #String. commit property: #short_id type: #String. commit property: #title type: #String. @@ -56,6 +58,7 @@ GLHMetamodelGenerator >> commitProperties [ commit property: #message type: #String. commit property: #web_url type: #String. + (commit property: #additions type: #Number) comment: 'The number of line that git consider added'. (commit property: #deletions type: #Number) comment: @@ -124,14 +127,16 @@ GLHMetamodelGenerator >> defineProperties [ project property: #creator_id type: #Number. "Pipelines properties" - (pipeline property: #status type: #String) comment: '#success or #failure'. - (pipeline property: #runDate type: #Object) comment: 'Date this pipeline was run'. + (pipeline property: #status type: #String) comment: + '#success or #failure'. + (pipeline property: #runDate type: #Object) comment: + 'Date this pipeline was run'. self userProperties. self branchProperties. self fileProperties. self commitProperties. - self diffProperties + self diffProperties ] { #category : #definition } diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index 7c91d2c..e434cc1 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -79,7 +79,7 @@ GLHModelImporter >> glhModel: anObject [ { #category : #'private - api' } GLHModelImporter >> importCommits: aGLHProject [ - + "limited to the last 20 commits" | results parsedResults | results := self glhApi commitsOfProject: aGLHProject id @@ -98,7 +98,9 @@ GLHModelImporter >> importCommits: aGLHProject [ parsedResults := self parseCommitsResult: results. aGLHProject repository mooseModel addAll: parsedResults. aGLHProject repository commits addAll: parsedResults. - self withCommitDiffs ifTrue: [ aGLHProject repository commits do: [ :commit | self importDiffOfCommit: commit ] ]. + self withCommitDiffs ifTrue: [ + aGLHProject repository commits do: [ :commit | + self importDiffOfCommit: commit ] ] ] { #category : #api } @@ -269,6 +271,7 @@ GLHModelImporter >> parseCommitsResult: result [ | reader | reader := NeoJSONReader on: result readStream. + reader for: GLHCommit do: [ :mapping | mapping mapInstVars: #( id short_id title author_name author_email committer_name @@ -276,6 +279,7 @@ GLHModelImporter >> parseCommitsResult: result [ (mapping mapInstVar: #authored_date) valueSchema: DateAndTime. (mapping mapInstVar: #committed_date) valueSchema: DateAndTime. (mapping mapInstVar: #created_at) valueSchema: DateAndTime. + (mapping mapInstVar: #parent_ids) valueSchema: #ArrayOfIds. mapping mapProperty: 'stats' getter: [ :el | "Not used" ] @@ -285,10 +289,16 @@ GLHModelImporter >> parseCommitsResult: result [ reader for: DateAndTime customDo: [ :mapping | mapping decoder: [ :string | DateAndTime fromString: string ] ]. + + reader + for: #ArrayOfIds + customDo: [ :mapping | mapping decoder: [ :string | string ] ]. + reader for: #ArrayOfCommit customDo: [ :customMappting | customMappting listOfElementSchema: GLHCommit ]. + ^ reader nextAs: #ArrayOfCommit ] diff --git a/src/GitLabHealth-Model/GLHCommit.class.st b/src/GitLabHealth-Model/GLHCommit.class.st index 28b262f..b829193 100644 --- a/src/GitLabHealth-Model/GLHCommit.class.st +++ b/src/GitLabHealth-Model/GLHCommit.class.st @@ -38,6 +38,7 @@ a commit attached to a repository | `deletions` | `Number` | nil | The number of line that git consider deleted| | `id` | `String` | nil | | | `message` | `String` | nil | | +| `parent_ids` | `Object` | 'OrderedCollection new' | | | `short_id` | `String` | nil | | | `title` | `String` | nil | | | `web_url` | `String` | nil | | @@ -62,6 +63,7 @@ Class { '#id => FMProperty', '#message => FMProperty', '#parentCommit => FMOne type: #GLHCommit opposite: #childCommit', + '#parent_ids => FMProperty defaultValue: ((OrderedCollection new))', '#repository => FMOne type: #GLHRepository opposite: #commits', '#short_id => FMProperty', '#title => FMProperty', @@ -318,6 +320,12 @@ GLHCommit >> parentCommit: anObject [ parentCommit := anObject ] +{ #category : #accessing } +GLHCommit >> parent_ids: anObject [ + + parent_ids := anObject +] + { #category : #accessing } GLHCommit >> repository [ "Relation named: #repository type: #GLHRepository opposite: #commits" diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index d909449..4545b9f 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -55,13 +55,12 @@ GLPHModelImporter >> completeImportProject: aGLHProject [ ] { #category : #import } -GLPHModelImporter >> importChangesForDiff: aGLHDiff [ - |result| +GLPHModelImporter >> importChangesForDiff: aGLHDiff [ + ('Import changes of diff ' , aGLHDiff printString) recordInfo. - self parseDiffString: aGLHDiff. - - ^ aGLHDiff diffRanges . + self parseDiffString: aGLHDiff. + ^ aGLHDiff diffRanges ] { #category : #import } @@ -75,8 +74,8 @@ GLPHModelImporter >> importCommitsOfBranch: aGLHBranch [ importCommitsOfBranch: aGLHBranch forRefName: aGLHBranch name since: Date today - 3 day. - - + + commits do: [ :aCommit | | diffs | 1 halt. From e42c688e15788319a40db8c547ce3eac4a33cf62 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Mon, 8 Apr 2024 09:20:26 +0200 Subject: [PATCH 13/77] fix an user import coming from a project import only --- .../GLHModelImporter.class.st | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index e434cc1..d43c338 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -238,12 +238,13 @@ GLHModelImporter >> importRepository: aGLHRepository [ GLHModelImporter >> importUser: aUserID [ | result userResult | - (glhModel allWithType: GLHUser) detect: [ :user | user id = aUserID ] ifFound: [ :user | ^ user ]. - ('Import user: ' , aUserID printString) recordInfo. - result := self glhApi user: aUserID. - userResult := self parseUserResult: result. - glhModel add: userResult. - ^ userResult + ^ (glhModel allWithType: GLHUser) detect: [ :user | user id = aUserID ] ifNone: [ + ('Import user: ' , aUserID printString) recordInfo. + result := self glhApi user: aUserID. + userResult := self parseUserResult: result. + glhModel add: userResult. + ^ userResult ]. + ] { #category : #initialization } From 3c902790eb2e54abfead6ee9ad4298809d02f8a0 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Mon, 8 Apr 2024 11:29:52 +0200 Subject: [PATCH 14/77] hope i havn't break everything --- .../GLHCommit.extension.st | 7 ++++ .../GLHMetamodelGenerator.class.st | 5 ++- .../GLHModelImporter.class.st | 14 +++++-- src/GitLabHealth-Model/GLHCommit.class.st | 42 ++++++++++++------- .../GLHTEntityCreator.trait.st | 7 ++++ src/GitLabHealth-Model/GLHUser.class.st | 33 ++++++++------- .../GLPHModelImporter.class.st | 41 ++++++++++++++++-- 7 files changed, 110 insertions(+), 39 deletions(-) create mode 100644 src/GitLabHealth-Model-Extension/GLHCommit.extension.st diff --git a/src/GitLabHealth-Model-Extension/GLHCommit.extension.st b/src/GitLabHealth-Model-Extension/GLHCommit.extension.st new file mode 100644 index 0000000..d458eb9 --- /dev/null +++ b/src/GitLabHealth-Model-Extension/GLHCommit.extension.st @@ -0,0 +1,7 @@ +Extension { #name : #GLHCommit } + +{ #category : #'*GitLabHealth-Model-Extension' } +GLHCommit >> parent_ids [ + + ^ parent_ids +] diff --git a/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st b/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st index 6343e10..535ee9e 100644 --- a/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st +++ b/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st @@ -102,6 +102,7 @@ GLHMetamodelGenerator >> defineHierarchy [ group --|> #TNamedEntity. project --|> #TNamedEntity. + user --|> #TNamedEntity. ] { #category : #definition } @@ -154,8 +155,10 @@ GLHMetamodelGenerator >> defineRelations [ <>-* (file property: #directoryOwner). (repository property: #commits) <>-* (commit property: #repository). (branch property: #commits) <>-* (commit property: #branch). + (commit property: #diffs) <>-* (diff property: #commit). - (commit property: #parentCommit) - (commit property: #childCommit) + "(user property: #commits) <>-* (commit property: #commiter)." + (commit property: #parentCommits) *-* (commit property: #childCommits) ] { #category : #definition } diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index d43c338..fbd4bd4 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -96,10 +96,14 @@ GLHModelImporter >> importCommits: aGLHProject [ perPage: nil page: nil. parsedResults := self parseCommitsResult: results. - aGLHProject repository mooseModel addAll: parsedResults. - aGLHProject repository commits addAll: parsedResults. + aGLHProject mooseModel addAll: parsedResults. + + parsedResults do: [ :commit | + commit repository: aGLHProject repository. + ]. + self withCommitDiffs ifTrue: [ - aGLHProject repository commits do: [ :commit | + parsedResults do: [ :commit | self importDiffOfCommit: commit ] ] ] @@ -138,6 +142,8 @@ GLHModelImporter >> importCommitsOf: aGLHProject withStats: aBoolean until: toDa existingCommit id = commitParsed id ] ] ]. aGLHProject mooseModel addAll: newlyFoundCommit. aGLHProject repository commits addAll: newlyFoundCommit ]. + + self withCommitDiffs ifTrue: [ aGLHProject repository commits do: [ :commit | self importDiffOfCommit: commit ] ] @@ -243,7 +249,7 @@ GLHModelImporter >> importUser: aUserID [ result := self glhApi user: aUserID. userResult := self parseUserResult: result. glhModel add: userResult. - ^ userResult ]. + userResult ]. ] diff --git a/src/GitLabHealth-Model/GLHCommit.class.st b/src/GitLabHealth-Model/GLHCommit.class.st index b829193..cd8ae19 100644 --- a/src/GitLabHealth-Model/GLHCommit.class.st +++ b/src/GitLabHealth-Model/GLHCommit.class.st @@ -18,8 +18,8 @@ a commit attached to a repository ### Other | Relation | Origin | Opposite | Type | Comment | |---| -| `childCommit` | `GLHCommit` | `parentCommit` | `GLHCommit` | | -| `parentCommit` | `GLHCommit` | `childCommit` | `GLHCommit` | | +| `childCommits` | `GLHCommit` | `parentCommits` | `GLHCommit` | | +| `parentCommits` | `GLHCommit` | `childCommits` | `GLHCommit` | | ## Properties @@ -53,7 +53,7 @@ Class { '#author_name => FMProperty', '#authored_date => FMProperty', '#branch => FMOne type: #GLHBranch opposite: #commits', - '#childCommit => FMOne type: #GLHCommit opposite: #parentCommit', + '#childCommits => FMMany type: #GLHCommit opposite: #parentCommits', '#committed_date => FMProperty', '#committer_email => FMProperty', '#committer_name => FMProperty', @@ -62,7 +62,7 @@ Class { '#diffs => FMMany type: #GLHDiff opposite: #commit', '#id => FMProperty', '#message => FMProperty', - '#parentCommit => FMOne type: #GLHCommit opposite: #childCommit', + '#parentCommits => FMMany type: #GLHCommit opposite: #childCommits', '#parent_ids => FMProperty defaultValue: ((OrderedCollection new))', '#repository => FMOne type: #GLHRepository opposite: #commits', '#short_id => FMProperty', @@ -81,12 +81,24 @@ GLHCommit class >> annotation [ ^ self ] +{ #category : #adding } +GLHCommit >> addChildCommit: anObject [ + + ^ self childCommits add: anObject +] + { #category : #adding } GLHCommit >> addDiff: anObject [ ^ self diffs add: anObject ] +{ #category : #adding } +GLHCommit >> addParentCommit: anObject [ + + ^ self parentCommits add: anObject +] + { #category : #accessing } GLHCommit >> additions [ @@ -168,19 +180,18 @@ GLHCommit >> branchGroup [ ] { #category : #accessing } -GLHCommit >> childCommit [ - "Relation named: #childCommit type: #GLHCommit opposite: #parentCommit" +GLHCommit >> childCommits [ + "Relation named: #childCommits type: #GLHCommit opposite: #parentCommits" - - ^ childCommit + ^ childCommits ] { #category : #accessing } -GLHCommit >> childCommit: anObject [ +GLHCommit >> childCommits: anObject [ - childCommit := anObject + childCommits value: anObject ] { #category : #accessing } @@ -306,18 +317,19 @@ GLHCommit >> message: anObject [ ] { #category : #accessing } -GLHCommit >> parentCommit [ - "Relation named: #parentCommit type: #GLHCommit opposite: #childCommit" +GLHCommit >> parentCommits [ + "Relation named: #parentCommits type: #GLHCommit opposite: #childCommits" - ^ parentCommit + + ^ parentCommits ] { #category : #accessing } -GLHCommit >> parentCommit: anObject [ +GLHCommit >> parentCommits: anObject [ - parentCommit := anObject + parentCommits value: anObject ] { #category : #accessing } diff --git a/src/GitLabHealth-Model/GLHTEntityCreator.trait.st b/src/GitLabHealth-Model/GLHTEntityCreator.trait.st index 84d43f3..518fc2c 100644 --- a/src/GitLabHealth-Model/GLHTEntityCreator.trait.st +++ b/src/GitLabHealth-Model/GLHTEntityCreator.trait.st @@ -108,3 +108,10 @@ GLHTEntityCreator >> newUser [ ^ self add: GLHUser new ] + +{ #category : #'entity creation' } +GLHTEntityCreator >> newUserNamed: aName [ + + + ^ self add: (GLHUser named: aName) +] diff --git a/src/GitLabHealth-Model/GLHUser.class.st b/src/GitLabHealth-Model/GLHUser.class.st index 3ad7340..f944512 100644 --- a/src/GitLabHealth-Model/GLHUser.class.st +++ b/src/GitLabHealth-Model/GLHUser.class.st @@ -26,6 +26,7 @@ A GitLab User | `linkedin` | `String` | nil | | | `location` | `String` | nil | | | `name` | `String` | nil | | +| `name` | `String` | nil | Basic name of the entity, not full reference.| | `organization` | `String` | nil | | | `pronouns` | `String` | nil | | | `public_email` | `String` | nil | | @@ -41,29 +42,31 @@ A GitLab User Class { #name : #GLHUser, #superclass : #GLHEntity, + #traits : 'FamixTNamedEntity', + #classTraits : 'FamixTNamedEntity classTrait', #instVars : [ - '#id => FMProperty', - '#username => FMProperty', - '#name => FMProperty', - '#state => FMProperty', '#avatar_url => FMProperty', - '#web_url => FMProperty', - '#created_at => FMProperty', '#bio => FMProperty', + '#bot => FMProperty', + '#createdProjects => FMMany type: #GLHProject opposite: #creator', + '#created_at => FMProperty', + '#followers => FMProperty', + '#following => FMProperty', + '#id => FMProperty', + '#job_title => FMProperty', + '#linkedin => FMProperty', '#location => FMProperty', + '#name => FMProperty', + '#organization => FMProperty', + '#pronouns => FMProperty', '#public_email => FMProperty', '#skype => FMProperty', - '#linkedin => FMProperty', + '#state => FMProperty', '#twitter => FMProperty', + '#username => FMProperty', + '#web_url => FMProperty', '#website_url => FMProperty', - '#organization => FMProperty', - '#job_title => FMProperty', - '#pronouns => FMProperty', - '#bot => FMProperty', - '#work_information => FMProperty', - '#followers => FMProperty', - '#following => FMProperty', - '#createdProjects => FMMany type: #GLHProject opposite: #creator' + '#work_information => FMProperty' ], #category : #'GitLabHealth-Model-Entities' } diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 4545b9f..642ea5d 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -63,6 +63,27 @@ GLPHModelImporter >> importChangesForDiff: aGLHDiff [ ^ aGLHDiff diffRanges ] +{ #category : #commit } +GLPHModelImporter >> importCommitsChainFromCommit: aCollection [ + + aCollection do: [ :commit | + | model| + + + model := commit mooseModel. + commit parent_ids do: [:parentId | + |parentCommit| + parentCommit := (model allWithType: GLHCommit) detect: [:c | c id = parentId] ifNone: [nil]. + parentCommit ifNotNil: [ + + commit parentCommit: parentCommit. + parentCommit childCommit: commit. + ]. + ] + + ] +] + { #category : #import } GLPHModelImporter >> importCommitsOfBranch: aGLHBranch [ @@ -73,12 +94,18 @@ GLPHModelImporter >> importCommitsOfBranch: aGLHBranch [ commits := self importCommitsOfBranch: aGLHBranch forRefName: aGLHBranch name - since: Date today - 3 day. + since: Date today - 1 week. + + + self importCommitsChainFromCommit: commits. + commits do: [ :aCommit | | diffs | 1 halt. + aCommit repository: aGLHBranch repository. + ('Import diff of commit: ' , aCommit short_id printString) recordInfo. diffs := self importDiffOfCommit: aCommit. @@ -247,23 +274,29 @@ GLPHModelImporter >> initialize [ { #category : #parsing } GLPHModelImporter >> parseDiffString: aDiff [ + "parse diff string to create entities for each lines" | lines range index | lines := aDiff diffString lines. index := 0. + range := nil. lines do: [ :line | | aChange | - aChange := GLPHEChange newFrom: line. + (line trim beginsWith: #'@@') ifTrue: [ range := GLPHEDiffRange newFrom: line. range diff: aDiff. aDiff mooseModel add: range ]. - + + self withCommitDiffs ifTrue: [ + aChange := GLPHEChange newFrom: line. aChange diffRange: range. aChange relativeIndex: index. index := index + 1. aDiff mooseModel add: aChange. - aChange ]. + ]. + + ]. ] { #category : #parsing } From 676276ab83c2a8071fd6a1a0f8ee628aa2fbf9b6 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Mon, 8 Apr 2024 11:42:29 +0200 Subject: [PATCH 15/77] add commit relation to one another --- .../GLHMetamodelGenerator.class.st | 2 +- .../GLPHModelImporter.class.st | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st b/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st index 535ee9e..7b2df19 100644 --- a/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st +++ b/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st @@ -157,7 +157,7 @@ GLHMetamodelGenerator >> defineRelations [ (branch property: #commits) <>-* (commit property: #branch). (commit property: #diffs) <>-* (diff property: #commit). - "(user property: #commits) <>-* (commit property: #commiter)." + (user property: #commits) <>-* (commit property: #commitCreator). (commit property: #parentCommits) *-* (commit property: #childCommits) ] diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 642ea5d..acab057 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -76,8 +76,8 @@ GLPHModelImporter >> importCommitsChainFromCommit: aCollection [ parentCommit := (model allWithType: GLHCommit) detect: [:c | c id = parentId] ifNone: [nil]. parentCommit ifNotNil: [ - commit parentCommit: parentCommit. - parentCommit childCommit: commit. + commit parentCommits add: parentCommit. + parentCommit childCommits add: commit. ]. ] @@ -103,7 +103,7 @@ GLPHModelImporter >> importCommitsOfBranch: aGLHBranch [ commits do: [ :aCommit | | diffs | - 1 halt. + aCommit repository: aGLHBranch repository. ('Import diff of commit: ' , aCommit short_id printString) From cab4c4421b2f5c8b517f9d4cf87c288f08b377bd Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Mon, 8 Apr 2024 12:31:32 +0200 Subject: [PATCH 16/77] add commit chain and correct some bug --- .../GLHCommit.extension.st | 9 +++++ .../GLHMetamodelGenerator.class.st | 1 + src/GitLabHealth-Model/GLHCommit.class.st | 28 ++++++++++++++ .../GLHTEntityCreator.trait.st | 7 ++++ src/GitLabHealth-Model/GLHUser.class.st | 37 ++++++++++++++++++- .../GLPHModelImporter.class.st | 8 ++-- 6 files changed, 85 insertions(+), 5 deletions(-) diff --git a/src/GitLabHealth-Model-Extension/GLHCommit.extension.st b/src/GitLabHealth-Model-Extension/GLHCommit.extension.st index d458eb9..a9c2664 100644 --- a/src/GitLabHealth-Model-Extension/GLHCommit.extension.st +++ b/src/GitLabHealth-Model-Extension/GLHCommit.extension.st @@ -1,5 +1,14 @@ Extension { #name : #GLHCommit } +{ #category : #'*GitLabHealth-Model-Extension' } +GLHCommit >> name [ + + + + + ^ title ifNil: [ 'commit #', short_id ]. +] + { #category : #'*GitLabHealth-Model-Extension' } GLHCommit >> parent_ids [ diff --git a/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st b/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st index 7b2df19..7db42f1 100644 --- a/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st +++ b/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st @@ -103,6 +103,7 @@ GLHMetamodelGenerator >> defineHierarchy [ group --|> #TNamedEntity. project --|> #TNamedEntity. user --|> #TNamedEntity. + commit --|> #TNamedEntity. ] { #category : #definition } diff --git a/src/GitLabHealth-Model/GLHCommit.class.st b/src/GitLabHealth-Model/GLHCommit.class.st index cd8ae19..d74f66a 100644 --- a/src/GitLabHealth-Model/GLHCommit.class.st +++ b/src/GitLabHealth-Model/GLHCommit.class.st @@ -8,6 +8,7 @@ a commit attached to a repository | Relation | Origin | Opposite | Type | Comment | |---| | `branch` | `GLHCommit` | `commits` | `GLHBranch` | | +| `commitCreator` | `GLHCommit` | `commits` | `GLHUser` | | | `repository` | `GLHCommit` | `commits` | `GLHRepository` | | ### Children @@ -38,6 +39,7 @@ a commit attached to a repository | `deletions` | `Number` | nil | The number of line that git consider deleted| | `id` | `String` | nil | | | `message` | `String` | nil | | +| `name` | `String` | nil | Basic name of the entity, not full reference.| | `parent_ids` | `Object` | 'OrderedCollection new' | | | `short_id` | `String` | nil | | | `title` | `String` | nil | | @@ -47,6 +49,8 @@ a commit attached to a repository Class { #name : #GLHCommit, #superclass : #GLHEntity, + #traits : 'FamixTNamedEntity', + #classTraits : 'FamixTNamedEntity classTrait', #instVars : [ '#additions => FMProperty', '#author_email => FMProperty', @@ -54,6 +58,7 @@ Class { '#authored_date => FMProperty', '#branch => FMOne type: #GLHBranch opposite: #commits', '#childCommits => FMMany type: #GLHCommit opposite: #parentCommits', + '#commitCreator => FMOne type: #GLHUser opposite: #commits', '#committed_date => FMProperty', '#committer_email => FMProperty', '#committer_name => FMProperty', @@ -194,6 +199,29 @@ GLHCommit >> childCommits: anObject [ childCommits value: anObject ] +{ #category : #accessing } +GLHCommit >> commitCreator [ + "Relation named: #commitCreator type: #GLHUser opposite: #commits" + + + + ^ commitCreator +] + +{ #category : #accessing } +GLHCommit >> commitCreator: anObject [ + + + commitCreator := anObject +] + +{ #category : #navigation } +GLHCommit >> commitCreatorGroup [ + + + ^ MooseSpecializedGroup with: self commitCreator +] + { #category : #accessing } GLHCommit >> committed_date [ diff --git a/src/GitLabHealth-Model/GLHTEntityCreator.trait.st b/src/GitLabHealth-Model/GLHTEntityCreator.trait.st index 518fc2c..5384b91 100644 --- a/src/GitLabHealth-Model/GLHTEntityCreator.trait.st +++ b/src/GitLabHealth-Model/GLHTEntityCreator.trait.st @@ -32,6 +32,13 @@ GLHTEntityCreator >> newCommit [ ^ self add: GLHCommit new ] +{ #category : #'entity creation' } +GLHTEntityCreator >> newCommitNamed: aName [ + + + ^ self add: (GLHCommit named: aName) +] + { #category : #'entity creation' } GLHTEntityCreator >> newDiff [ diff --git a/src/GitLabHealth-Model/GLHUser.class.st b/src/GitLabHealth-Model/GLHUser.class.st index f944512..7a212f1 100644 --- a/src/GitLabHealth-Model/GLHUser.class.st +++ b/src/GitLabHealth-Model/GLHUser.class.st @@ -4,6 +4,11 @@ A GitLab User ## Relations ====================== +### Children +| Relation | Origin | Opposite | Type | Comment | +|---| +| `commits` | `GLHUser` | `commitCreator` | `GLHCommit` | | + ### Other | Relation | Origin | Opposite | Type | Comment | |---| @@ -25,8 +30,8 @@ A GitLab User | `job_title` | `String` | nil | | | `linkedin` | `String` | nil | | | `location` | `String` | nil | | -| `name` | `String` | nil | | | `name` | `String` | nil | Basic name of the entity, not full reference.| +| `name` | `String` | nil | | | `organization` | `String` | nil | | | `pronouns` | `String` | nil | | | `public_email` | `String` | nil | | @@ -48,6 +53,7 @@ Class { '#avatar_url => FMProperty', '#bio => FMProperty', '#bot => FMProperty', + '#commits => FMMany type: #GLHCommit opposite: #commitCreator', '#createdProjects => FMMany type: #GLHProject opposite: #creator', '#created_at => FMProperty', '#followers => FMProperty', @@ -80,6 +86,12 @@ GLHUser class >> annotation [ ^ self ] +{ #category : #adding } +GLHUser >> addCommit: anObject [ + + ^ self commits add: anObject +] + { #category : #adding } GLHUser >> addCreatedProject: anObject [ @@ -128,6 +140,29 @@ GLHUser >> bot: anObject [ bot := anObject ] +{ #category : #accessing } +GLHUser >> commits [ + "Relation named: #commits type: #GLHCommit opposite: #commitCreator" + + + + ^ commits +] + +{ #category : #accessing } +GLHUser >> commits: anObject [ + + + commits value: anObject +] + +{ #category : #navigation } +GLHUser >> commitsGroup [ + + + ^ MooseSpecializedGroup withAll: self commits asSet +] + { #category : #accessing } GLHUser >> createdProjects [ "Relation named: #createdProjects type: #GLHProject opposite: #creator" diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index acab057..23190f2 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -77,7 +77,7 @@ GLPHModelImporter >> importCommitsChainFromCommit: aCollection [ parentCommit ifNotNil: [ commit parentCommits add: parentCommit. - parentCommit childCommits add: commit. + "self haltIf: [commit childCommits size = 2]." ]. ] @@ -109,7 +109,6 @@ GLPHModelImporter >> importCommitsOfBranch: aGLHBranch [ ('Import diff of commit: ' , aCommit short_id printString) recordInfo. diffs := self importDiffOfCommit: aCommit. - aCommit diffs addAll: diffs. aCommit mooseModel addAll: diffs. diffs do: [ :diff | "changes are added into the model during the import" @@ -285,8 +284,9 @@ GLPHModelImporter >> parseDiffString: aDiff [ (line trim beginsWith: #'@@') ifTrue: [ range := GLPHEDiffRange newFrom: line. - range diff: aDiff. - aDiff mooseModel add: range ]. + self glhModel add: range. + aDiff diffRanges add: range. + ]. self withCommitDiffs ifTrue: [ aChange := GLPHEChange newFrom: line. From 3368a9d3d6f0f8db70468d57d1818563081636bc Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Tue, 9 Apr 2024 09:31:05 +0200 Subject: [PATCH 17/77] finalize commits chain import --- .../GLPHModelImporter.class.st | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 23190f2..47dbb68 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -67,17 +67,11 @@ GLPHModelImporter >> importChangesForDiff: aGLHDiff [ GLPHModelImporter >> importCommitsChainFromCommit: aCollection [ aCollection do: [ :commit | - | model| - - - model := commit mooseModel. commit parent_ids do: [:parentId | |parentCommit| - parentCommit := (model allWithType: GLHCommit) detect: [:c | c id = parentId] ifNone: [nil]. + parentCommit := (self glhModel allWithType: GLHCommit) detect: [:c | c id = parentId] ifNone: [nil]. parentCommit ifNotNil: [ - - commit parentCommits add: parentCommit. - "self haltIf: [commit childCommits size = 2]." + parentCommit childCommits detect: [ :c | c id = commit id ] ifNone: [parentCommit addChildCommit: commit.]. ]. ] From 63569572a31887cbe3ec522d06592c088e0d8c61 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Tue, 9 Apr 2024 09:59:26 +0200 Subject: [PATCH 18/77] start work on chrun analysis --- .../CodeChurnAnalyzer.class.st | 15 +++++++ src/GitLabHealth-Model-Analysis/package.st | 1 + .../GLHModelImporter.class.st | 5 ++- .../GLPHModelImporter.class.st | 44 +++++++++++-------- 4 files changed, 44 insertions(+), 21 deletions(-) create mode 100644 src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st create mode 100644 src/GitLabHealth-Model-Analysis/package.st diff --git a/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st new file mode 100644 index 0000000..e9e0914 --- /dev/null +++ b/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st @@ -0,0 +1,15 @@ +Class { + #name : #CodeChurnAnalyzer, + #superclass : #Object, + #instVars : [ + 'glModel', + 'fromCommit' + ], + #category : #'GitLabHealth-Model-Analysis' +} + +{ #category : #initialization } +CodeChurnAnalyzer >> initialize [ + glModel := GLPHEModel new. + fromCommit := GLHCommit new. +] diff --git a/src/GitLabHealth-Model-Analysis/package.st b/src/GitLabHealth-Model-Analysis/package.st new file mode 100644 index 0000000..b7288f0 --- /dev/null +++ b/src/GitLabHealth-Model-Analysis/package.st @@ -0,0 +1 @@ +Package { #name : #'GitLabHealth-Model-Analysis' } diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index fbd4bd4..839ca06 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -5,7 +5,8 @@ Class { 'glhModel', 'glhApi', 'withFiles', - 'withCommitDiffs' + 'withCommitDiffs', + 'withCommitsSince' ], #category : #'GitLabHealth-Model-Importer' } @@ -232,7 +233,7 @@ GLHModelImporter >> importRepository: aGLHRepository [ resultBranches := self glhApi branchesOfRepository: aGLHRepository project id. branches := self parseBranchesResult: resultBranches. - branches do: [ :branch | aGLHRepository addBranch: branch. aGLHRepository mooseModel add: branch. ]. + branches do: [ :branch | aGLHRepository addBranch: branch. self glhModel add: branch. ]. self withFiles ifTrue: [ branches do: [ :branch | self importFilesOfBranch: branch ] ] ] on: NeoJSONParseError diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 47dbb68..ff538e1 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -4,6 +4,21 @@ Class { #category : #'GitLabProjectHealth-Model-Importer' } +{ #category : #commit } +GLPHModelImporter >> chainsCommitsFrom: commitsCollection [ + + commitsCollection do: [ :commit | + commit parent_ids do: [ :parentId | + | parentCommit | + parentCommit := (self glhModel allWithType: GLHCommit) + detect: [ :c | c id = parentId ] + ifNone: [ nil ]. + parentCommit ifNotNil: [ + parentCommit childCommits + detect: [ :c | c id = commit id ] + ifNone: [ parentCommit addChildCommit: commit ] ] ] ] +] + { #category : #commit } GLPHModelImporter >> commitsOfProject: aGLHProject forRefName: refName until: toDate [ @@ -63,21 +78,6 @@ GLPHModelImporter >> importChangesForDiff: aGLHDiff [ ^ aGLHDiff diffRanges ] -{ #category : #commit } -GLPHModelImporter >> importCommitsChainFromCommit: aCollection [ - - aCollection do: [ :commit | - commit parent_ids do: [:parentId | - |parentCommit| - parentCommit := (self glhModel allWithType: GLHCommit) detect: [:c | c id = parentId] ifNone: [nil]. - parentCommit ifNotNil: [ - parentCommit childCommits detect: [ :c | c id = commit id ] ifNone: [parentCommit addChildCommit: commit.]. - ]. - ] - - ] -] - { #category : #import } GLPHModelImporter >> importCommitsOfBranch: aGLHBranch [ @@ -88,12 +88,10 @@ GLPHModelImporter >> importCommitsOfBranch: aGLHBranch [ commits := self importCommitsOfBranch: aGLHBranch forRefName: aGLHBranch name - since: Date today - 1 week. - - + since: withCommitsSince. - self importCommitsChainFromCommit: commits. + self chainsCommitsFrom: commits. commits do: [ :aCommit | | diffs | @@ -387,3 +385,11 @@ GLPHModelImporter >> parseProjectResult: aResult [ ^ reader nextAs: GLHProject ] + +{ #category : #accessing } +GLPHModelImporter >> withCommitsSince: someDays [ + "substract the current date with a given number of days. Use to retrieve the commits submit in the last giving days" + "(GLPHModelImporter new withCommitsSince: 1 week ) >>> (Date today - 1 week) asDateAndTime" + "(GLPHModelImporter new withCommitsSince: 30 day ) >>> (Date today - 30 day) asDateAndTime" + withCommitsSince := (Date today - someDays) asDateAndTime +] From 27aa8cc56f4d49bed8a67ea6451922a08707b7cd Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Tue, 9 Apr 2024 17:07:20 +0200 Subject: [PATCH 19/77] add checker on instances add to the model --- .../FMSlotMultivalueLink.extension.st | 17 ++++++ .../MooseAbstractGroup.extension.st | 28 ++++++++++ .../GLHApi.class.st | 6 +++ .../GLHModelImporter.class.st | 5 +- .../GLPHApi.class.st | 6 +++ .../GLPHModelImporter.class.st | 52 ++++++++++++++++--- 6 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 src/GitLabHealth-Model-Extension/FMSlotMultivalueLink.extension.st create mode 100644 src/GitLabHealth-Model-Extension/MooseAbstractGroup.extension.st diff --git a/src/GitLabHealth-Model-Extension/FMSlotMultivalueLink.extension.st b/src/GitLabHealth-Model-Extension/FMSlotMultivalueLink.extension.st new file mode 100644 index 0000000..0dca839 --- /dev/null +++ b/src/GitLabHealth-Model-Extension/FMSlotMultivalueLink.extension.st @@ -0,0 +1,17 @@ +Extension { #name : #FMSlotMultivalueLink } + +{ #category : #'*GitLabHealth-Model-Extension' } +FMSlotMultivalueLink >> add: anElmt unless: aConditionAsBlock [ + "()>>>" + + self addAll: { anElmt } unless: aConditionAsBlock +] + +{ #category : #'*GitLabHealth-Model-Extension' } +FMSlotMultivalueLink >> addAll: anElmtCollection unless: aConditionAsBlock [ + "()>>>" + + self addAll: (anElmtCollection reject: [ :anElmt | + self anySatisfy: [ + :existingElmt | aConditionAsBlock value: existingElmt value: anElmt ] ]) +] diff --git a/src/GitLabHealth-Model-Extension/MooseAbstractGroup.extension.st b/src/GitLabHealth-Model-Extension/MooseAbstractGroup.extension.st new file mode 100644 index 0000000..6817f5f --- /dev/null +++ b/src/GitLabHealth-Model-Extension/MooseAbstractGroup.extension.st @@ -0,0 +1,28 @@ +Extension { #name : #MooseAbstractGroup } + +{ #category : #'*GitLabHealth-Model-Extension' } +MooseAbstractGroup >> add: anElmt unless: aConditionAsBlock [ + "()>>>" + + self addAll: { anElmt } unless: aConditionAsBlock +] + +{ #category : #'*GitLabHealth-Model-Extension' } +MooseAbstractGroup >> addAll: anElmtCollection ofType: aType unless: aConditionAsBlock [ + "()>>>" + + self addAll: (anElmtCollection reject: [ :anElmt | + (self allWithType: aType) anySatisfy: [ :existingElmt | + aConditionAsBlock value: existingElmt value: anElmt ] ]) +] + +{ #category : #'*GitLabHealth-Model-Extension' } +MooseAbstractGroup >> addAll: anElmtCollection unless: aConditionAsBlock [ + "()>>>" + + anElmtCollection ifEmpty: [ ^ self ]. + self + addAll: anElmtCollection + ofType: anElmtCollection first class + unless: aConditionAsBlock. +] diff --git a/src/GitLabHealth-Model-Importer/GLHApi.class.st b/src/GitLabHealth-Model-Importer/GLHApi.class.st index b47e0cc..7e3129e 100644 --- a/src/GitLabHealth-Model-Importer/GLHApi.class.st +++ b/src/GitLabHealth-Model-Importer/GLHApi.class.st @@ -261,3 +261,9 @@ GLHApi >> user: aUserID [ ^ self client get: self baseAPIUrl , '/users/' , aUserID printString ] + +{ #category : #api } +GLHApi >> userWithUsername: anUsername [ + + ^ self client get: self baseAPIUrl , '/users?username=' , anUsername +] diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index 839ca06..1b3eee6 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -165,7 +165,7 @@ GLHModelImporter >> importDiffOfCommit: aCommit [ unidiff: true. diffsResult := self parseDiffResult: result. aCommit diffs addAll: diffsResult. - aCommit mooseModel addAll: diffsResult. + self glhModel addAll: diffsResult. ^ diffsResult ] @@ -258,7 +258,8 @@ GLHModelImporter >> importUser: aUserID [ GLHModelImporter >> initialize [ withFiles := false. - withCommitDiffs := false + withCommitDiffs := false. + withCommitsSince := (Date today - 1 week) asDateAndTime. ] { #category : #private } diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st index 2d9135e..d616b4e 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st @@ -117,3 +117,9 @@ GLPHApi >> project: aProjectID [ ^ self client get: self baseAPIUrl , '/projects/' , aProjectID printString ] + +{ #category : #user } +GLPHApi >> usersSearchByUsername: aUserName [ + + ^ self client get: self baseAPIUrl , '/users?search=' , (aUserName replaceAllRegex: ' ' with: '+'). +] diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index ff538e1..19a296d 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -90,18 +90,31 @@ GLPHModelImporter >> importCommitsOfBranch: aGLHBranch [ forRefName: aGLHBranch name since: withCommitsSince. - - self chainsCommitsFrom: commits. + + self glhModel addAll: (commits reject: [ :commitParsed | + (self glhModel allWithType: GLHCommit) anySatisfy: [ + :existingCommit | existingCommit id = commitParsed id ] ]). + + + self chainsCommitsFrom: commits. + + commits do: [ :aCommit | | diffs | + (self importUserByUsername: aCommit author_name) commits + add: aCommit + unless: [ :existingCommit :newCommit | + existingCommit id = newCommit id ]. + + "(self importUserByUsername: aCommit author_name) addCommit: aCommit." aCommit repository: aGLHBranch repository. - + ('Import diff of commit: ' , aCommit short_id printString) recordInfo. diffs := self importDiffOfCommit: aCommit. - aCommit mooseModel addAll: diffs. + self glhModel addAll: diffs. diffs do: [ :diff | "changes are added into the model during the import" self importChangesForDiff: diff ] ] @@ -139,8 +152,6 @@ GLPHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName since newlyFoundCommit := parsedResults reject: [ :commitParsed | existingCommits anySatisfy: [ :existingCommit | existingCommit id = commitParsed id ] ]. - - aGLHBranch mooseModel addAll: newlyFoundCommit. aGLHBranch commits addAll: newlyFoundCommit ]. " (aGLHBranch commits reject: #isNil) do: [ :commit | @@ -187,7 +198,6 @@ GLPHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName until "newlyFoundCommit := self parseCommitsResult: results." - aGLHBranch mooseModel addAll: newlyFoundCommit. aGLHBranch commits addAll: newlyFoundCommit ]. aGLHBranch commits do: [ :commit | @@ -256,6 +266,34 @@ GLPHModelImporter >> importRepository: aGLHRepository [ self importCommitsOfBranch: branch ]. ] +{ #category : #user } +GLPHModelImporter >> importUserByEmail: anUsername [ + ^ (self glhModel allWithType: GLHUser) detect: [ :user | user username = anUsername ] ifNone: [ + + ] +] + +{ #category : #user } +GLPHModelImporter >> importUserByUsername: anUsername [ + + ^ (self glhModel allWithType: GLHUser) + detect: [ :user | user username = anUsername ] + ifNone: [ + | result userResult userId searchResult | + ('Import user with username: ' , anUsername printString) + recordInfo. + result := self glhApi usersSearchByUsername: anUsername. + searchResult := NeoJSONReader fromString: result. + searchResult ifEmpty: [ GLHUser new ] ifNotEmpty: [ + userId := searchResult first at: #id. + (self glhModel allWithType: GLHUser) + detect: [ :user | user id = userId ] + ifNone: [ + userResult := self parseUserResult: (self glhApi user: userId). + glhModel add: userResult. + userResult ] ] ] +] + { #category : #initialization } GLPHModelImporter >> initialize [ From 33a98b43cc1d77d22d014646c5c527f5f596b8cc Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Wed, 10 Apr 2024 12:17:28 +0200 Subject: [PATCH 20/77] rework add:unless and commit import. --- ...sion.st => OrderedCollection.extension.st} | 6 +- .../GLHModelImporter.class.st | 23 +- .../GLPHModelImporter.class.st | 196 +++++++++--------- 3 files changed, 117 insertions(+), 108 deletions(-) rename src/GitLabHealth-Model-Extension/{FMSlotMultivalueLink.extension.st => OrderedCollection.extension.st} (64%) diff --git a/src/GitLabHealth-Model-Extension/FMSlotMultivalueLink.extension.st b/src/GitLabHealth-Model-Extension/OrderedCollection.extension.st similarity index 64% rename from src/GitLabHealth-Model-Extension/FMSlotMultivalueLink.extension.st rename to src/GitLabHealth-Model-Extension/OrderedCollection.extension.st index 0dca839..2127d74 100644 --- a/src/GitLabHealth-Model-Extension/FMSlotMultivalueLink.extension.st +++ b/src/GitLabHealth-Model-Extension/OrderedCollection.extension.st @@ -1,14 +1,14 @@ -Extension { #name : #FMSlotMultivalueLink } +Extension { #name : #OrderedCollection } { #category : #'*GitLabHealth-Model-Extension' } -FMSlotMultivalueLink >> add: anElmt unless: aConditionAsBlock [ +OrderedCollection >> add: anElmt unless: aConditionAsBlock [ "()>>>" self addAll: { anElmt } unless: aConditionAsBlock ] { #category : #'*GitLabHealth-Model-Extension' } -FMSlotMultivalueLink >> addAll: anElmtCollection unless: aConditionAsBlock [ +OrderedCollection >> addAll: anElmtCollection unless: aConditionAsBlock [ "()>>>" self addAll: (anElmtCollection reject: [ :anElmt | diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index 1b3eee6..b7c0d87 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -28,6 +28,20 @@ GLHModelImporter >> beWithouFiles [ withFiles := false ] +{ #category : #equality } +GLHModelImporter >> blockForDifffEquality [ + ^ [ :existing :new | + existing diffString size = new diffString size and: [ + existing diffString = new diffString ] ] +] + +{ #category : #equality } +GLHModelImporter >> blockForEntitiesWithIdEquality [ + + ^ [ :existing :new | + existing id = new id ] +] + { #category : #'private - api' } GLHModelImporter >> completeImportProject: aGLHProject [ @@ -155,8 +169,7 @@ GLHModelImporter >> importDiffOfCommit: aCommit [ | result diffsResult | aCommit diffs ifNotEmpty: [ - 'Diff already importer: ' - , aCommit short_id printString recordInfo. + 'Diff already importer: ' , aCommit short_id printString recordInfo. ^ aCommit diffs ]. ('Import diff of commit: ' , aCommit short_id printString) recordInfo. result := self glhApi @@ -164,9 +177,9 @@ GLHModelImporter >> importDiffOfCommit: aCommit [ ofProject: aCommit repository project id unidiff: true. diffsResult := self parseDiffResult: result. - aCommit diffs addAll: diffsResult. - self glhModel addAll: diffsResult. - ^ diffsResult + + aCommit diffs addAll: diffsResult unless: self blockForDifffEquality . + ^ aCommit diffs ] { #category : #'private - api' } diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 19a296d..2defa01 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -14,9 +14,12 @@ GLPHModelImporter >> chainsCommitsFrom: commitsCollection [ detect: [ :c | c id = parentId ] ifNone: [ nil ]. parentCommit ifNotNil: [ - parentCommit childCommits + + parentCommit childCommits add: commit unless: [ :existing :new | existing id = new id ]. + + "parentCommit childCommits detect: [ :c | c id = commit id ] - ifNone: [ parentCommit addChildCommit: commit ] ] ] ] + ifNone: [ parentCommit addChildCommit: commit ] "] ] ] ] { #category : #commit } @@ -69,15 +72,6 @@ GLPHModelImporter >> completeImportProject: aGLHProject [ ] -{ #category : #import } -GLPHModelImporter >> importChangesForDiff: aGLHDiff [ - - ('Import changes of diff ' , aGLHDiff printString) recordInfo. - self parseDiffString: aGLHDiff. - - ^ aGLHDiff diffRanges -] - { #category : #import } GLPHModelImporter >> importCommitsOfBranch: aGLHBranch [ @@ -90,93 +84,58 @@ GLPHModelImporter >> importCommitsOfBranch: aGLHBranch [ forRefName: aGLHBranch name since: withCommitsSince. - - self glhModel addAll: (commits reject: [ :commitParsed | - (self glhModel allWithType: GLHCommit) anySatisfy: [ - :existingCommit | existingCommit id = commitParsed id ] ]). - + self chainsCommitsFrom: commits. - - commits do: [ :aCommit | | diffs | - (self importUserByUsername: aCommit author_name) commits add: aCommit unless: [ :existingCommit :newCommit | existingCommit id = newCommit id ]. - "(self importUserByUsername: aCommit author_name) addCommit: aCommit." aCommit repository: aGLHBranch repository. ('Import diff of commit: ' , aCommit short_id printString) recordInfo. + diffs := self importDiffOfCommit: aCommit. - self glhModel addAll: diffs. + self glhModel addAll: diffs unless: [ :existing :new | + existing diffString size = new diffString size and: [ + existing diffString = new diffString ] ]. diffs do: [ :diff | "changes are added into the model during the import" - self importChangesForDiff: diff ] ] + self importDiffRangesForDiff: diff ] ] ] { #category : #commit } GLPHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName since: fromDate [ - | newlyFoundCommit page | - page := 0. - newlyFoundCommit := { true }. - [ newlyFoundCommit isNotEmpty ] whileTrue: [ - | results parsedResults existingCommits | - page := page + 1. - ('import commit page ' , page printString) recordInfo. - results := self glhApi - commitsOfProject: aGLHBranch repository project id - forRefName: aGLHBranch name - since: fromDate asDateAndTime asString - until: nil - path: nil - author: nil - all: nil - with_stats: nil - firstParent: nil - order: nil - trailers: nil - perPage: 100 - page: page. - - - parsedResults := self parseCommitsResult: results. - existingCommits := aGLHBranch commits. - - newlyFoundCommit := parsedResults reject: [ :commitParsed | - existingCommits anySatisfy: [ :existingCommit | - existingCommit id = commitParsed id ] ]. - aGLHBranch commits addAll: newlyFoundCommit ]. - - " (aGLHBranch commits reject: #isNil) do: [ :commit | - 1 halt. - commit repository: aGLHBranch repository. - commit repository project: aGLHBranch repository project ]. -" - ^ aGLHBranch commits + ^ self + importCommitsOfBranch: aGLHBranch + forRefName: refName + since: fromDate + until: nil ] { #category : #commit } -GLPHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName until: toDate [ +GLPHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName since: fromDate until: toDate [ | newlyFoundCommit page | page := 0. newlyFoundCommit := { true }. [ newlyFoundCommit isNotEmpty ] whileTrue: [ - | results parsedResults existingCommits | + | results | page := page + 1. ('import commit page ' , page printString) recordInfo. results := self glhApi commitsOfProject: aGLHBranch repository project id forRefName: aGLHBranch name - since: nil - until: toDate asDateAndTime asString + since: + (fromDate ifNotNil: [ fromDate asDateAndTime asString ]) + until: + (toDate ifNotNil: [ toDate asDateAndTime asString ]) path: nil author: nil all: nil @@ -187,48 +146,70 @@ GLPHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName until perPage: 100 page: page. + newlyFoundCommit := self parseCommitsResult: results. + aGLHBranch commits + addAll: newlyFoundCommit + unless: self blockForEntitiesWithIdEquality ]. - parsedResults := self parseCommitsResult: results. - existingCommits := aGLHBranch commits. - - - newlyFoundCommit := parsedResults reject: [ :commitParsed | - existingCommits anySatisfy: [ :existingCommit | - existingCommit id = commitParsed id ] ]. + self glhModel + addAll: aGLHBranch commits + unless: [ :old :new | old id = new id ]. - "newlyFoundCommit := self parseCommitsResult: results." + ^ aGLHBranch commits +] - aGLHBranch commits addAll: newlyFoundCommit ]. +{ #category : #commit } +GLPHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName until: toDate [ - aGLHBranch commits do: [ :commit | - commit repository: aGLHBranch repository. - commit repository project: aGLHBranch repository project ]. - ^ aGLHBranch commits + ^ self + importCommitsOfBranch: aGLHBranch + forRefName: refName + since: nil + until: toDate ] { #category : #'as yet unclassified' } GLPHModelImporter >> importDiffOfMergeRequest: aMergeRequest [ | result diffsResult | - aMergeRequest diffs ifNotEmpty: [ 'Diff of already importer: ' , aMergeRequest iid printString recordInfo. ^ aMergeRequest diffs ]. - ('Import diff of mergeRequest ' , aMergeRequest iid printString) recordInfo. + ('Import diff of mergeRequest ' , aMergeRequest iid printString) + recordInfo. result := self glhApi - diffsMergeRequestOfProject: (aMergeRequest project_id) + diffsMergeRequestOfProject: aMergeRequest project_id withId: aMergeRequest iid. - + diffsResult := self parseDiffResult: result. - aMergeRequest diffs addAll: diffsResult. - aMergeRequest mooseModel addAll: aMergeRequest diffs. - - aMergeRequest diffs do: [ :diff | - self importChangesForDiff: diff. - ]. + + + aMergeRequest diffs + addAll: diffsResult + unless: self blockForDifffEquality. + self glhModel + addAll: aMergeRequest diffs + unless: self blockForDifffEquality. + + aMergeRequest diffs do: [ :diff | self importDiffRangesForDiff: diff ]. + + ^ aMergeRequest diffs +] + +{ #category : #import } +GLPHModelImporter >> importDiffRangesForDiff: aGLHDiff [ + + | diffRanges | + ('Import changes of diff ' , aGLHDiff printString) recordInfo. + diffRanges := self parseDiffString: aGLHDiff. - ^ diffsResult + self glhModel addAll: diffRanges unless: [ :existing :new | + (existing diff mooseID = new diff mooseID and: [ + existing originalLineRange = new originalLineRange ]) and: [ + existing newLineRange = new newLineRange ] ]. + + ^ aGLHDiff diffRanges ] { #category : #import } @@ -311,22 +292,37 @@ GLPHModelImporter >> parseDiffString: aDiff [ range := nil. lines do: [ :line | | aChange | - (line trim beginsWith: #'@@') ifTrue: [ range := GLPHEDiffRange newFrom: line. - self glhModel add: range. - aDiff diffRanges add: range. - ]. - + + aDiff diffRanges add: range unless: [ :existing :new | + existing originalLineRange = new originalLineRange and: [ + existing newLineRange = new newLineRange ] ] + + " self glhModel + add: range + unless: [:existing :new | ((existing diff mooseID = new diff mooseID) + and: [existing originalLineRange = new originalLineRange]) + and: [ existing newLineRange = new newLineRange ] + ]." ]. + self withCommitDiffs ifTrue: [ - aChange := GLPHEChange newFrom: line. - aChange diffRange: range. - aChange relativeIndex: index. - index := index + 1. - aDiff mooseModel add: aChange. - ]. - - ]. + aChange := GLPHEChange newFrom: line. + aChange relativeIndex: index. + range changes add: aChange unless: [ :existing :new | + existing sourceCode = new sourceCode and: [ + existing index = new index ] ]. + "self glhModel add: aChange ." + index := index + 1 ] ]. + + + aDiff diffRanges do: [ :diffRange | + self glhModel addAll: diffRange changes unless: [ :existing :new | + (existing diffRange mooseID = new diffRange mooseID and: [ + existing sourceCode = new sourceCode ]) and: [ + existing index = new index ] ] ]. + +^ aDiff diffRanges. ] { #category : #parsing } From 9f691eaa515655697908991670ceaf44ffe286e8 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Wed, 10 Apr 2024 14:39:08 +0200 Subject: [PATCH 21/77] add convinient block for equality checking --- .../GLHModelImporter.class.st | 50 +++++++++++++------ .../GLPHModelImporter.class.st | 25 ++++------ 2 files changed, 45 insertions(+), 30 deletions(-) diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index b7c0d87..b36c7d2 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -28,20 +28,32 @@ GLHModelImporter >> beWithouFiles [ withFiles := false ] +{ #category : #'as yet unclassified' } +GLHModelImporter >> blockEqualityOn: aSymbol [ + ^ [ :existing :new | + (existing perform: aSymbol) = (new perform: aSymbol) ] +] + { #category : #equality } -GLHModelImporter >> blockForDifffEquality [ +GLHModelImporter >> blockForDiffEquality [ ^ [ :existing :new | existing diffString size = new diffString size and: [ existing diffString = new diffString ] ] ] { #category : #equality } -GLHModelImporter >> blockForEntitiesWithIdEquality [ +GLHModelImporter >> blockOnIdEquality [ ^ [ :existing :new | existing id = new id ] ] +{ #category : #equality } +GLHModelImporter >> blockOnNameEquality [ + + ^ self blockEqualityOn: #name +] + { #category : #'private - api' } GLHModelImporter >> completeImportProject: aGLHProject [ @@ -178,7 +190,7 @@ GLHModelImporter >> importDiffOfCommit: aCommit [ unidiff: true. diffsResult := self parseDiffResult: result. - aCommit diffs addAll: diffsResult unless: self blockForDifffEquality . + aCommit diffs addAll: diffsResult unless: self blockForDiffEquality . ^ aCommit diffs ] @@ -191,14 +203,15 @@ GLHModelImporter >> importDirectoryFiles: aDirectoryFile OfBranch: aBranch [ ofBranch: aBranch name andPath: aDirectoryFile path , '/'. apiFiles := self parseFileTreeResult: result. - files := apiFiles collect: [ :apiFile | + files := apiFiles collect: [ :apiFile | self convertApiFileAsFile: apiFile ]. - files do: [ :file | + files do: [ :file | + self glhModel add: file. aDirectoryFile addFile: file ]. files select: [ :file | file isKindOf: GLHFileDirectory ] - thenCollect: [ :file | + thenCollect: [ :file | self importDirectoryFiles: file OfBranch: aBranch ] ] @@ -246,7 +259,15 @@ GLHModelImporter >> importRepository: aGLHRepository [ resultBranches := self glhApi branchesOfRepository: aGLHRepository project id. branches := self parseBranchesResult: resultBranches. - branches do: [ :branch | aGLHRepository addBranch: branch. self glhModel add: branch. ]. + + + aGLHRepository branches addAll: branches unless: self blockOnNameEquality. + self glhModel addAll: branches unless: self blockOnNameEquality. + + "branches do: [ :branch | + aGLHRepository addBranch: branch. + self glhModel add: branch ]." + self withFiles ifTrue: [ branches do: [ :branch | self importFilesOfBranch: branch ] ] ] on: NeoJSONParseError @@ -258,13 +279,14 @@ GLHModelImporter >> importRepository: aGLHRepository [ GLHModelImporter >> importUser: aUserID [ | result userResult | - ^ (glhModel allWithType: GLHUser) detect: [ :user | user id = aUserID ] ifNone: [ - ('Import user: ' , aUserID printString) recordInfo. - result := self glhApi user: aUserID. - userResult := self parseUserResult: result. - glhModel add: userResult. - userResult ]. - + ^ (glhModel allWithType: GLHUser) + detect: [ :user | user id = aUserID ] + ifNone: [ + ('Import user: ' , aUserID printString) recordInfo. + result := self glhApi user: aUserID. + userResult := self parseUserResult: result. + glhModel add: userResult unless: self blockOnIdEquality. + userResult ] ] { #category : #initialization } diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 2defa01..3f7f721 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -149,7 +149,7 @@ GLPHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName since newlyFoundCommit := self parseCommitsResult: results. aGLHBranch commits addAll: newlyFoundCommit - unless: self blockForEntitiesWithIdEquality ]. + unless: (self blockEqualityOn: #id) ]. self glhModel addAll: aGLHBranch commits @@ -187,10 +187,10 @@ GLPHModelImporter >> importDiffOfMergeRequest: aMergeRequest [ aMergeRequest diffs addAll: diffsResult - unless: self blockForDifffEquality. + unless: self blockForDiffEquality. self glhModel addAll: aMergeRequest diffs - unless: self blockForDifffEquality. + unless: self blockForDiffEquality. aMergeRequest diffs do: [ :diff | self importDiffRangesForDiff: diff ]. @@ -219,8 +219,9 @@ GLPHModelImporter >> importMergeRequests: aGLHProject [ results := self glhApi mergeRequestOfProject: aGLHProject id. parsedResults := self parseMergeRequestResult: results. - aGLHProject mergeRequests addAll: parsedResults. - aGLHProject mooseModel addAll: aGLHProject mergeRequests. + 1 halt. + aGLHProject mergeRequests addAll: parsedResults unless: self blockOnIdEquality . + self glhModel addAll: aGLHProject mergeRequests unless: self blockOnIdEquality. self withCommitDiffs ifTrue: [ aGLHProject mergeRequests do: [ :mr | @@ -247,20 +248,13 @@ GLPHModelImporter >> importRepository: aGLHRepository [ self importCommitsOfBranch: branch ]. ] -{ #category : #user } -GLPHModelImporter >> importUserByEmail: anUsername [ - ^ (self glhModel allWithType: GLHUser) detect: [ :user | user username = anUsername ] ifNone: [ - - ] -] - { #category : #user } GLPHModelImporter >> importUserByUsername: anUsername [ ^ (self glhModel allWithType: GLHUser) detect: [ :user | user username = anUsername ] ifNone: [ - | result userResult userId searchResult | + | result userId searchResult | ('Import user with username: ' , anUsername printString) recordInfo. result := self glhApi usersSearchByUsername: anUsername. @@ -270,9 +264,8 @@ GLPHModelImporter >> importUserByUsername: anUsername [ (self glhModel allWithType: GLHUser) detect: [ :user | user id = userId ] ifNone: [ - userResult := self parseUserResult: (self glhApi user: userId). - glhModel add: userResult. - userResult ] ] ] + self importUser: userId. + ] ] ] ] { #category : #initialization } From 6c3edef6313132af2d2da7ec76be5cdd63d870e1 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 11 Apr 2024 15:39:53 +0200 Subject: [PATCH 22/77] add inspection for commits --- .../GLPHEMergeRequest.extension.st | 10 ++++ src/GLPHExtended-Model/GLHCommit.extension.st | 8 +-- .../GLPHEMergeRequest.class.st | 47 +++++++++++++---- .../GLPHETEntityCreator.trait.st | 7 +++ .../GLHCommit.extension.st | 12 ++++- .../GLHApi.class.st | 15 ++++++ .../GLPHMetamodelGenerator.class.st | 9 ++-- .../GLPHModelImporter.class.st | 50 +++++++++++++++++-- 8 files changed, 136 insertions(+), 22 deletions(-) create mode 100644 src/GLPHExtended-Model-Extension/GLPHEMergeRequest.extension.st diff --git a/src/GLPHExtended-Model-Extension/GLPHEMergeRequest.extension.st b/src/GLPHExtended-Model-Extension/GLPHEMergeRequest.extension.st new file mode 100644 index 0000000..65292e2 --- /dev/null +++ b/src/GLPHExtended-Model-Extension/GLPHEMergeRequest.extension.st @@ -0,0 +1,10 @@ +Extension { #name : #GLPHEMergeRequest } + +{ #category : #'*GLPHExtended-Model-Extension' } +GLPHEMergeRequest >> name [ + + + + + ^ title +] diff --git a/src/GLPHExtended-Model/GLHCommit.extension.st b/src/GLPHExtended-Model/GLHCommit.extension.st index 1e3f944..74129a5 100644 --- a/src/GLPHExtended-Model/GLHCommit.extension.st +++ b/src/GLPHExtended-Model/GLHCommit.extension.st @@ -2,11 +2,11 @@ Extension { #name : #GLHCommit } { #category : #'*GLPHExtended-Model-accessing' } GLHCommit >> commitedMergeRequest [ - "Relation named: #commitedMergeRequest type: #GLPHEMergeRequest opposite: #mergeCommit" + "Relation named: #commitedMergeRequest type: #GLPHEMergeRequest opposite: #mergedCommit" - + ^ self attributeAt: #commitedMergeRequest ifAbsent: [ nil ] ] @@ -19,10 +19,10 @@ GLHCommit >> commitedMergeRequest: anObject [ anObject ifNil: [ | otherSide | otherSide := self commitedMergeRequest. self attributeAt: #commitedMergeRequest put: anObject. - otherSide mergeCommit: nil ] + otherSide mergedCommit: nil ] ifNotNil: [ self attributeAt: #commitedMergeRequest put: anObject. - anObject mergeCommit: self ] + anObject mergedCommit: self ] ] { #category : #'*GLPHExtended-Model-accessing' } diff --git a/src/GLPHExtended-Model/GLPHEMergeRequest.class.st b/src/GLPHExtended-Model/GLPHEMergeRequest.class.st index 2ea40f4..e403184 100644 --- a/src/GLPHExtended-Model/GLPHEMergeRequest.class.st +++ b/src/GLPHExtended-Model/GLPHEMergeRequest.class.st @@ -17,8 +17,9 @@ a gitlab merge request | `author` | `GLPHEMergeRequest` | `createdMergeRequests` | `GLHUser` | | | `closed_by` | `GLPHEMergeRequest` | `closedMergeRequests` | `GLHUser` | | | `diffs` | `GLPHEMergeRequest` | `mergeRequest` | `GLHDiff` | | -| `mergeCommit` | `GLPHEMergeRequest` | `commitedMergeRequest` | `GLHCommit` | | +| `mergeRequestCommit` | `GLPHEMergeRequest` | `commitedMergeRequest` | `GLHCommit` | | | `merge_user` | `GLPHEMergeRequest` | `usedMerges` | `GLHUser` | | +| `mergedCommit` | `GLPHEMergeRequest` | `commitedMergeRequest` | `GLHCommit` | | | `merged_by` | `GLPHEMergeRequest` | `mergerOfMergeRequests` | `GLHUser` | | | `pipeline` | `GLPHEMergeRequest` | `mergeResquest` | `GLHPipeline` | | | `reviewers` | `GLPHEMergeRequest` | `reviewedMergeResquest` | `GLHUser` | | @@ -52,6 +53,7 @@ a gitlab merge request | `merge_when_pipeline_succeeds` | `Boolean` | nil | | | `merged_at` | `String` | nil | | | `milestone` | `String` | nil | | +| `name` | `String` | nil | Basic name of the entity, not full reference.| | `project_id` | `Number` | nil | | | `reference` | `String` | nil | | | `references_full` | `String` | nil | | @@ -85,6 +87,8 @@ a gitlab merge request Class { #name : #GLPHEMergeRequest, #superclass : #GLPHEEntity, + #traits : 'FamixTNamedEntity', + #classTraits : 'FamixTNamedEntity classTrait', #instVars : [ '#blocking_discussions_resolved => FMProperty', '#changes_count => FMProperty', @@ -488,25 +492,25 @@ GLPHEMergeRequest >> latest_build_started_at: anObject [ ] { #category : #accessing } -GLPHEMergeRequest >> mergeCommit [ - "Relation named: #mergeCommit type: #GLHCommit opposite: #commitedMergeRequest" +GLPHEMergeRequest >> mergeRequestCommit [ + "Relation named: #mergeRequestCommit type: #GLHCommit opposite: #commitedMergeRequest" - - ^ self attributeAt: #mergeCommit ifAbsent: [ nil ] + + ^ self attributeAt: #mergeRequestCommit ifAbsent: [ nil ] ] { #category : #accessing } -GLPHEMergeRequest >> mergeCommit: anObject [ +GLPHEMergeRequest >> mergeRequestCommit: anObject [ - (self attributeAt: #mergeCommit ifAbsentPut: [nil]) == anObject ifTrue: [ ^ anObject ]. + (self attributeAt: #mergeRequestCommit ifAbsentPut: [nil]) == anObject ifTrue: [ ^ anObject ]. anObject ifNil: [ | otherSide | - otherSide := self mergeCommit. - self attributeAt: #mergeCommit put: anObject. + otherSide := self mergeRequestCommit. + self attributeAt: #mergeRequestCommit put: anObject. otherSide commitedMergeRequest: nil ] ifNotNil: [ - self attributeAt: #mergeCommit put: anObject. + self attributeAt: #mergeRequestCommit put: anObject. anObject commitedMergeRequest: self ] ] @@ -568,6 +572,29 @@ GLPHEMergeRequest >> merge_when_pipeline_succeeds: anObject [ merge_when_pipeline_succeeds := anObject ] +{ #category : #accessing } +GLPHEMergeRequest >> mergedCommit [ + "Relation named: #mergedCommit type: #GLHCommit opposite: #commitedMergeRequest" + + + + ^ self attributeAt: #mergedCommit ifAbsent: [ nil ] +] + +{ #category : #accessing } +GLPHEMergeRequest >> mergedCommit: anObject [ + + + (self attributeAt: #mergedCommit ifAbsentPut: [nil]) == anObject ifTrue: [ ^ anObject ]. + anObject ifNil: [ | otherSide | + otherSide := self mergedCommit. + self attributeAt: #mergedCommit put: anObject. + otherSide commitedMergeRequest: nil ] + ifNotNil: [ + self attributeAt: #mergedCommit put: anObject. + anObject commitedMergeRequest: self ] +] + { #category : #accessing } GLPHEMergeRequest >> merged_at [ diff --git a/src/GLPHExtended-Model/GLPHETEntityCreator.trait.st b/src/GLPHExtended-Model/GLPHETEntityCreator.trait.st index e47e98b..0e43fe1 100644 --- a/src/GLPHExtended-Model/GLPHETEntityCreator.trait.st +++ b/src/GLPHExtended-Model/GLPHETEntityCreator.trait.st @@ -59,3 +59,10 @@ GLPHETEntityCreator >> newMergeRequest [ ^ self add: GLPHEMergeRequest new ] + +{ #category : #'entity creation' } +GLPHETEntityCreator >> newMergeRequestNamed: aName [ + + + ^ self add: (GLPHEMergeRequest named: aName) +] diff --git a/src/GitLabHealth-Model-Extension/GLHCommit.extension.st b/src/GitLabHealth-Model-Extension/GLHCommit.extension.st index a9c2664..47a946e 100644 --- a/src/GitLabHealth-Model-Extension/GLHCommit.extension.st +++ b/src/GitLabHealth-Model-Extension/GLHCommit.extension.st @@ -1,12 +1,22 @@ Extension { #name : #GLHCommit } +{ #category : #'*GitLabHealth-Model-Extension' } +GLHCommit >> inspectionGLHCommit [ + + + | canvas | + canvas := RSCommit canvasFor: self . + + ^ canvas asPresenter +] + { #category : #'*GitLabHealth-Model-Extension' } GLHCommit >> name [ - ^ title ifNil: [ 'commit #', short_id ]. + ^ message ifNil: [ 'Commit#' , short_id ] ] { #category : #'*GitLabHealth-Model-Extension' } diff --git a/src/GitLabHealth-Model-Importer/GLHApi.class.st b/src/GitLabHealth-Model-Importer/GLHApi.class.st index 7e3129e..487c5ae 100644 --- a/src/GitLabHealth-Model-Importer/GLHApi.class.st +++ b/src/GitLabHealth-Model-Importer/GLHApi.class.st @@ -44,6 +44,21 @@ GLHApi >> client: anObject [ client := anObject ] +{ #category : #'api - commits' } +GLHApi >> commit: commitSHA ofProject: aProjectId [ + + self client path: (String streamContents: [ :str | + str + << self baseAPIUrl; + << '/projects/'; + << aProjectId asString; + << '/repository/commits/'; + << commitSHA asString; + << '' ]). + + ^ self client get +] + { #category : #'api - commits' } GLHApi >> commitDiff: commitSHA ofProject: aProjectId unidiff: unidiff [ diff --git a/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st b/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st index 99395c8..e8c2fcf 100644 --- a/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st +++ b/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st @@ -75,7 +75,6 @@ GLPHMetamodelGenerator >> defineClasses [ 'a indicate on which line of code are concerned by a diff (usually declared with @@ oldRange, newRange @@) '. - repository := self remoteEntity: #Repository withPrefix: #GLH. file := self remoteEntity: #File withPrefix: #GLH. diff := self remoteEntity: #Diff withPrefix: #GLH. @@ -92,7 +91,7 @@ GLPHMetamodelGenerator >> defineHierarchy [ (builder ensureClassNamed: #Entity) --|> #TEntityMetaLevelDependency. change <|-- addition. change <|-- deletion. - change <|-- lineOfCode. + change <|-- lineOfCode. mergeRequest --|> #TNamedEntity ] { #category : #definition } @@ -134,7 +133,10 @@ GLPHMetamodelGenerator >> defineRelations [ (mergeRequest property: #pipeline) - (pipeline property: #mergeResquest). - (mergeRequest property: #mergeCommit) + (mergeRequest property: #mergedCommit) + - (commit property: #commitedMergeRequest). + + (mergeRequest property: #mergeRequestCommit) - (commit property: #commitedMergeRequest). (mergeRequest property: #squashCommit) @@ -148,7 +150,6 @@ GLPHMetamodelGenerator >> defineRelations [ "DIFF" (diff property: #diffRanges) <>-* (diffRange property: #diff). (diffRange property: #changes) -* (change property: #diffRange) - ] { #category : #'as yet unclassified' } diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 3f7f721..7effca7 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -72,6 +72,21 @@ GLPHModelImporter >> completeImportProject: aGLHProject [ ] +{ #category : #'as yet unclassified' } +GLPHModelImporter >> importCommitOfProject: anProject withId: anID [ + |commit result| + + + anID ifNil: [ ^ nil ]. + + result := self glhApi commit: anID ofProject: anProject id. + commit := (self parseCommitsResult: ('[',result,']'))first. + + self glhModel add: commit unless: (self blockOnIdEquality ). + + ^commit +] + { #category : #import } GLPHModelImporter >> importCommitsOfBranch: aGLHBranch [ @@ -168,6 +183,25 @@ GLPHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName until until: toDate ] +{ #category : #importer } +GLPHModelImporter >> importCommitsOfMergeResquest: aGLPHEMergeRequest [ + |foundCommits| + foundCommits := OrderedCollection new. + + aGLPHEMergeRequest mergeRequestCommit: ((self + importCommitOfProject: aGLPHEMergeRequest project + withId: aGLPHEMergeRequest sha) ifNotNil: [:commit | foundCommits add: commit]). + + aGLPHEMergeRequest mergedCommit: ((self + importCommitOfProject: aGLPHEMergeRequest project + withId: aGLPHEMergeRequest merge_commit_sha) ifNotNil: [:commit | foundCommits add: commit]). + aGLPHEMergeRequest squashCommit: ((self + importCommitOfProject: aGLPHEMergeRequest project + withId: aGLPHEMergeRequest squash_commit_sha) ifNotNil: [:commit | foundCommits add: commit]). + + self chainsCommitsFrom: foundCommits. +] + { #category : #'as yet unclassified' } GLPHModelImporter >> importDiffOfMergeRequest: aMergeRequest [ @@ -219,9 +253,19 @@ GLPHModelImporter >> importMergeRequests: aGLHProject [ results := self glhApi mergeRequestOfProject: aGLHProject id. parsedResults := self parseMergeRequestResult: results. - 1 halt. - aGLHProject mergeRequests addAll: parsedResults unless: self blockOnIdEquality . - self glhModel addAll: aGLHProject mergeRequests unless: self blockOnIdEquality. + aGLHProject mergeRequests + addAll: parsedResults + unless: self blockOnIdEquality. + self glhModel + addAll: aGLHProject mergeRequests + unless: self blockOnIdEquality. + + + + "gets it related commits" + aGLHProject mergeRequests do: [ :mr | + self importCommitsOfMergeResquest: mr ]. + self withCommitDiffs ifTrue: [ aGLHProject mergeRequests do: [ :mr | From 4eeb8d39f859c59f96c821602bb39e15abd0d5be Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 11 Apr 2024 17:20:49 +0200 Subject: [PATCH 23/77] add diff name --- .../GLPHEChange.extension.st | 20 +++++++++--------- .../GLPHEMergeRequest.extension.st | 1 - .../GLHDiff.extension.st | 10 +++++++++ .../GLHMetamodelGenerator.class.st | 3 ++- src/GitLabHealth-Model/GLHDiff.class.st | 21 +++++++++++++++---- .../GLHTEntityCreator.trait.st | 7 +++++++ 6 files changed, 46 insertions(+), 16 deletions(-) create mode 100644 src/GitLabHealth-Model-Extension/GLHDiff.extension.st diff --git a/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st b/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st index 61fcbf3..d90cb42 100644 --- a/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st +++ b/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st @@ -1,18 +1,18 @@ Extension { #name : #GLPHEChange } { #category : #'*GLPHExtended-Model-Extension' } -GLPHEChange class >> createFrom: aDiffLine [ +GLPHEChange class >> createFrom: aDiffLine [ "Factory a Change from a loc" - |aChange| - aChange := nil. - (aDiffLine beginsWith: #'@@') ifTrue: [ - |infos| - aChange := GLPHELineOfCode newFromLoCRange: aDiffLine. - 1 halt. + + | aChange | + aChange := nil. + (aDiffLine beginsWith: #'@@') ifTrue: [ + | infos | + aChange := GLPHELineOfCode newFromLoCRange: aDiffLine. + infos := (aDiffLine splitOn: '@@') copyWithoutFirst. - infos := aDiffLine splitOn: ' ' . - ]. - ^ aChange. + infos := aDiffLine splitOn: ' ' ]. + ^ aChange ] { #category : #'*GLPHExtended-Model-Extension' } diff --git a/src/GLPHExtended-Model-Extension/GLPHEMergeRequest.extension.st b/src/GLPHExtended-Model-Extension/GLPHEMergeRequest.extension.st index 65292e2..c369eb7 100644 --- a/src/GLPHExtended-Model-Extension/GLPHEMergeRequest.extension.st +++ b/src/GLPHExtended-Model-Extension/GLPHEMergeRequest.extension.st @@ -4,7 +4,6 @@ Extension { #name : #GLPHEMergeRequest } GLPHEMergeRequest >> name [ - ^ title ] diff --git a/src/GitLabHealth-Model-Extension/GLHDiff.extension.st b/src/GitLabHealth-Model-Extension/GLHDiff.extension.st new file mode 100644 index 0000000..483fde4 --- /dev/null +++ b/src/GitLabHealth-Model-Extension/GLHDiff.extension.st @@ -0,0 +1,10 @@ +Extension { #name : #GLHDiff } + +{ #category : #'*GitLabHealth-Model-Extension' } +GLHDiff >> name [ + + + + + ^ new_path contractTo: 100 +] diff --git a/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st b/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st index 7db42f1..ec9f754 100644 --- a/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st +++ b/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st @@ -99,11 +99,12 @@ GLHMetamodelGenerator >> defineHierarchy [ (builder ensureClassNamed: #Entity) --|> #TEntityMetaLevelDependency. fileDirectory --|> file. fileBlob --|> file. - + group --|> #TNamedEntity. project --|> #TNamedEntity. user --|> #TNamedEntity. commit --|> #TNamedEntity. + diff --|> #TNamedEntity ] { #category : #definition } diff --git a/src/GitLabHealth-Model/GLHDiff.class.st b/src/GitLabHealth-Model/GLHDiff.class.st index 8c9be42..a9f3c74 100644 --- a/src/GitLabHealth-Model/GLHDiff.class.st +++ b/src/GitLabHealth-Model/GLHDiff.class.st @@ -17,6 +17,7 @@ The diff of a commit |---| | `deleted_file` | `Boolean` | false | Is the file of the diff has been removed| | `diffString` | `String` | nil | The string presenting the diff. It should follow the [Detailed-Unified](https://www.gnu.org/software/diffutils/manual/html_node/Detailed-Unified.html) format| +| `name` | `String` | nil | Basic name of the entity, not full reference.| | `new_file` | `Boolean` | false | Is the file of the diff is a new file| | `new_path` | `String` | nil | The path to the file in the resulted repository commitish| | `old_path` | `String` | nil | The path to the file in the origin repository commitish| @@ -26,14 +27,16 @@ The diff of a commit Class { #name : #GLHDiff, #superclass : #GLHEntity, + #traits : 'FamixTNamedEntity', + #classTraits : 'FamixTNamedEntity classTrait', #instVars : [ + '#commit => FMOne type: #GLHCommit opposite: #diffs', + '#deleted_file => FMProperty defaultValue: false', '#diffString => FMProperty', + '#new_file => FMProperty defaultValue: false', '#new_path => FMProperty', '#old_path => FMProperty', - '#new_file => FMProperty defaultValue: false', - '#renamed_file => FMProperty defaultValue: false', - '#deleted_file => FMProperty defaultValue: false', - '#commit => FMOne type: #GLHCommit opposite: #diffs' + '#renamed_file => FMProperty defaultValue: false' ], #category : #'GitLabHealth-Model-Entities' } @@ -100,6 +103,16 @@ GLHDiff >> diffString: anObject [ diffString := anObject ] +{ #category : #'as yet unclassified' } +GLHDiff >> inspectionGLHDiff [ + + + | canvas | + canvas := RSCommitDiff canvasFor: self. + + ^ canvas asPresenter +] + { #category : #accessing } GLHDiff >> new_file [ diff --git a/src/GitLabHealth-Model/GLHTEntityCreator.trait.st b/src/GitLabHealth-Model/GLHTEntityCreator.trait.st index 5384b91..c78fed0 100644 --- a/src/GitLabHealth-Model/GLHTEntityCreator.trait.st +++ b/src/GitLabHealth-Model/GLHTEntityCreator.trait.st @@ -46,6 +46,13 @@ GLHTEntityCreator >> newDiff [ ^ self add: GLHDiff new ] +{ #category : #'entity creation' } +GLHTEntityCreator >> newDiffNamed: aName [ + + + ^ self add: (GLHDiff named: aName) +] + { #category : #'entity creation' } GLHTEntityCreator >> newFile [ From d3388a77f6c142f7427cc06fc96c09014c34aa41 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 18 Apr 2024 15:13:31 +0200 Subject: [PATCH 24/77] new commit import --- .../GLPHEChange.extension.st | 9 + .../GLPHEDiffRange.extension.st | 6 + src/GLPHExtended-Model/GLHDiff.extension.st | 5 +- src/GLPHExtended-Model/GLPHEChange.class.st | 2 + .../GLHCommit.extension.st | 14 +- .../GLHModelImporter.class.st | 180 ++++++++++++++- src/GitLabHealth-Model/GLHCommit.class.st | 1 + .../GLPHMetamodelGenerator.class.st | 9 +- .../GLPHApi.class.st | 34 ++- .../GLPHModelImporter.class.st | 205 ++++++------------ 10 files changed, 306 insertions(+), 159 deletions(-) diff --git a/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st b/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st index d90cb42..ed6697e 100644 --- a/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st +++ b/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st @@ -15,6 +15,15 @@ GLPHEChange class >> createFrom: aDiffLine [ ^ aChange ] +{ #category : #'*GLPHExtended-Model-Extension' } +GLPHEChange >> name [ + + + + + ^ sourceCode +] + { #category : #'*GLPHExtended-Model-Extension' } GLPHEChange class >> newFrom: aDiffLine [ "Factory a Change from a loc" diff --git a/src/GLPHExtended-Model-Extension/GLPHEDiffRange.extension.st b/src/GLPHExtended-Model-Extension/GLPHEDiffRange.extension.st index 40ee7e9..712e717 100644 --- a/src/GLPHExtended-Model-Extension/GLPHEDiffRange.extension.st +++ b/src/GLPHExtended-Model-Extension/GLPHEDiffRange.extension.st @@ -1,5 +1,11 @@ Extension { #name : #GLPHEDiffRange } +{ #category : #'*GLPHExtended-Model-Extension' } +GLPHEDiffRange >> name [ + + ^ '[ OG: ' , originalLineRange , ' | NEW: ' , newLineRange , ']' +] + { #category : #'*GLPHExtended-Model-Extension' } GLPHEDiffRange class >> newFrom: aLine [ |range infos rangesInfo | diff --git a/src/GLPHExtended-Model/GLHDiff.extension.st b/src/GLPHExtended-Model/GLHDiff.extension.st index d9daf32..4967cf8 100644 --- a/src/GLPHExtended-Model/GLHDiff.extension.st +++ b/src/GLPHExtended-Model/GLHDiff.extension.st @@ -8,7 +8,10 @@ GLHDiff >> diffRanges [ - ^ self attributeAt: #diffRanges ifAbsentPut: [ FMMultivalueLink on: self opposite: #diff: ] + + ^ self + attributeAt: #diffRanges + ifAbsentPut: [ FMMultivalueLink on: self opposite: #diff: ] ] { #category : #'*GLPHExtended-Model-accessing' } diff --git a/src/GLPHExtended-Model/GLPHEChange.class.st b/src/GLPHExtended-Model/GLPHEChange.class.st index df2bd7f..836c483 100644 --- a/src/GLPHExtended-Model/GLPHEChange.class.st +++ b/src/GLPHExtended-Model/GLPHEChange.class.st @@ -23,6 +23,8 @@ super class for addition and deletion Class { #name : #GLPHEChange, #superclass : #GLPHEEntity, + #traits : 'FamixTNamedEntity', + #classTraits : 'FamixTNamedEntity classTrait', #instVars : [ '#diffRange => FMOne type: #GLPHEDiffRange opposite: #changes', '#index => FMProperty', diff --git a/src/GitLabHealth-Model-Extension/GLHCommit.extension.st b/src/GitLabHealth-Model-Extension/GLHCommit.extension.st index 47a946e..0564f0f 100644 --- a/src/GitLabHealth-Model-Extension/GLHCommit.extension.st +++ b/src/GitLabHealth-Model-Extension/GLHCommit.extension.st @@ -3,9 +3,19 @@ Extension { #name : #GLHCommit } { #category : #'*GitLabHealth-Model-Extension' } GLHCommit >> inspectionGLHCommit [ - + | canvas | - canvas := RSCommit canvasFor: self . + canvas := RSCommit canvasHierarchyFor: self. + + ^ canvas asPresenter +] + +{ #category : #'*GitLabHealth-Model-Extension' } +GLHCommit >> inspectionGLHCommitFileImpacts [ + + + | canvas | + canvas := RSCommit canvasDiffImpactFor: self. ^ canvas asPresenter ] diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index b36c7d2..b4ac60e 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -63,9 +63,27 @@ GLHModelImporter >> completeImportProject: aGLHProject [ aGLHProject addPipeline: pipeline ]. aGLHProject creator: (self importUser: aGLHProject creator_id). aGLHProject repository: GLHRepository new. - self glhModel add: aGLHProject. + self glhModel add: aGLHProject unless: (self blockEqualityOn: #id). self glhModel add: aGLHProject repository. - self importRepository: aGLHProject repository. + self importRepository: aGLHProject repository +] + +{ #category : #'as yet unclassified' } +GLHModelImporter >> completeImportedCommit: aCommit [ + + | diffs | + (self importUserByUsername: aCommit author_name) commits + add: aCommit + unless: [ :existingCommit :newCommit | + existingCommit id = newCommit id ]. + + ('Import diff of commit: ' , aCommit short_id printString) recordInfo. + + self withCommitDiffs ifTrue: [ + diffs := self importDiffOfCommit: aCommit. + self glhModel addAll: diffs unless: self blockForDiffEquality ]. + + ^ aCommit ] { #category : #private } @@ -107,6 +125,7 @@ GLHModelImporter >> glhModel: anObject [ { #category : #'private - api' } GLHModelImporter >> importCommits: aGLHProject [ "limited to the last 20 commits" + | results parsedResults | results := self glhApi commitsOfProject: aGLHProject id @@ -123,17 +142,76 @@ GLHModelImporter >> importCommits: aGLHProject [ perPage: nil page: nil. parsedResults := self parseCommitsResult: results. - aGLHProject mooseModel addAll: parsedResults. - + self glhModel addAll: parsedResults unless: self blockOnIdEquality. + parsedResults do: [ :commit | - commit repository: aGLHProject repository. - ]. - + commit repository: aGLHProject repository ]. + + self withCommitDiffs ifTrue: [ parsedResults do: [ :commit | + self importDiffOfCommit: commit ] ] ] +{ #category : #'as yet unclassified' } +GLHModelImporter >> importCommitsFollowing: aCommit upToDays: aNumberOfDay [ + "import the 'n' commits of a project starting from an initial 'aCommit' commit. + Lazy import does not import the entities inside the model" + + | date | + date := aCommit created_at asDateAndTime. + + ^ self + importCommitsOfBranch: aCommit branch + forRefName: aCommit branch name + since: date + until: (date + aNumberOfDay day) +] + +{ #category : #commit } +GLHModelImporter >> importCommitsOProject: aProject since: fromDate until: toDate [ + + | newlyFoundCommit page foundCommit | + page := 0. + foundCommit := OrderedCollection new. + newlyFoundCommit := { true }. + [ newlyFoundCommit isNotEmpty ] whileTrue: [ + | results | + page := page + 1. + ('import commit page ' , page printString) recordInfo. + results := self glhApi + commitsOfProject: aProject id + forRefName: nil + since: + (fromDate ifNotNil: [ fromDate asDateAndTime asString ]) + until: + (toDate ifNotNil: [ toDate asDateAndTime asString ]) + path: nil + author: nil + all: nil + with_stats: nil + firstParent: nil + order: nil + trailers: nil + perPage: 100 + page: page. + + newlyFoundCommit := self parseCommitsResult: results. + newlyFoundCommit do: [ :c | + c repository: aProject repository. + ]. + aProject repository commits + addAll: newlyFoundCommit + unless: self blockOnIdEquality. + foundCommit addAll: newlyFoundCommit ]. + + self glhModel addAll: foundCommit unless: self blockOnIdEquality. + + + ^ foundCommit +] + { #category : #api } GLHModelImporter >> importCommitsOf: aGLHProject withStats: aBoolean until: toDate [ @@ -176,6 +254,71 @@ GLHModelImporter >> importCommitsOf: aGLHProject withStats: aBoolean until: toDa self importDiffOfCommit: commit ] ] ] +{ #category : #commit } +GLHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName since: fromDate [ + + ^ self + importCommitsOfBranch: aGLHBranch + forRefName: aGLHBranch name + since: fromDate + until: nil +] + +{ #category : #commit } +GLHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName since: fromDate until: toDate [ + + | newlyFoundCommit page foundCommit| + page := 0. + foundCommit := OrderedCollection new. + newlyFoundCommit := { true }. + [ newlyFoundCommit isNotEmpty ] whileTrue: [ + | results | + page := page + 1. + ('import commit page ' , page printString) recordInfo. + results := self glhApi + commitsOfProject: aGLHBranch repository project id + forRefName: aGLHBranch name + since: + (fromDate ifNotNil: [ fromDate asDateAndTime asString ]) + until: + (toDate ifNotNil: [ toDate asDateAndTime asString ]) + path: nil + author: nil + all: nil + with_stats: nil + firstParent: nil + order: nil + trailers: nil + perPage: 100 + page: page. + + newlyFoundCommit := self parseCommitsResult: results. + aGLHBranch commits + addAll: newlyFoundCommit + unless: self blockOnIdEquality. + foundCommit addAll: newlyFoundCommit. + ]. + + self glhModel + addAll: aGLHBranch commits + unless: self blockOnIdEquality. + + "self withCommitDiffs ifTrue: [ + aGLHBranch commits: [ :commit | self importDiffOfCommit: commit ] ]." + + ^ foundCommit +] + +{ #category : #commit } +GLHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName until: toDate [ + + ^ self + importCommitsOfBranch: aGLHBranch + forRefName: aGLHBranch name + since: nil + until: toDate +] + { #category : #api } GLHModelImporter >> importDiffOfCommit: aCommit [ @@ -190,7 +333,8 @@ GLHModelImporter >> importDiffOfCommit: aCommit [ unidiff: true. diffsResult := self parseDiffResult: result. - aCommit diffs addAll: diffsResult unless: self blockForDiffEquality . + aCommit diffs addAll: diffsResult unless: self blockForDiffEquality. + ^ aCommit diffs ] @@ -289,6 +433,26 @@ GLHModelImporter >> importUser: aUserID [ userResult ] ] +{ #category : #user } +GLHModelImporter >> importUserByUsername: anUsername [ + + ^ (self glhModel allWithType: GLHUser) + detect: [ :user | user username = anUsername ] + ifNone: [ + | result userId searchResult | + ('Import user with username: ' , anUsername printString) + recordInfo. + result := self glhApi usersSearchByUsername: anUsername. + searchResult := NeoJSONReader fromString: result. + searchResult ifEmpty: [ GLHUser new ] ifNotEmpty: [ + userId := searchResult first at: #id. + (self glhModel allWithType: GLHUser) + detect: [ :user | user id = userId ] + ifNone: [ + self importUser: userId. + ] ] ] +] + { #category : #initialization } GLHModelImporter >> initialize [ diff --git a/src/GitLabHealth-Model/GLHCommit.class.st b/src/GitLabHealth-Model/GLHCommit.class.st index d74f66a..cf5c13c 100644 --- a/src/GitLabHealth-Model/GLHCommit.class.st +++ b/src/GitLabHealth-Model/GLHCommit.class.st @@ -299,6 +299,7 @@ GLHCommit >> diffs [ + ^ diffs ] diff --git a/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st b/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st index e8c2fcf..d3d8251 100644 --- a/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st +++ b/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st @@ -89,9 +89,11 @@ GLPHMetamodelGenerator >> defineHierarchy [ super defineHierarchy. (builder ensureClassNamed: #Entity) --|> #TEntityMetaLevelDependency. - change <|-- addition. + change --|> #TNamedEntity. + change <|-- addition. change <|-- deletion. - change <|-- lineOfCode. mergeRequest --|> #TNamedEntity + change <|-- lineOfCode. + mergeRequest --|> #TNamedEntity ] { #category : #definition } @@ -135,7 +137,7 @@ GLPHMetamodelGenerator >> defineRelations [ (mergeRequest property: #mergedCommit) - (commit property: #commitedMergeRequest). - + (mergeRequest property: #mergeRequestCommit) - (commit property: #commitedMergeRequest). @@ -148,6 +150,7 @@ GLPHMetamodelGenerator >> defineRelations [ <>-* (mergeRequest property: #project). "DIFF" + (diff property: #file) <>-* (file property: #diffs). (diff property: #diffRanges) <>-* (diffRange property: #diff). (diffRange property: #changes) -* (change property: #diffRange) ] diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st index d616b4e..b369760 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st @@ -5,9 +5,16 @@ Class { } { #category : #commit } -GLPHApi >> commitsOfProject: anInteger forRefName: refName [ - - ^ self commitsOfProject: anInteger +GLPHApi >> commitsOfProject: anInteger forRefName: refName [ + + self + deprecated: 'Use #unknow instead' + on: '12 April 2024' + in: + 'Pharo-11.0.0+build.726.sha.aece1b5473acf3830a0e082c1bc3a15d4ff3522b (64 Bit)'. + + ^ self + commitsOfProject: anInteger forRefName: refName since: nil until: nil @@ -19,13 +26,20 @@ GLPHApi >> commitsOfProject: anInteger forRefName: refName [ order: nil trailers: nil perPage: nil - page: nil. + page: nil ] { #category : #commits } -GLPHApi >> commitsOfProject: anInteger forRefName: refName since: since [ +GLPHApi >> commitsOfProject: anInteger forRefName: refName since: since [ - ^ self commitsOfProject: anInteger + self + deprecated: 'Use #unknow instead' + on: '12 April 2024' + in: + 'Pharo-11.0.0+build.726.sha.aece1b5473acf3830a0e082c1bc3a15d4ff3522b (64 Bit)'. + + ^ self + commitsOfProject: anInteger forRefName: refName since: since until: nil @@ -37,13 +51,18 @@ GLPHApi >> commitsOfProject: anInteger forRefName: refName since: since [ order: nil trailers: nil perPage: nil - page: nil. + page: nil ] { #category : #commit } GLPHApi >> commitsOfProject: aGLHProject forRefName: refName until: toDate [ | newlyFoundCommit page | + self + deprecated: 'Use #unknow instead' + on: '11 April 2024' + in: + 'Pharo-11.0.0+build.726.sha.aece1b5473acf3830a0e082c1bc3a15d4ff3522b (64 Bit)'. page := 0. newlyFoundCommit := { true }. [ newlyFoundCommit isNotEmpty ] whileTrue: [ @@ -78,7 +97,6 @@ GLPHApi >> commitsOfProject: aGLHProject forRefName: refName until: toDate [ self withCommitDiffs ifTrue: [ aGLHProject repository commits do: [ :commit | self importDiffOfCommit: commit ] ] - ] { #category : #'as yet unclassified' } diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 7effca7..bf87990 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -4,6 +4,15 @@ Class { #category : #'GitLabProjectHealth-Model-Importer' } +{ #category : #'as yet unclassified' } +GLPHModelImporter >> blockForDiffRangeEquality [ + +^ [ :existing :new | + (existing diff mooseID = new diff mooseID and: [ + existing originalLineRange = new originalLineRange ]) and: [ + existing newLineRange = new newLineRange ] ] +] + { #category : #commit } GLPHModelImporter >> chainsCommitsFrom: commitsCollection [ @@ -14,12 +23,15 @@ GLPHModelImporter >> chainsCommitsFrom: commitsCollection [ detect: [ :c | c id = parentId ] ifNone: [ nil ]. parentCommit ifNotNil: [ - - parentCommit childCommits add: commit unless: [ :existing :new | existing id = new id ]. + parentCommit childCommits + add: commit + unless: [ :existing :new | existing id = new id ]. + ^ commitsCollection + "parentCommit childCommits detect: [ :c | c id = commit id ] - ifNone: [ parentCommit addChildCommit: commit ] "] ] ] + ifNone: [ parentCommit addChildCommit: commit ] " ] ] ] ] { #category : #commit } @@ -55,8 +67,8 @@ GLPHModelImporter >> commitsOfProject: aGLHProject forRefName: refName until: to < toDate asDateAndTime ]) or: [ existingCommits anySatisfy: [ :existingCommit | existingCommit id = commitParsed id ] ] ]. - aGLHProject mooseModel addAll: newlyFoundCommit. - aGLHProject repository commits addAll: newlyFoundCommit ]. + self glhModel addAll: newlyFoundCommit unless: self blockOnIdEquality. + aGLHProject repository commits addAll: newlyFoundCommit unless: self blockOnIdEquality ]. self withCommitDiffs ifTrue: [ aGLHProject repository commits do: [ :commit | self importDiffOfCommit: commit ] ] @@ -74,17 +86,21 @@ GLPHModelImporter >> completeImportProject: aGLHProject [ { #category : #'as yet unclassified' } GLPHModelImporter >> importCommitOfProject: anProject withId: anID [ - |commit result| - - - anID ifNil: [ ^ nil ]. - + + | commit result | + anID ifNil: [ ^ nil ]. + result := self glhApi commit: anID ofProject: anProject id. - commit := (self parseCommitsResult: ('[',result,']'))first. + commit := (self parseCommitsResult: '[' , result , ']') first. + + self glhModel add: commit unless: self blockOnIdEquality. + commit repository: anProject repository . - self glhModel add: commit unless: (self blockOnIdEquality ). + self withCommitDiffs ifTrue: [ + self importDiffOfCommit: commit. + ]. - ^commit + ^ commit ] { #category : #import } @@ -99,107 +115,49 @@ GLPHModelImporter >> importCommitsOfBranch: aGLHBranch [ forRefName: aGLHBranch name since: withCommitsSince. - - self chainsCommitsFrom: commits. commits do: [ :aCommit | - | diffs | - (self importUserByUsername: aCommit author_name) commits - add: aCommit - unless: [ :existingCommit :newCommit | - existingCommit id = newCommit id ]. - aCommit repository: aGLHBranch repository. - - ('Import diff of commit: ' , aCommit short_id printString) - recordInfo. - - diffs := self importDiffOfCommit: aCommit. - self glhModel addAll: diffs unless: [ :existing :new | - existing diffString size = new diffString size and: [ - existing diffString = new diffString ] ]. - - diffs do: [ :diff | "changes are added into the model during the import" - self importDiffRangesForDiff: diff ] ] + self completeImportedCommit: aCommit. + ] ] -{ #category : #commit } -GLPHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName since: fromDate [ - - ^ self - importCommitsOfBranch: aGLHBranch - forRefName: refName - since: fromDate - until: nil -] +{ #category : #importer } +GLPHModelImporter >> importCommitsOfMergeResquest: aGLPHEMergeRequest [ -{ #category : #commit } -GLPHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName since: fromDate until: toDate [ + | foundCommits | + foundCommits := OrderedCollection new. - | newlyFoundCommit page | - page := 0. - newlyFoundCommit := { true }. - [ newlyFoundCommit isNotEmpty ] whileTrue: [ - | results | - page := page + 1. - ('import commit page ' , page printString) recordInfo. - results := self glhApi - commitsOfProject: aGLHBranch repository project id - forRefName: aGLHBranch name - since: - (fromDate ifNotNil: [ fromDate asDateAndTime asString ]) - until: - (toDate ifNotNil: [ toDate asDateAndTime asString ]) - path: nil - author: nil - all: nil - with_stats: nil - firstParent: nil - order: nil - trailers: nil - perPage: 100 - page: page. + "the founds commits are added to the model during their respective import" + aGLPHEMergeRequest mergeRequestCommit: ((self + importCommitOfProject: aGLPHEMergeRequest project + withId: aGLPHEMergeRequest sha) ifNotNil: [ :commit | + foundCommits add: commit ]). - newlyFoundCommit := self parseCommitsResult: results. - aGLHBranch commits - addAll: newlyFoundCommit - unless: (self blockEqualityOn: #id) ]. + aGLPHEMergeRequest mergedCommit: ((self + importCommitOfProject: aGLPHEMergeRequest project + withId: aGLPHEMergeRequest merge_commit_sha) ifNotNil: [ :commit | + foundCommits add: commit ]). - self glhModel - addAll: aGLHBranch commits - unless: [ :old :new | old id = new id ]. + aGLPHEMergeRequest squashCommit: ((self + importCommitOfProject: aGLPHEMergeRequest project + withId: aGLPHEMergeRequest squash_commit_sha) ifNotNil: [ :commit | + foundCommits add: commit ]). - ^ aGLHBranch commits + + self chainsCommitsFrom: foundCommits ] -{ #category : #commit } -GLPHModelImporter >> importCommitsOfBranch: aGLHBranch forRefName: refName until: toDate [ - - ^ self - importCommitsOfBranch: aGLHBranch - forRefName: refName - since: nil - until: toDate -] +{ #category : #api } +GLPHModelImporter >> importDiffOfCommit: aCommit [ -{ #category : #importer } -GLPHModelImporter >> importCommitsOfMergeResquest: aGLPHEMergeRequest [ - |foundCommits| - foundCommits := OrderedCollection new. - - aGLPHEMergeRequest mergeRequestCommit: ((self - importCommitOfProject: aGLPHEMergeRequest project - withId: aGLPHEMergeRequest sha) ifNotNil: [:commit | foundCommits add: commit]). + super importDiffOfCommit: aCommit. - aGLPHEMergeRequest mergedCommit: ((self - importCommitOfProject: aGLPHEMergeRequest project - withId: aGLPHEMergeRequest merge_commit_sha) ifNotNil: [:commit | foundCommits add: commit]). - aGLPHEMergeRequest squashCommit: ((self - importCommitOfProject: aGLPHEMergeRequest project - withId: aGLPHEMergeRequest squash_commit_sha) ifNotNil: [:commit | foundCommits add: commit]). - - self chainsCommitsFrom: foundCommits. + "changes are added into the model during the import" + aCommit diffs do: [ :diff | self importDiffRangesForDiff: diff ]. + + ^ aCommit diffs ] { #category : #'as yet unclassified' } @@ -236,12 +194,11 @@ GLPHModelImporter >> importDiffRangesForDiff: aGLHDiff [ | diffRanges | ('Import changes of diff ' , aGLHDiff printString) recordInfo. + "the import of 'diffRanges' entites is technically done inside the parseDiffString" diffRanges := self parseDiffString: aGLHDiff. - self glhModel addAll: diffRanges unless: [ :existing :new | - (existing diff mooseID = new diff mooseID and: [ - existing originalLineRange = new originalLineRange ]) and: [ - existing newLineRange = new newLineRange ] ]. + self glhModel addAll: diffRanges unless: self blockForDiffRangeEquality . + aGLHDiff diffRanges addAll: diffRanges unless: self blockForDiffRangeEquality . ^ aGLHDiff diffRanges ] @@ -256,12 +213,12 @@ GLPHModelImporter >> importMergeRequests: aGLHProject [ aGLHProject mergeRequests addAll: parsedResults unless: self blockOnIdEquality. + self glhModel addAll: aGLHProject mergeRequests unless: self blockOnIdEquality. - "gets it related commits" aGLHProject mergeRequests do: [ :mr | self importCommitsOfMergeResquest: mr ]. @@ -292,26 +249,6 @@ GLPHModelImporter >> importRepository: aGLHRepository [ self importCommitsOfBranch: branch ]. ] -{ #category : #user } -GLPHModelImporter >> importUserByUsername: anUsername [ - - ^ (self glhModel allWithType: GLHUser) - detect: [ :user | user username = anUsername ] - ifNone: [ - | result userId searchResult | - ('Import user with username: ' , anUsername printString) - recordInfo. - result := self glhApi usersSearchByUsername: anUsername. - searchResult := NeoJSONReader fromString: result. - searchResult ifEmpty: [ GLHUser new ] ifNotEmpty: [ - userId := searchResult first at: #id. - (self glhModel allWithType: GLHUser) - detect: [ :user | user id = userId ] - ifNone: [ - self importUser: userId. - ] ] ] -] - { #category : #initialization } GLPHModelImporter >> initialize [ @@ -332,16 +269,11 @@ GLPHModelImporter >> parseDiffString: aDiff [ (line trim beginsWith: #'@@') ifTrue: [ range := GLPHEDiffRange newFrom: line. - aDiff diffRanges add: range unless: [ :existing :new | - existing originalLineRange = new originalLineRange and: [ - existing newLineRange = new newLineRange ] ] - - " self glhModel - add: range - unless: [:existing :new | ((existing diff mooseID = new diff mooseID) - and: [existing originalLineRange = new originalLineRange]) - and: [ existing newLineRange = new newLineRange ] - ]." ]. + aDiff diffRanges + add: range + unless: [ :existing :new | + existing originalLineRange = new originalLineRange and: [ + existing newLineRange = new newLineRange ] ] ]. self withCommitDiffs ifTrue: [ aChange := GLPHEChange newFrom: line. @@ -349,7 +281,6 @@ GLPHModelImporter >> parseDiffString: aDiff [ range changes add: aChange unless: [ :existing :new | existing sourceCode = new sourceCode and: [ existing index = new index ] ]. - "self glhModel add: aChange ." index := index + 1 ] ]. @@ -358,8 +289,8 @@ GLPHModelImporter >> parseDiffString: aDiff [ (existing diffRange mooseID = new diffRange mooseID and: [ existing sourceCode = new sourceCode ]) and: [ existing index = new index ] ] ]. - -^ aDiff diffRanges. + + ^ aDiff diffRanges ] { #category : #parsing } From d3e8ee2a0e17f394f8c88d0cdf8784ec7736d45f Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Fri, 26 Apr 2024 16:14:16 +0200 Subject: [PATCH 25/77] add commit import from a commit upto a certain distant past --- .../GLPHEChange.extension.st | 3 - src/GLPHExtended-Model/GLHDiff.extension.st | 23 +++++- src/GLPHExtended-Model/GLHFile.extension.st | 19 +++++ src/GLPHExtended-Model/GLPHEChange.class.st | 1 + .../GLPHETEntityCreator.trait.st | 7 ++ .../GLHCommit.extension.st | 18 +++-- .../GLHDiff.extension.st | 2 - .../MooseAbstractGroup.extension.st | 2 + .../GLHModelImporter.class.st | 79 +++++++++++++++++++ src/GitLabHealth-Model/GLHCommit.class.st | 1 - src/GitLabHealth-Model/GLHUser.class.st | 2 +- .../GLPHModelImporter.class.st | 31 ++------ 12 files changed, 146 insertions(+), 42 deletions(-) create mode 100644 src/GLPHExtended-Model/GLHFile.extension.st diff --git a/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st b/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st index ed6697e..c51cb9c 100644 --- a/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st +++ b/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st @@ -18,9 +18,6 @@ GLPHEChange class >> createFrom: aDiffLine [ { #category : #'*GLPHExtended-Model-Extension' } GLPHEChange >> name [ - - - ^ sourceCode ] diff --git a/src/GLPHExtended-Model/GLHDiff.extension.st b/src/GLPHExtended-Model/GLHDiff.extension.st index 4967cf8..ac43e8b 100644 --- a/src/GLPHExtended-Model/GLHDiff.extension.st +++ b/src/GLPHExtended-Model/GLHDiff.extension.st @@ -8,10 +8,7 @@ GLHDiff >> diffRanges [ - - ^ self - attributeAt: #diffRanges - ifAbsentPut: [ FMMultivalueLink on: self opposite: #diff: ] + ^ self attributeAt: #diffRanges ifAbsentPut: [ FMMultivalueLink on: self opposite: #diff: ] ] { #category : #'*GLPHExtended-Model-accessing' } @@ -21,6 +18,24 @@ GLHDiff >> diffRanges: anObject [ self diffRanges value: anObject ] +{ #category : #'*GLPHExtended-Model-accessing' } +GLHDiff >> file [ + "Relation named: #file type: #GLHFile opposite: #diffs" + + + + + + ^ self attributeAt: #file ifAbsentPut: [ FMMultivalueLink on: self opposite: #diffs: ] +] + +{ #category : #'*GLPHExtended-Model-accessing' } +GLHDiff >> file: anObject [ + + + self file value: anObject +] + { #category : #'*GLPHExtended-Model-accessing' } GLHDiff >> mergeRequest [ "Relation named: #mergeRequest type: #GLPHEMergeRequest opposite: #diffs" diff --git a/src/GLPHExtended-Model/GLHFile.extension.st b/src/GLPHExtended-Model/GLHFile.extension.st new file mode 100644 index 0000000..8b03b68 --- /dev/null +++ b/src/GLPHExtended-Model/GLHFile.extension.st @@ -0,0 +1,19 @@ +Extension { #name : #GLHFile } + +{ #category : #'*GLPHExtended-Model-accessing' } +GLHFile >> diffs [ + "Relation named: #diffs type: #GLHDiff opposite: #file" + + + + + + ^ self attributeAt: #diffs ifAbsent: [ nil ] +] + +{ #category : #'*GLPHExtended-Model-accessing' } +GLHFile >> diffs: anObject [ + + + self attributeAt: #diffs put: (FMMultivalueLink on: self update: #file from: self diffs to: anObject). +] diff --git a/src/GLPHExtended-Model/GLPHEChange.class.st b/src/GLPHExtended-Model/GLPHEChange.class.st index 836c483..1796c32 100644 --- a/src/GLPHExtended-Model/GLPHEChange.class.st +++ b/src/GLPHExtended-Model/GLPHEChange.class.st @@ -16,6 +16,7 @@ super class for addition and deletion | Name | Type | Default value | Comment | |---| | `index` | `Number` | nil | | +| `name` | `String` | nil | Basic name of the entity, not full reference.| | `relativeIndex` | `Number` | nil | | | `sourceCode` | `String` | nil | | diff --git a/src/GLPHExtended-Model/GLPHETEntityCreator.trait.st b/src/GLPHExtended-Model/GLPHETEntityCreator.trait.st index 0e43fe1..fcc8d1b 100644 --- a/src/GLPHExtended-Model/GLPHETEntityCreator.trait.st +++ b/src/GLPHExtended-Model/GLPHETEntityCreator.trait.st @@ -32,6 +32,13 @@ GLPHETEntityCreator >> newChange [ ^ self add: GLPHEChange new ] +{ #category : #'entity creation' } +GLPHETEntityCreator >> newChangeNamed: aName [ + + + ^ self add: (GLPHEChange named: aName) +] + { #category : #'entity creation' } GLPHETEntityCreator >> newDeletion [ diff --git a/src/GitLabHealth-Model-Extension/GLHCommit.extension.st b/src/GitLabHealth-Model-Extension/GLHCommit.extension.st index 0564f0f..9284bc0 100644 --- a/src/GitLabHealth-Model-Extension/GLHCommit.extension.st +++ b/src/GitLabHealth-Model-Extension/GLHCommit.extension.st @@ -3,9 +3,19 @@ Extension { #name : #GLHCommit } { #category : #'*GitLabHealth-Model-Extension' } GLHCommit >> inspectionGLHCommit [ - + | canvas | - canvas := RSCommit canvasHierarchyFor: self. + canvas := RSCommit canvasHierarchyFor: self. + + ^ canvas asPresenter +] + +{ #category : #'*GitLabHealth-Model-Extension' } +GLHCommit >> inspectionGLHCommitChildren [ + + + | canvas | + canvas := RSCommit canvasCompleteHierachyFor: self. ^ canvas asPresenter ] @@ -22,10 +32,6 @@ GLHCommit >> inspectionGLHCommitFileImpacts [ { #category : #'*GitLabHealth-Model-Extension' } GLHCommit >> name [ - - - - ^ message ifNil: [ 'Commit#' , short_id ] ] diff --git a/src/GitLabHealth-Model-Extension/GLHDiff.extension.st b/src/GitLabHealth-Model-Extension/GLHDiff.extension.st index 483fde4..1b10673 100644 --- a/src/GitLabHealth-Model-Extension/GLHDiff.extension.st +++ b/src/GitLabHealth-Model-Extension/GLHDiff.extension.st @@ -4,7 +4,5 @@ Extension { #name : #GLHDiff } GLHDiff >> name [ - - ^ new_path contractTo: 100 ] diff --git a/src/GitLabHealth-Model-Extension/MooseAbstractGroup.extension.st b/src/GitLabHealth-Model-Extension/MooseAbstractGroup.extension.st index 6817f5f..605908c 100644 --- a/src/GitLabHealth-Model-Extension/MooseAbstractGroup.extension.st +++ b/src/GitLabHealth-Model-Extension/MooseAbstractGroup.extension.st @@ -5,6 +5,8 @@ MooseAbstractGroup >> add: anElmt unless: aConditionAsBlock [ "()>>>" self addAll: { anElmt } unless: aConditionAsBlock + + ] { #category : #'*GitLabHealth-Model-Extension' } diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index b4ac60e..28963d0 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -122,6 +122,25 @@ GLHModelImporter >> glhModel: anObject [ glhModel := anObject ] +{ #category : #'as yet unclassified' } +GLHModelImporter >> importCommitOfProject: anProject withId: anID [ + + | commit result | + anID ifNil: [ ^ nil ]. + + result := self glhApi commit: anID ofProject: anProject id. + commit := (self parseCommitsResult: '[' , result , ']') first. + + self glhModel add: commit unless: self blockOnIdEquality. + commit repository: anProject repository . + + self withCommitDiffs ifTrue: [ + self importDiffOfCommit: commit. + ]. + + ^ commit +] + { #category : #'private - api' } GLHModelImporter >> importCommits: aGLHProject [ "limited to the last 20 commits" @@ -395,6 +414,59 @@ GLHModelImporter >> importGroup: aGroupID [ ^ groupResult ] +{ #category : #commit } +GLHModelImporter >> importParentCommitsOfCommit: aGLHCommit forOnly: number [ + + | parentsIds commits | + commits := OrderedCollection new. + number = 0 ifTrue: [ + ^ commits + add: aGLHCommit; + yourself ]. + + parentsIds := aGLHCommit parent_ids. + + + + commits addAll: (parentsIds collect: [ :id | + self + importCommitOfProject: aGLHCommit repository project + withId: id. + (self selectEntityType: GLHCommit overAttribut: #id equalTo: id) + first ]). + + + + ^ commits , (commits collect: [ :parentCommit | + self + importParentCommitsOfCommit: parentCommit + forOnly: number - 1 ]) flatten +] + +{ #category : #commit } +GLHModelImporter >> importParentCommitsOfCommit: aGLHCommit since: aDate [ + + | parentsIds commits | + commits := OrderedCollection new. + aGLHCommit created_at asDateAndTime < aDate asDateAndTime ifTrue: [ + + ^ commits + add: aGLHCommit; + yourself ]. + + parentsIds := aGLHCommit parent_ids. + + commits addAll: (parentsIds collect: [ :id | + self + importCommitOfProject: aGLHCommit repository project + withId: id ]). + + + ^ (commits collect: [ :parentCommit | + self importParentCommitsOfCommit: parentCommit since: aDate ]) + flatten +] + { #category : #'private - api' } GLHModelImporter >> importRepository: aGLHRepository [ @@ -601,6 +673,13 @@ GLHModelImporter >> pipelinesOf: aProjectID [ ^ self parsePipelinesResult: result ] +{ #category : #'as yet unclassified' } +GLHModelImporter >> selectEntityType: aType overAttribut: aSelector equalTo: value [ + + ^ (self glhModel allWithType: aType) + select: [ :entity | (entity perform: aSelector) = value ] +] + { #category : #'private - api' } GLHModelImporter >> subGroupsOf: aGroupID [ diff --git a/src/GitLabHealth-Model/GLHCommit.class.st b/src/GitLabHealth-Model/GLHCommit.class.st index cf5c13c..d74f66a 100644 --- a/src/GitLabHealth-Model/GLHCommit.class.st +++ b/src/GitLabHealth-Model/GLHCommit.class.st @@ -299,7 +299,6 @@ GLHCommit >> diffs [ - ^ diffs ] diff --git a/src/GitLabHealth-Model/GLHUser.class.st b/src/GitLabHealth-Model/GLHUser.class.st index 7a212f1..c639b44 100644 --- a/src/GitLabHealth-Model/GLHUser.class.st +++ b/src/GitLabHealth-Model/GLHUser.class.st @@ -30,8 +30,8 @@ A GitLab User | `job_title` | `String` | nil | | | `linkedin` | `String` | nil | | | `location` | `String` | nil | | -| `name` | `String` | nil | Basic name of the entity, not full reference.| | `name` | `String` | nil | | +| `name` | `String` | nil | Basic name of the entity, not full reference.| | `organization` | `String` | nil | | | `pronouns` | `String` | nil | | | `public_email` | `String` | nil | | diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index bf87990..31a1ff6 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -17,6 +17,10 @@ GLPHModelImporter >> blockForDiffRangeEquality [ GLPHModelImporter >> chainsCommitsFrom: commitsCollection [ commitsCollection do: [ :commit | + + self haltIf: [ + commit id = 'f6fcfae70cf6a8506c90de7e33270cd95a35f260' ]. + commit parent_ids do: [ :parentId | | parentCommit | parentCommit := (self glhModel allWithType: GLHCommit) @@ -26,12 +30,8 @@ GLPHModelImporter >> chainsCommitsFrom: commitsCollection [ parentCommit childCommits add: commit unless: [ :existing :new | existing id = new id ]. - - ^ commitsCollection - - "parentCommit childCommits - detect: [ :c | c id = commit id ] - ifNone: [ parentCommit addChildCommit: commit ] " ] ] ] + ] ] ]. + ^ commitsCollection ] { #category : #commit } @@ -84,25 +84,6 @@ GLPHModelImporter >> completeImportProject: aGLHProject [ ] -{ #category : #'as yet unclassified' } -GLPHModelImporter >> importCommitOfProject: anProject withId: anID [ - - | commit result | - anID ifNil: [ ^ nil ]. - - result := self glhApi commit: anID ofProject: anProject id. - commit := (self parseCommitsResult: '[' , result , ']') first. - - self glhModel add: commit unless: self blockOnIdEquality. - commit repository: anProject repository . - - self withCommitDiffs ifTrue: [ - self importDiffOfCommit: commit. - ]. - - ^ commit -] - { #category : #import } GLPHModelImporter >> importCommitsOfBranch: aGLHBranch [ From 4a9139c223e587199051c0ad839f874d6424c66b Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Fri, 26 Apr 2024 17:12:07 +0200 Subject: [PATCH 26/77] add visualisation for commit chains --- .../GLPHModelImporter.class.st | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 31a1ff6..5088c0f 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -18,8 +18,6 @@ GLPHModelImporter >> chainsCommitsFrom: commitsCollection [ commitsCollection do: [ :commit | - self haltIf: [ - commit id = 'f6fcfae70cf6a8506c90de7e33270cd95a35f260' ]. commit parent_ids do: [ :parentId | | parentCommit | From 772af0188ff96b2b58354a935e4670641513a1ef Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 2 May 2024 13:27:46 +0200 Subject: [PATCH 27/77] add first visu for code churn --- .../CodeChurnAnalyzer.class.st | 85 ++++++++++++++++++- .../GLHModelImporter.class.st | 34 +++++--- 2 files changed, 104 insertions(+), 15 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st index e9e0914..cb7a60c 100644 --- a/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st @@ -3,13 +3,92 @@ Class { #superclass : #Object, #instVars : [ 'glModel', - 'fromCommit' + 'fromCommit', + 'glhImporter' ], #category : #'GitLabHealth-Model-Analysis' } +{ #category : #'as yet unclassified' } +CodeChurnAnalyzer >> analyseChurn [ + + |commitFiles| + + commitFiles := self impactedFilesInFollowUpCommitsOf: fromCommit. + + commitFiles associations do: [ :assoc | + |file impactedCommits ranges| + file := assoc key. + impactedCommits := assoc value. + 1 halt. + ranges := impactedCommits collect: [ :range | + ranges + ] + + ]. + + + 1 halt. + +] + +{ #category : #accessing } +CodeChurnAnalyzer >> fromCommit: aCommit [ + fromCommit := aCommit. +] + +{ #category : #accessing } +CodeChurnAnalyzer >> glhImporter: anImporter [ + glhImporter := anImporter . +] + +{ #category : #'as yet unclassified' } +CodeChurnAnalyzer >> impactedFilesInFollowUpCommitsOf: aGLHCommit [ + + | commitFiles | + commitFiles := (fromCommit diffs collect: [ :diff | + diff new_path -> Set new ]) + asDictionary. + + self + visitChildCommits: fromCommit childCommits + lookingForFiles: commitFiles. + + ^ commitFiles +] + { #category : #initialization } -CodeChurnAnalyzer >> initialize [ +CodeChurnAnalyzer >> initialize [ + glModel := GLPHEModel new. - fromCommit := GLHCommit new. + fromCommit := GLHCommit new. + glhImporter := GLPHModelImporter new. + +] + +{ #category : #'as yet unclassified' } +CodeChurnAnalyzer >> onModel: agitHealthModel [ + glModel := agitHealthModel +] + +{ #category : #visiting } +CodeChurnAnalyzer >> visitChildCommits: commits lookingForFiles: commitFiles [ + + commits ifEmpty: [ ^ commitFiles ]. + + commits do: [ :commit | + | files | + files := commit diffs collect: [ :diff | diff ]. + + files do: [ :diff | + commitFiles + at: diff new_path + ifPresent: [ :v | v add: commit -> diff diffRanges ] + ifAbsent: [ ] ]. + + self + visitChildCommits: commit childCommits + lookingForFiles: commitFiles ]. + + ^ commitFiles ] diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index 28963d0..9aec433 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -98,6 +98,13 @@ GLHModelImporter >> convertApiFileAsFile: aAPIFile [ yourself ] +{ #category : #'as yet unclassified' } +GLHModelImporter >> detectEntityType: aType overAttribut: aSelector equalTo: value [ + + ^ (self glhModel allWithType: aType) detect: [ :entity | + (entity perform: aSelector) = value ] ifNone: [ nil ]. +] + { #category : #accessing } GLHModelImporter >> glhApi [ @@ -128,16 +135,22 @@ GLHModelImporter >> importCommitOfProject: anProject withId: anID [ | commit result | anID ifNil: [ ^ nil ]. - result := self glhApi commit: anID ofProject: anProject id. - commit := (self parseCommitsResult: '[' , result , ']') first. + commit := (self + detectEntityType: GLHCommit + overAttribut: #id + equalTo: anID) ifNil: [ + result := self glhApi commit: anID ofProject: anProject id. + commit := (self parseCommitsResult: '[' , result , ']') + first. + + self glhModel add: commit unless: self blockOnIdEquality. + commit repository: anProject repository. + + commit ]. + + self withCommitDiffs ifTrue: [ + self importDiffOfCommit: commit ]. - self glhModel add: commit unless: self blockOnIdEquality. - commit repository: anProject repository . - - self withCommitDiffs ifTrue: [ - self importDiffOfCommit: commit. - ]. - ^ commit ] @@ -426,8 +439,6 @@ GLHModelImporter >> importParentCommitsOfCommit: aGLHCommit forOnly: number [ parentsIds := aGLHCommit parent_ids. - - commits addAll: (parentsIds collect: [ :id | self importCommitOfProject: aGLHCommit repository project @@ -436,7 +447,6 @@ GLHModelImporter >> importParentCommitsOfCommit: aGLHCommit forOnly: number [ first ]). - ^ commits , (commits collect: [ :parentCommit | self importParentCommitsOfCommit: parentCommit From 0210cca97d20a2843815b1844d91a15f2c9b21e8 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 2 May 2024 15:32:32 +0200 Subject: [PATCH 28/77] add visualization --- .../RSCommit.class.st | 437 ++++++++++++++++++ .../RSCommitDiff.class.st | 172 +++++++ src/GitLabHealth-Visualization/package.st | 1 + 3 files changed, 610 insertions(+) create mode 100644 src/GitLabHealth-Visualization/RSCommit.class.st create mode 100644 src/GitLabHealth-Visualization/RSCommitDiff.class.st create mode 100644 src/GitLabHealth-Visualization/package.st diff --git a/src/GitLabHealth-Visualization/RSCommit.class.st b/src/GitLabHealth-Visualization/RSCommit.class.st new file mode 100644 index 0000000..90f6c68 --- /dev/null +++ b/src/GitLabHealth-Visualization/RSCommit.class.st @@ -0,0 +1,437 @@ +Class { + #name : #RSCommit, + #superclass : #Object, + #instVars : [ + 'aCommit', + 'canvas' + ], + #category : #'GitLabHealth-Visualization' +} + +{ #category : #'as yet unclassified' } +RSCommit class >> canvasCompleteHierachyFor: aGLHCommit [ + + ^ self new + commit: aGLHCommit; + buildCommitCompleteHierachyCanvas +] + +{ #category : #'as yet unclassified' } +RSCommit class >> canvasDiffImpactFor: aGLHCommit [ + + ^ self new + commit: aGLHCommit; + buildCommitDiffFileImpact +] + +{ #category : #'as yet unclassified' } +RSCommit class >> canvasHierarchyFor: aGLHCommit [ + + ^ self new + commit: aGLHCommit; + buildCommitHierachyCanvas +] + +{ #category : #shape } +RSCommit >> allChildrenShapesFrom: aGLHCommit [ + + | children sum | + children := RSGroup new. + sum := RSGroup new. + + + children addAll: (aGLHCommit childCommits collect: [ :child | + self shapeForCommit: child ]). + + "^ children + , + ((children collect: [ :commit | + self allChildrenShapesFrom: commit model ]) flattened)" + sum := ((children collect: [ :commit | + self allChildrenShapesFrom: commit model ]) flattened). + + ^ children, sum . +] + +{ #category : #shape } +RSCommit >> allChildrenShapesFrom: aGLHCommit into: dic [ + + | children sum | + children := RSGroup new. + sum := RSGroup new. + + children addAll: (aGLHCommit childCommits collect: [ :child | + dic at: child put: (self shapeForCommit: child) ]). + + (children do: [ :commit | + self allChildrenShapesFrom: commit model into: dic ]) flattened +] + +{ #category : #canvas } +RSCommit >> buildCommitCompleteHierachyCanvas [ + + | shapes parents children commitShape dic churnAnalyzer files2Commits shape2change | + canvas := RSCanvas new. + shapes := RSGroup new. + parents := RSGroup new. + children := RSGroup new. + commitShape := RSGroup new. + dic := Dictionary new. + + + churnAnalyzer := CodeChurnAnalyzer new fromCommit: aCommit. + files2Commits := churnAnalyzer impactedFilesInFollowUpCommitsOf: + aCommit. + files2Commits := files2Commits select: [ :entry | + entry value isNotNil ]. + + + + + commitShape add: (self shapeForCommit: aCommit). + + self allChildrenShapesFrom: aCommit into: dic. + children addAll: dic values. + + shape2change := (children collect: [ :shape | + ((files2Commits values collect: [ :group | + group asDictionary + at: shape model + ifPresent: [ shape ] ]) reject: #isNil) asSet ]) + flattened. + + + shape2change do: [ :composite | + (composite children select: [ :b | b class = RSBox ]) color: + Color red ]. + + + + "canvas addAllShapes: commitShape , parents , children." + self createLinesBetweenCommits: commitShape , children. + + canvas addAll: commitShape , children. + RSHorizontalTreeLayout new + on: canvas nodes. + + canvas @ RSCanvasController. + + ^ canvas +] + +{ #category : #canvas } +RSCommit >> buildCommitDiffFileImpact [ + + | shapes | + canvas := RSCanvas new. + shapes := RSGroup new. + + + aCommit diffsGroup do: [ :diff | + shapes add: + (RSCommitDiff new + shapeForDiffRange: diff + withColor: Color lightBlue) ]. + + + + RSVerticalLineLayout new + gapSize: 5 ; + on: shapes. + + + canvas addAllShapes: shapes. + canvas @ RSCanvasController. + + ^ canvas +] + +{ #category : #canvas } +RSCommit >> buildCommitHierachyCanvas [ + "build the commit graph from a starting root commit toward all of the descendants (children commit). Regardless of branches. Also display in red border the following commits that touch on the same files commited by the root commit. " + + | shapes parents children commitShape group | + canvas := RSCanvas new. + shapes := RSGroup new. + parents := RSGroup new. + children := RSGroup new. + commitShape := RSGroup new. + + + commitShape add: (self shapeForCommit: aCommit). + + children addAll: (aCommit childCommits collect: [ :c | + self shapeForCommit: c withColor: self childColor ]). + + RSVerticalLineLayout new + alignCenter; + gapSize: 100; + on: children. + + parents addAll: (aCommit parentCommits collect: [ :c | + self shapeForCommit: c withColor: self parentColor ]). + + RSVerticalLineLayout new + alignCenter; + gapSize: 100; + on: parents. + + "canvas addAllShapes: commitShape , parents , children." + self createLinesBetweenCommits: commitShape , parents , children. + + group := RSGroup new + add: parents; + add: commitShape; + add: children; + yourself. + + RSHorizontalLineLayout new + gapSize: 100; + on: group. + + + + + + "canvas addShape: (RSComposite new + shapes: group; + updateConnectedLines; + yourself)." + + canvas addAll: commitShape , parents , children. + self createLinesBetweenCommits: commitShape , parents , children. + + canvas @ RSCanvasController. + + ^ canvas +] + +{ #category : #shape } +RSCommit >> buildCompositeLayer: shapes withName: name withColor: color [ + + | shapesCompo label boxGroup | + boxGroup := RSGroup new. + label := RSLabel new + text: name; + fontSize: 50; + bold; + yourself. + + RSVerticalLineLayout on: shapes. + shapesCompo := RSComposite new + shapes: shapes; + yourself. + "shapes do: [ :shp | canvas removeShape: shp ]." + shapesCompo extent: shapesCompo extent + 5. + + "create the group for one class and its method" + boxGroup + add: label; + add: shapesCompo. + RSVerticalLineLayout new alignCenter on: boxGroup. + + ^ RSComposite new + shapes: boxGroup; + " model: shapes;" + color: (color alpha: 0.1); + popup; + yourself +] + +{ #category : #canvas } +RSCommit >> buildDiffCanvas [ + + | shapes parents children commitShape group | + canvas := RSCanvas new. + shapes := RSGroup new. + parents := RSGroup new. + children := RSGroup new. + commitShape := RSGroup new. + + commitShape add: (self shapeForCommit: aCommit). + + children addAll: (aCommit childCommits collect: [ :c | + self shapeForCommit: c withColor: self childColor ]). + + RSVerticalLineLayout new + alignCenter; + gapSize: 100; + on: children. + + parents addAll: (aCommit parentCommits collect: [ :c | + self shapeForCommit: c withColor: self parentColor ]). + + RSVerticalLineLayout new + alignCenter; + gapSize: 100; + on: parents. + + "canvas addAllShapes: commitShape , parents , children." + self createLinesBetweenCommits: commitShape , parents , children. + + group := RSGroup new + add: parents; + add: commitShape; + add: children; + yourself. + + RSHorizontalLineLayout new + gapSize: 100; + on: group. + + + + "canvas addShape: (RSComposite new + shapes: group; + updateConnectedLines; + yourself)." + + canvas addAll: commitShape , parents , children. + self createLinesBetweenCommits: commitShape , parents , children. + + canvas @ RSCanvasController. + + ^ canvas +] + +{ #category : #color } +RSCommit >> childColor [ + + ^ Color green +] + +{ #category : #accessing } +RSCommit >> commit: aGLHCommit [ + aCommit := aGLHCommit + +] + +{ #category : #color } +RSCommit >> commitColor [ + + ^ Color blue +] + +{ #category : #shape } +RSCommit >> createChildrenGroupFor: aGLHCommit [ + |group| + group := RSGroup new. + + group add: ((aGLHCommit childCommits collect: [ :child | + |children| + children := RSGroup new. + + children addAll: (child childCommits collect: [ :c | + self shapeForCommit: c withColor: self childColor ]). + + RSVerticalLineLayout new + alignCenter; + gapSize: 100; + on: children. + + children + + ]) reject: #nil). + + + ^ group +] + +{ #category : #lines } +RSCommit >> createLinesBetweenCommits [ + + (RSLineBuilder arrowedLine + withBorderAttachPoint; + color: (Color gray alpha: 0.9); + shapes: canvas shapes; + canvas: canvas; + beDirectional; + connectToAll: [ :commit | commit childCommits ]) pushBack +] + +{ #category : #lines } +RSCommit >> createLinesBetweenCommits: shapes [ + + (RSLineBuilder arrowedLine + withBorderAttachPoint; + color: Color black; + canvas: canvas; + shapes: shapes; + beDirectional; + connectToAll: [ :commit | commit childCommits ]) pushFront +] + +{ #category : #color } +RSCommit >> defaultColor [ + + ^ Color lightBlue +] + +{ #category : #initialization } +RSCommit >> initialize [ + canvas := RSCanvas new. + aCommit := GLHCommit new. +] + +{ #category : #color } +RSCommit >> parentColor [ + ^ Color gray. +] + +{ #category : #shape } +RSCommit >> shapeForCommit: aGLHCommit [ + + ^ self shapeForCommit: aGLHCommit withColor: self defaultColor. +] + +{ #category : #shape } +RSCommit >> shapeForCommit: aGLHCommit withColor: color [ + + | box group label impactedFiles | + group := RSGroup new. + impactedFiles := RSGroup new. + label := RSLabel new + text: (aGLHCommit name truncateWithElipsisTo: 30); + yourself. + + "aGLHCommit diffsGroup do: [ :diff | + impactedFiles add: + (RSCommitDiff new + shapeForDiffRange: diff + withColor: Color lightBlue) ]. + RSVerticalLineLayout new alignCenter on: impactedFiles. + + + box := RSBox new + height: + (impactedFiles sum: [ :composite | composite height ]); + width: (impactedFiles max: [ :composite | composite width ]); + color: color. + + group + add: label; + add: (RSComposite new shapes: (RSGroup new + add: box; + addAll: impactedFiles; + yourself)). + RSVerticalLineLayout new alignCenter on: group. + " + + box := RSBox new + height: 50; + width: 60; + color: color. + + group + add: label; + add: box. + + RSVerticalLineLayout new alignCenter on: group. + + ^ RSComposite new + shapes: group; + model: aGLHCommit; + popup; + draggable; + yourself +] diff --git a/src/GitLabHealth-Visualization/RSCommitDiff.class.st b/src/GitLabHealth-Visualization/RSCommitDiff.class.st new file mode 100644 index 0000000..a6680b2 --- /dev/null +++ b/src/GitLabHealth-Visualization/RSCommitDiff.class.st @@ -0,0 +1,172 @@ +Class { + #name : #RSCommitDiff, + #superclass : #Object, + #instVars : [ + 'aDiff', + 'canvas' + ], + #category : #'GitLabHealth-Visualization' +} + +{ #category : #'as yet unclassified' } +RSCommitDiff class >> canvasFor: aGLHDiff [ + + ^ self new + diffCommit: aGLHDiff; + buildCanvas +] + +{ #category : #hooks } +RSCommitDiff class >> shapeFor: aCommitDiff [ + ^ self new + diffCommit: aCommitDiff; + shapeForDiff: aCommitDiff withColor: Color gray. + + +] + +{ #category : #canvas } +RSCommitDiff >> buildCanvas [ + + | shapes | + self + deprecated: 'Use #build instead' + on: '2 May 2024' + in: + 'Pharo-11.0.0+build.726.sha.aece1b5473acf3830a0e082c1bc3a15d4ff3522b (64 Bit)'. + canvas := RSCanvas new. + shapes := RSGroup new. + + aDiff diffRanges collect: [ :diffRange | + self shapeForDiff: diffRange withColor: Color gray ]. + + + canvas @ RSCanvasController. + + ^ canvas +] + +{ #category : #shape } +RSCommitDiff >> buildCompositeLayer: shapes withName: name withColor: color [ + + | shapesCompo label boxGroup | + boxGroup := RSGroup new. + label := RSLabel new + text: name; + fontSize: 20; + bold; + yourself. + + RSVerticalLineLayout on: shapes. + shapesCompo := RSComposite new + shapes: shapes; + yourself. + "shapes do: [ :shp | canvas removeShape: shp ]." + shapesCompo extent: shapesCompo extent + 5. + + "create the group for one class and its method" + boxGroup + add: label; + add: shapesCompo. + RSVerticalLineLayout new alignCenter on: boxGroup. + + ^ RSComposite new + shapes: boxGroup; + " model: shapes;"color: (color alpha: 0.1); + popup; + yourself +] + +{ #category : #lines } +RSCommitDiff >> createLinesBetweenCommits: shapes [ + + (RSLineBuilder arrowedLine + withBorderAttachPoint; + color: Color black; + canvas: canvas; + shapes: shapes; + beDirectional; + connectToAll: [ :diff | diff childCommits ]) pushFront +] + +{ #category : #setter } +RSCommitDiff >> diffCommit: aGLHDiff [ + aDiff := aGLHDiff +] + +{ #category : #shape } +RSCommitDiff >> shapeForDiff: aGLHDiff withColor: color [ + + | box group label | + group := RSGroup new. + label := RSLabel new + text: + (aGLHDiff new_path asPath segments last + truncateWithElipsisTo: 50); + yourself. + box := RSBox new + height: label height; + width: label width; + color: color. + group + add: label; + add: box. + "RSVerticalLineLayout new alignCenter on: group." + + ^ RSComposite new + shapes: group; + model: aGLHDiff; + popup; + draggable; + yourself +] + +{ #category : #shape } +RSCommitDiff >> shapeForDiffRange: aGLPHEDiffRange withColor: aColor [ + + | box group label | + group := RSGroup new. + label := RSLabel new + text: aGLPHEDiffRange new_path asPath segments last; + yourself. + box := RSBox new + height: label height + 5; + width: label width + 10; + color: aColor. + group + add: box; + add: label. + "RSVerticalLineLayout new alignCenter on: group." + + ^ RSComposite new + shapes: group; + model: aGLPHEDiffRange; + popup; + draggable; + yourself +] + +{ #category : #shape } +RSCommitDiff >> shapeForRange: aGLHDiff withColor: color [ + + | box group label | + group := RSGroup new. + label := RSLabel new + text: (aGLHDiff new_path truncateWithElipsisTo: 30); + yourself. + box := RSBox new + height: 50; + width: 50; + color: color. + group + add: label; + add: box. + RSVerticalLineLayout new alignCenter on: group. + + ^ RSComposite new + shapes: group; + model: aGLHDiff; + popup; + draggable; + yourself +] diff --git a/src/GitLabHealth-Visualization/package.st b/src/GitLabHealth-Visualization/package.st new file mode 100644 index 0000000..aaa0de3 --- /dev/null +++ b/src/GitLabHealth-Visualization/package.st @@ -0,0 +1 @@ +Package { #name : #'GitLabHealth-Visualization' } From fcab076f585ddfd100c63b75d0af3d9080f174ef Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 2 May 2024 15:40:01 +0200 Subject: [PATCH 29/77] change color --- .../RSCommit.class.st | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/GitLabHealth-Visualization/RSCommit.class.st b/src/GitLabHealth-Visualization/RSCommit.class.st index 90f6c68..7ee8c69 100644 --- a/src/GitLabHealth-Visualization/RSCommit.class.st +++ b/src/GitLabHealth-Visualization/RSCommit.class.st @@ -61,7 +61,7 @@ RSCommit >> allChildrenShapesFrom: aGLHCommit into: dic [ sum := RSGroup new. children addAll: (aGLHCommit childCommits collect: [ :child | - dic at: child put: (self shapeForCommit: child) ]). + dic at: child put: (self shapeForCommit: child withColor: self childColor) ]). (children do: [ :commit | self allChildrenShapesFrom: commit model into: dic ]) flattened @@ -100,19 +100,20 @@ RSCommit >> buildCommitCompleteHierachyCanvas [ ifPresent: [ shape ] ]) reject: #isNil) asSet ]) flattened. - shape2change do: [ :composite | (composite children select: [ :b | b class = RSBox ]) color: Color red ]. + parents addAll: (aCommit parentCommits collect: [ :c | + self shapeForCommit: c withColor: self parentColor ]). + "canvas addAllShapes: commitShape , parents , children." - self createLinesBetweenCommits: commitShape , children. + self createLinesBetweenCommits: parents , commitShape , children. - canvas addAll: commitShape , children. - RSHorizontalTreeLayout new - on: canvas nodes. + canvas addAll: parents, commitShape , children. + RSHorizontalTreeLayout new on: canvas nodes. canvas @ RSCanvasController. @@ -297,7 +298,7 @@ RSCommit >> buildDiffCanvas [ { #category : #color } RSCommit >> childColor [ - ^ Color green + ^ Color lightGreen ] { #category : #accessing } @@ -364,7 +365,7 @@ RSCommit >> createLinesBetweenCommits: shapes [ { #category : #color } RSCommit >> defaultColor [ - ^ Color lightBlue + ^ Color lightOrange ] { #category : #initialization } @@ -375,7 +376,8 @@ RSCommit >> initialize [ { #category : #color } RSCommit >> parentColor [ - ^ Color gray. + + ^ Color gray ] { #category : #shape } From b28aea30826558b7e687abfeae0f4be96cf45459 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Tue, 7 May 2024 10:50:25 +0200 Subject: [PATCH 30/77] add commit churn metric --- .../GLPHEAddition.extension.st | 13 +++ .../GLPHEChange.extension.st | 12 +++ .../GLPHEDeletion.extension.st | 11 +++ .../CodeChurnAnalyzer.class.st | 83 ++++++++++++++++--- .../RSCommit.class.st | 4 +- 5 files changed, 111 insertions(+), 12 deletions(-) create mode 100644 src/GLPHExtended-Model-Extension/GLPHEAddition.extension.st create mode 100644 src/GLPHExtended-Model-Extension/GLPHEDeletion.extension.st diff --git a/src/GLPHExtended-Model-Extension/GLPHEAddition.extension.st b/src/GLPHExtended-Model-Extension/GLPHEAddition.extension.st new file mode 100644 index 0000000..dfe9f2d --- /dev/null +++ b/src/GLPHExtended-Model-Extension/GLPHEAddition.extension.st @@ -0,0 +1,13 @@ +Extension { #name : #GLPHEAddition } + +{ #category : #'*GLPHExtended-Model-Extension' } +GLPHEAddition >> isAddition [ + ^ true + +] + +{ #category : #'*GLPHExtended-Model-Extension' } +GLPHEAddition >> isDeletion [ + + ^ false +] diff --git a/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st b/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st index c51cb9c..d3a8f24 100644 --- a/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st +++ b/src/GLPHExtended-Model-Extension/GLPHEChange.extension.st @@ -15,6 +15,18 @@ GLPHEChange class >> createFrom: aDiffLine [ ^ aChange ] +{ #category : #'*GLPHExtended-Model-Extension' } +GLPHEChange >> isAddition [ + + ^ false +] + +{ #category : #'*GLPHExtended-Model-Extension' } +GLPHEChange >> isDeletion [ + + ^ false +] + { #category : #'*GLPHExtended-Model-Extension' } GLPHEChange >> name [ diff --git a/src/GLPHExtended-Model-Extension/GLPHEDeletion.extension.st b/src/GLPHExtended-Model-Extension/GLPHEDeletion.extension.st new file mode 100644 index 0000000..010db38 --- /dev/null +++ b/src/GLPHExtended-Model-Extension/GLPHEDeletion.extension.st @@ -0,0 +1,11 @@ +Extension { #name : #GLPHEDeletion } + +{ #category : #'*GLPHExtended-Model-Extension' } +GLPHEDeletion >> isAddition [ + ^ false +] + +{ #category : #'*GLPHExtended-Model-Extension' } +GLPHEDeletion >> isDeletion [ + ^ false. +] diff --git a/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st index cb7a60c..bde0504 100644 --- a/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st @@ -12,24 +12,64 @@ Class { { #category : #'as yet unclassified' } CodeChurnAnalyzer >> analyseChurn [ - |commitFiles| - + | commitFiles churn| commitFiles := self impactedFilesInFollowUpCommitsOf: fromCommit. + + churn := self computeChurnOnFiles: commitFiles. + +] + +{ #category : #churn } +CodeChurnAnalyzer >> computeChurnOnFiles: aCollection [ - commitFiles associations do: [ :assoc | - |file impactedCommits ranges| + |changesDic perFileChanges churns | + + "1 -> (a GLPHEChange -> NumberOfChurnDetected)" + changesDic := Dictionary new. + + + perFileChanges := aCollection associations collect: [ :assoc | + + (assoc key -> (self computeSpecificChurnOf: assoc value )). + ] . + + churns := perFileChanges collect: [ :assoc | + |churn file results| + file := assoc key. - impactedCommits := assoc value. - 1 halt. - ranges := impactedCommits collect: [ :range | - ranges - ] + results := assoc value. + churn := ((results values select: [ :a | a value > 1 ] )size)/(results values sum: [ :a | a value ]). + (file -> churn) + ]. + + ^ {(#churns -> churns). + (#details -> perFileChanges)} asDictionary +] + +{ #category : #churn } +CodeChurnAnalyzer >> computeSpecificChurnOf: commit2Changes [ + |changesDic| + "1 -> (a GLPHEChange -> NumberOfChurnDetected)" + changesDic := OrderedDictionary new. + + commit2Changes do: [ :entry | + |commit diffRanges| + commit := entry key. + diffRanges := entry value. + + diffRanges do: [ :diff | + |from upTo| + + from := (diff originalLineRange copyFrom: (diff originalLineRange indexOf: $-)+1 to: (diff originalLineRange indexOf: $,)-1 ) asString asNumber . + self insertDiff: diff into: changesDic startingFrom: from. + ]. + ]. - 1 halt. + ^ self sortChangeDic: changesDic. ] { #category : #accessing } @@ -66,11 +106,34 @@ CodeChurnAnalyzer >> initialize [ ] +{ #category : #insertion } +CodeChurnAnalyzer >> insertDiff: aGLPHEDiffRange into: fileChangesDic startingFrom: from [ + |index| + index := from. + aGLPHEDiffRange changes do: [ :aChange | + + aChange isAddition ifTrue: [ + fileChangesDic at: index ifPresent: [ :current | + + current key add: aChange. + current value: current value + 1. ] ifAbsentPut: [((OrderedCollection new add: aChange; yourself) -> 1 ) ]. + ]. + + aChange isDeletion ifFalse: [ index := index + 1 ]. + + ] +] + { #category : #'as yet unclassified' } CodeChurnAnalyzer >> onModel: agitHealthModel [ glModel := agitHealthModel ] +{ #category : #sorting } +CodeChurnAnalyzer >> sortChangeDic: aCollection [ + ^ (aCollection associations sortAscending: [ :e | e key ] ) asOrderedDictionary +] + { #category : #visiting } CodeChurnAnalyzer >> visitChildCommits: commits lookingForFiles: commitFiles [ diff --git a/src/GitLabHealth-Visualization/RSCommit.class.st b/src/GitLabHealth-Visualization/RSCommit.class.st index 7ee8c69..975f3f9 100644 --- a/src/GitLabHealth-Visualization/RSCommit.class.st +++ b/src/GitLabHealth-Visualization/RSCommit.class.st @@ -136,8 +136,8 @@ RSCommit >> buildCommitDiffFileImpact [ - RSVerticalLineLayout new - gapSize: 5 ; + RSVerticalLineLayout new + gapSize: 5; on: shapes. From ab796000e9e8d9a67b13a82d6fb0c21af47d2935 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Mon, 13 May 2024 09:07:49 +0200 Subject: [PATCH 31/77] return churn data --- src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st index bde0504..56bff2a 100644 --- a/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st @@ -12,11 +12,11 @@ Class { { #category : #'as yet unclassified' } CodeChurnAnalyzer >> analyseChurn [ - | commitFiles churn| + | commitFiles churn | commitFiles := self impactedFilesInFollowUpCommitsOf: fromCommit. - churn := self computeChurnOnFiles: commitFiles. - + churn := self computeChurnOnFiles: commitFiles. + ^ churn. ] { #category : #churn } From 418b48d6dcb6b5d0213bc77d3bce08bd2474c8e7 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Tue, 14 May 2024 14:19:55 +0200 Subject: [PATCH 32/77] fix some bug in import from gitlab and some churn computation --- .../GLPHEDeletion.extension.st | 2 +- .../CodeChurnAnalyzer.class.st | 5 ++-- .../GLHModelImporter.class.st | 25 +++++++++++++++++-- .../GLPHModelImporter.class.st | 22 ++++++++++++---- 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/GLPHExtended-Model-Extension/GLPHEDeletion.extension.st b/src/GLPHExtended-Model-Extension/GLPHEDeletion.extension.st index 010db38..3178959 100644 --- a/src/GLPHExtended-Model-Extension/GLPHEDeletion.extension.st +++ b/src/GLPHExtended-Model-Extension/GLPHEDeletion.extension.st @@ -7,5 +7,5 @@ GLPHEDeletion >> isAddition [ { #category : #'*GLPHExtended-Model-Extension' } GLPHEDeletion >> isDeletion [ - ^ false. + ^ true. ] diff --git a/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st index 56bff2a..19097f2 100644 --- a/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st @@ -38,7 +38,8 @@ CodeChurnAnalyzer >> computeChurnOnFiles: aCollection [ file := assoc key. results := assoc value. - churn := ((results values select: [ :a | a value > 1 ] )size)/(results values sum: [ :a | a value ]). + results values ifEmpty: [ churn := nil. ] ifNotEmpty: [ churn := ((results values select: [ :a | a value > 1 ] )size) / (results values sum: [ :a | a value ]) . ]. + (file -> churn) ]. @@ -87,7 +88,7 @@ CodeChurnAnalyzer >> impactedFilesInFollowUpCommitsOf: aGLHCommit [ | commitFiles | commitFiles := (fromCommit diffs collect: [ :diff | - diff new_path -> Set new ]) + diff new_path -> (Set new add: (aGLHCommit -> (diff diffRanges) ); yourself) ]) asDictionary. self diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index 9aec433..d598c4e 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -105,6 +105,20 @@ GLHModelImporter >> detectEntityType: aType overAttribut: aSelector equalTo: val (entity perform: aSelector) = value ] ifNone: [ nil ]. ] +{ #category : #'as yet unclassified' } +GLHModelImporter >> filterCommitChanges: aCollection [ + |changes| + + "reject file infos" + changes := aCollection. + changes := changes reject: [ :line | line matchesRegex: '---.*' ]. + changes := changes reject: [ :line | line matchesRegex: '\+\+\+.*' ]. + changes := changes reject: [ :line | line matchesRegex: '\\ No newline at end of file' ]. + + +^ changes. +] + { #category : #accessing } GLHModelImporter >> glhApi [ @@ -526,13 +540,20 @@ GLHModelImporter >> importUserByUsername: anUsername [ recordInfo. result := self glhApi usersSearchByUsername: anUsername. searchResult := NeoJSONReader fromString: result. - searchResult ifEmpty: [ GLHUser new ] ifNotEmpty: [ + + ((searchResult at: #message) includesSubstring: '403 Forbidden') ifTrue: [ + GLHUser new username: anUsername. + ] ifFalse: [ + searchResult ifEmpty: [ GLHUser new ] ifNotEmpty: [ userId := searchResult first at: #id. (self glhModel allWithType: GLHUser) detect: [ :user | user id = userId ] ifNone: [ self importUser: userId. - ] ] ] + ] ] + ]. + + ] ] { #category : #initialization } diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 5088c0f..8bd2d01 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -239,15 +239,21 @@ GLPHModelImporter >> initialize [ GLPHModelImporter >> parseDiffString: aDiff [ "parse diff string to create entities for each lines" - | lines range index | + | lines range index firstChangeRange| lines := aDiff diffString lines. index := 0. range := nil. + + + + lines := self filterCommitChanges: lines. + lines do: [ :line | | aChange | + firstChangeRange := false. (line trim beginsWith: #'@@') ifTrue: [ range := GLPHEDiffRange newFrom: line. - + firstChangeRange := true. aDiff diffRanges add: range unless: [ :existing :new | @@ -255,14 +261,20 @@ GLPHModelImporter >> parseDiffString: aDiff [ existing newLineRange = new newLineRange ] ] ]. self withCommitDiffs ifTrue: [ - aChange := GLPHEChange newFrom: line. + + firstChangeRange ifFalse: [ + aChange := GLPHEChange newFrom: line. aChange relativeIndex: index. range changes add: aChange unless: [ :existing :new | existing sourceCode = new sourceCode and: [ existing index = new index ] ]. - index := index + 1 ] ]. - + index := index + 1 + ]. + + ] + ]. + "add changes to the diff range" aDiff diffRanges do: [ :diffRange | self glhModel addAll: diffRange changes unless: [ :existing :new | (existing diffRange mooseID = new diffRange mooseID and: [ From 08f0ed0ac8340d2257a5a53663788ab7f524e0b9 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Tue, 14 May 2024 15:09:10 +0200 Subject: [PATCH 33/77] correction for churn --- src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st index 19097f2..4667272 100644 --- a/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st @@ -53,7 +53,8 @@ CodeChurnAnalyzer >> computeSpecificChurnOf: commit2Changes [ "1 -> (a GLPHEChange -> NumberOfChurnDetected)" changesDic := OrderedDictionary new. - commit2Changes do: [ :entry | + + (commit2Changes sortAscending: [ :assoc | assoc key created_at ]) do: [ :entry | |commit diffRanges| commit := entry key. diffRanges := entry value. @@ -61,7 +62,8 @@ CodeChurnAnalyzer >> computeSpecificChurnOf: commit2Changes [ diffRanges do: [ :diff | |from upTo| - from := (diff originalLineRange copyFrom: (diff originalLineRange indexOf: $-)+1 to: (diff originalLineRange indexOf: $,)-1 ) asString asNumber . + from := (diff originalLineRange copyFrom: (diff originalLineRange indexOf: $-)+1 to: (diff originalLineRange indexOf: $,)-1 ) asString asNumber. + (from = 0) ifTrue:[from := 1]. self insertDiff: diff into: changesDic startingFrom: from. ]. From 2e4e05d120920c1de6bdd2275a70a47bac981a55 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Tue, 14 May 2024 16:25:30 +0200 Subject: [PATCH 34/77] add tests --- .../CodeChurnAnalyzerTest.class.st | 75 +++++++++++++++++++ .../package.st | 1 + .../CodeChurnAnalyzer.class.st | 2 +- .../GLHModelImporterTest.class.st | 34 +++++++++ .../package.st | 1 + 5 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 src/GitLabHealth-Model-Analysis-Tests/CodeChurnAnalyzerTest.class.st create mode 100644 src/GitLabHealth-Model-Analysis-Tests/package.st create mode 100644 src/GitLabProjectHealth-Model-Importer-Tests/package.st diff --git a/src/GitLabHealth-Model-Analysis-Tests/CodeChurnAnalyzerTest.class.st b/src/GitLabHealth-Model-Analysis-Tests/CodeChurnAnalyzerTest.class.st new file mode 100644 index 0000000..9efaad0 --- /dev/null +++ b/src/GitLabHealth-Model-Analysis-Tests/CodeChurnAnalyzerTest.class.st @@ -0,0 +1,75 @@ +" +A CodeChurnAnalyzerTest is a test class for testing the behavior of CodeChurnAnalyzer +" +Class { + #name : #CodeChurnAnalyzerTest, + #superclass : #TestCase, + #instVars : [ + 'glphModel', + 'glphApi', + 'glhImporter', + 'projects' + ], + #category : #'GitLabHealth-Model-Analysis-Tests' +} + +{ #category : #running } +CodeChurnAnalyzerTest >> setUp [ + + | since | + super setUp. + + "Put here a common initialization logic for tests" + glphModel := GLPHEModel new. + + glphApi := GLPHApi new + privateToken: ''; + baseAPIUrl: 'https://gitlab.com/api/v4'; + yourself. + + since := Date today - 2 day. + + glhImporter := GLPHModelImporter new + glhApi: glphApi; + glhModel: glphModel; + withFiles: false; + withCommitsSince: 0 day; + withCommitDiffs: true. + + glhImporter importProject: 57841283. + projects := (glhImporter glhModel allWithType: GLHProject). + + +glhImporter importCommitsOProject: projects first since: (since asDate ) until: nil. + + +] + +{ #category : #tests } +CodeChurnAnalyzerTest >> testChurnMetric [ + + | myCommits res wednesdayCommit mondayCommit| + glhImporter withCommitDiffs: true. + + wednesdayCommit := glhImporter importCommitOfProject: projects first id withId: 'f298ec312f81032b611a890f993d98096ab3bbc1'. + + myCommits := (glhImporter + importParentCommitsOfCommit: + wednesdayCommit + forOnly: 2) asSet. + myCommits add: wednesdayCommit. + + glhImporter chainsCommitsFrom: myCommits asSet. + + myCommits do: [ :c | glhImporter completeImportedCommit: c ]. + + mondayCommit := myCommits detect: [ :c | c id = '8a6f665118e6fcfa468d7cce1a7e944ba65812ab' ]. + + res := CodeChurnAnalyzer new + fromCommit: mondayCommit; + glhImporter: glhImporter; + onModel: glphModel; + analyseChurn. + + self assert: (res at:#churns) first value equals: 5/7 +] diff --git a/src/GitLabHealth-Model-Analysis-Tests/package.st b/src/GitLabHealth-Model-Analysis-Tests/package.st new file mode 100644 index 0000000..b9ed58f --- /dev/null +++ b/src/GitLabHealth-Model-Analysis-Tests/package.st @@ -0,0 +1 @@ +Package { #name : #'GitLabHealth-Model-Analysis-Tests' } diff --git a/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st index 4667272..329f2cf 100644 --- a/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st @@ -38,7 +38,7 @@ CodeChurnAnalyzer >> computeChurnOnFiles: aCollection [ file := assoc key. results := assoc value. - results values ifEmpty: [ churn := nil. ] ifNotEmpty: [ churn := ((results values select: [ :a | a value > 1 ] )size) / (results values sum: [ :a | a value ]) . ]. + results values ifEmpty: [ churn := nil. ] ifNotEmpty: [ churn := ((results values select: [ :a | a value > 1 ] )sum: [ :a | a value ]) / (results values sum: [ :a | a value ]) . ]. (file -> churn) ]. diff --git a/src/GitLabHealth-Model-Importer-Tests/GLHModelImporterTest.class.st b/src/GitLabHealth-Model-Importer-Tests/GLHModelImporterTest.class.st index 1105776..9b57af7 100644 --- a/src/GitLabHealth-Model-Importer-Tests/GLHModelImporterTest.class.st +++ b/src/GitLabHealth-Model-Importer-Tests/GLHModelImporterTest.class.st @@ -4,9 +4,43 @@ A GLHModelImporterTest is a test class for testing the behavior of GLHModelImpor Class { #name : #GLHModelImporterTest, #superclass : #TestCase, + #instVars : [ + 'glphModel', + 'glphApi', + 'since', + 'glhImporter' + ], #category : #'GitLabHealth-Model-Importer-Tests' } +{ #category : #'class initialization' } +GLHModelImporterTest class >> initialize [ + + +] + +{ #category : #running } +GLHModelImporterTest >> setUp [ + super setUp. + + "Put here a common initialization logic for tests" + glphModel := GLPHEModel new. + +glphApi := GLPHApi new + privateToken: ''; + baseAPIUrl:'https://gitlab.com/api/v4'; + yourself. + +since:= (Date today - 2 day) . + +glhImporter := GLPHModelImporter new + glhApi: glphApi; + glhModel: glphModel; + withFiles: false; + withCommitsSince: 0 day; + withCommitDiffs: true. +] + { #category : #tests } GLHModelImporterTest >> testImport [ ] diff --git a/src/GitLabProjectHealth-Model-Importer-Tests/package.st b/src/GitLabProjectHealth-Model-Importer-Tests/package.st new file mode 100644 index 0000000..d97321c --- /dev/null +++ b/src/GitLabProjectHealth-Model-Importer-Tests/package.st @@ -0,0 +1 @@ +Package { #name : #'GitLabProjectHealth-Model-Importer-Tests' } From c59e328a122b1fc969f515e71c116a05e8714856 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Wed, 15 May 2024 11:01:41 +0200 Subject: [PATCH 35/77] add amandment metric --- ...Test.class.st => GitAnalyzerTest.class.st} | 56 ++++++++++++++++-- ...Analyzer.class.st => GitAnalyzer.class.st} | 59 +++++++++++++++---- .../RSCommit.class.st | 2 +- 3 files changed, 99 insertions(+), 18 deletions(-) rename src/GitLabHealth-Model-Analysis-Tests/{CodeChurnAnalyzerTest.class.st => GitAnalyzerTest.class.st} (54%) rename src/GitLabHealth-Model-Analysis/{CodeChurnAnalyzer.class.st => GitAnalyzer.class.st} (66%) diff --git a/src/GitLabHealth-Model-Analysis-Tests/CodeChurnAnalyzerTest.class.st b/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st similarity index 54% rename from src/GitLabHealth-Model-Analysis-Tests/CodeChurnAnalyzerTest.class.st rename to src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st index 9efaad0..fe27ed4 100644 --- a/src/GitLabHealth-Model-Analysis-Tests/CodeChurnAnalyzerTest.class.st +++ b/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st @@ -2,19 +2,20 @@ A CodeChurnAnalyzerTest is a test class for testing the behavior of CodeChurnAnalyzer " Class { - #name : #CodeChurnAnalyzerTest, + #name : #GitAnalyzerTest, #superclass : #TestCase, #instVars : [ 'glphModel', 'glphApi', 'glhImporter', - 'projects' + 'projects', + 'res' ], #category : #'GitLabHealth-Model-Analysis-Tests' } { #category : #running } -CodeChurnAnalyzerTest >> setUp [ +GitAnalyzerTest >> setUp [ | since | super setUp. @@ -45,8 +46,53 @@ glhImporter importCommitsOProject: projects first since: (since asDate ) until: ] +{ #category : #test } +GitAnalyzerTest >> testAnalyseAmandment [ + + | myCommits mondayCommit tuesdayCommit wednesdayCommit | + glhImporter withCommitDiffs: true. + + "commit chain + mondayCommit -> tuesdayCommit -> wednesdayCommit + " + + wednesdayCommit := glhImporter + importCommitOfProject: projects first id + withId: + 'f298ec312f81032b611a890f993d98096ab3bbc1'. + + mondayCommit := glhImporter + importCommitOfProject: projects first id + withId: + '8a6f665118e6fcfa468d7cce1a7e944ba65812ab'. + + tuesdayCommit := glhImporter + importCommitOfProject: projects first id + withId: + '2c7308c727560dd425ef78f9a95ca7650d164bed'. + + myCommits := { mondayCommit . tuesdayCommit . wednesdayCommit } asSet . + + glhImporter chainsCommitsFrom: myCommits. + + myCommits do: [ :c | glhImporter completeImportedCommit: c ]. + + mondayCommit := myCommits detect: [ :c | + c id = '8a6f665118e6fcfa468d7cce1a7e944ba65812ab' ]. + + res := GitAnalyzer new + fromCommit: mondayCommit; + glhImporter: glhImporter; + onModel: glphModel; + analyseAmandment. + + + + self assert: res = tuesdayCommit equals: true. +] + { #category : #tests } -CodeChurnAnalyzerTest >> testChurnMetric [ +GitAnalyzerTest >> testChurnMetric [ | myCommits res wednesdayCommit mondayCommit| glhImporter withCommitDiffs: true. @@ -65,7 +111,7 @@ CodeChurnAnalyzerTest >> testChurnMetric [ mondayCommit := myCommits detect: [ :c | c id = '8a6f665118e6fcfa468d7cce1a7e944ba65812ab' ]. - res := CodeChurnAnalyzer new + res := GitAnalyzer new fromCommit: mondayCommit; glhImporter: glhImporter; onModel: glphModel; diff --git a/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st similarity index 66% rename from src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st rename to src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st index 329f2cf..46be8e5 100644 --- a/src/GitLabHealth-Model-Analysis/CodeChurnAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st @@ -1,5 +1,5 @@ Class { - #name : #CodeChurnAnalyzer, + #name : #GitAnalyzer, #superclass : #Object, #instVars : [ 'glModel', @@ -10,7 +10,19 @@ Class { } { #category : #'as yet unclassified' } -CodeChurnAnalyzer >> analyseChurn [ +GitAnalyzer >> analyseAmandment [ + "return the first commit that modify the same lines of code as the fromCommit" + + | churn res | + churn := self analyseChurn. + + res := self firstAmandmentFromChrun: churn. + + ^ res +] + +{ #category : #'as yet unclassified' } +GitAnalyzer >> analyseChurn [ | commitFiles churn | commitFiles := self impactedFilesInFollowUpCommitsOf: fromCommit. @@ -20,7 +32,7 @@ CodeChurnAnalyzer >> analyseChurn [ ] { #category : #churn } -CodeChurnAnalyzer >> computeChurnOnFiles: aCollection [ +GitAnalyzer >> computeChurnOnFiles: aCollection [ |changesDic perFileChanges churns | @@ -48,7 +60,7 @@ CodeChurnAnalyzer >> computeChurnOnFiles: aCollection [ ] { #category : #churn } -CodeChurnAnalyzer >> computeSpecificChurnOf: commit2Changes [ +GitAnalyzer >> computeSpecificChurnOf: commit2Changes [ |changesDic| "1 -> (a GLPHEChange -> NumberOfChurnDetected)" changesDic := OrderedDictionary new. @@ -76,17 +88,40 @@ CodeChurnAnalyzer >> computeSpecificChurnOf: commit2Changes [ ] { #category : #accessing } -CodeChurnAnalyzer >> fromCommit: aCommit [ +GitAnalyzer >> firstAmandmentFromChrun: aChurnAnalysis [ + |details whereChangesOccurs firstCommitsPerFile| + + whereChangesOccurs := (aChurnAnalysis at: #churns ) select: [ :file | file value > 0 ]. + + details := whereChangesOccurs collect: [ :file | + ((aChurnAnalysis at: #details) detect: [ :entry | entry key = file key] ) + ]. + + firstCommitsPerFile := details collect: [ :perFile | + |changes firstCommits first| + changes := perFile value. + changes := changes select: [ :line2changes | line2changes value value > 1 ]. + firstCommits := (changes collect: [ :line2changes | line2changes key second diffRange diff commit ]) values. + first := (firstCommits sortAscending: [:c | c created_at ]) first. + ]. + + + ^ (firstCommitsPerFile sortAscending: [:c | c created_at ]) first. + +] + +{ #category : #accessing } +GitAnalyzer >> fromCommit: aCommit [ fromCommit := aCommit. ] { #category : #accessing } -CodeChurnAnalyzer >> glhImporter: anImporter [ +GitAnalyzer >> glhImporter: anImporter [ glhImporter := anImporter . ] { #category : #'as yet unclassified' } -CodeChurnAnalyzer >> impactedFilesInFollowUpCommitsOf: aGLHCommit [ +GitAnalyzer >> impactedFilesInFollowUpCommitsOf: aGLHCommit [ | commitFiles | commitFiles := (fromCommit diffs collect: [ :diff | @@ -101,7 +136,7 @@ CodeChurnAnalyzer >> impactedFilesInFollowUpCommitsOf: aGLHCommit [ ] { #category : #initialization } -CodeChurnAnalyzer >> initialize [ +GitAnalyzer >> initialize [ glModel := GLPHEModel new. fromCommit := GLHCommit new. @@ -110,7 +145,7 @@ CodeChurnAnalyzer >> initialize [ ] { #category : #insertion } -CodeChurnAnalyzer >> insertDiff: aGLPHEDiffRange into: fileChangesDic startingFrom: from [ +GitAnalyzer >> insertDiff: aGLPHEDiffRange into: fileChangesDic startingFrom: from [ |index| index := from. aGLPHEDiffRange changes do: [ :aChange | @@ -128,17 +163,17 @@ CodeChurnAnalyzer >> insertDiff: aGLPHEDiffRange into: fileChangesDic startingFr ] { #category : #'as yet unclassified' } -CodeChurnAnalyzer >> onModel: agitHealthModel [ +GitAnalyzer >> onModel: agitHealthModel [ glModel := agitHealthModel ] { #category : #sorting } -CodeChurnAnalyzer >> sortChangeDic: aCollection [ +GitAnalyzer >> sortChangeDic: aCollection [ ^ (aCollection associations sortAscending: [ :e | e key ] ) asOrderedDictionary ] { #category : #visiting } -CodeChurnAnalyzer >> visitChildCommits: commits lookingForFiles: commitFiles [ +GitAnalyzer >> visitChildCommits: commits lookingForFiles: commitFiles [ commits ifEmpty: [ ^ commitFiles ]. diff --git a/src/GitLabHealth-Visualization/RSCommit.class.st b/src/GitLabHealth-Visualization/RSCommit.class.st index 975f3f9..3d3706b 100644 --- a/src/GitLabHealth-Visualization/RSCommit.class.st +++ b/src/GitLabHealth-Visualization/RSCommit.class.st @@ -79,7 +79,7 @@ RSCommit >> buildCommitCompleteHierachyCanvas [ dic := Dictionary new. - churnAnalyzer := CodeChurnAnalyzer new fromCommit: aCommit. + churnAnalyzer := GitAnalyzer new fromCommit: aCommit. files2Commits := churnAnalyzer impactedFilesInFollowUpCommitsOf: aCommit. files2Commits := files2Commits select: [ :entry | From d8ac080e8be1440c3dedf345989320ce76f3c5d6 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 16 May 2024 11:44:52 +0200 Subject: [PATCH 36/77] pre model update --- .../GitAnalyzerTest.class.st | 44 +++++---- .../GitAnalyzer.class.st | 12 ++- .../GLPHMetamodelGenerator.class.st | 8 +- .../GLPHModelImporter.class.st | 91 ++++++------------- 4 files changed, 67 insertions(+), 88 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st b/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st index fe27ed4..fb68810 100644 --- a/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st +++ b/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st @@ -8,8 +8,7 @@ Class { 'glphModel', 'glphApi', 'glhImporter', - 'projects', - 'res' + 'projects' ], #category : #'GitLabHealth-Model-Analysis-Tests' } @@ -49,7 +48,7 @@ glhImporter importCommitsOProject: projects first since: (since asDate ) until: { #category : #test } GitAnalyzerTest >> testAnalyseAmandment [ - | myCommits mondayCommit tuesdayCommit wednesdayCommit | + | myCommits res mondayCommit tuesdayCommit wednesdayCommit | glhImporter withCommitDiffs: true. "commit chain @@ -60,18 +59,19 @@ GitAnalyzerTest >> testAnalyseAmandment [ importCommitOfProject: projects first id withId: 'f298ec312f81032b611a890f993d98096ab3bbc1'. - + mondayCommit := glhImporter - importCommitOfProject: projects first id - withId: - '8a6f665118e6fcfa468d7cce1a7e944ba65812ab'. - - tuesdayCommit := glhImporter - importCommitOfProject: projects first id - withId: - '2c7308c727560dd425ef78f9a95ca7650d164bed'. + importCommitOfProject: projects first id + withId: '8a6f665118e6fcfa468d7cce1a7e944ba65812ab'. - myCommits := { mondayCommit . tuesdayCommit . wednesdayCommit } asSet . + tuesdayCommit := glhImporter + importCommitOfProject: projects first id + withId: '2c7308c727560dd425ef78f9a95ca7650d164bed'. + + myCommits := { + mondayCommit. + tuesdayCommit. + wednesdayCommit } asSet. glhImporter chainsCommitsFrom: myCommits. @@ -81,14 +81,22 @@ GitAnalyzerTest >> testAnalyseAmandment [ c id = '8a6f665118e6fcfa468d7cce1a7e944ba65812ab' ]. res := GitAnalyzer new - fromCommit: mondayCommit; - glhImporter: glhImporter; - onModel: glphModel; - analyseAmandment. + fromCommit: mondayCommit; + glhImporter: glhImporter; + onModel: glphModel; + analyseAmandment. + + self assert: res = tuesdayCommit equals: true +] +{ #category : #test } +GitAnalyzerTest >> testAnalyseMergeResquestValidation [ - self assert: res = tuesdayCommit equals: true. + + + + self assert: false ] { #category : #tests } diff --git a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st index 46be8e5..e19cbee 100644 --- a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st @@ -24,11 +24,17 @@ GitAnalyzer >> analyseAmandment [ { #category : #'as yet unclassified' } GitAnalyzer >> analyseChurn [ - | commitFiles churn | + | commitFiles | commitFiles := self impactedFilesInFollowUpCommitsOf: fromCommit. - churn := self computeChurnOnFiles: commitFiles. - ^ churn. + ^ self computeChurnOnFiles: commitFiles. + +] + +{ #category : #'as yet unclassified' } +GitAnalyzer >> analyseMergeResquestValidation: aGLHPMergeRequest [ + + 1 halt. ] { #category : #churn } diff --git a/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st b/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st index d3d8251..3cad16c 100644 --- a/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st +++ b/src/GitLabProjectHealth-ExtendModel-Generator/GLPHMetamodelGenerator.class.st @@ -174,10 +174,10 @@ GLPHMetamodelGenerator >> mergeRequestProperties [ mergeRequest property: #title type: #String. mergeRequest property: #description type: #String. mergeRequest property: #state type: #String. - mergeRequest property: #created_at type: #String. - mergeRequest property: #updated_at type: #String. - mergeRequest property: #merged_at type: #String. - mergeRequest property: #closed_at type: #String. + mergeRequest property: #created_at type: #Object. + mergeRequest property: #updated_at type: #Object. + mergeRequest property: #merged_at type: #Object. + mergeRequest property: #closed_at type: #Object. mergeRequest property: #target_branch type: #String. mergeRequest property: #source_branch type: #String. mergeRequest property: #user_notes_count type: #Number. diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 8bd2d01..bbab835 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -285,86 +285,51 @@ GLPHModelImporter >> parseDiffString: aDiff [ ] { #category : #parsing } -GLPHModelImporter >> parseMergeRequestResult: result [ - | reader | +GLPHModelImporter >> parseMergeRequestResult: result [ + + | reader | reader := NeoJSONReader on: result readStream. - + "declare quil y a un array a mapper" - reader - for: #ArrayOfMergeRequest - customDo: [ :customMappting | + reader for: #ArrayOfMergeRequest customDo: [ :customMappting | customMappting listOfElementSchema: GLPHEMergeRequest ]. - - "declare la liste des pro" - reader for: GLPHEMergeRequest do: [ :mapping | + "declare la liste des pro" + reader for: GLPHEMergeRequest do: [ :mapping | mapping mapInstVars: - #( blocking_discussions_resolved -changes_count -description -detailed_merge_status -discussion_locked -downvotes -draft -first_deployed_to_production_at -force_remove_source_branch -has_conflicts -id -iid -labels -latest_build_finished_at -latest_build_started_at -merge_commit_sha -merge_status -merge_when_pipeline_succeeds -merged_at -milestone -project_id -reference -references_full -references_relative -references_short -sha -should_remove_source_branch -source_branch -source_project_id -squash -squash_commit_sha -squash_on_merge -state -subscribed -target_branch -target_project_id -task_completion_status_completed_count -task_completion_status_count -time_stats_human_time_estimate -time_stats_human_total_time_spent -time_stats_time_estimate -time_stats_total_time_spent -title -updated_at -upvotes -user_notes_count -web_url -work_in_progress). -" (mapping mapInstVar: #closed_at) valueSchema: DateAndTime. + #( blocking_discussions_resolved changes_count description + detailed_merge_status discussion_locked downvotes draft first_deployed_to_production_at + force_remove_source_branch has_conflicts id iid labels latest_build_finished_at + latest_build_started_at merge_commit_sha merge_status + merge_when_pipeline_succeeds merged_at milestone project_id + reference references_full references_relative + references_short sha should_remove_source_branch + source_branch source_project_id squash squash_commit_sha + squash_on_merge state subscribed target_branch target_project_id + task_completion_status_completed_count + task_completion_status_count time_stats_human_time_estimate + time_stats_human_total_time_spent + time_stats_time_estimate time_stats_total_time_spent + title updated_at upvotes user_notes_count web_url work_in_progress ). + (mapping mapInstVar: #created_at) valueSchema: DateAndTime. + (mapping mapInstVar: #closed_at) valueSchema: DateAndTime. + " (mapping mapInstVar: #closed_at) valueSchema: DateAndTime. (mapping mapInstVar: #created_at) valueSchema: DateAndTime. (mapping mapInstVar: #assignee) valueSchema: GLHUser. (mapping mapInstVar: #author) valueSchema: GLHUser. (mapping mapInstVar: #closed_by) valueSchema: GLHUser. (mapping mapInstVar: #mergeCommit) valueSchema: GLHCommit." - " mapping + " mapping mapProperty: 'stats' getter: [ :el | ""Not used"" ] setter: [ :commit :value | commit deletions: (value at: #deletions). commit additions: (value at: #additions) ] -" -]. -" reader for: DateAndTime customDo: [ :mapping | +" ]. + " reader for: DateAndTime customDo: [ :mapping | mapping decoder: [ :string | DateAndTime fromString: string ] ]." - + ^ reader nextAs: #ArrayOfMergeRequest ] From ec13c4927ac40aadcc996c2cd10ce7d5e81a3d91 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 16 May 2024 14:14:59 +0200 Subject: [PATCH 37/77] add merge resquest duration metric --- .../GLPHEMergeRequest.class.st | 16 ++++++------ .../GitAnalyzer.class.st | 25 ++++++++++++++++++- .../GLPHModelImporter.class.st | 22 ++++++++-------- 3 files changed, 42 insertions(+), 21 deletions(-) diff --git a/src/GLPHExtended-Model/GLPHEMergeRequest.class.st b/src/GLPHExtended-Model/GLPHEMergeRequest.class.st index e403184..99551d9 100644 --- a/src/GLPHExtended-Model/GLPHEMergeRequest.class.st +++ b/src/GLPHExtended-Model/GLPHEMergeRequest.class.st @@ -33,8 +33,8 @@ a gitlab merge request |---| | `blocking_discussions_resolved` | `Boolean` | nil | | | `changes_count` | `String` | nil | | -| `closed_at` | `String` | nil | | -| `created_at` | `String` | nil | | +| `closed_at` | `Object` | nil | | +| `created_at` | `Object` | nil | | | `description` | `String` | nil | | | `detailed_merge_status` | `String` | nil | | | `discussion_locked` | `String` | nil | | @@ -51,7 +51,7 @@ a gitlab merge request | `merge_commit_sha` | `String` | nil | | | `merge_status` | `String` | nil | | | `merge_when_pipeline_succeeds` | `Boolean` | nil | | -| `merged_at` | `String` | nil | | +| `merged_at` | `Object` | nil | | | `milestone` | `String` | nil | | | `name` | `String` | nil | Basic name of the entity, not full reference.| | `project_id` | `Number` | nil | | @@ -77,7 +77,7 @@ a gitlab merge request | `time_stats_time_estimate` | `Number` | nil | | | `time_stats_total_time_spent` | `Number` | nil | | | `title` | `String` | nil | | -| `updated_at` | `String` | nil | | +| `updated_at` | `Object` | nil | | | `upvotes` | `Number` | nil | | | `user_notes_count` | `Number` | nil | | | `web_url` | `String` | nil | | @@ -251,7 +251,7 @@ GLPHEMergeRequest >> changes_count: anObject [ { #category : #accessing } GLPHEMergeRequest >> closed_at [ - + ^ closed_at ] @@ -281,7 +281,7 @@ GLPHEMergeRequest >> closed_by: anObject [ { #category : #accessing } GLPHEMergeRequest >> created_at [ - + ^ created_at ] @@ -598,7 +598,7 @@ GLPHEMergeRequest >> mergedCommit: anObject [ { #category : #accessing } GLPHEMergeRequest >> merged_at [ - + ^ merged_at ] @@ -1051,7 +1051,7 @@ GLPHEMergeRequest >> title: anObject [ { #category : #accessing } GLPHEMergeRequest >> updated_at [ - + ^ updated_at ] diff --git a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st index e19cbee..fe02da0 100644 --- a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st @@ -34,7 +34,30 @@ GitAnalyzer >> analyseChurn [ { #category : #'as yet unclassified' } GitAnalyzer >> analyseMergeResquestValidation: aGLHPMergeRequest [ - 1 halt. + |creationDate mergedDate response| + + response := { + #id_merge_resquest -> aGLHPMergeRequest iid. + #id_merge_commit -> nil. + #created_at -> aGLHPMergeRequest created_at. + #merged_at -> nil . + #duration -> nil. + #status -> (aGLHPMergeRequest merge_status)} asDictionary . + + + creationDate := aGLHPMergeRequest created_at asDateAndTime. + + mergedDate := aGLHPMergeRequest merged_at ifNil: [ + + ^ response + + ]. + + ^ response at: #duration put: (mergedDate - creationDate); + at: #id_merge_commit put: aGLHPMergeRequest merge_commit_sha ; + at: #merged_at put: aGLHPMergeRequest merged_at; + yourself. + ] { #category : #churn } diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index bbab835..22c6ad5 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -294,7 +294,7 @@ GLPHModelImporter >> parseMergeRequestResult: result [ reader for: #ArrayOfMergeRequest customDo: [ :customMappting | customMappting listOfElementSchema: GLPHEMergeRequest ]. - "declare la liste des pro" + "declare la liste des properties" reader for: GLPHEMergeRequest do: [ :mapping | mapping mapInstVars: #( blocking_discussions_resolved changes_count description @@ -312,23 +312,21 @@ GLPHModelImporter >> parseMergeRequestResult: result [ time_stats_time_estimate time_stats_total_time_spent title updated_at upvotes user_notes_count web_url work_in_progress ). (mapping mapInstVar: #created_at) valueSchema: DateAndTime. + (mapping mapInstVar: #updated_at) valueSchema: DateAndTime. + (mapping mapInstVar: #merged_at) valueSchema: DateAndTime. (mapping mapInstVar: #closed_at) valueSchema: DateAndTime. - " (mapping mapInstVar: #closed_at) valueSchema: DateAndTime. - (mapping mapInstVar: #created_at) valueSchema: DateAndTime. + " (mapping mapInstVar: #assignee) valueSchema: GLHUser. (mapping mapInstVar: #author) valueSchema: GLHUser. (mapping mapInstVar: #closed_by) valueSchema: GLHUser. (mapping mapInstVar: #mergeCommit) valueSchema: GLHCommit." - " mapping - mapProperty: 'stats' - getter: [ :el | ""Not used"" ] - setter: [ :commit :value | - commit deletions: (value at: #deletions). - commit additions: (value at: #additions) ] -" ]. - " reader for: DateAndTime customDo: [ :mapping | - mapping decoder: [ :string | DateAndTime fromString: string ] ]." + ]. + + "indique ce que doit faire le reader lorsqu'il parse une DateAndTime object" + reader for: DateAndTime customDo: [ :mapping | + mapping decoder: [ :string | string ifNil: [nil] ifNotNil: [ DateAndTime fromString: string ] ] ]. + ^ reader nextAs: #ArrayOfMergeRequest ] From d2a185df4e064ec1074d62e1158ed555d81ca10a Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 16 May 2024 14:55:55 +0200 Subject: [PATCH 38/77] add test --- .../GitAnalyzerTest.class.st | 80 ++++++++++++------- 1 file changed, 53 insertions(+), 27 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st b/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st index fb68810..9c48e48 100644 --- a/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st +++ b/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st @@ -8,7 +8,8 @@ Class { 'glphModel', 'glphApi', 'glhImporter', - 'projects' + 'projects', + 'gitAnalyzer' ], #category : #'GitLabHealth-Model-Analysis-Tests' } @@ -35,14 +36,19 @@ GitAnalyzerTest >> setUp [ withFiles: false; withCommitsSince: 0 day; withCommitDiffs: true. - - glhImporter importProject: 57841283. - projects := (glhImporter glhModel allWithType: GLHProject). + + glhImporter importProject: 57841283. + projects := glhImporter glhModel allWithType: GLHProject. -glhImporter importCommitsOProject: projects first since: (since asDate ) until: nil. + glhImporter + importCommitsOProject: projects first + since: since asDate + until: nil. - + gitAnalyzer := GitAnalyzer new + glhImporter: glhImporter; + onModel: glphModel. ] { #category : #test } @@ -93,37 +99,57 @@ GitAnalyzerTest >> testAnalyseAmandment [ { #category : #test } GitAnalyzerTest >> testAnalyseMergeResquestValidation [ - - - - self assert: false + | mergeResquest res| + mergeResquest := (glhImporter glhModel allWithType: GLPHEMergeRequest) + detect: [ :mr | mr iid = 1 ] . + + + res := gitAnalyzer analyseMergeResquestValidation: mergeResquest. + + + self assert: ((res at: #merged_at) - (res at: #created_at) ) equals: (res at: #duration) ] { #category : #tests } GitAnalyzerTest >> testChurnMetric [ - | myCommits res wednesdayCommit mondayCommit| + | myCommits res wednesdayCommit mondayCommit tuesdayCommit | glhImporter withCommitDiffs: true. - - wednesdayCommit := glhImporter importCommitOfProject: projects first id withId: 'f298ec312f81032b611a890f993d98096ab3bbc1'. - - myCommits := (glhImporter - importParentCommitsOfCommit: - wednesdayCommit - forOnly: 2) asSet. - myCommits add: wednesdayCommit. - - glhImporter chainsCommitsFrom: myCommits asSet. + + "commit chain + mondayCommit -> tuesdayCommit -> wednesdayCommit + " + wednesdayCommit := glhImporter + importCommitOfProject: projects first id + withId: + 'f298ec312f81032b611a890f993d98096ab3bbc1'. + + mondayCommit := glhImporter + importCommitOfProject: projects first id + withId: '8a6f665118e6fcfa468d7cce1a7e944ba65812ab'. + + tuesdayCommit := glhImporter + importCommitOfProject: projects first id + withId: '2c7308c727560dd425ef78f9a95ca7650d164bed'. + + myCommits := { + mondayCommit. + tuesdayCommit. + wednesdayCommit } asSet. + + glhImporter chainsCommitsFrom: myCommits. myCommits do: [ :c | glhImporter completeImportedCommit: c ]. - mondayCommit := myCommits detect: [ :c | c id = '8a6f665118e6fcfa468d7cce1a7e944ba65812ab' ]. + + mondayCommit := myCommits detect: [ :c | + c id = '8a6f665118e6fcfa468d7cce1a7e944ba65812ab' ]. res := GitAnalyzer new - fromCommit: mondayCommit; - glhImporter: glhImporter; - onModel: glphModel; - analyseChurn. + fromCommit: mondayCommit; + glhImporter: glhImporter; + onModel: glphModel; + analyseChurn. - self assert: (res at:#churns) first value equals: 5/7 + self assert: (res at: #churns) first value equals: 5 / 7 ] From 8c10145a8d71ede6cc2ae75f7b0ef1a3cc5571ed Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 16 May 2024 17:24:56 +0200 Subject: [PATCH 39/77] add frequency commit metric (for all commits) --- .../GitAnalyzerTest.class.st | 20 ++++++++- .../GitAnalyzer.class.st | 45 +++++++++++++++++-- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st b/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st index 9c48e48..78f73c3 100644 --- a/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st +++ b/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st @@ -60,7 +60,6 @@ GitAnalyzerTest >> testAnalyseAmandment [ "commit chain mondayCommit -> tuesdayCommit -> wednesdayCommit " - wednesdayCommit := glhImporter importCommitOfProject: projects first id withId: @@ -96,6 +95,25 @@ GitAnalyzerTest >> testAnalyseAmandment [ self assert: res = tuesdayCommit equals: true ] +{ #category : #test } +GitAnalyzerTest >> testAnalyseCommitFrequencySinceUntil [ + + | res from since | + + from := '13 May 2024'. + since := '16 May 2024'. + + res := gitAnalyzer + onProject: projects first; + analyseCommitFrequencySince: from asDate + until: since asDate. + + + + self assert: (res at: from) size equals: 1. + self assert: (res at: '14 May 2024') size equals: 4. +] + { #category : #test } GitAnalyzerTest >> testAnalyseMergeResquestValidation [ diff --git a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st index fe02da0..e48f5d7 100644 --- a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st @@ -4,7 +4,8 @@ Class { #instVars : [ 'glModel', 'fromCommit', - 'glhImporter' + 'glhImporter', + 'onProject' ], #category : #'GitLabHealth-Model-Analysis' } @@ -31,6 +32,26 @@ GitAnalyzer >> analyseChurn [ ] +{ #category : #'as yet unclassified' } +GitAnalyzer >> analyseCommitFrequencySince: since until: until [ + + | commits response | + response := { + (#numberOfCommit -> nil). + (#frequency -> nil) } asDictionary. + + commits := glhImporter + importCommitsOProject: onProject + since: since + until: until. + + commits := self arrangeCommitsByDate: commits. + commits := (commits associations sortAscending: [ :entry | + entry key asDate ]) asOrderedDictionary. + + ^ commits +] + { #category : #'as yet unclassified' } GitAnalyzer >> analyseMergeResquestValidation: aGLHPMergeRequest [ @@ -60,6 +81,19 @@ GitAnalyzer >> analyseMergeResquestValidation: aGLHPMergeRequest [ ] +{ #category : #filter } +GitAnalyzer >> arrangeCommitsByDate: commits [ + |date2commits| + date2commits := Dictionary new. + + commits do: [ :commit | + |date| + date := commit created_at asDate. + date2commits at: date printString ifPresent: [ :v | v add: commit ] ifAbsentPut: [ OrderedCollection new add: commit; yourself. ]. + ]. + ^ date2commits. +] + { #category : #churn } GitAnalyzer >> computeChurnOnFiles: aCollection [ @@ -169,8 +203,8 @@ GitAnalyzer >> initialize [ glModel := GLPHEModel new. fromCommit := GLHCommit new. - glhImporter := GLPHModelImporter new. - + glhImporter := GLPHModelImporter new. + onProject:= GLHProject new. ] { #category : #insertion } @@ -196,6 +230,11 @@ GitAnalyzer >> onModel: agitHealthModel [ glModel := agitHealthModel ] +{ #category : #accessing } +GitAnalyzer >> onProject: aGLHProject [ + onProject := aGLHProject +] + { #category : #sorting } GitAnalyzer >> sortChangeDic: aCollection [ ^ (aCollection associations sortAscending: [ :e | e key ] ) asOrderedDictionary From dd39fb256f6aa526ea7015aab42c05016ea82c89 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Tue, 21 May 2024 16:08:34 +0200 Subject: [PATCH 40/77] update test and add commit windows for children analysis --- .../GitAnalyzerTest.class.st | 7 +-- .../GitAnalyzer.class.st | 43 +++++++++++++++---- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st b/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st index 78f73c3..c68bee3 100644 --- a/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st +++ b/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st @@ -138,16 +138,16 @@ GitAnalyzerTest >> testChurnMetric [ mondayCommit -> tuesdayCommit -> wednesdayCommit " wednesdayCommit := glhImporter - importCommitOfProject: projects first id + importCommitOfProject: projects first withId: 'f298ec312f81032b611a890f993d98096ab3bbc1'. mondayCommit := glhImporter - importCommitOfProject: projects first id + importCommitOfProject: projects first withId: '8a6f665118e6fcfa468d7cce1a7e944ba65812ab'. tuesdayCommit := glhImporter - importCommitOfProject: projects first id + importCommitOfProject: projects first withId: '2c7308c727560dd425ef78f9a95ca7650d164bed'. myCommits := { @@ -167,6 +167,7 @@ GitAnalyzerTest >> testChurnMetric [ fromCommit: mondayCommit; glhImporter: glhImporter; onModel: glphModel; + maxChildCommit: 3; analyseChurn. self assert: (res at: #churns) first value equals: 5 / 7 diff --git a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st index e48f5d7..6abacf0 100644 --- a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st @@ -5,7 +5,8 @@ Class { 'glModel', 'fromCommit', 'glhImporter', - 'onProject' + 'onProject', + 'maxChildCommits' ], #category : #'GitLabHealth-Model-Analysis' } @@ -26,10 +27,11 @@ GitAnalyzer >> analyseAmandment [ GitAnalyzer >> analyseChurn [ | commitFiles | - commitFiles := self impactedFilesInFollowUpCommitsOf: fromCommit. + commitFiles := self + impactedFilesInFollowUpCommitsOf: fromCommit + withMaxCommits: self maxChildCommits. - ^ self computeChurnOnFiles: commitFiles. - + ^ self computeChurnOnFiles: commitFiles ] { #category : #'as yet unclassified' } @@ -186,14 +188,21 @@ GitAnalyzer >> glhImporter: anImporter [ { #category : #'as yet unclassified' } GitAnalyzer >> impactedFilesInFollowUpCommitsOf: aGLHCommit [ + ^ self impactedFilesInFollowUpCommitsOf: aGLHCommit withMaxCommits: -1 +] + +{ #category : #churn } +GitAnalyzer >> impactedFilesInFollowUpCommitsOf: aGLHCommit withMaxCommits: max [ + | commitFiles | commitFiles := (fromCommit diffs collect: [ :diff | - diff new_path -> (Set new add: (aGLHCommit -> (diff diffRanges) ); yourself) ]) - asDictionary. + diff new_path -> (Set new + add: aGLHCommit -> diff diffRanges; + yourself) ]) asDictionary. self visitChildCommits: fromCommit childCommits - lookingForFiles: commitFiles. + lookingForFiles: commitFiles upto: max. ^ commitFiles ] @@ -225,6 +234,16 @@ GitAnalyzer >> insertDiff: aGLPHEDiffRange into: fileChangesDic startingFrom: fr ] ] +{ #category : #accessing } +GitAnalyzer >> maxChildCommit: max [ + maxChildCommits := max +] + +{ #category : #accessing } +GitAnalyzer >> maxChildCommits [ + ^ maxChildCommits +] + { #category : #'as yet unclassified' } GitAnalyzer >> onModel: agitHealthModel [ glModel := agitHealthModel @@ -243,7 +262,14 @@ GitAnalyzer >> sortChangeDic: aCollection [ { #category : #visiting } GitAnalyzer >> visitChildCommits: commits lookingForFiles: commitFiles [ + ^ self visitChildCommits: commits lookingForFiles: commitFiles upto: -1 +] + +{ #category : #visiting } +GitAnalyzer >> visitChildCommits: commits lookingForFiles: commitFiles upto: nCommits [ + commits ifEmpty: [ ^ commitFiles ]. + (nCommits = 0) ifTrue: [ ^ commitFiles ]. commits do: [ :commit | | files | @@ -257,7 +283,8 @@ GitAnalyzer >> visitChildCommits: commits lookingForFiles: commitFiles [ self visitChildCommits: commit childCommits - lookingForFiles: commitFiles ]. + lookingForFiles: commitFiles + upto: nCommits - 1 ]. ^ commitFiles ] From 0658e1095b9afd33bdb0410bcfa6f63c6a940be3 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Tue, 21 May 2024 17:33:48 +0200 Subject: [PATCH 41/77] update tests --- .../GitAnalyzerTest.class.st | 6 +++--- src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st b/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st index c68bee3..2c6f2e1 100644 --- a/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st +++ b/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st @@ -61,16 +61,16 @@ GitAnalyzerTest >> testAnalyseAmandment [ mondayCommit -> tuesdayCommit -> wednesdayCommit " wednesdayCommit := glhImporter - importCommitOfProject: projects first id + importCommitOfProject: projects first withId: 'f298ec312f81032b611a890f993d98096ab3bbc1'. mondayCommit := glhImporter - importCommitOfProject: projects first id + importCommitOfProject: projects first withId: '8a6f665118e6fcfa468d7cce1a7e944ba65812ab'. tuesdayCommit := glhImporter - importCommitOfProject: projects first id + importCommitOfProject: projects first withId: '2c7308c727560dd425ef78f9a95ca7650d164bed'. myCommits := { diff --git a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st index 6abacf0..1d95513 100644 --- a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st @@ -213,7 +213,8 @@ GitAnalyzer >> initialize [ glModel := GLPHEModel new. fromCommit := GLHCommit new. glhImporter := GLPHModelImporter new. - onProject:= GLHProject new. + onProject := GLHProject new. + maxChildCommits := -1 ] { #category : #insertion } From 720de51254993a4211f808b5944d828a7916781e Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 23 May 2024 09:30:11 +0200 Subject: [PATCH 42/77] fix GLHProject injected into user commits --- .../GitAnalyzerTest.class.st | 7 +- .../GitMetric4User.class.st | 76 +++++++++++++++++++ .../GLHModelImporter.class.st | 29 +++++-- src/GitLabHealth-Model/GLHUser.class.st | 2 + .../GLPHModelImporter.class.st | 5 +- 5 files changed, 107 insertions(+), 12 deletions(-) create mode 100644 src/GitLabHealth-Model-Analysis/GitMetric4User.class.st diff --git a/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st b/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st index 2c6f2e1..0adfbb1 100644 --- a/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st +++ b/src/GitLabHealth-Model-Analysis-Tests/GitAnalyzerTest.class.st @@ -61,16 +61,16 @@ GitAnalyzerTest >> testAnalyseAmandment [ mondayCommit -> tuesdayCommit -> wednesdayCommit " wednesdayCommit := glhImporter - importCommitOfProject: projects first + importCommitOfProject: projects first withId: 'f298ec312f81032b611a890f993d98096ab3bbc1'. mondayCommit := glhImporter - importCommitOfProject: projects first + importCommitOfProject: projects first withId: '8a6f665118e6fcfa468d7cce1a7e944ba65812ab'. tuesdayCommit := glhImporter - importCommitOfProject: projects first + importCommitOfProject: projects first withId: '2c7308c727560dd425ef78f9a95ca7650d164bed'. myCommits := { @@ -89,6 +89,7 @@ GitAnalyzerTest >> testAnalyseAmandment [ fromCommit: mondayCommit; glhImporter: glhImporter; onModel: glphModel; + analyseAmandment. diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st new file mode 100644 index 0000000..21cc2d1 --- /dev/null +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -0,0 +1,76 @@ +Class { + #name : #GitMetric4User, + #superclass : #Object, + #instVars : [ + 'user', + 'itsProjects', + 'itsGroups', + 'itsCommits', + 'gitAnalyzer', + 'glhModel', + 'glhImporter' + ], + #category : #'GitLabHealth-Model-Analysis' +} + +{ #category : #'as yet unclassified' } +GitMetric4User >> findUserNamed: aUsername [ + + user := glhImporter importUserByUsername: aUsername. + ^ user +] + +{ #category : #accessing } +GitMetric4User >> glhImporter: aGLPHModelImporter [ + glhImporter := aGLPHModelImporter +] + +{ #category : #initialization } +GitMetric4User >> initialize [ + + user := GLHUser new. + itsProjects := Dictionary new. + itsCommits := Dictionary new. + itsGroups := Dictionary new +] + +{ #category : #accessing } +GitMetric4User >> itsCommits [ + ^ user commits +] + +{ #category : #churn } +GitMetric4User >> loadCommitOfProjects: aCollection since: aTimespan [ + |commits| + + commits := (aCollection collect: [ :id | + glhImporter + importCommitsOProject: (itsProjects at: id) + since: aTimespan + until: nil ]) flattened. + + commits do: [ :commit | + glhImporter completeImportedCommit: commit. + ]. + + ^ self itsCommits. +] + +{ #category : #loading } +GitMetric4User >> loadProjects: aCollection [ + + aCollection do: [ :id | + itsProjects at: id ifAbsentPut: (glhImporter importProject: id) ]. + + ^ itsProjects +] + +{ #category : #accessing } +GitMetric4User >> user [ + ^ user +] + +{ #category : #accessing } +GitMetric4User >> user: anUser [ + user := anUser. +] diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index d598c4e..c85d442 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -61,17 +61,22 @@ GLHModelImporter >> completeImportProject: aGLHProject [ (self pipelinesOf: aGLHProject id) do: [ :pipeline | self glhModel add: pipeline. aGLHProject addPipeline: pipeline ]. - aGLHProject creator: (self importUser: aGLHProject creator_id). + + "aGLHProject creator: (self importUser: aGLHProject creator_id)." + + ((self importUser: aGLHProject creator_id)) addCreatedProject: aGLHProject. aGLHProject repository: GLHRepository new. self glhModel add: aGLHProject unless: (self blockEqualityOn: #id). self glhModel add: aGLHProject repository. - self importRepository: aGLHProject repository + self importRepository: aGLHProject repository. + ^ aGLHProject ] { #category : #'as yet unclassified' } GLHModelImporter >> completeImportedCommit: aCommit [ | diffs | + (self importUserByUsername: aCommit author_name) commits add: aCommit unless: [ :existingCommit :newCommit | @@ -245,8 +250,9 @@ GLHModelImporter >> importCommitsOProject: aProject since: fromDate until: toDat newlyFoundCommit := self parseCommitsResult: results. newlyFoundCommit do: [ :c | - c repository: aProject repository. - ]. + + c repository: aProject repository ]. + aProject repository commits addAll: newlyFoundCommit unless: self blockOnIdEquality. @@ -262,6 +268,9 @@ GLHModelImporter >> importCommitsOProject: aProject since: fromDate until: toDat GLHModelImporter >> importCommitsOf: aGLHProject withStats: aBoolean until: toDate [ | newlyFoundCommit page | + + self deprecated: [ ] . + page := 0. newlyFoundCommit := { true }. [ newlyFoundCommit isNotEmpty ] whileTrue: [ @@ -293,8 +302,8 @@ GLHModelImporter >> importCommitsOf: aGLHProject withStats: aBoolean until: toDa existingCommit id = commitParsed id ] ] ]. aGLHProject mooseModel addAll: newlyFoundCommit. aGLHProject repository commits addAll: newlyFoundCommit ]. - - + + self withCommitDiffs ifTrue: [ aGLHProject repository commits do: [ :commit | self importDiffOfCommit: commit ] ] @@ -522,11 +531,15 @@ GLHModelImporter >> importUser: aUserID [ ^ (glhModel allWithType: GLHUser) detect: [ :user | user id = aUserID ] ifNone: [ + ('Import user: ' , aUserID printString) recordInfo. result := self glhApi user: aUserID. userResult := self parseUserResult: result. glhModel add: userResult unless: self blockOnIdEquality. - userResult ] + self flag: + 'please optimize by changing add:unless: so it return the add elmnt'. + (glhModel allWithType: GLHUser) detect: [ :user | + user id = aUserID ] ] ] { #category : #user } @@ -541,7 +554,7 @@ GLHModelImporter >> importUserByUsername: anUsername [ result := self glhApi usersSearchByUsername: anUsername. searchResult := NeoJSONReader fromString: result. - ((searchResult at: #message) includesSubstring: '403 Forbidden') ifTrue: [ + (searchResult class = Dictionary and: [((searchResult at: #message) includesSubstring: '403 Forbidden')]) ifTrue: [ GLHUser new username: anUsername. ] ifFalse: [ searchResult ifEmpty: [ GLHUser new ] ifNotEmpty: [ diff --git a/src/GitLabHealth-Model/GLHUser.class.st b/src/GitLabHealth-Model/GLHUser.class.st index c639b44..19654df 100644 --- a/src/GitLabHealth-Model/GLHUser.class.st +++ b/src/GitLabHealth-Model/GLHUser.class.st @@ -88,7 +88,9 @@ GLHUser class >> annotation [ { #category : #adding } GLHUser >> addCommit: anObject [ + + ^ self commits add: anObject ] diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 22c6ad5..07bac2f 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -79,6 +79,8 @@ GLPHModelImporter >> completeImportProject: aGLHProject [ self importMergeRequests: aGLHProject. + ^ aGLHProject + ] @@ -244,7 +246,8 @@ GLPHModelImporter >> parseDiffString: aDiff [ index := 0. range := nil. - + (lines ifEmpty: [ ^ aDiff diffRanges ]). + (lines first beginsWith: '@@') ifFalse: [ ^ aDiff diffRanges ]. lines := self filterCommitChanges: lines. From 0791627313310ce1bc50d41dc4f90b5788b3d008 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 23 May 2024 15:02:07 +0200 Subject: [PATCH 43/77] add frequency aggregator over Week, Month and Year --- .../GitMetric4User.class.st | 55 +++++++++++++++++ .../GLHCommit.extension.st | 12 +--- .../MooseAbstractGroup.extension.st | 22 ++++--- .../OrderedCollection.extension.st | 17 ++++-- .../GLHModelImporter.class.st | 24 ++++---- .../RSCommit.class.st | 60 +++++++++---------- 6 files changed, 122 insertions(+), 68 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index 21cc2d1..3308dd6 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -13,6 +13,61 @@ Class { #category : #'GitLabHealth-Model-Analysis' } +{ #category : #'as yet unclassified' } +GitMetric4User >> commitFrequencyOnProject: projectId since: sinceDate until: untilDate [ + + | frequencies | + + gitAnalyzer := GitAnalyzer new + glhImporter: glhImporter; + onModel: glhModel. + + frequencies := gitAnalyzer + onProject: (itsProjects at: projectId); + analyseCommitFrequencySince: sinceDate + until: untilDate. + + "complete each commit " + frequencies values flattened do: [ :commit | + glhImporter completeImportedCommit: commit. + ]. + + "Filter the associations in the 'frequencies' collection, removing any commits not created by the specified 'user'. " + frequencies associations do: [ :assoc | + |commits| + commits := assoc value. + assoc value: (commits reject: [ :commit | commit commitCreator ~= user ]). + ]. + + ^ frequencies +] + +{ #category : #frequency } +GitMetric4User >> commitFrequencyOnProject: aProjectId since: since until: until overA: aWeekOrMonthOrYear [ + "'aWeekOrMonthOrYear' should be the class of Week, Month or Year" + + |frequency aggregatedFrequencies| + aggregatedFrequencies := OrderedDictionary new. + frequency := self commitFrequencyOnProject: aProjectId since: since until: until. + + + frequency associations do: [ :assoc | + |date commits overDate| + date := assoc key. + commits := assoc value. + + + " convert the date as its week date. For some english reason, the week start in sunday so we should add a + 1 day " + overDate := ((date asDate) perform: (('as', aWeekOrMonthOrYear name) asSymbol )) asDate. + aggregatedFrequencies at: (overDate printString) ifPresent: [ :collection | collection addAll: commits ] ifAbsentPut: [ commits ]. + + ]. + + ^ aggregatedFrequencies + + +] + { #category : #'as yet unclassified' } GitMetric4User >> findUserNamed: aUsername [ diff --git a/src/GitLabHealth-Model-Extension/GLHCommit.extension.st b/src/GitLabHealth-Model-Extension/GLHCommit.extension.st index 9284bc0..54c5fa2 100644 --- a/src/GitLabHealth-Model-Extension/GLHCommit.extension.st +++ b/src/GitLabHealth-Model-Extension/GLHCommit.extension.st @@ -1,19 +1,9 @@ Extension { #name : #GLHCommit } -{ #category : #'*GitLabHealth-Model-Extension' } -GLHCommit >> inspectionGLHCommit [ - - - | canvas | - canvas := RSCommit canvasHierarchyFor: self. - - ^ canvas asPresenter -] - { #category : #'*GitLabHealth-Model-Extension' } GLHCommit >> inspectionGLHCommitChildren [ - + | canvas | canvas := RSCommit canvasCompleteHierachyFor: self. diff --git a/src/GitLabHealth-Model-Extension/MooseAbstractGroup.extension.st b/src/GitLabHealth-Model-Extension/MooseAbstractGroup.extension.st index 605908c..561dc81 100644 --- a/src/GitLabHealth-Model-Extension/MooseAbstractGroup.extension.st +++ b/src/GitLabHealth-Model-Extension/MooseAbstractGroup.extension.st @@ -4,18 +4,24 @@ Extension { #name : #MooseAbstractGroup } MooseAbstractGroup >> add: anElmt unless: aConditionAsBlock [ "()>>>" - self addAll: { anElmt } unless: aConditionAsBlock - - + ^ (self addAll: { anElmt } unless: aConditionAsBlock) first ] { #category : #'*GitLabHealth-Model-Extension' } MooseAbstractGroup >> addAll: anElmtCollection ofType: aType unless: aConditionAsBlock [ "()>>>" - self addAll: (anElmtCollection reject: [ :anElmt | - (self allWithType: aType) anySatisfy: [ :existingElmt | - aConditionAsBlock value: existingElmt value: anElmt ] ]) + | addedElement | + addedElement := anElmtCollection reject: [ :anElmt | + (self allWithType: aType) anySatisfy: [ + :existingElmt | + aConditionAsBlock value: existingElmt value: anElmt ] ]. + + self addAll: addedElement. + + ^ (self allWithType: aType) select: [ :existingElmt | + anElmtCollection anySatisfy: [ :anElmt | + aConditionAsBlock value: anElmt value: existingElmt ] ] ] { #category : #'*GitLabHealth-Model-Extension' } @@ -23,8 +29,8 @@ MooseAbstractGroup >> addAll: anElmtCollection unless: aConditionAsBlock [ "()>>>" anElmtCollection ifEmpty: [ ^ self ]. - self + ^ self addAll: anElmtCollection ofType: anElmtCollection first class - unless: aConditionAsBlock. + unless: aConditionAsBlock ] diff --git a/src/GitLabHealth-Model-Extension/OrderedCollection.extension.st b/src/GitLabHealth-Model-Extension/OrderedCollection.extension.st index 2127d74..0754d75 100644 --- a/src/GitLabHealth-Model-Extension/OrderedCollection.extension.st +++ b/src/GitLabHealth-Model-Extension/OrderedCollection.extension.st @@ -4,14 +4,21 @@ Extension { #name : #OrderedCollection } OrderedCollection >> add: anElmt unless: aConditionAsBlock [ "()>>>" - self addAll: { anElmt } unless: aConditionAsBlock + ^ (self addAll: { anElmt } unless: aConditionAsBlock) first ] { #category : #'*GitLabHealth-Model-Extension' } OrderedCollection >> addAll: anElmtCollection unless: aConditionAsBlock [ "()>>>" - - self addAll: (anElmtCollection reject: [ :anElmt | - self anySatisfy: [ - :existingElmt | aConditionAsBlock value: existingElmt value: anElmt ] ]) + |addedElements| + + addedElements := (anElmtCollection reject: [ :anElmt | + self anySatisfy: [ :existingElmt | + aConditionAsBlock value: existingElmt value: anElmt ] ]). + + self addAll: addedElements. + + ^ (self select: [ :existingElmt | + anElmtCollection anySatisfy: [ :anElmt | + aConditionAsBlock value: anElmt value: existingElmt ] ]). ] diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index c85d442..01814f5 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -75,20 +75,20 @@ GLHModelImporter >> completeImportProject: aGLHProject [ { #category : #'as yet unclassified' } GLHModelImporter >> completeImportedCommit: aCommit [ - | diffs | + | diffs addedCommit | + addedCommit := (self importUserByUsername: aCommit author_name) + commits + add: aCommit + unless: [ :existingCommit :newCommit | + existingCommit id = newCommit id ]. - (self importUserByUsername: aCommit author_name) commits - add: aCommit - unless: [ :existingCommit :newCommit | - existingCommit id = newCommit id ]. - - ('Import diff of commit: ' , aCommit short_id printString) recordInfo. + ('Import diff of commit: ' , addedCommit short_id printString) recordInfo. self withCommitDiffs ifTrue: [ - diffs := self importDiffOfCommit: aCommit. + diffs := self importDiffOfCommit: addedCommit. self glhModel addAll: diffs unless: self blockForDiffEquality ]. - ^ aCommit + ^ addedCommit ] { #category : #private } @@ -240,7 +240,7 @@ GLHModelImporter >> importCommitsOProject: aProject since: fromDate until: toDat (toDate ifNotNil: [ toDate asDateAndTime asString ]) path: nil author: nil - all: nil + all: true with_stats: nil firstParent: nil order: nil @@ -258,10 +258,8 @@ GLHModelImporter >> importCommitsOProject: aProject since: fromDate until: toDat unless: self blockOnIdEquality. foundCommit addAll: newlyFoundCommit ]. - self glhModel addAll: foundCommit unless: self blockOnIdEquality. + ^ self glhModel addAll: foundCommit unless: self blockOnIdEquality. - - ^ foundCommit ] { #category : #api } diff --git a/src/GitLabHealth-Visualization/RSCommit.class.st b/src/GitLabHealth-Visualization/RSCommit.class.st index 3d3706b..90034bf 100644 --- a/src/GitLabHealth-Visualization/RSCommit.class.st +++ b/src/GitLabHealth-Visualization/RSCommit.class.st @@ -70,7 +70,7 @@ RSCommit >> allChildrenShapesFrom: aGLHCommit into: dic [ { #category : #canvas } RSCommit >> buildCommitCompleteHierachyCanvas [ - | shapes parents children commitShape dic churnAnalyzer files2Commits shape2change | + | shapes parents children commitShape dic churnAnalyzer files2Commits shape2change allCommitShps | canvas := RSCanvas new. shapes := RSGroup new. parents := RSGroup new. @@ -109,10 +109,25 @@ RSCommit >> buildCommitCompleteHierachyCanvas [ self shapeForCommit: c withColor: self parentColor ]). + allCommitShps := parents , commitShape , children. + + allCommitShps do: [ :shp | + |commit| + commit := shp model. + (commit commitCreator = commitShape first model commitCreator) ifTrue: [ + (shp children detect: [ :c | c class = RSBox ]) border: (RSBorder new + width: 5; + color: self defaultColor ; + yourself). + ]. + ]. + + + "canvas addAllShapes: commitShape , parents , children." - self createLinesBetweenCommits: parents , commitShape , children. + self createLinesBetweenCommits: allCommitShps. - canvas addAll: parents, commitShape , children. + canvas addAll: allCommitShps. RSHorizontalTreeLayout new on: canvas nodes. canvas @ RSCanvasController. @@ -389,35 +404,17 @@ RSCommit >> shapeForCommit: aGLHCommit [ { #category : #shape } RSCommit >> shapeForCommit: aGLHCommit withColor: color [ - | box group label impactedFiles | + | box group labelTitle labelCommiter impactedFiles | group := RSGroup new. impactedFiles := RSGroup new. - label := RSLabel new - text: (aGLHCommit name truncateWithElipsisTo: 30); - yourself. - - "aGLHCommit diffsGroup do: [ :diff | - impactedFiles add: - (RSCommitDiff new - shapeForDiffRange: diff - withColor: Color lightBlue) ]. - RSVerticalLineLayout new alignCenter on: impactedFiles. + labelTitle := RSLabel new + text: (aGLHCommit name truncateWithElipsisTo: 40); + yourself. - - box := RSBox new - height: - (impactedFiles sum: [ :composite | composite height ]); - width: (impactedFiles max: [ :composite | composite width ]); - color: color. - - group - add: label; - add: (RSComposite new shapes: (RSGroup new - add: box; - addAll: impactedFiles; - yourself)). - RSVerticalLineLayout new alignCenter on: group. - " + labelCommiter := RSLabel new + text: 'by ', + (aGLHCommit author_name truncateWithElipsisTo: 50); + yourself. box := RSBox new height: 50; @@ -425,8 +422,9 @@ RSCommit >> shapeForCommit: aGLHCommit withColor: color [ color: color. group - add: label; - add: box. + add: labelTitle; + add: box; + add: labelCommiter. RSVerticalLineLayout new alignCenter on: group. From 7eec1cd607f62119ea02ce3fc4a9c6135eaa5844 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 23 May 2024 16:35:03 +0200 Subject: [PATCH 44/77] add avg code frequency --- .../GitMetric4User.class.st | 80 ++++++++++++++----- 1 file changed, 61 insertions(+), 19 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index 3308dd6..bc5eff7 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -14,7 +14,40 @@ Class { } { #category : #'as yet unclassified' } -GitMetric4User >> commitFrequencyOnProject: projectId since: sinceDate until: untilDate [ +GitMetric4User >> commitFrequencySince: since until: until overA: aWeekMonthOrYear [ + + | all aggregate periods total | + aggregate := OrderedDictionary new. + + all := itsProjects keys collect: [ :id | + self + commitsProducedOnProject: id + since: since + until: until + overA: aWeekMonthOrYear ]. + + + + + all do: [ :frequencies | + frequencies associations do: [ :assoc | + aggregate + at: assoc key + ifPresent: [ :collection | collection addAll: assoc value ] + ifAbsentPut: [ assoc value ] ] ]. + + periods := aggregate keys size. + total := aggregate values sum: [ :commits | commits size ]. + + ^ { + (#averageFrac -> (total / periods)). + (#averageFloat -> (total / periods) asFloat). + (#periode -> aWeekMonthOrYear name). + (#details -> aggregate) } asOrderedDictionary +] + +{ #category : #'as yet unclassified' } +GitMetric4User >> commitsProducedOnProject: projectId since: sinceDate until: untilDate [ | frequencies | @@ -43,29 +76,38 @@ GitMetric4User >> commitFrequencyOnProject: projectId since: sinceDate until: un ] { #category : #frequency } -GitMetric4User >> commitFrequencyOnProject: aProjectId since: since until: until overA: aWeekOrMonthOrYear [ +GitMetric4User >> commitsProducedOnProject: aProjectId since: since until: until overA: aWeekOrMonthOrYear [ "'aWeekOrMonthOrYear' should be the class of Week, Month or Year" - - |frequency aggregatedFrequencies| - aggregatedFrequencies := OrderedDictionary new. - frequency := self commitFrequencyOnProject: aProjectId since: since until: until. - - - frequency associations do: [ :assoc | - |date commits overDate| + "self commitFrequencyOnProject: 6462 since: (Date today - 60 day) until: (Date today) overA: Week. " + | frequency aggregatedFrequencies | + aggregatedFrequencies := OrderedDictionary new. + frequency := self + commitsProducedOnProject: aProjectId + since: since + until: until. + + + frequency associations do: [ :assoc | + | date commits overDate | date := assoc key. - commits := assoc value. - - - " convert the date as its week date. For some english reason, the week start in sunday so we should add a + 1 day " - overDate := ((date asDate) perform: (('as', aWeekOrMonthOrYear name) asSymbol )) asDate. - aggregatedFrequencies at: (overDate printString) ifPresent: [ :collection | collection addAll: commits ] ifAbsentPut: [ commits ]. + commits := assoc value. - ]. - - ^ aggregatedFrequencies + commits ifNotEmpty: [ + " convert the date as its week date. For some english reason, the week start in sunday so we should add a + 1 day " + overDate := (date asDate perform: + ('as' , aWeekOrMonthOrYear name) asSymbol) asDate. + aggregatedFrequencies + at: overDate printString + ifPresent: [ :collection | collection addAll: commits ] + ifAbsentPut: [ commits ] + + ] + + + ]. + ^ aggregatedFrequencies ] { #category : #'as yet unclassified' } From f6baf2eaa1de2246f720597192c35205c82fd49f Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Mon, 27 May 2024 17:13:52 +0200 Subject: [PATCH 45/77] add metrics for avgDelayUntilFirstChrun, commitFrequency and mergeRequestDurantion --- .../GitAnalyzer.class.st | 4 +- .../GitMetric4User.class.st | 163 ++++++++++++++++-- .../GLPHApi.class.st | 19 ++ .../GLPHModelImporter.class.st | 50 ++++++ 4 files changed, 219 insertions(+), 17 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st index 1d95513..f25d88b 100644 --- a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st @@ -115,7 +115,7 @@ GitAnalyzer >> computeChurnOnFiles: aCollection [ file := assoc key. results := assoc value. - results values ifEmpty: [ churn := nil. ] ifNotEmpty: [ churn := ((results values select: [ :a | a value > 1 ] )sum: [ :a | a value ]) / (results values sum: [ :a | a value ]) . ]. + results values ifEmpty: [ churn := -1. ] ifNotEmpty: [ churn := ((results values select: [ :a | a value > 1 ] )sum: [ :a | a value ]) / (results values sum: [ :a | a value ]) . ]. (file -> churn) ]. @@ -171,7 +171,7 @@ GitAnalyzer >> firstAmandmentFromChrun: aChurnAnalysis [ ]. - ^ (firstCommitsPerFile sortAscending: [:c | c created_at ]) first. + ^ (firstCommitsPerFile sortAscending: [:c | c created_at ]) ifEmpty: nil ifNotEmpty: [ :v | v first ] . ] diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index bc5eff7..317cf1f 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -38,12 +38,14 @@ GitMetric4User >> commitFrequencySince: since until: until overA: aWeekMonthOrYe periods := aggregate keys size. total := aggregate values sum: [ :commits | commits size ]. + ^ { + (#averageFrac -> (total / periods)). (#averageFloat -> (total / periods) asFloat). (#periode -> aWeekMonthOrYear name). - (#details -> aggregate) } asOrderedDictionary + (#details -> aggregate) } asOrderedDictionary ] { #category : #'as yet unclassified' } @@ -78,7 +80,9 @@ GitMetric4User >> commitsProducedOnProject: projectId since: sinceDate until: un { #category : #frequency } GitMetric4User >> commitsProducedOnProject: aProjectId since: since until: until overA: aWeekOrMonthOrYear [ "'aWeekOrMonthOrYear' should be the class of Week, Month or Year" + "self commitFrequencyOnProject: 6462 since: (Date today - 60 day) until: (Date today) overA: Week. " + | frequency aggregatedFrequencies | aggregatedFrequencies := OrderedDictionary new. frequency := self @@ -91,25 +95,75 @@ GitMetric4User >> commitsProducedOnProject: aProjectId since: since until: until | date commits overDate | date := assoc key. commits := assoc value. - - commits ifNotEmpty: [ - " convert the date as its week date. For some english reason, the week start in sunday so we should add a + 1 day " - overDate := (date asDate perform: - ('as' , aWeekOrMonthOrYear name) asSymbol) asDate. - aggregatedFrequencies - at: overDate printString - ifPresent: [ :collection | collection addAll: commits ] - ifAbsentPut: [ commits ] - - ] - - - ]. + commits ifNotEmpty: [ " convert the date as its week date. For some english reason, the week start in sunday so we should add a + 1 day " + overDate := self transformDate: date to: aWeekOrMonthOrYear. + aggregatedFrequencies + at: overDate printString + ifPresent: [ :collection | collection addAll: commits ] + ifAbsentPut: [ commits ] ] ]. ^ aggregatedFrequencies ] +{ #category : #churn } +GitMetric4User >> delayUntilFirstChurnSince: since until: until overA: aDateWeekMonthOrYear [ + + |commits groupedByDate res avg | + + groupedByDate := OrderedDictionary new. + + commits := self loadCommitOfProjects: (itsProjects keys) since: since until: until. + + "class commit by dates, filter none user commits" + commits do: [ :commit | + (commit commitCreator = user) ifTrue: [ + |overDate| + overDate := self transformDate: commit created_at to: aDateWeekMonthOrYear. + + groupedByDate + at: overDate printString + ifPresent: [ :arrayOfCommits | arrayOfCommits add: commit ] + ifAbsentPut: [ + OrderedCollection new + add: commit; + yourself ]. + ]. + ]. + + + + res := groupedByDate collect: [ :commits4Date| + |durationFromA2B| + + commits4Date collect: [ :commitA | + |commitB| + + commitB := gitAnalyzer glhImporter: glhImporter; + onModel: glhModel; + fromCommit: commitA; + analyseAmandment. + + durationFromA2B := (commitB ifNil: [0] ifNotNil: [commitB created_at - commitA created_at]) . + durationFromA2B. + ]. + ]. + + res := res collect: [ :durationsByDate | + (durationsByDate sum:[:v | v asDuration ])/(durationsByDate size). + ]. + + avg := (res values sum) /(res keys size). + + ^ { + (#avg -> avg). + (#periode -> aDateWeekMonthOrYear name). + (#forOver -> (groupedByDate keys size printString + , aDateWeekMonthOrYear printString)). + (#details -> groupedByDate). + } asDictionary . +] + { #category : #'as yet unclassified' } GitMetric4User >> findUserNamed: aUsername [ @@ -153,6 +207,23 @@ GitMetric4User >> loadCommitOfProjects: aCollection since: aTimespan [ ^ self itsCommits. ] +{ #category : #churn } +GitMetric4User >> loadCommitOfProjects: aCollection since: since until: until [ + + | commits | + commits := (aCollection collect: [ :id | + glhImporter + importCommitsOProject: (itsProjects at: id) + since: since + until: until ]) flattened first. + + commits do: [ :commit | glhImporter completeImportedCommit: commit ]. + + glhImporter chainsCommitsFrom: commits. + + ^ commits +] + { #category : #loading } GitMetric4User >> loadProjects: aCollection [ @@ -162,6 +233,68 @@ GitMetric4User >> loadProjects: aCollection [ ^ itsProjects ] +{ #category : #metrics } +GitMetric4User >> mergeRequestDurationSince: since until: until overA: aDateWeekMonthOrYear [ + + | mergeRequest res groupedByDate avg | + groupedByDate := OrderedDictionary new. + + gitAnalyzer := GitAnalyzer new + glhImporter: glhImporter; + onModel: glhModel. + + mergeRequest := itsProjects collect: [ :project | + glhImporter + importMergeRequests: project + since: since + until: until ]. + + mergeRequest := mergeRequest values flattened. + + res := mergeRequest collect: [ :mr | + gitAnalyzer analyseMergeResquestValidation: mr ]. + + + res do: [ :dic | + | overDate | + overDate := self + transformDate: (dic at: #created_at) + to: aDateWeekMonthOrYear. + + groupedByDate + at: overDate printString + ifPresent: [ :durations | durations add: (dic at: #duration) ] + ifAbsentPut: [ + OrderedCollection new + add: (dic at: #duration); + yourself ] ]. + + groupedByDate associations do: [ :assoc | + | sum denominator | + denominator := assoc value size. + sum := assoc value sum: [ :v | v ]. + assoc value: sum / denominator ]. + + + avg := (groupedByDate values sum: [ :v | v ]) + / groupedByDate keys size. + + + ^ { + (#avgDuration -> avg). + (#overEach -> aDateWeekMonthOrYear name). + (#forOver -> (groupedByDate keys size printString + , aDateWeekMonthOrYear printString)). + (#totalMergeRequest -> mergeRequest size). + (#details -> groupedByDate) } asDictionary +] + +{ #category : #'as yet unclassified' } +GitMetric4User >> transformDate: date to: aWeekOrMonthOrYear [ + ^ (date asDate perform: + ('as' , aWeekOrMonthOrYear name) asSymbol) asDate +] + { #category : #accessing } GitMetric4User >> user [ ^ user diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st index b369760..1f285a9 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st @@ -129,6 +129,25 @@ GLPHApi >> mergeRequestOfProject: aProjectID withId: anMRiid [ , '/merge_requests/' , anMRiid printString ] +{ #category : #mergeRequest } +GLPHApi >> mergeRequestsOfProject: aProjectId createdAfter: createdAfter createdBefore: createdBefore authorId: authorId authorUsername: authorUsername scope: scope orderBy: orderBy page: page [ + self client path: + self baseAPIUrl , '/projects/' , aProjectId printString + , '/merge_requests'. + + createdAfter ifNotNil: [ self client queryAt: #created_after put: createdAfter ]. + createdBefore ifNotNil: [ self client queryAt: #created_before put: createdBefore ]. + authorId ifNotNil: [ self client queryAt: #author_id put: authorId ]. + authorUsername ifNotNil: [ self client queryAt: #author_username put: authorUsername ]. + scope ifNotNil: [ + self client queryAt: #scope put: scope ]. + orderBy ifNotNil: [ + self client queryAt: #order_by put: orderBy ]. + page ifNotNil: [ self client queryAt: #page put: page ]. + + ^ self client get +] + { #category : #accessing } GLPHApi >> project: aProjectID [ diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 07bac2f..18750c7 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -210,6 +210,56 @@ GLPHModelImporter >> importMergeRequests: aGLHProject [ self importDiffOfMergeRequest: mr ] ] ] +{ #category : #import } +GLPHModelImporter >> importMergeRequests: aGLHProject since: fromDate until: toDate [ + + | newlyFoundMR page foundMR | + page := 0. + foundMR := OrderedCollection new. + newlyFoundMR := { true }. + [ newlyFoundMR isNotEmpty ] whileTrue: [ + | results | + page := page + 1. + ('import merge request page ' , page printString) recordInfo. + results := self glhApi + mergeRequestsOfProject: aGLHProject id + createdAfter: + (fromDate ifNotNil: [ fromDate asDateAndTime asString ]) + createdBefore: + (toDate ifNotNil: [ toDate asDateAndTime asString ]) + authorId: nil + authorUsername: nil + scope: 'all' + orderBy: 'created_at' + page: page. + + newlyFoundMR := self parseMergeRequestResult: results. + "newlyFoundMR do: [ :mr | + ""1 halt. mr project: aGLHProject"". + aGLHProject mergeRequests add: mr unless: self blockOnIdEquality. + ]." + "TODO : pourquoi le merge resquest est rajouté dans le model ??? " + "newlyFoundMR = (self glhModel allWithType: GLPHEMergeRequest) first . + newlyFoundMR = aGLHProject mergeRequests first." + + newlyFoundMR := aGLHProject mergeRequests + addAll: newlyFoundMR + unless: self blockOnIdEquality. + foundMR addAll: newlyFoundMR ]. + + + "gets it related commits" + aGLHProject mergeRequests do: [ :mr | + self importCommitsOfMergeResquest: mr ]. + + + self withCommitDiffs ifTrue: [ + aGLHProject mergeRequests do: [ :mr | + self importDiffOfMergeRequest: mr ] ]. + + ^ foundMR +] + { #category : #'as yet unclassified' } GLPHModelImporter >> importProject: aProjectID [ | result projectResult | From c640822abf4d5e3840864bfaf60cfe1a9427de24 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Tue, 28 May 2024 17:14:41 +0200 Subject: [PATCH 46/77] add commit churn metric --- .../GitAnalyzer.class.st | 35 +++- .../GitMetric4User.class.st | 160 ++++++++++++------ .../MooseAbstractGroup.extension.st | 2 +- .../GLHModelImporter.class.st | 9 +- 4 files changed, 146 insertions(+), 60 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st index f25d88b..815f4bc 100644 --- a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st @@ -26,12 +26,26 @@ GitAnalyzer >> analyseAmandment [ { #category : #'as yet unclassified' } GitAnalyzer >> analyseChurn [ - | commitFiles | + | commitFiles totalContribution childCommits | + childCommits := OrderedCollection new. + + totalContribution := self + visitChildCommits: fromCommit childCommits + toStoreThemIn: childCommits + upto: self maxChildCommits. + totalContribution := totalContribution sum: [ :commit | + commit additions ]. + + commitFiles := self impactedFilesInFollowUpCommitsOf: fromCommit withMaxCommits: self maxChildCommits. - ^ self computeChurnOnFiles: commitFiles + + + ^ (self computeChurnOnFiles: commitFiles) + at: #totalContribution + ifAbsentPut: totalContribution; yourself. ] { #category : #'as yet unclassified' } @@ -289,3 +303,20 @@ GitAnalyzer >> visitChildCommits: commits lookingForFiles: commitFiles upto: nCo ^ commitFiles ] + +{ #category : #visiting } +GitAnalyzer >> visitChildCommits: commits toStoreThemIn: commitsFound upto: nCommits [ + + commits ifEmpty: [ ^ commitsFound ]. + nCommits = 0 ifTrue: [ ^ commitsFound ]. + + commits do: [ :commit | + commitsFound add: commit. + + self + visitChildCommits: commit childCommits + toStoreThemIn: commitsFound + upto: nCommits - 1 ]. + + ^ commitsFound +] diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index 317cf1f..518de83 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -13,6 +13,55 @@ Class { #category : #'GitLabHealth-Model-Analysis' } +{ #category : #churn } +GitMetric4User >> codeChurnSince: since until: until onACommitWindowOf: commitLimit overA: aDateWeekMonthOrYear [ + |commits res groupedByDate| + + groupedByDate := OrderedDictionary new. + gitAnalyzer := GitAnalyzer new glhImporter: glhImporter; onModel: glhModel; maxChildCommit: commitLimit. + + commits := self loadCommitOfProjects: itsProjects keys since: since until: until. + commits := commits reject: [ :commit | commit commitCreator ~= user ]. + + + "for each commit, we get its churn result (a dictionnary)" + res := commits collect: [ :commit | + |overDate| + commit -> (gitAnalyzer fromCommit: commit; analyseChurn). + ]. + + + res do: [ :assoc | + |commit churnResult sumChurn overDate contribution fraction| + commit := assoc key. + churnResult := assoc value. + sumChurn := (churnResult at: #churns) sum: [ :churn | + |numerator| + numerator := churn value asFraction numerator. + self flag: 'must investigate what does it mean to have a -1 in churn'. + (numerator = -1) ifTrue: [ 0 ] ifFalse: [ numerator ] . + ]. + contribution := (churnResult at: #totalContribution). + + fraction := (sumChurn / contribution). + overDate := self transformDate: commit created_at to: aDateWeekMonthOrYear. + groupedByDate at: overDate printString + ifPresent: [ :array | array add: fraction. ] + ifAbsentPut: [ OrderedCollection new add: fraction; yourself ]. + ]. + + groupedByDate := groupedByDate collect: [ :churns | + |totalChurn totalContribution| + totalChurn := churns sum: [ :churn | churn asFraction numerator ]. + totalContribution := churns sum: [ :churn | churn asFraction denominator ]. + totalChurn / totalContribution. + ]. + + + + ^ groupedByDate. +] + { #category : #'as yet unclassified' } GitMetric4User >> commitFrequencySince: since until: until overA: aWeekMonthOrYear [ @@ -38,12 +87,13 @@ GitMetric4User >> commitFrequencySince: since until: until overA: aWeekMonthOrYe periods := aggregate keys size. total := aggregate values sum: [ :commits | commits size ]. - + ^ { - (#averageFrac -> (total / periods)). (#averageFloat -> (total / periods) asFloat). + (#forOver -> (aggregate keys size printString + , aWeekMonthOrYear printString)). (#periode -> aWeekMonthOrYear name). (#details -> aggregate) } asOrderedDictionary ] @@ -107,61 +157,69 @@ GitMetric4User >> commitsProducedOnProject: aProjectId since: since until: until ] { #category : #churn } -GitMetric4User >> delayUntilFirstChurnSince: since until: until overA: aDateWeekMonthOrYear [ - - |commits groupedByDate res avg | +GitMetric4User >> delayUntilFirstChurnSince: since until: until overA: aDateWeekMonthOrYear [ + + | commits groupedByDate res avg | + groupedByDate := OrderedDictionary new. + + commits := self + loadCommitOfProjects: itsProjects keys + since: since + until: until. - groupedByDate := OrderedDictionary new. - - commits := self loadCommitOfProjects: (itsProjects keys) since: since until: until. - "class commit by dates, filter none user commits" commits do: [ :commit | - (commit commitCreator = user) ifTrue: [ - |overDate| - overDate := self transformDate: commit created_at to: aDateWeekMonthOrYear. - - groupedByDate - at: overDate printString - ifPresent: [ :arrayOfCommits | arrayOfCommits add: commit ] - ifAbsentPut: [ - OrderedCollection new - add: commit; - yourself ]. - ]. - ]. - - - - res := groupedByDate collect: [ :commits4Date| - |durationFromA2B| - - commits4Date collect: [ :commitA | - |commitB| - - commitB := gitAnalyzer glhImporter: glhImporter; - onModel: glhModel; - fromCommit: commitA; - analyseAmandment. - - durationFromA2B := (commitB ifNil: [0] ifNotNil: [commitB created_at - commitA created_at]) . - durationFromA2B. - ]. - ]. + commit commitCreator = user ifTrue: [ + | overDate | + overDate := self + transformDate: commit created_at + to: aDateWeekMonthOrYear. + + groupedByDate + at: overDate printString + ifPresent: [ :arrayOfCommits | arrayOfCommits add: commit ] + ifAbsentPut: [ + OrderedCollection new + add: commit; + yourself ] ] ]. + + + + res := groupedByDate collect: [ :commits4Date | + | durationFromA2B | + commits4Date collect: [ :commitA | + | commitB | + commitB := gitAnalyzer + glhImporter: glhImporter; + onModel: glhModel; + fromCommit: commitA; + analyseAmandment. + + durationFromA2B := commitB + ifNil: [ 0 ] + ifNotNil: [ + commitB created_at - commitA created_at ]. + durationFromA2B ] ]. + + res := res collect: [ :durationsByDate | - (durationsByDate sum:[:v | v asDuration ])/(durationsByDate size). - ]. - - avg := (res values sum) /(res keys size). - + |filtered| + filtered := durationsByDate reject: [ :value | value = 0 ]. + filtered isEmpty ifTrue: [ nil ] ifFalse: [ (filtered sum: [ :v | v asDuration ]) + / filtered size ] + ]. + + res := res reject: #isNil. + + avg := res values sum / res keys size. + ^ { - (#avg -> avg). - (#periode -> aDateWeekMonthOrYear name). - (#forOver -> (groupedByDate keys size printString + (#avg -> avg). + (#periode -> aDateWeekMonthOrYear name). + (#forOver -> (groupedByDate keys size printString , aDateWeekMonthOrYear printString)). - (#details -> groupedByDate). - } asDictionary . + (#details -> groupedByDate) } asDictionary ] { #category : #'as yet unclassified' } @@ -272,7 +330,7 @@ GitMetric4User >> mergeRequestDurationSince: since until: until overA: aDateWeek groupedByDate associations do: [ :assoc | | sum denominator | denominator := assoc value size. - sum := assoc value sum: [ :v | v ]. + sum := assoc value sum: [ :v | v ifNil: [denominator := denominator -1. 0 asDuration] ]. assoc value: sum / denominator ]. diff --git a/src/GitLabHealth-Model-Extension/MooseAbstractGroup.extension.st b/src/GitLabHealth-Model-Extension/MooseAbstractGroup.extension.st index 561dc81..e0b3f81 100644 --- a/src/GitLabHealth-Model-Extension/MooseAbstractGroup.extension.st +++ b/src/GitLabHealth-Model-Extension/MooseAbstractGroup.extension.st @@ -28,7 +28,7 @@ MooseAbstractGroup >> addAll: anElmtCollection ofType: aType unless: aConditionA MooseAbstractGroup >> addAll: anElmtCollection unless: aConditionAsBlock [ "()>>>" - anElmtCollection ifEmpty: [ ^ self ]. + anElmtCollection ifEmpty: [ ^ {} ]. ^ self addAll: anElmtCollection ofType: anElmtCollection first class diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index 01814f5..d37a861 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -241,7 +241,7 @@ GLHModelImporter >> importCommitsOProject: aProject since: fromDate until: toDat path: nil author: nil all: true - with_stats: nil + with_stats: true firstParent: nil order: nil trailers: nil @@ -258,6 +258,7 @@ GLHModelImporter >> importCommitsOProject: aProject since: fromDate until: toDat unless: self blockOnIdEquality. foundCommit addAll: newlyFoundCommit ]. + ^ self glhModel addAll: foundCommit unless: self blockOnIdEquality. ] @@ -529,15 +530,11 @@ GLHModelImporter >> importUser: aUserID [ ^ (glhModel allWithType: GLHUser) detect: [ :user | user id = aUserID ] ifNone: [ - ('Import user: ' , aUserID printString) recordInfo. result := self glhApi user: aUserID. userResult := self parseUserResult: result. glhModel add: userResult unless: self blockOnIdEquality. - self flag: - 'please optimize by changing add:unless: so it return the add elmnt'. - (glhModel allWithType: GLHUser) detect: [ :user | - user id = aUserID ] ] + ] ] { #category : #user } From 266f96b16c4540d3c61c071b6b7ecdb102924a22 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Tue, 28 May 2024 17:40:36 +0200 Subject: [PATCH 47/77] add details result on churn metrics --- .../GitMetric4User.class.st | 98 ++++++++++++------- 1 file changed, 60 insertions(+), 38 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index 518de83..1b44460 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -14,52 +14,74 @@ Class { } { #category : #churn } -GitMetric4User >> codeChurnSince: since until: until onACommitWindowOf: commitLimit overA: aDateWeekMonthOrYear [ - |commits res groupedByDate| - - groupedByDate := OrderedDictionary new. - gitAnalyzer := GitAnalyzer new glhImporter: glhImporter; onModel: glhModel; maxChildCommit: commitLimit. - - commits := self loadCommitOfProjects: itsProjects keys since: since until: until. +GitMetric4User >> codeChurnSince: since until: until onACommitWindowOf: commitLimit overA: aDateWeekMonthOrYear [ + + | commits res groupedByDate | + groupedByDate := OrderedDictionary new. + gitAnalyzer := GitAnalyzer new + glhImporter: glhImporter; + onModel: glhModel; + maxChildCommit: commitLimit. + + commits := self + loadCommitOfProjects: itsProjects keys + since: since + until: until. commits := commits reject: [ :commit | commit commitCreator ~= user ]. - - + + "for each commit, we get its churn result (a dictionnary)" res := commits collect: [ :commit | - |overDate| - commit -> (gitAnalyzer fromCommit: commit; analyseChurn). - ]. - - + commit -> (gitAnalyzer + fromCommit: commit; + analyseChurn) ]. + + res do: [ :assoc | - |commit churnResult sumChurn overDate contribution fraction| + | commit churnResult sumChurn overDate contribution fraction | commit := assoc key. churnResult := assoc value. sumChurn := (churnResult at: #churns) sum: [ :churn | - |numerator| - numerator := churn value asFraction numerator. - self flag: 'must investigate what does it mean to have a -1 in churn'. - (numerator = -1) ifTrue: [ 0 ] ifFalse: [ numerator ] . - ]. - contribution := (churnResult at: #totalContribution). - - fraction := (sumChurn / contribution). - overDate := self transformDate: commit created_at to: aDateWeekMonthOrYear. - groupedByDate at: overDate printString - ifPresent: [ :array | array add: fraction. ] - ifAbsentPut: [ OrderedCollection new add: fraction; yourself ]. - ]. - - groupedByDate := groupedByDate collect: [ :churns | - |totalChurn totalContribution| - totalChurn := churns sum: [ :churn | churn asFraction numerator ]. - totalContribution := churns sum: [ :churn | churn asFraction denominator ]. - totalChurn / totalContribution. - ]. - - + | numerator | + numerator := churn value asFraction numerator. + self flag: + 'must investigate what does it mean to have a -1 in churn'. + numerator = -1 + ifTrue: [ 0 ] + ifFalse: [ numerator ] ]. + contribution := churnResult at: #totalContribution. + + fraction := sumChurn / contribution. + overDate := self + transformDate: commit created_at + to: aDateWeekMonthOrYear. + groupedByDate + at: overDate printString + ifPresent: [ :array | array add: fraction ] + ifAbsentPut: [ + OrderedCollection new + add: fraction; + yourself ] ]. + + groupedByDate := groupedByDate collect: [ :churns | + | totalChurn totalContribution | + totalChurn := churns sum: [ :churn | + churn asFraction numerator ]. + totalContribution := churns sum: [ :churn | + churn asFraction denominator ]. + totalChurn / totalContribution ]. + + - ^ groupedByDate. + ^ { + (#details -> groupedByDate). + (#totalChurn -> (groupedByDate sum: [ :frac | frac numerator ])). + (#avgChurn -> ((groupedByDate sum: [ :frac | frac numerator ]) / groupedByDate keys size) ) . + (#overEach -> aDateWeekMonthOrYear name). + (#forOver -> (groupedByDate keys size printString + , aDateWeekMonthOrYear printString)). + (#userCommits -> commits size). + (#churnWindow -> commitLimit) } asDictionary ] { #category : #'as yet unclassified' } From 68a4f55299e2d967a41acfb11ea559262d393117 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Tue, 28 May 2024 18:23:40 +0200 Subject: [PATCH 48/77] fix diffrange import --- src/GitLabHealth-Model-Importer/GLHModelImporter.class.st | 7 ++----- .../GLPHModelImporter.class.st | 6 +++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index d37a861..7baa610 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -249,9 +249,7 @@ GLHModelImporter >> importCommitsOProject: aProject since: fromDate until: toDat page: page. newlyFoundCommit := self parseCommitsResult: results. - newlyFoundCommit do: [ :c | - - c repository: aProject repository ]. + newlyFoundCommit do: [ :c | c repository: aProject repository ]. aProject repository commits addAll: newlyFoundCommit @@ -259,8 +257,7 @@ GLHModelImporter >> importCommitsOProject: aProject since: fromDate until: toDat foundCommit addAll: newlyFoundCommit ]. - ^ self glhModel addAll: foundCommit unless: self blockOnIdEquality. - + ^ self glhModel addAll: foundCommit unless: self blockOnIdEquality ] { #category : #api } diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 18750c7..80a5854 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -179,9 +179,8 @@ GLPHModelImporter >> importDiffRangesForDiff: aGLHDiff [ diffRanges := self parseDiffString: aGLHDiff. self glhModel addAll: diffRanges unless: self blockForDiffRangeEquality . - aGLHDiff diffRanges addAll: diffRanges unless: self blockForDiffRangeEquality . + ^ aGLHDiff diffRanges addAll: diffRanges unless: self blockForDiffRangeEquality . - ^ aGLHDiff diffRanges ] { #category : #import } @@ -297,9 +296,10 @@ GLPHModelImporter >> parseDiffString: aDiff [ range := nil. (lines ifEmpty: [ ^ aDiff diffRanges ]). + lines := self filterCommitChanges: lines. (lines first beginsWith: '@@') ifFalse: [ ^ aDiff diffRanges ]. - lines := self filterCommitChanges: lines. + lines do: [ :line | | aChange | From f6f92812190c9cbab190d70152f5600743a72d59 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Wed, 29 May 2024 15:39:26 +0200 Subject: [PATCH 49/77] add contribution metrics --- .../GitAnalyzer.class.st | 7 +++ .../GitMetric4User.class.st | 57 ++++++++++++++++++- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st index 815f4bc..0c370c3 100644 --- a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st @@ -48,6 +48,13 @@ GitAnalyzer >> analyseChurn [ ifAbsentPut: totalContribution; yourself. ] +{ #category : #commit } +GitAnalyzer >> analyseCommitContribution [ + + ^ { (#addition -> fromCommit additions). + (#deletion -> fromCommit deletions). } asDictionary +] + { #category : #'as yet unclassified' } GitAnalyzer >> analyseCommitFrequencySince: since until: until [ diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index 1b44460..1699e3f 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -16,7 +16,8 @@ Class { { #category : #churn } GitMetric4User >> codeChurnSince: since until: until onACommitWindowOf: commitLimit overA: aDateWeekMonthOrYear [ - | commits res groupedByDate | + | commits res groupedByDate totalContributions| + totalContributions := OrderedCollection new. groupedByDate := OrderedDictionary new. gitAnalyzer := GitAnalyzer new glhImporter: glhImporter; @@ -50,7 +51,8 @@ GitMetric4User >> codeChurnSince: since until: until onACommitWindowOf: commitLi ifTrue: [ 0 ] ifFalse: [ numerator ] ]. contribution := churnResult at: #totalContribution. - + totalContributions add:contribution. + fraction := sumChurn / contribution. overDate := self transformDate: commit created_at @@ -75,8 +77,10 @@ GitMetric4User >> codeChurnSince: since until: until onACommitWindowOf: commitLi ^ { (#details -> groupedByDate). + (#totalContribution -> totalContributions sum). (#totalChurn -> (groupedByDate sum: [ :frac | frac numerator ])). - (#avgChurn -> ((groupedByDate sum: [ :frac | frac numerator ]) / groupedByDate keys size) ) . + (#avgChurn -> ((groupedByDate sum: [ :frac | frac numerator ]) + / groupedByDate keys size ifEmpty: 1)). (#overEach -> aDateWeekMonthOrYear name). (#forOver -> (groupedByDate keys size printString , aDateWeekMonthOrYear printString)). @@ -84,6 +88,53 @@ GitMetric4User >> codeChurnSince: since until: until onACommitWindowOf: commitLi (#churnWindow -> commitLimit) } asDictionary ] +{ #category : #'as yet unclassified' } +GitMetric4User >> codeContributionsSince: since until: until overA: aDateWeekMonthOrYear [ + |commits contributions groupedByDate| + + + groupedByDate := OrderedDictionary new. + gitAnalyzer := GitAnalyzer new onModel: glhModel; glhImporter: glhImporter. + + commits := self + loadCommitOfProjects: itsProjects keys + since: since + until: until. + commits := commits reject: [ :commit | commit commitCreator ~= user ]. + + contributions := commits collect: [ :commit | + commit -> (gitAnalyzer + fromCommit: commit; + analyseCommitContribution) ]. + + contributions do: [ :assoc | + |dateOver| + dateOver := self transformDate: (assoc key created_at) to: aDateWeekMonthOrYear. + groupedByDate at: (dateOver printString) ifPresent: [ :v | v add: assoc value ] ifAbsentPut: [ OrderedCollection new add: assoc value; yourself ]. + ]. + + groupedByDate := groupedByDate collect: [ :contribs | + |totalAdd totalDele| + totalAdd := contribs sum: [ :v | v at: #addition ]. + totalDele := contribs sum: [ :v | v at: #deletion ]. + { (#addition -> totalAdd). (#deletion -> totalDele) } asDictionary + ]. + + + ^ { + (#overEach -> aDateWeekMonthOrYear name). + (#forOver -> (groupedByDate keys size printString + , aDateWeekMonthOrYear printString)). + (#avgAddition -> ((groupedByDate values sum: [ :v | v at: #addition ]) / (groupedByDate keys size)) asFloat ). + (#avgDeletion -> (((groupedByDate values sum: [ :v | v at: #deletion ]) / (groupedByDate keys size)) asFloat) ). + (#details -> groupedByDate). + (#userCommits -> commits size). + } asDictionary + + + +] + { #category : #'as yet unclassified' } GitMetric4User >> commitFrequencySince: since until: until overA: aWeekMonthOrYear [ From c22f6618c5a1bd4fca416ad92b0a1a5b8f36be18 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 30 May 2024 13:59:17 +0200 Subject: [PATCH 50/77] fix metric period --- .../GitAnalyzer.class.st | 15 ++ .../GitMetric4User.class.st | 244 +++++++++++++----- 2 files changed, 197 insertions(+), 62 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st index 0c370c3..9aabada 100644 --- a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st @@ -48,6 +48,21 @@ GitAnalyzer >> analyseChurn [ ifAbsentPut: totalContribution; yourself. ] +{ #category : #'as yet unclassified' } +GitAnalyzer >> analyseCommentContributions [ + |additions comments| + + additions := (fromCommit diffs collect: [:diff | + diff diffRanges collect: [ :range | + range changes select: [ :change | + change class = GLPHEAddition. ] ] ]) flattened collect: [ :add | add sourceCode] . + + additions := additions collect: [ :codeSource | (codeSource withoutPrefix: '+') trim ]. + additions := additions select: [ :codeSource | codeSource beginsWithAnyOf: { '//' . '/*' . '*' . '*/' } ]. + + ^ additions size +] + { #category : #commit } GitAnalyzer >> analyseCommitContribution [ diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index 1699e3f..1a85738 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -16,9 +16,12 @@ Class { { #category : #churn } GitMetric4User >> codeChurnSince: since until: until onACommitWindowOf: commitLimit overA: aDateWeekMonthOrYear [ - | commits res groupedByDate totalContributions| - totalContributions := OrderedCollection new. - groupedByDate := OrderedDictionary new. + | commits res groupedByDate totalContributions | + totalContributions := OrderedCollection new. + groupedByDate := self + setupGroupedDateFrom: since + to: until + over: aDateWeekMonthOrYear. gitAnalyzer := GitAnalyzer new glhImporter: glhImporter; onModel: glhModel; @@ -51,8 +54,8 @@ GitMetric4User >> codeChurnSince: since until: until onACommitWindowOf: commitLi ifTrue: [ 0 ] ifFalse: [ numerator ] ]. contribution := churnResult at: #totalContribution. - totalContributions add:contribution. - + totalContributions add: contribution. + fraction := sumChurn / contribution. overDate := self transformDate: commit created_at @@ -71,16 +74,17 @@ GitMetric4User >> codeChurnSince: since until: until onACommitWindowOf: commitLi churn asFraction numerator ]. totalContribution := churns sum: [ :churn | churn asFraction denominator ]. - totalChurn / totalContribution ]. + totalContribution = 0 ifTrue: [ totalChurn / 1] ifFalse: [ totalChurn / totalContribution ] + ]. ^ { (#details -> groupedByDate). - (#totalContribution -> totalContributions sum). + (#totalContribution -> totalContributions sum). (#totalChurn -> (groupedByDate sum: [ :frac | frac numerator ])). (#avgChurn -> ((groupedByDate sum: [ :frac | frac numerator ]) - / groupedByDate keys size ifEmpty: 1)). + / (groupedByDate keys ifEmpty: [ 1 ] ifNotEmpty: [ groupedByDate keys size ]) )). (#overEach -> aDateWeekMonthOrYear name). (#forOver -> (groupedByDate keys size printString , aDateWeekMonthOrYear printString)). @@ -88,57 +92,133 @@ GitMetric4User >> codeChurnSince: since until: until onACommitWindowOf: commitLi (#churnWindow -> commitLimit) } asDictionary ] -{ #category : #'as yet unclassified' } +{ #category : #metrics } GitMetric4User >> codeContributionsSince: since until: until overA: aDateWeekMonthOrYear [ - |commits contributions groupedByDate| + | commits contributions groupedByDate | + groupedByDate := self + setupGroupedDateFrom: since + to: until + over: aDateWeekMonthOrYear. + + gitAnalyzer := GitAnalyzer new + onModel: glhModel; + glhImporter: glhImporter. - groupedByDate := OrderedDictionary new. - gitAnalyzer := GitAnalyzer new onModel: glhModel; glhImporter: glhImporter. - commits := self loadCommitOfProjects: itsProjects keys since: since until: until. commits := commits reject: [ :commit | commit commitCreator ~= user ]. - + contributions := commits collect: [ :commit | - commit -> (gitAnalyzer - fromCommit: commit; - analyseCommitContribution) ]. - + commit -> (gitAnalyzer + fromCommit: commit; + analyseCommitContribution) ]. + contributions do: [ :assoc | - |dateOver| - dateOver := self transformDate: (assoc key created_at) to: aDateWeekMonthOrYear. - groupedByDate at: (dateOver printString) ifPresent: [ :v | v add: assoc value ] ifAbsentPut: [ OrderedCollection new add: assoc value; yourself ]. - ]. - + | dateOver | + dateOver := self + transformDate: assoc key created_at + to: aDateWeekMonthOrYear. + groupedByDate + at: dateOver printString + ifPresent: [ :v | v add: assoc value ] + ifAbsentPut: [ + OrderedCollection new + add: assoc value; + yourself ] ]. + groupedByDate := groupedByDate collect: [ :contribs | - |totalAdd totalDele| - totalAdd := contribs sum: [ :v | v at: #addition ]. - totalDele := contribs sum: [ :v | v at: #deletion ]. - { (#addition -> totalAdd). (#deletion -> totalDele) } asDictionary - ]. - - - ^ { - (#overEach -> aDateWeekMonthOrYear name). - (#forOver -> (groupedByDate keys size printString + | totalAdd totalDele | + totalAdd := contribs sum: [ :v | v at: #addition ]. + totalDele := contribs sum: [ :v | v at: #deletion ]. + { + (#addition -> totalAdd). + (#deletion -> totalDele) } asDictionary ]. + + + ^ { + (#overEach -> aDateWeekMonthOrYear name). + (#forOver -> (groupedByDate keys size printString , aDateWeekMonthOrYear printString)). - (#avgAddition -> ((groupedByDate values sum: [ :v | v at: #addition ]) / (groupedByDate keys size)) asFloat ). - (#avgDeletion -> (((groupedByDate values sum: [ :v | v at: #deletion ]) / (groupedByDate keys size)) asFloat) ). - (#details -> groupedByDate). - (#userCommits -> commits size). - } asDictionary - - - + (#avgAddition + -> ((groupedByDate values sum: [ :v | v at: #addition ]) + / groupedByDate keys size) asFloat). + (#avgDeletion + -> ((groupedByDate values sum: [ :v | v at: #deletion ]) + / groupedByDate keys size) asFloat). + (#details -> groupedByDate). + (#userCommits -> commits size) } asDictionary ] -{ #category : #'as yet unclassified' } -GitMetric4User >> commitFrequencySince: since until: until overA: aWeekMonthOrYear [ +{ #category : #metrics } +GitMetric4User >> commentsContributionsSince: since until: until overA: aDateWeekMonthOrYear [ + + | commits contributions groupedByDate | + groupedByDate := self + setupGroupedDateFrom: since + to: until + over: aDateWeekMonthOrYear. + + + + gitAnalyzer := GitAnalyzer new + onModel: glhModel; + glhImporter: glhImporter. + + commits := self + loadCommitOfProjects: itsProjects keys + since: since + until: until. + commits := commits reject: [ :commit | commit commitCreator ~= user ]. + + contributions := commits collect: [ :commit | + commit -> (gitAnalyzer + fromCommit: commit; + analyseCommentContributions) ]. + + + contributions do: [ :assoc | + | dateOver | + dateOver := self + transformDate: assoc key created_at + to: aDateWeekMonthOrYear. + groupedByDate + at: dateOver printString + ifPresent: [ :v | v add: assoc value ] + ifAbsentPut: [ + OrderedCollection new + add: assoc value; + yourself ] ]. + + groupedByDate := groupedByDate collect: [ :contribs | + + contribs + ifNotEmpty: [ contribs sum ] + ifEmpty: [ 0 ] ]. + + + ^ { + (#details -> groupedByDate). + (#totalComments -> groupedByDate values sum). + (#avgComments + -> (groupedByDate values sum / groupedByDate keys size)). + (#overEach -> aDateWeekMonthOrYear name). + (#forOver -> (groupedByDate keys size printString + , aDateWeekMonthOrYear printString)). + (#userCommits -> commits size) } asDictionary +] + +{ #category : #metrics } +GitMetric4User >> commitFrequencySince: since until: until overA: aDateWeekMonthOrYear [ + + | all aggregate periods total groupedByDate | + groupedByDate := self + setupGroupedDateFrom: since + to: until + over: aDateWeekMonthOrYear. - | all aggregate periods total | aggregate := OrderedDictionary new. all := itsProjects keys collect: [ :id | @@ -146,9 +226,7 @@ GitMetric4User >> commitFrequencySince: since until: until overA: aWeekMonthOrYe commitsProducedOnProject: id since: since until: until - overA: aWeekMonthOrYear ]. - - + overA: aDateWeekMonthOrYear ]. all do: [ :frequencies | @@ -165,9 +243,11 @@ GitMetric4User >> commitFrequencySince: since until: until overA: aWeekMonthOrYe ^ { (#averageFrac -> (total / periods)). (#averageFloat -> (total / periods) asFloat). - (#forOver -> (aggregate keys size printString - , aWeekMonthOrYear printString)). - (#periode -> aWeekMonthOrYear name). + (#forOver + -> + (aggregate keys size printString + , aDateWeekMonthOrYear printString)). + (#periode -> aDateWeekMonthOrYear name). (#details -> aggregate) } asOrderedDictionary ] @@ -200,14 +280,14 @@ GitMetric4User >> commitsProducedOnProject: projectId since: sinceDate until: un ^ frequencies ] -{ #category : #frequency } +{ #category : #metrics } GitMetric4User >> commitsProducedOnProject: aProjectId since: since until: until overA: aWeekOrMonthOrYear [ "'aWeekOrMonthOrYear' should be the class of Week, Month or Year" "self commitFrequencyOnProject: 6462 since: (Date today - 60 day) until: (Date today) overA: Week. " | frequency aggregatedFrequencies | - aggregatedFrequencies := OrderedDictionary new. + aggregatedFrequencies := self setupGroupedDateFrom: since to: until over: aWeekOrMonthOrYear. frequency := self commitsProducedOnProject: aProjectId since: since @@ -233,7 +313,10 @@ GitMetric4User >> commitsProducedOnProject: aProjectId since: since until: until GitMetric4User >> delayUntilFirstChurnSince: since until: until overA: aDateWeekMonthOrYear [ | commits groupedByDate res avg | - groupedByDate := OrderedDictionary new. + groupedByDate := self + setupGroupedDateFrom: since + to: until + over: aDateWeekMonthOrYear. commits := self loadCommitOfProjects: itsProjects keys @@ -274,14 +357,15 @@ GitMetric4User >> delayUntilFirstChurnSince: since until: until overA: aDateWeek commitB created_at - commitA created_at ]. durationFromA2B ] ]. - + res := res collect: [ :durationsByDate | - |filtered| - filtered := durationsByDate reject: [ :value | value = 0 ]. - filtered isEmpty ifTrue: [ nil ] ifFalse: [ (filtered sum: [ :v | v asDuration ]) - / filtered size ] - ]. + | filtered | + filtered := durationsByDate reject: [ :value | value = 0 ]. + filtered isEmpty + ifTrue: [ nil ] + ifFalse: [ + (filtered sum: [ :v | v asDuration ]) / filtered size ] ]. res := res reject: #isNil. @@ -368,7 +452,10 @@ GitMetric4User >> loadProjects: aCollection [ GitMetric4User >> mergeRequestDurationSince: since until: until overA: aDateWeekMonthOrYear [ | mergeRequest res groupedByDate avg | - groupedByDate := OrderedDictionary new. + groupedByDate := self + setupGroupedDateFrom: since + to: until + over: aDateWeekMonthOrYear. gitAnalyzer := GitAnalyzer new glhImporter: glhImporter; @@ -379,9 +466,11 @@ GitMetric4User >> mergeRequestDurationSince: since until: until overA: aDateWeek importMergeRequests: project since: since until: until ]. - mergeRequest := mergeRequest values flattened. + mergeRequest ifEmpty: [ + ^ { (#totalMergeRequest -> mergeRequest size) } asDictionary ]. + res := mergeRequest collect: [ :mr | gitAnalyzer analyseMergeResquestValidation: mr ]. @@ -403,10 +492,14 @@ GitMetric4User >> mergeRequestDurationSince: since until: until overA: aDateWeek groupedByDate associations do: [ :assoc | | sum denominator | denominator := assoc value size. - sum := assoc value sum: [ :v | v ifNil: [denominator := denominator -1. 0 asDuration] ]. + sum := assoc value sum: [ :v | + v ifNil: [ + denominator := denominator - 1. + 0 asDuration ] ]. assoc value: sum / denominator ]. + avg := (groupedByDate values sum: [ :v | v ]) / groupedByDate keys size. @@ -420,6 +513,33 @@ GitMetric4User >> mergeRequestDurationSince: since until: until overA: aDateWeek (#details -> groupedByDate) } asDictionary ] +{ #category : #setup } +GitMetric4User >> setupGroupedDateFrom: since to: until over: aDateWeekMonthOrYear [ + |groupedByDate start end| + + groupedByDate := OrderedDictionary new. + + start := self transformDate: since to: aDateWeekMonthOrYear. + end := self transformDate: until to: aDateWeekMonthOrYear. + + groupedByDate at: (start printString) ifAbsentPut: [ OrderedCollection new ]. + + [(groupedByDate keys last) asDateAndTime < end] whileTrue: [ + |index over| + + over := (aDateWeekMonthOrYear name asLowercase) asSymbol. + (over = #date) ifTrue: [ over := #day ]. + index := (groupedByDate keys last asDateAndTime) + (1 perform: over ) . + index := self transformDate: index to: aDateWeekMonthOrYear. + groupedByDate at: (index printString) ifAbsentPut: [ OrderedCollection new ]. + + ] . + + groupedByDate at: (end printString) ifAbsentPut: [ OrderedCollection new ]. + +^ groupedByDate +] + { #category : #'as yet unclassified' } GitMetric4User >> transformDate: date to: aWeekOrMonthOrYear [ ^ (date asDate perform: From 7167da7f00f5ed092a6b9980284d6c88eb385f15 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 30 May 2024 15:11:11 +0200 Subject: [PATCH 51/77] fix Month export for metric and merge duration avg --- .../GitMetric4User.class.st | 90 +++++++++++++------ 1 file changed, 62 insertions(+), 28 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index 1a85738..66136f1 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -243,7 +243,8 @@ GitMetric4User >> commitFrequencySince: since until: until overA: aDateWeekMonth ^ { (#averageFrac -> (total / periods)). (#averageFloat -> (total / periods) asFloat). - (#forOver + (#userCommit -> total). + (#forOver -> (aggregate keys size printString , aDateWeekMonthOrYear printString)). @@ -372,8 +373,8 @@ GitMetric4User >> delayUntilFirstChurnSince: since until: until overA: aDateWeek avg := res values sum / res keys size. ^ { - (#avg -> avg). - (#periode -> aDateWeekMonthOrYear name). + (#avgDelay -> avg). + (#overEach -> aDateWeekMonthOrYear name). (#forOver -> (groupedByDate keys size printString , aDateWeekMonthOrYear printString)). (#details -> groupedByDate) } asDictionary @@ -451,7 +452,7 @@ GitMetric4User >> loadProjects: aCollection [ { #category : #metrics } GitMetric4User >> mergeRequestDurationSince: since until: until overA: aDateWeekMonthOrYear [ - | mergeRequest res groupedByDate avg | + | mergeRequest res groupedByDate filterGroups avg | groupedByDate := self setupGroupedDateFrom: since to: until @@ -489,19 +490,30 @@ GitMetric4User >> mergeRequestDurationSince: since until: until overA: aDateWeek add: (dic at: #duration); yourself ] ]. - groupedByDate associations do: [ :assoc | + + filterGroups := groupedByDate reject: [ :array | array isEmpty ]. + + filterGroups associations do: [ + :assoc | | sum denominator | + + denominator := assoc value size. + sum := assoc value sum: [ :v | v ifNil: [ denominator := denominator - 1. 0 asDuration ] ]. - assoc value: sum / denominator ]. + denominator = 0 ifTrue: [ denominator := 1 ]. + filterGroups at: assoc key put: sum / denominator ]. - avg := (groupedByDate values sum: [ :v | v ]) - / groupedByDate keys size. + filterGroups keys ifEmpty: [ avg := 0 ] ifNotEmpty: [ + avg := (filterGroups values sum ) + / filterGroups keys size. + + ]. ^ { @@ -514,36 +526,58 @@ GitMetric4User >> mergeRequestDurationSince: since until: until overA: aDateWeek ] { #category : #setup } -GitMetric4User >> setupGroupedDateFrom: since to: until over: aDateWeekMonthOrYear [ - |groupedByDate start end| +GitMetric4User >> setupGroupedDateFrom: since to: until over: aDateWeekMonthOrYear [ - groupedByDate := OrderedDictionary new. - + | groupedByDate start end over increment| + groupedByDate := OrderedDictionary new. + + increment := 1. start := self transformDate: since to: aDateWeekMonthOrYear. end := self transformDate: until to: aDateWeekMonthOrYear. - - groupedByDate at: (start printString) ifAbsentPut: [ OrderedCollection new ]. + + groupedByDate + at: start printString + ifAbsentPut: [ OrderedCollection new ]. - [(groupedByDate keys last) asDateAndTime < end] whileTrue: [ - |index over| - - over := (aDateWeekMonthOrYear name asLowercase) asSymbol. - (over = #date) ifTrue: [ over := #day ]. - index := (groupedByDate keys last asDateAndTime) + (1 perform: over ) . + over := aDateWeekMonthOrYear name asLowercase asSymbol. + over = #date ifTrue: [ over := #day ]. + over = #month ifTrue: [ increment := 32. over := #day ]. + + + [ groupedByDate keys last asDateAndTime < end ] whileTrue: [ + | index | + index := groupedByDate keys last asDateAndTime + (increment perform: over). index := self transformDate: index to: aDateWeekMonthOrYear. - groupedByDate at: (index printString) ifAbsentPut: [ OrderedCollection new ]. - - ] . - - groupedByDate at: (end printString) ifAbsentPut: [ OrderedCollection new ]. + groupedByDate + at: index printString + ifAbsentPut: [ OrderedCollection new ] ]. + -^ groupedByDate + over = #day ifTrue: [ + groupedByDate keys do: [ :date | + |aWeekday| + aWeekday := date asDate weekday. + ((aWeekday = #Sunday) or: [ aWeekday = #Saturday ] )ifTrue: [ + groupedByDate removeKey: date. + ] + ]. + ]. + + + groupedByDate + at: end printString + ifAbsentPut: [ OrderedCollection new ]. + + ^ groupedByDate ] { #category : #'as yet unclassified' } GitMetric4User >> transformDate: date to: aWeekOrMonthOrYear [ - ^ (date asDate perform: - ('as' , aWeekOrMonthOrYear name) asSymbol) asDate + + aWeekOrMonthOrYear = Month ifTrue: [ ^ date asDate month asDate ]. + + ^ (date asDate perform: ('as' , aWeekOrMonthOrYear name) asSymbol) + asDate ] { #category : #accessing } From 8661e4774e776e5b43b79a3290835123311fddcf Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Tue, 4 Jun 2024 14:53:38 +0200 Subject: [PATCH 52/77] latest patch --- .../GitMetric4User.class.st | 41 +++++---- .../GLHModelImporter.class.st | 83 +++++++++++-------- .../GLPHApi.class.st | 8 +- .../GLPHModelImporter.class.st | 25 ++++-- 4 files changed, 96 insertions(+), 61 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index 66136f1..4acfd81 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -81,7 +81,7 @@ GitMetric4User >> codeChurnSince: since until: until onACommitWindowOf: commitLi ^ { (#details -> groupedByDate). - (#totalContribution -> totalContributions sum). + (#totalContribution -> (totalContributions ifEmpty: [ 0 ] ifNotEmpty: [ totalContributions sum ]) ). (#totalChurn -> (groupedByDate sum: [ :frac | frac numerator ])). (#avgChurn -> ((groupedByDate sum: [ :frac | frac numerator ]) / (groupedByDate keys ifEmpty: [ 1 ] ifNotEmpty: [ groupedByDate keys size ]) )). @@ -96,6 +96,9 @@ GitMetric4User >> codeChurnSince: since until: until onACommitWindowOf: commitLi GitMetric4User >> codeContributionsSince: since until: until overA: aDateWeekMonthOrYear [ | commits contributions groupedByDate | + + glhImporter withCommitDiffs: false. + groupedByDate := self setupGroupedDateFrom: since to: until @@ -137,6 +140,7 @@ GitMetric4User >> codeContributionsSince: since until: until overA: aDateWeekMon (#addition -> totalAdd). (#deletion -> totalDele) } asDictionary ]. + glhImporter withCommitDiffs: true. ^ { (#overEach -> aDateWeekMonthOrYear name). @@ -161,8 +165,6 @@ GitMetric4User >> commentsContributionsSince: since until: until overA: aDateWee to: until over: aDateWeekMonthOrYear. - - gitAnalyzer := GitAnalyzer new onModel: glhModel; glhImporter: glhImporter. @@ -193,7 +195,6 @@ GitMetric4User >> commentsContributionsSince: since until: until overA: aDateWee yourself ] ]. groupedByDate := groupedByDate collect: [ :contribs | - contribs ifNotEmpty: [ contribs sum ] ifEmpty: [ 0 ] ]. @@ -426,18 +427,26 @@ GitMetric4User >> loadCommitOfProjects: aCollection since: aTimespan [ { #category : #churn } GitMetric4User >> loadCommitOfProjects: aCollection since: since until: until [ - | commits | - commits := (aCollection collect: [ :id | - glhImporter - importCommitsOProject: (itsProjects at: id) - since: since - until: until ]) flattened first. - - commits do: [ :commit | glhImporter completeImportedCommit: commit ]. - - glhImporter chainsCommitsFrom: commits. - - ^ commits + | allCommits | + allCommits := itsCommits + at: since printString , '-' , until printString + ifPresent: [ :c | c ] + ifAbsentPut: [ + | commits | + commits := (aCollection collect: [ :id | + glhImporter + importCommitsOProject: + (itsProjects at: id) + since: since + until: until ]) flattened. + commits := commits flatCollect: [ :c | c ]. + commits ]. + + allCommits do: [ :commit | glhImporter completeImportedCommit: commit ]. + + glhImporter chainsCommitsFrom: allCommits. + + ^ allCommits ] { #category : #loading } diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index 7baa610..3209956 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -75,18 +75,29 @@ GLHModelImporter >> completeImportProject: aGLHProject [ { #category : #'as yet unclassified' } GLHModelImporter >> completeImportedCommit: aCommit [ - | diffs addedCommit | - addedCommit := (self importUserByUsername: aCommit author_name) + | addedCommit | + + addedCommit := aCommit. + + addedCommit commitCreator ifNil: [ + addedCommit := (self importUserByUsername: aCommit author_name) commits add: aCommit unless: [ :existingCommit :newCommit | existingCommit id = newCommit id ]. + ]. + ('Import diff of commit: ' , addedCommit short_id printString) recordInfo. self withCommitDiffs ifTrue: [ - diffs := self importDiffOfCommit: addedCommit. - self glhModel addAll: diffs unless: self blockForDiffEquality ]. + |diffs| + addedCommit diffs ifEmpty: [ + diffs := self importDiffOfCommit: addedCommit. + self glhModel addAll: diffs unless: self blockForDiffEquality + ] + + ]. ^ addedCommit ] @@ -249,12 +260,11 @@ GLHModelImporter >> importCommitsOProject: aProject since: fromDate until: toDat page: page. newlyFoundCommit := self parseCommitsResult: results. - newlyFoundCommit do: [ :c | c repository: aProject repository ]. + "newlyFoundCommit do: [ :c | c repository: aProject repository ]." - aProject repository commits + foundCommit addAll: (aProject repository commits addAll: newlyFoundCommit - unless: self blockOnIdEquality. - foundCommit addAll: newlyFoundCommit ]. + unless: self blockOnIdEquality). ]. ^ self glhModel addAll: foundCommit unless: self blockOnIdEquality @@ -384,9 +394,8 @@ GLHModelImporter >> importDiffOfCommit: aCommit [ unidiff: true. diffsResult := self parseDiffResult: result. - aCommit diffs addAll: diffsResult unless: self blockForDiffEquality. + ^ aCommit diffs addAll: diffsResult unless: self blockForDiffEquality. - ^ aCommit diffs ] { #category : #'private - api' } @@ -524,41 +533,43 @@ GLHModelImporter >> importRepository: aGLHRepository [ GLHModelImporter >> importUser: aUserID [ | result userResult | - ^ (glhModel allWithType: GLHUser) - detect: [ :user | user id = aUserID ] - ifNone: [ - ('Import user: ' , aUserID printString) recordInfo. - result := self glhApi user: aUserID. - userResult := self parseUserResult: result. - glhModel add: userResult unless: self blockOnIdEquality. - ] + + result := self glhApi user: aUserID. + userResult := self parseUserResult: result. + ^ glhModel add: userResult unless: self blockOnIdEquality . ] { #category : #user } GLHModelImporter >> importUserByUsername: anUsername [ - - ^ (self glhModel allWithType: GLHUser) - detect: [ :user | user username = anUsername ] - ifNone: [ + |dicUsername| + + dicUsername := ((self glhModel allWithType: GLHUser) collect: [ :user | + user username -> user. + ]) asSet asDictionary. + + ^ dicUsername at: anUsername + ifPresent: [ :user | user ] + ifAbsent: [ | result userId searchResult | ('Import user with username: ' , anUsername printString) recordInfo. result := self glhApi usersSearchByUsername: anUsername. searchResult := NeoJSONReader fromString: result. - - (searchResult class = Dictionary and: [((searchResult at: #message) includesSubstring: '403 Forbidden')]) ifTrue: [ - GLHUser new username: anUsername. - ] ifFalse: [ - searchResult ifEmpty: [ GLHUser new ] ifNotEmpty: [ - userId := searchResult first at: #id. - (self glhModel allWithType: GLHUser) - detect: [ :user | user id = userId ] - ifNone: [ - self importUser: userId. - ] ] - ]. - - ] + + (searchResult class = Dictionary and: [ + (searchResult at: #message) includesSubstring: '403 Forbidden' ]) + ifTrue: [ GLHUser new username: anUsername ] + ifFalse: [ + searchResult + ifEmpty: [ GLHUser new username: anUsername; yourself ] + ifNotEmpty: [ + + userId := searchResult first at: #id. + (self glhModel allWithType: GLHUser) + detect: [ :user | user id = userId ] + ifNone: [ self importUser: userId ] . + + ] ] ] ] { #category : #initialization } diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st index 1f285a9..0bd383e 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHApi.class.st @@ -157,6 +157,10 @@ GLPHApi >> project: aProjectID [ { #category : #user } GLPHApi >> usersSearchByUsername: aUserName [ - - ^ self client get: self baseAPIUrl , '/users?search=' , (aUserName replaceAllRegex: ' ' with: '+'). + |name| + + name := (aUserName splitOn: ' ') collect: [:part | part urlEncoded]. + name := '+' join: name. + + ^ self client get: self baseAPIUrl , '/users?search=' , name ] diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 80a5854..9415b53 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -16,19 +16,31 @@ GLPHModelImporter >> blockForDiffRangeEquality [ { #category : #commit } GLPHModelImporter >> chainsCommitsFrom: commitsCollection [ + |dic| + + dic := ((self glhModel allWithType: GLHCommit) collect: [ :commit | + (commit id) -> commit. + ]) asSet asDictionary . + commitsCollection do: [ :commit | commit parent_ids do: [ :parentId | - | parentCommit | - parentCommit := (self glhModel allWithType: GLHCommit) + " | parentCommit |" + +" parentCommit := (self glhModel allWithType: GLHCommit) detect: [ :c | c id = parentId ] ifNone: [ nil ]. parentCommit ifNotNil: [ parentCommit childCommits add: commit unless: [ :existing :new | existing id = new id ]. - ] ] ]. + ] " + + dic at: parentId ifPresent: [:parentCommit | parentCommit childCommits add: commit unless: self blockOnIdEquality] ifAbsent: []. + + ] + ]. ^ commitsCollection ] @@ -176,9 +188,8 @@ GLPHModelImporter >> importDiffRangesForDiff: aGLHDiff [ | diffRanges | ('Import changes of diff ' , aGLHDiff printString) recordInfo. "the import of 'diffRanges' entites is technically done inside the parseDiffString" - diffRanges := self parseDiffString: aGLHDiff. - self glhModel addAll: diffRanges unless: self blockForDiffRangeEquality . + diffRanges := self glhModel addAll: (self parseDiffString: aGLHDiff) unless: self blockForDiffRangeEquality . ^ aGLHDiff diffRanges addAll: diffRanges unless: self blockForDiffRangeEquality . ] @@ -307,7 +318,7 @@ GLPHModelImporter >> parseDiffString: aDiff [ (line trim beginsWith: #'@@') ifTrue: [ range := GLPHEDiffRange newFrom: line. firstChangeRange := true. - aDiff diffRanges + range := aDiff diffRanges add: range unless: [ :existing :new | existing originalLineRange = new originalLineRange and: [ @@ -318,7 +329,7 @@ GLPHModelImporter >> parseDiffString: aDiff [ firstChangeRange ifFalse: [ aChange := GLPHEChange newFrom: line. aChange relativeIndex: index. - range changes add: aChange unless: [ :existing :new | + aChange := range changes add: aChange unless: [ :existing :new | existing sourceCode = new sourceCode and: [ existing index = new index ] ]. index := index + 1 From 9accddf052210ee8c4cafa75561cf5fa0ac87408 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Wed, 5 Jun 2024 11:33:08 +0200 Subject: [PATCH 53/77] add some logs and fix some bugs --- .../GitMetric4User.class.st | 6 +- .../GLHModelImporter.class.st | 47 +++++--- .../GLPHModelImporter.class.st | 104 +++++++++--------- 3 files changed, 90 insertions(+), 67 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index 4acfd81..ff01d73 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -450,10 +450,10 @@ GitMetric4User >> loadCommitOfProjects: aCollection since: since until: until [ ] { #category : #loading } -GitMetric4User >> loadProjects: aCollection [ +GitMetric4User >> loadProjects: projectIds [ - aCollection do: [ :id | - itsProjects at: id ifAbsentPut: (glhImporter importProject: id) ]. + projectIds do: [ :id | + itsProjects at: id ifAbsentPut: [ glhImporter importProject: id ] ]. ^ itsProjects ] diff --git a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st index 3209956..0f2d2a5 100644 --- a/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GLHModelImporter.class.st @@ -57,14 +57,15 @@ GLHModelImporter >> blockOnNameEquality [ { #category : #'private - api' } GLHModelImporter >> completeImportProject: aGLHProject [ - ('Complete import project: ' , aGLHProject id printString) recordInfo. - (self pipelinesOf: aGLHProject id) do: [ :pipeline | - self glhModel add: pipeline. - aGLHProject addPipeline: pipeline ]. + ('Complete import of project: ' , aGLHProject id printString) + recordInfo. + + self importPipelinesOfProject: aGLHProject. "aGLHProject creator: (self importUser: aGLHProject creator_id)." - ((self importUser: aGLHProject creator_id)) addCreatedProject: aGLHProject. + (self importUser: aGLHProject creator_id) addCreatedProject: + aGLHProject. aGLHProject repository: GLHRepository new. self glhModel add: aGLHProject unless: (self blockEqualityOn: #id). self glhModel add: aGLHProject repository. @@ -165,6 +166,9 @@ GLHModelImporter >> importCommitOfProject: anProject withId: anID [ | commit result | anID ifNil: [ ^ nil ]. + ('looking for commit ' , anID printString , ' in project : ' + , anProject id printString) recordInfo. + commit := (self detectEntityType: GLHCommit overAttribut: #id @@ -178,8 +182,7 @@ GLHModelImporter >> importCommitOfProject: anProject withId: anID [ commit ]. - self withCommitDiffs ifTrue: [ - self importDiffOfCommit: commit ]. + self withCommitDiffs ifTrue: [ self importDiffOfCommit: commit ]. ^ commit ] @@ -505,23 +508,37 @@ GLHModelImporter >> importParentCommitsOfCommit: aGLHCommit since: aDate [ flatten ] +{ #category : #'private - api' } +GLHModelImporter >> importPipelinesOfProject: aGLHProject [ + + (self pipelinesOf: aGLHProject id) do: [ :pipeline | + self glhModel add: pipeline. + aGLHProject addPipeline: pipeline ] +] + { #category : #'private - api' } GLHModelImporter >> importRepository: aGLHRepository [ | resultBranches branches | [ + ('import the repository of project ' , aGLHRepository project name) + recordInfo. + resultBranches := self glhApi branchesOfRepository: aGLHRepository project id. branches := self parseBranchesResult: resultBranches. - - aGLHRepository branches addAll: branches unless: self blockOnNameEquality. + ('import the branches of project ') recordInfo. + + aGLHRepository branches + addAll: branches + unless: self blockOnNameEquality. self glhModel addAll: branches unless: self blockOnNameEquality. - + "branches do: [ :branch | aGLHRepository addBranch: branch. self glhModel add: branch ]." - + self withFiles ifTrue: [ branches do: [ :branch | self importFilesOfBranch: branch ] ] ] on: NeoJSONParseError @@ -533,10 +550,10 @@ GLHModelImporter >> importRepository: aGLHRepository [ GLHModelImporter >> importUser: aUserID [ | result userResult | - - result := self glhApi user: aUserID. - userResult := self parseUserResult: result. - ^ glhModel add: userResult unless: self blockOnIdEquality . + ('Import user with id: ', aUserID printString) recordInfo. + result := self glhApi user: aUserID. + userResult := self parseUserResult: result. + ^ glhModel add: userResult unless: self blockOnIdEquality ] { #category : #user } diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index 9415b53..c53337a 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -16,31 +16,22 @@ GLPHModelImporter >> blockForDiffRangeEquality [ { #category : #commit } GLPHModelImporter >> chainsCommitsFrom: commitsCollection [ - |dic| + | dic | + + ('Chains ', commitsCollection size printString , ' commits') recordInfo. dic := ((self glhModel allWithType: GLHCommit) collect: [ :commit | - (commit id) -> commit. - ]) asSet asDictionary . + commit id -> commit ]) asSet asDictionary. commitsCollection do: [ :commit | - - - commit parent_ids do: [ :parentId | - " | parentCommit |" - -" parentCommit := (self glhModel allWithType: GLHCommit) - detect: [ :c | c id = parentId ] - ifNone: [ nil ]. - parentCommit ifNotNil: [ - parentCommit childCommits - add: commit - unless: [ :existing :new | existing id = new id ]. - ] " - - dic at: parentId ifPresent: [:parentCommit | parentCommit childCommits add: commit unless: self blockOnIdEquality] ifAbsent: []. - - ] - ]. + commit parent_ids do: [ :parentId | + dic + at: parentId + ifPresent: [ :parentCommit | + parentCommit childCommits + add: commit + unless: self blockOnIdEquality ] + ifAbsent: [ ] ] ]. ^ commitsCollection ] @@ -89,11 +80,13 @@ GLPHModelImporter >> completeImportProject: aGLHProject [ super completeImportProject: aGLHProject. - self importMergeRequests: aGLHProject. - - ^ aGLHProject - + "for this initial import, with only look at some but not all MR" + self + importMergeRequests: aGLHProject + since: DateAndTime today + until: DateAndTime now. + ^ aGLHProject ] { #category : #import } @@ -122,24 +115,31 @@ GLPHModelImporter >> importCommitsOfMergeResquest: aGLPHEMergeRequest [ | foundCommits | foundCommits := OrderedCollection new. + ('Import commit sha of MR: ' , aGLPHEMergeRequest iid printString) + recordInfo. "the founds commits are added to the model during their respective import" aGLPHEMergeRequest mergeRequestCommit: ((self importCommitOfProject: aGLPHEMergeRequest project withId: aGLPHEMergeRequest sha) ifNotNil: [ :commit | foundCommits add: commit ]). + ('Import commit merge_commit_sha of MR: ' + , aGLPHEMergeRequest iid printString) recordInfo. aGLPHEMergeRequest mergedCommit: ((self importCommitOfProject: aGLPHEMergeRequest project withId: aGLPHEMergeRequest merge_commit_sha) ifNotNil: [ :commit | foundCommits add: commit ]). + ('Import commit squash_commit_sha of MR: ' + , aGLPHEMergeRequest iid printString) recordInfo. aGLPHEMergeRequest squashCommit: ((self importCommitOfProject: aGLPHEMergeRequest project withId: aGLPHEMergeRequest squash_commit_sha) ifNotNil: [ :commit | foundCommits add: commit ]). - - self chainsCommitsFrom: foundCommits + + self chainsCommitsFrom: foundCommits. + ^ foundCommits ] { #category : #api } @@ -161,7 +161,7 @@ GLPHModelImporter >> importDiffOfMergeRequest: aMergeRequest [ 'Diff of already importer: ' , aMergeRequest iid printString recordInfo. ^ aMergeRequest diffs ]. - ('Import diff of mergeRequest ' , aMergeRequest iid printString) + ('Import diff commits of MR ' , aMergeRequest iid printString) recordInfo. result := self glhApi diffsMergeRequestOfProject: aMergeRequest project_id @@ -186,25 +186,36 @@ GLPHModelImporter >> importDiffOfMergeRequest: aMergeRequest [ GLPHModelImporter >> importDiffRangesForDiff: aGLHDiff [ | diffRanges | + aGLHDiff diffRanges ifNotEmpty: [ + ('Diffs already imported for ' , aGLHDiff printString) recordInfo. + ^ aGLHDiff diffRanges ]. + ('Import changes of diff ' , aGLHDiff printString) recordInfo. "the import of 'diffRanges' entites is technically done inside the parseDiffString" - - diffRanges := self glhModel addAll: (self parseDiffString: aGLHDiff) unless: self blockForDiffRangeEquality . - ^ aGLHDiff diffRanges addAll: diffRanges unless: self blockForDiffRangeEquality . + + diffRanges := self glhModel + addAll: (self parseDiffString: aGLHDiff) + unless: self blockForDiffRangeEquality. + ^ aGLHDiff diffRanges + addAll: diffRanges + unless: self blockForDiffRangeEquality ] { #category : #import } GLPHModelImporter >> importMergeRequests: aGLHProject [ | results parsedResults | + + ('Import merge request of Project: ' , aGLHProject id printString) recordInfo. + results := self glhApi mergeRequestOfProject: aGLHProject id. parsedResults := self parseMergeRequestResult: results. aGLHProject mergeRequests addAll: parsedResults unless: self blockOnIdEquality. - + self glhModel addAll: aGLHProject mergeRequests unless: self blockOnIdEquality. @@ -224,13 +235,15 @@ GLPHModelImporter >> importMergeRequests: aGLHProject [ GLPHModelImporter >> importMergeRequests: aGLHProject since: fromDate until: toDate [ | newlyFoundMR page foundMR | + ('import MR of Project ' , aGLHProject id) recordInfo. + page := 0. foundMR := OrderedCollection new. newlyFoundMR := { true }. [ newlyFoundMR isNotEmpty ] whileTrue: [ | results | page := page + 1. - ('import merge request page ' , page printString) recordInfo. + ('import MR page ' , page printString) recordInfo. results := self glhApi mergeRequestsOfProject: aGLHProject id createdAfter: @@ -243,18 +256,10 @@ GLPHModelImporter >> importMergeRequests: aGLHProject since: fromDate until: toD orderBy: 'created_at' page: page. - newlyFoundMR := self parseMergeRequestResult: results. - "newlyFoundMR do: [ :mr | - ""1 halt. mr project: aGLHProject"". - aGLHProject mergeRequests add: mr unless: self blockOnIdEquality. - ]." - "TODO : pourquoi le merge resquest est rajouté dans le model ??? " - "newlyFoundMR = (self glhModel allWithType: GLPHEMergeRequest) first . - newlyFoundMR = aGLHProject mergeRequests first." - + newlyFoundMR := aGLHProject mergeRequests - addAll: newlyFoundMR - unless: self blockOnIdEquality. + addAll: (self parseMergeRequestResult: results) + unless: self blockOnIdEquality. foundMR addAll: newlyFoundMR ]. @@ -266,18 +271,19 @@ GLPHModelImporter >> importMergeRequests: aGLHProject since: fromDate until: toD self withCommitDiffs ifTrue: [ aGLHProject mergeRequests do: [ :mr | self importDiffOfMergeRequest: mr ] ]. - - ^ foundMR + + ^ foundMR ] { #category : #'as yet unclassified' } GLPHModelImporter >> importProject: aProjectID [ - | result projectResult | - ('Import group: ' , aProjectID printString) recordInfo. + + | result projectResult | + ('Import project with id: ' , aProjectID printString) recordInfo. result := self glhApi project: aProjectID. projectResult := self parseProjectResult: result. - + ^ self completeImportProject: projectResult ] From f2c096976d9fbba25df6bfac3d01a30d66f03320 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Wed, 5 Jun 2024 11:43:59 +0200 Subject: [PATCH 54/77] bug fix --- .../GitMetric4User.class.st | 28 ++++++++++--------- .../GLPHModelImporter.class.st | 2 +- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index ff01d73..a9374b1 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -56,7 +56,7 @@ GitMetric4User >> codeChurnSince: since until: until onACommitWindowOf: commitLi contribution := churnResult at: #totalContribution. totalContributions add: contribution. - fraction := sumChurn / contribution. + fraction := sumChurn / ((contribution = 0) ifTrue: [1] ifFalse: [contribution ]) . overDate := self transformDate: commit created_at to: aDateWeekMonthOrYear. @@ -347,7 +347,7 @@ GitMetric4User >> delayUntilFirstChurnSince: since until: until overA: aDateWeek | durationFromA2B | commits4Date collect: [ :commitA | | commitB | - commitB := gitAnalyzer + commitB := GitAnalyzer new glhImporter: glhImporter; onModel: glhModel; fromCommit: commitA; @@ -479,7 +479,13 @@ GitMetric4User >> mergeRequestDurationSince: since until: until overA: aDateWeek mergeRequest := mergeRequest values flattened. mergeRequest ifEmpty: [ - ^ { (#totalMergeRequest -> mergeRequest size) } asDictionary ]. + ^ { + (#avgDuration -> nil). + (#overEach -> aDateWeekMonthOrYear name). + (#forOver -> (groupedByDate keys size printString + , aDateWeekMonthOrYear printString)). + (#totalMergeRequest -> 0). + (#details -> nil) } asDictionary ]. res := mergeRequest collect: [ :mr | gitAnalyzer analyseMergeResquestValidation: mr ]. @@ -502,11 +508,8 @@ GitMetric4User >> mergeRequestDurationSince: since until: until overA: aDateWeek filterGroups := groupedByDate reject: [ :array | array isEmpty ]. - filterGroups associations do: [ - :assoc | + filterGroups associations do: [ :assoc | | sum denominator | - - denominator := assoc value size. sum := assoc value sum: [ :v | @@ -515,14 +518,13 @@ GitMetric4User >> mergeRequestDurationSince: since until: until overA: aDateWeek 0 asDuration ] ]. denominator = 0 ifTrue: [ denominator := 1 ]. - filterGroups at: assoc key put: sum / denominator ]. + filterGroups at: assoc key put: sum / denominator ]. - filterGroups keys ifEmpty: [ avg := 0 ] ifNotEmpty: [ - avg := (filterGroups values sum ) - / filterGroups keys size. - - ]. + filterGroups keys + ifEmpty: [ avg := 0 ] + ifNotEmpty: [ + avg := filterGroups values sum / filterGroups keys size ]. ^ { diff --git a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st index c53337a..1114333 100644 --- a/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st +++ b/src/GitLabProjectHealth-Model-Importer/GLPHModelImporter.class.st @@ -235,7 +235,7 @@ GLPHModelImporter >> importMergeRequests: aGLHProject [ GLPHModelImporter >> importMergeRequests: aGLHProject since: fromDate until: toDate [ | newlyFoundMR page foundMR | - ('import MR of Project ' , aGLHProject id) recordInfo. + ('import MR of Project ' , aGLHProject id printString) recordInfo. page := 0. foundMR := OrderedCollection new. From db08a63cdcad56d226d5a0413da96d4217177675 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Wed, 5 Jun 2024 11:54:34 +0200 Subject: [PATCH 55/77] fix code churn parsing bug --- src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st index 9aabada..fd282b1 100644 --- a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st @@ -175,7 +175,7 @@ GitAnalyzer >> computeSpecificChurnOf: commit2Changes [ diffRanges do: [ :diff | |from upTo| - from := (diff originalLineRange copyFrom: (diff originalLineRange indexOf: $-)+1 to: (diff originalLineRange indexOf: $,)-1 ) asString asNumber. + from := (diff originalLineRange copyFrom: (diff originalLineRange indexOf: $-)+1 to: (diff originalLineRange indexOf: $, ifAbsent: [ diff originalLineRange size + 1 ] )-1 ) asString asNumber. (from = 0) ifTrue:[from := 1]. self insertDiff: diff into: changesDic startingFrom: from. From bbf98a333e1beeec16463890922468350427362f Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Wed, 5 Jun 2024 13:14:23 +0200 Subject: [PATCH 56/77] fix delay churn --- src/GitLabHealth-Model-Analysis/GitMetric4User.class.st | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index a9374b1..6a185be 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -370,8 +370,9 @@ GitMetric4User >> delayUntilFirstChurnSince: since until: until overA: aDateWeek (filtered sum: [ :v | v asDuration ]) / filtered size ] ]. res := res reject: #isNil. + + res ifEmpty: [ avg:= nil ] ifNotEmpty: [ avg := res values sum / res keys size. ]. - avg := res values sum / res keys size. ^ { (#avgDelay -> avg). From a60f337732ec2a8e1c5e0c0a4fd42c0bd8766b34 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 6 Jun 2024 08:50:33 +0200 Subject: [PATCH 57/77] opti on MR --- .../GitMetric4User.class.st | 54 ++++++++++++++----- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index 6a185be..b8f6a08 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -8,7 +8,8 @@ Class { 'itsCommits', 'gitAnalyzer', 'glhModel', - 'glhImporter' + 'glhImporter', + 'itsMergeRequests' ], #category : #'GitLabHealth-Model-Analysis' } @@ -400,7 +401,8 @@ GitMetric4User >> initialize [ user := GLHUser new. itsProjects := Dictionary new. itsCommits := Dictionary new. - itsGroups := Dictionary new + itsGroups := Dictionary new. + itsMergeRequests := Dictionary new ] { #category : #accessing } @@ -450,6 +452,31 @@ GitMetric4User >> loadCommitOfProjects: aCollection since: since until: until [ ^ allCommits ] +{ #category : #churn } +GitMetric4User >> loadMergeRequestsOfProjects: aCollection since: since until: until [ + + | allMr | + itsMergeRequests ifNil: [ itsMergeRequests := Dictionary new ]. + + + + allMr := itsMergeRequests + at: since printString , '-' , until printString + ifPresent: [ :c | c ] + ifAbsentPut: [ + | mr | + mr := (aCollection collect: [ :id | + glhImporter + importMergeRequests: (itsProjects at: id) + since: since + until: until ]) flattened. + mr := mr flatCollect: [ :c | c ]. + mr ]. + + + ^ allMr +] + { #category : #loading } GitMetric4User >> loadProjects: projectIds [ @@ -472,21 +499,20 @@ GitMetric4User >> mergeRequestDurationSince: since until: until overA: aDateWeek glhImporter: glhImporter; onModel: glhModel. - mergeRequest := itsProjects collect: [ :project | - glhImporter - importMergeRequests: project - since: since - until: until ]. - mergeRequest := mergeRequest values flattened. + mergeRequest := self + loadMergeRequestsOfProjects: itsProjects keys + since: since + until: until. + mergeRequest ifEmpty: [ ^ { - (#avgDuration -> nil). - (#overEach -> aDateWeekMonthOrYear name). - (#forOver -> (groupedByDate keys size printString - , aDateWeekMonthOrYear printString)). - (#totalMergeRequest -> 0). - (#details -> nil) } asDictionary ]. + (#avgDuration -> nil). + (#overEach -> aDateWeekMonthOrYear name). + (#forOver -> (groupedByDate keys size printString + , aDateWeekMonthOrYear printString)). + (#totalMergeRequest -> 0). + (#details -> nil) } asDictionary ]. res := mergeRequest collect: [ :mr | gitAnalyzer analyseMergeResquestValidation: mr ]. From d6c270f141f215c0d934f717ed5e52ee04beaf58 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 6 Jun 2024 10:44:14 +0200 Subject: [PATCH 58/77] fix churn --- src/GitLabHealth-Model-Analysis/GitMetric4User.class.st | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index b8f6a08..48babde 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -78,14 +78,16 @@ GitMetric4User >> codeChurnSince: since until: until onACommitWindowOf: commitLi totalContribution = 0 ifTrue: [ totalChurn / 1] ifFalse: [ totalChurn / totalContribution ] ]. - +1 halt. ^ { (#details -> groupedByDate). (#totalContribution -> (totalContributions ifEmpty: [ 0 ] ifNotEmpty: [ totalContributions sum ]) ). (#totalChurn -> (groupedByDate sum: [ :frac | frac numerator ])). - (#avgChurn -> ((groupedByDate sum: [ :frac | frac numerator ]) - / (groupedByDate keys ifEmpty: [ 1 ] ifNotEmpty: [ groupedByDate keys size ]) )). + "(#avgChurn -> ((groupedByDate sum: [ :frac | frac numerator ]) + / (groupedByDate keys ifEmpty: [ 1 ] ifNotEmpty: [ groupedByDate keys size ]) ))." + (#churn -> ((groupedByDate sum: [ :frac | frac numerator ]) -> + (groupedByDate sum: [ :frac | frac denominator ]))). (#overEach -> aDateWeekMonthOrYear name). (#forOver -> (groupedByDate keys size printString , aDateWeekMonthOrYear printString)). From da252db3cb5016cf951aa929e2c51bb3a90f00e9 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 6 Jun 2024 10:56:11 +0200 Subject: [PATCH 59/77] remove halt and fix mr load on metrics --- src/GitLabHealth-Model-Analysis/GitMetric4User.class.st | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index 48babde..8b2be67 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -78,7 +78,7 @@ GitMetric4User >> codeChurnSince: since until: until onACommitWindowOf: commitLi totalContribution = 0 ifTrue: [ totalChurn / 1] ifFalse: [ totalChurn / totalContribution ] ]. -1 halt. + ^ { (#details -> groupedByDate). @@ -472,7 +472,7 @@ GitMetric4User >> loadMergeRequestsOfProjects: aCollection since: since until: u importMergeRequests: (itsProjects at: id) since: since until: until ]) flattened. - mr := mr flatCollect: [ :c | c ]. + " mr := mr flatCollect: [ :c | c ]." mr ]. From b0a3117008014e858c0a415b54d05518a3423453 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 6 Jun 2024 12:12:43 +0200 Subject: [PATCH 60/77] add comment for python --- .../GitAnalyzer.class.st | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st index fd282b1..3b0af07 100644 --- a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st @@ -50,17 +50,21 @@ GitAnalyzer >> analyseChurn [ { #category : #'as yet unclassified' } GitAnalyzer >> analyseCommentContributions [ - |additions comments| - - additions := (fromCommit diffs collect: [:diff | - diff diffRanges collect: [ :range | - range changes select: [ :change | - change class = GLPHEAddition. ] ] ]) flattened collect: [ :add | add sourceCode] . - - additions := additions collect: [ :codeSource | (codeSource withoutPrefix: '+') trim ]. - additions := additions select: [ :codeSource | codeSource beginsWithAnyOf: { '//' . '/*' . '*' . '*/' } ]. - - ^ additions size + + | additions | + additions := (fromCommit diffs collect: [ :diff | + diff diffRanges collect: [ :range | + range changes select: [ :change | + change class = GLPHEAddition ] ] ]) flattened + collect: [ :add | add sourceCode ]. + + additions := additions collect: [ :codeSource | + (codeSource withoutPrefix: '+') trim ]. + additions := additions select: [ :codeSource | + codeSource beginsWithAnyOf: + { '#'. '//'. '/*'. '*'. '*/' } ]. + + ^ additions size ] { #category : #commit } From 67239e598d6914c4a866d8371be534c98df2f8f5 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 6 Jun 2024 13:58:58 +0200 Subject: [PATCH 61/77] add export class --- .../GitMetricExporter.class.st | 211 ++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st diff --git a/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st b/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st new file mode 100644 index 0000000..c283b9f --- /dev/null +++ b/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st @@ -0,0 +1,211 @@ +Class { + #name : #GitMetricExporter, + #superclass : #Object, + #instVars : [ + 'glhImporter', + 'beforeExp', + 'duringExp' + ], + #category : #'GitLabHealth-Model-Analysis' +} + +{ #category : #'as yet unclassified' } +GitMetricExporter >> beforeDic: aDictionnarySinceUntil [ + + beforeExp := aDictionnarySinceUntil +] + +{ #category : #'as yet unclassified' } +GitMetricExporter >> duringDic: aDictionnarySinceUntil [ + duringExp := aDictionnarySinceUntil +] + +{ #category : #'as yet unclassified' } +GitMetricExporter >> exportFor: usersWithProjects [ + + self exportFor: usersWithProjects over: { Date. Week . Month . Year }. +] + +{ #category : #'as yet unclassified' } +GitMetricExporter >> exportFor: usersWithProjects over: aCollectionOfDateWeekMonthOrYear [ + + | over entities file exportBrowserModel | + over := Date. + + + entities := usersWithProjects associations collect: [ :assoc | + | username projects metrics | + username := assoc key. + projects := assoc value. + + metrics := GitMetric4User new. + metrics + glhImporter: glhImporter; + findUserNamed: username. + metrics loadProjects: projects. + + metrics ]. + + + + exportBrowserModel := MiExportModel new. + exportBrowserModel entitiesList: entities. + exportBrowserModel removeColumnForQueryNamed: #Type. + exportBrowserModel removeColumnForQueryNamed: #Name. + + exportBrowserModel + addColumnForQuery: [ :metrics | metrics user name printString ] + withName: #'User name'. + + + "Code Contribution " + exportBrowserModel + addColumnForQuery: [ :metrics | + ((metrics + codeContributionsSince: (beforeExp at: #since) + until: (beforeExp at: #until) + overA: over) at: #avgAddition) printString ] + withName: #'(B) code additions (avg)'. + exportBrowserModel + addColumnForQuery: [ :metrics | + ((metrics + codeContributionsSince: (beforeExp at: #since) + until: (beforeExp at: #until) + overA: over) at: #avgDeletion) printString ] + withName: #'(B) code deletion (avg)'. + + exportBrowserModel + addColumnForQuery: [ :metrics | + ((metrics + codeContributionsSince: (duringExp at: #since) + until: (duringExp at: #until) + overA: over) at: #avgAddition) printString ] + withName: #'(D) code additions (avg)'. + exportBrowserModel + addColumnForQuery: [ :metrics | + ((metrics + codeContributionsSince: (duringExp at: #since) + until: (duringExp at: #until) + overA: over) at: #avgDeletion) printString ] + withName: #'(D) code deletion (avg)'. + + + "Commit frequencies " + exportBrowserModel + addColumnForQuery: [ :metrics | + ((metrics + commitFrequencySince: (beforeExp at: #since) + until: (beforeExp at: #until) + overA: over) at: #averageFloat) printString ] + withName: #'(B) commits frequency (avg)'. + + exportBrowserModel + addColumnForQuery: [ :metrics | + ((metrics + commitFrequencySince: (duringExp at: #since) + until: (duringExp at: #until) + overA: over) at: #averageFloat) printString ] + withName: #'(D) commits frequency (avg)'. + + + "comment contribution " + exportBrowserModel + addColumnForQuery: [ :metrics | + ((metrics + commentsContributionsSince: (beforeExp at: #since) + until: (beforeExp at: #until) + overA: over) at: #avgComments) printString ] + withName: #'(B) comment contribution (avg)'. + + exportBrowserModel + addColumnForQuery: [ :metrics | + ((metrics + commentsContributionsSince: (duringExp at: #since) + until: (duringExp at: #until) + overA: over) at: #avgComments) printString ] + withName: #'(D) comment contribution (avg)'. + + + "merge Request Duration " + exportBrowserModel + addColumnForQuery: [ :metrics | + ((metrics + mergeRequestDurationSince: (beforeExp at: #since) + until: (beforeExp at: #until) + overA: over) at: #avgDuration) printString ] + withName: #'(B) merge Request Duration'. + + exportBrowserModel + addColumnForQuery: [ :metrics | + ((metrics + mergeRequestDurationSince: (duringExp at: #since) + until: (duringExp at: #until) + overA: over) at: #avgDuration) printString ] + withName: #'(D) merge Request Duration'. + + + "code churn" + exportBrowserModel + addColumnForQuery: [ :metrics | + ((metrics + codeChurnSince: (beforeExp at: #since) + until: (beforeExp at: #until) + onACommitWindowOf: 3 + overA: over) at: #churn) printString ] + withName: #'(B) code churn (window=3)'. + + exportBrowserModel + addColumnForQuery: [ :metrics | + ((metrics + codeChurnSince: (duringExp at: #since) + until: (duringExp at: #until) + onACommitWindowOf: 3 + overA: over) at: #churn) printString ] + withName: #'(D) code churn (window=3)'. + + "delay Until First Churn" + exportBrowserModel + addColumnForQuery: [ :metrics | + ((metrics + delayUntilFirstChurnSince: (beforeExp at: #since) + until: (beforeExp at: #until) + overA: over) at: #avgDelay) printString ] + withName: #'(B) delay Until First Churn'. + + exportBrowserModel + addColumnForQuery: [ :metrics | + ((metrics + delayUntilFirstChurnSince: (duringExp at: #since) + until: (duringExp at: #until) + overA: over) at: #avgDelay) printString ] + withName: #'(D) delay until first churn'. + + + + + aCollectionOfDateWeekMonthOrYear do: [ :period | + over := period. + + file := (FileLocator home + / ('IA4Code-DRIT-' , over printString , '-' + , (DateAndTime now printString replaceAll: $: with: $-) + , '.csv')) asFileReference. + + file writeStreamDo: [ :aStream | + aStream + << 'sep=,'; + << OSPlatform current lineEnding. + exportBrowserModel writeCSVOn: aStream. + + aStream << OSPlatform current lineEnding. + + aStream << ('(B): ' , beforeExp printString) + << OSPlatform current lineEnding. + aStream << ('(D): ' , duringExp printString) + << OSPlatform current lineEnding ] ] +] + +{ #category : #accessing } +GitMetricExporter >> glhImporter: anImporter [ + glhImporter := anImporter. +] From 9d0b68275694e9ec06bc0e11ba471c9c7573212c Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Mon, 10 Jun 2024 10:47:05 +0200 Subject: [PATCH 62/77] fix churn amendment --- src/GitLabHealth-Model-Analysis/GitMetric4User.class.st | 1 + 1 file changed, 1 insertion(+) diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index 8b2be67..e37ac13 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -354,6 +354,7 @@ GitMetric4User >> delayUntilFirstChurnSince: since until: until overA: aDateWeek glhImporter: glhImporter; onModel: glhModel; fromCommit: commitA; + maxChildCommit: 5; analyseAmandment. durationFromA2B := commitB From b7427b9a7a5885023e798b1b556983ca118ebb7b Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Mon, 10 Jun 2024 10:57:22 +0200 Subject: [PATCH 63/77] add logger --- .../GitAnalyzer.class.st | 24 +++++++++++++++++++ .../GitMetric4User.class.st | 16 +++++++++++++ 2 files changed, 40 insertions(+) diff --git a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st index 3b0af07..d66d9a7 100644 --- a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st @@ -16,6 +16,10 @@ GitAnalyzer >> analyseAmandment [ "return the first commit that modify the same lines of code as the fromCommit" | churn res | + + ('GitAnalyzer, analyse amandment onProject: ', onProject printString ) + recordInfo. + churn := self analyseChurn. res := self firstAmandmentFromChrun: churn. @@ -27,6 +31,10 @@ GitAnalyzer >> analyseAmandment [ GitAnalyzer >> analyseChurn [ | commitFiles totalContribution childCommits | + + ('GitAnalyzer, analyse chrun onProject: ', onProject printString ) + recordInfo. + childCommits := OrderedCollection new. totalContribution := self @@ -52,6 +60,10 @@ GitAnalyzer >> analyseChurn [ GitAnalyzer >> analyseCommentContributions [ | additions | + + ('GitAnalyzer, analyse comment contributions onProject: ', onProject printString ) + recordInfo. + additions := (fromCommit diffs collect: [ :diff | diff diffRanges collect: [ :range | range changes select: [ :change | @@ -70,6 +82,10 @@ GitAnalyzer >> analyseCommentContributions [ { #category : #commit } GitAnalyzer >> analyseCommitContribution [ + + ('GitAnalyzer, analyse commit contribution of: ', fromCommit printString ) + recordInfo. + ^ { (#addition -> fromCommit additions). (#deletion -> fromCommit deletions). } asDictionary ] @@ -78,6 +94,10 @@ GitAnalyzer >> analyseCommitContribution [ GitAnalyzer >> analyseCommitFrequencySince: since until: until [ | commits response | + + ('GitAnalyzer, analyse commit Frequency on: ', onProject printString ) + recordInfo. + response := { (#numberOfCommit -> nil). (#frequency -> nil) } asDictionary. @@ -99,6 +119,10 @@ GitAnalyzer >> analyseMergeResquestValidation: aGLHPMergeRequest [ |creationDate mergedDate response| + + ('GitAnalyzer, analyse merge request delay of: ', aGLHPMergeRequest printString ) + recordInfo. + response := { #id_merge_resquest -> aGLHPMergeRequest iid. #id_merge_commit -> nil. diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index e37ac13..13852e5 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -18,6 +18,10 @@ Class { GitMetric4User >> codeChurnSince: since until: until onACommitWindowOf: commitLimit overA: aDateWeekMonthOrYear [ | commits res groupedByDate totalContributions | + + ('GitMetric4User: code churn') + recordInfo. + totalContributions := OrderedCollection new. groupedByDate := self setupGroupedDateFrom: since @@ -100,6 +104,8 @@ GitMetric4User >> codeContributionsSince: since until: until overA: aDateWeekMon | commits contributions groupedByDate | + ('GitMetric4User: code contribution') recordInfo. + glhImporter withCommitDiffs: false. groupedByDate := self @@ -163,6 +169,9 @@ GitMetric4User >> codeContributionsSince: since until: until overA: aDateWeekMon GitMetric4User >> commentsContributionsSince: since until: until overA: aDateWeekMonthOrYear [ | commits contributions groupedByDate | + + ('GitMetric4User: comment contribution') recordInfo. + groupedByDate := self setupGroupedDateFrom: since to: until @@ -218,6 +227,9 @@ GitMetric4User >> commentsContributionsSince: since until: until overA: aDateWee GitMetric4User >> commitFrequencySince: since until: until overA: aDateWeekMonthOrYear [ | all aggregate periods total groupedByDate | + + ('GitMetric4User: commit frequency') recordInfo. + groupedByDate := self setupGroupedDateFrom: since to: until @@ -261,6 +273,7 @@ GitMetric4User >> commitsProducedOnProject: projectId since: sinceDate until: un | frequencies | + gitAnalyzer := GitAnalyzer new glhImporter: glhImporter; onModel: glhModel. @@ -318,6 +331,9 @@ GitMetric4User >> commitsProducedOnProject: aProjectId since: since until: until GitMetric4User >> delayUntilFirstChurnSince: since until: until overA: aDateWeekMonthOrYear [ | commits groupedByDate res avg | + + ('GitMetric4User: delay until first churn') recordInfo. + groupedByDate := self setupGroupedDateFrom: since to: until From 1a7f7cc0902b87d54b60dfce210489c6126bc158 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Tue, 11 Jun 2024 09:31:55 +0200 Subject: [PATCH 64/77] fix some stuff --- .../GitMetricExporter.class.st | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st b/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st index c283b9f..3b3342b 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st @@ -4,7 +4,8 @@ Class { #instVars : [ 'glhImporter', 'beforeExp', - 'duringExp' + 'duringExp', + 'label' ], #category : #'GitLabHealth-Model-Analysis' } @@ -187,7 +188,9 @@ GitMetricExporter >> exportFor: usersWithProjects over: aCollectionOfDateWeekMon over := period. file := (FileLocator home - / ('IA4Code-DRIT-' , over printString , '-' + / + ('IA4Code-' , label printString , '-' , over printString + , '-' , (DateAndTime now printString replaceAll: $: with: $-) , '.csv')) asFileReference. @@ -202,7 +205,9 @@ GitMetricExporter >> exportFor: usersWithProjects over: aCollectionOfDateWeekMon aStream << ('(B): ' , beforeExp printString) << OSPlatform current lineEnding. aStream << ('(D): ' , duringExp printString) - << OSPlatform current lineEnding ] ] + << OSPlatform current lineEnding ] ]. + + 'Done computing' recordInfo ] { #category : #accessing } From 44af6db09cb83fcafe701714440b3d30350af1cb Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Wed, 12 Jun 2024 10:50:03 +0200 Subject: [PATCH 65/77] change commit freq metrics --- .../GitAnalyzer.class.st | 19 ++++++++++++ .../GitMetric4User.class.st | 29 +++++++++++++++---- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st index d66d9a7..5b120ad 100644 --- a/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st +++ b/src/GitLabHealth-Model-Analysis/GitAnalyzer.class.st @@ -90,6 +90,25 @@ GitAnalyzer >> analyseCommitContribution [ (#deletion -> fromCommit deletions). } asDictionary ] +{ #category : #'as yet unclassified' } +GitAnalyzer >> analyseCommitFrequencyFromCommits: initialCommits [ + + | commits response | + + ('GitAnalyzer, analyse commit Frequency on: ', onProject printString ) + recordInfo. + + response := { (#numberOfCommit -> nil). + (#frequency -> nil) } asDictionary. + + + commits := self arrangeCommitsByDate: initialCommits. + commits := (commits associations sortAscending: [ :entry | + entry key asDate ]) asOrderedDictionary. + + ^ commits +] + { #category : #'as yet unclassified' } GitAnalyzer >> analyseCommitFrequencySince: since until: until [ diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index 13852e5..fefd7e5 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -226,7 +226,7 @@ GitMetric4User >> commentsContributionsSince: since until: until overA: aDateWee { #category : #metrics } GitMetric4User >> commitFrequencySince: since until: until overA: aDateWeekMonthOrYear [ - | all aggregate periods total groupedByDate | + | all aggregate periods total groupedByDate userCommits| ('GitMetric4User: commit frequency') recordInfo. @@ -234,8 +234,26 @@ GitMetric4User >> commitFrequencySince: since until: until overA: aDateWeekMonth setupGroupedDateFrom: since to: until over: aDateWeekMonthOrYear. + + + userCommits := self loadCommitOfProjects: (itsProjects keys) since: since until: until. + userCommits := userCommits reject: [ :c | c commitCreator ~= user ]. + + userCommits do: [ :c | + | dateOver | + dateOver := self + transformDate: c created_at + to: aDateWeekMonthOrYear. + groupedByDate at: dateOver printString ifPresent: [:v | v add: c]. + ]. - aggregate := OrderedDictionary new. + + periods := groupedByDate keys size. + total := groupedByDate values sum: [ :commits | commits size ]. + + + + "aggregate := OrderedDictionary new. all := itsProjects keys collect: [ :id | self @@ -254,7 +272,7 @@ GitMetric4User >> commitFrequencySince: since until: until overA: aDateWeekMonth periods := aggregate keys size. total := aggregate values sum: [ :commits | commits size ]. - +" ^ { (#averageFrac -> (total / periods)). @@ -262,10 +280,10 @@ GitMetric4User >> commitFrequencySince: since until: until overA: aDateWeekMonth (#userCommit -> total). (#forOver -> - (aggregate keys size printString + (groupedByDate keys size printString , aDateWeekMonthOrYear printString)). (#periode -> aDateWeekMonthOrYear name). - (#details -> aggregate) } asOrderedDictionary + (#details -> groupedByDate) } asOrderedDictionary ] { #category : #'as yet unclassified' } @@ -306,6 +324,7 @@ GitMetric4User >> commitsProducedOnProject: aProjectId since: since until: until | frequency aggregatedFrequencies | aggregatedFrequencies := self setupGroupedDateFrom: since to: until over: aWeekOrMonthOrYear. + frequency := self commitsProducedOnProject: aProjectId since: since From 35a4d3dca0f0530775532a0e3a46a9887a22f144 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Wed, 12 Jun 2024 17:48:14 +0200 Subject: [PATCH 66/77] improve exporter --- .../GitMetricExporter.class.st | 72 +++++++++++++------ 1 file changed, 49 insertions(+), 23 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st b/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st index 3b3342b..30392a2 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st @@ -5,7 +5,8 @@ Class { 'glhImporter', 'beforeExp', 'duringExp', - 'label' + 'label', + 'entities' ], #category : #'GitLabHealth-Model-Analysis' } @@ -16,6 +17,19 @@ GitMetricExporter >> beforeDic: aDictionnarySinceUntil [ beforeExp := aDictionnarySinceUntil ] +{ #category : #'as yet unclassified' } +GitMetricExporter >> constructFilePath: over [ + + | file | + file := (FileLocator home + / + ('IA4Code-' , label printString , '-' , over printString + , '-' + , (DateAndTime now printString replaceAll: $: with: $-) + , '.csv')) asFileReference. + ^ file +] + { #category : #'as yet unclassified' } GitMetricExporter >> duringDic: aDictionnarySinceUntil [ duringExp := aDictionnarySinceUntil @@ -30,23 +44,17 @@ GitMetricExporter >> exportFor: usersWithProjects [ { #category : #'as yet unclassified' } GitMetricExporter >> exportFor: usersWithProjects over: aCollectionOfDateWeekMonthOrYear [ - | over entities file exportBrowserModel | - over := Date. - - - entities := usersWithProjects associations collect: [ :assoc | - | username projects metrics | - username := assoc key. - projects := assoc value. + entities ifNil: [ + self initEntitiesFromUserProjects: usersWithProjects ]. - metrics := GitMetric4User new. - metrics - glhImporter: glhImporter; - findUserNamed: username. - metrics loadProjects: projects. + self exportFor: aCollectionOfDateWeekMonthOrYear. +] - metrics ]. +{ #category : #'as yet unclassified' } +GitMetricExporter >> exportOver: aCollectionOfDateWeekMonthOrYear [ + | over file exportBrowserModel | + over := Date. exportBrowserModel := MiExportModel new. @@ -58,7 +66,6 @@ GitMetricExporter >> exportFor: usersWithProjects over: aCollectionOfDateWeekMon addColumnForQuery: [ :metrics | metrics user name printString ] withName: #'User name'. - "Code Contribution " exportBrowserModel addColumnForQuery: [ :metrics | @@ -186,13 +193,7 @@ GitMetricExporter >> exportFor: usersWithProjects over: aCollectionOfDateWeekMon aCollectionOfDateWeekMonthOrYear do: [ :period | over := period. - - file := (FileLocator home - / - ('IA4Code-' , label printString , '-' , over printString - , '-' - , (DateAndTime now printString replaceAll: $: with: $-) - , '.csv')) asFileReference. + file := self constructFilePath: over. file writeStreamDo: [ :aStream | aStream @@ -214,3 +215,28 @@ GitMetricExporter >> exportFor: usersWithProjects over: aCollectionOfDateWeekMon GitMetricExporter >> glhImporter: anImporter [ glhImporter := anImporter. ] + +{ #category : #'as yet unclassified' } +GitMetricExporter >> initEntitiesFromUserProjects: usersWithProjects [ + + entities := (usersWithProjects associations collect: [ :assoc | + | username projects metrics | + username := assoc key. + projects := assoc value. + + metrics := GitMetric4User new. + metrics + glhImporter: glhImporter; + findUserNamed: username. + metrics loadProjects: projects. + + metrics ]). + + ^ self +] + +{ #category : #initialization } +GitMetricExporter >> initialize [ + + +] From 0c6452d168093fd02d82bb1ed8b2bf247a38f4ff Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Wed, 12 Jun 2024 17:58:30 +0200 Subject: [PATCH 67/77] add snapchot checkpoint during export --- .../GitMetricExporter.class.st | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st b/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st index 30392a2..5f2fa30 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st @@ -74,6 +74,7 @@ GitMetricExporter >> exportOver: aCollectionOfDateWeekMonthOrYear [ until: (beforeExp at: #until) overA: over) at: #avgAddition) printString ] withName: #'(B) code additions (avg)'. + exportBrowserModel addColumnForQuery: [ :metrics | ((metrics @@ -84,6 +85,7 @@ GitMetricExporter >> exportOver: aCollectionOfDateWeekMonthOrYear [ exportBrowserModel addColumnForQuery: [ :metrics | + Smalltalk snapshot: true andQuit: false. ((metrics codeContributionsSince: (duringExp at: #since) until: (duringExp at: #until) @@ -119,6 +121,7 @@ GitMetricExporter >> exportOver: aCollectionOfDateWeekMonthOrYear [ "comment contribution " exportBrowserModel addColumnForQuery: [ :metrics | + Smalltalk snapshot: true andQuit: false. ((metrics commentsContributionsSince: (beforeExp at: #since) until: (beforeExp at: #until) @@ -137,6 +140,7 @@ GitMetricExporter >> exportOver: aCollectionOfDateWeekMonthOrYear [ "merge Request Duration " exportBrowserModel addColumnForQuery: [ :metrics | + Smalltalk snapshot: true andQuit: false. ((metrics mergeRequestDurationSince: (beforeExp at: #since) until: (beforeExp at: #until) @@ -240,3 +244,8 @@ GitMetricExporter >> initialize [ ] + +{ #category : #accessing } +GitMetricExporter >> label: aString [ + label := aString +] From c38127e32b2323e6bec947af82dafe9bd981a30c Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 13 Jun 2024 09:33:02 +0200 Subject: [PATCH 68/77] change commits --- .../GitMetric4User.class.st | 12 ++++++------ .../GitMetricExporter.class.st | 5 +++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st index fefd7e5..4122768 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetric4User.class.st @@ -443,11 +443,6 @@ GitMetric4User >> initialize [ itsMergeRequests := Dictionary new ] -{ #category : #accessing } -GitMetric4User >> itsCommits [ - ^ user commits -] - { #category : #churn } GitMetric4User >> loadCommitOfProjects: aCollection since: aTimespan [ |commits| @@ -462,7 +457,7 @@ GitMetric4User >> loadCommitOfProjects: aCollection since: aTimespan [ glhImporter completeImportedCommit: commit. ]. - ^ self itsCommits. + ^ self userCommits. ] { #category : #churn } @@ -665,3 +660,8 @@ GitMetric4User >> user [ GitMetric4User >> user: anUser [ user := anUser. ] + +{ #category : #accessing } +GitMetric4User >> userCommits [ + ^ user commits +] diff --git a/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st b/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st index 5f2fa30..971b116 100644 --- a/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st +++ b/src/GitLabHealth-Model-Analysis/GitMetricExporter.class.st @@ -35,6 +35,11 @@ GitMetricExporter >> duringDic: aDictionnarySinceUntil [ duringExp := aDictionnarySinceUntil ] +{ #category : #accessing } +GitMetricExporter >> entities: aCollection [ + entities := aCollection +] + { #category : #'as yet unclassified' } GitMetricExporter >> exportFor: usersWithProjects [ From d95cee00738e3857891f7cb8daa5d98cd7142462 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 13 Jun 2024 10:36:10 +0200 Subject: [PATCH 69/77] add dependencies in Baseline --- .../BaselineOfGitLabHealth.class.st | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/BaselineOfGitLabHealth/BaselineOfGitLabHealth.class.st b/src/BaselineOfGitLabHealth/BaselineOfGitLabHealth.class.st index 8adfec8..a60fed2 100644 --- a/src/BaselineOfGitLabHealth/BaselineOfGitLabHealth.class.st +++ b/src/BaselineOfGitLabHealth/BaselineOfGitLabHealth.class.st @@ -45,6 +45,24 @@ BaselineOfGitLabHealth >> definePackages: spec [ with: [ spec requires: #( 'NeoJSON' 'MoreLogger' ) ]; package: 'GitHubHealth-Model-Importer' with: [ spec requires: #( 'NeoJSON' 'MoreLogger' ) ]; - package: 'GitLabHealth-Model-Importer-Tests' - with: [ spec requires: #( 'GitLabHealth-Model-Importer' 'GitHubHealth-Model-Importer' ) ] + package: 'GitLabHealth-Model-Importer-Tests' with: [ + spec requires: + #( 'GitLabHealth-Model-Importer' 'GitHubHealth-Model-Importer' ) ]. + + "model extension" + spec + package: 'GLPHExtended-Model'; + package: 'GLPHExtended-Model-Extension' + with: [ spec requires: #( 'GLPHExtended-Model' ) ]; + package: 'GitLabHealth-Model-Analysis'; + package: 'GitLabHealth-Model-Analysis-Tests' with: [spec requires: #('GitLabHealth-Model-Analysis')]; + package: 'GitLabHealth-Visualization'; + package: 'GitLabProjectHealth-ExtendModel-Generator'; + package: 'GitLabProjectHealth-Model-Importer'; + package: 'GitLabProjectHealth-Model-Importer-Test' with: [spec requires: #('GitLabProjectHealth-Model-Importer') ]. + + + + + ] From 51e69ad8ce8f20add97ab129a8c416b22136d897 Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 13 Jun 2024 14:20:32 +0200 Subject: [PATCH 70/77] baseline fix --- .../BaselineOfGitLabHealth.class.st | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/BaselineOfGitLabHealth/BaselineOfGitLabHealth.class.st b/src/BaselineOfGitLabHealth/BaselineOfGitLabHealth.class.st index a60fed2..8dd5881 100644 --- a/src/BaselineOfGitLabHealth/BaselineOfGitLabHealth.class.st +++ b/src/BaselineOfGitLabHealth/BaselineOfGitLabHealth.class.st @@ -55,14 +55,11 @@ BaselineOfGitLabHealth >> definePackages: spec [ package: 'GLPHExtended-Model-Extension' with: [ spec requires: #( 'GLPHExtended-Model' ) ]; package: 'GitLabHealth-Model-Analysis'; - package: 'GitLabHealth-Model-Analysis-Tests' with: [spec requires: #('GitLabHealth-Model-Analysis')]; + package: 'GitLabHealth-Model-Analysis-Tests' + with: [ spec requires: #( 'GitLabHealth-Model-Analysis' ) ]; package: 'GitLabHealth-Visualization'; package: 'GitLabProjectHealth-ExtendModel-Generator'; package: 'GitLabProjectHealth-Model-Importer'; - package: 'GitLabProjectHealth-Model-Importer-Test' with: [spec requires: #('GitLabProjectHealth-Model-Importer') ]. - - - - - + package: 'GitLabProjectHealth-Model-Importer-Tests' + with: [ spec requires: #( 'GitLabProjectHealth-Model-Importer' ) ] ] From c88a18300b6b98de77c72d05fa3376b01459ffca Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 13 Jun 2024 15:14:00 +0200 Subject: [PATCH 71/77] fix baseline dep --- src/BaselineOfGitLabHealth/BaselineOfGitLabHealth.class.st | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/BaselineOfGitLabHealth/BaselineOfGitLabHealth.class.st b/src/BaselineOfGitLabHealth/BaselineOfGitLabHealth.class.st index 8dd5881..47e810c 100644 --- a/src/BaselineOfGitLabHealth/BaselineOfGitLabHealth.class.st +++ b/src/BaselineOfGitLabHealth/BaselineOfGitLabHealth.class.st @@ -51,7 +51,9 @@ BaselineOfGitLabHealth >> definePackages: spec [ "model extension" spec - package: 'GLPHExtended-Model'; + package: 'GLPHExtended-Model' with: [ + spec requires: + #( 'GitLabHealth-Model' 'GitLabHealth-Model-Extension' ) ]; package: 'GLPHExtended-Model-Extension' with: [ spec requires: #( 'GLPHExtended-Model' ) ]; package: 'GitLabHealth-Model-Analysis'; From 63852c75fa8b722e00a845308d1902e92a95f50a Mon Sep 17 00:00:00 2001 From: HLAD Nicolas Date: Thu, 13 Jun 2024 15:14:42 +0200 Subject: [PATCH 72/77] fix baseline dep 2 --- src/BaselineOfGitLabHealth/BaselineOfGitLabHealth.class.st | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BaselineOfGitLabHealth/BaselineOfGitLabHealth.class.st b/src/BaselineOfGitLabHealth/BaselineOfGitLabHealth.class.st index 47e810c..4efbc13 100644 --- a/src/BaselineOfGitLabHealth/BaselineOfGitLabHealth.class.st +++ b/src/BaselineOfGitLabHealth/BaselineOfGitLabHealth.class.st @@ -53,9 +53,9 @@ BaselineOfGitLabHealth >> definePackages: spec [ spec package: 'GLPHExtended-Model' with: [ spec requires: - #( 'GitLabHealth-Model' 'GitLabHealth-Model-Extension' ) ]; + #( 'GitLabHealth-Model' 'GitLabHealth-Model-Extension' ) ]; package: 'GLPHExtended-Model-Extension' - with: [ spec requires: #( 'GLPHExtended-Model' ) ]; + with: [ spec requires: #( 'GLPHExtended-Model' 'GitLabHealth-Model' 'GitLabHealth-Model-Extension' ) ]; package: 'GitLabHealth-Model-Analysis'; package: 'GitLabHealth-Model-Analysis-Tests' with: [ spec requires: #( 'GitLabHealth-Model-Analysis' ) ]; From 9314057aa7ae7f4c8aea99a9b9c19269be25acbc Mon Sep 17 00:00:00 2001 From: nhlad Date: Thu, 13 Jun 2024 17:17:10 +0200 Subject: [PATCH 73/77] Update README.md --- README.md | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/README.md b/README.md index 1410fa5..7f6712d 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,115 @@ classDiagram User -- Project : creator ``` + + ## Contributor This work has been first developed by the [research department of Berger-Levrault](https://www.research-bl.com/) + + +## Running metrics with docker + +### running locally +```smalltalk + +|glphModel glphApi glhImporter beforeExp duringExp usersWithProjects gme| + +"This example set up and run a GitProjectHealth metrics over two period of time of a given set of users and their projects. +It ouputs a csv files containing : churn code, commits frequencies, code addition and deletion, comments added (e.g. // # /**/ ), avg delay before first churn and merge request duration. +" + +"load githealth project into your image" +Metacello new + repository: 'github://moosetechnology/GitProjectHealth:GLPH-importer-new-changes/src'; + baseline: 'GitLabHealth'; + onConflict: [ :ex | ex useIncoming ]; + onUpgrade: [ :ex | ex useIncoming ]; + onDowngrade: [ :ex | ex useLoaded ]; + ignoreImage; + load. + +"set up a log at your root" +TinyLogger default + addFileLoggerNamed: 'pharo-code-churn.log'. + +"new model instance" +glphModel := GLPHEModel new. + +"new API class instance" +glphApi := GLPHApi new + privateToken: ''; + baseAPIUrl:'https://gitlab.com/api/v4'; + yourself. + +"new importer instance" +glhImporter := GLPHModelImporter new + glhApi: glphApi; + glhModel: glphModel; + withFiles: false; + withCommitDiffs: true. + +"setting up the period to compare (e.g. before a experience and during an experience)" +beforeExp := { + #since -> ('1 march 2023' asDate). + #until -> ('24 may 2023' asDate). + } asDictionary . +duringExp := { + #since -> ('1 march 2024' asDate). + #until -> ('24 may 2024' asDate). + } asDictionary . + +usersWithProjects := { +" 'dev nameA' -> { projectID1 . projectID2 }." +" 'dev nameB' -> { projectID3 . projectID2 }." +'John Do' -> { 14 . 543 . 2455 }. + } asDictionary. + + +gme := GitMetricExporter new glhImporter: glhImporter; + initEntitiesFromUserProjects: usersWithProjects; + beforeDic: beforeExp; duringDic: duringExp; label: 'GitLabHealth'. + +"select among the following calendar class (at least one) " +gme exportOver: { Date . Week . Month . Year .}. + +"the output files are located at 'FileLocator home/*.csv' " +Smalltalk snapshot: true andQuit: true. +``` + +### deploying with docker + +```dockerfile +# Use the official smalltalkci image from Docker Hub +FROM hpiswa/smalltalkci + +# Set environment variables +ENV ORIGIN_IMAGE_NAME=Moose64-11 +ENV IMAGE_NAME=PharoServer + +# Set the working directory +WORKDIR /usr/src/app + +# Copy your Smalltalk project files into the container +COPY . . + +# Run the CI script commands +RUN smalltalkci -s "${ORIGIN_IMAGE_NAME}" .smalltalk.ston +RUN mkdir ${IMAGE_NAME} +RUN mv /root/smalltalkCI-master/_builds/* ./${IMAGE_NAME} +RUN mv ./${IMAGE_NAME}/*/* ./${IMAGE_NAME} +RUN mv ${IMAGE_NAME}/TravisCI.changes ${IMAGE_NAME}/${IMAGE_NAME}.changes +RUN mv ${IMAGE_NAME}/TravisCI.image ${IMAGE_NAME}/${IMAGE_NAME}.image +RUN rm ${IMAGE_NAME}/build_status.txt +RUN rm -rf ${IMAGE_NAME}/vm + +# Expose any ports the application might need (if applicable) +# EXPOSE 8080 + +# Set the command to run your Smalltalk application +CMD ["/root/smalltalkCI-master/_cache/vms/Moose64-11/pharo", "--headless", "PharoServer/PharoServer.image", "st", "./launchAnalyze.st"] +# the output csv files will be under /root/*.csv + +``` + + From a199e0e03e8bbe4a99005dcda8adcf48201a9229 Mon Sep 17 00:00:00 2001 From: nhlad Date: Thu, 13 Jun 2024 17:18:49 +0200 Subject: [PATCH 74/77] Create launchAnalyze.ts for docker deployement --- launchAnalyze.ts | 62 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 launchAnalyze.ts diff --git a/launchAnalyze.ts b/launchAnalyze.ts new file mode 100644 index 0000000..2fe3ae9 --- /dev/null +++ b/launchAnalyze.ts @@ -0,0 +1,62 @@ +|glphModel glphApi glhImporter beforeExp duringExp usersWithProjects gme| + +"This example set up and run a GitProjectHealth metrics over two period of time of a given set of users and their projects. +It ouputs a csv files containing : churn code, commits frequencies, code addition and deletion, comments added (e.g. // # /**/ ), avg delay before first churn and merge request duration. +" + +"load githealth project into your image" +Metacello new + repository: 'github://moosetechnology/GitProjectHealth:GLPH-importer-new-changes/src'; + baseline: 'GitLabHealth'; + onConflict: [ :ex | ex useIncoming ]; + onUpgrade: [ :ex | ex useIncoming ]; + onDowngrade: [ :ex | ex useLoaded ]; + ignoreImage; + load. + +"set up a log at your root" +TinyLogger default + addFileLoggerNamed: 'pharo-code-churn.log'. + +"new model instance" +glphModel := GLPHEModel new. + +"new API class instance" +glphApi := GLPHApi new + privateToken: ''; + baseAPIUrl:'https://gitlab.com/api/v4'; + yourself. + +"new importer instance" +glhImporter := GLPHModelImporter new + glhApi: glphApi; + glhModel: glphModel; + withFiles: false; + withCommitDiffs: true. + +"setting up the period to compare (e.g. before a experience and during an experience)" +beforeExp := { + #since -> ('1 march 2023' asDate). + #until -> ('24 may 2023' asDate). + } asDictionary . +duringExp := { + #since -> ('1 march 2024' asDate). + #until -> ('24 may 2024' asDate). + } asDictionary . + +usersWithProjects := { +" 'dev nameA' -> { projectID1 . projectID2 }." +" 'dev nameB' -> { projectID3 . projectID2 }." +'John Do' -> { 14 . 543 . 2455 }. + } asDictionary. + + +gme := GitMetricExporter new glhImporter: glhImporter; + initEntitiesFromUserProjects: usersWithProjects; + beforeDic: beforeExp; duringDic: duringExp; label: 'GitLabHealth'. + +"select among the following calendar class (at least one) " +gme exportOver: { Date . Week . Month . Year .}. + +"the output files are located at 'FileLocator home/*.csv' " +Smalltalk snapshot: true andQuit: true. From 4d80dcda5bbacfbef6fefb2cf75f64c521e2a94d Mon Sep 17 00:00:00 2001 From: nhlad Date: Thu, 13 Jun 2024 17:21:55 +0200 Subject: [PATCH 75/77] Update README.md docker deployment --- README.md | 36 +++++++----------------------------- 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 7f6712d..41cd4a6 100644 --- a/README.md +++ b/README.md @@ -245,38 +245,16 @@ Smalltalk snapshot: true andQuit: true. ``` ### deploying with docker + -```dockerfile -# Use the official smalltalkci image from Docker Hub -FROM hpiswa/smalltalkci +```bash +git clone https://github.com/moosetechnology/GitProjectHealth.git +git checkout GLPH-importer-new-changes -# Set environment variables -ENV ORIGIN_IMAGE_NAME=Moose64-11 -ENV IMAGE_NAME=PharoServer - -# Set the working directory -WORKDIR /usr/src/app - -# Copy your Smalltalk project files into the container -COPY . . - -# Run the CI script commands -RUN smalltalkci -s "${ORIGIN_IMAGE_NAME}" .smalltalk.ston -RUN mkdir ${IMAGE_NAME} -RUN mv /root/smalltalkCI-master/_builds/* ./${IMAGE_NAME} -RUN mv ./${IMAGE_NAME}/*/* ./${IMAGE_NAME} -RUN mv ${IMAGE_NAME}/TravisCI.changes ${IMAGE_NAME}/${IMAGE_NAME}.changes -RUN mv ${IMAGE_NAME}/TravisCI.image ${IMAGE_NAME}/${IMAGE_NAME}.image -RUN rm ${IMAGE_NAME}/build_status.txt -RUN rm -rf ${IMAGE_NAME}/vm - -# Expose any ports the application might need (if applicable) -# EXPOSE 8080 +sudo docker build -t code-churn-pharo . +sudo docker run code-churn-pharo & +``` -# Set the command to run your Smalltalk application -CMD ["/root/smalltalkCI-master/_cache/vms/Moose64-11/pharo", "--headless", "PharoServer/PharoServer.image", "st", "./launchAnalyze.st"] -# the output csv files will be under /root/*.csv -``` From aef0849c9c86e4528658dd8044cb15ff938a0535 Mon Sep 17 00:00:00 2001 From: nhlad Date: Thu, 13 Jun 2024 17:22:27 +0200 Subject: [PATCH 76/77] Create Dockerfile --- Dockerfile | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a267e4a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# Use the official smalltalkci image from Docker Hub +FROM hpiswa/smalltalkci + +# Set environment variables +ENV ORIGIN_IMAGE_NAME=Moose64-11 +ENV IMAGE_NAME=PharoServer + +# Set the working directory +WORKDIR /usr/src/app + +# Copy your Smalltalk project files into the container +COPY . . + +# Run the CI script commands +RUN smalltalkci -s "${ORIGIN_IMAGE_NAME}" .smalltalk.ston +RUN mkdir ${IMAGE_NAME} +RUN mv /root/smalltalkCI-master/_builds/* ./${IMAGE_NAME} +RUN mv ./${IMAGE_NAME}/*/* ./${IMAGE_NAME} +RUN mv ${IMAGE_NAME}/TravisCI.changes ${IMAGE_NAME}/${IMAGE_NAME}.changes +RUN mv ${IMAGE_NAME}/TravisCI.image ${IMAGE_NAME}/${IMAGE_NAME}.image +RUN rm ${IMAGE_NAME}/build_status.txt +RUN rm -rf ${IMAGE_NAME}/vm + +# Expose any ports the application might need (if applicable) +# EXPOSE 8080 + +# Set the command to run your Smalltalk application +CMD ["/root/smalltalkCI-master/_cache/vms/Moose64-11/pharo", "--headless", "PharoServer/PharoServer.image", "st", "./launchAnalyze.st"] +# the output csv files will be under /root/*.csv From ac34dd5eb26a713e33ee275c98b246f553a3cdb8 Mon Sep 17 00:00:00 2001 From: nhlad Date: Thu, 13 Jun 2024 17:25:32 +0200 Subject: [PATCH 77/77] Update README.md --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 41cd4a6..f0c1790 100644 --- a/README.md +++ b/README.md @@ -249,11 +249,18 @@ Smalltalk snapshot: true andQuit: true. ```bash git clone https://github.com/moosetechnology/GitProjectHealth.git +cd GitProjectHealth git checkout GLPH-importer-new-changes sudo docker build -t code-churn-pharo . sudo docker run code-churn-pharo & ``` +locate and retrieve csv output files: +```bash +sudo docker ps +sudo docker exec -it find / -type f -name 'IA4Code*.csv' 2>/dev/null +``` +