From bc7d6d8934a211c1e11ea3fb49b0b02e70aa47cf Mon Sep 17 00:00:00 2001 From: Georgios Gousios Date: Wed, 5 Oct 2016 15:51:47 +0200 Subject: [PATCH] Make code nicer - remove dead code - use upsert method where necessary --- README.md | 2 +- lib/ghtorrent/commands/full_repo_retriever.rb | 2 +- lib/ghtorrent/ghtorrent.rb | 93 +++---------------- lib/ghtorrent/retriever.rb | 19 +--- lib/ghtorrent/transacted_gh_torrent.rb | 5 - 5 files changed, 15 insertions(+), 106 deletions(-) diff --git a/README.md b/README.md index 08b6f25..33c990e 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ For distributed mirroring you also need RabbitMQ >= 3.3 ### 1. Install GHTorrent -GHTorrent is written in Ruby (tested with 2.0). To install it as a Gem do: +GHTorrent is written in Ruby (tested with Ruby > 2.0). To install it as a Gem do: sudo gem install ghtorrent diff --git a/lib/ghtorrent/commands/full_repo_retriever.rb b/lib/ghtorrent/commands/full_repo_retriever.rb index a8ae4f7..9e3b52e 100644 --- a/lib/ghtorrent/commands/full_repo_retriever.rb +++ b/lib/ghtorrent/commands/full_repo_retriever.rb @@ -11,7 +11,7 @@ module FullRepoRetriever def stages %w(ensure_commits ensure_forks ensure_pull_requests - ensure_issues ensure_watchers ensure_labels ensure_languages) #ensure_project_members + ensure_issues ensure_watchers ensure_labels ensure_languages) end def settings diff --git a/lib/ghtorrent/ghtorrent.rb b/lib/ghtorrent/ghtorrent.rb index 7c9cd81..f2a1227 100644 --- a/lib/ghtorrent/ghtorrent.rb +++ b/lib/ghtorrent/ghtorrent.rb @@ -593,7 +593,7 @@ def ensure_repo(user, repo, recursive = false) repos.filter(:owner_id => curuser[:id], :name => repo).update(:forked_from => parent[:id]) info "Repo #{user}/#{repo} is a fork of #{parent_owner}/#{parent_repo}" - forked_commit = ensure_fork_point(user, repo) + ensure_fork_point(user, repo) end end @@ -813,71 +813,6 @@ def ensure_fork_point(owner, repo) db[:commits].where(:sha => forked_sha).first end - ## - # Make sure that a project has all the registered members defined - def ensure_project_members(user, repo, refresh = false) - currepo = ensure_repo(user, repo) - time = currepo[:created_at] - - project_members = db.from(:project_members, :users).\ - where(:project_members__user_id => :users__id).\ - where(:project_members__repo_id => currepo[:id]).select(:login).all - - retrieve_repo_collaborators(user, repo).reduce([]) do |acc, x| - if project_members.find {|y| y[:login] == x['login']}.nil? - acc << x - else - acc - end - end.map { |x| save{ensure_project_member(user, repo, x['login'], time) }}.select{|x| !x.nil?} - end - - ## - # Make sure that a project member exists in a project - def ensure_project_member(owner, repo, new_member, date_added) - pr_members = db[:project_members] - project = ensure_repo(owner, repo) - new_user = ensure_user(new_member, false, false) - - if project.nil? or new_user.nil? - warn "Could not find repo #{owner}/#{repo} or member #{new_member}" - return - end - - memb_exist = pr_members.first(:user_id => new_user[:id], - :repo_id => project[:id]) - - if memb_exist.nil? - added = if date_added.nil? - max(project[:created_at], new_user[:created_at]) - else - date_added - end - retrieved = retrieve_repo_collaborator(owner, repo, new_member) - - if retrieved.nil? - warn "Could not retrieve member #{new_member} of #{owner}/#{repo}" - return - end - - pr_members.insert( - :user_id => new_user[:id], - :repo_id => project[:id], - :created_at => date(added) - ) - info "Added project_member #{repo} -> #{new_member}" - else - debug "Project member #{repo} -> #{new_member} exists" - end - - unless date_added.nil? - pr_members.filter(:user_id => new_user[:id], - :repo_id => project[:id])\ - .update(:created_at => date(date_added)) - info "Updated project member #{repo} -> #{new_member}, created_at -> #{date(date_added)}" - end - end - ## # Make sure that the organizations the user participates into exist # @@ -1003,7 +938,7 @@ def ensure_commit_comment(owner, repo, sha, comment_id) ## # Make sure that all watchers exist for a repository - def ensure_watchers(owner, repo, refresh = false) + def ensure_watchers(owner, repo) currepo = ensure_repo(owner, repo) if currepo.nil? @@ -1023,12 +958,12 @@ def ensure_watchers(owner, repo, refresh = false) else acc end - end.map { |x| save{ensure_watcher(owner, repo, x['login'], nil) }}.select{|x| !x.nil?} + end.map { |x| save{ensure_watcher(owner, repo, x['login']) }}.select{|x| !x.nil?} end ## # Make sure that a watcher/stargazer exists for a repository - def ensure_watcher(owner, repo, watcher, date_added = nil) + def ensure_watcher(owner, repo, watcher, date_added: nil) project = ensure_repo(owner, repo) new_watcher = ensure_user(watcher, false, false) @@ -1448,23 +1383,19 @@ def ensure_fork(owner, repo, fork_id) ## # Make sure all issues exist for a project - def ensure_issues(owner, repo, refresh = false) + def ensure_issues(owner, repo) currepo = ensure_repo(owner, repo) if currepo.nil? warn "Could not find repo #{owner}/#{repo} for retrieving issues" return end - raw_issues = if refresh - retrieve_issues(owner, repo, refresh = true) - else - issues = db[:issues].filter(:repo_id => currepo[:id]).all - retrieve_issues(owner, repo).reduce([]) do |acc, x| - if issues.find { |y| y[:issue_id] == x['number'] }.nil? - acc << x - else + issues = db[:issues].filter(:repo_id => currepo[:id]).all + raw_issues = retrieve_issues(owner, repo).reduce([]) do |acc, x| + if issues.find { |y| y[:issue_id] == x['number'] }.nil? + acc << x + else acc - end end end @@ -1719,7 +1650,7 @@ def ensure_issue_comment(owner, repo, issue_id, comment_id, pull_req_id = nil) ## # Retrieve repository issue labels - def ensure_labels(owner, repo, refresh = false) + def ensure_labels(owner, repo) currepo = ensure_repo(owner, repo) if currepo.nil? @@ -1729,7 +1660,7 @@ def ensure_labels(owner, repo, refresh = false) repo_labels = db[:repo_labels].filter(:repo_id => currepo[:id]).all - retrieve_repo_labels(owner, repo, refresh).reduce([]) do |acc, x| + retrieve_repo_labels(owner, repo).reduce([]) do |acc, x| if repo_labels.find {|y| y[:name] == x['name']}.nil? acc << x else diff --git a/lib/ghtorrent/retriever.rb b/lib/ghtorrent/retriever.rb index b2ff78a..bcd95e0 100644 --- a/lib/ghtorrent/retriever.rb +++ b/lib/ghtorrent/retriever.rb @@ -288,22 +288,6 @@ def retrieve_commit_comment(owner, repo, sha, id) end end - # Retrieve all collaborators for a repository - def retrieve_repo_collaborators(user, repo) - repo_bound_items(user, repo, :repo_collaborators, - ["repos/#{user}/#{repo}/collaborators"], - {'repo' => repo, 'owner' => user}, - 'login', item = nil, refresh = false, order = :asc) - end - - # Retrieve a single repository collaborator - def retrieve_repo_collaborator(user, repo, new_member) - repo_bound_item(user, repo, new_member, :repo_collaborators, - ["repos/#{user}/#{repo}/collaborators"], - {'repo' => repo, 'owner' => user}, - 'login') - end - # Retrieve all watchers for a repository def retrieve_watchers(user, repo) repo_bound_items(user, repo, :watchers, @@ -654,8 +638,7 @@ def repo_bound_items(user, repo, entity, urls, selector, discriminator, end instance_selector = selector.merge({discriminator => id}) - persister.del(entity, instance_selector) - persister.store(entity, x) + persister.upsert(entity, instance_selector, x) debug "Refreshing #{entity} #{user}/#{repo} -> #{x[discriminator]}" end else diff --git a/lib/ghtorrent/transacted_gh_torrent.rb b/lib/ghtorrent/transacted_gh_torrent.rb index 42ed7b9..e1f0a01 100644 --- a/lib/ghtorrent/transacted_gh_torrent.rb +++ b/lib/ghtorrent/transacted_gh_torrent.rb @@ -73,11 +73,6 @@ def ensure_issue_label(owner, repo, issue_id, name) end end - def ensure_project_member(owner, repo, new_member, date_added) - check_transaction do - super(owner, repo, new_member, date_added) - end - end def ensure_watcher(owner, repo, watcher, date_added = nil) check_transaction do