parser_diputados_popit

#!/usr/bin/env ruby
# encoding: utf-8

require 'nokogiri'
require 'open-uri'
require 'json'
require 'optparse'
require 'i18n'
require 'popit'

@options = {}
optparse = OptionParser.new do |option_parser|
  option_parser.banner = "Usage: parser_diputados.rb [@options]"

  option_parser.on("-o", "--output FILENAME", "Save the output to a file") do |o|
    @options[:output] = o
  end

  option_parser.on("-i n", "--instance=n", "Send every row this instance of the PopIt Api on popit.mysociety.org. Default legisladores-ar [INSTANCE]") do |instance|
    @options[:instance_name] = instance
  end

  option_parser.on("-u n", "--user=n", "Api user [USERNAME]") do |user|
    @options[:api_user] = user
  end

  option_parser.on("-p n", "--password=n", "Api password [PASSWORD]") do |pass|
    @options[:api_pass] = pass
  end
end
optparse.parse!

#Initialize API
@api = PopIt.new :instance_name => @options[:instance_name], :user => @options[:api_user], :password => @options[:api_pass]

URL_DIPUTADOS = "http://www.diputados.gov.ar/diputados/listadip.html"

def fetch_and_clean_html
  cachefile = "html.cache"
  
  if File.exist?(cachefile)
    content = File.read(cachefile)
  else 
    content = open(URL_DIPUTADOS)
      .read
      .gsub("<tbody>", "<tbody><tr>")
      .gsub("</tr>", "</tr><tr>");

    File.open(cachefile, 'w+') {|f| f.write(content) }
  end
  return content
end

def parse_email element
  username = element
    .css("td a")
    .first['href']
    .split("/")[2]
    .strip

  "#{username}@diputados.gob.ar"
end

def parse_url element
  path = element.css("td a").first["href"]
  "http://www.diputados.gov.ar#{path}"
end

def slug text
  return I18n.transliterate(text).gsub(" ","-").downcase
end

def update_membership organization, person, role, start_date, end_date, district
  membership = {}
  membership[:organization_id] = organization["id"]
  membership[:person_id]    = person["id"]
  membership[:role] = role
  membership[:start_date]      = start_date
  membership[:end_date]        = end_date
  membership[:district]    = district


  # Check to see if content exists
  exists = @api.search.memberships.get(
    :q => 'organization_id: "'+membership[:organization_id]+'" &&
           person_id: "'+membership[:person_id]+'" &&
           start_date: "'+membership[:start_date].to_s+'" &&
           end_date: "'+membership[:end_date].to_s+'"
    ')

  #puts exists

  if !exists.empty?
    puts "Membership already exists for " + person["id"].to_s + " as " + role + " to " + organization["id"  ].to_s
  else
      #Create content
      puts "Adding membership for " + person["id"].to_s + " as " + role + " to " + organization["id"  ].to_s
      response = @api.memberships.post membership
  end
  rescue
    p membership
  raise
    
  return response
end

def update_person content
  # Check to see if content exists
  exists = @api.search.persons.get(:q => 'slug: "'+content[:slug]+'"')

  if !exists.empty?
    puts "Already exists " + content[:slug]
    response = exists.first
    #puts "Updating " + content[:slug]
    #response = @api.persons(exists.first["id"].to_s).put content
  else
      #Create content
      puts "Creating " + content[:slug]
      response = @api.persons.post content
  end
  rescue
    p content
  raise
    
  return response

end

def update_organization content
  # Check to see if content exists
  exists = @api.search.organizations.get(:q => 'slug: "'+content[:slug]+'"')

  if !exists.empty?
    #Update content with new information
    #puts "Updating " + content[:slug]
    puts "Already exists " + content[:slug]
    response = exists.first
    #response = @api.organizations(exists.first["id"].to_s).put content
  else
      #Create content
      puts "Creating " + content[:slug]
      response = @api.organizations.post content
  end
  rescue
    p content
  raise
    
  return response

end


doc = Nokogiri::HTML fetch_and_clean_html

data = {}
data[:bloques] = []
data[:diputados] = []

doc.css("#tablesorter tbody tr").reverse().each do |el|
  next if el.css("td").empty?

  diputado = {}
  name = el
    .css("td")[1]
    .text
    .split(",")
    .map(&:strip)
  diputado[:name]             = name.last.strip.capitalize + " " + name.first.strip.capitalize
  diputado[:slug]             = slug diputado[:name];

  diputado[:images]           =  {}
  diputado[:images][:url]     = el.css("td img")
                                .first['src']
                                .gsub("_medium", "")

  diputado[:contact_details]  = []

  contact = {}
  contact[:label]             = "email"
  contact[:type]              = "email"
  contact[:value]             = parse_email el

  diputado[:contact_details] << contact

  diputado[:memberships] = []

  bloque = {}
  bloque[:name] = el.css("td")[5].text.strip.capitalize
  bloque[:slug] = slug bloque[:name]

  camara = {}
  camara[:name] = "Honorable Cámara de Diputados de la Nación"
  camara[:slug] = "hcdn"

  start_date      = Date.strptime(Date.parse(el.css("td")[3].text.strip).to_s,'%Y-%m-%d')
  end_date        = Date.strptime(Date.parse(el.css("td")[4].text.strip).to_s,'%Y-%m-%d')
  district        = el.css("td")[2].text.strip.capitalize


  ## Missing info for the new structure

  # diputado[:links]               = {}
  # diputado[:links][:note]  = "Link name"
  # diputado[:links][:url]    = "url"

  # diputado[:commissions]      = {}

  # diputado[:birth_date]       = ""
  # diputado[:personal_info]      = {}
  # diputado[:electoral_info]      = {}
  # diputado[:represent]      = {}
  # diputado[:represent][:district]    = el.css("td")[2].text

  data[:diputados] << diputado
  data[:bloques] << bloque

  ##Request to API
  if @options[:instance_name] 
   
    diputado_response = update_person diputado
    bloque_response = update_organization bloque
    camara_response = update_organization camara

    ##Memberships
    update_membership camara_response, diputado_response, "Diputado", start_date, end_date, district
    update_membership bloque_response, diputado_response, "Integrante", start_date, end_date, district

  end

end

data[:bloques].uniq!.sort!


if @options[:output]
  File.open(@options[:output], "w") do |file|
    file.puts data.to_json
  end
else
  $stdout.puts data.to_json
end