ct_analyse

#!/usr/bin/env ruby

require 'json'
require 'find'
require 'fileutils'
require 'open3'

E_NOT_DIR=10

$version="1.02"
$progname=File.basename( $PROGRAM_NAME )
$debug=false
$verbose=false
$keep_logs=false # if true, save a file of concatenated records

def verbose_msg(msg)
  if $verbose
    $stderr.puts("#{msg}\n")
  end
end

def debug_msg(msg)
  if $debug
    $stderr.puts("DEBUG: #{msg}\n")
  end
end

def usage()
  print("Usage #{$progname} { -s startdir }")
end

def main(args)
  if args.include?('-d')
    $debug=true
    args.delete('-d')
  end

  if args.include?('-v')
    $verbose=true
    args.delete('-v')
  end

  args.reverse!
  until args.empty? do
    arg=args.pop
    debug_msg ("Arg=#{arg}\n")
    case arg
      when '-s'
        start_dir=args.pop
      when '-k'
        $keep_logs=true
    else
      warn("Ignoring arg [#{arg}]")
    end
  end
  if start_dir==nil then
    start_dir="."
  end
  debug_msg("Analysing all .json files below starting dir \"#{start_dir}\"")

  load_logs(start_dir)
    
end

# Amazon delivers CT logs as gzipped json files, so only work with those:
def load_logs(start_dir)
  record_count=0
  file_count=0
  records=Array.new
  Find.find(start_dir) do | path|
    if path.match(".json.gz$")
      if ! is_already_loaded(path)
        debug_msg("Loading data from file #{path}")
        file_count+=1
        record_count+=load_records_from_json_file(path,records)
        FileUtils.touch("#{path}.loaded")
      end
    end
  end
  debug_msg("#{record_count} records loaded from #{file_count} files")
  verbose_msg("#{record_count} records loaded from #{file_count} files")
end

def is_already_loaded(file_path)
  if File.exist?("#{file_path}.loaded")
    debug_msg("File #{file_path} has already been loaded ; skipping")
    return(true)
  end
  return(false)
end

def load_records_from_json_file(file_path,record_list)
  file_data,stderr,status=Open3.capture3("/bin/zcat #{file_path}")
  data=JSON.parse(file_data);
  data['Records'].each do |record|
    ev_name=record['eventName']
    ev_username=record['userIdentity']['userName']
    ev_id_type=record['userIdentity']['type']
    ev_time=record['eventTime']
    ev_username=record['userIdentity']['userName']
    ev_type=record['eventType']
    ev_region=record['awsRegion']
    ev_error=record['errorCode']       
    ev_source=record['eventSource']
    printf("\"%s\",\"%s\",\"%s/%s\",\"%s\",\"%s\",\"%s\",\"%s\"\n",\
           ev_type, ev_region, ev_id_type,(ev_username == nil ? "not_applicable" : ev_username), ev_name, ev_source, ev_time,(ev_error == nil ? "" : ev_error))
  end
  debug_msg("Parsing file #{file_path} - #{data['Records'].length} records in #{file_data.length} lines." )
  return data['Records'].length
end

main(ARGV)