From a8f268102ecc0753c163248be09300b4944c1417 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Mon, 12 Aug 2013 17:26:23 +0200 Subject: [PATCH 01/12] Use Mail for parsing and working with email, replacing RMail. Initial steps for moving to using Mail in place of RMail. Also various Ruby 2.0.0 compatability fixes have been made. --- bin/sup-migrate-index | 110 +++++++ bin/sup-tweak-labels | 2 +- devel/console.sh | 2 +- lib/sup.rb | 3 +- lib/sup/index.rb | 20 +- lib/sup/maildir.rb | 14 +- lib/sup/mbox.rb | 10 +- lib/sup/message.rb | 512 ++++++++++++++++++----------- lib/sup/message_chunks.rb | 12 +- lib/sup/modes/edit_message_mode.rb | 21 +- lib/sup/modes/forward_mode.rb | 9 +- lib/sup/sent.rb | 4 + lib/sup/thread.rb | 2 + lib/sup/util.rb | 125 ++----- sup.gemspec | 4 +- test/dummy_source.rb | 4 +- test/test_helper.rb | 1 + test/test_message.rb | 4 +- 18 files changed, 534 insertions(+), 325 deletions(-) create mode 100755 bin/sup-migrate-index mode change 100644 => 100755 devel/console.sh diff --git a/bin/sup-migrate-index b/bin/sup-migrate-index new file mode 100755 index 000000000..ff63f732d --- /dev/null +++ b/bin/sup-migrate-index @@ -0,0 +1,110 @@ +#!/usr/bin/env ruby + +require 'uri' +require 'rubygems' +require 'trollop' +require "sup"; Redwood::check_library_version_against "git" + +PROGRESS_UPDATE_INTERVAL = 15 # seconds + +class AbortExecution < SystemExit +end + +opts = Trollop::options do + version "sup-migrate-index (sup #{Redwood::VERSION})" + banner < "-n" + opt :version, "Show version information", :short => :none + + conflicts :ignore_missing, :warn_missing, :abort_missing +end +Trollop::die "Extra arguments given" if ARGV.length > 1 +missing_action = [:ignore_missing, :warn_missing, :abort_missing].find { |x| opts[x] } || :abort_missing + +BASE_DIR = ENV["SUP_BASE"] || File.join(ENV["HOME"], ".sup") + + +Redwood::start +index = Redwood::Index.init + +#index.lock_interactively or exit + + +info "Starting migration to version 5.." + +index.load true + +xapian = index.get_xapian +db_version = xapian.get_metadata 'version' +db_version = '0' if db_version.empty? + +fail "It only makes sense to migrate from version 4 to 5, you have an index version #{db_version}" unless db_version == '4' + +num_docs = 0 +xapian.postlist('Kmail').each do |x| + info "Updating document: #{x.inspect}" + num_docs += 1 + + doc = xapian.document(x.docid) + entry = doc.entry + + debug "Working on entry: #{entry[:message_id]}." + + debug "Adding missing fields.." + entry[:safe_id] = Redwood::Message.munge_msgid(entry[:message_id]) + entry[:safe_refs] += entry[:refs].nil? ? [] : entry[:refs].compact.map { |r| rr = Redwood::Message.munge_msgid(r); debug "addin ref #{r} -> #{rr}.."; rr } + entry[:safe_refs] = entry[:safe_refs].uniq + + + entry[:safe_replytos] = entry[:replytos].nil? ? [] : entry[:replytos].compact.map { |r| rr = Redwood::Message.munge_msgid(r); debug "addin replyto #{r} -> #{rr}.."; rr } + + next if opts[:dry_run] + doc.entry = entry + + info "Storing message.." + xapian.replace_document x.docid, doc + k = xapian.document x.docid + info "stored safe_id: " + k.value(0) + + debug "Loading message.." + #locations = entry[:locations].map do |source_id,source_info| + #source = Redwood::SourceManager[source_id] + #raise "invalid source #{source_id}" unless source + #Redwood::Location.new source, source_info + #end + + #m = Redwood::Message.new :locations => locations, + #:labels => entry[:labels], + #:snippet => entry[:snippet] + + ## Try to find person from contacts before falling back to + ## generating it from the address. + #mk_person = lambda { |x| Redwood::Person.from_name_and_email(*x.reverse!) } + #entry[:from] = mk_person[entry[:from]] + #entry[:to].map!(&mk_person) + #entry[:cc].map!(&mk_person) + #entry[:bcc].map!(&mk_person) + + #m.load_from_index! entry + #m.load_from_source! + + break + + +end + +info "done, updated: #{num_docs} documents." + diff --git a/bin/sup-tweak-labels b/bin/sup-tweak-labels index 36947d556..38bcbd9bb 100755 --- a/bin/sup-tweak-labels +++ b/bin/sup-tweak-labels @@ -85,7 +85,7 @@ begin parsed_query = index.parse_query query parsed_query.merge! :load_spam => true, :load_deleted => true, :load_killed => true - ids = Enumerator.new(index, :each_id, parsed_query) + ids = index.to_enum(:each_id, parsed_query) num_total = index.num_results_for parsed_query $stderr.puts "Found #{num_total} documents across #{source_ids.length} sources. Scanning..." diff --git a/devel/console.sh b/devel/console.sh old mode 100644 new mode 100755 index ac119345c..dc7077563 --- a/devel/console.sh +++ b/devel/console.sh @@ -1,3 +1,3 @@ #!/bin/sh -irb -I lib -r devel/start-console.rb +irb -I lib -r ./devel/start-console.rb diff --git a/lib/sup.rb b/lib/sup.rb index ad6d405b1..218961dfe 100644 --- a/lib/sup.rb +++ b/lib/sup.rb @@ -2,13 +2,12 @@ require 'rubygems' require 'yaml' -YAML::ENGINE.yamler = 'psych' require 'zlib' require 'thread' require 'fileutils' require 'locale' require 'curses' -require 'rmail' +require 'mail' begin require 'fastthread' rescue LoadError diff --git a/lib/sup/index.rb b/lib/sup/index.rb index dbfb14a4b..62392d860 100644 --- a/lib/sup/index.rb +++ b/lib/sup/index.rb @@ -32,7 +32,7 @@ module Redwood class Index include InteractiveLock - INDEX_VERSION = '4' + INDEX_VERSION = '5' ## dates are converted to integers for xapian, and are used for document ids, ## so we must ensure they're reasonably valid. this typically only affect @@ -98,9 +98,9 @@ def unlock end end - def load + def load failsafe=false SourceManager.load_sources - load_index + load_index failsafe end def save @@ -110,7 +110,11 @@ def save save_index end - def load_index + def get_xapian + @xapian + end + + def load_index failsafe=false path = File.join(@dir, 'xapian') if File.exists? path @xapian = Xapian::WritableDatabase.new(path, Xapian::DB_OPEN) @@ -119,8 +123,12 @@ def load_index if false info "Upgrading index format #{db_version} to #{INDEX_VERSION}" @xapian.set_metadata 'version', INDEX_VERSION + + elsif (db_version == '4') + fail "This Sup has a new index version v#{INDEX_VERSION}, but you have v#{db_version}. If you have just upgraded Sup there has been a major change in the index format and a migration tool need to be run. Please first back up your existing index using sup-dump and back up #{path}, then run sup-migrate-index to upgrade it." unless failsafe + elsif db_version != INDEX_VERSION - fail "This Sup version expects a v#{INDEX_VERSION} index, but you have an existing v#{db_version} index. Please run sup-dump to save your labels, move #{path} out of the way, and run sup-sync --restore." + fail "This Sup version expects a v#{INDEX_VERSION} index, but you have an existing v#{db_version} index. Please run sup-dump to save your labels, move #{path} out of the way, and run sup-sync --restore." unless failsafe end else @xapian = Xapian::WritableDatabase.new(path, Xapian::DB_CREATE) @@ -299,7 +307,7 @@ def source_for_id id synchronize { get_entry(id)[:source_id] } end - ## Yields each tearm in the index that starts with prefix + ## Yields each term in the index that starts with prefix def each_prefixed_term prefix term = @xapian._dangerous_allterms_begin prefix lastTerm = @xapian._dangerous_allterms_end prefix diff --git a/lib/sup/maildir.rb b/lib/sup/maildir.rb index 0c3061c8c..e6fcfb6d1 100644 --- a/lib/sup/maildir.rb +++ b/lib/sup/maildir.rb @@ -23,6 +23,10 @@ def initialize uri, usual=true, archived=false, id=nil, labels=[] @mtimes = { 'cur' => Time.at(0), 'new' => Time.at(0) } end + def init_with coder + initialize coder['uri'], coder['usual'], coder['archived'], coder['id'], coder['labels'] + end + def file_path; @dir end def self.suggest_labels_for path; [] end def is_source_for? uri; super || (uri == @expanded_uri); end @@ -68,7 +72,13 @@ def load_header id end def load_message id - with_file_for(id) { |f| RMail::Parser.read f } + with_file_for(id) do |f| + begin + Mail.read_from_string f.read + rescue + raise SourceError + end + end end def raw_header id @@ -95,7 +105,7 @@ def poll next if prev_mtime >= mtime @mtimes[d] = mtime - old_ids = benchmark(:maildir_read_index) { Enumerator.new(Index.instance, :each_source_info, self.id, "#{d}/").to_a } + old_ids = benchmark(:maildir_read_index) { Index.to_enum(:each_source_info, self.id, "#{d}/").to_a } new_ids = benchmark(:maildir_read_dir) { Dir.glob("#{subdir}/*").map { |x| File.basename x }.sort } added = new_ids - old_ids deleted = old_ids - new_ids diff --git a/lib/sup/mbox.rb b/lib/sup/mbox.rb index 95753c4e1..e2d428f3e 100644 --- a/lib/sup/mbox.rb +++ b/lib/sup/mbox.rb @@ -34,6 +34,10 @@ def initialize uri_or_fp, usual=true, archived=false, id=nil, labels=nil super uri_or_fp, usual, archived, id end + def init_with coder + initialize coder['uri'], coder['usual'], coder['archived'], coder['id'], coder['labels'] + end + def file_path; @path end def is_source_for? uri; super || (uri == @expanded_uri) end @@ -81,8 +85,8 @@ def load_message offset until @f.eof? || MBox::is_break_line?(l = @f.gets) string << l end - RMail::Parser.read string - rescue RMail::Parser::Error => e + Mail.read_from_string string + rescue e raise FatalSourceError, "error parsing mbox file: #{e.message}" end end @@ -161,7 +165,7 @@ def next_offset offset ## TODO optimize this by iterating over allterms list backwards or ## storing source_info negated def last_indexed_message - benchmark(:mbox_read_index) { Enumerator.new(Index.instance, :each_source_info, self.id).map(&:to_i).max } + benchmark(:mbox_read_index) { Index.to_enum(:each_source_info, self.id).map(&:to_i).max } end ## offset of first new message or nil diff --git a/lib/sup/message.rb b/lib/sup/message.rb index e4f16d457..50238afbb 100644 --- a/lib/sup/message.rb +++ b/lib/sup/message.rb @@ -1,4 +1,8 @@ +# encoding: UTF-8 + require 'time' +require 'uri' + module Redwood @@ -65,83 +69,79 @@ def initialize opts #parse_header(opts[:header] || @source.load_header(@source_info)) end - def decode_header_field v - return unless v - return v unless v.is_a? String - return unless v.size < MAX_HEADER_VALUE_SIZE # avoid regex blowup on spam - d = v.dup - d = d.transcode($encoding, 'ASCII') - Rfc2047.decode_to $encoding, d - end + def parse_header m + # load + # @id sanitized message id + # @from + # @date + # @subj + # @to + # @cc + # @bcc + # @refs + # @replyto + # @replytos + # @list_address + # @receipient_email + # @source_marked_read + # @source_starred + # @list_subscribe + # @list_unsubscribe + + unless m.message_id + m.message_id = "<#{Time.now.to_i}-defaulted-#{munge_msgid m.header.to_s}@sup-faked>" + end - def parse_header encoded_header - header = SavingHash.new { |k| decode_header_field encoded_header[k] } + @id = m.message_id - @id = '' - if header["message-id"] - mid = header["message-id"] =~ /<(.+?)>/ ? $1 : header["message-id"] - @id = sanitize_message_id mid - end - if (not @id.include? '@') || @id.length < 6 - @id = "sup-faked-" + Digest::MD5.hexdigest(raw_header) - #from = header["from"] - #debug "faking non-existent message-id for message from #{from}: #{id}" - end + @from = Person.from_address m.fetch_header(:from) + @from = Person.from_address "Sup Auto-generated Fake Sender " unless @from + @sender = Person.from_address m.fetch_header(:sender) - @from = Person.from_address(if header["from"] - header["from"] - else - name = "Sup Auto-generated Fake Sender " - #debug "faking non-existent sender for message #@id: #{name}" - name - end) - - @date = case(date = header["date"]) - when Time - date - when String - begin - Time.parse date - rescue ArgumentError => e - #debug "faking mangled date header for #{@id} (orig #{header['date'].inspect} gave error: #{e.message})" - Time.now - end - else - #debug "faking non-existent date header for #{@id}" - Time.now + begin + @date = (m.date || Time.now).to_time + rescue NoMethodError + # TODO: remove this rescue once mail/#564 is fixed + warn "Invalid date for message #{@id}, using 'now'." + @date = Time.now.to_time end - subj = header["subject"] - subj = subj ? subj.fix_encoding : nil - @subj = subj ? subj.gsub(/\s+/, " ").gsub(/\s+$/, "") : DEFAULT_SUBJECT - @to = Person.from_address_list header["to"] - @cc = Person.from_address_list header["cc"] - @bcc = Person.from_address_list header["bcc"] + @to = Person.from_address_list (m.fetch_header (:to)) + @cc = Person.from_address_list (m.fetch_header (:cc)) + @bcc = Person.from_address_list (m.fetch_header (:bcc)) + + @subj = (m.subject || DEFAULT_SUBJECT) + @replyto = Person.from_address (m.fetch_header (:reply_to)) ## before loading our full header from the source, we can actually ## have some extra refs set by the UI. (this happens when the user ## joins threads manually). so we will merge the current refs values ## in here. - refs = (header["references"] || "").scan(/<(.+?)>/).map { |x| sanitize_message_id x.first } - @refs = (@refs + refs).uniq - @replytos = (header["in-reply-to"] || "").scan(/<(.+?)>/).map { |x| sanitize_message_id x.first } - - @replyto = Person.from_address header["reply-to"] - @list_address = if header["list-post"] - address = if header["list-post"] =~ /mailto:(.*?)[>\s$]/ - $1 - elsif header["list-post"] =~ /@/ - header["list-post"] # just try the whole fucking thing - end - address && Person.from_address(address) - elsif header["x-mailing-list"] - Person.from_address header["x-mailing-list"] + begin + @refs += m.fetch_message_ids(:references) + @replytos = m.fetch_message_ids(:in_reply_to) + rescue Mail::Field::FieldError => e + raise InvalidMessageError, e.message end + @refs += @replytos unless @refs.member?(@replytos.first) + @refs = @refs.uniq # user may have set some refs manually + + @recipient_email = (m.fetch_header(:Envelope_to) || m.fetch_header(:x_original_to) || m.fetch_header(:delivered_to)) + + @list_subscribe = m.fetch_header(:list_subscribe) + @list_unsubscribe = m.fetch_header(:list_unsubscribe) + @list_address = Person.from_address(get_email_from_mailto(m.fetch_header(:list_post) || m.fetch_header(:x_mailing_list))) - @recipient_email = header["envelope-to"] || header["x-original-to"] || header["delivered-to"] - @source_marked_read = header["status"] == "RO" - @list_subscribe = header["list-subscribe"] - @list_unsubscribe = header["list-unsubscribe"] + @source_marked_read = m.fetch_header(:status) == 'RO' + end + + # get to field from mailto:example@example uri + def get_email_from_mailto field + return nil if field.nil? + u = URI + k = u.extract field, 'mailto' + a = u.parse k[0] + return a.to end ## Expected index entry format: @@ -259,10 +259,12 @@ def load_from_source! ## actually, it's also the differentiation between to/cc/bcc, ## so i will keep this. rmsg = location.parsed_message - parse_header rmsg.header + + parse_header rmsg message_to_chunks rmsg - rescue SourceError, SocketError, RMail::EncodingUnsupportedError => e + rescue SourceError, SocketError => e warn "problem reading message #{id}" + puts location.inspect [Chunk::Text.new(error_message.split("\n"))] debug "could not load message: #{location.inspect}, exception: #{e.inspect}" @@ -305,7 +307,7 @@ def indexable_content end def indexable_body - indexable_chunks.map { |c| c.lines }.flatten.compact.join " " + indexable_chunks.map { |c| c.lines.each { |l| l.fix_encoding} }.flatten.compact.join " " end def indexable_chunks @@ -373,14 +375,14 @@ def multipart_signed_to_chunks m end ## this probably will never happen - if payload.header.content_type && payload.header.content_type.downcase == "application/pgp-signature" - warn "multipart/signed with payload content type #{payload.header.content_type}" + if payload.content_type && payload.content_type.downcase == "application/pgp-signature" + warn "multipart/signed with payload content type #{payload.content_type}" return end - if signature.header.content_type && signature.header.content_type.downcase != "application/pgp-signature" + if signature.content_type && signature.content_type.downcase != "application/pgp-signature" ## unknown signature type; just ignore. - #warn "multipart/signed with signature content type #{signature.header.content_type}" + #warn "multipart/signed with signature content type #{signature.content_type}" return end @@ -399,13 +401,13 @@ def multipart_encrypted_to_chunks m return end - if payload.header.content_type && payload.header.content_type.downcase != "application/octet-stream" - warn "multipart/encrypted with payload content type #{payload.header.content_type}" + if payload.content_type && payload.content_type.downcase != "application/octet-stream" + warn "multipart/encrypted with payload content type #{payload.content_type}" return end - if control.header.content_type && control.header.content_type.downcase != "application/pgp-encrypted" - warn "multipart/encrypted with control content type #{signature.header.content_type}" + if control.content_type && control.content_type.downcase != "application/pgp-encrypted" + warn "multipart/encrypted with control content type #{signature.content_type}" return end @@ -418,128 +420,268 @@ def multipart_encrypted_to_chunks m end end - ## takes a RMail::Message, breaks it into Chunk:: classes. + ## takes Mail and breaks it into Chunk::s def message_to_chunks m, encrypted=false, sibling_types=[] - if m.multipart? - chunks = - case m.header.content_type.downcase - when "multipart/signed" - multipart_signed_to_chunks m - when "multipart/encrypted" - multipart_encrypted_to_chunks m - end + if encrypted + info "Encrypted message not implemented, skipping." + return [] + #raise NotImplementedError + end - unless chunks - sibling_types = m.body.map { |p| p.header.content_type } - chunks = m.body.map { |p| message_to_chunks p, encrypted, sibling_types }.flatten.compact - end + preferred_type = "text/plain" # TODO: im just gonna assume this is preferred + + chunks = [] + + mime_parts(m, preferred_type).each do |type, filename, id, content| + if type == "message/rfc822" + info "RFC822 messages not implemented, skipping." + next + #raise NotImplementedError + elsif type == "application/pgp" + info "Encrypted PGP message not implemented, skipping." + next + #raise NotImplementedError + else + ## if there is a filename, we'll treat it as an attachment + if filename + @attachments.push filename.downcase unless filename =~ /^sup-attachment-/ + add_label :attachment unless filename =~ /^sup-attachment-/ + + chunks << Chunk::Attachment.new(type, filename, content, sibling_types) - chunks - elsif m.header.content_type && m.header.content_type.downcase == "message/rfc822" - encoding = m.header["Content-Transfer-Encoding"] - if m.body - body = - case encoding - when "base64" - m.body.unpack("m")[0] - when "quoted-printable" - m.body.unpack("M")[0] - when "7bit", "8bit", nil - m.body else - raise RMail::EncodingUnsupportedError, encoding.inspect + body = content + + # check for inline PGP + #ch = inline_gpg_to_chunks body, $encoding, 'UTF-8' + #chunks << ch if ch + + text_to_chunks(body.normalize_whitespace.split("\n"), encrypted).each do |tc| + chunks << tc + end end - body = body.normalize_whitespace - payload = RMail::Parser.read(body) - from = payload.header.from.first ? payload.header.from.first.format : "" - to = payload.header.to.map { |p| p.format }.join(", ") - cc = payload.header.cc.map { |p| p.format }.join(", ") - subj = decode_header_field(payload.header.subject) || DEFAULT_SUBJECT - subj = Message.normalize_subj(subj.gsub(/\s+/, " ").gsub(/\s+$/, "")) - msgdate = payload.header.date - from_person = from ? Person.from_address(decode_header_field(from)) : nil - to_people = to ? Person.from_address_list(decode_header_field(to)) : nil - cc_people = cc ? Person.from_address_list(decode_header_field(cc)) : nil - [Chunk::EnclosedMessage.new(from_person, to_people, cc_people, msgdate, subj)] + message_to_chunks(payload, encrypted) - else - debug "no body for message/rfc822 enclosure; skipping" - [] - end - elsif m.header.content_type && m.header.content_type.downcase == "application/pgp" && m.body - ## apparently some versions of Thunderbird generate encryped email that - ## does not follow RFC3156, e.g. messages with X-Enigmail-Version: 0.95.0 - ## they have no MIME multipart and just set the body content type to - ## application/pgp. this handles that. - ## - ## TODO: unduplicate code between here and multipart_encrypted_to_chunks - notice, sig, decryptedm = CryptoManager.decrypt m.body - if decryptedm # managed to decrypt - children = message_to_chunks decryptedm, true - [notice, sig].compact + children - else - [notice] end - else - filename = - ## first, paw through the headers looking for a filename. - ## RFC 2183 (Content-Disposition) specifies that disposition-parms are - ## separated by ";". So, we match everything up to " and ; (if present). - if m.header["Content-Disposition"] && m.header["Content-Disposition"] =~ /filename="?(.*?[^\\])("|;|\z)/m - $1 - elsif m.header["Content-Type"] && m.header["Content-Type"] =~ /name="?(.*?[^\\])("|;|\z)/im - $1 - - ## haven't found one, but it's a non-text message. fake - ## it. - ## - ## TODO: make this less lame. - elsif m.header["Content-Type"] && m.header["Content-Type"] !~ /^text\/plain/i - extension = - case m.header["Content-Type"] - when /text\/html/ then "html" - when /image\/(.*)/ then $1 - end - - ["sup-attachment-#{Time.now.to_i}-#{rand 10000}", extension].join(".") - end + end - ## if there's a filename, we'll treat it as an attachment. - if filename - ## filename could be 2047 encoded - filename = Rfc2047.decode_to $encoding, filename - # add this to the attachments list if its not a generated html - # attachment (should we allow images with generated names?). - # Lowercase the filename because searches are easier that way - @attachments.push filename.downcase unless filename =~ /^sup-attachment-/ - add_label :attachment unless filename =~ /^sup-attachment-/ - content_type = (m.header.content_type || "application/unknown").downcase # sometimes RubyMail gives us nil - [Chunk::Attachment.new(content_type, filename, m, sibling_types)] - - ## otherwise, it's body text - else - ## Decode the body, charset conversion will follow either in - ## inline_gpg_to_chunks (for inline GPG signed messages) or - ## a few lines below (messages without inline GPG) - body = m.body ? m.decode : "" - - ## Check for inline-PGP - chunks = inline_gpg_to_chunks body, $encoding, (m.charset || $encoding) - return chunks if chunks - - if m.body - ## if there's no charset, use the current encoding as the charset. - ## this ensures that the body is normalized to avoid non-displayable - ## characters - body = m.decode.transcode($encoding, m.charset) + chunks + + end + + def mime_parts m, preferred_type + decode_mime_parts m, preferred_type + end + + def mime_part_types part + ptype = part.fetch_header(:content_type) + [ptype] + (part.multipart? ? part.body.parts.map { |sub| mime_part_types sub } : []) + end + + ## unnests all the mime stuff and returns a list of [type, filename, content] + ## tuples. + ## + ## for multipart/alternative parts, will only return the subpart that matches + ## preferred_type. if none of them, will only return the first subpart. + def decode_mime_parts part, preferred_type, level=0 + if part.multipart? + if mime_type_for(part) =~ /multipart\/alternative/ + target = part.body.parts.find { |p| mime_type_for(p).index(preferred_type) } || part.body.parts.first + if target # this can be nil + decode_mime_parts target, preferred_type, level + 1 else - body = "" + [] end - - text_to_chunks(body.normalize_whitespace.split("\n"), encrypted) + else # decode 'em all + part.body.parts.compact.map { |subpart| decode_mime_parts subpart, preferred_type, level + 1 }.flatten 1 end + else + type = mime_type_for part + filename = mime_filename_for part + id = mime_id_for part + content = mime_content_for part, preferred_type + [[type, filename, id, content]] + end + end + + def mime_type_for part + (part.fetch_header(:content_type) || "text/plain").gsub(/\s+/, " ").strip.downcase + end + + def mime_id_for part + header = part.fetch_header(:content_id) + case header + when /<(.+?)>/; $1 + else header end end + ## a filename, or nil + def mime_filename_for part + cd = part.fetch_header(:content_disposition) + ct = part.fetch_header(:content_type) + + ## RFC 2183 (Content-Disposition) specifies that disposition-parms are + ## separated by ";". So, we match everything up to " and ; (if present). + filename = if ct && ct =~ /name="?(.*?[^\\])("|;|\z)/im # find in content-type + $1 + elsif cd && cd =~ /filename="?(.*?[^\\])("|;|\z)/m # find in content-disposition + $1 + end + end + + + #CONVERSIONS = { + #["text/html", "text/plain"] => :html_to_text + #} + + def mime_content_for mime_part, preferred_type + return "" unless mime_part.body # sometimes this happens. not sure why. + + content_type = mime_part.fetch_header(:content_type) || "text/plain" + source_charset = mime_part.charset || "UTF-8" + + content = mime_part.decoded + #converted_content, converted_charset = if(converter = CONVERSIONS[[content_type, preferred_type]]) + #send converter, content, source_charset + #else + #[content, source_charset] + #end + + # decode + #if content_type =~ /^text\// + #Decoder.transcode "utf-8", converted_charset, converted_content + #else + #converted_content + #end + content + end + + def has_attachment? m + m.has_attachments? + end + + ## takes a RMail::Message, breaks it into Chunk:: classes. + #def _message_to_chunks m, encrypted=false, sibling_types=[] + #if m.multipart? + #chunks = + #case m.content_type.downcase + #when "multipart/signed" + #multipart_signed_to_chunks m + #when "multipart/encrypted" + #multipart_encrypted_to_chunks m + #end + + #unless chunks + #sibling_types = m.body.map { |p| p.content_type } + #chunks = m.body.map { |p| message_to_chunks p, encrypted, sibling_types }.flatten.compact + #end + + #chunks + #elsif m[:content_type] && m.fetch_header(:content_type).downcase == "message/rfc822" + #encoding = m.header["Content-Transfer-Encoding"] + #if m.body + #body = + #case encoding + #when "base64" + #m.body.unpack("m")[0] + #when "quoted-printable" + #m.body.unpack("M")[0] + #when "7bit", "8bit", nil + #m.body + #else + #raise RMail::EncodingUnsupportedError, encoding.inspect + #end + #body = body.normalize_whitespace + #payload = RMail::Parser.read(body) + #from = payload.header.from.first ? payload.header.from.first.format : "" + #to = payload.header.to.map { |p| p.format }.join(", ") + #cc = payload.header.cc.map { |p| p.format }.join(", ") + #subj = decode_header_field(payload.header.subject) || DEFAULT_SUBJECT + #subj = Message.normalize_subj(subj.gsub(/\s+/, " ").gsub(/\s+$/, "")) + #msgdate = payload.header.date + #from_person = from ? Person.from_address(decode_header_field(from)) : nil + #to_people = to ? Person.from_address_list(decode_header_field(to)) : nil + #cc_people = cc ? Person.from_address_list(decode_header_field(cc)) : nil + #[Chunk::EnclosedMessage.new(from_person, to_people, cc_people, msgdate, subj)] + message_to_chunks(payload, encrypted) + #else + #debug "no body for message/rfc822 enclosure; skipping" + #[] + #end + #elsif m[:content_type] && m.fetch_header(:content_type).downcase == "application/pgp" && m.body + ### apparently some versions of Thunderbird generate encryped email that + ### does not follow RFC3156, e.g. messages with X-Enigmail-Version: 0.95.0 + ### they have no MIME multipart and just set the body content type to + ### application/pgp. this handles that. + ### + ### TODO: unduplicate code between here and multipart_encrypted_to_chunks + #notice, sig, decryptedm = CryptoManager.decrypt m.body + #if decryptedm # managed to decrypt + #children = message_to_chunks decryptedm, true + #[notice, sig].compact + children + #else + #[notice] + #end + #else + #filename = + ### first, paw through the headers looking for a filename. + ### RFC 2183 (Content-Disposition) specifies that disposition-parms are + ### separated by ";". So, we match everything up to " and ; (if present). + #if m.header["Content-Disposition"] && m.header["Content-Disposition"] =~ /filename="?(.*?[^\\])("|;|\z)/m + #$1 + #elsif m.header["Content-Type"] && m.header["Content-Type"] =~ /name="?(.*?[^\\])("|;|\z)/im + #$1 + + ### haven't found one, but it's a non-text message. fake + ### it. + ### + ### TODO: make this less lame. + #elsif m.header["Content-Type"] && m.header["Content-Type"] !~ /^text\/plain/i + #extension = + #case m.header["Content-Type"] + #when /text\/html/ then "html" + #when /image\/(.*)/ then $1 + #end + + #["sup-attachment-#{Time.now.to_i}-#{rand 10000}", extension].join(".") + #end + + ### if there's a filename, we'll treat it as an attachment. + #if filename + ### filename could be 2047 encoded + #filename = Rfc2047.decode_to $encoding, filename + ## add this to the attachments list if its not a generated html + ## attachment (should we allow images with generated names?). + ## Lowercase the filename because searches are easier that way + #@attachments.push filename.downcase unless filename =~ /^sup-attachment-/ + #add_label :attachment unless filename =~ /^sup-attachment-/ + #content_type = (m.content_type || "application/unknown").downcase # sometimes RubyMail gives us nil + #[Chunk::Attachment.new(content_type, filename, m, sibling_types)] + + ### otherwise, it's body text + #else + ### Decode the body, charset conversion will follow either in + ### inline_gpg_to_chunks (for inline GPG signed messages) or + ### a few lines below (messages without inline GPG) + #body = m.body ? m.decoded : "" + + ### Check for inline-PGP + #chunks = inline_gpg_to_chunks body, $encoding, (m.charset || $encoding) + #return chunks if chunks + + #if m.body + ### if there's no charset, use the current encoding as the charset. + ### this ensures that the body is normalized to avoid non-displayable + ### characters + ##body = Iconv.easy_decode($encoding, m.charset || $encoding, m.decoded) + #body = m.body.decoded + #else + #body = "" + #end + + #text_to_chunks(body.normalize_whitespace.split("\n"), encrypted) + #end + #end + #end + ## looks for gpg signed (but not encrypted) inline messages inside the ## message body (there is no extra header for inline GPG) or for encrypted ## (and possible signed) inline GPG messages diff --git a/lib/sup/message_chunks.rb b/lib/sup/message_chunks.rb index f3f807b43..2392af12d 100644 --- a/lib/sup/message_chunks.rb +++ b/lib/sup/message_chunks.rb @@ -1,3 +1,5 @@ +# encoding: UTF-8 + require 'tempfile' require 'rbconfig' @@ -105,12 +107,7 @@ def initialize content_type, filename, encoded_content, sibling_types @filename = filename @quotable = false # changed to true if we can parse it through the # mime-decode hook, or if it's plain text - @raw_content = - if encoded_content.body - encoded_content.decode - else - "For some bizarre reason, RubyMail was unable to parse this attachment.\n" - end + @raw_content = encoded_content text = case @content_type when /^text\/plain\b/ @@ -118,13 +115,12 @@ def initialize content_type, filename, encoded_content, sibling_types else HookManager.run "mime-decode", :content_type => content_type, :filename => lambda { write_to_disk }, - :charset => encoded_content.charset, :sibling_types => sibling_types end @lines = nil if text - text = text.transcode(encoded_content.charset || $encoding, text.encoding) + text = text.transcode(encoded_content.encoding || $encoding, text.encoding) @lines = text.gsub("\r\n", "\n").gsub(/\t/, " ").gsub(/\r/, "").split("\n") @quotable = true end diff --git a/lib/sup/modes/edit_message_mode.rb b/lib/sup/modes/edit_message_mode.rb index e203ed0f4..9ce00fb6c 100644 --- a/lib/sup/modes/edit_message_mode.rb +++ b/lib/sup/modes/edit_message_mode.rb @@ -63,7 +63,7 @@ class EditMessageMode < LineCursorMode The message will be saved after this hook is run, so any modification to it will be recorded. Variables: - message: RMail::Message instance of the mail to send + message: Mail instance of the mail to send account: Account instance matching the From address Return value: True if mail has been sent successfully, false otherwise. @@ -105,7 +105,7 @@ def initialize opts={} begin hostname = File.open("/etc/mailname", "r").gets.chomp rescue - nil + nil end hostname = Socket.gethostname if hostname.nil? or hostname.empty? @@ -511,18 +511,19 @@ def save_as_draft end def build_message date - m = RMail::Message.new - m.header["Content-Type"] = "text/plain; charset=#{$encoding}" - m.body = @body.join("\n") - m.body += "\n" + sig_lines.join("\n") unless @sig_edited + m = Mail.new + m.content_type = "text/plain; charset=#{$encoding}" + bb = @body.join ("\n") + bb += "\n" + sig_lines.join("\n") unless @sig_edited ## body must end in a newline or GPG signatures will be WRONG! - m.body += "\n" unless m.body =~ /\n\Z/ + bb += "\n" unless bb =~ /\n\Z/ + m.body = bb ## there are attachments, so wrap body in an attachment of its own unless @attachments.empty? body_m = m - body_m.header["Content-Disposition"] = "inline" - m = RMail::Message.new + body_m.content_disposition = "inline" + m = Mail.new m.add_part body_m @attachments.each { |a| m.add_part a } @@ -533,7 +534,7 @@ def build_message date from_email = Person.from_address(@header["From"]).email to_email = [@header["To"], @header["Cc"], @header["Bcc"]].flatten.compact.map { |p| Person.from_address(p).email } if m.multipart? - m.each_part {|p| p = transfer_encode p} + m.parts.each {|p| p = transfer_encode p} else m = transfer_encode m end diff --git a/lib/sup/modes/forward_mode.rb b/lib/sup/modes/forward_mode.rb index 5d1ec58b1..35f396ec8 100644 --- a/lib/sup/modes/forward_mode.rb +++ b/lib/sup/modes/forward_mode.rb @@ -7,9 +7,10 @@ def initialize opts={} "From" => AccountManager.default_account.full_address, } + @m = opts[:message] header["Subject"] = - if opts[:message] - "Fwd: " + opts[:message].subj + if @m + "Fwd: " + @m.subj elsif opts[:attachments] "Fwd: " + opts[:attachments].keys.join(", ") end @@ -19,8 +20,8 @@ def initialize opts={} header["Bcc"] = opts[:bcc].map { |p| p.full_address }.join(", ") if opts[:bcc] body = - if opts[:message] - forward_body_lines(opts[:message]) + if @m + forward_body_lines @m elsif opts[:attachments] ["Note: #{opts[:attachments].size.pluralize 'attachment'}."] end diff --git a/lib/sup/sent.rb b/lib/sup/sent.rb index 05d9a8a05..f859be8db 100644 --- a/lib/sup/sent.rb +++ b/lib/sup/sent.rb @@ -44,6 +44,10 @@ def initialize super "mbox://" + @filename, true, $config[:archive_sent] end + def init_with coder + initialize + end + def file_path; @filename end def to_s; 'sup://sent'; end diff --git a/lib/sup/thread.rb b/lib/sup/thread.rb index f1414bad1..20d5c80c2 100644 --- a/lib/sup/thread.rb +++ b/lib/sup/thread.rb @@ -1,3 +1,5 @@ +# encoding: UTF-8 +# ## Herein lies all the code responsible for threading messages. It's ## basically an online version of the JWZ threading algorithm: ## http://www.jwz.org/doc/threading.html diff --git a/lib/sup/util.rb b/lib/sup/util.rb index bf33dcff2..efc97b04f 100644 --- a/lib/sup/util.rb +++ b/lib/sup/util.rb @@ -9,6 +9,36 @@ require 'benchmark' require 'unicode' +module Mail + class Message + # a common interface that matches all the field + # IMPORTANT: if not existing, it must return nil + def fetch_header field + sym = field.to_sym + begin + self[sym] ? self[sym].to_s : nil + rescue + info "Error while fetching header field: #{field}." + nil + end + end + + # make sure the message has valid message ids for the message, and + # fetch them + def fetch_message_ids field + if self[field] + begin + self[field].message_ids || [] + rescue + [] + end + else + [] + end + end + end +end + ## time for some monkeypatching! class Symbol unless method_defined? :to_proc @@ -65,101 +95,6 @@ def human_time end end -## more monkeypatching! -module RMail - class EncodingUnsupportedError < StandardError; end - - class Message - def self.make_file_attachment fn - bfn = File.basename fn - t = MIME::Types.type_for(bfn).first || MIME::Types.type_for("exe").first - make_attachment IO.read(fn), t.content_type, t.encoding, bfn.to_s - end - - def charset - if header.field?("content-type") && header.fetch("content-type") =~ /charset="?(.*?)"?(;|$)/i - $1 - end - end - - def self.make_attachment payload, mime_type, encoding, filename - a = Message.new - a.header.add "Content-Disposition", "attachment; filename=#{filename.inspect}" - a.header.add "Content-Type", "#{mime_type}; name=#{filename.inspect}" - a.header.add "Content-Transfer-Encoding", encoding if encoding - a.body = - case encoding - when "base64" - [payload].pack "m" - when "quoted-printable" - [payload].pack "M" - when "7bit", "8bit", nil - payload - else - raise EncodingUnsupportedError, encoding.inspect - end - a - end - end - - class Serialize - ## Don't add MIME-Version headers on serialization. Sup sometimes want's to serialize - ## message parts where these headers are not needed and messing with the message on - ## serialization breaks gpg signatures. The commented section shows the original RMail - ## code. - def calculate_boundaries(message) - calculate_boundaries_low(message, []) - # unless message.header['MIME-Version'] - # message.header['MIME-Version'] = "1.0" - # end - end - end - - class Header - - # Convert to ASCII before trying to match with regexp - class Field - - EXTRACT_FIELD_NAME_RE = /\A([^\x00-\x1f\x7f-\xff :]+):\s*/no - - class << self - def parse(field) - field = field.dup.to_s - field = field.fix_encoding.ascii - if field =~ EXTRACT_FIELD_NAME_RE - [ $1, $'.chomp ] - else - [ "", Field.value_strip(field) ] - end - end - end - end - - ## Be more cautious about invalid content-type headers - ## the original RMail code calls - ## value.strip.split(/\s*;\s*/)[0].downcase - ## without checking if split returned an element - - # This returns the full content type of this message converted to - # lower case. - # - # If there is no content type header, returns the passed block is - # executed and its return value is returned. If no block is passed, - # the value of the +default+ argument is returned. - def content_type(default = nil) - if value = self['content-type'] and ct = value.strip.split(/\s*;\s*/)[0] - return ct.downcase - else - if block_given? - yield - else - default - end - end - end - end -end - class Range ## only valid for integer ranges (unless I guess it's exclusive) def size diff --git a/sup.gemspec b/sup.gemspec index 0d3f1b83c..d0effe565 100644 --- a/sup.gemspec +++ b/sup.gemspec @@ -6,7 +6,7 @@ require 'sup/version' # Files SUP_EXECUTABLES = %w(sup sup-add sup-config sup-dump sup-import-dump sup-recover-sources sup-sync sup-sync-back sup-tweak-labels - sup-psych-ify-config-files) + sup-migrate-index sup-psych-ify-config-files) SUP_EXTRA_FILES = %w(CONTRIBUTORS README.md LICENSE History.txt ReleaseNotes) SUP_FILES = SUP_EXTRA_FILES + @@ -45,7 +45,7 @@ SUP: Please run `sup-psych-ify-config-files` to migrate from 0.13 to 0.14 s.add_runtime_dependency "xapian-ruby", "~> 1.2.15" s.add_runtime_dependency "ncursesw-sup", "~> 1.3.1" - s.add_runtime_dependency "rmail", ">= 0.17" + s.add_runtime_dependency "mail", "~> 2.5" s.add_runtime_dependency "highline" s.add_runtime_dependency "trollop", ">= 1.12" s.add_runtime_dependency "lockfile" diff --git a/test/dummy_source.rb b/test/dummy_source.rb index abedf7133..36df8f5ae 100644 --- a/test/dummy_source.rb +++ b/test/dummy_source.rb @@ -1,8 +1,6 @@ #!/usr/bin/ruby -require 'sup' require 'stringio' -require 'rmail' require 'uri' module Redwood @@ -30,7 +28,7 @@ def load_header offset end def load_message offset - RMail::Parser.read raw_message(offset) + Mail.read_from_string raw_message(offset) end def raw_header offset diff --git a/test/test_helper.rb b/test/test_helper.rb index adb09b729..1e679c994 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,3 +1,4 @@ +require 'sup' require "rubygems" rescue nil require 'minitest/autorun' require "rr" diff --git a/test/test_message.rb b/test/test_message.rb index 62831746b..efd62baed 100644 --- a/test/test_message.rb +++ b/test/test_message.rb @@ -1,7 +1,6 @@ #!/usr/bin/ruby require 'test_helper' -require 'sup' require 'stringio' require 'dummy_source' @@ -514,8 +513,7 @@ def test_blank_header_lines # Look at another header field whose first line was blank. list_unsubscribe = sup_message.list_unsubscribe - assert_equal(",\n \t" + - "", + assert_equal(", ", list_unsubscribe) end From 1dab80b4645c223cd47da9ab4d0c4e3173809476 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Mon, 12 Aug 2013 18:00:01 +0200 Subject: [PATCH 02/12] @source_starred does not belong here --- lib/sup/message.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/sup/message.rb b/lib/sup/message.rb index 50238afbb..880c74cef 100644 --- a/lib/sup/message.rb +++ b/lib/sup/message.rb @@ -84,7 +84,6 @@ def parse_header m # @list_address # @receipient_email # @source_marked_read - # @source_starred # @list_subscribe # @list_unsubscribe From 9e9d39b2febd1baa8c5513a0ab7ede3786faefec Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Mon, 12 Aug 2013 18:24:07 +0200 Subject: [PATCH 03/12] use Digest::MD5 directly --- lib/sup/message.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sup/message.rb b/lib/sup/message.rb index 880c74cef..0d3f31c91 100644 --- a/lib/sup/message.rb +++ b/lib/sup/message.rb @@ -88,7 +88,7 @@ def parse_header m # @list_unsubscribe unless m.message_id - m.message_id = "<#{Time.now.to_i}-defaulted-#{munge_msgid m.header.to_s}@sup-faked>" + m.message_id = "<#{Time.now.to_i}-defaulted-#{Digest::MD5.hexdigest m.header.to_s}@sup-faked>" end @id = m.message_id From 54dad4d9f55b9cb51adfdbac294958cdc68b9e06 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Mon, 12 Aug 2013 20:27:07 +0200 Subject: [PATCH 04/12] Don't re-create auto-generated (invalid) fields, but use field from index --- lib/sup/message.rb | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/lib/sup/message.rb b/lib/sup/message.rb index 0d3f31c91..dc45a3812 100644 --- a/lib/sup/message.rb +++ b/lib/sup/message.rb @@ -64,6 +64,9 @@ def initialize opts ## we need to initialize this. see comments in parse_header as to ## why. + @date = nil # may or may not have been set when the message was loaded + # or discovered for the first time. + @id = nil # same as @date for invalid ids @refs = [] #parse_header(opts[:header] || @source.load_header(@source_info)) @@ -88,21 +91,29 @@ def parse_header m # @list_unsubscribe unless m.message_id - m.message_id = "<#{Time.now.to_i}-defaulted-#{Digest::MD5.hexdigest m.header.to_s}@sup-faked>" + m.message_id = @id || "<#{Time.now.to_i}-defaulted-#{Digest::MD5.hexdigest m.header.to_s}@sup-faked>" + debug "Using fake id (newly created or existing): #{id} for message located at: #{location.inspect}." end - @id = m.message_id + @id = sanitize_message_id m.message_id @from = Person.from_address m.fetch_header(:from) @from = Person.from_address "Sup Auto-generated Fake Sender " unless @from @sender = Person.from_address m.fetch_header(:sender) begin - @date = (m.date || Time.now).to_time + if m.date + @date = m.date.to_time + else + warn "Invalid date for message #{@id}, using 'now' or previously fake 'now'." + BufferManager.flash "Invalid date for message #{@id}, using 'now' or previously fake 'now'." if BufferManager.instantiated? + @date = @date || Time.now.to_time + end rescue NoMethodError # TODO: remove this rescue once mail/#564 is fixed - warn "Invalid date for message #{@id}, using 'now'." - @date = Time.now.to_time + warn "Invalid date for message #{@id}, using 'now' or previously fake 'now'." + BufferManager.flash "Invalid date for message #{@id}, using 'now' or previously fake 'now'." if BufferManager.instantiated? + @date = (@date || Time.now.to_time) end @to = Person.from_address_list (m.fetch_header (:to)) From cd52a3e39c3aa034f02464eb95a5567dff9df022 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Thu, 15 Aug 2013 17:39:23 +0200 Subject: [PATCH 05/12] Temporarily disable crypto on Mail branch --- lib/sup/crypto.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/sup/crypto.rb b/lib/sup/crypto.rb index 8f042cd5a..4faec8785 100644 --- a/lib/sup/crypto.rb +++ b/lib/sup/crypto.rb @@ -57,6 +57,7 @@ def initialize @mutex = Mutex.new @not_working_reason = nil + @not_working_reason = "The crypto code needs to be ported to use Mail not RMail" # test if the gpgme gem is available @gpgme_present = From 48adfffb270ab8e9a05641b3a01ec9c1dc0129ba Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Thu, 15 Aug 2013 17:45:10 +0200 Subject: [PATCH 06/12] rm migration script: with same message id Mail msg-id conform to old RMail ids and we do not need to migrate --- bin/sup-migrate-index | 110 ------------------------------------------ lib/sup/index.rb | 5 +- 2 files changed, 1 insertion(+), 114 deletions(-) delete mode 100755 bin/sup-migrate-index diff --git a/bin/sup-migrate-index b/bin/sup-migrate-index deleted file mode 100755 index ff63f732d..000000000 --- a/bin/sup-migrate-index +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env ruby - -require 'uri' -require 'rubygems' -require 'trollop' -require "sup"; Redwood::check_library_version_against "git" - -PROGRESS_UPDATE_INTERVAL = 15 # seconds - -class AbortExecution < SystemExit -end - -opts = Trollop::options do - version "sup-migrate-index (sup #{Redwood::VERSION})" - banner < "-n" - opt :version, "Show version information", :short => :none - - conflicts :ignore_missing, :warn_missing, :abort_missing -end -Trollop::die "Extra arguments given" if ARGV.length > 1 -missing_action = [:ignore_missing, :warn_missing, :abort_missing].find { |x| opts[x] } || :abort_missing - -BASE_DIR = ENV["SUP_BASE"] || File.join(ENV["HOME"], ".sup") - - -Redwood::start -index = Redwood::Index.init - -#index.lock_interactively or exit - - -info "Starting migration to version 5.." - -index.load true - -xapian = index.get_xapian -db_version = xapian.get_metadata 'version' -db_version = '0' if db_version.empty? - -fail "It only makes sense to migrate from version 4 to 5, you have an index version #{db_version}" unless db_version == '4' - -num_docs = 0 -xapian.postlist('Kmail').each do |x| - info "Updating document: #{x.inspect}" - num_docs += 1 - - doc = xapian.document(x.docid) - entry = doc.entry - - debug "Working on entry: #{entry[:message_id]}." - - debug "Adding missing fields.." - entry[:safe_id] = Redwood::Message.munge_msgid(entry[:message_id]) - entry[:safe_refs] += entry[:refs].nil? ? [] : entry[:refs].compact.map { |r| rr = Redwood::Message.munge_msgid(r); debug "addin ref #{r} -> #{rr}.."; rr } - entry[:safe_refs] = entry[:safe_refs].uniq - - - entry[:safe_replytos] = entry[:replytos].nil? ? [] : entry[:replytos].compact.map { |r| rr = Redwood::Message.munge_msgid(r); debug "addin replyto #{r} -> #{rr}.."; rr } - - next if opts[:dry_run] - doc.entry = entry - - info "Storing message.." - xapian.replace_document x.docid, doc - k = xapian.document x.docid - info "stored safe_id: " + k.value(0) - - debug "Loading message.." - #locations = entry[:locations].map do |source_id,source_info| - #source = Redwood::SourceManager[source_id] - #raise "invalid source #{source_id}" unless source - #Redwood::Location.new source, source_info - #end - - #m = Redwood::Message.new :locations => locations, - #:labels => entry[:labels], - #:snippet => entry[:snippet] - - ## Try to find person from contacts before falling back to - ## generating it from the address. - #mk_person = lambda { |x| Redwood::Person.from_name_and_email(*x.reverse!) } - #entry[:from] = mk_person[entry[:from]] - #entry[:to].map!(&mk_person) - #entry[:cc].map!(&mk_person) - #entry[:bcc].map!(&mk_person) - - #m.load_from_index! entry - #m.load_from_source! - - break - - -end - -info "done, updated: #{num_docs} documents." - diff --git a/lib/sup/index.rb b/lib/sup/index.rb index 62392d860..a895f1083 100644 --- a/lib/sup/index.rb +++ b/lib/sup/index.rb @@ -32,7 +32,7 @@ module Redwood class Index include InteractiveLock - INDEX_VERSION = '5' + INDEX_VERSION = '4' ## dates are converted to integers for xapian, and are used for document ids, ## so we must ensure they're reasonably valid. this typically only affect @@ -124,9 +124,6 @@ def load_index failsafe=false info "Upgrading index format #{db_version} to #{INDEX_VERSION}" @xapian.set_metadata 'version', INDEX_VERSION - elsif (db_version == '4') - fail "This Sup has a new index version v#{INDEX_VERSION}, but you have v#{db_version}. If you have just upgraded Sup there has been a major change in the index format and a migration tool need to be run. Please first back up your existing index using sup-dump and back up #{path}, then run sup-migrate-index to upgrade it." unless failsafe - elsif db_version != INDEX_VERSION fail "This Sup version expects a v#{INDEX_VERSION} index, but you have an existing v#{db_version} index. Please run sup-dump to save your labels, move #{path} out of the way, and run sup-sync --restore." unless failsafe end From 9b8a9f116fb9b3c6d2d68925f2672bb3472e0216 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Thu, 15 Aug 2013 18:55:00 +0200 Subject: [PATCH 07/12] rm sup-migrate-index from SUP_FILES --- sup.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sup.gemspec b/sup.gemspec index e23c05728..81a660338 100644 --- a/sup.gemspec +++ b/sup.gemspec @@ -6,7 +6,7 @@ require 'sup/version' # Files SUP_EXECUTABLES = %w(sup sup-add sup-config sup-dump sup-import-dump sup-recover-sources sup-sync sup-sync-back sup-tweak-labels - sup-migrate-index sup-psych-ify-config-files) + sup-psych-ify-config-files) SUP_EXTRA_FILES = %w(CONTRIBUTORS README.md LICENSE History.txt ReleaseNotes) SUP_FILES = SUP_EXTRA_FILES + From 87dcfdf3029ad67ad07c06bc80f31c0f9d7a42b3 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Mon, 19 Aug 2013 09:57:46 +0200 Subject: [PATCH 08/12] init_with is defined dynamically in util.rb --- lib/sup/maildir.rb | 4 ---- lib/sup/mbox.rb | 4 ---- lib/sup/sent.rb | 4 ---- 3 files changed, 12 deletions(-) diff --git a/lib/sup/maildir.rb b/lib/sup/maildir.rb index e6fcfb6d1..2f5e44dc3 100644 --- a/lib/sup/maildir.rb +++ b/lib/sup/maildir.rb @@ -23,10 +23,6 @@ def initialize uri, usual=true, archived=false, id=nil, labels=[] @mtimes = { 'cur' => Time.at(0), 'new' => Time.at(0) } end - def init_with coder - initialize coder['uri'], coder['usual'], coder['archived'], coder['id'], coder['labels'] - end - def file_path; @dir end def self.suggest_labels_for path; [] end def is_source_for? uri; super || (uri == @expanded_uri); end diff --git a/lib/sup/mbox.rb b/lib/sup/mbox.rb index e2d428f3e..5933bf7b3 100644 --- a/lib/sup/mbox.rb +++ b/lib/sup/mbox.rb @@ -34,10 +34,6 @@ def initialize uri_or_fp, usual=true, archived=false, id=nil, labels=nil super uri_or_fp, usual, archived, id end - def init_with coder - initialize coder['uri'], coder['usual'], coder['archived'], coder['id'], coder['labels'] - end - def file_path; @path end def is_source_for? uri; super || (uri == @expanded_uri) end diff --git a/lib/sup/sent.rb b/lib/sup/sent.rb index f859be8db..05d9a8a05 100644 --- a/lib/sup/sent.rb +++ b/lib/sup/sent.rb @@ -44,10 +44,6 @@ def initialize super "mbox://" + @filename, true, $config[:archive_sent] end - def init_with coder - initialize - end - def file_path; @filename end def to_s; 'sup://sent'; end From b9892186de47327957ba1691a15f43669df4ba74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20B=C3=A4hr?= Date: Mon, 17 Mar 2014 20:04:27 +0800 Subject: [PATCH 09/12] Mail throws an error UnknownEncodingType --- lib/sup/message.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sup/message.rb b/lib/sup/message.rb index 675f50fe8..b0c35ce56 100644 --- a/lib/sup/message.rb +++ b/lib/sup/message.rb @@ -272,7 +272,7 @@ def load_from_source! parse_header rmsg message_to_chunks rmsg - rescue SourceError, SocketError => e + rescue SourceError, SocketError, Mail::UnknownEncodingType => e warn "problem reading message #{id}" debug "could not load message: #{location.inspect}, exception: #{e.inspect}" From 5e36249b4f6e038e796f6a08369ee9ef767e31f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20B=C3=A4hr?= Date: Sat, 22 Mar 2014 13:12:29 +0800 Subject: [PATCH 10/12] rename Singleton to SupSingleton to sidestep conflicts with ruby's Singleton --- lib/sup/account.rb | 2 +- lib/sup/buffer.rb | 2 +- lib/sup/contact.rb | 2 +- lib/sup/crypto.rb | 2 +- lib/sup/draft.rb | 2 +- lib/sup/hook.rb | 2 +- lib/sup/idle.rb | 2 +- lib/sup/index.rb | 2 +- lib/sup/label.rb | 2 +- lib/sup/logger.rb | 2 +- lib/sup/poll.rb | 2 +- lib/sup/search.rb | 2 +- lib/sup/sent.rb | 2 +- lib/sup/source.rb | 2 +- lib/sup/undo.rb | 2 +- lib/sup/update.rb | 2 +- lib/sup/util.rb | 2 +- lib/sup/util/ncurses.rb | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/sup/account.rb b/lib/sup/account.rb index 7fc1aa8b1..c637dd670 100644 --- a/lib/sup/account.rb +++ b/lib/sup/account.rb @@ -25,7 +25,7 @@ def bounce_sendmail end class AccountManager - include Singleton + include SupSingleton attr_accessor :default_account diff --git a/lib/sup/buffer.rb b/lib/sup/buffer.rb index 74d15672e..b2d0df6e1 100644 --- a/lib/sup/buffer.rb +++ b/lib/sup/buffer.rb @@ -99,7 +99,7 @@ def blur end class BufferManager - include Singleton + include SupSingleton attr_reader :focus_buf diff --git a/lib/sup/contact.rb b/lib/sup/contact.rb index 82c059edb..6e1e6e064 100644 --- a/lib/sup/contact.rb +++ b/lib/sup/contact.rb @@ -3,7 +3,7 @@ module Redwood class ContactManager - include Singleton + include SupSingleton def initialize fn @fn = fn diff --git a/lib/sup/crypto.rb b/lib/sup/crypto.rb index 728fcc764..1289bf260 100644 --- a/lib/sup/crypto.rb +++ b/lib/sup/crypto.rb @@ -6,7 +6,7 @@ module Redwood class CryptoManager - include Singleton + include SupSingleton class Error < StandardError; end diff --git a/lib/sup/draft.rb b/lib/sup/draft.rb index 27dd556ba..061731650 100644 --- a/lib/sup/draft.rb +++ b/lib/sup/draft.rb @@ -1,7 +1,7 @@ module Redwood class DraftManager - include Singleton + include SupSingleton attr_accessor :source def initialize dir diff --git a/lib/sup/hook.rb b/lib/sup/hook.rb index 31d7921ee..b418cf3fd 100644 --- a/lib/sup/hook.rb +++ b/lib/sup/hook.rb @@ -69,7 +69,7 @@ def __run __hook, __filename, __locals end end - include Singleton + include SupSingleton @descs = {} diff --git a/lib/sup/idle.rb b/lib/sup/idle.rb index a3a272f20..afcb6a3ca 100644 --- a/lib/sup/idle.rb +++ b/lib/sup/idle.rb @@ -3,7 +3,7 @@ module Redwood class IdleManager - include Singleton + include SupSingleton IDLE_THRESHOLD = 60 diff --git a/lib/sup/index.rb b/lib/sup/index.rb index 79bebbae9..14ca3987d 100644 --- a/lib/sup/index.rb +++ b/lib/sup/index.rb @@ -55,7 +55,7 @@ def initialize h def method_missing m; @h[m.to_s] end end - include Singleton + include SupSingleton def initialize dir=BASE_DIR @dir = dir diff --git a/lib/sup/label.rb b/lib/sup/label.rb index 3cdfa1484..b396b7b8c 100644 --- a/lib/sup/label.rb +++ b/lib/sup/label.rb @@ -3,7 +3,7 @@ module Redwood class LabelManager - include Singleton + include SupSingleton ## labels that have special semantics. user will be unable to ## add/remove these via normal label mechanisms. diff --git a/lib/sup/logger.rb b/lib/sup/logger.rb index 514b78e88..348045753 100644 --- a/lib/sup/logger.rb +++ b/lib/sup/logger.rb @@ -8,7 +8,7 @@ module Redwood ## also keeps a record of all messages, so that adding a new sink will send all ## previous messages to it by default. class Logger - include Singleton + include SupSingleton LEVELS = %w(debug info warn error) # in order! diff --git a/lib/sup/poll.rb b/lib/sup/poll.rb index 0ed44962f..d1ad261c9 100644 --- a/lib/sup/poll.rb +++ b/lib/sup/poll.rb @@ -3,7 +3,7 @@ module Redwood class PollManager - include Singleton + include SupSingleton HookManager.register "before-add-message", < Date: Sat, 22 Mar 2014 11:31:55 +0100 Subject: [PATCH 11/12] test: binary-content-transfer-encoding should not fail anymore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As suggested by Martin Bähr. --- test/messages/binary-content-transfer-encoding-2.eml | 1 + test/test_messages_dir.rb | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/messages/binary-content-transfer-encoding-2.eml b/test/messages/binary-content-transfer-encoding-2.eml index 142f8d8cb..29aaa861f 100644 --- a/test/messages/binary-content-transfer-encoding-2.eml +++ b/test/messages/binary-content-transfer-encoding-2.eml @@ -10,6 +10,7 @@ Content-Type: text/plain; charset="ISO-8859-1" Content-Disposition: inline Content-Transfer-Encoding: quoted-printable +test --======11647==82899====== Content-Type: message/rfc822 diff --git a/test/test_messages_dir.rb b/test/test_messages_dir.rb index 6341559a4..6f25f0490 100644 --- a/test/test_messages_dir.rb +++ b/test/test_messages_dir.rb @@ -64,12 +64,11 @@ def test_binary_content_transfer_encoding indexable_chunks = sup_message.indexable_chunks # there should be only one chunk - #assert_equal(1, chunks.length) - + assert_equal(1, chunks.length) lines = chunks[0].lines # lines should contain an error message - assert (lines.join.include? "An error occurred while loading this message."), "This message should not load successfully" + assert (lines.join == "test"), "The message could not be loaded successfully" end def test_bad_content_transfer_encoding From 3cbbc7a4661de440489892b6eede17047f1f450a Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Sat, 22 Mar 2014 11:54:40 +0100 Subject: [PATCH 12/12] missing excl on fix_encoding --- lib/sup/message.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sup/message.rb b/lib/sup/message.rb index b0c35ce56..e18b45172 100644 --- a/lib/sup/message.rb +++ b/lib/sup/message.rb @@ -342,7 +342,7 @@ def indexable_content end def indexable_body - indexable_chunks.map { |c| c.lines.each { |l| l.fix_encoding} }.flatten.compact.join " " + indexable_chunks.map { |c| c.lines.each { |l| l.fix_encoding!} }.flatten.compact.join " " end def indexable_chunks