Applying two small fixes:

dpkt_http_replacement.py

@@ -29,8 +29,20 @@ def parse_headers(f):
             d[k] = v
     return d
 
-def parse_body(f, headers):
-    """Return HTTP body parsed from a file object, given HTTP header dict."""
+def parse_body(f, headers, strict=True):
+    """Return HTTP body parsed from a file object, given HTTP header dict.
+
+    Args:
+      f: File object to parse packets from.
+      headers: Dict containing HTTP headers.
+      strict: Whether the body size must match the content-length header.
+
+    Returns:
+      The parsed HTTP body.
+
+    Raises:
+      dpkt.NeedData: If the body is incomplete.
+    """
     if headers.get('transfer-encoding', '').lower() == 'chunked':
         l = []
         found_end = False
@@ -55,7 +67,7 @@ def parse_body(f, headers):
     elif 'content-length' in headers:
         n = int(headers['content-length'])
         body = f.read(n)
-        if len(body) != n:
+        if strict and len(body) != n:
             raise dpkt.NeedData('short body (missing %d bytes)' % (n - len(body)))
     else:
         # XXX - need to handle HTTP/0.9
@@ -85,7 +97,7 @@ def unpack(self, buf):
         # Parse headers
         self.headers = parse_headers(f)
         # Parse body
-        self.body = parse_body(f, self.headers)
+        self.body = parse_body(f, self.headers, strict=False)
         # Save the rest
         self.data = f.read()
 

http/flow.py

@@ -88,6 +88,7 @@ def gather_messages(MessageClass, tcpdir):
         try:
             msg = MessageClass(tcpdir, pointer)
         except dpkt.Error as error: # if the message failed
+            logging.exception('dpkt Error parsing HTTP')
             if pointer == 0: # if this is the first message
                 raise http.Error('Invalid http')
             else: # we're done parsing messages

http/request.py

@@ -21,4 +21,3 @@ def __init__(self, tcpdir, pointer):
         self.fullurl = fullurl.geturl()
         self.url, frag = urlparse.urldefrag(self.fullurl)
         self.query = urlparse.parse_qs(uri.query, keep_blank_values=True)
-

http/response.py

@@ -73,6 +73,10 @@ def handle_compression(self):
             elif encoding == 'identity':
                 # no compression
                 self.body = self.raw_body
+            elif 'sdch' in encoding:
+                # ignore sdch, a Google proposed modification to HTTP/1.1
+                # not in RFC 2616.
+                self.body = self.raw_body
             else:
                 # I'm pretty sure the above are the only allowed encoding types
                 # see RFC 2616 sec 3.5 (http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.5)

httpsession.py

@@ -10,6 +10,8 @@
 import logging as log
 import settings
 
+print 'HI THERE MDW YOU GOT THE RIGHT VERSION'
+
 class Entry:
     '''
     represents an HTTP request/response in a form suitable for writing to a HAR

main.py

@@ -28,7 +28,8 @@
 settings.process_pages = options.pages
 
 # setup logs
-logging.basicConfig(filename='pcap2har.log', level=logging.INFO)
+#logging.basicConfig(filename='pcap2har.log', level=logging.INFO)
+logging.basicConfig(level=logging.DEBUG)
 
 # get filenames, or bail out with usage error
 if len(args) == 2:

tcp/flow.py

@@ -30,13 +30,21 @@ def add(self, pkt):
         called for every packet coming in, instead of iterating through
         a list
         '''
-        # make sure packet is in time order
-        if len(self.packets): # if we have received packets before...
-            if self.packets[-1].ts > pkt.ts: # if this one is out of order...
-                # error out
-                raise ValueError("packet added to tcp.Flow out of "
-                                 "chronological order")
-        self.packets.append(pkt)
+        # maintain an invariant that packets are ordered by ts;
+        # perform ordered insertion (as in insertion sort) if they're
+        # not in order because sometimes libpcap writes packets out of
+        # order.
+
+        # the correct position for pkt is found by looping i from
+        # len(self.packets) descending back to 0 (inclusive);
+        # normally, this loop will only run for one iteration.
+        for i in xrange(len(self.packets), -1, -1):
+            # pkt is at the correct position if it is at the
+            # beginning, or if it is >= the packet at its previous
+            # position.
+            if i == 0 or self.packets[i - 1].ts <= pkt.ts: break
+        self.packets.insert(i, pkt)
+
         # look out for handshake
         # add it to the appropriate direction, if we've found or given up on
         # finding handshake