Skip to content

Commit

Permalink
Close issue #500
Browse files Browse the repository at this point in the history 
Add the following general configuration options:

- `always_use_local_files` - Disables the download of the reverse DNS map
- `local_reverse_dns_map_path` - Overrides the default local file path to use for the reverse DNS map
  • Loading branch information
@seanthegeek
seanthegeek committed Apr 1, 2024
1 parent 041296b commit fd5b792
Show file tree
Hide file tree
Showing 3 changed files with 160 additions and 53 deletions.
148 changes: 106 additions & 42 deletions parsedmarc/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,14 +72,21 @@ class InvalidForensicReport(InvalidDMARCReport):
"""Raised when an invalid DMARC forensic report is encountered"""


def _parse_report_record(record, ip_db_path=None, offline=False,
def _parse_report_record(record, ip_db_path=None,
always_use_local_files=False,
reverse_dns_map_path=None,
reverse_dns_map_url=None,
offline=False,
nameservers=None, dns_timeout=2.0):
"""
Converts a record from a DMARC aggregate report into a more consistent
format
Args:
record (OrderedDict): The record to convert
always_use_local_files (bool): Do not download files
reverse_dns_map_path (str): Path to a reverse DNS map file
reverse_dns_map_url (str): URL to a reverse DNS map file
ip_db_path (str): Path to a MMDB file from MaxMind or DBIP
offline (bool): Do not query online for geolocation or DNS
nameservers (list): A list of one or more nameservers to use
Expand All @@ -91,12 +98,16 @@ def _parse_report_record(record, ip_db_path=None, offline=False,
"""
record = record.copy()
new_record = OrderedDict()
new_record_source = get_ip_address_info(record["row"]["source_ip"],
cache=IP_ADDRESS_CACHE,
ip_db_path=ip_db_path,
offline=offline,
nameservers=nameservers,
timeout=dns_timeout)
new_record_source = get_ip_address_info(
record["row"]["source_ip"],
cache=IP_ADDRESS_CACHE,
ip_db_path=ip_db_path,
always_use_local_files=always_use_local_files,
reverse_dns_map_path=reverse_dns_map_path,
reverse_dns_map_url=reverse_dns_map_url,
offline=offline,
nameservers=nameservers,
timeout=dns_timeout)
new_record["source"] = new_record_source
new_record["count"] = int(record["row"]["count"])
policy_evaluated = record["row"]["policy_evaluated"].copy()
Expand Down Expand Up @@ -387,14 +398,24 @@ def parsed_smtp_tls_reports_to_csv(reports):
return csv_file_object.getvalue()


def parse_aggregate_report_xml(xml, ip_db_path=None, offline=False,
nameservers=None, timeout=2.0,
keep_alive=None):
def parse_aggregate_report_xml(
xml,
ip_db_path=None,
always_use_local_files=False,
reverse_dns_map_path=None,
reverse_dns_map_url=None,
offline=False,
nameservers=None,
timeout=2.0,
keep_alive=None):
"""Parses a DMARC XML report string and returns a consistent OrderedDict
Args:
xml (str): A string of DMARC aggregate report XML
ip_db_path (str): Path to a MMDB file from MaxMind or DBIP
always_use_local_files (bool): Do not download files
reverse_dns_map_path (str): Path to a reverse DNS map file
reverse_dns_map_url (str): URL to a reverse DNS map file
offline (bool): Do not query online for geolocation or DNS
nameservers (list): A list of one or more nameservers to use
(Cloudflare's public DNS resolvers by default)
Expand Down Expand Up @@ -516,19 +537,27 @@ def parse_aggregate_report_xml(xml, ip_db_path=None, offline=False,
keep_alive()
logger.debug("Processed {0}/{1}".format(
i, len(report["record"])))
report_record = _parse_report_record(report["record"][i],
ip_db_path=ip_db_path,
offline=offline,
nameservers=nameservers,
dns_timeout=timeout)
report_record = _parse_report_record(
report["record"][i],
ip_db_path=ip_db_path,
offline=offline,
always_use_local_files=always_use_local_files,
reverse_dns_map_path=reverse_dns_map_path,
reverse_dns_map_url=reverse_dns_map_url,
nameservers=nameservers,
dns_timeout=timeout)
records.append(report_record)

else:
report_record = _parse_report_record(report["record"],
ip_db_path=ip_db_path,
offline=offline,
nameservers=nameservers,
dns_timeout=timeout)
report_record = _parse_report_record(
report["record"],
ip_db_path=ip_db_path,
always_use_local_files=always_use_local_files,
reverse_dns_map_path=reverse_dns_map_path,
reverse_dns_map_url=reverse_dns_map_url,
offline=offline,
nameservers=nameservers,
dns_timeout=timeout)
records.append(report_record)

new_report["records"] = records
Expand Down Expand Up @@ -607,16 +636,25 @@ def extract_report(input_):
return report


def parse_aggregate_report_file(_input, offline=False, ip_db_path=None,
nameservers=None,
dns_timeout=2.0,
keep_alive=None):
def parse_aggregate_report_file(
_input,
offline=False,
always_use_local_files=None,
reverse_dns_map_path=None,
reverse_dns_map_url=None,
ip_db_path=None,
nameservers=None,
dns_timeout=2.0,
keep_alive=None):
"""Parses a file at the given path, a file-like object. or bytes as an
aggregate DMARC report
Args:
_input: A path to a file, a file like object, or bytes
offline (bool): Do not query online for geolocation or DNS
always_use_local_files (bool): Do not download files
reverse_dns_map_path (str): Path to a reverse DNS map file
reverse_dns_map_url (str): URL to a reverse DNS map file
ip_db_path (str): Path to a MMDB file from MaxMind or DBIP
nameservers (list): A list of one or more nameservers to use
(Cloudflare's public DNS resolvers by default)
Expand All @@ -632,12 +670,16 @@ def parse_aggregate_report_file(_input, offline=False, ip_db_path=None,
except Exception as e:
raise InvalidAggregateReport(e)

return parse_aggregate_report_xml(xml,
ip_db_path=ip_db_path,
offline=offline,
nameservers=nameservers,
timeout=dns_timeout,
keep_alive=keep_alive)
return parse_aggregate_report_xml(
xml,
always_use_local_files=always_use_local_files,
reverse_dns_map_path=reverse_dns_map_path,
reverse_dns_map_url=reverse_dns_map_url,
ip_db_path=ip_db_path,
offline=offline,
nameservers=nameservers,
timeout=dns_timeout,
keep_alive=keep_alive)


def parsed_aggregate_reports_to_csv_rows(reports):
Expand Down Expand Up @@ -781,18 +823,28 @@ def parsed_aggregate_reports_to_csv(reports):
return csv_file_object.getvalue()


def parse_forensic_report(feedback_report, sample, msg_date,
offline=False, ip_db_path=None,
nameservers=None, dns_timeout=2.0,
def parse_forensic_report(feedback_report,
sample,
msg_date,
always_use_local_files=False,
reverse_dns_map_path=None,
reverse_dns_map_url=None,
offline=False,
ip_db_path=None,
nameservers=None,
dns_timeout=2.0,
strip_attachment_payloads=False):
"""
Converts a DMARC forensic report and sample to a ``OrderedDict``
Args:
feedback_report (str): A message's feedback report as a string
sample (str): The RFC 822 headers or RFC 822 message sample
ip_db_path (str): Path to a MMDB file from MaxMind or DBIP
always_use_local_files (bool): Do not download files
reverse_dns_map_path (str): Path to a reverse DNS map file
reverse_dns_map_url (str): URL to a reverse DNS map file
offline (bool): Do not query online for geolocation or DNS
sample (str): The RFC 822 headers or RFC 822 message sample
msg_date (str): The message's date header
nameservers (list): A list of one or more nameservers to use
(Cloudflare's public DNS resolvers by default)
Expand Down Expand Up @@ -840,12 +892,16 @@ def parse_forensic_report(feedback_report, sample, msg_date,
parsed_report["arrival_date_utc"] = arrival_utc

ip_address = re.split(r'\s', parsed_report["source_ip"]).pop(0)
parsed_report_source = get_ip_address_info(ip_address,
cache=IP_ADDRESS_CACHE,
ip_db_path=ip_db_path,
offline=offline,
nameservers=nameservers,
timeout=dns_timeout)
parsed_report_source = get_ip_address_info(
ip_address,
cache=IP_ADDRESS_CACHE,
ip_db_path=ip_db_path,
always_use_local_files=always_use_local_files,
reverse_dns_map_path=reverse_dns_map_path,
reverse_dns_map_url=reverse_dns_map_url,
offline=offline,
nameservers=nameservers,
timeout=dns_timeout)
parsed_report["source"] = parsed_report_source
del parsed_report["source_ip"]

Expand Down Expand Up @@ -1144,6 +1200,9 @@ def parse_report_email(input_, offline=False, ip_db_path=None,

def parse_report_file(input_, nameservers=None, dns_timeout=2.0,
strip_attachment_payloads=False, ip_db_path=None,
always_use_local_files=False,
reverse_dns_map_path=None,
reverse_dns_map_url=None,
offline=False, keep_alive=None):
"""Parses a DMARC aggregate or forensic file at the given path, a
file-like object. or bytes
Expand All @@ -1156,6 +1215,9 @@ def parse_report_file(input_, nameservers=None, dns_timeout=2.0,
strip_attachment_payloads (bool): Remove attachment payloads from
forensic report results
ip_db_path (str): Path to a MMDB file from MaxMind or DBIP
always_use_local_files (bool): Do not download files
reverse_dns_map_path (str): Path to a reverse DNS map
reverse_dns_map_url (str): URL to a reverse DNS map
offline (bool): Do not make online queries for geolocation or DNS
keep_alive (callable): Keep alive function
Expand All @@ -1173,8 +1235,10 @@ def parse_report_file(input_, nameservers=None, dns_timeout=2.0,
content = file_object.read()
file_object.close()
try:
report = parse_aggregate_report_file(content,
ip_db_path=ip_db_path,
report = parse_aggregate_report_file(
content,
ip_db_path=ip_db_path,
always_use_local_files=always_use_local_files,
offline=offline,
nameservers=nameservers,
dns_timeout=dns_timeout,
Expand Down
37 changes: 30 additions & 7 deletions parsedmarc/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,23 @@ def _str_to_list(s):


def cli_parse(file_path, sa, nameservers, dns_timeout,
ip_db_path, offline, conn):
ip_db_path, offline,
always_use_local_files,
reverse_dns_map_path,
reverse_dns_map_url,
conn):
"""Separated this function for multiprocessing"""
try:
file_results = parse_report_file(file_path,
ip_db_path=ip_db_path,
offline=offline,
nameservers=nameservers,
dns_timeout=dns_timeout,
strip_attachment_payloads=sa)
file_results = parse_report_file(
file_path,
ip_db_path=ip_db_path,
offline=offline,
always_use_local_files=always_use_local_files,
reverse_dns_map_path=reverse_dns_map_path,
reverse_dns_map_url=reverse_dns_map_url,
nameservers=nameservers,
dns_timeout=dns_timeout,
strip_attachment_payloads=sa)
conn.send([file_results, file_path])
except ParserError as error:
conn.send([error, file_path])
Expand Down Expand Up @@ -473,6 +481,9 @@ def process_reports(reports_):
log_file=args.log_file,
n_procs=1,
ip_db_path=None,
always_use_local_files=False,
reverse_dns_map_path=None,
reverse_dns_map_url=None,
la_client_id=None,
la_client_secret=None,
la_tenant_id=None,
Expand Down Expand Up @@ -545,6 +556,15 @@ def process_reports(reports_):
opts.ip_db_path = general_config["ip_db_path"]
else:
opts.ip_db_path = None
if "always_use_local_files" in general_config:
opts.always_use_local_files = general_config.getboolean(
"always_use_local_files")
if "reverse_dns_map_path" in general_config:
opts.reverse_dns_map_path = general_config[
"reverse_dns_path"]
if "reverse_dns_map_url" in general_config:
opts.reverse_dns_map_url = general_config[
"reverse_dns_url"]

if "mailbox" in config.sections():
mailbox_config = config["mailbox"]
Expand Down Expand Up @@ -1164,6 +1184,9 @@ def process_reports(reports_):
opts.dns_timeout,
opts.ip_db_path,
opts.offline,
opts.always_use_local_files,
opts.reverse_dns_map_path,
opts.reverse_dns_map_url,
child_conn,
))
processes.append(process)
Expand Down
Loading

0 comments on commit fd5b792

Please sign in to comment.