Add basic support for IP indicators in MVT

[DonnchaC]

This PR reworks #465 which was started by renini.

The initial PR had support for specifying IP address in CIDR format. I've remove this for now as it could risk making MVT slow if large CIDRs are specified with thousands or millions of IPs.

This first version supports IP addresses by treated as domains internally in MVT which will automatically be matching in existing check_domain and check_url functions.

Closes #465.

[github-actions]

Coverage Report

File	Stmts	Miss	Cover	Missing
src/mvt/android
cli.py	157	54	66%	61, 69, 88–120, 156–180, 218–219, 226, 280–281, 288, 344–345, 352, 375–382, 390–391
cmd_check_adb.py	10	3	70%	26–37
cmd_check_androidqf.py	34	5	85%	57–60, 64
cmd_check_backup.py	63	18	71%	60, 69–70, 78–79, 85–88, 95–108, 112
cmd_check_bugreport.py	39	7	82%	51, 54–57, 70, 76
cmd_download_apks.py	87	69	21%	38–42, 51–53, 65–99, 105–112, 116–172, 175–177, 180–184
utils.py	16	5	69%	11–20
src/mvt/android/artifacts
dumpsys_appops.py	90	6	93%	23, 89–90, 138–140, 146
dumpsys_battery_daily.py	42	2	95%	42–43
dumpsys_battery_history.py	48	6	88%	47–55
dumpsys_dbinfo.py	42	5	88%	60–65
dumpsys_package_activities.py	40	4	90%	12, 64, 70–71
dumpsys_packages.py	121	5	96%	18–24, 161, 168
dumpsys_receivers.py	57	6	89%	24, 29, 34, 97, 103–104
getprop.py	31	3	90%	40, 48, 51
processes.py	34	9	74%	20, 24–25, 31, 55, 59, 63–65
src/mvt/android/modules/adb
base.py	148	112	24%	51–61, 66–73, 77–138, 142, 146–148, 157, 166–167, 171–172, 185, 198–200, 218–224, 234–269, 282–306, 309–352, 356
chrome_history.py	37	24	35%	30–38, 41, 50–56, 64–98, 101–110
dumpsys_accessibility.py	15	8	47%	26, 36–47
dumpsys_activities.py	14	7	50%	28–37, 40–45
dumpsys_appops.py	14	6	57%	28, 38–44
dumpsys_battery_daily.py	13	6	54%	26, 36–42
dumpsys_battery_history.py	13	6	54%	26, 36–42
dumpsys_dbinfo.py	14	6	57%	28, 38–44
dumpsys_full.py	16	9	44%	25, 35–45
dumpsys_receivers.py	14	7	50%	26–35, 38–44
files.py	72	57	21%	37–45, 48–56, 59–70, 73–88, 93–121, 124–155
getprop.py	14	7	50%	26–35, 38–43
logcat.py	21	14	33%	25, 35–57
packages.py	168	148	12%	39–47, 50–72, 75–109, 113–162, 166–176, 181–212, 215–312
processes.py	13	6	54%	26, 36–42
root_binaries.py	25	18	28%	24, 34–36, 39–70
selinux_status.py	17	10	41%	26–35, 38–48
settings.py	25	18	28%	26–35, 38–58
sms.py	79	60	24%	56–65, 68–69, 77–90, 98–127, 137–151, 154–179
whatsapp.py	53	39	26%	31, 41–42, 50–60, 68–103, 106–113
src/mvt/android/modules/androidqf
dumpsys_packages.py	24	3	88%	52–53, 56
packages.py	41	4	90%	44–50, 71
settings.py	24	2	92%	51–52
sms.py	49	13	73%	48, 51, 56–57, 73–78, 81, 85–91, 96–97
src/mvt/android/modules/backup
base.py	33	2	94%	64–65
helpers.py	22	1	95%	26
sms.py	33	4	88%	40, 44, 47–48
src/mvt/android/modules/bugreport
accessibility.py	17	3	82%	38–42, 51
activities.py	16	2	88%	42–46
appops.py	15	2	87%	38–42
base.py	51	16	69%	48–49, 54–55, 62–67, 71, 86–93
battery_daily.py	15	2	87%	38–42
battery_history.py	15	2	87%	38–42
dbinfo.py	16	2	88%	40–44
getprop.py	26	7	73%	40–44, 51–52, 57–60
packages.py	24	3	88%	39–43, 56
receivers.py	16	2	88%	40–44
src/mvt/android/parsers
backup.py	109	9	92%	62, 102–103, 109, 129, 132, 175, 190–191
src/mvt/common
artifact.py	10	2	80%	22, 28
cmd_check_iocs.py	38	29	24%	26–36, 39–80
command.py	132	47	64%	67–73, 79–97, 103–109, 118–146, 152–156, 159–161, 167, 201, 218, 222–223
indicators.py	302	60	80%	38–40, 53–59, 137, 149, 186–191, 274, 286, 298, 340–343, 360, 386–404, 431, 453, 484, 503, 518–524, 537–545, 556, 580, 605, 614–621, 636, 661–674, 685, 688, 739, 757
logo.py	40	33	18%	16–74, 78–85
module.py	119	36	70%	71–75, 80–84, 99–119, 160, 169, 174, 184, 203–204, 220–221, 237–246
options.py	13	3	77%	27–33
updates.py	143	118	17%	26–33, 38–51, 56–64, 67–69, 76–84, 87–89, 92–104, 107–122, 125–163, 170–199, 206–216, 219–249
url.py	25	6	76%	327, 366, 372–376
utils.py	107	37	65%	48–50, 63, 95–96, 110, 124–125, 164–181, 196–197, 210–211, 218–227, 242, 252, 260
virustotal.py	23	13	43%	25–52
src/mvt/ios
cli.py	139	62	55%	61, 69, 98–142, 161–185, 226–227, 260–281, 304–311, 319–320
cmd_check_fs.py	13	4	69%	28–40, 43
decrypt.py	114	92	19%	33–36, 39, 48–56, 61–64, 73–123, 131–181, 192–221, 227–231, 244–255
versions.py	32	3	91%	21, 30, 48
src/mvt/ios/modules
base.py	90	23	74%	54, 64, 71–96, 114, 122, 129, 137–138, 157–160, 195–196
net_base.py	119	46	61%	74–75, 156–211, 226–237, 249, 294–295, 303–304, 308
src/mvt/ios/modules/backup
backup_info.py	29	2	93%	43, 79
configuration_profiles.py	67	46	31%	43–48, 58, 61–88, 103–178
manifest.py	81	8	90%	59, 66, 112–119, 124, 170–171
profile_events.py	51	32	37%	44, 55, 58–67, 71–97, 103–110, 113
src/mvt/ios/modules/fs
analytics.py	65	50	23%	34, 44, 52–77, 80–137, 140–143, 146–152
analytics_ios_versions.py	36	26	28%	30, 40, 48–86
cache_files.py	46	35	24%	24, 34–45, 48–62, 65–80, 92–99
filesystem.py	42	8	81%	52, 56–57, 61, 77–78, 89–90
net_netusage.py	18	10	44%	34, 44–57
safari_favicon.py	37	26	30%	31, 41, 50–60, 63–115, 118–124
shutdownlog.py	65	54	17%	30, 40, 49–69, 72–127, 130–133
version_history.py	20	9	55%	32, 42, 50–65
webkit_base.py	22	16	27%	17–24, 27–38
webkit_indexeddb.py	13	4	69%	34, 44, 53–54
webkit_localstorage.py	12	4	67%	32, 42, 51–52
webkit_safariviewservice.py	10	3	70%	32, 42–43
src/mvt/ios/modules/mixed
applications.py	74	46	38%	44–51, 55–93, 99–107, 113–118, 128–140, 146, 148–150
calendar.py	49	2	96%	75–78
calls.py	22	10	55%	41, 53–82
chrome_favicon.py	36	23	36%	42, 50–60, 66–104
chrome_history.py	30	17	43%	44, 54–61, 67–102
contacts.py	28	17	39%	45–75
firefox_favicon.py	32	19	41%	43, 52–62, 68–106
firefox_history.py	30	17	43%	47, 55–62, 68–101
global_preferences.py	25	1	96%	45
idstatuscache.py	56	39	30%	46, 55–72, 75–105, 110–120
interactionc.py	55	40	27%	251–275, 281–320
locationd.py	83	65	22%	58–70, 73–133, 136–155, 160–172
osanalytics_addaily.py	31	17	45%	45, 56–63, 70–98
safari_browserstate.py	73	27	63%	68, 71–75, 96–98, 112–132, 167, 173–180
safari_history.py	70	44	37%	48, 59–98, 107, 110–113, 116–151, 163–171
shortcuts.py	67	50	25%	47–55, 71–78, 84–153
sms.py	69	13	81%	74, 86, 110–127, 140, 154
sms_attachments.py	42	24	43%	44, 57–74, 97–128
tcc.py	80	28	65%	68, 85, 108–125, 140–143, 163–205
webkit_resource_load_statistics.py	52	10	81%	67–68, 93–94, 128–135
webkit_session_resource_log.py	75	52	31%	56–66, 70, 73–113, 119–149, 156–173
whatsapp.py	51	38	25%	43–48, 56–63, 69–135
TOTAL	5812	2400	59%

Tests	Skipped	Failures	Errors	Time
100	0 💤	0 ❌	0 🔥	7.864s ⏱️

[github-actions]

Coverage Report

File	Stmts	Miss	Cover	Missing
src/mvt/android
cli.py	157	54	66%	61, 69, 88–120, 156–180, 218–219, 226, 280–281, 288, 344–345, 352, 375–382, 390–391
cmd_check_adb.py	10	3	70%	26–37
cmd_check_androidqf.py	34	5	85%	57–60, 64
cmd_check_backup.py	63	18	71%	60, 69–70, 78–79, 85–88, 95–108, 112
cmd_check_bugreport.py	39	7	82%	51, 54–57, 70, 76
cmd_download_apks.py	86	68	21%	38–42, 51–53, 65–99, 105–112, 116–172, 175–177, 180–184
utils.py	16	5	69%	11–20
src/mvt/android/artifacts
dumpsys_appops.py	90	6	93%	23, 89–90, 138–140, 146
dumpsys_battery_daily.py	42	2	95%	42–43
dumpsys_battery_history.py	48	7	85%	47–55, 68
dumpsys_dbinfo.py	42	5	88%	60–65
dumpsys_package_activities.py	40	4	90%	12, 64, 70–71
dumpsys_packages.py	121	5	96%	18–24, 161, 168
dumpsys_receivers.py	57	6	89%	24, 29, 34, 97, 103–104
getprop.py	31	3	90%	40, 48, 51
processes.py	34	9	74%	20, 24–25, 31, 55, 59, 63–65
src/mvt/android/modules/adb
base.py	147	111	24%	51–61, 66–73, 77–138, 142, 146–148, 157, 166–167, 171–172, 185, 198–200, 218–224, 234–269, 282–306, 309–352, 356
chrome_history.py	37	24	35%	30–38, 41, 50–56, 64–98, 101–110
dumpsys_accessibility.py	15	8	47%	26, 36–47
dumpsys_activities.py	14	7	50%	28–37, 40–45
dumpsys_appops.py	14	6	57%	28, 38–44
dumpsys_battery_daily.py	13	6	54%	26, 36–42
dumpsys_battery_history.py	13	6	54%	26, 36–42
dumpsys_dbinfo.py	14	6	57%	28, 38–44
dumpsys_full.py	16	9	44%	25, 35–45
dumpsys_receivers.py	14	7	50%	26–35, 38–44
files.py	72	57	21%	37–45, 48–56, 59–70, 73–88, 93–121, 124–155
getprop.py	14	7	50%	26–35, 38–43
logcat.py	21	14	33%	25, 35–57
packages.py	168	148	12%	39–47, 50–72, 75–109, 113–162, 166–176, 181–212, 215–312
processes.py	13	6	54%	26, 36–42
root_binaries.py	25	18	28%	24, 34–36, 39–70
selinux_status.py	17	10	41%	26–35, 38–48
settings.py	25	18	28%	26–35, 38–58
sms.py	79	60	24%	56–65, 68–69, 77–90, 98–127, 137–151, 154–179
whatsapp.py	53	39	26%	31, 41–42, 50–60, 68–103, 106–113
src/mvt/android/modules/androidqf
dumpsys_packages.py	24	3	88%	52–53, 56
packages.py	40	3	92%	44–50
settings.py	24	2	92%	51–52
sms.py	49	13	73%	48, 51, 56–57, 73–78, 81, 85–91, 96–97
src/mvt/android/modules/backup
base.py	33	2	94%	64–65
helpers.py	22	1	95%	26
sms.py	33	4	88%	40, 44, 47–48
src/mvt/android/modules/bugreport
accessibility.py	17	3	82%	38–42, 51
activities.py	16	2	88%	42–46
appops.py	15	2	87%	38–42
base.py	51	16	69%	48–49, 54–55, 62–67, 71, 86–93
battery_daily.py	15	2	87%	38–42
battery_history.py	15	2	87%	38–42
dbinfo.py	16	2	88%	40–44
getprop.py	26	7	73%	40–44, 51–52, 57–60
packages.py	24	3	88%	39–43, 56
receivers.py	16	2	88%	40–44
src/mvt/android/parsers
backup.py	109	9	92%	62, 102–103, 109, 129, 132, 175, 190–191
src/mvt/common
artifact.py	10	2	80%	22, 28
cmd_check_iocs.py	38	29	24%	26–36, 39–80
command.py	132	47	64%	67–73, 79–97, 103–109, 118–146, 152–156, 159–161, 167, 201, 218, 222–223
indicators.py	302	60	80%	38–40, 53–59, 137, 149, 186–191, 274, 286, 298, 340–343, 360, 386–404, 431, 453, 484, 503, 518–524, 537–545, 556, 580, 605, 614–621, 636, 661–674, 685, 688, 739, 757
logo.py	40	33	18%	16–74, 78–85
module.py	119	36	70%	71–75, 80–84, 99–119, 160, 169, 174, 184, 203–204, 220–221, 237–246
options.py	13	3	77%	27–33
updates.py	143	118	17%	26–33, 38–51, 56–64, 67–69, 76–84, 87–89, 92–104, 107–122, 125–163, 170–199, 206–216, 219–249
url.py	25	6	76%	327, 366, 372–376
utils.py	107	37	65%	48–50, 63, 95–96, 110, 124–125, 164–181, 196–197, 210–211, 218–227, 242, 252, 260
virustotal.py	23	13	43%	25–52
src/mvt/ios
cli.py	139	62	55%	61, 69, 98–142, 161–185, 226–227, 260–281, 304–311, 319–320
cmd_check_fs.py	13	4	69%	28–40, 43
decrypt.py	114	92	19%	33–36, 39, 48–56, 61–64, 73–123, 131–181, 192–221, 227–231, 244–255
versions.py	32	3	91%	21, 30, 48
src/mvt/ios/modules
base.py	90	23	74%	54, 64, 71–96, 114, 122, 129, 137–138, 157–160, 195–196
net_base.py	119	46	61%	74–75, 156–211, 226–237, 249, 294–295, 303–304, 308
src/mvt/ios/modules/backup
backup_info.py	29	2	93%	43, 79
configuration_profiles.py	67	46	31%	43–48, 58, 61–88, 103–178
manifest.py	81	8	90%	59, 66, 112–119, 124, 170–171
profile_events.py	51	32	37%	44, 55, 58–67, 71–97, 103–110, 113
src/mvt/ios/modules/fs
analytics.py	65	50	23%	34, 44, 52–77, 80–137, 140–143, 146–152
analytics_ios_versions.py	36	26	28%	30, 40, 48–86
cache_files.py	46	35	24%	24, 34–45, 48–62, 65–80, 92–99
filesystem.py	42	8	81%	52, 56–57, 61, 77–78, 89–90
net_netusage.py	18	10	44%	34, 44–57
safari_favicon.py	37	26	30%	31, 41, 50–60, 63–115, 118–124
shutdownlog.py	65	54	17%	30, 40, 49–69, 72–127, 130–133
version_history.py	20	9	55%	32, 42, 50–65
webkit_base.py	22	16	27%	17–24, 27–38
webkit_indexeddb.py	13	4	69%	34, 44, 53–54
webkit_localstorage.py	12	4	67%	32, 42, 51–52
webkit_safariviewservice.py	10	3	70%	32, 42–43
src/mvt/ios/modules/mixed
applications.py	74	46	38%	44–51, 55–93, 99–107, 113–118, 128–140, 146, 148–150
calendar.py	49	2	96%	75–78
calls.py	22	10	55%	41, 53–82
chrome_favicon.py	36	23	36%	42, 50–60, 66–104
chrome_history.py	30	17	43%	44, 54–61, 67–102
contacts.py	28	17	39%	45–75
firefox_favicon.py	32	19	41%	43, 52–62, 68–106
firefox_history.py	30	17	43%	47, 55–62, 68–101
global_preferences.py	25	1	96%	45
idstatuscache.py	56	39	30%	46, 55–72, 75–105, 110–120
interactionc.py	55	40	27%	251–275, 281–320
locationd.py	83	65	22%	58–70, 73–133, 136–155, 160–172
osanalytics_addaily.py	31	17	45%	45, 56–63, 70–98
safari_browserstate.py	73	27	63%	68, 71–75, 96–98, 112–132, 167, 173–180
safari_history.py	70	44	37%	48, 59–98, 107, 110–113, 116–151, 163–171
shortcuts.py	67	50	25%	47–55, 71–78, 84–153
sms.py	69	13	81%	74, 86, 110–127, 140, 154
sms_attachments.py	42	24	43%	44, 57–74, 97–128
tcc.py	80	28	65%	68, 85, 108–125, 140–143, 163–205
webkit_resource_load_statistics.py	52	10	81%	67–68, 93–94, 128–135
webkit_session_resource_log.py	75	52	31%	56–66, 70, 73–113, 119–149, 156–173
whatsapp.py	51	38	25%	43–48, 56–63, 69–135
TOTAL	5809	2398	59%

Tests	Skipped	Failures	Errors	Time
100	0 💤	0 ❌	0 🔥	7.985s ⏱️