diff --git a/.github/workflows/super-linter.yml b/.github/workflows/super-linter.yml index 81aa963..ec4cb8f 100644 --- a/.github/workflows/super-linter.yml +++ b/.github/workflows/super-linter.yml @@ -32,3 +32,5 @@ jobs: VALIDATE_BASH_EXEC: true DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + IGNORE_GITIGNORED_FILES: true + FILTER_REGEX_EXCLUDE: *.sh \ No newline at end of file diff --git a/config.py b/config.py index 65688fd..b2bbf2e 100644 --- a/config.py +++ b/config.py @@ -1,15 +1,13 @@ import hashlib import hmac -import os +import os, sys import secrets import shlex from pathlib import Path -from sys import platform - +import platform import logging import logging.handlers as handlers - def setup_logger(): """ Set up a logger with a rotating file handler. @@ -57,8 +55,8 @@ def setup_logger(): IOS_DUMPFILES = { "Jailbroken-FS": "ios_jailbroken.log", "Jailbroken-SSH": "ios_jailbreak_ssh.retcode", - "Apps": "ios_apps.plist", - "Info": "ios_info.xml", + "Apps": "ios_apps.json", + "Info": "ios_info.json", } TEST_APP_LIST = "static_data/android.test.apps_list" @@ -106,26 +104,15 @@ def set_test_mode(test): # TODO: We should get rid of this, ADB_PATH is very confusing ANDROID_HOME = os.getenv("ANDROID_HOME", "") -PLATFORM = ( - "darwin" - if platform == "darwin" - else ( - "linux" - if platform.startswith("linux") - else "win32" if platform == "win32" else None - ) -) - -ADB_PATH = shlex.quote(os.path.join(ANDROID_HOME, "adb")) - -# LIBIMOBILEDEVICE_PATH = shlex.quote(str(STATIC_DATA / ("libimobiledevice-" + PLATFORM))) -LIBIMOBILEDEVICE_PATH = "" -# MOBILEDEVICE_PATH = 'mobiledevice' -# MOBILEDEVICE_PATH = os.path.join(THISDIR, "mdf") #'python2 -m MobileDevice' -if PLATFORM: - MOBILEDEVICE_PATH = shlex.quote(str(STATIC_DATA / ("ios-deploy-" + PLATFORM))) -else: - MOBILEDEVICE_PATH = shlex.quote(str(STATIC_DATA / ("ios-deploy-none"))) +PLATFORM = platform.system().lower() +if 'microsoft' in platform.release().lower(): ## Check for wsl + PLATFORM = "wsl" + +LIBIMOBILEDEVICE_PATH = "pymobiledevice3.exe" if PLATFORM == "wsl" \ + else "pymobiledevice3" +ADB_PATH = shlex.quote(os.path.join(ANDROID_HOME, "adb")) \ + + (".exe" if PLATFORM in ("wsl", "win32") else "") + DUMP_DIR = THIS_DIR / "phone_dumps" SCRIPT_DIR = THIS_DIR / "scripts" diff --git a/phone_scanner/__init__.py b/phone_scanner/__init__.py index 991aa36..8cd492d 100644 --- a/phone_scanner/__init__.py +++ b/phone_scanner/__init__.py @@ -2,10 +2,11 @@ # -*- coding: utf-8 -*- import os +import sys import re +import json import shlex import sqlite3 -import sys import config import pandas as pd from collections import defaultdict @@ -15,7 +16,6 @@ from .android_permissions import all_permissions from .runcmd import catch_err, run_command - class AppScan(object): device_type = "" # app_info_conn = dataset.connect(config.APP_INFO_SQLITE_FILE) @@ -216,7 +216,7 @@ def __init__(self): # self.setup() def setup(self): - p = run_command("{cli} kill-server; {cli} start-server") + p = run_command("{cli} kill-server; {cli} start-server", cli=self.cli) if p != 0: print( ">> Setup failed with returncode={}. ~~ ex={!r}".format( @@ -229,7 +229,7 @@ def _get_apps_from_device(self, serialno, flag) -> list: """get apps from the device""" cmd = "{cli} -s {serial} shell pm list packages {flag} | sed 's/^package://g' | sort" s = catch_err( - run_command(cmd, serial=serialno, flag=flag), + run_command(cmd, cli=self.cli, serial=serialno, flag=flag), msg="App search failed", cmd=cmd, ) @@ -256,6 +256,7 @@ def get_apps(self, serialno: str, from_dump: bool = True) -> list: if installed_apps: q = run_command( "bash scripts/android_scan.sh scan {ser} {hmac_serial}", + cli=self.cli, ser=serialno, hmac_serial=hmac_serial, nowait=True, @@ -308,7 +309,7 @@ def devices(self): # cmd = '{cli} kill-server; {cli} start-server' # s = catch_err(run_command(cmd), time=30, msg="ADB connection failed", cmd=cmd) cmd = "{cli} devices | tail -n +2" - runcmd = catch_err(run_command(cmd), cmd=cmd).strip().split("\n") + runcmd = catch_err(run_command(cmd, cli=self.cli), cmd=cmd).strip().split("\n") conn_devices = [] for rc in runcmd: d = rc.split() @@ -328,15 +329,15 @@ def device_info(self, serial): m = {} cmd = "{cli} -s {serial} shell getprop ro.product.brand" m["brand"] = ( - run_command(cmd, serial=serial).stdout.read().decode("utf-8").title() + run_command(cmd, cli=self.cli, serial=serial).stdout.read().decode("utf-8").title() ) cmd = "{cli} -s {serial} shell getprop ro.product.model" - m["model"] = run_command(cmd, serial=serial).stdout.read().decode("utf-8") + m["model"] = run_command(cmd, cli=self.cli, serial=serial).stdout.read().decode("utf-8") cmd = "{cli} -s {serial} shell getprop ro.build.version.release" m["version"] = ( - run_command(cmd, serial=serial).stdout.read().decode("utf-8").strip() + run_command(cmd, cli=self.cli, serial=serial).stdout.read().decode("utf-8").strip() ) cmd = '{cli} -s {serial} shell dumpsys batterystats | grep -i "Start clock time:" | head -n1' @@ -361,7 +362,7 @@ def device_info(self, serial): def uninstall(self, serial, appid): cmd = "{cli} uninstall {appid!r}" s = catch_err( - run_command(cmd, appid=shlex.quote(appid)), + run_command(cmd, cli=self.cli, appid=shlex.quote(appid)), cmd=cmd, msg="Could not uninstall", ) @@ -444,7 +445,7 @@ def isrooted(self, serial): } for k, v in root_checks.items(): cmd = "{cli} -s {serial} shell '{v[0]}'" - s = catch_err(run_command(cmd, serial=shlex.quote(serial), v=v)) + s = catch_err(run_command(cmd, cli=self.cli, serial=shlex.quote(serial), v=v)) if s.strip() == v[1]: return (True, f"The device is rooted: Found: {k!r}.") return (False, "The device is probably not rooted.") @@ -461,39 +462,6 @@ def __init__(self): self.serialno = None self.parse_dump = None - def setup(self, attempt_remount=False): - """FIXME: iOS setup.""" - if config.PLATFORM == "linux" and attempt_remount: - # should show GUI prompt for password. sudo apt install policykit-1 if not there. - cmd = "pkexec '" + config.SCRIPT_DIR + "/ios_mount_linux.sh' mount" - # mountmsg = run_command(cmd).stderr.read().decode('utf-8') - if catch_err(run_command(cmd)) == -1: - return ( - False, - "Couldn't detect device. See {}/ios_mount_linux.sh.".format( - config.SCRIPT_DIR - ), - ) - cmd = "{}idevicepair pair".format(self.cli) - pairmsg = run_command(cmd).stdout.read().decode("utf-8") - if "No device found, is it plugged in?" in pairmsg: - return (False, pairmsg) - elif "Please enter the passcode on the device and retry." in pairmsg: - return ( - False, - "Please unlock your device and follow the trust dialog" - " (you will need to enter your passcode). Then try to scan again.", - ) - elif "SUCCESS: Paired with device" in pairmsg: - return (True, "Device successfully paired. Setup complete.") - elif "said that the user denied the trust dialog." in pairmsg: - return ( - False, - "The trust dialog was denied. Please unplug the device" - ", reconnect it, and scan again -- accept the trust dialog to proceed.", - ) - return (True, "Follow trust dialog on iOS device to continue.") - # TODO: This might send titles out of order. Fix this to send both appid and # titles. def get_app_titles(self, serialno): @@ -524,16 +492,16 @@ def _is_device(x): """Is it looks like a serial number""" return re.match(r"[a-f0-9]+", x) is not None - # cmd = '{cli} --detect -t1 | tail -n 1' - cmd = "{}idevice_id -l | tail -n 1".format(self.cli) + # cmd = "{}idevice_id -l | tail -n 1".format(self.cli) + + cmd = "{cli} usbmux list" self.serialno = None - s = catch_err(run_command(cmd), cmd=cmd, msg="") - - d = [ - line.strip() - for line in s.split("\n") - if line.strip() and _is_device(line.strip()) - ] + s = catch_err(run_command(cmd, cli=self.cli), cmd=cmd, msg="") + try: + d = [a.get("Identifier", "") for a in json.loads(s)] + except json.JSONDecodeError as e: + print(f">>> ERROR: {e!r}", file=sys.stderr) + d = [] config.logging.info("Devices found:", d) return d @@ -557,19 +525,15 @@ def _load_dump(self, serial) -> parse_dump.IosDump: def _dump_phone(self, serial: str) -> bool: print("DUMPING iOS INFO...") - connected, connected_reason = self.setup() - if not connected: - print("Couldn't connect to the device. Trying to reconnect. Over here.") - print(connected_reason) - return False hmac_serial = config.hmac_serial(serial) cmd = ( - "'{}/ios_dump.sh' {} {Apps} {Info} {Jailbroken-FS} {Jailbroken-SSH}".format( - config.SCRIPT_DIR, hmac_serial, **config.IOS_DUMPFILES + f"'{config.SCRIPT_DIR}/ios_dump.sh' {hmac_serial} " + "{Apps} {Info} {Jailbroken-FS} {Jailbroken-SSH}".format( + **config.IOS_DUMPFILES ) ) print(cmd) - dumped = catch_err(run_command(cmd), cmd).strip() + dumped = catch_err(run_command(cmd, cli=self.cli), cmd).strip() if dumped: print("iOS DUMP RESULTS for {}:".format(hmac_serial)) print(dumped) @@ -581,10 +545,10 @@ def _dump_phone(self, serial: str) -> bool: return False def uninstall(self, serial, appid): - # cmd = '{cli} -i {serial} --uninstall_only --bundle_id {appid!r}' + # cmd = '{self.cli} -i {serial} --uninstall_only --bundle_id {appid!r}' # cmd = 'ideviceinstaller --udid {} --uninstall {appid!r}'.format(serial, appid) - cmd = f"{self.cli}ideviceinstaller --uninstall {appid!r}" - s = catch_err(run_command(cmd, appid=appid), cmd=cmd, msg="Could not uninstall") + cmd = "{cli} apps uninstall {appid!r}" + s = catch_err(run_command(cmd, cli=self.cli, appid=appid), cmd=cmd, msg="Could not uninstall") return s != -1 def isrooted(self, serial): diff --git a/phone_scanner/parse_dump.py b/phone_scanner/parse_dump.py index 2b31971..0f0f5fa 100644 --- a/phone_scanner/parse_dump.py +++ b/phone_scanner/parse_dump.py @@ -9,7 +9,7 @@ from collections import OrderedDict from functools import reduce from pathlib import Path -from plistlib import load +# from plistlib import load from typing import List, Dict import pandas as pd from rsonlite import simpleparse @@ -463,7 +463,7 @@ def __len__(self): def load_device_info(self): try: with open(self.finfo, "rb") as data: - device_info = load(data) + device_info = json.load(data) return device_info except Exception as ex: @@ -479,11 +479,14 @@ def load_device_info(self): def load_file(self): # d = pd.read_json(self.fname)[self.COLS].set_index(self.INDEX) try: - # FIXME: somehow, get the ios_apps.plist into a dataframe. print("fname is: {}".format(self.fname)) - with open(self.fname, "rb") as app_data: - apps_plist = load(app_data) - d = pd.DataFrame(apps_plist) + apps_list = [] + with open(self.fname, "r") as app_data: + apps_json = json.load(app_data) + for k in apps_json: + apps_list.append(apps_json[k]) + + d = pd.DataFrame(apps_list) d["appId"] = d["CFBundleIdentifier"] return d except Exception as ex: diff --git a/phone_scanner/runcmd.py b/phone_scanner/runcmd.py index 03665b1..2cb7b35 100644 --- a/phone_scanner/runcmd.py +++ b/phone_scanner/runcmd.py @@ -89,7 +89,7 @@ def catch_err( def run_command(cmd, **kwargs): - _cmd = cmd.format(cli="adb", **kwargs) + _cmd = cmd.format(**kwargs) print(_cmd) if kwargs.get("nowait", False) or kwargs.get("NOWAIT", False): pid = subprocess.Popen( diff --git a/requirements.txt b/requirements.txt index e95412b..67ddb4e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,6 +18,7 @@ Mako>=1.2.4 MarkupSafe>=2.1.1 numpy>=1.23.5 pandas>=1.5.2 +pymobiledevice3==4.20.17 pycodestyle>=2.10.0 python-dateutil>=2.8.2 python-dotenv>=0.10.3 diff --git a/scripts/android_scan.sh b/scripts/android_scan.sh index 018a7ef..14706d0 100755 --- a/scripts/android_scan.sh +++ b/scripts/android_scan.sh @@ -42,7 +42,6 @@ hmac_serial="-s $3" dump_dir="./phone_dumps/" ofname=$dump_dir/${hmac_serial:3}_android.txt -email="^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}$" function scan { act=$1 $adb "$serial" shell dumpsys "$act" | \ @@ -69,21 +68,21 @@ function retrieve { fi app="$1" process_uid=$(grep -A1 "Package \[$app\]" $ofname | sed '1d;s/.*userId=\([0-9]\+\).*/\1/g') - process_uidu=$(grep -Eo "app=ProcessRecord{[a-f0-9]+ [0-9]+:$app/([a-z0-9]*)" $ofname | cut -d '/' -f 2 | sort -u) + process_uidu=$(grep -Eo "app=ProcessRecord{[a-f0-9]+ [0-9]+:$app/([a-z0-9]*)" "$ofname" | cut -d '/' -f 2 | sort -u) echo "$process_uid $process_uidu" # Install date - printf 'Install: %s\n' "$(awk "/Package \[$app\]/{flag=1;next}/install permissions:/{flag=0}flag" $ofname | grep Time | tr -d ' ')" + printf 'Install: %s\n' "$(awk "/Package \[$app\]/{flag=1;next}/install permissions:/{flag=0}flag" "$ofname" | grep Time | tr -d ' ')" # Memory info - printf 'Memory: %s\n' "$(awk '/Total PSS by process/{flag=1;next}/Total PSS/{flag=0}flag' $ofname | grep $app | sed '/^\s*$/d')" + printf 'Memory: %s\n' "$(awk '/Total PSS by process/{flag=1;next}/Total PSS/{flag=0}flag' "$ofname" | grep $app | sed '/^\s*$/d')" # Network info - cnt_set=0 => background,rx_bytes - printf 'DataUsage (%s): %s\n' "${process_uid}" "$(awk "/DUMP OF SERVICE net_stats/{flag=1;next}/DUMP OF SERVICE/{flag=0}flag" $ofname | sed 's/ /,/g' | csvgrep -c 4 -m ${process_uid} | csvcut -c 4,5,6,8)" + printf 'DataUsage (%s): %s\n' "${process_uid}" "$(awk "/DUMP OF SERVICE net_stats/{flag=1;next}/DUMP OF SERVICE/{flag=0}flag" "$ofname" | sed 's/ /,/g' | csvgrep -c 4 -m ${process_uid} | csvcut -c 4,5,6,8)" # awk "/DUMP OF SERVICE net_stats/{flag=1;next}/DUMP OF SERVICE/{flag=0}flag" $ofname | sed 's/ /,/g' | grep -a "${process_uid}" # Battery info - printf "Battery (%s): %s\n" "${process_uidu}" "$(awk '/Estimated power use/{flag=1;next}/^\s*$/{flag=0}flag' $ofname | grep "Uid ${process_uidu}")" + printf "Battery (%s): %s\n" "${process_uidu}" "$(awk '/Estimated power use/{flag=1;next}/^\s*$/{flag=0}flag' "$ofname" | grep "Uid ${process_uidu}")" # Device Admin } @@ -94,7 +93,7 @@ services=(package location media.camera netpolicy mount activity appops) function dump { - for a in ${services[*]}; do + for a in "${services[@]}"; do echo "DUMP OF SERVICE $a" scan "$a" done @@ -127,7 +126,7 @@ if [[ "$1" == "scan" ]]; then $adb $serial shell pm clear com.android.settings elif [[ "$1" == "info" ]]; then (>&2 echo "------ Running app info ------- $2 $3") - retrieve $3 + retrieve "$3" else echo "$ bash $0 [appId]" exit 1; diff --git a/scripts/ios_dump.sh b/scripts/ios_dump.sh index 421ff56..9d01266 100755 --- a/scripts/ios_dump.sh +++ b/scripts/ios_dump.sh @@ -1,5 +1,6 @@ #!/bin/bash DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +idb=pymobiledevice3 platform='unknown' unamestr=`uname` if [[ "$unamestr" == 'Linux' ]]; then @@ -10,28 +11,37 @@ elif [[ "$unamestr" == 'FreeBSD' ]]; then platform='freebsd' fi -echo "$platform" "$adb" +if grep -qi microsoft /proc/version; then + platform="wsl" + idb=pymobiledevice3.exe +fi + +echo "$platform" + -serial=$(idevice_id -l 2>&1 | tail -n 1) +# serial=$(idevice_id -l 2>&1 | tail -n 1) +serial=$(${idb} usbmux list | awk -F'"' '/Identifier/ {print $4}') mkdir -p phone_dumps/"$1"_ios cd phone_dumps/"$1"_ios # gets all of the details about each app (basically what ios_deploy does but with extra fields) -ideviceinstaller -u "$serial" -l -o xml -o list_all > "$2" +# ideviceinstaller -u "$serial" -l -o xml -o list_all > "$2" +${idb} apps list > "$2" # get around bug in Python 3 that doesn't recognize utf-8 encodings. # sed -i -e 's///g' $2 # sed -i -e 's/<\/data>/<\/string>/g' $2 # maybe for macOS... -# plutil -convert json $2 +# plutil -convert json $2 # gets OS version, serial, etc. -x for xml. Raw is easy to parse, too. -ideviceinfo -u "$serial" -x > $3 +# ideviceinfo -u "$serial" -x > $3 +${idb} lockdown info > "$3" # sed -i -e 's///g' $3 # sed -i -e 's/<\/data>/<\/string>/g' $3 -# remove identifying info (delete file after saving +# remove identifying info (delete file after saving # relevant bits of scan in server.py, actually) #sed -i -e '/<\key>DeviceName<\/key>/,+1d' $3 #sed -i -e '/<\key>MobileEquipmentIdentifier<\/key>/,+1d' $3 @@ -45,7 +55,7 @@ ideviceinfo -u "$serial" -x > $3 # delete this after hashing when session ends. #sed -i -e '/<\key>InternationalMobileEquipmentIdentity<\/key>/,+1d' $3 -# try to check for jailbroken by mounting the entire filesystem. +# try to check for jailbroken by mounting the entire filesystem. # Gets output: # "Failed to start AFC service 'com.apple.afc2' on the device. # This service enables access to the root filesystem of your device. @@ -56,9 +66,9 @@ mkdir -p /tmp/phonescanmnt # ifuse -u "$serial" --root /tmp/phonescanmnt &> $4 #lsof -ti tcp:2222 | xargs kill -iproxy 2222 22 & "${DIR}/ios_ssh_expect.sh" localhost -echo $? > $5 -cd .. +# iproxy 2222 22 & "${DIR}/ios_ssh_expect.sh" localhost +# echo $? > $5 +# cd .. # for consumption by python echo $serial