Pymobile

.github/workflows/super-linter.yml

@@ -32,3 +32,5 @@ jobs:
           VALIDATE_BASH_EXEC: true
           DEFAULT_BRANCH: main
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          IGNORE_GITIGNORED_FILES: true
+          FILTER_REGEX_EXCLUDE: *.sh

config.py

@@ -1,15 +1,13 @@
 import hashlib
 import hmac
-import os
+import os, sys
 import secrets
 import shlex
 from pathlib import Path
-from sys import platform
-
+import platform
 import logging
 import logging.handlers as handlers
 
-
 def setup_logger():
     """
     Set up a logger with a rotating file handler.
@@ -57,8 +55,8 @@ def setup_logger():
 IOS_DUMPFILES = {
     "Jailbroken-FS": "ios_jailbroken.log",
     "Jailbroken-SSH": "ios_jailbreak_ssh.retcode",
-    "Apps": "ios_apps.plist",
-    "Info": "ios_info.xml",
+    "Apps": "ios_apps.json",
+    "Info": "ios_info.json",
 }
 
 TEST_APP_LIST = "static_data/android.test.apps_list"
@@ -106,26 +104,15 @@ def set_test_mode(test):
 
 # TODO: We should get rid of this, ADB_PATH is very confusing
 ANDROID_HOME = os.getenv("ANDROID_HOME", "")
-PLATFORM = (
-    "darwin"
-    if platform == "darwin"
-    else (
-        "linux"
-        if platform.startswith("linux")
-        else "win32" if platform == "win32" else None
-    )
-)
-
-ADB_PATH = shlex.quote(os.path.join(ANDROID_HOME, "adb"))
-
-# LIBIMOBILEDEVICE_PATH = shlex.quote(str(STATIC_DATA / ("libimobiledevice-" + PLATFORM)))
-LIBIMOBILEDEVICE_PATH = ""
-# MOBILEDEVICE_PATH = 'mobiledevice'
-# MOBILEDEVICE_PATH = os.path.join(THISDIR, "mdf")  #'python2 -m MobileDevice'
-if PLATFORM:
-    MOBILEDEVICE_PATH = shlex.quote(str(STATIC_DATA / ("ios-deploy-" + PLATFORM)))
-else:
-    MOBILEDEVICE_PATH = shlex.quote(str(STATIC_DATA / ("ios-deploy-none")))
+PLATFORM = platform.system().lower()
+if 'microsoft' in platform.release().lower(): ## Check for wsl
+    PLATFORM = "wsl" 
+
+LIBIMOBILEDEVICE_PATH = "pymobiledevice3.exe" if PLATFORM == "wsl" \
+    else "pymobiledevice3"
+ADB_PATH = shlex.quote(os.path.join(ANDROID_HOME, "adb")) \
+    + (".exe" if PLATFORM in ("wsl", "win32") else "")
+
 
 DUMP_DIR = THIS_DIR / "phone_dumps"
 SCRIPT_DIR = THIS_DIR / "scripts"

phone_scanner/__init__.py

@@ -2,10 +2,11 @@
 # -*- coding: utf-8 -*-
 
 import os
+import sys
 import re
+import json
 import shlex
 import sqlite3
-import sys
 import config
 import pandas as pd
 from collections import defaultdict
@@ -15,7 +16,6 @@
 from .android_permissions import all_permissions
 from .runcmd import catch_err, run_command
 
-
 class AppScan(object):
     device_type = ""
     # app_info_conn = dataset.connect(config.APP_INFO_SQLITE_FILE)
@@ -216,7 +216,7 @@ def __init__(self):
         # self.setup()
 
     def setup(self):
-        p = run_command("{cli} kill-server; {cli} start-server")
+        p = run_command("{cli} kill-server; {cli} start-server", cli=self.cli)
         if p != 0:
             print(
                 ">> Setup failed with returncode={}. ~~ ex={!r}".format(
@@ -229,7 +229,7 @@ def _get_apps_from_device(self, serialno, flag) -> list:
         """get apps from the device"""
         cmd = "{cli} -s {serial} shell pm list packages {flag} | sed 's/^package://g' | sort"
         s = catch_err(
-            run_command(cmd, serial=serialno, flag=flag),
+            run_command(cmd, cli=self.cli, serial=serialno, flag=flag),
             msg="App search failed",
             cmd=cmd,
         )
@@ -256,6 +256,7 @@ def get_apps(self, serialno: str, from_dump: bool = True) -> list:
             if installed_apps:
                 q = run_command(
                     "bash scripts/android_scan.sh scan {ser} {hmac_serial}",
+                    cli=self.cli,
                     ser=serialno,
                     hmac_serial=hmac_serial,
                     nowait=True,
@@ -308,7 +309,7 @@ def devices(self):
         # cmd = '{cli} kill-server; {cli} start-server'
         # s = catch_err(run_command(cmd), time=30, msg="ADB connection failed", cmd=cmd)
         cmd = "{cli} devices | tail -n +2"
-        runcmd = catch_err(run_command(cmd), cmd=cmd).strip().split("\n")
+        runcmd = catch_err(run_command(cmd, cli=self.cli), cmd=cmd).strip().split("\n")
         conn_devices = []
         for rc in runcmd:
             d = rc.split()
@@ -328,15 +329,15 @@ def device_info(self, serial):
         m = {}
         cmd = "{cli} -s {serial} shell getprop ro.product.brand"
         m["brand"] = (
-            run_command(cmd, serial=serial).stdout.read().decode("utf-8").title()
+            run_command(cmd, cli=self.cli, serial=serial).stdout.read().decode("utf-8").title()
         )
 
         cmd = "{cli} -s {serial} shell getprop ro.product.model"
-        m["model"] = run_command(cmd, serial=serial).stdout.read().decode("utf-8")
+        m["model"] = run_command(cmd, cli=self.cli, serial=serial).stdout.read().decode("utf-8")
 
         cmd = "{cli} -s {serial} shell getprop ro.build.version.release"
         m["version"] = (
-            run_command(cmd, serial=serial).stdout.read().decode("utf-8").strip()
+            run_command(cmd, cli=self.cli, serial=serial).stdout.read().decode("utf-8").strip()
         )
 
         cmd = '{cli} -s {serial} shell dumpsys batterystats | grep -i "Start clock time:" | head -n1'
@@ -361,7 +362,7 @@ def device_info(self, serial):
     def uninstall(self, serial, appid):
         cmd = "{cli} uninstall {appid!r}"
         s = catch_err(
-            run_command(cmd, appid=shlex.quote(appid)),
+            run_command(cmd, cli=self.cli, appid=shlex.quote(appid)),
             cmd=cmd,
             msg="Could not uninstall",
         )
@@ -444,7 +445,7 @@ def isrooted(self, serial):
         }
         for k, v in root_checks.items():
             cmd = "{cli} -s {serial} shell '{v[0]}'"
-            s = catch_err(run_command(cmd, serial=shlex.quote(serial), v=v))
+            s = catch_err(run_command(cmd, cli=self.cli, serial=shlex.quote(serial), v=v))
             if s.strip() == v[1]:
                 return (True, f"The device is rooted: Found:  {k!r}.")
         return (False, "The device is probably not rooted.")
@@ -461,39 +462,6 @@ def __init__(self):
         self.serialno = None
         self.parse_dump = None
 
-    def setup(self, attempt_remount=False):
-        """FIXME: iOS setup."""
-        if config.PLATFORM == "linux" and attempt_remount:
-            # should show GUI prompt for password. sudo apt install policykit-1 if not there.
-            cmd = "pkexec '" + config.SCRIPT_DIR + "/ios_mount_linux.sh' mount"
-            # mountmsg = run_command(cmd).stderr.read().decode('utf-8')
-            if catch_err(run_command(cmd)) == -1:
-                return (
-                    False,
-                    "Couldn't detect device. See {}/ios_mount_linux.sh.".format(
-                        config.SCRIPT_DIR
-                    ),
-                )
-        cmd = "{}idevicepair pair".format(self.cli)
-        pairmsg = run_command(cmd).stdout.read().decode("utf-8")
-        if "No device found, is it plugged in?" in pairmsg:
-            return (False, pairmsg)
-        elif "Please enter the passcode on the device and retry." in pairmsg:
-            return (
-                False,
-                "Please unlock your device and follow the trust dialog"
-                " (you will need to enter your passcode). Then try to scan again.",
-            )
-        elif "SUCCESS: Paired with device" in pairmsg:
-            return (True, "Device successfully paired. Setup complete.")
-        elif "said that the user denied the trust dialog." in pairmsg:
-            return (
-                False,
-                "The trust dialog was denied. Please unplug the device"
-                ", reconnect it, and scan again -- accept the trust dialog to proceed.",
-            )
-        return (True, "Follow trust dialog on iOS device to continue.")
-
     # TODO: This might send titles out of order. Fix this to send both appid and
     # titles.
     def get_app_titles(self, serialno):
@@ -524,16 +492,16 @@ def _is_device(x):
             """Is it looks like a serial number"""
             return re.match(r"[a-f0-9]+", x) is not None
 
-        # cmd = '{cli} --detect -t1 | tail -n 1'
-        cmd = "{}idevice_id -l | tail -n 1".format(self.cli)
+        # cmd = "{}idevice_id -l | tail -n 1".format(self.cli)
+
+        cmd = "{cli} usbmux list"
         self.serialno = None
-        s = catch_err(run_command(cmd), cmd=cmd, msg="")
-
-        d = [
-            line.strip()
-            for line in s.split("\n")
-            if line.strip() and _is_device(line.strip())
-        ]
+        s = catch_err(run_command(cmd, cli=self.cli), cmd=cmd, msg="")
+        try:
+            d = [a.get("Identifier", "") for a in json.loads(s)]
+        except json.JSONDecodeError as e:
+            print(f">>> ERROR: {e!r}", file=sys.stderr)
+            d = []
         config.logging.info("Devices found:", d)
         return d
 
@@ -557,19 +525,15 @@ def _load_dump(self, serial) -> parse_dump.IosDump:
 
     def _dump_phone(self, serial: str) -> bool:
         print("DUMPING iOS INFO...")
-        connected, connected_reason = self.setup()
-        if not connected:
-            print("Couldn't connect to the device. Trying to reconnect. Over here.")
-            print(connected_reason)
-            return False
         hmac_serial = config.hmac_serial(serial)
         cmd = (
-            "'{}/ios_dump.sh' {} {Apps} {Info} {Jailbroken-FS} {Jailbroken-SSH}".format(
-                config.SCRIPT_DIR, hmac_serial, **config.IOS_DUMPFILES
+            f"'{config.SCRIPT_DIR}/ios_dump.sh' {hmac_serial} "
+            "{Apps} {Info} {Jailbroken-FS} {Jailbroken-SSH}".format(
+                **config.IOS_DUMPFILES
             )
         )
         print(cmd)
-        dumped = catch_err(run_command(cmd), cmd).strip()
+        dumped = catch_err(run_command(cmd, cli=self.cli), cmd).strip()
         if dumped:
             print("iOS DUMP RESULTS for {}:".format(hmac_serial))
             print(dumped)
@@ -581,10 +545,10 @@ def _dump_phone(self, serial: str) -> bool:
             return False
 
     def uninstall(self, serial, appid):
-        # cmd = '{cli} -i {serial} --uninstall_only --bundle_id {appid!r}'
+        # cmd = '{self.cli} -i {serial} --uninstall_only --bundle_id {appid!r}'
         # cmd = 'ideviceinstaller --udid {} --uninstall {appid!r}'.format(serial, appid)
-        cmd = f"{self.cli}ideviceinstaller --uninstall {appid!r}"
-        s = catch_err(run_command(cmd, appid=appid), cmd=cmd, msg="Could not uninstall")
+        cmd = "{cli} apps uninstall {appid!r}"
+        s = catch_err(run_command(cmd, cli=self.cli, appid=appid), cmd=cmd, msg="Could not uninstall")
         return s != -1
 
     def isrooted(self, serial):

phone_scanner/parse_dump.py

@@ -9,7 +9,7 @@
 from collections import OrderedDict
 from functools import reduce
 from pathlib import Path
-from plistlib import load
+# from plistlib import load
 from typing import List, Dict
 import pandas as pd
 from rsonlite import simpleparse
@@ -463,7 +463,7 @@ def __len__(self):
     def load_device_info(self):
         try:
             with open(self.finfo, "rb") as data:
-                device_info = load(data)
+                device_info = json.load(data)
             return device_info
 
         except Exception as ex:
@@ -479,11 +479,14 @@ def load_device_info(self):
     def load_file(self):
         # d = pd.read_json(self.fname)[self.COLS].set_index(self.INDEX)
         try:
-            # FIXME: somehow, get the ios_apps.plist into a dataframe.
             print("fname is: {}".format(self.fname))
-            with open(self.fname, "rb") as app_data:
-                apps_plist = load(app_data)
-            d = pd.DataFrame(apps_plist)
+            apps_list = []
+            with open(self.fname, "r") as app_data:
+                apps_json = json.load(app_data)
+                for k in apps_json:
+                    apps_list.append(apps_json[k])
+
+            d = pd.DataFrame(apps_list)
             d["appId"] = d["CFBundleIdentifier"]
             return d
         except Exception as ex:

phone_scanner/runcmd.py

@@ -89,7 +89,7 @@ def catch_err(
 
 
 def run_command(cmd, **kwargs):
-    _cmd = cmd.format(cli="adb", **kwargs)
+    _cmd = cmd.format(**kwargs)
     print(_cmd)
     if kwargs.get("nowait", False) or kwargs.get("NOWAIT", False):
         pid = subprocess.Popen(

requirements.txt

@@ -18,6 +18,7 @@ Mako>=1.2.4
 MarkupSafe>=2.1.1
 numpy>=1.23.5
 pandas>=1.5.2
+pymobiledevice3==4.20.17
 pycodestyle>=2.10.0
 python-dateutil>=2.8.2
 python-dotenv>=0.10.3

scripts/android_scan.sh

@@ -42,7 +42,6 @@ hmac_serial="-s $3"
 dump_dir="./phone_dumps/"
 ofname=$dump_dir/${hmac_serial:3}_android.txt
 
-email="^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}$"
 function scan {
     act=$1
     $adb "$serial" shell dumpsys "$act" | \
@@ -69,21 +68,21 @@ function retrieve {
     fi
     app="$1"
     process_uid=$(grep -A1 "Package \[$app\]" $ofname | sed '1d;s/.*userId=\([0-9]\+\).*/\1/g')
-    process_uidu=$(grep -Eo "app=ProcessRecord{[a-f0-9]+ [0-9]+:$app/([a-z0-9]*)" $ofname  | cut -d '/' -f 2 | sort -u)
+    process_uidu=$(grep -Eo "app=ProcessRecord{[a-f0-9]+ [0-9]+:$app/([a-z0-9]*)" "$ofname" | cut -d '/' -f 2 | sort -u)
 
     echo "$process_uid  $process_uidu"
     # Install date
-    printf 'Install: %s\n' "$(awk "/Package \[$app\]/{flag=1;next}/install permissions:/{flag=0}flag" $ofname | grep Time | tr -d '  ')"
+    printf 'Install: %s\n' "$(awk "/Package \[$app\]/{flag=1;next}/install permissions:/{flag=0}flag" "$ofname" | grep Time | tr -d '  ')"
 
     # Memory info 
-    printf 'Memory: %s\n' "$(awk '/Total PSS by process/{flag=1;next}/Total PSS/{flag=0}flag' $ofname | grep $app | sed '/^\s*$/d')"
+    printf 'Memory: %s\n' "$(awk '/Total PSS by process/{flag=1;next}/Total PSS/{flag=0}flag' "$ofname" | grep $app | sed '/^\s*$/d')"
 
     # Network info - cnt_set=0 => background,rx_bytes
-    printf 'DataUsage (%s): %s\n' "${process_uid}" "$(awk "/DUMP OF SERVICE net_stats/{flag=1;next}/DUMP OF SERVICE/{flag=0}flag" $ofname | sed 's/ /,/g' | csvgrep -c 4 -m ${process_uid} | csvcut -c 4,5,6,8)"
+    printf 'DataUsage (%s): %s\n' "${process_uid}" "$(awk "/DUMP OF SERVICE net_stats/{flag=1;next}/DUMP OF SERVICE/{flag=0}flag" "$ofname" | sed 's/ /,/g' | csvgrep -c 4 -m ${process_uid} | csvcut -c 4,5,6,8)"
     # awk "/DUMP OF SERVICE net_stats/{flag=1;next}/DUMP OF SERVICE/{flag=0}flag" $ofname | sed 's/ /,/g' | grep -a "${process_uid}"
 
     # Battery info
-    printf "Battery (%s): %s\n" "${process_uidu}" "$(awk '/Estimated power use/{flag=1;next}/^\s*$/{flag=0}flag' $ofname | grep "Uid ${process_uidu}")"
+    printf "Battery (%s): %s\n" "${process_uidu}" "$(awk '/Estimated power use/{flag=1;next}/^\s*$/{flag=0}flag' "$ofname" | grep "Uid ${process_uidu}")"
     # Device Admin
 }
 
@@ -94,7 +93,7 @@ services=(package location media.camera netpolicy mount
           activity appops)
 
 function dump {
-    for a in ${services[*]}; do
+    for a in "${services[@]}"; do
         echo "DUMP OF SERVICE $a"
         scan "$a"
     done
@@ -127,7 +126,7 @@ if [[ "$1" == "scan" ]]; then
     $adb $serial shell pm clear com.android.settings
 elif [[ "$1" == "info" ]]; then
     (>&2 echo "------ Running app info ------- $2 $3")
-    retrieve $3
+    retrieve "$3"
 else
     echo "$ bash $0 <scan|info> <serial_no> [appId]"
     exit 1;