diff --git a/Pipfile b/Pipfile index 69eaa20..12425bf 100644 --- a/Pipfile +++ b/Pipfile @@ -6,12 +6,14 @@ name = "pypi" [packages] pydantic = "~=2.9" flask-ml = "0.0.8" +colorama = "*" [dev-packages] black = "~=24.8" mypy = "~=1.11" flake8 = "~=7.1" isort = "~=5.13" +hapless = "*" [requires] python_version = "3.11" diff --git a/Pipfile.lock b/Pipfile.lock index af937e9..a84dc0e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "820053c841e6255f06bec43306d9e840af7cccca86850e25278feb07fb186541" + "sha256": "8a8fe82d208d7d253b6bccb4bfd0d83145cabfdb728312f650f62575b92fdef1" }, "pipfile-spec": 6, "requires": { @@ -144,6 +144,15 @@ "markers": "python_version >= '3.7'", "version": "==8.1.7" }, + "colorama": { + "hashes": [ + "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", + "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" + ], + "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'", + "version": "==0.4.6" + }, "flask": { "hashes": [ "sha256:34e815dfaa43340d1d15a5c3a02b8476004037eb4840b34910c6e21679d288f3", @@ -435,6 +444,23 @@ "markers": "python_full_version >= '3.8.1'", "version": "==7.1.1" }, + "hapless": { + "hashes": [ + "sha256:a1d04dcd591e534bea7a132cd668bca1c1320e1ff9e02afa2fd67e477c4986ea", + "sha256:ef430336df69e53700432468b18c32028f430d6c23ad80675b8a6954dffa5ad4" + ], + "index": "pypi", + "markers": "python_version >= '3.7' and python_version < '4.0'", + "version": "==0.4.1" + }, + "humanize": { + "hashes": [ + "sha256:06b6eb0293e4b85e8d385397c5868926820db32b9b654b932f57fa41c23c9978", + "sha256:39e7ccb96923e732b5c2e27aeaa3b10a8dfeeba3eb965ba7b74a3eb0e30040a6" + ], + "markers": "python_version >= '3.8'", + "version": "==4.10.0" + }, "isort": { "hashes": [ "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109", @@ -444,6 +470,14 @@ "markers": "python_full_version >= '3.8.0'", "version": "==5.13.2" }, + "markdown-it-py": { + "hashes": [ + "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", + "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb" + ], + "markers": "python_version >= '3.8'", + "version": "==3.0.0" + }, "mccabe": { "hashes": [ "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", @@ -452,6 +486,14 @@ "markers": "python_version >= '3.6'", "version": "==0.7.0" }, + "mdurl": { + "hashes": [ + "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", + "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba" + ], + "markers": "python_version >= '3.7'", + "version": "==0.1.2" + }, "mypy": { "hashes": [ "sha256:06d26c277962f3fb50e13044674aa10553981ae514288cb7d0a738f495550b36", @@ -518,6 +560,28 @@ "markers": "python_version >= '3.8'", "version": "==4.3.6" }, + "psutil": { + "hashes": [ + "sha256:02615ed8c5ea222323408ceba16c60e99c3f91639b07da6373fb7e6539abc56d", + "sha256:05806de88103b25903dff19bb6692bd2e714ccf9e668d050d144012055cbca73", + "sha256:26bd09967ae00920df88e0352a91cff1a78f8d69b3ecabbfe733610c0af486c8", + "sha256:27cc40c3493bb10de1be4b3f07cae4c010ce715290a5be22b98493509c6299e2", + "sha256:36f435891adb138ed3c9e58c6af3e2e6ca9ac2f365efe1f9cfef2794e6c93b4e", + "sha256:50187900d73c1381ba1454cf40308c2bf6f34268518b3f36a9b663ca87e65e36", + "sha256:611052c4bc70432ec770d5d54f64206aa7203a101ec273a0cd82418c86503bb7", + "sha256:6be126e3225486dff286a8fb9a06246a5253f4c7c53b475ea5f5ac934e64194c", + "sha256:7d79560ad97af658a0f6adfef8b834b53f64746d45b403f225b85c5c2c140eee", + "sha256:8cb6403ce6d8e047495a701dc7c5bd788add903f8986d523e3e20b98b733e421", + "sha256:8db4c1b57507eef143a15a6884ca10f7c73876cdf5d51e713151c1236a0e68cf", + "sha256:aee678c8720623dc456fa20659af736241f575d79429a0e5e9cf88ae0605cc81", + "sha256:bc56c2a1b0d15aa3eaa5a60c9f3f8e3e565303b465dbf57a1b730e7a2b9844e0", + "sha256:bd1184ceb3f87651a67b2708d4c3338e9b10c5df903f2e3776b62303b26cb631", + "sha256:d06016f7f8625a1825ba3732081d77c94589dca78b7a3fc072194851e88461a4", + "sha256:d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==5.9.8" + }, "pycodestyle": { "hashes": [ "sha256:46f0fb92069a7c28ab7bb558f05bfc0110dac69a0cd23c61ea0040283a9d78b3", @@ -534,6 +598,22 @@ "markers": "python_version >= '3.8'", "version": "==3.2.0" }, + "pygments": { + "hashes": [ + "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199", + "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a" + ], + "markers": "python_version >= '3.8'", + "version": "==2.18.0" + }, + "rich": { + "hashes": [ + "sha256:1760a3c0848469b97b558fc61c85233e3dafb69c7a071b4d60c38099d3cd4c06", + "sha256:8260cda28e3db6bf04d2d1ef4dbc03ba80a824c88b0e7668a0f23126a424844a" + ], + "markers": "python_full_version >= '3.7.0'", + "version": "==13.8.1" + }, "typing-extensions": { "hashes": [ "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", diff --git a/client_example.py b/client_example.py index 59408b9..dc63461 100644 --- a/client_example.py +++ b/client_example.py @@ -12,19 +12,19 @@ { "input": { "input_type": "TARGET_FOLDER", - "file_path": f"{root.joinpath("examples", "target_folder")}", + "file_path": f'{root.joinpath("examples", "target_folder")}', }, }, { "input": { "input_type": "KNOWN_DATASET", - "file_path": f"{root.joinpath("examples", "known_dataset")}", + "file_path": f'{root.joinpath("examples", "known_dataset")}', }, }, { "input": { "input_type": "OUTPUT_SQL_PATH", - "file_path": f"{root.joinpath("examples", "out", "known_content_hashes.sqlite")}", + "file_path": f'{root.joinpath("examples", "out", "known_content_hashes.sqlite")}', }, }, ] @@ -32,7 +32,7 @@ data_type = DataTypes.CUSTOM # The type of the input data # Parameters of the model -parameters = {"block_size": 4096, "target_probability": 0.99} +parameters = {"block_size": 4, "target_probability": 0.99} response = client.request(inputs, data_type, parameters) # Send a request to the server diff --git a/small_blk_forensics/ml/model.py b/small_blk_forensics/ml/model.py index c41dc20..12ea5fb 100644 --- a/small_blk_forensics/ml/model.py +++ b/small_blk_forensics/ml/model.py @@ -42,6 +42,7 @@ def run_with_known_content_directory( # Hash the target directory and check for matches using random blocks response = self._hash_directory_random_blocks(target_directory, db_conn) + db_conn.close() return response @@ -60,6 +61,7 @@ def run_with_known_content_sqlite( # Hash the target directory and check for matches response = self._hash_directory_random_blocks(target_directory, db_conn) + db_conn.close() return response @@ -275,6 +277,7 @@ def hash_directory(self, directory: Path, out_sql_path: Path) -> None: # Fully hash the known content directory and store hashes in the output directory's database self._hash_directory(directory, db_conn, out_sql_path) + db_conn.close() def _hash_directory(self, directory: Path, db_conn: sqlite3.Connection, out_path: Path) -> None: """ diff --git a/test/rere.py b/test/rere.py index 4df44ae..99ed383 100644 --- a/test/rere.py +++ b/test/rere.py @@ -51,14 +51,26 @@ def write_blob_field(f: BinaryIO, name: bytes, blob: bytes): f.write(b'\n') def capture(shell: str) -> dict: - print(f"CAPTURING: {shell}") - process = subprocess.run(['sh', '-c', shell], capture_output = True) - return { - 'shell': shell, - 'returncode': process.returncode, - 'stdout': process.stdout, - 'stderr': process.stderr, - } + if shell.startswith("(NOCAPTURE)"): + new_shell = " ".join(shell.split(" ")[1:]) + print(f"RUNNING: {new_shell}") + process = subprocess.run(['sh', '-c', new_shell]) + return { + 'shell': shell, + 'returncode': 0, + 'stdout': b'', + 'stderr': b'', + } + + else: + print(f"CAPTURING: {shell}") + process = subprocess.run(['sh', '-c', shell], capture_output = True) + return { + 'shell': shell, + 'returncode': process.returncode, + 'stdout': process.stdout, + 'stderr': process.stderr, + } def load_list(file_path: str) -> list[str]: with open(file_path) as f: diff --git a/test/test.list b/test/test.list index 36e4e04..239105f 100644 --- a/test/test.list +++ b/test/test.list @@ -6,3 +6,9 @@ python cmd_interface.py generate_hashes --output_sql ./examples/out/known_conten # Run SBF on a pre-generated known content directory SQLite DB and target directory python cmd_interface.py hash_random_blocks --input_sql ./examples/out/known_content_hashes.sqlite --target_directory ./examples/target_folder --block_size 4 | head -n -2 + +# Server-client test +(NOCAPTURE) hap run --check python -m small_blk_forensics.backend.server +sleep 3 && python client_example.py +hap kill --all +hap cleanall diff --git a/test/test.list.bi b/test/test.list.bi index 88b5b46..cadd83a 100644 --- a/test/test.list.bi +++ b/test/test.list.bi @@ -1,4 +1,4 @@ -:i count 8 +:i count 14 :b shell 59 # Run SBF on a known content directory and target directory :i returncode 0 @@ -77,3 +77,49 @@ Results: :b stderr 0 +:b shell 0 + +:i returncode 0 +:b stdout 0 + +:b stderr 0 + +:b shell 20 +# Server-client test +:i returncode 0 +:b stdout 0 + +:b stderr 0 + +:b shell 72 +(NOCAPTURE) hap run --check python -m small_blk_forensics.backend.server +:i returncode 0 +:b stdout 0 + +:b stderr 0 + +:b shell 35 +sleep 3 && python client_example.py +:i returncode 0 +:b stdout 591 +INFO: Received a response +[{'result': {'found': True, 'target_file': '/Users/atharvakale/workspace/umass/596e-cs/individual-project/small-block-forensics/examples/target_folder/sample.txt', 'known_dataset_file': '/Users/atharvakale/workspace/umass/596e-cs/individual-project/small-block-forensics/examples/known_dataset/sample.txt', 'block_num_in_known_dataset': 0, 'block_num_in_target': 1}, 'text': 'RESULTS'}, {'result': '/Users/atharvakale/workspace/umass/596e-cs/individual-project/small-block-forensics/examples/out/known_content_hashes.sqlite', 'text': 'Successfully stored hashes'}] + +:b stderr 0 + +:b shell 14 +hap kill --all +:i returncode 0 +:b stdout 26 +💀 Killed 1 active haps + +:b stderr 0 + +:b shell 12 +hap cleanall +:i returncode 0 +:b stdout 29 +🧲 Deleted 1 finished haps + +:b stderr 0 +