Modernize the packaging (#416)

- Gets rid of deprecation notices (#440)
- Will be publishable to Pypi
- Using modern pyproject.toml
- While we are there, breaking change: scripts names without .py
- Minimum Python version is 3.7

.coveragerc

@@ -3,7 +3,6 @@ branch = True
 source =
     pyrdp
     twisted
-    bin
 
 [report]
 # Regexes for lines to exclude from consideration

.github/workflows/ci.yml

@@ -16,20 +16,20 @@ jobs:
     - uses: actions/checkout@v2
     - name: Build Docker image
       run: docker build -t pyrdp .
-    - name: "Smoke test docker image: pyrdp-convert.py"
-      run: docker run pyrdp pyrdp-convert.py -h
-    - name: "Smoke test docker image: pyrdp-player.py"
-      run: docker run pyrdp pyrdp-player.py -h
-    - name: "Smoke test docker image: pyrdp-mitm.py"
-      run: docker run pyrdp pyrdp-mitm.py -h
+    - name: "Smoke test docker image: pyrdp-convert"
+      run: docker run pyrdp pyrdp-convert -h
+    - name: "Smoke test docker image: pyrdp-player"
+      run: docker run pyrdp pyrdp-player -h
+    - name: "Smoke test docker image: pyrdp-mitm"
+      run: docker run pyrdp pyrdp-mitm -h
     - name: Build slim Docker image
       run: docker build -f Dockerfile.slim -t pyrdp .
-    - name: "Smoke test docker image: pyrdp-convert.py"
-      run: docker run pyrdp pyrdp-convert.py -h
-    - name: "Smoke test docker image: pyrdp-player.py"
-      run: docker run pyrdp pyrdp-player.py -h
-    - name: "Smoke test docker image: pyrdp-mitm.py"
-      run: docker run pyrdp pyrdp-mitm.py -h
+    - name: "Smoke test docker image: pyrdp-convert"
+      run: docker run pyrdp pyrdp-convert -h
+    - name: "Smoke test docker image: pyrdp-player"
+      run: docker run pyrdp pyrdp-player -h
+    - name: "Smoke test docker image: pyrdp-mitm"
+      run: docker run pyrdp pyrdp-mitm -h
 
   install-and-test-ubuntu:
     runs-on: ubuntu-latest
@@ -83,7 +83,7 @@ jobs:
         working-directory: ./
         run: coverage run test/test_prerecorded.py
 
-      - name: pyrdp-mitm.py initialization integration test
+      - name: pyrdp-mitm initialization integration test
         working-directory: ./
         run: coverage run --append test/test_mitm_initialization.py dummy_value
 
@@ -141,17 +141,17 @@ jobs:
         working-directory: ./
         run: coverage run test/test_prerecorded.py
 
-      - name: pyrdp-mitm.py initialization test
+      - name: pyrdp-mitm initialization test
         working-directory: ./
         run: coverage run --append test/test_mitm_initialization.py dummy_value
 
-      - name: pyrdp-player.py read a replay in headless mode test
+      - name: pyrdp-player read a replay in headless mode test
         working-directory: ./
-        run: coverage run --append bin/pyrdp-player.py --headless test/files/test_session.replay
+        run: coverage run --append -m pyrdp.bin.player --headless test/files/test_session.replay
 
-      - name: pyrdp-convert.py to MP4
+      - name: pyrdp-convert to MP4
         working-directory: ./
-        run: coverage run --append bin/pyrdp-convert.py test/files/test_convert.pyrdp -f mp4
+        run: coverage run --append -m pyrdp.bin.convert test/files/test_convert.pyrdp -f mp4
 
       - name: Verify the MP4 file
         working-directory: ./

CHANGELOG.adoc

@@ -9,6 +9,11 @@ For a detailed view of what has changed, refer to the {uri-repo}/commits/master[
 
 == v<unreleased> - <date>
 
+=== Backwards Compatibility Changes
+
+* All tools lost their `.py` suffix. For example, `pyrdp-mitm.py` is now `pyrdp-mitm`.
+* Requires Python 3.7
+
 === Enhancements
 
 * Support for RDP version 10.11 ({uri-issue}433[#433])
@@ -24,6 +29,7 @@ For a detailed view of what has changed, refer to the {uri-repo}/commits/master[
 
 === Infrastructure
 
+* Project packaging modernized, getting rid of `pkg_resources` deprecation warnings ({uri-issue}416[#416], {uri-issue}440[#440])
 * Added Python 3.10 tests on Windows to CI test configuration ({uri-issue}439[#439])
 * Replaced Python 3.10 with Python 3.11 for CI test configuration ({uri-issue}438[#438])
 

Dockerfile

@@ -19,23 +19,27 @@ RUN python3 -m venv /opt/venv
 # Make sure we use the virtualenv:
 ENV PATH="/opt/venv/bin:$PATH"
 
+# Python packaging tooling evolved quickly, we need to get latest, especially on old Pythons
+RUN pip --no-cache-dir install -U pip setuptools wheel
+
 # Install dependencies only (speeds repetitive builds)
 COPY requirements.txt /pyrdp/requirements.txt
 RUN cd /pyrdp && \
-    pip3 install wheel && \
-    pip3 --no-cache-dir install --default-timeout=100 -r requirements.txt
+    pip --no-cache-dir install --default-timeout=100 -r requirements.txt
 
 # Compile only our C extension and install
 # This way changes to source tree will not trigger full images rebuilds
-COPY ext/rle.c /pyrdp/ext/rle.c
-COPY setup.py /pyrdp/setup.py
+COPY ext/rle.c /pyrdp/ext/
+COPY setup.py /pyrdp/
+COPY README.md /pyrdp/
+COPY pyproject.toml /pyrdp/
+COPY pyrdp/ /pyrdp/pyrdp/
 RUN cd /pyrdp \
-    && python setup.py build_ext \
-    && python setup.py install_lib
+    && pip --no-cache-dir install '.[full]'
 
 
 # Handles runtime only (minimize size for distribution)
-FROM ubuntu:20.04 AS docker-image
+FROM ubuntu:20.04 AS runtime-image
 
 # Install runtime dependencies except pre-built venv
 ARG DEBIAN_FRONTEND=noninteractive
@@ -63,14 +67,10 @@ RUN useradd --create-home --home-dir /home/pyrdp pyrdp
 # Make sure we use the virtualenv
 ENV PATH="/opt/venv/bin:$PATH"
 
-# Install python source and package
+# Install python source
 # NOTE: we are no longer doing this in the compile image to avoid long image rebuilds in development
 COPY --from=compile-image /pyrdp /pyrdp
-COPY bin/ /pyrdp/bin/
 COPY pyrdp/ /pyrdp/pyrdp/
-COPY setup.py /pyrdp/setup.py
-RUN cd /pyrdp \
-    && python setup.py install
 
 USER pyrdp
 

Dockerfile.slim

@@ -24,23 +24,25 @@ ENV PATH="/opt/venv/bin:$PATH"
 
 # Required for ARM builds
 # Building dependencies didn't work without an upgraded pip and wheel on ARM
-RUN pip3 --no-cache-dir install -U pip wheel
+RUN pip3 --no-cache-dir install -U pip setuptools wheel
 
 # Install dependencies only (speeds repetitive builds)
 COPY requirements-slim.txt /pyrdp/requirements.txt
 RUN cd /pyrdp && pip3 --no-cache-dir install -r requirements.txt
 
 # Compile only our C extension and install
 # This way changes to source tree will not trigger full images rebuilds
-COPY ext/rle.c /pyrdp/ext/rle.c
-COPY setup.py /pyrdp/setup.py
+COPY ext/rle.c /pyrdp/ext/
+COPY setup.py /pyrdp/
+COPY README.md /pyrdp/
+COPY pyproject.toml /pyrdp/
+COPY pyrdp/ /pyrdp/pyrdp/
 RUN cd /pyrdp \
-    && python setup.py build_ext \
-    && python setup.py install_lib
+    && pip --no-cache-dir install .
 
 
 # Handles runtime only (minimize size for distribution)
-FROM ubuntu:20.04 AS docker-image
+FROM ubuntu:20.04 AS runtime-image
 
 # Install runtime dependencies except pre-built venv
 ARG DEBIAN_FRONTEND=noninteractive
@@ -63,11 +65,7 @@ ENV PATH="/opt/venv/bin:$PATH"
 # Install python source and package
 # NOTE: we are no longer doing this in the compile image to avoid long image rebuilds in development
 COPY --from=compile-image /pyrdp /pyrdp
-COPY bin/ /pyrdp/bin/
 COPY pyrdp/ /pyrdp/pyrdp/
-COPY setup.py /pyrdp/setup.py
-RUN cd /pyrdp \
-    && python setup.py install
 
 USER pyrdp
 

README.md

@@ -59,9 +59,9 @@ research use cases in mind.
 
 
 ## Supported Systems
-PyRDP should work on Python 3.6 and up on the x86-64, ARM and ARM64 platforms.
+PyRDP should work on Python 3.7 and up on the x86-64, ARM and ARM64 platforms.
 
-This tool has been tested to work on Python 3.6 on Linux (Ubuntu 18.04, 20.04), Raspberry Pi and Windows
+This tool has been tested to work on Python 3.7 on Linux (Ubuntu 20.04, 22.04), Raspberry Pi and Windows
 (see section [Installing on Windows](#installing-on-windows)). It has not been tested on macOS.
 
 ## Installing
@@ -192,7 +192,7 @@ This should install all the dependencies required to run PyRDP.
 For example, to open the player:
 
 ```
-python venv\Scripts\pyrdp-player.py
+python venv\Scripts\pyrdp-player
 ```
 
 If you ever want to leave your virtual environment, you can simply deactivate it:
@@ -232,12 +232,12 @@ docker buildx build --platform linux/arm,linux/amd64 -t pyrdp -f Dockerfile.slim
 ## Using PyRDP
 
 ### Using the PyRDP Monster-in-the-Middle
-Use `pyrdp-mitm.py <ServerIP>` or `pyrdp-mitm.py <ServerIP>:<ServerPort>` to run the MITM.
+Use `pyrdp-mitm <ServerIP>` or `pyrdp-mitm <ServerIP>:<ServerPort>` to run the MITM.
 
 Assuming you have an RDP server running on `192.168.1.10` and listening on port 3389, you would run:
 
 ```
-pyrdp-mitm.py 192.168.1.10
+pyrdp-mitm 192.168.1.10
 ```
 
 When running the MITM for the first time a directory called `pyrdp_output/`
@@ -289,7 +289,7 @@ If key generation didn't work or you want to use a custom key and certificate, y
 `-c` and `-k` arguments:
 
 ```
-pyrdp-mitm.py 192.168.1.10 -k private_key.pem -c certificate.pem
+pyrdp-mitm 192.168.1.10 -k private_key.pem -c certificate.pem
 ```
 
 ##### Monster-in-the-Middle Network Level Authentication (NLA) connections
@@ -344,7 +344,7 @@ If you want to see live RDP connections through the PyRDP player, you will need
 player is listening using the `-i` and `-d` arguments. Note: the port argument is optional, the default port is 3000.
 
 ```
-pyrdp-mitm.py 192.168.1.10 -i 127.0.0.1 -d 3000
+pyrdp-mitm 192.168.1.10 -i 127.0.0.1 -d 3000
 ```
 
 ##### Connecting to a PyRDP player when the MITM is running on a server
@@ -355,7 +355,7 @@ port as arguments to the MITM. For example, if port 4000 on the server is forwar
 this would be the command to use:
 
 ```
-pyrdp-mitm.py 192.168.1.10 -i 127.0.0.1 -d 4000
+pyrdp-mitm 192.168.1.10 -i 127.0.0.1 -d 4000
 ```
 
 #### Running payloads on new connections
@@ -412,7 +412,7 @@ This will block the client's input / output for 5 seconds to hide the console an
 After 5 seconds, input / output is restored back to normal.
 
 #### Other MITM arguments
-Run `pyrdp-mitm.py --help` for a full list of arguments.
+Run `pyrdp-mitm --help` for a full list of arguments.
 
 ##### `--no-downgrade`
 
@@ -461,34 +461,34 @@ support can be added as required. (Make sure that the trace does not contain sen
 [gdi]: https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-rdpegdi/745f2eee-d110-464c-8aca-06fc1814f6ad
 
 ### Using the PyRDP Player
-Use `pyrdp-player.py` to run the player.
+Use `pyrdp-player` to run the player.
 
 #### Playing a replay file
 You can use the menu to open a new replay file: File > Open.
 
 You can also open replay files when launching the player:
 
 ```
-pyrdp-player.py <FILE1> <FILE2> ...
+pyrdp-player <FILE1> <FILE2> ...
 ```
 
 #### Listening for live connections
 The player always listens for live connections. By default, the listening port is 3000, but it can be changed:
 
 ```
-pyrdp-player.py -p <PORT>
+pyrdp-player -p <PORT>
 ```
 
 #### Changing the listening address
 By default, the player only listens to connections coming from the local machine. We do not recommend opening up the player
 to other machines. If you still want to change the listening address, you can do it with `-b`:
 
 ```
-pyrdp-player.py -b <ADDRESS>
+pyrdp-player -b <ADDRESS>
 ```
 
 #### Other player arguments
-Run `pyrdp-player.py --help` for a full list of arguments.
+Run `pyrdp-player --help` for a full list of arguments.
 
 ### Using the PyRDP Certificate Cloner
 
@@ -502,10 +502,10 @@ you're trying to trick a legitimate user into going through your MITM. Using a c
 certificate could increase your success rate.
 
 #### Cloning a certificate
-You can clone a certificate by using `pyrdp-clonecert.py`:
+You can clone a certificate by using `pyrdp-clonecert`:
 
 ```
-pyrdp-clonecert.py 192.168.1.10 cert.pem -o key.pem
+pyrdp-clonecert 192.168.1.10 cert.pem -o key.pem
 ```
 
 The `-o` parameter defines the path name to use for the generated private key.
@@ -514,11 +514,11 @@ The `-o` parameter defines the path name to use for the generated private key.
 If you want to use your own private key instead of generating a new one:
 
 ```
-pyrdp-clonecert.py 192.168.1.10 cert.pem -i input_key.pem
+pyrdp-clonecert 192.168.1.10 cert.pem -i input_key.pem
 ```
 
 #### Other cloner arguments
-Run `pyrdp-clonecert.py --help` for a full list of arguments.
+Run `pyrdp-clonecert --help` for a full list of arguments.
 
 ### Using PyRDP Convert
 
@@ -535,19 +535,19 @@ The following outputs are supported:
 
 - MP4 video file
 - JSON: a sequence of low-level events serialized in JSON format
-- Replay file compatible with `pyrdp-player.py`
+- Replay file compatible with `pyrdp-player`
 
 Encrypted (TLS) network captures require the TLS master secrets to be provided using `--secrets ssl.log`.
 
 ```
 # Export the session coming client 10.2.0.198 to a .pyrdp file.
-pyrdp-convert.py --src 10.2.0.198 --secrets ssl.log -o path/to/output capture.pcap
+pyrdp-convert --src 10.2.0.198 --secrets ssl.log -o path/to/output capture.pcap
 
 # Or as an MP4 video
-pyrdp-convert.py --src 10.2.0.198 --secrets ssl.log -o path/to/output -f mp4 capture.pcap
+pyrdp-convert --src 10.2.0.198 --secrets ssl.log -o path/to/output -f mp4 capture.pcap
 
 # List the sessions in a network trace, along with the decryptable ones.
-pyrdp-convert.py --list-only capture.pcap
+pyrdp-convert --list-only capture.pcap
 ```
 
 Note that MP4 conversion requires libavcodec and ffmpeg, so this may require extra steps on Windows.
@@ -609,15 +609,15 @@ In most of the monster-in-the-middle cases you will need to map a port of your h
 For example, to listen on 3389 (RDP's default port) on all interfaces, use:
 
 ```
-docker run -p 3389:3389 gosecure/pyrdp pyrdp-mitm.py 192.168.1.10
+docker run -p 3389:3389 gosecure/pyrdp pyrdp-mitm 192.168.1.10
 ```
 
 #### Logs and Artifacts Storage
 
 To store the PyRDP output permanently (logs, files, etc.), add the `--volume` (`-v`) option to the previous command. In this example we store the files relatively to the current directory in `pyrdp_output`:
 
 ```
-docker run -v $PWD/pyrdp_output:/home/pyrdp/pyrdp_output -p 3389:3389 gosecure/pyrdp pyrdp-mitm.py 192.168.1.10
+docker run -v $PWD/pyrdp_output:/home/pyrdp/pyrdp_output -p 3389:3389 gosecure/pyrdp pyrdp-mitm 192.168.1.10
 ```
 
 Make sure that your destination directory is owned by a user with a UID of 1000, otherwise you will get permission denied errors.
@@ -628,7 +628,7 @@ If you are the only non-root user on the system, usually your user will be assig
 If you want PyRDP to log the host IP address in its logs, you can set the `HOST_IP` environment variable when using `docker run`:
 
 ```
-docker run -p 3389:3389 -e HOST_IP=192.168.1.9 gosecure/pyrdp pyrdp-mitm.py 192.168.1.10
+docker run -p 3389:3389 -e HOST_IP=192.168.1.9 gosecure/pyrdp pyrdp-mitm 192.168.1.10
 ```
 
 #### Using the GUI Player in Docker
@@ -639,7 +639,7 @@ You also need to expose the host's network and prevent Qt from using the MIT-SHM
 To do so, add the `-e` and `--net` options to the run command:
 
 ```
-docker run -e DISPLAY=$DISPLAY -e QT_X11_NO_MITSHM=1 --net=host gosecure/pyrdp pyrdp-player.py
+docker run -e DISPLAY=$DISPLAY -e QT_X11_NO_MITSHM=1 --net=host gosecure/pyrdp pyrdp-player
 ```
 
 Keep in mind that exposing the host's network to docker can compromise the isolation between your container and the host.