Added visualiser. (#32)

* Added visualiser.

* Removed binary files.

* Removed .DS_Store binary file.

visualiser/README.md

@@ -0,0 +1,36 @@
+# Visualiser
+
+This visualiser works by showing the nodes being communicated with during an operation. It shows the nodes involved in the lookup relative to the hash space. 
+
+~~You can either use the command line version (`index.py`) or the Dash web application (`app.py`).~~
+
+## Usage
+
+To run it just run:
+```
+python app.py <logs_dir>
+```
+
+Where the `<logs_dir>` is the path that points to the folder with the logs in it (the `message.csv` and `operation.csv` files).
+
+<!-- To visualise a specific operation (using the command line tool - `index.py`) just pass in the operation ID:
+```
+python index.py <operation_id>
+```
+Example:
+```
+python index.py 21
+``` -->
+
+## How It Works
+1. Read and store the contents of operations.csv.
+2. Read and store the contents of msg.csv.
+3. Visualise a single or multiple operations.
+4. Get the src node of the operation.
+5. Get all the message IDs involved in that operation.
+6. Get all the message destinations nodes for each message ID.
+7. Plot and annotate (label) the operation source node.
+8. Plot and annotate (label) all of the message destination nodes.
+9. Make the x-axis scale in a log-based manner so that the nodes are nicely presented (otherwise the few nodes with smaller values will get squashed up together).
+10. Hide the axes and their information
+11. If multiple operations are being visualised then go back to 4.

visualiser/app.py

@@ -0,0 +1,218 @@
+from dash import dcc, Input, Output, callback, html, dash_table
+from plotly.subplots import make_subplots
+from rich.console import Console
+from pprint import pprint
+from decimal import Decimal
+
+import os
+import sys
+import dash
+import dash_bootstrap_components as dbc
+import pandas as pd
+import plotly.graph_objects as go
+
+console = Console()
+
+args = sys.argv[1:]
+
+if len(args) > 0:
+    logsdir = args[0]
+else:
+    logsdir = "log_folder"
+    console.print(f"No <logs_dir> argument given. Using 'log_folder'.", style="bold white")
+
+if not os.path.exists(logsdir):
+    console.print(f"{logsdir} is NOT a valid path.", style="bold red")
+    sys.exit()
+
+csv_files = [_ for _ in os.listdir(logsdir) if '.csv' in _]
+
+if len(csv_files) == 0:
+    console.print(f"No files found in {logsdir}.", style="bold red")
+    sys.exit()
+
+op_file = [_ for _ in csv_files if 'op' in _][0]
+msg_file = [_ for _ in csv_files if 'message' in _ or 'msg' in _][0]
+
+op_df = pd.read_csv(os.path.join(logsdir, op_file), index_col=False)
+
+old_ids = op_df['src']
+mapping = {}
+for i in range(len(old_ids)):
+    mapping[old_ids[i]] = i + 1
+
+new_ids = []
+for id in old_ids:
+    new_id = mapping[id]
+    new_ids.append(new_id)
+
+op_df["new_src"] = new_ids
+
+columns = [{"name": i, "id": i} for i in ['id', 'new_src', 'messages', 'type', 'src', 'start', 'stop']]
+
+app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
+
+app.layout = html.Div(
+    [
+        dbc.Row(
+            [
+                html.H1(
+                    "Kademlia Visualiser",
+                    style={
+                        "width": "25%"
+                    }
+                ),
+                # html.Div(
+                #     dcc.Input(
+                #         id="path-input",
+                #         type="text",
+                #         placeholder="Enter the path to the logs here",
+                #         # TODO: Delete the next line.
+                #         value="./log_folder",
+                #         style={
+                #             "width": "100%",
+                #             "padding": "0.5vh 0.5vw"
+                #         }
+                #     ),
+                #     style={
+                #         "width": "20%"
+                #     }
+                # )
+            ],
+            style={
+                # "outline": "1px solid black",
+                "display": "flex",
+                "justifyContent": "space-between",
+                "alignItems": "center",
+                "padding": "0vh 1vw",
+                "maxWidth": "100vw",
+                "borderBottom": "2px solid #aaa",
+                "overflowX": "hidden"
+            }
+        ),
+        dbc.Row(
+            [
+                dbc.Col(
+                    [
+                        html.P(
+                            "Click on an operation to plot it.",
+                            style={"textAlign": "center"}
+                        ),
+                        html.Div(
+                            dash_table.DataTable(
+                                data=op_df.to_dict("records"), 
+                                columns=columns,
+                                id="table",
+                                style_cell={"textAlign": "center"}
+                            ),
+                            style={
+                                "maxHeight": "90vh",
+                                "overflowY": "scroll"
+                            }
+                        )
+                    ],
+                    width=4
+                ),
+                dbc.Col(
+                    [
+                        dcc.Graph(
+                            id="graph", 
+                            style={
+                                "height": "95vh"
+                            }
+                        ),
+                    ],
+                    width=8
+                )
+            ],
+            style={
+                "padding": "1vh 1vw",
+                "maxWidth": "100vw"
+            }
+        )
+    ],
+    style={
+        "maxWidth": "100vw",
+        "maxHeight": "100vh",
+        "overflow": "hidden"
+    }
+)
+
+@callback(Output("graph", "figure"), Input("table", "active_cell"))
+def update_graphs(active_cell):
+    msg_df = pd.read_csv(os.path.join(logsdir, msg_file), index_col=False)
+
+    if active_cell:
+        op_id = active_cell["row_id"]
+
+        for _, row in op_df.iterrows():
+            if int(row["id"]) == int(op_id):
+                op = row
+
+        message_ids = [int(x) for x in op["messages"].split("|") if x]
+        msg_dsts = list((msg_df.loc[ msg_df["id"].isin(message_ids) ])["dst"])
+
+        # ? op_id is the id of the op so that we can plot multiple operations by using the op_id on the y-axis
+        # ? op_src is the src node (node all the way on the left)
+        # ? msg_dsts is the list of nodes that have received from the op_src (nodes on the line in an ordered fashion)
+        x = [op["src"]] + msg_dsts
+        y = [op_id] * len(msg_dsts) + [op_id]
+
+        fig = go.Figure(
+            go.Scatter(
+                x = x, 
+                y = y, 
+                marker=dict(size = 50)
+            )
+        )
+
+        fig.update_xaxes(type="log")
+        fig.update_layout(title="Operation " + str(op_id))
+
+        return fig
+    else:
+        if len(op_df) > 1:
+
+            op_srcs = []
+            for index, row in op_df.iterrows():
+                op_srcs.append(Decimal(row.loc["src"]))
+
+            max_op_id = op_df.iloc[ op_df['src'].astype(float).idxmax() ]
+            min_op_id = op_df.iloc[ op_df['src'].astype(float).idxmin() ]
+        else:
+            max_op_id = op_df["id"]
+            min_op_id = op_df["id"]
+
+        fig = make_subplots(rows=1, cols=1)
+
+        for _, row in op_df.iterrows():
+            message_ids = [int(x) for x in row["messages"].split("|") if len(x) > 0]
+            msg_dsts = list((msg_df.loc[ msg_df["id"].isin(message_ids) ])["dst"])
+
+            # ? op_id is the id of the op so that we can plot multiple operations by using the op_id on the y-axis
+            # ? op_src is the src node (node all the way on the left)
+            # ? msg_dsts is the list of nodes that have received from the op_src (nodes on the line in an ordered fashion)
+
+            x = [row["src"]] + msg_dsts
+            y = [row["id"]] * len(x)
+
+            fig.add_trace(
+                go.Scatter(
+                    x = x, 
+                    y = y, 
+
+                    marker=dict(size = 50)
+                )
+            )
+
+            fig.update_xaxes(type="log")
+            fig.update_layout(
+                yaxis_range=[min_op_id, max_op_id],
+                showlegend=False,
+                margin=dict(t=0, r=0)
+            )
+
+        return fig
+
+if __name__ == "__main__":
+    app.run_server(debug=True)

visualiser/index.py

@@ -0,0 +1,101 @@
+from pprint import pprint
+import matplotlib.pyplot as plt
+import pandas as pd
+import sys
+
+from rich.console import Console
+console = Console(record = True)
+
+args = sys.argv[1:]
+if len(args) > 1:
+    console.print("Too many arguments passed in.", style="bold red")
+    sys.exit()
+elif len(args) == 1:
+    op_id = int(args[0])
+
+def visualise_op(op_id, op_df, msg_df):
+    fig, ax = plt.subplots(figsize=(30, 30))
+
+    for _, row in op_df.iterrows():
+        if int(row["id"]) == int(op_id):
+            op_src = row["src"]
+
+    try:
+        op_src
+    except NameError:
+        console.print("Couldn't find an operation with the specified ID: " + str(op_id), style="bold red")
+        sys.exit()
+
+    ax.set_ylim(ymin=op_id * 0.8, ymax= op_id * 1.2)
+
+    message_ids = [int(x) for x in row["messages"].split("|")]
+    msg_dsts = list((msg_df.loc[ msg_df["id"].isin(message_ids) ])["dst"])
+
+    # ? op_id is the id of the op so that we can plot multiple operations by using the op_id on the y-axis
+    # ? op_src is the src node (node all the way on the left)
+    # ? msg_dsts is the list of nodes that have received from the op_src (nodes on the line in an ordered fashion)
+
+    node_color = "#c1e7ff"
+    node_label_color = "#004c6d"
+    dot_size = 2000
+
+    ax.scatter([op_src], [op_id], s=dot_size, color=node_color, edgecolors=node_label_color)
+    ax.annotate(str(op_src)[:5], (op_src, op_id), color=node_label_color, fontweight="bold", fontsize=40)
+
+    ax.scatter(msg_dsts, [op_id] * len(msg_dsts), s=dot_size, color=node_color, edgecolors=node_label_color)
+
+    for dst in msg_dsts:
+        ax.annotate(str(dst)[:5], (dst, op_id), color=node_label_color, fontweight="bold", fontsize=40)
+
+    ax.set_xscale("log", base=2)
+    ax.axis("off")
+
+    plt.savefig("op_" +str(op_id)+ ".png")
+
+def visualise_ops(op_df, msg_df):
+    _, ax = plt.subplots(figsize=(30, 30))
+
+    max_op_id = op_df.loc[op_df["src"].idxmax()]["id"]
+    min_op_id = op_df.loc[op_df["src"].idxmin()]["id"]
+
+    ax.set_ylim(ymin=min_op_id * 0.8, ymax= max_op_id * 1.2)
+    for _, row in op_df.iterrows():
+        op_id = row["id"]
+        op_src = row["src"]
+
+        message_ids = [int(x) for x in row["messages"].split("|")]
+        msg_dsts = list((msg_df.loc[ msg_df["id"].isin(message_ids) ])["dst"])
+
+        # ? op_id is the id of the op so that we can plot multiple operations by using the op_id on the y-axis
+        # ? op_src is the src node (node all the way on the left)
+        # ? msg_dsts is the list of nodes that have received from the op_src (nodes on the line in an ordered fashion)
+
+        node_color = "#c1e7ff"
+        node_label_color = "#004c6d"
+        dot_size = 2000
+
+        ax.scatter([op_src], [op_id], s=dot_size, color=node_color, edgecolors=node_label_color)
+        ax.annotate(str(op_src)[:5], (op_src, op_id), color=node_label_color, fontweight="bold", fontsize=40)
+
+        ax.scatter(msg_dsts, [op_id] * len(msg_dsts), s=dot_size, color=node_color, edgecolors=node_label_color)
+
+        for dst in msg_dsts:
+            ax.annotate(str(dst)[:5], (dst, op_id), color=node_label_color, fontweight="bold", fontsize=40)
+
+        ax.set_xscale("log")
+        ax.axis("off")
+
+        plt.savefig("output.png")
+
+with console.status("Visualising..."):
+    # ? Read contents of operation
+    op_df = pd.read_csv('log_folder/operations.csv')
+    # src | id | type | messages
+    # ? Read contents of msg
+    msg_df = pd.read_csv('log_folder/msg.csv')
+    # dst | src | id | type | status
+    try:
+        op_id
+        visualise_op(op_id, op_df, msg_df)
+    except NameError:
+        visualise_ops(op_df, msg_df)

visualiser/log_folder/msg.csv

@@ -0,0 +1,10 @@
+dst,src,id,type,status
+2,1,1,MSG_FIND,received
+20,1,2,MSG_FIND,received
+100,1,3,MSG_FIND,received
+256000,1,4,MSG_FIND,received
+6,1,5,MSG_FIND,received
+2,2,6,MSG_FIND,received
+3,2,7,MSG_FIND,received
+4,2,8,MSG_FIND,received
+5,2,9,MSG_FIND,received

visualiser/log_folder/operations.csv

@@ -0,0 +1,3 @@
+src,id,type,messages
+1,21,OP_FIND,1|2|3|4|5
+2,22,OP_FIND,6|7|8|9