Add Outline VPN stats API POC

apps/climatemappedafrica/src/components/ExplorePage/useExplore.js

@@ -107,7 +107,7 @@ function reducer(state, action) {
         newState.secondary = undefined;
       }
       newState.secondary = undefined;
-      newState.slug = newState.primary.geography.code.toLowerCase();
+      newState.slug = newState.primary.geography?.code?.toLowerCase();
 
       return newState;
     }

apps/vpnmanager/package.json

@@ -27,6 +27,7 @@
     "@sentry/nextjs": "^8.22.0",
     "@svgr/webpack": "^8.1.0",
     "@types/jest": "^29.5.12",
+    "better-sqlite3": "^11.2.1",
     "googleapis": "^133.0.0",
     "jest": "^29.7.0",
     "next": "^14.2.5",
@@ -37,6 +38,7 @@
   "devDependencies": {
     "@babel/core": "^7.25.2",
     "@commons-ui/testing-library": "workspace:*",
+    "@types/better-sqlite3": "^7.6.11",
     "@types/node": "^20.14.14",
     "@types/react": "^18.3.3",
     "@types/react-dom": "^18.3.0",

apps/vpnmanager/src/lib/data/database.ts

@@ -0,0 +1,151 @@
+// lib/Database.ts
+import betterSqlite3 from "better-sqlite3";
+import path from "path";
+
+// Define the database file path
+const dbPath = path.resolve(process.cwd(), "data", "database.sqlite");
+const db = betterSqlite3(dbPath);
+
+class Record {
+  ID?: number;
+  userId: string;
+  usage: number;
+  date: string;
+  cumulativeData: number;
+  email: string;
+  accessUrl?: string;
+  createdAt: string;
+
+  constructor(
+    userId: string,
+    usage: number,
+    date: string,
+    cumulativeData: number,
+    email: string,
+    accessUrl?: string,
+    createdAt?: string,
+    ID?: number,
+  ) {
+    this.ID = ID;
+    this.userId = userId;
+    this.usage = usage;
+    this.date = date;
+    this.cumulativeData = cumulativeData;
+    this.email = email;
+    this.accessUrl = accessUrl;
+    this.createdAt = createdAt || new Date().toISOString();
+  }
+}
+
+export interface Filters {
+  email?: string;
+  date?: string;
+  dateBetween?: { start: string; end: string };
+  userId?: string;
+  ID?: number;
+  groupBy?: "email" | "date";
+  orderBy?: string;
+}
+
+class Model {
+  static initialize() {
+    const createTable = `
+      CREATE TABLE IF NOT EXISTS records (
+        ID INTEGER PRIMARY KEY AUTOINCREMENT,
+        userId TEXT NOT NULL,
+        usage INTEGER NOT NULL,
+        date TEXT NOT NULL,
+        cumulativeData INTEGER NOT NULL,
+        email TEXT NOT NULL,
+        accessUrl TEXT,
+        createdAt TEXT NOT NULL
+      )
+    `;
+    db.exec(createTable);
+  }
+
+  static create(record: Record) {
+    const insertData = db.prepare(`
+      INSERT INTO records (userId, usage, date, cumulativeData, email, accessUrl, createdAt)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+    `);
+    const info = insertData.run(
+      record.userId,
+      record.usage,
+      record.date,
+      record.cumulativeData,
+      record.email,
+      record.accessUrl,
+      record.createdAt,
+    );
+    return { ...record, ID: info.lastInsertRowid };
+  }
+
+  static update(ID: number, updates: Partial<Record>) {
+    const setClause = Object.keys(updates)
+      .map((key) => `${key} = ?`)
+      .join(", ");
+    const query = `UPDATE records SET ${setClause} WHERE ID = ?`;
+    const stmt = db.prepare(query);
+    return stmt.run([...Object.values(updates), ID]);
+  }
+
+  static delete(ID: number) {
+    const stmt = db.prepare("DELETE FROM records WHERE ID = ?");
+    return stmt.run(ID);
+  }
+
+  static getAll(filters: Filters = {}) {
+    let query = "SELECT";
+    const params: any[] = [];
+    if (filters.groupBy === "email" || filters.groupBy === "date") {
+      if (filters.groupBy === "email") {
+        query +=
+          " email, userId, accessUrl, SUM(usage) as totalUsage FROM records WHERE 1=1";
+      }
+      if (filters.groupBy === "date") {
+        query += " date, SUM(usage) as totalUsage FROM records WHERE 1=1";
+      }
+    } else {
+      query += " * FROM records WHERE 1=1";
+    }
+    if (filters.email) {
+      query += " AND email = ?";
+      params.push(filters.email);
+    }
+    if (filters.date) {
+      query += " AND date = ?";
+      params.push(filters.date);
+    }
+    if (filters.dateBetween && !filters.date) {
+      query += " AND date BETWEEN ? AND ?";
+      params.push(filters.dateBetween.start, filters.dateBetween.end);
+    }
+    if (filters.userId) {
+      query += " AND userId = ?";
+      params.push(filters.userId);
+    }
+    if (filters.ID) {
+      query += " AND ID = ?";
+      params.push(filters.ID);
+    }
+
+    if (filters.groupBy) {
+      if (filters.groupBy === "email") {
+        query += " GROUP BY email";
+      } else if (filters.groupBy === "date") {
+        query += " GROUP BY date";
+      }
+    }
+    if (filters.orderBy) {
+      query += ` ORDER BY ${filters.orderBy}`;
+    }
+    const stmt = db.prepare(query);
+    return stmt.all(params);
+  }
+}
+
+// Initialize the database
+Model.initialize();
+
+export { Model, Record };

apps/vpnmanager/src/lib/userStatistics.ts

@@ -0,0 +1,83 @@
+import { OutlineVPN } from "./outline";
+import { Model, Record } from "@/vpnmanager/lib/data/database";
+
+interface UserDataUsage {
+  outlineId: string | number;
+  usage: number;
+}
+
+const vpnManager = new OutlineVPN({
+  apiUrl: process.env.NEXT_APP_VPN_API_URL as string,
+});
+
+function calculateDailyDataUsage(userData: UserDataUsage) {
+  if (!userData) {
+    return 0;
+  }
+  const date = new Date();
+  date.setDate(date.getDate() - 1);
+  const { usage, outlineId } = userData;
+  const [res] = Model.getAll({
+    orderBy: "date DESC",
+    userId: outlineId?.toString(),
+  }) as any[];
+  return usage - (res?.cumulativeData || 0);
+}
+
+function addUserStatsToDb(record: Omit<Record, "ID">) {
+  // Find in DB if userId and date exists then update else create
+  const date = new Date();
+  const [res] = Model.getAll({
+    date: `${date.getFullYear()}-${date.getMonth() + 1}-${date.getDate()}`,
+    userId: record?.userId?.toString(),
+  }) as Record[];
+  if (res) {
+    Model.update(res.ID as number, record);
+    return;
+  }
+  Model.create(record);
+}
+// Process Daily user stats. Doesn't matter the time of the day, it just updates.
+export async function processUserStats() {
+  const date = `${new Date().getFullYear()}-${new Date().getMonth() + 1}-${new Date().getDate()}`;
+  const { bytesTransferredByUserId = {} } = await vpnManager.getDataUsage();
+  const allUsers = await vpnManager.getUsers();
+  const unprocessedUsers: Omit<Record, "ID" | "createdAt">[] = Object.keys(
+    bytesTransferredByUserId,
+  ).map((key: string) => {
+    const userDetails = allUsers.find(({ id }) => id === key);
+    const newData = {
+      userId: key,
+      usage: calculateDailyDataUsage({
+        outlineId: key,
+        usage: bytesTransferredByUserId[key],
+      }),
+      date,
+      cumulativeData: bytesTransferredByUserId[key],
+      email: userDetails?.name || "",
+      accessUrl: userDetails?.accessUrl,
+    };
+    addUserStatsToDb({ ...newData, createdAt: new Date().toISOString() });
+    return newData;
+  });
+  return unprocessedUsers;
+}
+
+export async function getStats(filters: { [key: string]: string }) {
+  const validFilters = {
+    email: filters.email,
+    ID: parseInt(filters.ID),
+    userId: filters.userId,
+    groupBy: filters.groupBy as "email" | "date",
+    orderBy: filters.orderBy,
+    dateBetween:
+      filters["dateBetween.start"] && filters["dateBetween.end"]
+        ? {
+            start: filters["dateBetween.start"],
+            end: filters["dateBetween.end"],
+          }
+        : undefined,
+  };
+
+  return Model.getAll(validFilters);
+}

apps/vpnmanager/src/pages/api/processGsheet.ts

@@ -5,7 +5,7 @@ export async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
     const key: string = req.headers["x-api-key"] as string;
     const API_SECRET_KEY = process.env.API_SECRET_KEY;
-    if (!(key && key !== API_SECRET_KEY)) {
+    if (!(key && key === API_SECRET_KEY)) {
       return res.status(403).json({ message: "INVALID_API_KEY" });
     }
     processNewUsers();

apps/vpnmanager/src/pages/api/userStatistics.ts

@@ -0,0 +1,21 @@
+import { NextApiResponse, NextApiRequest } from "next";
+import { processUserStats, getStats } from "@/vpnmanager/lib/userStatistics";
+
+export async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const key: string = req.headers["x-api-key"] as string;
+    const API_SECRET_KEY = process.env.API_SECRET_KEY;
+    if (!(key && key === API_SECRET_KEY)) {
+      return res.status(403).json({ message: "INVALID_API_KEY" });
+    }
+    if (req.method === "POST") {
+      const data = await processUserStats();
+      return res.status(200).json(data);
+    }
+    const response = await getStats(req.query as { [key: string]: string });
+    return res.status(200).json(response);
+  } catch (error) {
+    return res.status(500).json(error);
+  }
+}
+export default handler;

docker-compose.yml

@@ -175,7 +175,11 @@ services:
         - API_SECRET_KEY
     environment:
       NODE_ENV: ${NODE_ENV:-production}
+      NODE_TLS_REJECT_UNAUTHORIZED: 0
+      NEXT_APP_VPN_API_URL: ${NEXT_APP_VPN_API_URL}
     ports:
       - ${VPN_MANAGER_PORT:-3000}:3000
+    volumes:
+      - ./db_data:/apps/vpnmanager/data
 volumes:
   db_data:

packages/hurumap-next/src/Map/Layers.js

@@ -223,7 +223,7 @@ function Layers({
           });
         }
       } else {
-        const mark = new L.Marker(layer.getBounds().getCenter(), {
+        const mark = new L.Marker(layer.getBounds()?.getCenter(), {
           icon: pinIcon,
         });
         mark.on("click", () => {