add preference ordering

server/config.ts

@@ -21,7 +21,7 @@ const defaults = {
   HETZNER_GATEWAY: '', // Gateway handling SSL termination
   HETZNER_SSH_KEYS: '', // IDs of Hetzner SSH keys to access vbrowsers
   HETZNER_IMAGE: '', // ID of Hetzner snapshot image to use for vbrowser
-  VM_MANAGER_CONFIG: '', // Comma-separated list of the pools of VMs to run (provider:size:region:minSize:limitSize:hostname), e.g. Docker:large:US:0:0:localhost,Docker:standard:US:0:0:localhost
+  VM_MANAGER_CONFIG: '', // Comma-separated list of the pools of VMs to run (provider:size:region:minSize:limitSize:hostname), e.g. Docker:large:US:0:1:localhost,Docker:standard:US:0:1:localhost
   SCW_SECRET_KEY: '', // Optional, for Scaleway VMs
   SCW_ORGANIZATION_ID: '', // Optional, for Scaleway VMs
   SCW_GATEWAY: '', // Gateway handling SSL termination

server/server.ts

@@ -627,8 +627,8 @@ async function minuteMetrics() {
     if (room.vBrowser && room.vBrowser.id) {
       // Update the heartbeat
       await postgres?.query(
-        `UPDATE vbrowser SET "heartbeatTime" = $1 WHERE "roomId" = $2 and vmid = $3`,
-        [new Date(), room.roomId, room.vBrowser.id],
+        `UPDATE vbrowser SET "heartbeatTime" = NOW() WHERE "roomId" = $1 and vmid = $2`,
+        [room.roomId, room.vBrowser.id],
       );
 
       const expireTime = getStartOfDay() / 1000 + 86400;

server/vm/base.ts

@@ -129,36 +129,38 @@ export abstract class VMManager {
     if (!roomId || !uid) {
       return undefined;
     }
-    if (this.getMinSize() === 0) {
-      // If the min size is 0 we may never create a VM
-      // So start one if we don't already have one ready or staging
-      const availableCount = await this.getAvailableCount();
-      const stagingCount = await this.getStagingCount();
-      if (!availableCount && !stagingCount) {
-        await this.startVMWrapper();
+    if (this.id === 'Docker' && this.getLimitSize() === 0) {
+      // For Docker if the size is 0, just create a VM directly
+      const vmid = await this.startVM(uuidv4());
+      await postgres.query(
+        `INSERT INTO vbrowser(pool, vmid, "creationTime", state, "roomId", uid, "heartbeatTime", "assignTime") VALUES ($1, $2, NOW(), 'used', $3, $4, NOW(), NOW())`,
+        [this.getPoolName(), vmid, roomId, uid],
+      );
+      redisCount('vBrowserLaunches');
+      const vm = await this.getVM(vmid);
+      if (!vm) {
+        return;
       }
+      return { ...vm, assignTime: Number(new Date()) };
     }
     // Update and use SKIP LOCKED to ensure each consumer gets a different one
-    const getAssignedVM = async (): Promise<VM | undefined> => {
-      const { rows } = await postgres.query(
-        `
-      UPDATE vbrowser 
-      SET "roomId" = $1, uid = $2, "heartbeatTime" = $3, "assignTime" = $3, state = 'used'
-      WHERE id = (
-        SELECT id
-        FROM vbrowser
-        WHERE state = 'available'
-        AND pool = $4
-        ORDER BY id ASC
-        FOR UPDATE SKIP LOCKED
-        LIMIT 1
-      )
-      RETURNING data`,
-        [roomId, uid, new Date(), this.getPoolName()],
-      );
-      return rows[0]?.data;
-    };
-    let selected: VM | undefined = await getAssignedVM();
+    const { rows } = await postgres.query(
+      `
+    UPDATE vbrowser 
+    SET "roomId" = $1, uid = $2, "heartbeatTime" = NOW(), "assignTime" = NOW(), state = 'used'
+    WHERE id = (
+      SELECT id
+      FROM vbrowser
+      WHERE state = 'available'
+      AND pool = $3
+      ORDER BY id ASC
+      FOR UPDATE SKIP LOCKED
+      LIMIT 1
+    )
+    RETURNING data`,
+      [roomId, uid, this.getPoolName()],
+    );
+    let selected: VM | undefined = rows[0]?.data;
     if (!selected) {
       return;
     }
@@ -195,12 +197,12 @@ export abstract class VMManager {
       const result = await postgres.query(
         `
         INSERT INTO vbrowser(pool, vmid, "creationTime", state)
-        VALUES($1, $2, $3, 'staging')
+        VALUES($1, $2, NOW(), 'staging')
         ON CONFLICT(pool, vmid) DO
         UPDATE SET state = 'staging',
         "roomId" = NULL, uid = NULL, retries = 0, "heartbeatTime" = NULL, "assignTime" = NULL, data = NULL
         `,
-        [this.getPoolName(), vmid, new Date()],
+        [this.getPoolName(), vmid],
       );
       console.log('UPSERT', result.rowCount);
       // Normally this should be an update, but we could insert if:
@@ -213,26 +215,17 @@ export abstract class VMManager {
 
   public startVMWrapper = async () => {
     // generate credentials and boot a VM
-    try {
-      const password = uuidv4();
-      const id = await this.startVM(password);
-      // We might fail to record it if crashing here but cleanup will reset it
-      await postgres.query(
-        `
-      INSERT INTO vbrowser(pool, vmid, "creationTime", state) 
-      VALUES($1, $2, $3, 'staging')`,
-        [this.getPoolName(), id, new Date()],
-      );
-      redisCount('vBrowserLaunches');
-      return id;
-    } catch (e: any) {
-      console.log(
-        e.response?.status,
-        JSON.stringify(e.response?.data),
-        e.config?.url,
-        e.config?.data,
-      );
-    }
+    const password = uuidv4();
+    const id = await this.startVM(password);
+    // We might fail to record it if crashing here but cleanup will reset it
+    await postgres.query(
+      `
+    INSERT INTO vbrowser(pool, vmid, "creationTime", state) 
+    VALUES($1, $2, NOW(), 'staging')`,
+      [this.getPoolName(), id],
+    );
+    redisCount('vBrowserLaunches');
+    return id;
   };
 
   protected terminateVMWrapper = async (vmid: string) => {
@@ -272,7 +265,16 @@ export abstract class VMManager {
           'limit:',
           this.getLimitSize(),
         );
-        this.startVMWrapper();
+        try {
+          this.startVMWrapper();
+        } catch (e: any) {
+          console.log(
+            e.response?.status,
+            JSON.stringify(e.response?.data),
+            e.config?.url,
+            e.config?.data,
+          );
+        }
       }
     };
 
@@ -483,9 +485,7 @@ export abstract class VMManager {
     console.log('[VMWORKER] %s: starting background jobs', this.getPoolName());
 
     setInterval(resizeVMGroupIncr, incrInterval);
-    if (this.id !== 'Hetzner') {
-      setInterval(resizeVMGroupDecr, decrInterval);
-    }
+    setInterval(resizeVMGroupDecr, decrInterval);
     setInterval(async () => {
       console.log(
         '[STATS] %s: currentSize %s, available %s, staging %s, buffer %s',

server/vm/digitalocean.ts

@@ -10,8 +10,8 @@ const imageId = Number(config.DO_IMAGE);
 const sshKeys = config.DO_SSH_KEYS.split(',');
 
 export class DigitalOcean extends VMManager {
-  size = 's-2vcpu-2gb'; // s-1vcpu-1gb, s-1vcpu-2gb, s-2vcpu-2gb, s-4vcpu-8gb, c-2
-  largeSize = 's-2vcpu-2gb';
+  size = 's-1vcpu-2gb'; // s-1vcpu-1gb, s-1vcpu-2gb, s-2vcpu-2gb, s-2vcpu-4gb, c-2
+  largeSize = 's-2vcpu-4gb';
   minRetries = 20;
   reuseVMs = true;
   id = 'DO';

server/vm/utils.ts

@@ -18,7 +18,7 @@ export type PoolConfig = {
   hostname: string | undefined;
 };
 
-function createVMManager(poolConfig: PoolConfig): VMManager | null {
+function createVMManager(poolConfig: PoolConfig): VMManager {
   let vmManager: VMManager | null = null;
   if (
     config.SCW_SECRET_KEY &&
@@ -45,6 +45,9 @@ function createVMManager(poolConfig: PoolConfig): VMManager | null {
   } else if (poolConfig.provider === 'Docker') {
     vmManager = new Docker(poolConfig);
   }
+  if (!vmManager) {
+    throw new Error('failed to create vmManager');
+  }
   return vmManager;
 }
 
@@ -62,8 +65,8 @@ export function getVMManagerConfig(): PoolConfig[] {
   });
 }
 
-export function getBgVMManagers(): { [key: string]: VMManager | null } {
-  const result: { [key: string]: VMManager | null } = {};
+export function getBgVMManagers(): { [key: string]: VMManager } {
+  const result: { [key: string]: VMManager } = {};
   const conf = getVMManagerConfig();
   conf.forEach((c) => {
     const mgr = createVMManager(c);

server/vmWorker.ts

@@ -17,25 +17,26 @@ app.post('/assignVM', async (req, res) => {
     // Find a pool that matches the size and region requirements
     const pools = Object.values(vmManagers).filter((mgr) => {
       return (
-        mgr?.getIsLarge() === Boolean(req.body.isLarge) &&
-        mgr?.getRegion() === req.body.region
+        mgr.getIsLarge() === Boolean(req.body.isLarge) &&
+        mgr.getRegion() === req.body.region
       );
     });
-    // maybe there's more than one, randomly load balance between them?
-    const pool = pools[Math.floor(Math.random() * pools.length)];
-    // TODO (howard) However if the pool is 0 sized we need to consistently request the same pool for a given uid/roomid
-    // Otherwise we may spawn multiple VMs on retries
-    if (pool) {
+    let vm = null;
+    for (let i = 0; i < pools.length; i++) {
+      // maybe there's more than one, randomly load balance between them
+      const pool = pools[i];
       console.log(
-        'assignVM from pool:',
+        'try assignVM from pool:',
         pool.getPoolName(),
         req.body.roomId,
         req.body.uid,
       );
-      const vm = await pool.assignVM(req.body.roomId, req.body.uid);
-      return res.json(vm ?? null);
+      vm = await pool.assignVM(req.body.roomId, req.body.uid);
+      if (vm) {
+        return res.json(vm);
+      }
     }
-    return res.status(400).end();
+    return res.json(null);
   } catch (e) {
     console.warn(e);
     return res.status(500).end();