Adding support for client-only build.

A few things needed to be tweaked for certain assets not being in their
expected places, or an API connection not being present.

Created scripts for automating downloading of projects, building the client
from those projects' specs, and deploying files to S3.

packages/client/client-only-build/vite.config.ts

@@ -0,0 +1,95 @@
+/*
+CPAL-1.0 License
+
+The contents of this file are subject to the Common Public Attribution License
+Version 1.0. (the "License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE.
+The License is based on the Mozilla Public License Version 1.1, but Sections 14
+and 15 have been added to cover use of software over a computer network and 
+provide for limited attribution for the Original Developer. In addition, 
+Exhibit A has been modified to be consistent with Exhibit B.
+
+Software distributed under the License is distributed on an "AS IS" basis,
+WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the
+specific language governing rights and limitations under the License.
+
+The Original Code is Ethereal Engine.
+
+The Original Developer is the Initial Developer. The Initial Developer of the
+Original Code is the Ethereal Engine team.
+
+All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 
+Ethereal Engine. All Rights Reserved.
+*/
+
+import packageRoot from 'app-root-path'
+import dotenv from 'dotenv'
+import fs from 'fs'
+import path from 'path'
+import { defineConfig, UserConfig } from 'vite'
+
+export default defineConfig(async () => {
+  dotenv.config({
+    path: packageRoot.path + '/.env.local'
+  })
+
+  const isDevOrLocal = process.env.APP_ENV === 'development' || process.env.VITE_LOCAL_BUILD === 'true'
+
+  const base = `https://${process.env['STATIC_BUILD_HOST'] ?? 'localhost:3000'}/`
+
+  const returned = {
+    server: {
+      cors: isDevOrLocal ? false : true,
+      hmr:
+        process.env.VITE_HMR === 'true'
+          ? {
+              port: process.env['VITE_APP_PORT'],
+              host: process.env['VITE_APP_HOST'],
+              overlay: false
+            }
+          : false,
+      host: process.env['VITE_APP_HOST'],
+      port: process.env['VITE_APP_PORT'],
+      headers: {
+        'Origin-Agent-Cluster': '?1'
+      },
+      ...(isDevOrLocal
+        ? {
+            https: {
+              key: fs.readFileSync(path.join(packageRoot.path, 'certs/key.pem')),
+              cert: fs.readFileSync(path.join(packageRoot.path, 'certs/cert.pem'))
+            }
+          }
+        : {})
+    },
+    base,
+    optimizeDeps: {
+      entries: ['./src/main.tsx'],
+      exclude: ['@etherealengine/volumetric'],
+      esbuildOptions: {
+        target: 'es2020'
+      }
+    },
+    plugins: [],
+    build: {
+      target: 'esnext',
+      sourcemap: 'inline',
+      minify: 'esbuild',
+      dynamicImportVarsOptions: {
+        warnOnError: true
+      },
+      rollupOptions: {
+        external: ['dotenv-flow'],
+        output: {
+          dir: 'dist',
+          format: 'es', // 'commonjs' | 'esm' | 'module' | 'systemjs'
+          // ignore files under 1mb
+          experimentalMinChunkSize: 1000000
+        }
+      }
+    }
+  } as UserConfig
+
+  return returned
+})

packages/engine/src/avatar/state/AvatarNetworkState.tsx

@@ -133,7 +133,7 @@ const AvatarReactor = React.memo(({ entityUUID }: { entityUUID: EntityUUID }) =>
   }, [state.animationState, entityUUID])
 
   useEffect(() => {
-    if (!state.avatarID.value) return
+    if (!state.avatarID.value || !Engine.instance.api) return
 
     Engine.instance.api
       .service(avatarPath)

packages/engine/src/avatar/systems/AvatarLoadingSystem.ts

@@ -303,17 +303,25 @@ const execute = () => {
 
 const reactor = () => {
   useEffect(() => {
-    AssetLoader.loadAsync('/static/itemLight.png').then((texture) => {
-      texture.colorSpace = SRGBColorSpace
-      texture.needsUpdate = true
-      light.material.map = texture
-    })
-
-    AssetLoader.loadAsync('/static/itemPlate.png').then((texture) => {
-      texture.colorSpace = SRGBColorSpace
-      texture.needsUpdate = true
-      plate.material.map = texture
-    })
+    AssetLoader.loadAsync('/static/itemLight.png')
+      .catch((err) => {
+        return
+      })
+      .then((texture) => {
+        texture.colorSpace = SRGBColorSpace
+        texture.needsUpdate = true
+        light.material.map = texture
+      })
+
+    AssetLoader.loadAsync('/static/itemPlate.png')
+      .catch((err) => {
+        return
+      })
+      .then((texture) => {
+        texture.colorSpace = SRGBColorSpace
+        texture.needsUpdate = true
+        plate.material.map = texture
+      })
   }, [])
   return null
 }

packages/engine/src/ecs/classes/Scene.ts

@@ -25,7 +25,9 @@ Ethereal Engine. All Rights Reserved.
 
 import { Color, Texture } from 'three'
 
+import config from '@etherealengine/common/src/config'
 import { SceneData } from '@etherealengine/common/src/interfaces/SceneInterface'
+import { parseStorageProviderURLs } from '@etherealengine/engine/src/common/functions/parseSceneJSON'
 import { defineState, getMutableState } from '@etherealengine/hyperflux'
 
 import { Engine } from './Engine'
@@ -44,8 +46,26 @@ export const SceneState = defineState({
 
 export const SceneServices = {
   setCurrentScene: async (projectName: string, sceneName: string) => {
-    const sceneData = await Engine.instance.api.service('scene').get({ projectName, sceneName, metadataOnly: null }, {})
-    getMutableState(SceneState).sceneData.set(sceneData.data)
+    //If there's an API connection, fetch the scene information from the API. If not, e.g. a client-only build, use
+    //a templated URL of where the scene is expected to be, and replace the __$project$__ template with the file
+    //server URL
+    if (Engine.instance.api) {
+      const sceneData = await Engine.instance.api
+        .service('scene')
+        .get({ projectName, sceneName, metadataOnly: null }, {})
+      getMutableState(SceneState).sceneData.set(sceneData.data)
+    } else {
+      let sceneData = await (
+        await fetch(`${config.client.fileServer}/projects/${projectName}/${sceneName}.scene.json`)
+      ).json()
+      sceneData = parseStorageProviderURLs(sceneData)
+      getMutableState(SceneState).sceneData.set({
+        name: sceneName,
+        project: projectName,
+        scene: sceneData,
+        thumbnailUrl: `${config.client.fileServer}/projects/${projectName}/${sceneName}.thumbnail.ktx2`
+      })
+    }
   }
 }
 // export const

scripts/build-client-only-project.ts

@@ -0,0 +1,179 @@
+/*
+CPAL-1.0 License
+
+The contents of this file are subject to the Common Public Attribution License
+Version 1.0. (the "License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE.
+The License is based on the Mozilla Public License Version 1.1, but Sections 14
+and 15 have been added to cover use of software over a computer network and
+provide for limited attribution for the Original Developer. In addition,
+Exhibit A has been modified to be consistent with Exhibit B.
+
+Software distributed under the License is distributed on an "AS IS" basis,
+WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the
+specific language governing rights and limitations under the License.
+
+The Original Code is Ethereal Engine.
+
+The Original Developer is the Initial Developer. The Initial Developer of the
+Original Code is the Ethereal Engine team.
+
+All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023
+Ethereal Engine. All Rights Reserved.
+*/
+
+import appRootPath from "app-root-path";
+import { exec, spawnSync } from 'child_process'
+import cli from 'cli'
+import dotenv from 'dotenv-flow'
+import fs from 'fs'
+import path from 'path'
+import util from 'util'
+
+const promiseExec = util.promisify(exec)
+
+import config from '@etherealengine/server-core/src/appconfig'
+import { useGit } from "@etherealengine/server-core/src/util/gitHelperFunctions";
+import { getContentType } from '@etherealengine/server-core/src/util/fileUtils'
+import {
+    createStorageProvider,
+    getStorageProvider
+} from '@etherealengine/server-core/src/media/storageprovider/storageprovider'
+import { getFilesRecursive } from "@etherealengine/server-core/src/util/fsHelperFunctions";
+import LocalStorage from "@etherealengine/server-core/src/media/storageprovider/local.storage";
+import S3Storage from "@etherealengine/server-core/src/media/storageprovider/s3.storage";
+import logger from "@etherealengine/server-core/src/ServerLogger";
+
+dotenv.config({
+    path: appRootPath.path,
+    silent: true
+})
+
+const options = cli.parse({
+    projects: [true, 'Comma-delimited list of projects to use, first project is one to build from, specify branch by appending via colon to project name, include org and repo separated by slash, e.g. etherealengine/ee-example-project:dev,etherealengine/ee-example-project-2:prod', 'string'],
+    storageProvider: [true, 'Storage Provider to use', 'string'],
+    token: [true, 'GitHub OAuth Token', 'string'],
+    user: [true, 'GitHub username', 'string'],
+    bucketName: [true, 'Name of bucket to upload to', 'string'],
+    region: [true, 'AWS region of bucket', 'string'],
+    domain: [true, 'Domain to serve files from', 'string'],
+    accessKey: [false, 'Public access key', 'string'],
+    secretKey: [false, 'Secret access key', 'string'],
+    distributionId: [false, 'Cloudfront Distribution ID', 'string'],
+    intervalMinutes: [false, 'Number of minutes between re-deploying client', 'string']
+})
+
+cli.main(async () => {
+    let interval
+    if (options.intervalMinutes) interval = parseInt(options.intervalMinutes)
+    const projectLocalDirectory = path.resolve(appRootPath.path, `packages/projects/projects/`)
+    const StorageProvider = options.storageProvider === 's3' ? S3Storage: LocalStorage
+    if (options.storageProvider === 's3') {
+        config.server.storageProvider = 's3'
+        config.aws.s3.staticResourceBucket = options.bucketName
+        config.aws.s3.region = options.region
+        config.aws.s3.accessKeyId = options.accessKey
+        config.aws.s3.secretAccessKey = options.secretKey
+        config.aws.cloudfront.distributionId = options.distributionId
+        config.aws.cloudfront.region = options.region
+        config.aws.cloudfront.domain = options.domain
+    }
+    console.log('config', config)
+    const storageProvider = createStorageProvider(StorageProvider)
+    console.log(storageProvider)
+    const projects = options.projects.split(',')
+    let rootProject, rootProjectDirectory
+
+    const runDeploy = async () => {
+        const promises = [] as Promise<void>[]
+        for (const [index, project] of projects.entries()) {
+            promises.push(new Promise(async (resolve, reject) => {
+                const split = project.split(':')
+                const projectSplit = split[0].split('/')
+                const org = projectSplit[0]
+                const projectName = projectSplit[1]
+                const branch = split.length > 1 ? split[1] : 'main'
+                const projectDirectory = path.resolve(appRootPath.path, `packages/projects/projects/${projectName}/`)
+                if (index === 0) {
+                    rootProject = projectName
+                    rootProjectDirectory = projectDirectory
+                }
+                await promiseExec(`npx rimraf packages/projects/projects/${projectName}`)
+                const authRepo = `https://${options.user}:${options.token}@github.com/${org}/${projectName}`
+                const gitCloner = useGit(projectLocalDirectory)
+                await gitCloner.clone(authRepo, projectDirectory)
+                const git = useGit(projectDirectory)
+                try {
+                    await git.checkout(branch)
+                } catch (e) {
+                    logger.warn(e)
+                }
+                resolve()
+            }))
+        }
+
+        await Promise.all(promises)
+        await promiseExec(`cp packages/client/client-only-build/vite.config.ts packages/projects/projects/${rootProject}/`)
+        await promiseExec(`cp -r packages/client/public packages/projects/projects/${rootProject}`)
+        const rootPath = path.resolve(appRootPath.path)
+        // const npmInstallPromise = new Promise<void>((resolve) => {
+        //     const npmInstallProcess = spawn('npm', ['install', '--legacy-peer-deps'], { cwd: rootPath })
+        //         npmInstallProcess.once('exit', () => {
+        //           logger.info('Finished npm installing %s', rootProject)
+        //           resolve()
+        //         })
+        //         npmInstallProcess.once('error', resolve)
+        //         npmInstallProcess.once('disconnect', resolve)
+        //         npmInstallProcess.stdout.on('data', (data) => logger.info(data.toString()))
+        //         npmInstallProcess.stderr.on('data', (data) => logger.error(data.toString()))
+        //       })
+        // await npmInstallPromise
+        const projectPath = path.resolve(appRootPath.path, `packages/projects/projects/${rootProject}`)
+        const projectFiles = getFilesRecursive(projectPath)
+        const projectFilesFiltered = projectFiles.filter((file) => !file.includes(`projects/${rootProject}/.git/`))
+        for (let file of projectFilesFiltered) {
+            const fileName = file.split(`packages/projects/`)[1]
+            const fileContent = fs.readFileSync(file)
+            await storageProvider.putObject(
+                {
+                    Body: fileContent,
+                    ContentType: getContentType(fileName),
+                    Key: fileName
+                }
+            )
+        }
+
+        await promiseExec(`cd packages/projects/projects/${rootProject} && VITE_FILE_SERVER=https://${options.domain} STATIC_BUILD_HOST=${options.domain} cross-env NODE_OPTIONS=--max_old_space_size=10240 vite build`)
+
+        const clientBuildPath = path.resolve(appRootPath.path, `packages/projects/projects/${rootProject}/dist`)
+        const clientFiles = getFilesRecursive(clientBuildPath)
+        const filtered = clientFiles.filter((file) => !file.includes(`projects/${rootProject}/.git/`))
+        for (let file of filtered) {
+            const fileName = file.split('dist/')[1]
+            const fileContent = fs.readFileSync(file)
+            await storageProvider.putObject(
+                {
+                    Body: fileContent,
+                    ContentType: getContentType(fileName),
+                    Key: fileName
+                }
+            )
+        }
+        console.log('Calling createInvalidation')
+        await storageProvider.createInvalidation(['/*'])
+        cli.ok(`Built and uploaded client-only build of project ${rootProject} at ${new Date()}`)
+    }
+
+    if (interval) {
+        console.log('calling runDeploy at', new Date())
+        runDeploy()
+        setInterval(() => {
+            console.log('calling runDeploy again at', new Date())
+            runDeploy()
+        }, interval * 1000 * 60)
+    } else {
+       await runDeploy()
+       process.exit(0)
+    }
+})