From a387843127e3f59d8e42e09d52ea8c525128b105 Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Wed, 10 Jan 2024 09:46:03 -0500 Subject: [PATCH 01/22] adds checkFileExists route to object router/controller --- controllers/object.js | 11 +++++++++++ routers/object.js | 14 ++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/controllers/object.js b/controllers/object.js index 74bc22c50..bb38df5da 100644 --- a/controllers/object.js +++ b/controllers/object.js @@ -375,6 +375,16 @@ const setFrameSharingEnabled = function (objectKey, shouldBeEnabled, callback) { console.warn('TODO: implement frame sharing... need to set property and implement all side-effects / consequences'); }; +const checkFileExists = async (objectId, filePath) => { + let obj = utilities.getObject(objects, objectId); + if (!obj) { + return false; + } + let objectIdentityDir = path.join(objectsPath, obj.name, identityFolderName); + let absoluteFilePath = path.join(objectIdentityDir, filePath); + return fileExists(absoluteFilePath); +}; + const getObject = function (objectID, excludeUnpinned) { let fullObject = utilities.getObject(objects, objectID); if (!fullObject) { return null; } @@ -425,6 +435,7 @@ module.exports = { zipBackup: zipBackup, generateXml: generateXml, setFrameSharingEnabled: setFrameSharingEnabled, + checkFileExists: checkFileExists, getObject: getObject, setup: setup }; diff --git a/routers/object.js b/routers/object.js index 45ed539d8..d3dcea6d2 100644 --- a/routers/object.js +++ b/routers/object.js @@ -403,6 +403,20 @@ router.post('/:objectName/frame/:frameName/pinned/', function (req, res) { }); }); +router.get('/:objectName/checkFileExists/*', (req, res) => { + if (!utilities.isValidId(req.params.objectName)) { + res.status(400).send('Invalid object name. Must be alphanumeric.'); + return; + } + // Extract the file path from the URL + const filePath = req.params[0]; + objectController.checkFileExists(req.params.objectName, filePath).then(exists => { + res.json({ exists: exists }); + }).catch(e => { + res.status(404).json({ error: e, exists: false }); + }); +}); + const setupDeveloperRoutes = function() { // normal nodes router.post('/:objectName/frame/:frameName/node/:nodeName/size/', function (req, res) { From f884b17369aa4b478cda761eebb9aabe0612ae49 Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Thu, 11 Jan 2024 15:33:02 -0500 Subject: [PATCH 02/22] world object entry on localhost:8080 is clickable to go to remote operator, even if object doesnt have target data yet (enables placeholder world objects) --- libraries/webInterface/gui/index.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libraries/webInterface/gui/index.js b/libraries/webInterface/gui/index.js index be96cddc2..e2f7b93c3 100644 --- a/libraries/webInterface/gui/index.js +++ b/libraries/webInterface/gui/index.js @@ -543,12 +543,16 @@ realityServer.updateManageObjects = function (thisItem2) { } else { // if not initializes with target files... - thisObject.dom.querySelector('.name').classList.add('inactive'); thisObject.dom.querySelector('.zone').classList.add('inactive'); thisObject.dom.querySelector('.sharing').classList.add('inactive'); thisObject.dom.querySelector('.download').classList.add('inactive'); thisObject.dom.querySelector('.active').classList.add('inactive'); + // world object names are always clickable to get to remote operator + if (isRemoteOperatorSupported) { // world object button only needs to be clickable in this case + realityServer.changeActiveState(thisObject.dom, true, objectKey); + } + // make on/off button yellow always if not properly initialized realityServer.switchClass(thisObject.dom.querySelector('.active'), 'green', 'yellow'); thisObject.dom.querySelector('.active').innerText = 'Off'; From 88c24e7d322fe97c71e531a154a2647dda7c49b9 Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Fri, 12 Jan 2024 10:32:13 -0500 Subject: [PATCH 03/22] send world object heartbeats even if they have no target files (similar to how avatars, anchors, humanPoses get sent anyways --- server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.js b/server.js index 78c5487ca..83ce430e5 100644 --- a/server.js +++ b/server.js @@ -1312,7 +1312,7 @@ async function objectBeatSender(PORT, thisId, thisIp, oneTimeOnly = false, immed tcs: objects[thisId].tcs, zone: zone }; - let sendWithoutTargetFiles = objects[thisId].isAnchor || objects[thisId].type === 'anchor' || objects[thisId].type === 'human' || objects[thisId].type === 'avatar'; + let sendWithoutTargetFiles = objects[thisId].isAnchor || objects[thisId].type === 'anchor' || objects[thisId].type === 'human' || objects[thisId].type === 'avatar' || objects[thisId].type === 'world'; if (objects[thisId].tcs || sendWithoutTargetFiles) { utilities.sendWithFallback(client, PORT, HOST, messageObj, { closeAfterSending: false, From d80b29e85d1acc90575645e8e9baf9f299fab1f4 Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Fri, 12 Jan 2024 12:18:23 -0500 Subject: [PATCH 04/22] dont crash if no authoringMesh.glb inside the upload.zip --- server.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/server.js b/server.js index 83ce430e5..d3697132b 100644 --- a/server.js +++ b/server.js @@ -3028,10 +3028,14 @@ function objectWebServer() { console.warn('target zip already cleaned up', folderName); } // let newFolderFiles = fs.readdirSync(path.join(folderD, identityFolderName, 'target')); - await fsProm.rename( - path.join(folderD, identityFolderName, 'target', 'authoringMesh.glb'), - path.join(folderD, identityFolderName, 'target', 'target.glb') - ); + try { + await fsProm.rename( + path.join(folderD, identityFolderName, 'target', 'authoringMesh.glb'), + path.join(folderD, identityFolderName, 'target', 'target.glb') + ); + } catch (e) { + console.warn('no authoringMesh.glb to rename to target.glb', e); + } }; } const finish = finishFn(folderFile); From 8ed0cfc792d2862a8eebb7fd0ea068c7f2bbb748 Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Fri, 12 Jan 2024 12:48:32 -0500 Subject: [PATCH 05/22] more completely cleanup target folder after unzipping target files --- server.js | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/server.js b/server.js index d3697132b..7d0fb80c1 100644 --- a/server.js +++ b/server.js @@ -3003,7 +3003,7 @@ function objectWebServer() { unzipper.on('extract', async function (_log) { const targetFolderPath = path.join(folderD, identityFolderName, 'target'); const folderFiles = await fsProm.readdir(targetFolderPath); - const targetTypes = ['xml', 'dat', 'glb', 'unitypackage', '3dt']; + const targetTypes = ['xml', 'dat', 'glb', 'unitypackage', '3dt', 'jpg']; let anyTargetsUploaded = false; @@ -3022,19 +3022,36 @@ function objectWebServer() { let deferred = false; function finishFn(folderName) { return async function() { + let nestedTargetDirPath = path.join(folderD, identityFolderName, 'target', folderName); + + try { + // Attempt to delete .DS_Store if it exists + await fsProm.unlink(path.join(nestedTargetDirPath, '.DS_Store')); + } catch (error) { + // Ignore if .DS_Store doesn't exist; display other warnings + if (error.code !== 'ENOENT') console.warn(error); + } + try { - await fsProm.rmdir(path.join(folderD, identityFolderName, 'target', folderName)); + await fsProm.rmdir(nestedTargetDirPath); } catch (e) { - console.warn('target zip already cleaned up', folderName); + console.warn('target zip already cleaned up', folderName, e); } - // let newFolderFiles = fs.readdirSync(path.join(folderD, identityFolderName, 'target')); + try { await fsProm.rename( path.join(folderD, identityFolderName, 'target', 'authoringMesh.glb'), path.join(folderD, identityFolderName, 'target', 'target.glb') ); } catch (e) { - console.warn('no authoringMesh.glb to rename to target.glb', e); + console.log('no authoringMesh.glb to rename to target.glb', e); + } + + try { + // Attempt to delete __MACOSX zip artifact, if it exists + await fsProm.rmdir(path.join(folderD, identityFolderName, 'target', '__MACOSX'), { recursive: true }); + } catch (error) { + if (error.code !== 'ENOENT') console.warn(error); } }; } From dc2a8d067bb77135c9f39146c30817252b09ff7a Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Fri, 16 Feb 2024 11:50:27 -0500 Subject: [PATCH 06/22] dont generate XML file when world object is created or when other target files are uploaded (if autogeneratexml=false in the headers) --- libraries/webInterface/gui/index.js | 12 ++++++------ server.js | 9 +++++++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/libraries/webInterface/gui/index.js b/libraries/webInterface/gui/index.js index e2f7b93c3..26ad23aa5 100644 --- a/libraries/webInterface/gui/index.js +++ b/libraries/webInterface/gui/index.js @@ -1969,10 +1969,10 @@ realityServer.gotClick = function (event) { // this is how world objects get instantly initialized try { let msgContent = JSON.parse(state); - // generate a placeholder xml file for this object - let defaultSize = 0.3; - realityServer.sendRequest('/object/' + msgContent.id + '/generateXml/', 'POST', function (stateGen) { - if (stateGen === 'ok') { + // // generate a placeholder xml file for this object + // let defaultSize = 0.3; + // realityServer.sendRequest('/object/' + msgContent.id + '/generateXml/', 'POST', function (stateGen) { + // if (stateGen === 'ok') { realityServer.objects[msgContent.id] = new Objects(); realityServer.objects[msgContent.id].name = msgContent.name; realityServer.objects[msgContent.id].isWorldObject = true; @@ -1987,8 +1987,8 @@ realityServer.gotClick = function (event) { realityServer.update(); }); }, 100); - } - }, 'name=' + msgContent.name + '&width=' + defaultSize + '&height=' + defaultSize); + // } + // }, 'name=' + msgContent.name + '&width=' + defaultSize + '&height=' + defaultSize); } catch (e) { console.error('json parse error for (action=new&name=\'' + objectName + '\') response: ' + state); diff --git a/server.js b/server.js index 1d493fb4a..d583d74d4 100644 --- a/server.js +++ b/server.js @@ -2811,6 +2811,7 @@ function objectWebServer() { form.on('end', function () { var folderD = form.uploadDir; + let autoGenerateXml = typeof req.headers.autogeneratexml !== 'undefined' ? JSON.parse(req.headers.autogeneratexml) : true; fileInfoList = fileInfoList.filter(fileInfo => !fileInfo.completed); // Don't repeat processing for completed files fileInfoList.forEach(async fileInfo => { if (!await fileExists(path.join(form.uploadDir, fileInfo.name))) { // Ignore files that haven't finished uploading @@ -2883,13 +2884,17 @@ function objectWebServer() { } } else { - continueProcessingUpload(); + continueProcessingUpload(folderD, fileExtension); } // Step 2) - Generate a default XML file if needed async function continueProcessingUpload() { // eslint-disable-line no-inner-declarations - var objectName = req.params.id + utilities.uuidTime(); + if (!autoGenerateXml) { + await onXmlVerified(); + return; + } + var objectName = req.params.id + utilities.uuidTime(); var documentcreate = '\n' + '\n' + ' \n' + From 6640c048d298acee427c62e564d5ebd9d3445d5c Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Fri, 16 Feb 2024 14:57:26 -0500 Subject: [PATCH 07/22] allow upload of target.splat to the object --- server.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/server.js b/server.js index d583d74d4..54ae5d9d3 100644 --- a/server.js +++ b/server.js @@ -974,7 +974,7 @@ async function setAnchors() { if (!objects[key]) { continue; } - if (objects[key].isWorldObject || objects[key].type === 'world') { + if (objects[key].isWorldObject || objects[key].type === 'world') { // TODO: is this still necessary? world objects are considered initialized by default, now // check if the object is correctly initialized with tracking targets let datExists = await fileExists(path.join(objectsPath, objects[key].name, identityFolderName, '/target/target.dat')); let xmlExists = await fileExists(path.join(objectsPath, objects[key].name, identityFolderName, '/target/target.xml')); @@ -1268,7 +1268,8 @@ async function objectBeatSender(PORT, thisId, thisIp, oneTimeOnly = false, immed let xmlPath = path.join(targetDir, 'target.xml'); let glbPath = path.join(targetDir, 'target.glb'); let tdtPath = path.join(targetDir, 'target.3dt'); - var fileList = [jpgPath, xmlPath, datPath, glbPath, tdtPath]; + let splatPath = path.join(targetDir, 'target.splat'); + var fileList = [jpgPath, xmlPath, datPath, glbPath, tdtPath, splatPath]; const tcs = await utilities.generateChecksums(objects, fileList); if (objects[thisId]) { // if no target files exist, checksum will be undefined, so mark @@ -1731,6 +1732,7 @@ function objectWebServer() { 'target.glb', 'target.unitypackage', 'target.3dt', + 'target.splat', ]; if (targetFiles.includes(filename) && urlArray[urlArray.length - 2] === 'target') { @@ -2826,7 +2828,7 @@ function objectWebServer() { fileExtension = 'jpg'; } - if (fileExtension === 'jpg' || fileExtension === 'dat' || fileExtension === 'xml' || fileExtension === 'glb') { + if (fileExtension === 'jpg' || fileExtension === 'dat' || fileExtension === 'xml' || fileExtension === 'glb' || fileExtension === '3dt' || fileExtension === 'splat') { if (!await fileExists(folderD + '/' + identityFolderName + '/target/')) { try { await fsProm.mkdir(folderD + '/' + identityFolderName + '/target/', '0766'); @@ -2938,8 +2940,9 @@ function objectWebServer() { let xmlPath = path.join(folderD, identityFolderName, '/target/target.xml'); let glbPath = path.join(folderD, identityFolderName, '/target/target.glb'); let tdtPath = path.join(folderD, identityFolderName, '/target/target.3dt'); + let splatPath = path.join(folderD, identityFolderName, '/target/target.splat'); - var fileList = [jpgPath, xmlPath, datPath, glbPath, tdtPath]; + var fileList = [jpgPath, xmlPath, datPath, glbPath, tdtPath, splatPath]; if (typeof objects[thisObjectId] !== 'undefined') { var thisObject = objects[thisObjectId]; @@ -2947,6 +2950,8 @@ function objectWebServer() { var dat = await fileExists(datPath); var xml = await fileExists(xmlPath); var glb = await fileExists(glbPath); + var tdt = await fileExists(tdtPath); + var splat = await fileExists(splatPath); var sendObject = { id: thisObjectId, @@ -2955,7 +2960,9 @@ function objectWebServer() { jpgExists: jpg, xmlExists: xml, datExists: dat, - glbExists: glb + glbExists: glb, + tdtExists: tdt, + splatExists: splat }; thisObject.tcs = await utilities.generateChecksums(objects, fileList); From 8a7068ef53fcd189b0b616947b71b7b0ac981cbd Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Wed, 21 Feb 2024 10:41:42 -0500 Subject: [PATCH 08/22] adds new all-in-one /checkTargetFiles/ API instead of needing to do an individual /checkFileExists/ for each one --- controllers/object.js | 23 +++++++++++++++++++++++ routers/object.js | 13 +++++++++++++ 2 files changed, 36 insertions(+) diff --git a/controllers/object.js b/controllers/object.js index cd8a246c4..6da3a9036 100644 --- a/controllers/object.js +++ b/controllers/object.js @@ -402,6 +402,28 @@ const checkFileExists = async (objectId, filePath) => { return fileExists(absoluteFilePath); }; +const checkTargetFiles = async (objectId) => { + let [ + glbExists, xmlExists, datExists, + jpgExists, _3dtExists, splatExists + ] = await Promise.all([ + checkFileExists(objectId, '/target/target.glb'), + checkFileExists(objectId, '/target/target.xml'), + checkFileExists(objectId, '/target/target.dat'), + checkFileExists(objectId, '/target/target.jpg'), + checkFileExists(objectId, '/target/target.3dt'), + checkFileExists(objectId, '/target/target.splat'), + ]); + return { + glbExists, + xmlExists, + datExists, + jpgExists, + _3dtExists, + splatExists + } +}; + const getObject = function (objectID, excludeUnpinned) { let fullObject = utilities.getObject(objects, objectID); if (!fullObject) { return null; } @@ -453,6 +475,7 @@ module.exports = { generateXml: generateXml, setFrameSharingEnabled: setFrameSharingEnabled, checkFileExists: checkFileExists, + checkTargetFiles: checkTargetFiles, getObject: getObject, setup: setup, requestGaussianSplatting, diff --git a/routers/object.js b/routers/object.js index 5b0a8c3ee..4e74164d7 100644 --- a/routers/object.js +++ b/routers/object.js @@ -417,6 +417,19 @@ router.get('/:objectName/checkFileExists/*', (req, res) => { }); }); +router.get('/:objectName/checkTargetFiles/', (req, res) => { + if (!utilities.isValidId(req.params.objectName)) { + res.status(400).send('Invalid object name. Must be alphanumeric.'); + return; + } + // Extract the file path from the URL + objectController.checkTargetFiles(req.params.objectName).then(existsJson => { + res.json(existsJson); + }).catch(e => { + res.status(404).json({ error: e, exists: false }); + }); +}); + router.post('/:objectName/requestGaussianSplatting/', async function (req, res) { if (!utilities.isValidId(req.params.objectName)) { res.status(400).send('Invalid object name. Must be alphanumeric.'); From 7b35a378709ca4643a21cff2b6944b7f8a7a31bf Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Tue, 27 Feb 2024 16:24:25 -0500 Subject: [PATCH 09/22] adds new /object/:id/uploadTarget/ route with cleaner logic than old targetUpload route --- controllers/object.js | 146 ++++++++++++++++++++++++++++++++++++++++++ routers/object.js | 7 ++ 2 files changed, 153 insertions(+) diff --git a/controllers/object.js b/controllers/object.js index 6da3a9036..2fc9aa0a6 100644 --- a/controllers/object.js +++ b/controllers/object.js @@ -4,6 +4,8 @@ const formidable = require('formidable'); const utilities = require('../libraries/utilities'); const {fileExists, unlinkIfExists, mkdirIfNotExists} = utilities; const {startSplatTask} = require('./object/SplatTask.js'); +const server = require('../server'); +const {/*objectsPath,*/ beatPort} = require('../config.js'); // Variables populated from server.js with setup() var objects = {}; @@ -424,6 +426,149 @@ const checkTargetFiles = async (objectId) => { } }; +const uploadTarget = async (objectName, req, res) => { + let thisObjectId = utilities.readObject(objectLookup, objectName); + let object = utilities.getObject(objects, thisObjectId); + if (!object) { + res.status(404).send('object ' + thisObjectId + ' not found'); + return; + } + + // first upload to a temporary directory before moving it to the target directory + let uploadDir = path.join(objectsPath, objectName, identityFolderName, 'tmp'); + await mkdirIfNotExists(uploadDir); + + let targetDir = path.join(objectsPath, objectName, identityFolderName, 'target'); + await mkdirIfNotExists(targetDir); + + let form = new formidable.IncomingForm({ + uploadDir: uploadDir, + keepExtensions: true + // accept: 'image/jpeg' // we don't include this anymore, because any filetype can be uploaded + }); + + form.on('error', function (err) { + res.status(500).send(err); + }); + + let fileInfoList = []; + + form.on('fileBegin', (name, file) => { + if (!file.name) { + file.name = file.newFilename; + } + fileInfoList.push({ + name: file.name, + completed: false + }); + file.path = path.join(form.uploadDir, file.name); + }); + + form.parse(req); + + /** + * Returns the file extension (portion after the last dot) of the given filename. + * If a file name starts with a dot, returns an empty string. + * + * @author VisioN @ StackOverflow + * @param {string} fileName - The name of the file, such as foo.zip + * @return {string} The lowercase extension of the file, such has "zip" + */ + function getFileExtension(fileName) { + return fileName.substr((~-fileName.lastIndexOf('.') >>> 0) + 2).toLowerCase(); + } + + function makeFileProcessPromise(fileInfo) { + return new Promise(async (resolve, reject) => { + if (!await fileExists(path.join(form.uploadDir, fileInfo.name))) { // Ignore files that haven't finished uploading + reject(`File doesn't exist at ${path.join(form.uploadDir, fileInfo.name)}`); + return; + } + fileInfo.completed = true; // File has downloaded + let fileExtension = getFileExtension(fileInfo.name); + + // only accept these predetermined file types + if (!(fileExtension === 'jpg' || fileExtension === 'dat' || + fileExtension === 'xml' || fileExtension === 'glb' || + fileExtension === '3dt' || fileExtension === 'splat')) { + reject(`File extension not acceptable for targetUpload (${fileExtension})`); + return; + } + + let originalFilepath = path.join(uploadDir, fileInfo.name); + let newFilepath = path.join(targetDir, `target.${fileExtension}`); + + try { + await fsProm.rename(originalFilepath, newFilepath); + resolve(); + } catch (e) { + reject(`error renaming ${originalFilepath} to ${newFilepath}`); + } + }); + } + + form.on('end', async () => { + fileInfoList = fileInfoList.filter(fileInfo => !fileInfo.completed); // Don't repeat processing for completed files + + let filePromises = []; + fileInfoList.forEach(fileInfo => { + filePromises.push(makeFileProcessPromise(fileInfo)); + }); + + try { + await Promise.all(filePromises); + // Code continues when all promises are resolved + } catch (error) { + console.warn(error); + // res.status(500).send('error') + } + + let jpgPath = path.join(targetDir, 'target.jpg'); + let datPath = path.join(targetDir, 'target.dat'); + let xmlPath = path.join(targetDir, 'target.xml'); + let glbPath = path.join(targetDir, 'target.glb'); + let tdtPath = path.join(targetDir, 'target.3dt'); + let splatPath = path.join(targetDir, 'target.splat'); + let fileList = [jpgPath, xmlPath, datPath, glbPath, tdtPath, splatPath]; + + let jpg = await fileExists(jpgPath); + let dat = await fileExists(datPath); + let xml = await fileExists(xmlPath); + let glb = await fileExists(glbPath); + let tdt = await fileExists(tdtPath); + let splat = await fileExists(splatPath); + + let sendObject = { + id: thisObjectId, + name: objectName, + // initialized: (jpg && xml), + jpgExists: jpg, + xmlExists: xml, + datExists: dat, + glbExists: glb, + tdtExists: tdt, + splatExists: splat + }; + + object.tcs = utilities.generateChecksums(objects, fileList); + await utilities.writeObjectToFile(objects, thisObjectId, globalVariables.saveToDisk); + await setAnchors(); + + // await objectBeatSender(beatPort, thisObjectId, objects[thisObjectId].ip, true); + await server.objectBeatSender(beatPort, thisObjectId, objects[thisObjectId].ip, true); + + // delete the tmp folder and any files within it + await utilities.rmdirIfExists(uploadDir); + + // res.status(200).send('ok'); + try { + res.status(200).json(sendObject); + } catch (e) { + console.error('unable to send res', e); + } + }); +}; + const getObject = function (objectID, excludeUnpinned) { let fullObject = utilities.getObject(objects, objectID); if (!fullObject) { return null; } @@ -476,6 +621,7 @@ module.exports = { setFrameSharingEnabled: setFrameSharingEnabled, checkFileExists: checkFileExists, checkTargetFiles: checkTargetFiles, + uploadTarget: uploadTarget, getObject: getObject, setup: setup, requestGaussianSplatting, diff --git a/routers/object.js b/routers/object.js index 4e74164d7..a3f283e76 100644 --- a/routers/object.js +++ b/routers/object.js @@ -605,6 +605,13 @@ const setupDeveloperRoutes = function() { let excludeUnpinned = (req.query.excludeUnpinned === 'true'); res.json(objectController.getObject(req.params.objectName, excludeUnpinned)).end(); }); + router.post('/:objectName/uploadTarget/', (req, res) => { + if (!utilities.isValidId(req.params.objectName)) { + res.status(400).send('Invalid object name. Must be alphanumeric.'); + return; + } + objectController.uploadTarget(req.params.objectName, req, res); + }); }; const setup = function(globalVariables) { From f54bda11b4bd5638d9bb9bd18b52f9d897485344 Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Tue, 27 Feb 2024 16:41:12 -0500 Subject: [PATCH 10/22] properly await checkFileExists --- controllers/object.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/object.js b/controllers/object.js index 2fc9aa0a6..52f8a7d01 100644 --- a/controllers/object.js +++ b/controllers/object.js @@ -401,7 +401,7 @@ const checkFileExists = async (objectId, filePath) => { } let objectIdentityDir = path.join(objectsPath, obj.name, identityFolderName); let absoluteFilePath = path.join(objectIdentityDir, filePath); - return fileExists(absoluteFilePath); + return await fileExists(absoluteFilePath); }; const checkTargetFiles = async (objectId) => { From 1b8b3fc8fa5ee8e25fb2b3366d8ba7ed9e3f927f Mon Sep 17 00:00:00 2001 From: James Hobin Date: Wed, 28 Feb 2024 09:55:04 -0500 Subject: [PATCH 11/22] Update toolsocket --- package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 90cc937d3..ce8019cbc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8030,7 +8030,7 @@ }, "node_modules/toolsocket": { "version": "1.0.0", - "resolved": "git+ssh://git@github.com/ptcrealitylab/toolsocket.git#a0ac24c4fc04cf2ad7306ae23c166505ff23d8a7", + "resolved": "git+ssh://git@github.com/ptcrealitylab/toolsocket.git#48a6cee0c28bb8957018514ffbee4c9fcce5e293", "license": "MPL-2.0", "dependencies": { "ip": "^1.1.5", @@ -14692,7 +14692,7 @@ } }, "toolsocket": { - "version": "git+ssh://git@github.com/ptcrealitylab/toolsocket.git#a0ac24c4fc04cf2ad7306ae23c166505ff23d8a7", + "version": "git+ssh://git@github.com/ptcrealitylab/toolsocket.git#48a6cee0c28bb8957018514ffbee4c9fcce5e293", "from": "toolsocket@github:ptcrealitylab/toolsocket", "requires": { "ip": "^1.1.5", From 37aa15072ed50f50a88181f4e15b433a7eec484c Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Fri, 1 Mar 2024 11:08:32 -0500 Subject: [PATCH 12/22] add try-catch around formidable parse to guard against malformed target uploads --- server.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server.js b/server.js index 59244d958..8ab09cae1 100644 --- a/server.js +++ b/server.js @@ -2813,7 +2813,13 @@ function objectWebServer() { } }); - form.parse(req); + try { + form.parse(req); + } catch (e) { + console.warn('error parsing formidable', e); + res.status(500).send(`error parsing formidable: ${e}`); + return; + } form.on('end', function () { var folderD = form.uploadDir; From 2abc9b73269f1f6039d42611676370ec7bb0adac Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Fri, 1 Mar 2024 11:18:52 -0500 Subject: [PATCH 13/22] fix linting for updated target uploads --- controllers/object.js | 54 +++++++++++++++-------------- libraries/webInterface/gui/index.js | 50 +++++++++++++++----------- 2 files changed, 58 insertions(+), 46 deletions(-) diff --git a/controllers/object.js b/controllers/object.js index 52f8a7d01..ac33a5736 100644 --- a/controllers/object.js +++ b/controllers/object.js @@ -423,7 +423,7 @@ const checkTargetFiles = async (objectId) => { jpgExists, _3dtExists, splatExists - } + }; }; const uploadTarget = async (objectName, req, res) => { @@ -479,31 +479,33 @@ const uploadTarget = async (objectName, req, res) => { } function makeFileProcessPromise(fileInfo) { - return new Promise(async (resolve, reject) => { - if (!await fileExists(path.join(form.uploadDir, fileInfo.name))) { // Ignore files that haven't finished uploading - reject(`File doesn't exist at ${path.join(form.uploadDir, fileInfo.name)}`); - return; - } - fileInfo.completed = true; // File has downloaded - let fileExtension = getFileExtension(fileInfo.name); - - // only accept these predetermined file types - if (!(fileExtension === 'jpg' || fileExtension === 'dat' || - fileExtension === 'xml' || fileExtension === 'glb' || - fileExtension === '3dt' || fileExtension === 'splat')) { - reject(`File extension not acceptable for targetUpload (${fileExtension})`); - return; - } - - let originalFilepath = path.join(uploadDir, fileInfo.name); - let newFilepath = path.join(targetDir, `target.${fileExtension}`); - - try { - await fsProm.rename(originalFilepath, newFilepath); - resolve(); - } catch (e) { - reject(`error renaming ${originalFilepath} to ${newFilepath}`); - } + return new Promise((resolve, reject) => { + (async () => { + if (!await fileExists(path.join(form.uploadDir, fileInfo.name))) { // Ignore files that haven't finished uploading + reject(`File doesn't exist at ${path.join(form.uploadDir, fileInfo.name)}`); + return; + } + fileInfo.completed = true; // File has downloaded + let fileExtension = getFileExtension(fileInfo.name); + + // only accept these predetermined file types + if (!(fileExtension === 'jpg' || fileExtension === 'dat' || + fileExtension === 'xml' || fileExtension === 'glb' || + fileExtension === '3dt' || fileExtension === 'splat')) { + reject(`File extension not acceptable for targetUpload (${fileExtension})`); + return; + } + + let originalFilepath = path.join(uploadDir, fileInfo.name); + let newFilepath = path.join(targetDir, `target.${fileExtension}`); + + try { + await fsProm.rename(originalFilepath, newFilepath); + resolve(); + } catch (e) { + reject(`error renaming ${originalFilepath} to ${newFilepath}`); + } + })(); }); } diff --git a/libraries/webInterface/gui/index.js b/libraries/webInterface/gui/index.js index 26ad23aa5..a225a3d1c 100644 --- a/libraries/webInterface/gui/index.js +++ b/libraries/webInterface/gui/index.js @@ -1969,26 +1969,36 @@ realityServer.gotClick = function (event) { // this is how world objects get instantly initialized try { let msgContent = JSON.parse(state); - // // generate a placeholder xml file for this object - // let defaultSize = 0.3; - // realityServer.sendRequest('/object/' + msgContent.id + '/generateXml/', 'POST', function (stateGen) { - // if (stateGen === 'ok') { - realityServer.objects[msgContent.id] = new Objects(); - realityServer.objects[msgContent.id].name = msgContent.name; - realityServer.objects[msgContent.id].isWorldObject = true; - // realityServer.objects[msgContent.id].initialized = true; - - // make them automatically activate after a slight delay - setTimeout(function () { - realityServer.sendRequest('/object/' + msgContent.id + '/activate/', 'GET', function (stateActivate) { - if (stateActivate === 'ok') { - // realityServer.objects[msgContent.id].active = true; - } - realityServer.update(); - }); - }, 100); - // } - // }, 'name=' + msgContent.name + '&width=' + defaultSize + '&height=' + defaultSize); + + const finishSettingUpObject = () => { + realityServer.objects[msgContent.id] = new Objects(); + realityServer.objects[msgContent.id].name = msgContent.name; + realityServer.objects[msgContent.id].isWorldObject = true; + // realityServer.objects[msgContent.id].initialized = true; + + // make them automatically activate after a slight delay + setTimeout(function () { + realityServer.sendRequest('/object/' + msgContent.id + '/activate/', 'GET', function (stateActivate) { + if (stateActivate === 'ok') { + // realityServer.objects[msgContent.id].active = true; + } + realityServer.update(); + }); + }, 100); + }; + + const AUTO_GENERATE_XML = false; + if (AUTO_GENERATE_XML) { + // generate a placeholder xml file for this object + let defaultSize = 0.3; + realityServer.sendRequest('/object/' + msgContent.id + '/generateXml/', 'POST', function (stateGen) { + if (stateGen === 'ok') { + finishSettingUpObject(); + } + }, 'name=' + msgContent.name + '&width=' + defaultSize + '&height=' + defaultSize); + } else { + finishSettingUpObject(); + } } catch (e) { console.error('json parse error for (action=new&name=\'' + objectName + '\') response: ' + state); From f1ec50c13db8130db04ef78dab5e280d3a22508f Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Mon, 4 Mar 2024 16:23:14 -0500 Subject: [PATCH 14/22] cleanup uploadTarget function --- controllers/object.js | 55 +++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/controllers/object.js b/controllers/object.js index ac33a5736..f48271b3d 100644 --- a/controllers/object.js +++ b/controllers/object.js @@ -5,7 +5,7 @@ const utilities = require('../libraries/utilities'); const {fileExists, unlinkIfExists, mkdirIfNotExists} = utilities; const {startSplatTask} = require('./object/SplatTask.js'); const server = require('../server'); -const {/*objectsPath,*/ beatPort} = require('../config.js'); +const { beatPort } = require('../config.js'); // Variables populated from server.js with setup() var objects = {}; @@ -394,6 +394,12 @@ const setFrameSharingEnabled = function (objectKey, shouldBeEnabled, callback) { console.warn('TODO: implement frame sharing... need to set property and implement all side-effects / consequences'); }; +/** + * Check whether a specific file within the object's .identity folder exists + * @param {string} objectId + * @param {string} filePath + * @returns {Promise} + */ const checkFileExists = async (objectId, filePath) => { let obj = utilities.getObject(objects, objectId); if (!obj) { @@ -404,6 +410,11 @@ const checkFileExists = async (objectId, filePath) => { return await fileExists(absoluteFilePath); }; +/** + * Check the status of all the possible target files that an object might have + * @param {string} objectId + * @returns {Promise<{glbExists: boolean, xmlExists: boolean, datExists: boolean, jpgExists: boolean, _3dtExists: boolean, splatExists: boolean}>} + */ const checkTargetFiles = async (objectId) => { let [ glbExists, xmlExists, datExists, @@ -426,6 +437,15 @@ const checkTargetFiles = async (objectId) => { }; }; +/** + * Uploads a target file to an object. + * Imitates much of the logic of the content/:id route in a simpler way. + * Note: not supported by Cloud Proxy as of 3-4-2024, use content/:id instead. + * @param {string} objectName + * @param {Request} req + * @param {Response} res + * @returns {Promise} + */ const uploadTarget = async (objectName, req, res) => { let thisObjectId = utilities.readObject(objectLookup, objectName); let object = utilities.getObject(objects, thisObjectId); @@ -434,17 +454,15 @@ const uploadTarget = async (objectName, req, res) => { return; } - // first upload to a temporary directory before moving it to the target directory + // first upload to a temporary directory (tmp/) before moving it to the target directory (target/) let uploadDir = path.join(objectsPath, objectName, identityFolderName, 'tmp'); await mkdirIfNotExists(uploadDir); - let targetDir = path.join(objectsPath, objectName, identityFolderName, 'target'); await mkdirIfNotExists(targetDir); let form = new formidable.IncomingForm({ uploadDir: uploadDir, keepExtensions: true - // accept: 'image/jpeg' // we don't include this anymore, because any filetype can be uploaded }); form.on('error', function (err) { @@ -455,7 +473,7 @@ const uploadTarget = async (objectName, req, res) => { form.on('fileBegin', (name, file) => { if (!file.name) { - file.name = file.newFilename; + file.name = file.newFilename; // some versions of formidable use .name, others use .newFilename } fileInfoList.push({ name: file.name, @@ -478,6 +496,11 @@ const uploadTarget = async (objectName, req, res) => { return fileName.substr((~-fileName.lastIndexOf('.') >>> 0) + 2).toLowerCase(); } + /** + * Returns a promise that can be executed later to move the uploaded file from the tmp dir to the target dir + * @param {{name: string, completed: boolean}} fileInfo + * @returns {Promise} + */ function makeFileProcessPromise(fileInfo) { return new Promise((resolve, reject) => { (async () => { @@ -519,7 +542,7 @@ const uploadTarget = async (objectName, req, res) => { try { await Promise.all(filePromises); - // Code continues when all promises are resolved + // Code continues when all promises are resolved (when all files have been moved from tmp/ to target/) } catch (error) { console.warn(error); // res.status(500).send('error') @@ -533,23 +556,16 @@ const uploadTarget = async (objectName, req, res) => { let splatPath = path.join(targetDir, 'target.splat'); let fileList = [jpgPath, xmlPath, datPath, glbPath, tdtPath, splatPath]; - let jpg = await fileExists(jpgPath); - let dat = await fileExists(datPath); - let xml = await fileExists(xmlPath); - let glb = await fileExists(glbPath); - let tdt = await fileExists(tdtPath); - let splat = await fileExists(splatPath); - let sendObject = { id: thisObjectId, name: objectName, // initialized: (jpg && xml), - jpgExists: jpg, - xmlExists: xml, - datExists: dat, - glbExists: glb, - tdtExists: tdt, - splatExists: splat + jpgExists: await fileExists(jpgPath), + xmlExists: await fileExists(xmlPath), + datExists: await fileExists(datPath), + glbExists: await fileExists(glbPath), + tdtExists: await fileExists(tdtPath), + splatExists: await fileExists(splatPath) }; object.tcs = utilities.generateChecksums(objects, fileList); @@ -562,7 +578,6 @@ const uploadTarget = async (objectName, req, res) => { // delete the tmp folder and any files within it await utilities.rmdirIfExists(uploadDir); - // res.status(200).send('ok'); try { res.status(200).json(sendObject); } catch (e) { From c00ce1ac8873cbe78269768d16a4fa7322558fc8 Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Mon, 4 Mar 2024 16:37:01 -0500 Subject: [PATCH 15/22] cleanup gs-upload-test --- server.js | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/server.js b/server.js index 1552785af..f30d6dc5e 100644 --- a/server.js +++ b/server.js @@ -990,7 +990,8 @@ async function setAnchors() { if (!objects[key]) { continue; } - if (objects[key].isWorldObject || objects[key].type === 'world') { // TODO: is this still necessary? world objects are considered initialized by default, now + // TODO: world objects are now considered initialized by default... update how anchor objects work (do they still require that the world has target data?) + if (objects[key].isWorldObject || objects[key].type === 'world') { // check if the object is correctly initialized with tracking targets let datExists = await fileExists(path.join(objectsPath, objects[key].name, identityFolderName, '/target/target.dat')); let xmlExists = await fileExists(path.join(objectsPath, objects[key].name, identityFolderName, '/target/target.xml')); @@ -1332,7 +1333,13 @@ async function objectBeatSender(PORT, thisId, thisIp, oneTimeOnly = false, immed tcs: objects[thisId].tcs, zone: zone }; - let sendWithoutTargetFiles = objects[thisId].isAnchor || objects[thisId].type === 'anchor' || objects[thisId].type === 'human' || objects[thisId].type === 'avatar' || objects[thisId].type === 'world'; + // we enumerate the object types that can be sent without target files. + // currently, only regular objects (type='object') require target files to send heartbeats. + let sendWithoutTargetFiles = objects[thisId].isAnchor || + objects[thisId].type === 'anchor' || + objects[thisId].type === 'human' || + objects[thisId].type === 'avatar' || + objects[thisId].type === 'world'; if (objects[thisId].tcs || sendWithoutTargetFiles) { utilities.sendWithFallback(client, PORT, HOST, messageObj, { closeAfterSending: false, @@ -2838,6 +2845,8 @@ function objectWebServer() { form.on('end', function () { var folderD = form.uploadDir; + // by default, uploading any other target file will auto-generate a target.xml file + // specify `autogeneratexml: false` in the headers to prevent this behavior let autoGenerateXml = typeof req.headers.autogeneratexml !== 'undefined' ? JSON.parse(req.headers.autogeneratexml) : true; fileInfoList = fileInfoList.filter(fileInfo => !fileInfo.completed); // Don't repeat processing for completed files fileInfoList.forEach(async fileInfo => { @@ -2853,7 +2862,8 @@ function objectWebServer() { fileExtension = 'jpg'; } - if (fileExtension === 'jpg' || fileExtension === 'dat' || fileExtension === 'xml' || fileExtension === 'glb' || fileExtension === '3dt' || fileExtension === 'splat') { + if (fileExtension === 'jpg' || fileExtension === 'dat' || fileExtension === 'xml' || + fileExtension === 'glb' || fileExtension === '3dt' || fileExtension === 'splat') { if (!await fileExists(folderD + '/' + identityFolderName + '/target/')) { try { await fsProm.mkdir(folderD + '/' + identityFolderName + '/target/', '0766'); @@ -3040,7 +3050,7 @@ function objectWebServer() { unzipper.on('extract', async function (_log) { const targetFolderPath = path.join(folderD, identityFolderName, 'target'); const folderFiles = await fsProm.readdir(targetFolderPath); - const targetTypes = ['xml', 'dat', 'glb', 'unitypackage', '3dt', 'jpg']; + const targetTypes = ['xml', 'dat', 'glb', 'unitypackage', '3dt', 'jpg', 'splat']; let anyTargetsUploaded = false; From 4383f4ef80d478cf04e7de87a8fea913eb498fb7 Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Wed, 6 Mar 2024 16:02:00 -0500 Subject: [PATCH 16/22] adds /object/renderMode API to post updates from mesh to ai rendering mode that sync to all clients --- controllers/object.js | 16 ++++++++++++++++ models/ObjectModel.js | 1 + routers/object.js | 11 +++++++++++ 3 files changed, 28 insertions(+) diff --git a/controllers/object.js b/controllers/object.js index f48271b3d..c87a3fee2 100644 --- a/controllers/object.js +++ b/controllers/object.js @@ -221,6 +221,21 @@ const setMatrix = function(objectID, body, callback) { callback(200, {success: true}); }; +const setRenderMode = async (objectID, body, callback) => { + let object = utilities.getObject(objects, objectID); + if (!object) { + callback(404, {failure: true, error: 'Object ' + objectID + ' not found'}); + return; + } + + object.renderMode = body.renderMode; + console.log(`server set renderMode of ${objectID} to ${body.renderMode}`); + + await utilities.writeObjectToFile(objects, objectID, globalVariables.saveToDisk); + utilities.actionSender({reloadObject: {object: objectID}, lastEditor: body.lastEditor}); + callback(200, {success: true}); +}; + /** * Upload an image file to the object's metadata folder. * The image is stored in a form, which can be parsed and written to the filesystem. @@ -629,6 +644,7 @@ module.exports = { saveCommit: saveCommit, resetToLastCommit: resetToLastCommit, setMatrix: setMatrix, + setRenderMode: setRenderMode, memoryUpload: memoryUpload, deactivate: deactivate, activate: activate, diff --git a/models/ObjectModel.js b/models/ObjectModel.js index 58382573f..39debd0bd 100644 --- a/models/ObjectModel.js +++ b/models/ObjectModel.js @@ -62,6 +62,7 @@ function ObjectModel(ip, version, protocol, objectId) { this.type = 'object'; // or: 'world' or 'human' or 'avatar' etc... this.timestamp = null; // timestamp optionally stores when the object was first created this.gaussianSplatRequestId = null; // Optional id for in-progress request to GS server + this.renderMode = null; // Can be set to 'mesh' or 'ai' to synchronize render mode across clients } /** diff --git a/routers/object.js b/routers/object.js index a3f283e76..5bd5e25ab 100644 --- a/routers/object.js +++ b/routers/object.js @@ -287,6 +287,17 @@ router.post('/:objectName/matrix', function (req, res) { res.status(statusCode).json(responseContents).end(); }); }); + +router.post('/:objectName/renderMode', (req, res) => { + if (!utilities.isValidId(req.params.objectName)) { + res.status(400).send('Invalid object name. Must be alphanumeric.'); + return; + } + objectController.setRenderMode(req.params.objectName, req.body, function(statusCode, responseContents) { + res.status(statusCode).json(responseContents).end(); + }); +}); + router.post('/:objectName/memory', function (req, res) { if (!utilities.isValidId(req.params.objectName)) { res.status(400).send('Invalid object name. Must be alphanumeric.'); From 9409e4956091bc5c76f7ea4c88816d8ebea6f78e Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Thu, 14 Mar 2024 15:51:53 -0400 Subject: [PATCH 17/22] cleanup gs-upload-test --- controllers/object.js | 12 ++++++++++-- libraries/webInterface/gui/index.js | 7 ++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/controllers/object.js b/controllers/object.js index c87a3fee2..97a03ec75 100644 --- a/controllers/object.js +++ b/controllers/object.js @@ -221,15 +221,23 @@ const setMatrix = function(objectID, body, callback) { callback(200, {success: true}); }; +/** + * Sets the renderMode of an object (e.g. 'mesh' or 'ai') and sends the new state to all clients via action message + * @param {string} objectID + * @param {Request.body} body + * @param {function} callback + */ const setRenderMode = async (objectID, body, callback) => { let object = utilities.getObject(objects, objectID); if (!object) { callback(404, {failure: true, error: 'Object ' + objectID + ' not found'}); return; } + if (typeof body.renderMode === 'undefined') { + callback(400, {failure: true, error: `Bad request: no renderMode specified in body`}); + } object.renderMode = body.renderMode; - console.log(`server set renderMode of ${objectID} to ${body.renderMode}`); await utilities.writeObjectToFile(objects, objectID, globalVariables.saveToDisk); utilities.actionSender({reloadObject: {object: objectID}, lastEditor: body.lastEditor}); @@ -241,7 +249,7 @@ const setRenderMode = async (objectID, body, callback) => { * The image is stored in a form, which can be parsed and written to the filesystem. * @param {string} objectID * @param {express.Request} req - * @param {express.Response} res + * @param {function} callback */ const memoryUpload = async function(objectID, req, callback) { if (!objects.hasOwnProperty(objectID)) { diff --git a/libraries/webInterface/gui/index.js b/libraries/webInterface/gui/index.js index 0a38b6729..7301a62a4 100644 --- a/libraries/webInterface/gui/index.js +++ b/libraries/webInterface/gui/index.js @@ -548,8 +548,8 @@ realityServer.updateManageObjects = function (thisItem2) { thisObject.dom.querySelector('.download').classList.add('inactive'); thisObject.dom.querySelector('.active').classList.add('inactive'); - // world object names are always clickable to get to remote operator - if (isRemoteOperatorSupported) { // world object button only needs to be clickable in this case + // world object names are always clickable to get to remote operator (assuming remote operator addon exists) + if (isRemoteOperatorSupported) { realityServer.changeActiveState(thisObject.dom, true, objectKey); } @@ -1987,7 +1987,8 @@ realityServer.gotClick = function (event) { }, 100); }; - const AUTO_GENERATE_XML = false; + // World objects can be totally empty, regular objects generate an XML file when you drop another target file e.g. jpg + const AUTO_GENERATE_XML = !shouldAddWorldObject; if (AUTO_GENERATE_XML) { // generate a placeholder xml file for this object let defaultSize = 0.3; From cad75c86104da7b398e751ccd125ca40ef935073 Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Thu, 14 Mar 2024 16:30:28 -0400 Subject: [PATCH 18/22] sanitize checkFileExists path --- controllers/object.js | 7 +++++++ libraries/webInterface/gui/index.js | 4 ++-- routers/object.js | 5 ++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/controllers/object.js b/controllers/object.js index 97a03ec75..0709ae055 100644 --- a/controllers/object.js +++ b/controllers/object.js @@ -428,6 +428,13 @@ const checkFileExists = async (objectId, filePath) => { if (!obj) { return false; } + + // prevent upward traversal so this can't be used maliciously. Only allow certain characters, and prevent ../ + let isSafePath = /^[a-zA-Z0-9_.\-\/]+$/.test(filePath) && !/\.\.\//.test(filePath); + if (!isSafePath) { + return false; + } + let objectIdentityDir = path.join(objectsPath, obj.name, identityFolderName); let absoluteFilePath = path.join(objectIdentityDir, filePath); return await fileExists(absoluteFilePath); diff --git a/libraries/webInterface/gui/index.js b/libraries/webInterface/gui/index.js index 7301a62a4..1a69cf180 100644 --- a/libraries/webInterface/gui/index.js +++ b/libraries/webInterface/gui/index.js @@ -1966,8 +1966,8 @@ realityServer.gotClick = function (event) { realityServer.objects[objectName].isWorldObject = true; } } else { - // this is how world objects get instantly initialized try { + // check if we need to generate an XML file, then setup and activate the object... let msgContent = JSON.parse(state); const finishSettingUpObject = () => { @@ -1987,7 +1987,7 @@ realityServer.gotClick = function (event) { }, 100); }; - // World objects can be totally empty, regular objects generate an XML file when you drop another target file e.g. jpg + // World objects can be totally empty, regular objects generate a default XML target file on creation const AUTO_GENERATE_XML = !shouldAddWorldObject; if (AUTO_GENERATE_XML) { // generate a placeholder xml file for this object diff --git a/routers/object.js b/routers/object.js index 5bd5e25ab..9141cafca 100644 --- a/routers/object.js +++ b/routers/object.js @@ -288,6 +288,7 @@ router.post('/:objectName/matrix', function (req, res) { }); }); +// Set an object's renderMode, and send the new state to all clients router.post('/:objectName/renderMode', (req, res) => { if (!utilities.isValidId(req.params.objectName)) { res.status(400).send('Invalid object name. Must be alphanumeric.'); @@ -414,7 +415,8 @@ router.post('/:objectName/frame/:frameName/pinned/', function (req, res) { }); }); -router.get('/:objectName/checkFileExists/*', (req, res) => { +// Check the existence of a file at a filepath within the .identity directory of the specified object +router.get('/:objectName/checkFileExists/*', async (req, res) => { if (!utilities.isValidId(req.params.objectName)) { res.status(400).send('Invalid object name. Must be alphanumeric.'); return; @@ -428,6 +430,7 @@ router.get('/:objectName/checkFileExists/*', (req, res) => { }); }); +// Check the existence of all the target files for the specified object router.get('/:objectName/checkTargetFiles/', (req, res) => { if (!utilities.isValidId(req.params.objectName)) { res.status(400).send('Invalid object name. Must be alphanumeric.'); From 3cd5b17406729757127f98acc0a2abe71c194241 Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Thu, 14 Mar 2024 16:39:17 -0400 Subject: [PATCH 19/22] cleanup gs-upload-test --- server.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server.js b/server.js index b0681fea7..0917f37e6 100644 --- a/server.js +++ b/server.js @@ -2922,7 +2922,7 @@ function objectWebServer() { } } else { - continueProcessingUpload(folderD, fileExtension); + continueProcessingUpload(); } // Step 2) - Generate a default XML file if needed @@ -3070,6 +3070,7 @@ function objectWebServer() { let deferred = false; function finishFn(folderName) { return async function() { + // cleanup the target directory after uploading files let nestedTargetDirPath = path.join(folderD, identityFolderName, 'target', folderName); try { From 2c332498732efe48f4171e0460b8ddf4d0017b8a Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Fri, 15 Mar 2024 14:14:35 -0400 Subject: [PATCH 20/22] fix linting error --- controllers/object.js | 1 + 1 file changed, 1 insertion(+) diff --git a/controllers/object.js b/controllers/object.js index 0709ae055..e591d9da4 100644 --- a/controllers/object.js +++ b/controllers/object.js @@ -430,6 +430,7 @@ const checkFileExists = async (objectId, filePath) => { } // prevent upward traversal so this can't be used maliciously. Only allow certain characters, and prevent ../ + // eslint-disable-next-line no-useless-escape let isSafePath = /^[a-zA-Z0-9_.\-\/]+$/.test(filePath) && !/\.\.\//.test(filePath); if (!isSafePath) { return false; From dcbdbbeaf565ae533b7eb533346100615b7a8d1a Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Fri, 15 Mar 2024 15:22:28 -0400 Subject: [PATCH 21/22] extracts targetId from dat file when uploaded individually, in addition to when its part of a zip --- server.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/server.js b/server.js index 0917f37e6..2bbef3501 100644 --- a/server.js +++ b/server.js @@ -2880,9 +2880,19 @@ function objectWebServer() { console.error(`error renaming ${filename} to target.${fileExtension}`, e); } - // Step 1) - resize image if necessary. Vuforia can make targets from jpgs of up to 2048px - // but we scale down to 1024px for a larger margin of error and (even) smaller filesize - if (fileExtension === 'jpg') { + // extract the targetId from the dat file when the dat file is uploaded + if (fileExtension === 'dat') { + try { + let targetUniqueId = await utilities.getTargetIdFromTargetDat(path.join(folderD, identityFolderName, 'target')); + let thisObjectId = utilities.readObject(objectLookup, req.params.id); + objects[thisObjectId].targetId = targetUniqueId; + console.log(`set targetId for ${thisObjectId} to ${targetUniqueId}`); + } catch (e) { + console.log('unable to extract targetId from dat file'); + } + // Step 1) - resize image if necessary. Vuforia can make targets from jpgs of up to 2048px + // but we scale down to 1024px for a larger margin of error and (even) smaller filesize + } else if (fileExtension === 'jpg') { var rawFilepath = folderD + '/' + identityFolderName + '/target/target.' + fileExtension; var tempFilepath = folderD + '/' + identityFolderName + '/target/target-temp.' + fileExtension; From a4ac496d9d6631b4b0e044e149bb4ccf5b5708e9 Mon Sep 17 00:00:00 2001 From: Ben Reynolds Date: Fri, 15 Mar 2024 15:32:47 -0400 Subject: [PATCH 22/22] fix circular dependency and add renderMode to test expected result --- controllers/object.js | 8 ++++---- server.js | 2 +- tests/upload-target.test.js | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/controllers/object.js b/controllers/object.js index e591d9da4..a006b8a03 100644 --- a/controllers/object.js +++ b/controllers/object.js @@ -4,7 +4,6 @@ const formidable = require('formidable'); const utilities = require('../libraries/utilities'); const {fileExists, unlinkIfExists, mkdirIfNotExists} = utilities; const {startSplatTask} = require('./object/SplatTask.js'); -const server = require('../server'); const { beatPort } = require('../config.js'); // Variables populated from server.js with setup() @@ -28,6 +27,7 @@ let objectLookup; let activeHeartbeats; let knownObjects; let setAnchors; +let objectBeatSender; const deleteObject = async function(objectID) { let object = utilities.getObject(objects, objectID); @@ -603,8 +603,7 @@ const uploadTarget = async (objectName, req, res) => { await utilities.writeObjectToFile(objects, thisObjectId, globalVariables.saveToDisk); await setAnchors(); - // await objectBeatSender(beatPort, thisObjectId, objects[thisObjectId].ip, true); - await server.objectBeatSender(beatPort, thisObjectId, objects[thisObjectId].ip, true); + await objectBeatSender(beatPort, thisObjectId, objects[thisObjectId].ip, true); // delete the tmp folder and any files within it await utilities.rmdirIfExists(uploadDir); @@ -641,7 +640,7 @@ const getObject = function (objectID, excludeUnpinned) { }; const setup = function (objects_, globalVariables_, hardwareAPI_, objectsPath_, sceneGraph_, - objectLookup_, activeHeartbeats_, knownObjects_, setAnchors_) { + objectLookup_, activeHeartbeats_, knownObjects_, setAnchors_, objectBeatSender_) { objects = objects_; globalVariables = globalVariables_; hardwareAPI = hardwareAPI_; @@ -651,6 +650,7 @@ const setup = function (objects_, globalVariables_, hardwareAPI_, objectsPath_, activeHeartbeats = activeHeartbeats_; knownObjects = knownObjects_; setAnchors = setAnchors_; + objectBeatSender = objectBeatSender_; }; module.exports = { diff --git a/server.js b/server.js index 2bbef3501..a5c5ff5dc 100644 --- a/server.js +++ b/server.js @@ -4632,7 +4632,7 @@ function setupControllers() { linkController.setup(objects, knownObjects, socketArray, globalVariables, hardwareAPI, objectsPath, socketUpdater, engine); logicNodeController.setup(objects, globalVariables, objectsPath); nodeController.setup(objects, globalVariables, objectsPath, sceneGraph); - objectController.setup(objects, globalVariables, hardwareAPI, objectsPath, sceneGraph, objectLookup, activeHeartbeats, knownObjects, setAnchors); + objectController.setup(objects, globalVariables, hardwareAPI, objectsPath, sceneGraph, objectLookup, activeHeartbeats, knownObjects, setAnchors, objectBeatSender); spatialController.setup(objects, globalVariables, hardwareAPI, sceneGraph); } diff --git a/tests/upload-target.test.js b/tests/upload-target.test.js index 01dfda70e..e45724f4d 100644 --- a/tests/upload-target.test.js +++ b/tests/upload-target.test.js @@ -102,6 +102,7 @@ test('target upload to /content/:objectName', async () => { version: '3.2.2', deactivated: false, protocol: 'R2', + renderMode: null, visible: false, visibleText: false, visibleEditing: false,