diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8200311 --- /dev/null +++ b/.gitignore @@ -0,0 +1,63 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (http://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Typescript v1 declaration files +typings/ + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env + +# Elastic Beanstalk Files +.elasticbeanstalk/* +!.elasticbeanstalk/*.cfg.yml +!.elasticbeanstalk/*.global.yml diff --git a/WS-node/.ebextensions/files.config b/WS-node/.ebextensions/files.config new file mode 100644 index 0000000..cc71e79 --- /dev/null +++ b/WS-node/.ebextensions/files.config @@ -0,0 +1,31 @@ +files: + "/etc/nginx/conf.d/01_websockets.conf" : + mode: "000644" + owner: root + group: root + content : | + upstream nodejs { + server 127.0.0.1:8081; + keepalive 256; + } + + server { + listen 8080; + + location / { + proxy_pass http://nodejs; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + } + + "/opt/elasticbeanstalk/hooks/appdeploy/enact/41_remove_eb_nginx_confg.sh": + mode: "000755" + owner: root + group: root + content : | + mv /etc/nginx/conf.d/00_elastic_beanstalk_proxy.conf /etc/nginx/conf.d/00_elastic_beanstalk_proxy.conf.old diff --git a/WS-node/.gitignore b/WS-node/.gitignore new file mode 100644 index 0000000..bca646a --- /dev/null +++ b/WS-node/.gitignore @@ -0,0 +1,5 @@ + +# Elastic Beanstalk Files +.elasticbeanstalk/* +!.elasticbeanstalk/*.cfg.yml +!.elasticbeanstalk/*.global.yml diff --git a/WS-node/.vscode/launch.json b/WS-node/.vscode/launch.json new file mode 100644 index 0000000..22ce6df --- /dev/null +++ b/WS-node/.vscode/launch.json @@ -0,0 +1,21 @@ +{ + // Use IntelliSense to learn about possible Node.js debug attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Launch Program", + "program": "${workspaceRoot}/app.js", + "cwd": "${workspaceRoot}" + }, + { + "type": "node", + "request": "attach", + "name": "Attach to Process", + "port": 5858 + } + ] +} \ No newline at end of file diff --git a/WS-node/app.js b/WS-node/app.js new file mode 100644 index 0000000..70cb58c --- /dev/null +++ b/WS-node/app.js @@ -0,0 +1,197 @@ + +require('dotenv').config() + +var WebSocketServer = require('websocket').server; + +var http = require('http'); +var HttpDispatcher = require('httpdispatcher'); +var dispatcher = new HttpDispatcher(); +const fs = require('fs'); +const winston = require('winston') +winston.level = process.env.LOG_LEVEL || 'info' + +var AsrClient = require('./lib/asrClient') +var asrActive = false +var myAsrClient; +var engineStartedMs; +var connections = [] + + +//Create a server +var server = http.createServer(function(req, res) { + handleRequest(req,res); +}); + +// Loading socket.io +var io = require('socket.io').listen(server); + +// When a client connects, we note it in the console +io.sockets.on('connection', function (socket) { + winston.log('info','A client is connected!'); +}); + + +var wsServer = new WebSocketServer({ + httpServer: server, + autoAcceptConnections: true, + binaryType: 'arraybuffer' +}); + + +//Lets use our dispatcher +function handleRequest(request, response){ + try { + //log the request on console + winston.log('info', 'handleRequest',request.url); + //Dispatch + dispatcher.dispatch(request, response); + } catch(err) { + console.log(err); + } +} +dispatcher.setStatic('/public'); +dispatcher.setStaticDirname('public'); +dispatcher.onGet("/", function(req, res) { + winston.log('info', 'loading index'); + winston.log('info', 'port', process.env.PORT) + fs.readFile('./public/index.html', 'utf-8', function(error, content) { + winston.log('debug', 'loading Index'); + res.writeHead(200, {"Content-Type": "text/html"}); + res.end(content); + }); +}); +// Serve the ncco +dispatcher.onGet("/ncco", function(req, res) { + fs.readFile('./ncco.json', function(error, data) { + winston.log('debug', 'loading ncco'); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(data, 'utf-8'); + }); +}); + +dispatcher.onPost("/terminate", function(req, res) { + winston.log('info', 'terminate called'); + wsServer.closeAllConnections(); + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(); +}); + +wsServer.on('connect', function(connection) { + connections.push(connection); + + winston.log('info', (new Date()) + ' Connection accepted' + ' - Protocol Version ' + connection.webSocketVersion); + connection.on('message', function(message) { + + if (message.type === 'utf8') { + try { + var json = JSON.parse(message.utf8Data); + winston.log('info', "json", json['app']); + + if (json['app'] == "audiosocket") { + VBConnect(); + winston.log('info', 'connecting to VB'); + } + + } catch (e) { + winston.log('error', 'message error catch', e) + } + winston.log('info', "utf ",message.utf8Data); + } + else if (message.type === 'binary') { + // Reflect the message back + // connection.sendBytes(message.binaryData); + if (myAsrClient != null && asrActive) { + winston.log('debug', "sendingDate ",message.binaryData); + myAsrClient.sendData(message.binaryData) + } + } + }); + + connection.on('close', function(reasonCode, description) { + winston.log('info', (new Date()) + ' Peer ' + connection.remoteAddress + ' disconnected.'); + wsServer.closeAllConnections(); + + }); +}); + +wsServer.on('close', function(connection) { + winston.log('info', 'socket closed'); + if (asrActive) { + io.sockets.emit('status', "disconnected"); + winston.log('info', 'trying to close ASR client'); + myAsrClient.close(); + myAsrClient = null; + asrActive = false; + } + else { + winston.log('info', 'asr not active, cant close'); + } +}) + +wsServer.on('error', function(error) { + winston.log('error', 'Websocket error', error); +}) + +var port = process.env.PORT || 8000 +server.listen(port, function(){ + winston.log('info', "Server listening on :%s", port); +}); + +function VBConnect() { + + winston.log('debug', 'load AsrClient'); + myAsrClient = new AsrClient() + var url = process.env.ASR_URL; + client_key = process.env.ASR_CLIENT_KEY; + client_secret = process.env.ASR_CLIENT_SECRET; + myAsrClient.setup(url, client_key, client_secret, (err) => { + if (err) { + return console.error('AsrClient error:', err) + } + var controlMessage = {} + controlMessage.language = 'en-US' // default starting value + controlMessage.targetSampleRate = '8' // default starting value + controlMessage.sampleRate = '16000' + controlMessage.windowSize = 10 + myAsrClient.reserveAsr(controlMessage) + winston.log('debug', "sending control message", controlMessage); + + myAsrClient.subscribeEvent('engineState', (msg) => { + winston.log('info', 'Engine State Event', msg) + if (msg === 'ready') { + asrActive = true + engineStartedMs = Date.now() + winston.log('info', 'Setting asrActive to true: ', asrActive, ' this.asrActive ', this.asrActive) + io.sockets.emit('status', "connected"); + } + }) + + myAsrClient.subscribeEvent('transcript', (msg) => { + winston.log('debug', 'transcript', msg); + io.sockets.emit('transcript', msg); + }) + + myAsrClient.subscribeEvent('sentiment', (msg) => { + winston.log('debug', 'sentiment', msg); + io.sockets.emit('sentiment', msg); + + }) + + myAsrClient.subscribeEvent('nlp', (msg) => { + winston.log('debug', 'nlp', msg); + io.sockets.emit('nlp', msg); + }) + + myAsrClient.subscribeEvent('keywords', (msg) => { + winston.log('info', 'keywords', msg); + io.sockets.emit('keywords', msg); + }) + + + myAsrClient.subscribeEvent('latency', (msg) => { + winston.log('info', 'latency', msg); + io.sockets.emit('latency', msg); + }) + }) +} \ No newline at end of file diff --git a/WS-node/lib/asrClient.js b/WS-node/lib/asrClient.js new file mode 100644 index 0000000..eb11c46 --- /dev/null +++ b/WS-node/lib/asrClient.js @@ -0,0 +1,243 @@ +// events are: sentiment, engineState, keywords, nlp, transcript +const io = require('socket.io-client') +const request = require('request') +const url = require('url') +var socket; + +function getBearerToken (asrUrl, clientKey, clientSecret, callback) { + request({ + url: asrUrl+'/api/oauth2/token', + method: 'POST', + json: true, + headers: { 'content-type': 'application/json' }, + auth: { + user: clientKey, + pass: clientSecret, + sendImmediately: false + }, + body: { 'grant_type': 'client_credentials' } + }, (err, res, body) => { + if (err) { + return callback(new Error('Authentication to API error:' + err)) + } + if (res.statusCode !== 200) { + return callback(new Error('Authentication to API got error code: ' + + res.statusCode)) + } + if (body.token_type === 'Bearer') { + var bearerToken = body.access_token + return callback(null, bearerToken) + } + callback(new Error('Wrong Bearer token')) + }) +} + +function AsrClient () { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0' + var self = this + + this.defaultControlMessage = { language: 'en-US', sampleRate: '16' } + this.transcript = [] + this.text = '' + this.events = [] + this.socket = null + this.sendAction = false + + self.setup = function (asrUrl, clientKey, clientSecret, callback) { + getBearerToken(asrUrl, clientKey, clientSecret, (err, bearerToken) => { + if (err) { + return callback(err) + } + + socket = io.connect(asrUrl, { + secure: true, + extraHeaders: { + Authorization: 'Bearer ' + bearerToken + } + }) + //console.log('new socket ', socket) + + socket.on('connect_error', (error) => { + console.log('websocket connect error ', error.description, ' (', + error.type, ') to ', asrUrl) + console.log(error.message) + }) + + socket.on('disconnect', () => { + console.log('disconnected') + // self.sendEndOfStream() + }) + + socket.on('engine state', (msg) => { + console.log('Engine state message: ', msg) + if (msg === "ready") { + engineStartedMs = Date.now() + } + self.emit('engineState', msg) // ready means start streaming + }) + + socket.on('sentiment estimate', (msg) => { + //console.log('sentiment: ', msg) + let x = JSON.parse(msg) + self.emit('sentiment', x) + }) + + socket.on('basic keywords event', (msg) => { + //console.log('kw: ', msg) + let x = JSON.parse(msg) + self.emit('keywords', x) + }) + + socket.on('nlp event', (msg) => { + //console.log('nlp: ', msg) + let x = JSON.parse(msg) + self.emit('nlp', x) + }) + '' + + socket.on('transcript segment', (msg) => { + var messageArrivedMs = Date.now() + var tsinceEngineStarted = messageArrivedMs - engineStartedMs + let x + try { + x = JSON.parse(msg) + } catch (err) { + //console.log('json parse err ', err) + //console.log(' and msg is:', msg, ':') + x = { words: [] } + } + + // collect the transcript fragments into one long transcript array, removing old words as we go + let tlen = self.transcript.length + let xlen = x.words.length + if (xlen > 0) { + let xP0 = x.words[0].p + if (tlen > 0) { + let tPn = self.transcript[tlen - 1].p + let nRemove = tPn - xP0 + 1 + if (nRemove > 0) { + for (let i = 0; i < nRemove; i++) { + self.transcript.pop() + } + } + } + + x.words.forEach((item, index, array) => { + var object = new Object() + object.item = item.p + object.time = (tsinceEngineStarted -item.s)/1000.0 + console.log("item", item); + + + + + + self.emit('latency',object) + + self.transcript.push(item) + }) + + var actionText = '' + var newIndex + // extract just the text for dsiplay and replace the silence tag with ellipses + var text = '' + self.transcript.forEach((item, index, array) => { + text = text + ' ' + item.w + console.log("item.w " + item.w); + if (item.w == "action" || item.w == "actions" ) { + newIndex = index + } + + if (index > newIndex) { + actionText += item.w + ' ' + } + + }) + var re = /<\/s>/gi + text = text.replace(re, '

') + + self.emit('transcript', text) + console.log('actionText ',actionText); + + if (actionText != '') { + this.sendAction = true + console.log('emmit action ', actionText); + self.emit('action', actionText) + } + + } else { + console.log('Empty transcript event!') + } + }) + + function WordCount(str) { + return str.split(" ").length; + } + + + self.subscribeEvent = function (eventName, fn) { + self.events[eventName] = self.events[eventName] || [] + let token = 'uid_' + String(self.uid++) + let item = { fn, token } + self.events[eventName].push(item) + return token + } + + self.unsubscribeEvent = function (eventName, token) { + if (self.events[eventName]) { + for (var i = 0; i < self.events[eventName].length; i++) { + if (self.events[eventName][i].token === token) { + self.events[eventName].splice(i, 1) + break + } + } + } + } + + // used internally only + self.emit = function (eventName, data) { + //console.log('emitting for eventName: ', eventName, ' and data ', data, ' ', self.events[eventName]) + if (self.events[eventName]) { + self.events[eventName].forEach(item => { + item.fn(data) + }) + } + } + + self.onAudio = function (data) { + socket.emit('audio-packet', data) + } + + self.endOfAudio = function () { + console.log('sending stream end') + socket.emit('stream-close', 'goodbye stream') + } + + self.reserveAsr = function (controlMessage) { + console.log('sending stream open') + socket.emit('stream-open', JSON.stringify(controlMessage)) + } + + callback(null) + }) + } + + self.sendData = function(data) { + self.onAudio(data); + } + + self.close = function() { + self.endOfAudio() + } +} + +AsrClient.prototype.convertFloat32ToInt16 = (buffer) => { + var l = buffer.length + var buf = new Int16Array(l) + while (l--) { + buf[l] = Math.min(1, buffer[l]) * 0x7FFF + } + return buf.buffer +} + +module.exports = AsrClient diff --git a/WS-node/lib/asrClient.py b/WS-node/lib/asrClient.py new file mode 100644 index 0000000..2de72b1 --- /dev/null +++ b/WS-node/lib/asrClient.py @@ -0,0 +1,33 @@ +import websocket +import thread +import time + +def on_message(ws, message): + print message + +def on_error(ws, error): + print error + +def on_close(ws): + print "### closed ###" + +def on_open(ws): + def run(*args): + for i in range(3): + time.sleep(1) + ws.send("Hello %d" % i) + time.sleep(1) + ws.close() + print "thread terminating..." + thread.start_new_thread(run, ()) + + +if __name__ == "__main__": + websocket.enableTrace(True) + ws = websocket.WebSocketApp("ws://echo.websocket.org/", + on_message = on_message, + on_error = on_error, + on_close = on_close) + ws.on_open = on_open + + ws.run_forever() diff --git a/WS-node/package.json b/WS-node/package.json new file mode 100644 index 0000000..f07bd13 --- /dev/null +++ b/WS-node/package.json @@ -0,0 +1,21 @@ +{ + "name": "ws-node", + "version": "1.0.0", + "description": "", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "buffer-to-arraybuffer": "0.0.4", + "dotenv": "^4.0.0", + "express": "^4.15.2", + "http": "0.0.0", + "httpdispatcher": "^2.0.1", + "request": "^2.79.0", + "socket.io-client": "^1.7.2", + "websocket": "^1.0.24", + "winston": "^2.3.1" + } +} diff --git a/WS-node/public/index.html b/WS-node/public/index.html new file mode 100644 index 0000000..d810302 --- /dev/null +++ b/WS-node/public/index.html @@ -0,0 +1,74 @@ + + + + + Socket.io + + + + + + + + + + + + + + +
+ + +
+
+

Communication with Nexmo Websocket and Voicebase Real-time transcription engine

+

Dial: (213)237-1836 to get started

+
+ + + +
status:
+
+
+
CanvasJS.com
+
+
+
CanvasJS.com
+
+
+ +
+
+
+
+
+
+
+
+
+ + + + \ No newline at end of file diff --git a/WS-node/public/js/charts.js b/WS-node/public/js/charts.js new file mode 100644 index 0000000..661bab1 --- /dev/null +++ b/WS-node/public/js/charts.js @@ -0,0 +1,289 @@ +var keywordChart = (function(){ + var chart; + var dps = []; // dataPoints + + + + var updateInterval = 100; + var dataLength = 1000; // number of dataPoints visible at any point + //window.onload = function () { + + //init + chart = new CanvasJS.Chart("keywordContainer",{ + title :{ + text: "Keywords" + }, + legend: { + horizontalAlign: "center", // "center" , "right" + verticalAlign: "top", // "top" , "bottom" + fontSize: 15 + }, + //axisY:{ + // minimum: -1.0, + // maximum: 1.0, + //}, + data: [ + {type: "bar", dataPoints: dps }, + ] + }); + resetChart(); + + function resetChart() { + //this.dps=[]; + dps.length=0 + chart.render(); + }; + + function updateChart(kws) { + dps.length=0; + + //console.log(kws); + var arrayLength = kws.words.length; + for (var i = 0; i < arrayLength; i++) { + //console.log(" "+i+ " "+kws.words[i]); + // workaround to get reasonably comparable + if (kws.words[i][1] <0) { + metric = 1.0 + kws.words[i][1]/100.0 + } else { + metric =kws.words[i][1]; + } + dps.push({ + label: kws.words[i][0], + y: metric + }); + } + chart.render(); + + }; + return { + reset: resetChart, + update: updateChart + }; +})(); + + + +var sentimentChart = (function(){ + var chart; + var pos = []; // dataPoints + var neg = []; // dataPoints + var neu = []; // dataPoints + var compound = []; // dataPoints + + + var updateInterval = 100; + var dataLength = 1000; // number of dataPoints visible at any point + //window.onload = function () { + + //init + chart = new CanvasJS.Chart("sentimentContainer",{ + title :{ + text: "Sentiment +/-" + }, + legend: { + horizontalAlign: "center", // "center" , "right" + verticalAlign: "top", // "top" , "bottom" + fontSize: 15 + }, + axisY:{ + minimum: -1.0, + maximum: 1.0, + + }, + data: [ + //{type: "spline", legendText: "positive", showInLegend: true, color: "green", lineThickness: 1, markerType: "none", dataPoints: pos }, + //{type: "spline", legendText: "negative", showInLegend: true, color: "red", lineThickness: 1, markerType: "none", dataPoints: neg }, + //{type: "spline", legendText: "neutral", showInLegend: true, color: "gray", lineThickness: 1, markerType: "none", dataPoints: neu }, + {type: "spline", legendText: "compound", showInLegend: true, color: "blue", lineThickness: 3, markerType: "none", dataPoints: compound} + ] + }); + resetChart(); + + function resetChart() { + //this.dps=[]; + compound.length=0 + pos.length=0 + neg.length=0 + neu.length=0 + chart.render(); + }; + + function updateChart(sentiment) { + pos.push({ + //x: sentiment.time, + y: sentiment.pos + }); + neg.push({ + //x: sentiment.time, + y: sentiment.neg + }); + neu.push({ + //x: sentiment.time, + y: sentiment.neu + }); + compound.push({ + //x: sentiment.time, + y: sentiment.compound + }); + if (compound.length > dataLength) { + pos.shift(); + neg.shift(); + neu.shift(); + compound.shift(); + + } + + chart.render(); + + }; + + function updateChartToDate(sentimentList) { + resetChart(); + for (var i = 0; i < sentimentList.length; i++) { + + sentiment =sentimentList[i]; + pos.push({ + //x: sentiment.time, + y: sentiment.pos + }); + neg.push({ + //x: sentiment.time, + y: sentiment.neg + }); + neu.push({ + //x: sentiment.time, + y: sentiment.neu + }); + compound.push({ + //x: sentiment.time, + y: sentiment.compound + }); + if (compound.length > dataLength) { + pos.shift(); + neg.shift(); + neu.shift(); + compound.shift(); + + } + } + + chart.render(); + + }; + return { + reset: resetChart, + updateChartToDate:updateChartToDate, + update: updateChart + }; +})(); + +var wordLengthDistChart = (function(){ + var chart; + var dps = []; // dataPoints + + var updateInterval = 100; + var dataLength = 1000; // number of dataPoints visible at any point + //window.onload = function () { + + //init + chart = new CanvasJS.Chart("wordLengthDist",{ + title :{ + text: "Word Length Distribution" + }, + legend: { + horizontalAlign: "center", // "center" , "right" + verticalAlign: "top", // "top" , "bottom" + fontSize: 15 + }, + //axisY:{ + // minimum: -1.0, + // maximum: 1.0, + //}, + data: [ + {type: "pie", dataPoints: dps }, + ] + }); + resetChart(); + + function resetChart() { + //this.dps=[]; + dps.length=0 + chart.render(); + }; + + function updateChart(wordLengthDist) { + dps.length=0; + + //console.log(kws); + var arrayLength = wordLengthDist.length; + for (var i = 0; i < arrayLength; i++) { + //console.log(" "+i+ " "+kws.words[i]); + dps.push({ + label: wordLengthDist[i][0], + y: wordLengthDist[i][1] + }); + } + chart.render(); + + }; + return { + reset: resetChart, + update: updateChart + }; +})(); + +var posTagDistChart = (function(){ + var chart; + var dps = []; // dataPoints + + var updateInterval = 100; + var dataLength = 1000; // number of dataPoints visible at any point + //window.onload = function () { + + //init + chart = new CanvasJS.Chart("posTagDist",{ + title :{ + text: "Part of Speech Tag Distribution" + }, + legend: { + horizontalAlign: "center", // "center" , "right" + verticalAlign: "top", // "top" , "bottom" + fontSize: 15 + }, + //axisY:{ + // minimum: -1.0, + // maximum: 1.0, + //}, + data: [ + {type: "pie", dataPoints: dps }, + ] + }); + resetChart(); + + function resetChart() { + //this.dps=[]; + dps.length=0 + chart.render(); + }; + + function updateChart(posTagDist) { + dps.length=0; + + //console.log(kws); + var arrayLength = posTagDist.length; + for (var i = 0; i < arrayLength; i++) { + //console.log(" "+i+ " "+kws.words[i]); + dps.push({ + label: posTagDist[i][0], + y: posTagDist[i][1] + }); + } + chart.render(); + + }; + return { + reset: resetChart, + update: updateChart + }; +})(); + diff --git a/WS-node/public/js/socket.js b/WS-node/public/js/socket.js new file mode 100644 index 0000000..fb4bc34 --- /dev/null +++ b/WS-node/public/js/socket.js @@ -0,0 +1,64 @@ +var socket = io(); +var actionItems = [] +var beginTakingAction = false + +var strOut; +socket.on('transcript', function(x) { + var div = $('div.transcription') + div.html(x); + console.log("transcript " + x); + if (!scrolling) { + div.scrollTop(div[0].scrollHeight); + } + + // //find the world action + // if (!beginTakingAction) { + // var actionIndex = x.lastIndexOf("action") + // console.log("actionIndex " + actionIndex); + // if(actionIndex > 0) { + // beginTakingAction = true + // strOut = x.substr(actionIndex); + // console.log("strOut " + strOut); + + // if (strOut.lastIndexOf("

") > 0) { + // console.log('saving action'); + // beginTakingAction = false + // actionItems.push(strOut) + // $('div.action').html(actionItems[actionItems.length-1]); + // } + // } + // } + +}) + +socket.on('action', function(x) { + console.log('sending action',x); + actionItems.push(x) + $('div.action').html(actionItems[actionItems.length-1]); + +}) +socket.on('sentiment', function(x) { + sentimentChart.update(x) +}) + +socket.on('nlp', function(x) { + wordLengthDistChart.update(x.wordLenghDist); + posTagDistChart.update(x.posTagDist); +}) + +socket.on('keywords', function(x) { + keywordChart.update(x) +}) + +socket.on('status', function(status) { + $('div.status').html("status: " + status); + + if (status == "connected") { + sentimentChart.reset() + keywordChart.reset() + wordLengthDistChart.reset() + posTagDistChart.reset() + $('div.transcription').html(''); + } + +}) \ No newline at end of file diff --git a/WS-node/public/nexmo.png b/WS-node/public/nexmo.png new file mode 100644 index 0000000..76f6014 Binary files /dev/null and b/WS-node/public/nexmo.png differ diff --git a/WS-node/public/voicebase.png b/WS-node/public/voicebase.png new file mode 100644 index 0000000..836f20b Binary files /dev/null and b/WS-node/public/voicebase.png differ diff --git a/github/Nexmo-RTS-Voicebase b/github/Nexmo-RTS-Voicebase new file mode 160000 index 0000000..2586f84 --- /dev/null +++ b/github/Nexmo-RTS-Voicebase @@ -0,0 +1 @@ +Subproject commit 2586f84789497c1f59b66ffcc5add45ecaeb1665 diff --git a/server/.gitignore b/server/.gitignore new file mode 100644 index 0000000..bca646a --- /dev/null +++ b/server/.gitignore @@ -0,0 +1,5 @@ + +# Elastic Beanstalk Files +.elasticbeanstalk/* +!.elasticbeanstalk/*.cfg.yml +!.elasticbeanstalk/*.global.yml diff --git a/server/.vscode/launch.json b/server/.vscode/launch.json new file mode 100644 index 0000000..96b0be7 --- /dev/null +++ b/server/.vscode/launch.json @@ -0,0 +1,22 @@ +{ + // Use IntelliSense to learn about possible Node.js debug attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + + { + "type": "node", + "request": "launch", + "name": "Launch Program", + "program": "${workspaceRoot}/index.js", + "cwd": "${workspaceRoot}" + }, + { + "type": "node", + "request": "attach", + "name": "Attach to Process", + "port": 5858 + } + ] +} \ No newline at end of file diff --git a/server/index.js b/server/index.js new file mode 100644 index 0000000..cb8de40 --- /dev/null +++ b/server/index.js @@ -0,0 +1,363 @@ +require('dotenv').config() +var express = require('express') +var app = express() +var bodyParser = require('body-parser'); +var Nexmo = require('nexmo'); +var Promise = require('bluebird'); +var util = require("util"); +const winston = require('winston') + +var executed = false; +var didStart = false +var USER_NUMBER = "" +var WEB_SOCKET = 'ws://' + process.env.WEB_SOCKET_URL + '/socket'; + +var SMS_TEXT = "View the transcription service here: " + "http://" + process.env.WEB_SOCKET_URL; +var converstationIDs = []; +var connectedUsers = []; + +var nexmo = new Nexmo({ + apiKey: process.env.API_KEY, + apiSecret: process.env.API_SECRET, + applicationId: process.env.APPLICATION_ID, + privateKey: __dirname + '/' + process.env.PRIVATE_KEY_PATH +}, + { debug: process.env.NEXMO_DEBUG } +); + +winston.level = process.env.LOG_LEVEL +var calls = Promise.promisifyAll(nexmo.calls); + +app.set('port', process.env.PORT || 3000) +app.use(bodyParser.json()); +app.use(bodyParser.urlencoded({ extended: false })); + +winston.log('debug', 'env vars', process.env); + +app.get('/answer', function (req, res) { + res.header("Access-Control-Allow-Origin", "*"); + res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); + winston.log('info', 'answer route called', req.body); + reset() + var url = require('url'); + var url_parts = url.parse(req.url, true); + var query = url_parts.query; + + connectedUsers.push(query.from); + winston.log('debug', 'adding user to sms list', connectedUsers); + if (process.env.SINGLE_USER) { + winston.log('debug', 'In single user mode'); + var json = [ + { + "action": "connect", + "eventUrl": [ + "http://" + req.headers.host + "/events" + ], + "from": process.env.NEXMO_NUMBER, + "endpoint": [ + { + "type": "websocket", + "uri": WEB_SOCKET, + "content-type": "audio/l16;rate=16000", + "headers": { + "app": "audiosocket" + } + } + ] + } + ] + winston.log('info', 'conference JSON', json); + res.send(json) + return + } + + if (process.env.TO_NUMBER) { + var json = [ + { + "action": "talk", + "text": "Calling user" + }, + { + "action": "conversation", + "name": process.env.CONFERENCE_NAME, + "endOnExit": "true" + } + ] + + var to = { + type: 'phone', + number: process.env.TO_NUMBER, + } + winston.log('info', 'answer JSON', json); + dial(to, process.env.NEXMO_NUMBER, req.headers.host, function (result) { + winston.log('info', 'dial result', result); + res.send(json) + }) + + } else { + var baseURL = "http://" + req.headers.host; + var json = [ + { + "action": "talk", + "text": "Please enter a phone number to call, press the pound key when complete", + "bargeIn": "false" + }, + { + "action": "input", + "submitOnHash": true, + "timeOut": 60, + "maxDigits": 20, + "eventUrl": [baseURL + "/events_ivr"] + }, + { + "action": "conversation", + "name": process.env.CONFERENCE_NAME, + "endOnExit": "true", + } + ] + + winston.log('info', 'answer JSON', json); + res.send(json) + } +}) + +app.get('/conference', function (req, res) { + + res.header("Access-Control-Allow-Origin", "*"); + res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); + var url = require('url'); + var url_parts = url.parse(req.url, true); + var query = url_parts.query; + var json = [ + { + "action": "talk", + "text": "Dialing into the conference now", + }, + { + "action": "conversation", + "name": process.env.CONFERENCE_NAME, + "startOnEnter": "false" + } + ] + winston.log('info', 'conference JSON', json); + res.send(json) +}) + +app.post('/events_ivr', function (req, res, next) { + res.header("Access-Control-Allow-Origin", "*"); + res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); + winston.log('info', 'events_ivr', req.body); + + var baseURL = req.headers.host; + var number = req.body.dtmf; + var from = req.body.from; + + USER_NUMBER = number + + if (number == '' || number.length < 11) { + winston.log('debug', 'could not get dtmf number', number); + var json = [ + { + "action": "talk", + "text": "Sorry, I did not get that phone number. Goodbye" + } + ] + res.send(json); + } else { + var to = { + type: 'phone', + number: number, + } + dial(to, process.env.NEXMO_NUMBER, baseURL, function (result) { + winston.log('info', 'IVR Dial result', result); + }) + + res.sendStatus(200); + } + +}); + +app.post('/events', function (req, res, next) { + res.header("Access-Control-Allow-Origin", "*"); + res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); + winston.log('info', 'events', req.body); + + var baseURL = req.headers.host; + var from = req.body.from; + + if (req.body.status == "answered") { + converstationIDs.push(req.body.uuid); + winston.log('debug', 'adding converstaion uuid', converstationIDs); + + if (req.body.to != process.env.NEXMO_NUMBER && req.body.to != WEB_SOCKET) { + connectedUsers.push(req.body.to); + winston.log('debug', 'adding user to sms list', connectedUsers); + } + + if (req.body.to == USER_NUMBER || req.body.to == process.env.TO_NUMBER) { + + var to = { + type: 'websocket', + uri: WEB_SOCKET, + "content-type": "audio/l16;rate=16000", + "headers": { + "app": "audiosocket" + } + } + winston.log('debug', 'calling websocket', req.body); + dial(to, from, baseURL, function (result) { + winston.log('debug', 'called websocket', result); + sendAllSms(SMS_TEXT, function () { + winston.log('info', 'all sms sent'); + }) + }) + res.sendStatus(200); + return + + } + } + else if (req.body.status == "completed") { + winston.log('debug', 'called ended', req.body); + winston.log('debug', 'calling hangup'); + performHangup() + } + res.sendStatus(200); +}); + +var performHangup = (function () { + return function () { + winston.log('debug', "executed " + executed + " didStart " + didStart) + if (!executed && !didStart) { + didStart = true + hangupCalls(function () { + executed = true; + winston.log('info', 'hangup complete'); + }) + } + }; +})(); + +process.on('unhandledRejection', (reason) => { + winston.log('error', 'unhandledRejection', reason) +}); + +; + +app.listen(process.env.PORT || 3000, function () { + winston.log('info', 'Nexmo Phone app listening on port ' + (process.env.PORT || 3000)) +}) + + +function dial(to, from, serverURL, callback) { + var json = { + to: [to], + from: { + type: 'phone', + number: from + }, + answer_url: ['http://' + serverURL + '/conference'], + event_url: ['http://' + serverURL + '/events', 'http://' + process.env.WEB_SOCKET_URL + '/events'] + } + winston.debug('debug', 'dial JSON', json); + calls.createAsync(json).then(function (res) { + winston.log('debug', 'call created', res) + callback(res) + }) +} + +function hangupCalls(callback) { + Promise.each(converstationIDs, function (converstationID) { + return new Promise(function (resolve, reject) { + calls.updateAsync(converstationID, { action: 'hangup' }) + .then(function (resp) { + setTimeout(function () { + winston.log('info', 'hangup result: for id: ' + converstationID, resp) + resolve(); + }, 2000) + + }) + }); + + }) + .then(function (allItems) { + winston.log('debug', 'all items', allItems) + callback(); + }) +} + +function terminate(callback) { + winston.log('debug', 'calling terminate'); + var request = require('request'); + request({ + url: 'http://' + process.env.WEB_SOCKET_URL + '/terminate', + method: 'POST', + json: true, + headers: { 'content-type': 'application/json' }, + }, (err, res, body) => { + winston.log('debug', 'terminate called'); + callback() + }) +} + +function sendAllSms(message, callback) { + + Promise.each(connectedUsers, function (phoneNumber) { + return new Promise(function (resolve, reject) { + sendSMS(phoneNumber, SMS_TEXT, function (resp) { + setTimeout(function () { + winston.log('info', 'sending sms to phoneNumber: ' + phoneNumber, resp) + resolve(); + }, 1000) + }) + }); + }) + .then(function (allItems) { + winston.log('debug', 'all items', allItems) + callback(); + }) +} + +function sendSMS(phoneNumber, message, callback) { + var https = require('https'); + var data = JSON.stringify({ + api_key: process.env.API_KEY, + api_secret: process.env.API_SECRET, + to: phoneNumber, + from: process.env.NEXMO_NUMBER, + text: message + }); + + var options = { + host: 'rest.nexmo.com', + path: '/sms/json', + port: 443, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(data) + } + }; + + var req = https.request(options); + + req.write(data); + req.end(); + + var responseData = ''; + req.on('response', function (res) { + res.on('data', function (chunk) { + responseData += chunk; + }); + + res.on('end', function () { + callback(JSON.parse(responseData)) + }); + }); +} + +function reset() { + connectedUsers.length = 0 + converstationIDs.length = 0; + executed = false; + didStart = false +} \ No newline at end of file diff --git a/server/package.json b/server/package.json new file mode 100644 index 0000000..1ac0e1e --- /dev/null +++ b/server/package.json @@ -0,0 +1,21 @@ +{ + "name": "nexmoserver", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node index.js" + }, + "author": "", + "license": "ISC", + "dependencies": { + "bluebird": "^3.4.7", + "body-parser": "^1.16.1", + "dotenv": "^4.0.0", + "express": "^4.14.1", + "nexmo": "*", + "request": "^2.80.0", + "winston": "^2.3.1" + } +} diff --git a/server/private.key b/server/private.key new file mode 100644 index 0000000..88317d3 --- /dev/null +++ b/server/private.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCpxBBDKv+lrQ6I +U6Z3ZY0gBJc+IDdn1+lz+pBdkCz1Bpgt8UUkuFs5Dhun2+YAtLtSX5yoFKI/fI4U +oSg5tg2OzOdI9CTRGmMQZJWDAKavF79HVLZwUiLvmb+FcqEAPmaVKfpw/Ny8b/L2 +xyTTpjUqRygfb/XwYC1KnO2W50bQlmZOIB/JyQ69LQal1+5e7reX46A485MBuYSB +hiONUDYN2PeFKB6FD9bdb29A9Ffgjc2npvGJnJAoOp0oywOG1T6VyiSbMLv2jcoN +YXdOR3dEdCYtQhEhPaxgSmv9I2HbdX7qa1eYVQgIBWY1sI0PbQb5yEpHSVE6rgxt +3kwxdf7bAgMBAAECggEAPMQ39/rzsDd+TLD4lKNOfSf3hR7eBjckUsXdGAiL9xbp +sQQ8NjUhPg59Orv9G8KVjJo5xnZAT05Dw0GeVu4B81wH67asWQbDb/I5DD/W6jSr +XJqHhTP5Wl9jzR4nYCF18a2pffkc//xocQn+dh33yM7Yn0whse2TOLJsXwgg8ndW +H+PBywnU6ThLrCxYBtHFT8ZXhXp0YgnI6NpYyJAmUva24gfVK60fIj+q4mSWbS4L +Apl/dVd8ugg18c/kegOALoomv4Ef5gzXoU8Z53THgAIBUqWfjeWEAWP/707sX0IX +CbHkHW1bqs5IMyv26F8KCYk1dw4+s3FKWBm73IZK6QKBgQDRCy/txyI3G9Mkm/WD +ikRTwhhx3ifY67oAmQA7Ej9RNT34Pm0g6e+cqxQMAYqbTIJlDOYPssc6/y4cB2v+ +z7xK+f3+dIE4U1G1uyuWZ3A1nbG69jBL7CLlZ7rFG0dnAhp+MudjIN7lbJC2Vm5f +Q++GGs4HZEIGtVKG8pWMQcSfgwKBgQDP5kA9SR1pURhwhejENJvD6WWiJz5bfjsR +RbjIU66egBn8lsvC9cJ7c6VAB/nSTCf/kP9K2EIP/ocHdPRrDUbcKAZBzBaTm1Ne ++BbcnMG3Asx3Sd93Flnvx9zA22aBkRGkpEbceJ+9Uqw/+jCR8FUuaC1cYcS7B+Ax +FR3qxaNryQKBgQC6BTmGX1D8Nz+B8WLdxhmdazgzg9xztPjU9VH2PvunOTjHAk48 +HtCB2t9A+9Iq9j+Gyp6jU5Tk1BVlDdm1a4iEM8gpOeWPiN1KWZ+k/LEmlQiGdfNu +bFh1SgQyyF6WtJF8wbGUTbjr8dso1urqzVqFPK8gdsAmlg41VgNsR05AHwKBgQCb +71PXo/OPfugZ3hbMs7jxpSo6Cb1X2sJrHiSO/VVIRUU4k9KZlMQe3IriEBlthvmp +UxRpCvPiE0MscKizcu3nS44F5BI6/JH39ZUQO/OAVXUPNDWMmRM0KnCFpwqnvCvD +lgQgcck+s7fG9N1hWSZK/JCpbhulPS8HIZOi+EKvIQKBgCKt7uztn1MDK1rMLL0F +/Zt4RULdM3S0sSQEjpcu1WE2cBv+5H1c8kkgYas00Q55++lWYK+3c+4NEaxdOAfS +gJMGRk1s+KC4TsaJoyQWvWJSaKWHZO9EiplcKvriTwA8gXOA/X/H6WivUXwCWnIS +AQEQfWItj4IGwB1UlUHVde3e +-----END PRIVATE KEY----- diff --git a/server/private_local.key b/server/private_local.key new file mode 100644 index 0000000..ee23b08 --- /dev/null +++ b/server/private_local.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQClmVsQNh8ohjpa +PZRe0uO/D/thAai3Tk+npkex0IAldpGPRAl8A3iaO51VK6bxCeubOQg186qAGwuh +dNreHqcIKNTveds63hjae6/wFqykMp6fyyfAPtcPFBt/rWmwplqVd4t53fkF0mAH +BXBDJ86s7jSM76lmdRXU9DBPlDHh5ahJKgWOZxpNKNRUZrRDQriz9+zjzIbMHmRa +wcK/glt7ULXtfqViRz7y8S7L5HCqPijuwAKwcEix1KHTU5yc2ru79o3ozL13E4ar ++3IeL3k4pSxzApQsQvYwiOJYShNqcoEpmE9HuwP3iGrUIrh9wZaE79PYo1in5tBD +M6UEhPi1AgMBAAECggEAAMHljnqs5E7O3ARNlixxQbm1POGYz7PSPfJPACNzvpge +/NCDGZfbeJseslnHgvE4bGaToMmLRVg6UqP570ulGG61YJ4GUGvdIbywlpDH+91G +Z/UyKpku+yDsqkYVBmuQXQAluWo4PhxS951BAPGg6p4RS/a5p+APuEBgqzCClOWp +3mtpkF6WS6R5l/dWcAHt+ZnfGerPTQycYF1GrIcZMSh65+SMkxNg6MJpoS4PSE0A +ER6cTxkO/zWbPd43yw/xKWxCTO8Q5SXk2cXOAgBy3ad/Z1qUGziGJh7eKgn5HF7h +Ga2FZBwVWGeHNdl6QhZRymmIDHzPS3zs/ZExN4g/kQKBgQDfDPdr5/HMOuzYDKHW +VmsT6XSSV3nVTw+diwPN0fRibrN/BvArU7WEnUfYCXlqW+t+ctTBD1L3ykHENoNn +wLWaeuHIrg8gug4SzSN6vlQIUr1TX8Jz6E3nkeJu/1iGm1+GJJmYsmM85yYA5co7 +dYWqRx8KNEEH50VRISHG1LMLcQKBgQC+D8K7/5OyObdoJ1++q9W8rZtOxZ67AN6W +JvAZZHYV3iKauAywm/QrYLPlTlHcylhUjNkrPwMvy4P2MCuqQ+ySUSmA9GVdWzMl +J/H9F0ztttYTtEDM9DTLdCWsbz62k7n4n0KNDO5XaHWII/1wzoUMSKSfxxf8X+9y +VBa/49b3hQKBgHuU/JU9yrvEPiuqPBS8htFefOcELo2gI1+/gRkNZeEPwsXzuyfW +VnEmj4LEJRBn43+I8hYRfn+qAWxMY9wdasEvql99CQax71A9dpXwVDDUXu+N0/hS +Uq+mJZYoRg02kuOI7c0rsU4yJ91BdB4jjC5+/1SxBQLzAXXk7Ij0mksRAoGBALPT +xbSLBPvEkDwDxRtrCjgTKrdFVAIruG7pOJNN8kyOeL9bFOalKElCnfOAPwRgj2Kw +QWohnKpELP9qZGYdDmECWfqhQqcp+yJUwSluOmNQcw0Bp65EAQ/fPSYBu5yT+Ym7 +ZgR/D6O0OkAtjUaGoGwW72wdvBwVyUCrPzsgH+zhAoGAKjCp+mlmpizw0+IsZkbe +miPYuHQPCc+0vH9v7l/OBN3RBf24xDwwjHbfm/K7p8iPE0Qea5qy8Y9i2MbyB4G9 +kUml3Mn4Mk12py6GunK51huxZacGO/9yT8tzyWzQsAkMB+bDXvkRqyAL/0YTFNbB +bQM1voPCxjwSuY4azzyzGa0= +-----END PRIVATE KEY----- \ No newline at end of file