diff --git a/.gitignore b/.gitignore index f49cdfb..b1b270f 100644 --- a/.gitignore +++ b/.gitignore @@ -108,3 +108,5 @@ package-lock.json demo.js !test/fixtures/files/package-lock.json + +local-test \ No newline at end of file diff --git a/commands/check_process_status.js b/commands/check_process_status.js index ee963d2..1a20f0e 100644 --- a/commands/check_process_status.js +++ b/commands/check_process_status.js @@ -45,6 +45,7 @@ async function checkInstalled(nodeExe, cwd) { async function checkStatus() { const nodeExe = await getNodeExe(pid); status.nodeVersion = await execute(`${nodeExe} -v`); + status.alinodeVersion = await execute(`${nodeExe} -V`); const processCwd = await getCwd(pid); await checkInstalled(nodeExe, processCwd); diff --git a/commands/get_os_info.js b/commands/get_os_info.js index 70b2cf8..ade77d0 100644 --- a/commands/get_os_info.js +++ b/commands/get_os_info.js @@ -10,6 +10,9 @@ async function getOsInfo() { // node version results.nodeVersion = process.versions.node; + // alinode version + results.alinodeVersion = process.versions.alinode; + // xtransit version results.xtransitVersion = require('../package.json').version; diff --git a/commands/upload_file.js b/commands/upload_file.js index 6596407..957b947 100644 --- a/commands/upload_file.js +++ b/commands/upload_file.js @@ -4,7 +4,7 @@ const fs = require('fs'); const path = require('path'); const zlib = require('zlib'); const qs = require('querystring'); -const formstream = require('formstream'); +const FormData = require('form-data'); const urllib = require('urllib'); const { promisify } = require('util'); const exists = promisify(fs.exists); @@ -60,11 +60,7 @@ async function removeExists(files) { } } -async function uploadFile() { - if (!utils.isNumber(fileId) || !fileType || !filePath || !server || !token) { - throw new Error('wrong args: node upload_file.js fileId fileType filePath server token'); - } - +async function checkFile(filePath) { if (!await exists(filePath)) { throw new Error(`file ${filePath} not exists`); } @@ -72,12 +68,32 @@ async function uploadFile() { if (!(await stat(filePath)).size) { throw new Error(`file ${filePath} is empty`); } +} + +async function uploadFile() { + if (!utils.isNumber(fileId) || !fileType || !filePath || !server || !token) { + throw new Error('wrong args: node upload_file.js fileId fileType filePath server token'); + } + + let files = []; + if (fileType === 'core') { + // core_path::node_executable_path + files = filePath.split('::'); + } else { + files = [filePath]; + } + + const removeFiles = [files[0]]; // create form - const gzippedFile = await gzipFile(filePath); - const form = formstream(); - const size = (await stat(gzippedFile)).size; - form.file('file', gzippedFile, gzippedFile, size); + const formdata = new FormData(); + for (const filePath of files) { + await checkFile(filePath); + const gzippedFile = await gzipFile(filePath); + formdata.append(path.basename(filePath), fs.createReadStream(gzippedFile)); + + removeFiles.push(gzippedFile); + } // create url const nonce = '' + (1 + parseInt((Math.random() * 100000000000), 10)); @@ -89,13 +105,14 @@ async function uploadFile() { dataType: 'json', type: 'POST', timeout: process.env.XTRANSIT_EXPIRED_TIME || 20 * 60 * 1000, - headers: form.headers(), - stream: form, + headers: formdata.getHeaders(), + stream: formdata, }; const result = await request(url, opts); if (result.storage && cleanAfterUpload === 'YES') { - await removeExists([filePath, gzippedFile]); + await removeExists(removeFiles); + result.removeFiles = removeFiles; } console.log(JSON.stringify(result)); } diff --git a/lib/agent.js b/lib/agent.js index cd9a171..3180297 100644 --- a/lib/agent.js +++ b/lib/agent.js @@ -37,6 +37,8 @@ class XtransitAgent extends EventEmitter { this.disks = Array.from(new Set(config.disks || [])); this.errors = Array.from(new Set(config.errors || [])); this.packages = Array.from(new Set(config.packages || [])); + this.coredirs = Array.from(new Set(config.coredirs || [])); + this.coreprefix = Array.from(new Set(config.coreprefix || [])); this.titles = Array.from(new Set(config.titles || [])); // global var diff --git a/lib/handler.js b/lib/handler.js index 3c22781..648c7d8 100644 --- a/lib/handler.js +++ b/lib/handler.js @@ -94,6 +94,6 @@ module.exports = async function(message) { return this.sendMessage('response', { ok: true, data: { stdout, stderr } }, traceId); } } catch (err) /* istanbul ignore next */ { - this.logger.error(`handle message failed: ${err}, raw message: ${message}`); + this.logger.error(`handle message failed: ${err.stack}, raw message: ${message}`); } }; diff --git a/orders/report_core.js b/orders/report_core.js new file mode 100644 index 0000000..c5c29c2 --- /dev/null +++ b/orders/report_core.js @@ -0,0 +1,190 @@ +/* istanbul ignore file */ +'use strict'; + +const os = require('os'); +const fs = require('fs'); +const cp = require('child_process'); +const path = require('path'); +const { promisify } = require('util'); +const getNodeExe = require('../common/exe'); +const exec = promisify(cp.exec); +const exists = promisify(fs.exists); +const readFile = promisify(fs.readFile); +const readdir = promisify(fs.readdir); +const access = promisify(fs.access); +const stat = promisify(fs.stat); +const realpath = promisify(fs.realpath); + +const CORE_PATTERN = '/proc/sys/kernel/core_pattern'; +const MAX_CORE_FILES = 500; +const REPORT_INTERVAL = 60; + +// system +const isLinux = os.platform() === 'linux'; +const isWindows = os.platform() === 'win32'; +const isMacOS = os.platform() === 'darwin'; + +let logger; +let coredirs = isMacOS ? ['/cores'] : []; +let coreprefix = ['core']; + +async function getNodePwd(pid) { + let pwd; + try { + // linux + if (isLinux) { + const { stdout } = await exec(`cat /proc/${pid}/environ`); + pwd = stdout.trim() + .split('\u0000') + .map(env => env.includes('PWD') && env.split('=')[1]) + .filter(pwd => pwd); + pwd = pwd[0]; + } + + // macos + if (isMacOS) { + const { stdout } = await exec(`lsof -a -d cwd -p ${pid} | grep cwd`); + pwd = stdout.split(' ').pop().trim(); + } + + // not support windows + if (isWindows) { + pwd = null; + } + } catch (err) { + logger.error(`getNodePwd failed: ${err}`); + } + + return pwd; +} + +async function getNodePwds(titles) { + let pwds = []; + + try { + const nodeExe = await getNodeExe(process.pid); + const file = path.join(__dirname, '../commands/get_node_processes.js'); + const cmd = `${nodeExe} ${JSON.stringify(file)}`; + const { stdout } = await exec(cmd, { + encoding: 'utf8', + stdio: 'ignore', + env: Object.assign({ + XTRANSIT_TITLES: JSON.stringify(titles), + }, process.env), + }); + + pwds = await Promise.all( + stdout + .split('\n') + .filter(proc => proc) + .map(proc => proc.split('\u0000')[0]) + .filter(pid => pid) + .map(pid => getNodePwd(pid)) + ); + + pwds = Array.from(new Set(pwds.filter(pwd => pwd))); + } catch (err) { + logger.error(`getNodePwds failed: ${err}`); + } + + return pwds; +} + +async function findCoreFile(coredir) { + const corefiles = []; + + try { + const files = await readdir(coredir); + for (const file of files) { + if (coreprefix.every(prefix => !file.startsWith(prefix) || file.endsWith('.gz'))) { + continue; + } + + // check core file stat + const corefile = path.join(coredir, file); + const filestat = await stat(corefile); + + if (!filestat.isFile()) { + continue; + } + + if (filestat.ctimeMs < Date.now() - REPORT_INTERVAL * 1000) { + continue; + } + + corefiles.push(corefile); + } + } catch (err) { + logger.error(`findCoreFile failed: ${err}`); + } + + return corefiles; +} + +async function findCoreFiles() { + const tasks = coredirs.map(coredir => findCoreFile(coredir)); + const nodeExe = await realpath(await getNodeExe(process.pid)); + const fileList = await Promise.all(tasks); + + // get corefiles + let corefiles = fileList + .reduce((list, files) => list.concat(files), []) + .map(corefile => ({ + core_path: corefile, + executable_path: nodeExe, + node_version: process.versions.node, + alinode_version: process.versions.alinode, + })); + if (corefiles.length > MAX_CORE_FILES) { + corefiles = corefiles.slice(0, MAX_CORE_FILES); + } + + return corefiles; +} + +exports = module.exports = async function() { + const message = { type: 'core_files', data: { list: [] } }; + + // concat coredirs + const pwds = await getNodePwds(this.titles); + coredirs = Array.from(new Set(coredirs.concat(pwds))); + + logger.debug(`[report_core] coredirs: ${coredirs.join(', ')}`); + + // find core files + message.data.list = await findCoreFiles(); + + return message; +}; + +exports.init = async function() { + logger = this.logger; + + // 1. from user config + coredirs = this.coredirs.concat(coredirs); + coreprefix = this.coreprefix.concat(coreprefix); + + // 2. from core_pattern + if (!isLinux || !await exists(CORE_PATTERN)) { + return; + } + let patt = await readFile(CORE_PATTERN, 'utf8'); + patt = patt.trim().split(' ')[0]; + if (patt.includes('%')) { + const coredir = path.parse(patt).dir; + if (await exists(coredir)) { + try { + await access(coredir, fs.R_OK); + coredirs.push(coredir); + const prefix = path.parse(patt).name.split('%')[0]; + if (!coreprefix.includes(prefix)) { + coreprefix.push(prefix); + } + } catch (e) { + logger.error(coredir + ' is unaccessible: ' + e.message); + } + } + } +}; + +exports.interval = process.env.UNIT_TEST_TRANSIT_LOG_INTERVAL || REPORT_INTERVAL; diff --git a/orders/system_log.js b/orders/system_log.js index 9a7489f..8d72f9a 100644 --- a/orders/system_log.js +++ b/orders/system_log.js @@ -439,7 +439,7 @@ async function getFreeMemory() { err; free = await linuxFreeMemroy(); } finally { - free = os.freemem(); + free = free || os.freemem(); } } else if (isLinux) { free = await linuxFreeMemroy(); diff --git a/package.json b/package.json index 49bd2a5..0e29f5d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "xtransit", - "version": "1.2.5", + "version": "1.3.0-beta", "description": "The agent that can collect or transfer xprofiler's performance logs.", "main": "xtransit.js", "bin": { @@ -41,7 +41,7 @@ "homepage": "https://github.com/X-Profiler/xtransit#readme", "dependencies": { "address": "^1.1.2", - "formstream": "^1.1.0", + "form-data": "^4.0.0", "moment": "^2.28.0", "nounou": "^1.2.1", "p-map": "^4.0.0",