-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
5 changed files
with
185 additions
and
78 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| #!/bin/bash | ||
| . ./utils.lib | ||
|
|
||
| # NOTE: This file does the same as logextract.sh | ||
| # but was modified to parse just one file at a time. | ||
| # the file has to be passed through as the first argument | ||
| FILE_EXTENSION=".log" | ||
| DIRECTORY="mediciones" # this is the directory where the log files are in | ||
| file="$DIRECTORY/$1" # first argument: the file name in the folder named as $DIRECTORY | ||
| DIR="../GeneratedData" | ||
| TEMP_FILE_NAME="temp_$1" | ||
| echo "$TEMP_FILE_NAME" | ||
| # ALGORITHM | ||
| # read files | ||
| # create temp file to save grep result | ||
| # filter lines getting the line number | ||
|
|
||
| echo "$file" | ||
| echo "$DIR" | ||
|
|
||
| mkdir -p $DIR # create folder if does not exists | ||
|
|
||
|
|
||
| if [[ ${file: -${#FILE_EXTENSION}} == $FILE_EXTENSION ]]; # checks if file has the FILE_EXTENSION criteria | ||
| then | ||
| # get lines with specified lenght (min: 510 chars max: 511) | ||
| grep -anx '.\{510,511\}' $file > "$DIR/$TEMP_FILE_NAME" | ||
|
|
||
| # replace ':' with '-' | ||
| sed -ri 's/[:]+/-/g' "$DIR/$TEMP_FILE_NAME" | ||
|
|
||
| # read temp file and filter the desired GeneratedCSV | ||
| input="$DIR/$TEMP_FILE_NAME" | ||
|
|
||
| i=0 | ||
| while IFS= read -r line | ||
| do | ||
|
|
||
| i=$((i+1)) | ||
| delimeter='-' | ||
| SplitLineByDelimeter line delimeter split_result | ||
| line_number="${split_result[0]}" | ||
| data="${split_result[-1]}" # encoded data from the ekm metering | ||
|
|
||
| re='^[0-9a-zA-Z]+$' # regex to check if variable is alphanumeric | ||
| if [[ $data =~ $re ]] ; | ||
| then | ||
| date_line_number=$(($line_number - 2)) | ||
| # printf " line number: $line_number - 2 : $date_line_number\n" | ||
| # show that line | ||
| date_line=$(sed -n "${date_line_number}p" "$file") | ||
| date_line_clean=$(echo ${date_line%demos*}) # remove from word demos to the end of line | ||
| # add date to temp file | ||
| sed -i "${i}s/$/-${date_line_clean}/" "$input" | ||
|
|
||
| fi | ||
|
|
||
|
|
||
| done < "$input" | ||
|
|
||
|
|
||
| # create new file with temp info | ||
| delimeter='/' | ||
| SplitLineByDelimeter file delimeter split_result | ||
|
|
||
| new_file_name=$(echo "$DIR/${split_result[-1]}.filtered") | ||
| cp $input $new_file_name | ||
|
|
||
| echo "file: $file has $i rows of data" | ||
| fi | ||
|
|
||
| rm $input | ||
|
|
||
| exit 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,14 +1,47 @@ | ||
| const { exec } = require('child_process'); | ||
|
|
||
| const dir = exec("ls -la", function(err, stdout, stderr) { | ||
| if (err) { | ||
| // should have err.code here? | ||
| console.log('error ', err) | ||
| } | ||
| console.log(stdout); | ||
| }); | ||
|
|
||
| dir.on('exit', function (code) { | ||
| // exit code is code | ||
| console.log('exit ', code); | ||
| const { exec, execSync } = require('child_process'); | ||
| const fs = require('fs'); | ||
| const converter_to_csv = require('json-2-csv'); | ||
|
|
||
| const DATA_DIR = 'GeneratedData'; | ||
| const DIR_TO_PUT_CSV = 'GeneratedCSV'; | ||
| const DIR_TO_LOG_DATA = 'bash/mediciones'; | ||
| const helper = require('./helper.js'); | ||
|
|
||
|
|
||
| fs.readdirSync(DIR_TO_LOG_DATA).forEach( async (file) => { | ||
| console.log(file); | ||
| const command = exec("bash logextract_ver2.sh " + file, { cwd: './bash' }, function(err, stdout, stderr) { | ||
| if (err) | ||
| Promise.reject('error'); | ||
|
|
||
| console.log(stdout); | ||
| helper.processLineByLine(DATA_DIR + `/${file}` + '.filtered') | ||
| .then(decoded_data => { | ||
|
|
||
| // once we received the created csv we want to write a new file with it | ||
| converter_to_csv.json2csv(decoded_data, function (err, csv) { | ||
| if (err) throw err; | ||
| fs.writeFile(DIR_TO_PUT_CSV + '/' + file + '.csv', csv, 'utf8', (err) => | ||
| { | ||
| if (err) { | ||
| console.log('Some error occured - file either not saved or corrupted file saved.'); | ||
| console.log(err); | ||
| } else { | ||
| console.log('It\'s saved!'); | ||
| } | ||
| }); | ||
| }); | ||
|
|
||
| }); | ||
|
|
||
| }); | ||
|
|
||
| command.on('exit', async (code) => { | ||
| // exit code is code | ||
| // once finish the bash script to be processed | ||
| console.log(`exit with code on file: ${file}`, code); | ||
| }); | ||
| }); | ||
|
|
||
| return 0; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| const fs = require('fs'); | ||
| const readline = require('readline'); | ||
|
|
||
|
|
||
| function filter_data_in_line(line, array_to_fill) { | ||
|
|
||
| //Remove all the characters in even position in the string | ||
| splitted_data = line.split('-'); | ||
| data = splitted_data[1]; | ||
| data = data.replace(/.(.)/g, '$1'); | ||
| //Get the GeneratedCSV from the each position | ||
| var decoded = { | ||
| "name": data.substr(8 , 8), | ||
| "kwhtot":data.substr(16, 8), | ||
| "volt1": data.substr(23 + 73, 3) + "." + data.substr(26 + 73, 1), | ||
| "volt2": data.substr(27 + 73, 3) + "." + data.substr(30 + 73, 1), | ||
| "volt3": data.substr(31 + 73, 3) + "." + data.substr(34 + 73, 1), | ||
| "amp1": data.substr(35 + 73, 4) + "." + data.substr(39 + 73, 1), | ||
| "amp2": data.substr(40 + 73, 4) + "." + data.substr(44 + 73, 1), | ||
| "amp3": data.substr(45 + 73, 4) + "." + data.substr(49 + 73, 1), | ||
| "watts1":data.substr(50 + 73, 6) + "." + data.substr(56 + 73, 1), | ||
| "watts2":data.substr(57 + 73, 6) + "." + data.substr(63 + 73, 1), | ||
| "watts3":data.substr(64 + 73, 6) + "." + data.substr(70 + 73, 1), | ||
| "date": splitted_data[2] | ||
| }; | ||
| array_to_fill.push(decoded) | ||
| } | ||
|
|
||
| async function processLineByLine(file) { | ||
| const fileStream = fs.createReadStream(file); | ||
|
|
||
| // Note: we use the crlfDelay option to recognize all instances of CR LF | ||
| // ('\r\n') in input.txt as a single line break. | ||
| const rl = readline.createInterface({ | ||
| input: fileStream, | ||
| crlfDelay: Infinity | ||
| }); | ||
|
|
||
| let decoded_data = []; | ||
|
|
||
| for await (const line of rl) { | ||
| // Each line in input.txt will be successively available here as `line`. | ||
| console.log(`Line from file: ${line}`); | ||
| filter_data_in_line(line, decoded_data) | ||
| } | ||
|
|
||
| return decoded_data; | ||
| } | ||
|
|
||
| module.exports = { | ||
| processLineByLine | ||
| }; |