Untitled
unknown
plain_text
a month ago
29 kB
5
Indexable
Never
var config = require("../config.json"); const outputPath = "./test_data/output"; const inputPath = "./test_data/input"; const path = require("path"); const fs = require("fs"); const audioRepo = "./test_data/audio"; const fs1 = require("fs").promises; const helper = require("./helper"); const translate = require('./translate'); const similarCheck = require('./similarCheck'); const excel = require("exceljs"); const Log = require("./Logger.js"); const TAG = "Interpreter"; const result_table = require("./result_table.js"); const front_end_data_updater = require("./front_end_data_updater.js"); const execution_util = require("./execution_util.js"); // const final_report = require('./report'); const compare_util = require("./compare_util"); const ffplayPath = path.join(__dirname, '..', 'ffplay', 'bin'); const ffplayExecutable = 'ffplay'; const middleware = require('./middleware.js'); const commonVariable = require("./shared_fields.js"); const cmd_executor = require("./cmd_executor.js"); let currentDevice = config.testDeviceSerial; let supportDeviceLanguage = config.supportDeviceLanguage; let testDeviceLanguage = config.testDeviceLanguage; let translationOption = config.translationModelType; let intermediate_report_name = ""; let memory_report_name = ""; let jsonFile = []; let totalTestCases = 0, asrPassCases = 0, asrFailCases = 0, asrBlockedCases = 0, nmtHighCases = 0, nmtMidCases = 0, nmtLowCases = 0; let testOpenInterpreter, setInterpreterLang, getInterpreterText, startNstopRecording; const keyStrings = require("./keystrings.json"); const logmonitor = require("./logmonitor"); const nfr = require('./nfr.js'); function executeInterpreter() { logmonitor.resetFields(); helper.initializeTestFolders(); helper.initializeTempFolder(`${audioRepo}/Temp`); fs.readdir(audioRepo, (err, files) => { if (err) { Log.debug(TAG, err); return; } audioFiles = files; }); fetch('../config.json') .then(response => response.json()) .then(data => { config = data; currentDevice = config.testDeviceSerial; supportDeviceLanguage = config.supportDeviceLanguage; testDeviceLanguage = config.testDeviceLanguage; translationOption = config.translationModelType; testOpenInterpreter = [ "-s", currentDevice, "shell", "am", "instrument", "-w", "-e", "class", "com.samsung.test.uiau.BEAT#testOpenInterpreter", "com.samsung.test.uiau.test/android.support.test.runner.AndroidJUnitRunner", ]; setInterpreterLang = [ "-s", currentDevice, "shell", "am", "instrument", "-w", "-e", "class", "com.samsung.test.uiau.BEAT#setInterpreterLang", "-e", "beat", supportDeviceLanguage + "::" + testDeviceLanguage, "com.samsung.test.uiau.test/android.support.test.runner.AndroidJUnitRunner", ]; getInterpreterText = [ "-s", currentDevice, "shell", "am", "instrument", "-w", "-e", "class", "com.samsung.test.uiau.BEAT#getInterpreterText", "com.samsung.test.uiau.test/android.support.test.runner.AndroidJUnitRunner", ]; startNstopRecording = [ "-s", currentDevice, "shell", "input", "tap", "899 2055", ]; readInputFile(config.inputFileName).then((_) => { middleware.notifyUser('Reading input file'); readFile(path.join(inputPath, config.outputJSONName)).then((data) => { totalTestCases = data.length; helper .checkDevices() .then((value) => { if (value) { helper .execute_and_monitor_uiauto_process( "adb", testOpenInterpreter, false ) .then(() => { helper .execute_and_monitor_uiauto_process( "adb", setInterpreterLang, false ) .then(() => { let i = 0; intermediate_report_name = helper.getIntermediateReportName(); memory_report_name = helper.getMemoryReportName(); fs.writeFile(path.join(outputPath, intermediate_report_name), "TC_NO::Input::Word_Count::ASR::Compare_Analysis::Words_Failed::Words_Failed_Percent::Remarks::Translation::Translation_Expected::AutoDetect_Language::AutoDetect_Language_Score::Similarity_Score", (err) => { if (err) throw err; Log.debug(TAG, "Output file created"); }); fs.writeFile(path.join(outputPath, memory_report_name), "Input::Native Heap Size::Native Heap Allocation::Dalvik Heap Size::Dalvik Heap Allocation::ASR Latency::NMT Latency::Battery::Temperature\n", (err) => { if (err) throw err; Log.debug(TAG, "Output file created"); }); extractAndFetchMsg(data, i, 1); }); }); } else { Log.debug(TAG, "Check the devices connected"); } }) .catch((err) => { Log.debug(TAG, "Error while checking status of the device " + err); }); }); }); }).catch(e => { Log.debug(TAG, "Error in fetching config" + e) }) } function readInputFile(filePath) { return new Promise((resolve, reject) => { fs.readFile(filePath, "utf8", (err, data) => { if (err) { console.error("Error reading the file:", err); return; } // Split the data into lines const lines = data.split("\r\n").join("\n").split("\n"); let conversations = []; lines.forEach((line) => { // Check if line is not empty if (line.trim() !== "") { try { // Split line into parts based on the delimiter (#) var parts = line.trim().split("#"); var tcNo = ""; var testcase = ""; if (parts.length == 1) tcNo = parts[0]; if (parts.length > 1) { tcNo = parts[0]; testcase = parts[1]; } // const tcNo = parts[0]; // const testcase = parts[1]; const conversationObj = { TC_NO: tcNo, conversation: testcase, }; // Add conversation object to conversations array conversations.push(conversationObj); } catch (err) { Log.debug(TAG, "Error occurred while creating input JSON " + err); } } }); // Store conversations as JSON const jsonConversations = JSON.stringify(conversations, null, 2); // Write JSON data to a new file fs.writeFile( path.join(inputPath, config.outputJSONName), jsonConversations, "utf8", (err) => { if (err) { console.error("Error writing JSON file:", err); return; } Log.debug(TAG, "Conversations stored as JSON successfully in output.json" ); resolve("done"); } ); }); }); } function readFile(filename) { return new Promise((resolve, rejects) => { try { fs.readFile(filename, "utf8", (err, data) => { if (err) { Log.debug(TAG, "Error reading the file:" + err); } else { const jsonData = JSON.parse(data); resolve(jsonData); } }); } catch (error) { Log.debug(TAG, error); rejects(error); } }); } function writeExcel(excel_data) { return new Promise(function (resolve, reject) { const workbook = new excel.Workbook(); const summary_worksheet = workbook.addWorksheet("Result Summary"); helper.updateSummaryWorksheet(summary_worksheet, totalTestCases, asrPassCases, asrFailCases, asrBlockedCases, nmtHighCases, nmtMidCases, nmtLowCases).then(() => { const result_worksheet = workbook.addWorksheet("Detailed Results"); result_worksheet.columns = [ { header: "TC No.", key: "tcno", width: 10 }, { header: "Speech Input", key: "input", width: 50 }, { header: "Word Count", key: "wordCount", width: 30 }, { header: "ASR Output", key: "asr", width: 50 }, { header: "Compare Analysis (Words Highlight)", key: "wordsHighlight", width: 50 }, { header: "Words Failed (Count)", key: "wordsFailed", width: 30 }, { header: "Word Recognition Rate (%)", key: "wordRecognitionRate", width: 30 }, { header: "Compare Analysis (Remarks)", key: "remarks", width: 30 }, { header: "ITN Categories", key: "triageSummary", width: 30 }, { header: "NMT (Samsung)", key: "translation", width: 50 }, { header: "NMT (" + config.translationModelType + ")", key: "expectedTranslation", width: 50 }, { header: "Auto Detect Language (NMT (Samsung))", key: "autoDetectLanguage", width: 30 }, { header: "Auto Detect Language (NMT (Samsung)) Score (%)", key: "autoDetectLanguageScore", width: 30 }, { header: "NMT Similarity Score (%)", key: "similarityScore", width: 30 }, { header: "ASR Latency (MO) (ms)", key: "asr_latency", width: 30 }, { header: "NMT Latency (MO) (ms)", key: "nmt_latency", width: 30 }, { header: "Dalvik Memory Total (MO) (KB)", key: "dalvik_heap_size", width: 30 }, { header: "Dalvik Memory Alloc (MO) {KB}", key: "dalvik_heap_alloc", width: 30 }, { header: "Native Memory Total (MO) (KB)", key: "native_heap_size", width: 30 }, { header: "Native Memory Alloc (MO) (KB)", key: "native_heap_alloc", width: 30 }, { header: "Battery (MO) (%)", key: "battery", width: 30 }, { header: "Device Temperature (MO) (°C)", key: "thermal", width: 30 } ]; excel_data.forEach((entry) => { result_worksheet.addRow(entry); }) result_worksheet.eachRow(function (row, rowNum) { row.eachCell(function (cell, cellNum) { cell.alignment = { vertical: 'middle', horizontal: 'center', wrapText: true }; cell.border = { top: { style: 'thin' }, left: { style: 'thin' }, bottom: { style: 'thin' }, right: { style: 'thin' } }; if (rowNum == 1) { row.height = 35; cell.font = { bold: true }; cell.fill = { type: 'pattern', pattern: 'solid', fgColor: { argb: '90ee90' } }; } var cellText = []; const tagPattern = /<.*?>(.*?)<\/.*?>/g; const spanPattern = /<span.*?>(.*?)<\/span>/; const delPattern = /<del.*?>(.*?)<\/del>/; const insPattern = /<ins.*?>(.*?)<\/ins>/; if (tagPattern.test(cell.value)) { const tagList = cell.value.match(new RegExp(tagPattern)); tagList.forEach((tag) => { if (spanPattern.test(tag)) { const match = tag.match(new RegExp(spanPattern)); cellText.push({ text: match[1] }); } else if (delPattern.test(tag)) { const match = tag.match(new RegExp(delPattern)); cellText.push({ text: match[1], font: { strike: true, color: { argb: 'ff0055' } } }); } else if (insPattern.test(tag)) { const match = tag.match(new RegExp(insPattern)); cellText.push({ text: match[1], font: { color: { argb: '0000ff' } } }) } }) cell.value = { richText: cellText }; } }); }); let excel_report_name = `Report_${config.featureName}_${config.suiteName}_${config.testDeviceLanguage }_${config.supportDeviceLanguage}_${config.audioType }_${helper.getCurrentDateAndTime()}.xlsx`; workbook.xlsx .writeFile(path.join(outputPath, excel_report_name)) .then(() => { middleware.notifyUser(`Successfully created report: ${excel_report_name}`); Log.debug(TAG, `Successfully created report: ${excel_report_name}`); Log.debug(TAG, "Completed!!"); var cmd = `start .\\test_data\\output\\${excel_report_name}`; cmd_executor.execute(cmd); }) .catch((error) => { Log.debug(TAG, "Error occurred " + error); }); resolve("Done"); }); }); } let excelObj = []; function extractRemoteMessage(data, result, index, retryCount) { middleware.notifyUser('Fetching screen data'); let asrLat = ""; let nmtLat = ""; return new Promise((resolve, reject) => { let rs = ""; try { rs = result .split("INTERPRETER :")[1] .split("INSTRUMENTATION_STATUS_CODE:")[0] .replace("]", ""); } catch (error) { Log.debug(TAG, "Fetch text was empty " + error); rs = ""; } let excel_obj = {}; let nfrDetails = {}; let tc = index + 1; var asr = ""; var translation = ""; let wordCount = 0, wordsFailed = 0, wordRecognitionRate = 0, testCaseResult = 0, wordsHighlight = "", remarks = "", triageSummary = ""; var expectedTranslation = "", autoDetectLanguage = "", autoDetectLanguageScore = "", similarityScore = ""; if (rs.includes("::")) { if (rs.split("::").length > 1) { asr = rs.split("::")[1].trim(); translation = rs.split("::")[0].trim(); } if (rs.split("::").length == 1) { translation = rs.split("::")[0].trim(); } if (config.retry == "TRUE" && retryCount <= parseInt(config.retryCount) && (asr == "" || !helper.checkASR(data.conversation, asr))) { console.log("RETRY " + retryCount); resolve("RETRY"); } else { //for nfr info try { Log.debug(TAG, 'Getting latency info from this file '); logmonitor.stop_log_monitor().then((logfilePath) => { Log.debug(TAG, 'Getting latency info from this file ' + logfilePath); //ASR nfr.calculateLatency(logfilePath, keyStrings.asr_start_log, keyStrings.asr_end_log, "ASR").then((asrLatency) => { Log.debug(TAG, "ASR ----------- " + asrLatency); asrLat = asrLatency; nfr.calculateLatency(logfilePath, keyStrings.int_nmt_start_log, keyStrings.int_nmt_end_log, "NMT").then((lastOccurrence) => { Log.debug(TAG, "NMT ---------- " + lastOccurrence); nmtLat = lastOccurrence; nfr.getMemInfo(data.conversation, config.testDeviceLanguage).then((memObj) => { memObj.asr_latency = asrLat; memObj.nmt_latency = nmtLat; nfrDetails = { ...memObj }; Log.debug(TAG, "NFRrrrrrrrrrrrrrrrrrrrrrrrrrrr " + JSON.stringify(memObj)) let data_test = data.conversation + "::" + memObj.native_heap_size + "::" + memObj.native_heap_alloc + "::" + memObj.dalvik_heap_size + "::" + memObj.dalvik_heap_alloc + "::" + asrLat + "::" + nmtLat + "::" + memObj.battery + "::" + memObj.thermal + "\n"; Log.debug(TAG, "MEMORY INFO==========" + data_test); fs1.appendFile( path.join(outputPath, memory_report_name), data_test ); let input_formated = compare_util.get_formated_data(data.conversation); let asr_formated = compare_util.get_formated_data(asr); let [insertedCount, deletedCount, replacedCount, insertedArray, deletedArray, replacedArray, wordsHighlight, triageCategories] = compare_util.get_comparison(input_formated, asr_formated, helper.isCharBased(config.testDeviceLanguage)); wordCount = data.conversation.split(" ").length; wordsFailed = deletedCount + replacedCount; if (wordsFailed > 0) asrFailCases++; else asrPassCases++; wordRecognitionRate = (((wordCount - wordsFailed) / wordCount) * 100).toFixed(2); commonVariable.setWrr(wordRecognitionRate); testCaseResult = (wordsFailed + insertedCount) === 0 ? 1 : 0; commonVariable.setSrr(testCaseResult * 100); if (insertedArray.length > 0) remarks += "\nInserted -> " + insertedArray.join(" | "); if (deletedArray.length > 0) remarks += "\nDeleted -> " + deletedArray.join(" | "); if (replacedArray.length > 0) remarks += "\nReplaced -> " + replacedArray.join(" | "); remarks = remarks.trim(); if (remarks.length == 0) remarks = "NA"; triageSummary = triageCategories.join(" | "); translate.detect_language(translation).then((res) => { autoDetectLanguage = res.detected_lang; autoDetectLanguageScore = res.lang_score; let fromLang = config.testDeviceLanguage.substring(0, 2).toLowerCase(); let toLang = config.supportDeviceLanguage.substring(0, 2).toLowerCase(); /** * This function translates the given ASR text into the specified language using either the Native or Google translation API. * @param {string} asr - The ASR text to be translated. * @param {string} lang - The language code to translate the ASR text into. * @param {string} translationOption - The translation API to use, either "Native" or "Google". * @return {Promise} - A promise that resolves when the translation is complete, or rejects if there is an error. */ const translationPromise = new Promise((resolve, reject) => { if (translationOption == "Libre") { Log.debug(TAG, "Inside Libre Translate") translate.get_translate(asr, toLang).then((res) => { Log.debug(TAG, "Expected translation from AI Model : " + res); resolve(res); }).catch(reject); } else if (translationOption == "Google") { Log.debug(TAG, "Inside Google Translate"); let source_language, target_language; if(fromLang == "zh") { source_language = "zh-CN" } else { source_language = fromLang; } if(toLang == "zh"){ target_language = "zh-CN" } else { target_language = toLang; } translate.googleTranslate(asr,source_language,target_language).then((res) => { Log.debug(TAG, "Expected translation from AI Model : " + res.data); resolve(res.data); }).catch(reject); } else { reject(new Error("Invalid translation option")); } }); translationPromise.then((res) => { expectedTranslation = res; similarCheck.get_similarity(expectedTranslation, translation).then((res) => { if (res == "NA") { similarityScore = "NA"; } else { similarityScore = (res * 100).toFixed(2); if (similarityScore > 90) nmtHighCases++; else if (similarityScore > 30) nmtMidCases++; else nmtLowCases++; } excel_obj = { tcno: data.TC_NO, input: data.conversation, wordCount: wordCount, asr: asr, wordsHighlight: wordsHighlight, wordsFailed: wordsFailed, wordRecognitionRate: wordRecognitionRate, testCaseResult: testCaseResult, remarks: remarks, triageSummary: triageSummary, translation: translation, expectedTranslation: expectedTranslation, autoDetectLanguage: autoDetectLanguage, autoDetectLanguageScore: autoDetectLanguageScore, similarityScore: similarityScore, ...nfrDetails }; let intermediate_result = "\n" + data.TC_NO + "::" + data.conversation + "::" + wordCount + "::" + asr + "::" + wordsHighlight + "::" + wordsFailed + "::" + wordRecognitionRate + "::" + remarks + "::" + triageSummary + "::" + translation + "::" + expectedTranslation + "::" + autoDetectLanguage + "::" + autoDetectLanguageScore + "::" + similarityScore; Log.debug(TAG, "RESULT>>>>>>>>>>>>>>>>>>>>>>>> " + intermediate_result); fs1.appendFile( path.join(outputPath, intermediate_report_name), intermediate_result ); excel_obj.testType = 'Interpreter'; result_table.insert_new_row(); result_table.update_table_results(excel_obj); jsonFile.push(excel_obj); fs.writeFileSync(path.join(__dirname, '..', 'test_data', 'output', 'out1.json'), JSON.stringify(jsonFile, null, 2)); // excel_obj.totalCount = totalTestCases; front_end_data_updater.update_summary_table_gui(excel_obj, totalTestCases); front_end_data_updater.update_execution_count_gui(tc, totalTestCases); excelObj.push(excel_obj); Log.debug(TAG, "Final excel object is " + JSON.stringify(excelObj)); Log.debug(TAG, "All details " + JSON.stringify(nfr.getNFRValuesforUI())); middleware.updateNfr(nfr.getNFRValuesforUI()); resolve(excel_obj); }).catch((error) => { Log.debug(TAG, error); }); }).catch((error) => { Log.debug(TAG, error); }); }).catch((error) => { Log.debug(TAG, error); }); }); }) }).catch(error => { Log.debug(TAG, "Error occurred " + error); }); }); } catch (error) { Log.debug(TAG, "Error occurred while Getting latency " + error); } } } else { try { Log.debug(TAG, 'Getting latency info from this file '); logmonitor.stop_log_monitor().then((logfilePath) => { Log.debug(TAG, 'Getting latency info from this file ' + logfilePath); //ASR nfr.calculateLatency(logfilePath, keyStrings.asr_start_log, keyStrings.asr_end_log, "ASR").then((asrLatency) => { Log.debug(TAG, "ASR ----------- " + asrLatency); asrLat = asrLatency; nfr.calculateLatency(logfilePath, keyStrings.int_nmt_start_log, keyStrings.int_nmt_end_log, "NMT").then((lastOccurrence) => { Log.debug(TAG, "NMT ---------- " + lastOccurrence); nmtLat = lastOccurrence; nfr.getMemInfo(asr, config.testDeviceLanguage).then((memObj) => { memObj.asr_latency = asrLat; memObj.nmt_latency = nmtLat; nfrDetails = { ...memObj }; Log.debug(TAG, "NFRrrrrrrrrrrrrrrrrrrrrrrrrrrr " + JSON.stringify(memObj)) let data_test = data.conversation + "::" + memObj.native_heap_size + "::" + memObj.native_heap_alloc + "::" + memObj.dalvik_heap_size + "::" + memObj.dalvik_heap_alloc + "::" + asrLat + "::" + nmtLat + "::" + memObj.battery + "::" + memObj.thermal + "\n"; Log.debug(TAG, "MEMORY INFO==========" + data_test); fs1.appendFile( path.join(outputPath, memory_report_name), data_test ); excel_obj = { tcno: data.TC_NO, input: data.conversation, wordCount: wordCount, asr: asr, wordsHighlight: wordsHighlight, wordsFailed: wordsFailed, wordRecognitionRate: wordRecognitionRate, testCaseResult: testCaseResult, remarks: remarks, triageSummary: triageSummary, translation: translation, expectedTranslation: expectedTranslation, autoDetectLanguage: autoDetectLanguage, autoDetectLanguageScore: autoDetectLanguageScore, similarityScore: similarityScore, ...nfrDetails }; let intermediate_result = "\n" + data.TC_NO + "::" + data.conversation + "::" + wordCount + "::" + asr + "::" + wordsHighlight + "::" + wordsFailed + "::" + wordRecognitionRate + "::" + remarks + "::" + triageSummary + "::" + translation + "::" + expectedTranslation + "::" + autoDetectLanguage + "::" + autoDetectLanguageScore + "::" + similarityScore; Log.debug(TAG, "RESULT>>>>>>>>>>>>>>>>>>>>>>>> " + intermediate_result); fs1.appendFile( path.join(outputPath, intermediate_report_name), intermediate_result ); excelObj.push(excel_obj); Log.debug(TAG, "Final excel object is " + JSON.stringify(excelObj)); resolve(excel_obj); }); }) }).catch(error => { Log.debug(TAG, "Error occurred " + error); }); }); } catch (error) { Log.debug(TAG, "Error occurred while Getting latency " + error); } } }); } function extractAndFetchMsg(data, index, retryCount) { if (index >= data.length || execution_util.get_execution_tracker() == 'STOPPED') { writeExcel(excelObj); execution_util.stop_execution(config); middleware.notifyUser('Execution Completed...'); return Promise.resolve("done"); } return new Promise((resolve, reject) => { let text = data[index].conversation; helper .createAudioFile(text, audioRepo, config.testDeviceLanguage) .then((dataObj) => { if ("error" in dataObj) { Log.debug(TAG, dataObj.error); if (dataObj.error.includes("Text given is empty") || dataObj.error.includes("Audio file is corrupted")) { extractRemoteMessage(data[index], "", index, retryCount).then((result) => { // extractAndFetchMsg(data, index + 1,1).then(resolve); if (result == "RETRY") { extractAndFetchMsg( data, index, retryCount + 1 ).then(resolve); } else extractAndFetchMsg( data, index + 1, 1 ).then(resolve); }); } } else { let m4aDuration = 2000; let mainCmd = `${path.join(ffplayPath, ffplayExecutable)}`; if (!playAudioCmd.includes("-autoexit")) { mainCmd = "adb"; m4aDuration = dataObj.duration + 3000; } helper .execute_and_monitor_uiauto_process( "adb", testOpenInterpreter, false ) .then(() => { helper .execute_and_monitor_uiauto_process( "adb", ["-s", currentDevice, "shell", "input", "keyevent 85"], false ) .then(() => { helper .execute_and_monitor_uiauto_process( "adb", startNstopRecording, false ) .then(() => { // middleware.changeAudioImage(); middleware.notifyUser('Audio started playing'); middleware.displayGifInPlayer1(); Log.debug(TAG, "Duration is " + m4aDuration); helper .execute_and_monitor_uiauto_process( mainCmd, playAudioCmd, false ) .then(() => { // middleware.changeAudioImage(); middleware.notifyUser('Completed playing audio'); middleware.displayGifInPlayer1(); helper.sleep(m4aDuration).then(() => { helper .execute_and_monitor_uiauto_process( "adb", [ "-s", currentDevice, "shell", "input", "keyevent 85", ], false ) .then(() => { helper.sleep(2000).then(() => { helper .execute_and_monitor_uiauto_process( "adb", getInterpreterText, true ) .then((fetchData) => { extractRemoteMessage( data[index], fetchData, index, retryCount ).then((result) => { // console.log(JSON.stringify(result)); if (result == "RETRY") { extractAndFetchMsg( data, index, retryCount + 1 ).then(resolve); } else extractAndFetchMsg( data, index + 1, 1 ).then(resolve); }); }); }); }); }); }); }); }); }); } }); }); } module.exports = { executeInterpreter: executeInterpreter }
Leave a Comment