diff --git a/dist/maidr.js b/dist/maidr.js index 2fab5dd1..376dcdfb 100644 --- a/dist/maidr.js +++ b/dist/maidr.js @@ -654,6 +654,13 @@ class Constants { clientToken = null; + /** + * Mark and recall vars. Used to store the current mark and recall state of the chart. + * @type {Array} + * @default Array(10).fill(null) + */ + mark = Array(10).fill(null); + /** * Stops the autoplay if it is currently running. * @@ -2179,9 +2186,9 @@ class ChatLLM { img = await constants.ConvertSVGtoJPG(singleMaidr.id, 'gemini'); } if (constants.emailAuthKey) { - chatLLM.GeminiPromptAPI(text, img); + chatLLM.GeminiPromptRemote(text, img); } else { - chatLLM.GeminiPrompt(text, img); + chatLLM.GeminiPromptLocal(text, img); } } @@ -2304,10 +2311,10 @@ class ChatLLM { if (model == 'openai') { text = data.choices[0].message.content; - let i = this.requestJson.messages.length; - this.requestJson.messages[i] = {}; - this.requestJson.messages[i].role = 'assistant'; - this.requestJson.messages[i].content = text; + let i = this.requestJsonOpenAI.messages.length; + this.requestJsonOpenAI.messages[i] = {}; + this.requestJsonOpenAI.messages[i].role = 'assistant'; + this.requestJsonOpenAI.messages[i].content = text; if (data.error) { chatLLM.DisplayChatMessage(LLMName, 'Error processing request.', true); @@ -2318,6 +2325,12 @@ class ChatLLM { } else if (model == 'gemini') { if (data.text()) { text = data.text(); + if (this.requestJsonGemini.contents.length > 2) { + let i = this.requestJsonGemini.contents.length; + this.requestJsonGemini.contents[i] = {}; + this.requestJsonGemini.contents[i].role = 'model'; + this.requestJsonGemini.contents[i].content = text; + } chatLLM.DisplayChatMessage(LLMName, text); } else { if (!data.error) { @@ -2360,7 +2373,7 @@ class ChatLLM { */ fakeLLMResponseData() { let responseText = {}; - if (this.requestJson.messages.length > 2) { + if (this.requestJsonOpenAI.messages.length > 2) { // subsequent responses responseText = { id: 'chatcmpl-8Y44iRCRrohYbAqm8rfBbJqTUADC7', @@ -2567,32 +2580,32 @@ class ChatLLM { let backupMessage = 'Describe ' + singleMaidr.type + ' charts to a blind person'; // headers and sys message - if (!this.requestJson) { - this.requestJson = {}; - //this.requestJson.model = 'gpt-4-vision-preview'; - this.requestJson.model = 'gpt-4o-2024-11-20'; - this.requestJson.max_tokens = constants.LLMmaxResponseTokens; // note: if this is too short (tested with less than 200), the response gets cut off + if (!this.requestJsonOpenAI) { + this.requestJsonOpenAI = {}; + //this.requestJsonOpenAI.model = 'gpt-4-vision-preview'; + this.requestJsonOpenAI.model = 'gpt-4o-2024-11-20'; + this.requestJsonOpenAI.max_tokens = constants.LLMmaxResponseTokens; // note: if this is too short (tested with less than 200), the response gets cut off // sys message - this.requestJson.messages = []; - this.requestJson.messages[0] = {}; - this.requestJson.messages[0].role = 'system'; - this.requestJson.messages[0].content = sysMessage; + this.requestJsonOpenAI.messages = []; + this.requestJsonOpenAI.messages[0] = {}; + this.requestJsonOpenAI.messages[0].role = 'system'; + this.requestJsonOpenAI.messages[0].content = sysMessage; if (constants.LLMPreferences) { - this.requestJson.messages[1] = {}; - this.requestJson.messages[1].role = 'system'; - this.requestJson.messages[1].content = constants.LLMPreferences; + this.requestJsonOpenAI.messages[1] = {}; + this.requestJsonOpenAI.messages[1].role = 'system'; + this.requestJsonOpenAI.messages[1].content = constants.LLMPreferences; } } // user message // if we have an image (first time only), send the image and the text, otherwise just the text - let i = this.requestJson.messages.length; - this.requestJson.messages[i] = {}; - this.requestJson.messages[i].role = 'user'; + let i = this.requestJsonOpenAI.messages.length; + this.requestJsonOpenAI.messages[i] = {}; + this.requestJsonOpenAI.messages[i].role = 'user'; if (img) { // first message, include the img - this.requestJson.messages[i].content = [ + this.requestJsonOpenAI.messages[i].content = [ { type: 'text', text: text, @@ -2604,10 +2617,10 @@ class ChatLLM { ]; } else { // just the text - this.requestJson.messages[i].content = text; + this.requestJsonOpenAI.messages[i].content = text; } - return this.requestJson; + return this.requestJsonOpenAI; } GeminiJson(text, img = null) { @@ -2672,7 +2685,7 @@ class ChatLLM { return payload; } - async GeminiPromptAPI(text, imgBase64 = null) { + async GeminiPromptRemote(text, imgBase64 = null) { let url = constants.baseURL + 'gemini' + constants.code; // Create the prompt @@ -2689,7 +2702,20 @@ class ChatLLM { } constants.LLMImage = imgBase64; - let requestJson = chatLLM.GeminiJson(prompt, imgBase64); + if (!this.requestJsonGemini) { + // this is our first message, do the full construction + this.requestJsonGemini = chatLLM.GeminiJson(prompt, imgBase64); + } else { + // subsequent messages, just add the new user message + let i = this.requestJsonGemini.contents.length; + this.requestJsonGemini.contents[i] = {}; + this.requestJsonGemini.contents[i].role = 'user'; + this.requestJsonGemini.contents[i].parts = [ + { + text: text, + }, + ]; + } const response = await fetch(url, { method: 'POST', @@ -2697,7 +2723,7 @@ class ChatLLM { 'Content-Type': 'application/json', Authentication: constants.emailAuthKey + ' ' + constants.clientToken, }, - body: JSON.stringify(requestJson), + body: JSON.stringify(this.requestJsonGemini), }); if (response.ok) { const responseJson = await response.json(); @@ -2713,7 +2739,7 @@ class ChatLLM { } } - async GeminiPrompt(text, imgBase64 = null) { + async GeminiPromptLocal(text, imgBase64 = null) { // https://ai.google.dev/docs/gemini_api_overview#node.js try { // Save the image for next time @@ -2735,21 +2761,24 @@ class ChatLLM { }); // old model was 'gemini-pro-vision' // Create the prompt - let prompt = constants.LLMSystemMessage; - if (constants.LLMPreferences) { - prompt += constants.LLMPreferences; + if (!this.requestJsonGemini) { + // this is our first message, do the full construction + this.requestJsonGemini = chatLLM.GeminiJson(prompt, imgBase64); + } else { + // subsequent messages, just add the new user message + let i = this.requestJsonGemini.contents.length; + this.requestJsonGemini.contents[i] = {}; + this.requestJsonGemini.contents[i].role = 'user'; + this.requestJsonGemini.contents[i].parts = [ + { + text: text, + }, + ]; } - prompt += '\n\n' + text; // Use the text parameter as the prompt - const image = { - inlineData: { - data: imgBase64, // Use the base64 image string - mimeType: 'image/png', // Or the appropriate mime type of your image - }, - }; // Generate the content //console.log('LLM request: ', prompt, image); - const result = await model.generateContent([prompt, image]); + const result = await model.generateContent(this.requestJsonGemini); //console.log(result.response.text()); // Process the response @@ -2757,7 +2786,7 @@ class ChatLLM { } catch (error) { chatLLM.WaitingSound(false); chatLLM.DisplayChatMessage('Gemini', 'Error processing request.', true); - console.error('Error in GeminiPrompt:', error); + console.error('Error in GeminiPromptLocal:', error); throw error; // Rethrow the error for further handling if necessary } } @@ -2822,7 +2851,7 @@ class ChatLLM { document.getElementById('chatLLM_chat_history').innerHTML = ''; // reset the data - this.requestJson = null; + this.requestJsonOpenAI = null; this.firstTime = true; // and start over, if enabled, or window is open @@ -9085,8 +9114,10 @@ class Control { * @returns {void} */ async SetKeyControls() { + // home / end: first / last element + // not available in review mode constants.events.push([ - document, + [constants.chart, constants.brailleInput], 'keydown', function (e) { // ctrl/cmd: stop autoplay @@ -9134,6 +9165,44 @@ class Control { }, ]); + // mark and recall + // mark with M + # (0-9), recall with m + # (0-9) + // available in chart and braille, not review + let lastKeytime = 0; + let lastKey = null; + constants.events.push([ + [constants.chart, constants.brailleInput], + 'keydown', + function (e) { + // setup + const now = new Date().getTime(); + const key = e.key; + + // check for keypress within threshold + if (now - lastKeytime < constants.keypressInterval) { + // mark with M + if (lastKey == 'M' && /[0-9]/.test(key)) { + const markIndex = parseInt(key, 10); + constants.mark[markIndex] = JSON.parse(JSON.stringify(position)); // deep copy + display.announceText('Marked position ' + markIndex); + } + + // recall with m + if (lastKey == 'm' && /[0-9]/.test(key)) { + const recallIndex = parseInt(key, 10); + if (constants.mark[recallIndex]) { + position = constants.mark[recallIndex]; + control.UpdateAll(); + } + } + } + + // update last key and time + lastKey = key; + lastKeytime = now; + }, + ]); + // Init a few things let lastPlayed = ''; if ([].concat(singleMaidr.type).includes('bar')) { diff --git a/dist/maidr.min.js b/dist/maidr.min.js index 2c34bd58..e2f7a413 100644 --- a/dist/maidr.min.js +++ b/dist/maidr.min.js @@ -1 +1 @@ -class Constants{chart_container_id="chart-container";main_container_id="maidr-container";braille_container_id="braille-div";braille_input_id="braille-input";info_id="info";announcement_container_id="announcements";end_chime_id="end_chime";container_id="container";project_id="maidr";review_id_container="review_container";review_id="review";reviewSaveSpot;reviewSaveBrailleMode;chartId="";events=[];postLoadEvents=[];constructor(){}textMode="verbose";brailleMode="off";lockSelection=!1;sonifMode="on";reviewMode="off";minX=0;maxX=0;minY=0;maxY=0;plotId="";chartType="";navigation=1;plotOrientation="horz";MAX_FREQUENCY=1e3;MIN_FREQUENCY=200;NULL_FREQUENCY=100;combinedVolMin=.25;combinedVolMax=1.25;MAX_SPEED=500;MIN_SPEED=50;DEFAULT_SPEED=250;INTERVAL=20;AUTOPLAY_DURATION=2e3;vol=.5;MAX_VOL=30;autoPlayRate=this.DEFAULT_SPEED;colorSelected="#03C809";brailleDisplayLength=32;showRect=1;hasRect=1;hasSmooth=1;duration=.3;outlierDuration=.06;autoPlayOutlierRate=50;autoPlayPointsRate=50;colorUnselected="#595959";canTrack=1;visualBraille=!1;ariaMode="assertive";userSettingsKeys=["vol","autoPlayRate","brailleDisplayLength","colorSelected","MIN_FREQUENCY","MAX_FREQUENCY","AUTOPLAY_DURATION","ariaMode","openAIAuthKey","geminiAuthKey","claudeAuthKey","emailAuthKey","skillLevel","skillLevelOther","LLMModel","LLMPreferences","LLMOpenAiMulti","LLMGeminiMulti","LLMModels","autoInitLLM","clientToken"];openAIAuthKey=null;geminiAuthKey=null;LLMmaxResponseTokens=1e3;playLLMWaitingSound=!0;LLMDetail="high";LLMModel="openai";LLMModels={openai:!0};LLMSystemMessage="You are a helpful assistant describing the chart to a blind person. ";skillLevel="basic";skillLevelOther="";autoInitLLM=!0;verboseText="";waitingQueue=0;showDisplay=1;showDisplayInBraille=1;showDisplayInAutoplay=0;outlierInterval=null;isMac=navigator.userAgent.toLowerCase().includes("mac");control=this.isMac?"Cmd":"Ctrl";alt=this.isMac?"option":"Alt";home=this.isMac?"fn + Left arrow":"Home";end=this.isMac?"fn + Right arrow":"End";keypressInterval=2e3;tabMovement=null;debugLevel=3;canPlayEndChime=!1;manualData=!0;baseURL="https://maidr-service.azurewebsites.net/api/";code="?code=I8Aa2PlPspjQ8Hks0QzGyszP8_i2-XJ3bq7Xh8-ykEe4AzFuYn_QWA%3D%3D";clientToken=null;KillAutoplay(){clearInterval(this.autoplayId),this.autoplayId=null}KillSepPlay(){this.sepPlayId&&(clearInterval(this.sepPlayId),this.sepPlayId=null)}SpeedUp(){constants.autoPlayRate-this.INTERVAL>this.MIN_SPEED&&(constants.autoPlayRate-=this.INTERVAL)}SpeedDown(){constants.autoPlayRate+this.INTERVAL<=this.MAX_SPEED&&(constants.autoPlayRate+=this.INTERVAL)}SpeedReset(){constants.autoPlayRate=constants.DEFAULT_SPEED}ConvertHexToRGBString(t){return"rgb("+parseInt(t.slice(1,3),16)+","+parseInt(t.slice(3,5),16)+","+parseInt(t.slice(5,7),16)+")"}ConvertRGBStringToHex(t){let e=t.replace(/[^\d,]/g,"").split(",");return"#"+e[0].toString(16).padStart(2,"0")+e[1].toString(16).padStart(2,"0")+e[2].toString(16).padStart(2,"0")}ColorInvert(t){let e=t.replace(/[^\d,]/g,"").split(",");return"rgb("+(255-e[0])+","+(255-e[1])+","+(255-e[2])+")"}GetBetterColor(t){-1!==t.indexOf("#")&&(t=this.ConvertHexToRGBString(t));let e=this.ColorInvert(t),n=e.replace(/[^\d,]/g,"").split(",");return n[1]n[0]-10&&n[2]n[0]-10&&(n[0]>86||n[0]<169)&&(e=this.colorSelected),e}GetStyleArrayFromString(t){return t.replaceAll(" ","").split(/[:;]/)}GetStyleStringFromArray(t){let e="";for(let n=0;n{var s=document.createElement("canvas"),i=s.getContext("2d"),l=(new XMLSerializer).serializeToString(n);l.startsWith("\n \n \n `;CreateMenu(){document.querySelector("body").insertAdjacentHTML("beforeend",this.menuHtml);let t=document.querySelectorAll("#close_menu, #menu .close");for(let e=0;e{t.addEventListener("change",(()=>{const e=document.querySelectorAll('input[name="LLM_model"]:checked');e.length>2?(t.checked=!1,alert("You can select up to 2 AI models.")):0===e.length&&(t.checked=!0,alert("You must select at least one AI model."))}))}))}Destroy(){let t=document.getElementById("menu");t&&t.remove();let e=document.getElementById("menu_modal_backdrop");e&&e.remove()}Toggle(t=!1){void 0===t&&(t=!!document.getElementById("menu").classList.contains("hidden")),t&&document.getElementById("chatLLM")&&!document.getElementById("chatLLM").classList.contains("hidden")||(t?(this.whereWasMyFocus=document.activeElement,this.PopulateData(),constants.tabMovement=0,document.getElementById("menu").classList.remove("hidden"),document.getElementById("menu_modal_backdrop").classList.remove("hidden"),document.querySelector("#menu .close").focus()):(document.getElementById("menu").classList.add("hidden"),document.getElementById("menu_modal_backdrop").classList.add("hidden"),this.whereWasMyFocus.focus(),this.whereWasMyFocus=null))}PopulateData(){if(document.getElementById("vol").value=constants.vol,document.getElementById("braille_display_length").value=constants.brailleDisplayLength,document.getElementById("color_selected").value=constants.colorSelected,document.getElementById("min_freq").value=constants.MIN_FREQUENCY,document.getElementById("max_freq").value=constants.MAX_FREQUENCY,document.getElementById("AUTOPLAY_DURATION").value=constants.AUTOPLAY_DURATION,"string"==typeof constants.openAIAuthKey&&(document.getElementById("openai_auth_key").value=constants.openAIAuthKey),"string"==typeof constants.emailAuthKey&&(document.getElementById("email_auth_key").value=constants.emailAuthKey),"string"==typeof constants.geminiAuthKey&&(document.getElementById("gemini_auth_key").value=constants.geminiAuthKey),"string"==typeof constants.claudeAuthKey&&(document.getElementById("claude_auth_key").value=constants.claudeAuthKey),document.getElementById("skill_level").value=constants.skillLevel,constants.skillLevelOther&&(document.getElementById("skill_level_other").value=constants.skillLevelOther),"assertive"==constants.ariaMode?(document.getElementById("aria_mode_assertive").checked=!0,document.getElementById("aria_mode_polite").checked=!1):(document.getElementById("aria_mode_polite").checked=!0,document.getElementById("aria_mode_assertive").checked=!1),constants.emailAuthKey&&constants.clientToken){for(let t in constants.LLMModels)document.getElementById(`LLM_model_${t}`).checked=!0;this.DisableLLMAPIKeys()}else for(let t in constants.LLMModels)document.getElementById(`LLM_model_${t}`).checked=!0,document.getElementById(`${t}_auth_key_container`).classList.remove("hidden");document.getElementById("claude_auth_key_container").classList.add("hidden"),constants.emailAuthKey||(document.getElementById("delete_email_key").style="display: none"),"other"==constants.skillLevel&&document.getElementById("skill_level_other_container").classList.remove("hidden"),constants.LLMPreferences&&(document.getElementById("LLM_preferences").value=constants.LLMPreferences),document.getElementById("LLM_reset_notification")&&document.getElementById("LLM_reset_notification").remove()}SaveData(){let t=this.ShouldLLMReset();constants.vol=document.getElementById("vol").value,constants.brailleDisplayLength=document.getElementById("braille_display_length").value,constants.colorSelected=document.getElementById("color_selected").value,constants.MIN_FREQUENCY=document.getElementById("min_freq").value,constants.MAX_FREQUENCY=document.getElementById("max_freq").value,constants.AUTOPLAY_DURATION=document.getElementById("AUTOPLAY_DURATION").value,constants.openAIAuthKey=document.getElementById("openai_auth_key").value,constants.geminiAuthKey=document.getElementById("gemini_auth_key").value,constants.claudeAuthKey=document.getElementById("claude_auth_key").value,constants.emailAuthKey=document.getElementById("email_auth_key").value,constants.skillLevel=document.getElementById("skill_level").value,constants.skillLevelOther=document.getElementById("skill_level_other").value;document.querySelectorAll('input[name="LLM_model"]').forEach((t=>{t.checked?constants.LLMModels[t.value]=!0:delete constants.LLMModels[t.value]})),constants.LLMPreferences=document.getElementById("LLM_preferences").value,constants.LLMOpenAiMulti=document.getElementById("LLM_model_openai").checked,constants.LLMGeminiMulti=document.getElementById("LLM_model_gemini").checked,constants.LLMClaudeMulti=document.getElementById("LLM_model_claude").checked,constants.autoInitLLM=document.getElementById("init_llm_on_load").checked,document.getElementById("aria_mode_assertive").checked?constants.ariaMode="assertive":document.getElementById("aria_mode_polite").checked&&(constants.ariaMode="polite"),this.SaveDataToLocalStorage(),this.UpdateHtml(),t&&chatLLM&&chatLLM.ResetLLM()}DisableLLMAPIKeys(){document.getElementById("email_auth_key").disabled=!0,document.getElementById("verify").classList.add("hidden")}isEmailTriggered=!1;VerifyEmail(){if(console.log("verify email"),!this.isEmailTriggered){this.isEmailTriggered=!0;let t=document.getElementById("email_auth_key").value;if(t&&-1!==t.indexOf("@")){let e=constants.baseURL+"send_email"+constants.code,n={email:t};fetch(e,{method:"POST",headers:{"Content-Type":"application/json",Authentication:constants.emailAuthKey},body:JSON.stringify(n)}).then((t=>t.json())).then((e=>{if(e&&e.message&&e.client_token){alert(e.message),constants.clientToken=e.client_token,constants.emailAuthKey=t,this.DisableLLMAPIKeys();for(let t in constants.LLMModels)document.getElementById(`${t}_auth_key_container`).classList.add("hidden");this.SaveDataToLocalStorage(),this.isEmailTriggered=!1}else console.log(e),alert(e.data),this.isEmailTriggered=!1})).catch((t=>{console.log(t),alert(t.data),this.isEmailTriggered=!1}))}else alert("Please enter a valid email address."),this.isEmailTriggered=!1}}UpdateHtml(){constants.infoDiv.setAttribute("aria-live",constants.ariaMode),document.getElementById(constants.announcement_container_id).setAttribute("aria-live",constants.ariaMode),document.getElementById("init_llm_on_load").checked=constants.autoInitLLM;const t=document.getElementsByClassName("highlight_point"),e=document.getElementById("highlight_rect"),n=document.getElementById("highlight_point");if(null!==t&&t.length>0)for(let e=0;eNote: Changes in LLM settings will reset any existing conversation.

'),document.getElementById("save_and_close_menu").setAttribute("aria-labelledby","save_and_close_text LLM_reset_notification")}ShouldLLMReset(){let t=!1;t||constants.skillLevel==document.getElementById("skill_level").value||(t=!0),t||constants.LLMPreferences==document.getElementById("LLM_preferences").value||(t=!0),t||constants.LLMModel==document.getElementById("LLM_model").value||(t=!0);let e=document.querySelectorAll('input[name="LLM_model"]');for(let n=0;n