From 357b11d83af297fb6c3d3ff955ff1e08e7b63cf6 Mon Sep 17 00:00:00 2001 From: mkrupskis Date: Sat, 10 Feb 2024 22:28:41 -0500 Subject: [PATCH 01/12] try bluetooth connection --- .gitignore | 1 + app/android/app/capacitor.build.gradle | 1 + app/android/capacitor.settings.gradle | 3 + app/ios/App/App.xcodeproj/project.pbxproj | 6 ++ app/ios/App/App/Info.plist | 4 ++ app/ios/App/Podfile | 1 + app/ios/App/Podfile.lock | 8 ++- app/package-lock.json | 17 +++++ app/package.json | 1 + app/src/pages/index.tsx | 79 +++++++++++++++++++++-- app/src/pages/setup.tsx | 28 ++++++++ 11 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 app/src/pages/setup.tsx diff --git a/.gitignore b/.gitignore index 4a148a9..5c44b40 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ /node_modules /.pnp .pnp.js +venv # testing /coverage diff --git a/app/android/app/capacitor.build.gradle b/app/android/app/capacitor.build.gradle index ac381a7..f3cf20c 100644 --- a/app/android/app/capacitor.build.gradle +++ b/app/android/app/capacitor.build.gradle @@ -9,6 +9,7 @@ android { apply from: "../capacitor-cordova-android-plugins/cordova.variables.gradle" dependencies { + implementation project(':capacitor-community-bluetooth-le') implementation project(':capacitor-preferences') } diff --git a/app/android/capacitor.settings.gradle b/app/android/capacitor.settings.gradle index a1c665a..d935a8a 100644 --- a/app/android/capacitor.settings.gradle +++ b/app/android/capacitor.settings.gradle @@ -2,5 +2,8 @@ include ':capacitor-android' project(':capacitor-android').projectDir = new File('../node_modules/@capacitor/android/capacitor') +include ':capacitor-community-bluetooth-le' +project(':capacitor-community-bluetooth-le').projectDir = new File('../node_modules/@capacitor-community/bluetooth-le/android') + include ':capacitor-preferences' project(':capacitor-preferences').projectDir = new File('../node_modules/@capacitor/preferences/android') diff --git a/app/ios/App/App.xcodeproj/project.pbxproj b/app/ios/App/App.xcodeproj/project.pbxproj index fe47a99..69d2ee9 100644 --- a/app/ios/App/App.xcodeproj/project.pbxproj +++ b/app/ios/App/App.xcodeproj/project.pbxproj @@ -345,8 +345,10 @@ baseConfigurationReference = FC68EB0AF532CFC21C3344DD /* Pods-App.debug.xcconfig */; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = Q892SK2RNM; INFOPLIST_FILE = App/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; @@ -354,6 +356,7 @@ OTHER_SWIFT_FLAGS = "$(inherited) \"-D\" \"COCOAPODS\" \"-DDEBUG\""; PRODUCT_BUNDLE_IDENTIFIER = ai.adeus; PRODUCT_NAME = "$(TARGET_NAME)"; + PROVISIONING_PROFILE_SPECIFIER = ""; SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; @@ -365,14 +368,17 @@ baseConfigurationReference = AF51FD2D460BCFE21FA515B2 /* Pods-App.release.xcconfig */; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = Q892SK2RNM; INFOPLIST_FILE = App/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = ai.adeus; PRODUCT_NAME = "$(TARGET_NAME)"; + PROVISIONING_PROFILE_SPECIFIER = ""; SWIFT_ACTIVE_COMPILATION_CONDITIONS = ""; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; diff --git a/app/ios/App/App/Info.plist b/app/ios/App/App/Info.plist index 5f0ed1c..b578aff 100644 --- a/app/ios/App/App/Info.plist +++ b/app/ios/App/App/Info.plist @@ -45,5 +45,9 @@ UIViewControllerBasedStatusBarAppearance + NSBluetoothPeripheralUsageDescription + We require Bluetooth access to connect to nearby devices. + NSBluetoothAlwaysUsageDescription + We require Bluetooth access to connect to nearby devices. diff --git a/app/ios/App/Podfile b/app/ios/App/Podfile index e403f39..cf00fc5 100644 --- a/app/ios/App/Podfile +++ b/app/ios/App/Podfile @@ -11,6 +11,7 @@ install! 'cocoapods', :disable_input_output_paths => true def capacitor_pods pod 'Capacitor', :path => '../../node_modules/@capacitor/ios' pod 'CapacitorCordova', :path => '../../node_modules/@capacitor/ios' + pod 'CapacitorCommunityBluetoothLe', :path => '../../node_modules/@capacitor-community/bluetooth-le' pod 'CapacitorPreferences', :path => '../../node_modules/@capacitor/preferences' end diff --git a/app/ios/App/Podfile.lock b/app/ios/App/Podfile.lock index 6cb3b9a..28f0abe 100644 --- a/app/ios/App/Podfile.lock +++ b/app/ios/App/Podfile.lock @@ -1,18 +1,23 @@ PODS: - Capacitor (5.6.0): - CapacitorCordova + - CapacitorCommunityBluetoothLe (3.1.1): + - Capacitor - CapacitorCordova (5.6.0) - CapacitorPreferences (5.0.7): - Capacitor DEPENDENCIES: - "Capacitor (from `../../node_modules/@capacitor/ios`)" + - "CapacitorCommunityBluetoothLe (from `../../node_modules/@capacitor-community/bluetooth-le`)" - "CapacitorCordova (from `../../node_modules/@capacitor/ios`)" - "CapacitorPreferences (from `../../node_modules/@capacitor/preferences`)" EXTERNAL SOURCES: Capacitor: :path: "../../node_modules/@capacitor/ios" + CapacitorCommunityBluetoothLe: + :path: "../../node_modules/@capacitor-community/bluetooth-le" CapacitorCordova: :path: "../../node_modules/@capacitor/ios" CapacitorPreferences: @@ -20,9 +25,10 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: Capacitor: ebfc16cdb8116d04c101686b080342872da42d43 + CapacitorCommunityBluetoothLe: 86ca83c89199336039bad94f45f8363114ae1464 CapacitorCordova: 931b48fcdbc9bc985fc2f16cec9f77c794a27729 CapacitorPreferences: 77ac427e98db83bace772455f8ba447430382c4c -PODFILE CHECKSUM: 769e120bf4dfe4ef1095b83775e36bafeeeb3cdd +PODFILE CHECKSUM: da5221e2db218790239ebdc591ce1525f4e8ad73 COCOAPODS: 1.15.2 diff --git a/app/package-lock.json b/app/package-lock.json index 3837192..b51c098 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -8,6 +8,7 @@ "name": "app", "version": "0.1.0", "dependencies": { + "@capacitor-community/bluetooth-le": "^3.1.1", "@capacitor/android": "^5.6.0", "@capacitor/core": "^5.6.0", "@capacitor/ios": "^5.6.0", @@ -74,6 +75,17 @@ "node": ">=6.9.0" } }, + "node_modules/@capacitor-community/bluetooth-le": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@capacitor-community/bluetooth-le/-/bluetooth-le-3.1.1.tgz", + "integrity": "sha512-ymKt9aEhkMD1ER8976bZ4JVywcmSnLAUjaL6SZp+pytp02yNxzqhvyxCw2gLGEMqjEXC7urL/RtUN5ELf2oZfA==", + "dependencies": { + "@types/web-bluetooth": "^0.0.16" + }, + "peerDependencies": { + "@capacitor/core": "^5.0.0" + } + }, "node_modules/@capacitor/android": { "version": "5.6.0", "resolved": "https://registry.npmjs.org/@capacitor/android/-/android-5.6.0.tgz", @@ -1545,6 +1557,11 @@ "integrity": "sha512-+OpjSaq85gvlZAYINyzKpLeiFkSC4EsC6IIiT6v6TLSU5k5U83fHGj9Lel8oKEXM0HqgrMVCjXPDPVICtxF7EQ==", "dev": true }, + "node_modules/@types/web-bluetooth": { + "version": "0.0.16", + "resolved": "https://registry.npmjs.org/@types/web-bluetooth/-/web-bluetooth-0.0.16.tgz", + "integrity": "sha512-oh8q2Zc32S6gd/j50GowEjKLoOVOwHP/bWVjKJInBwQqdOYMdPrf1oVlelTlyfFK3CKxL1uahMDAr+vy8T7yMQ==" + }, "node_modules/@types/ws": { "version": "8.5.10", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.10.tgz", diff --git a/app/package.json b/app/package.json index 4f6d9d1..ec4afa8 100644 --- a/app/package.json +++ b/app/package.json @@ -12,6 +12,7 @@ "android": "npm run app && npx cap open android" }, "dependencies": { + "@capacitor-community/bluetooth-le": "^3.1.1", "@capacitor/android": "^5.6.0", "@capacitor/core": "^5.6.0", "@capacitor/ios": "^5.6.0", diff --git a/app/src/pages/index.tsx b/app/src/pages/index.tsx index cc1f5f6..231afd9 100644 --- a/app/src/pages/index.tsx +++ b/app/src/pages/index.tsx @@ -3,12 +3,35 @@ import React, { useState, useEffect } from "react"; import { useSupabase } from "@/utils/useSupabaseConfig"; import LoginForm from "@/components/LoginForm"; import Chat from "@/components/Chat"; +import { useRouter } from "next/router"; +import { + BleClient, + ScanResult, + numberToUUID, +} from "@capacitor-community/bluetooth-le"; +import { Button } from "@/components/ui/button"; + +const SERVICE_ID = "4fafc201-1fb5-459e-8fcc-c5c9c331914b"; +const CHARACTERISTIC_ID = "beb5483e-36e1-4688-b7f5-ea07361b26a8"; export default function Index() { + const [devices, setDevices] = useState([]); + + const connect = async (deviceId: string) => { + BleClient.connect(deviceId); + }; + + const router = useRouter(); const [loggedIn, setLoggedIn] = useState(false); + const [data, setData] = useState(null); + const [connected, setConnected] = useState(false); const { user, supabaseClient } = useSupabase(); + useEffect(() => { + scan(); + }, []); + useEffect(() => { if (user) { setLoggedIn(true); @@ -17,13 +40,61 @@ export default function Index() { } }, [user]); + async function scan(): Promise { + try { + await BleClient.initialize(); + + await BleClient.requestLEScan({}, (result: ScanResult) => { + setDevices((prev) => { + return [...prev, result]; + }); + console.log("received new scan result", result); + }); + + setTimeout(async () => { + await BleClient.stopLEScan(); + console.log("stopped scanning"); + }, 5000); + } catch (error) { + console.error(error); + } + } + return ( - <> - {loggedIn && user ? ( +
+

Index

+ {/* {loggedIn && user ? ( ) : ( - )} - + )} */} + {devices.map((device, index) => { + if (device.uuids?.includes("4fafc201-1fb5-459e-8fcc-c5c9c331914b")) { + return ( + + ); + } + })} + {connected &&

status: connected

} + {data &&

data: {data}

} +
); } diff --git a/app/src/pages/setup.tsx b/app/src/pages/setup.tsx new file mode 100644 index 0000000..b83240e --- /dev/null +++ b/app/src/pages/setup.tsx @@ -0,0 +1,28 @@ +import { BleClient, numberToUUID } from "@capacitor-community/bluetooth-le"; + +// const HEART_RATE_SERVICE = numberToUUID(0x180d); + +export default function Setup() { + async function scan(): Promise { + try { + await BleClient.initialize(); + + await BleClient.requestLEScan({}, (result) => { + console.log("received new scan result", result); + }); + + setTimeout(async () => { + await BleClient.stopLEScan(); + console.log("stopped scanning"); + }, 5000); + } catch (error) { + console.error(error); + } + } + scan(); + return ( +
+

Setup

+
+ ); +} From b22cf7bda6fdca299569798f4326240dc929d321 Mon Sep 17 00:00:00 2001 From: mkrupskis Date: Mon, 19 Feb 2024 17:35:02 -0500 Subject: [PATCH 02/12] Bluetooth work --- app/src/pages/index.tsx | 50 ++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/app/src/pages/index.tsx b/app/src/pages/index.tsx index 231afd9..67d435e 100644 --- a/app/src/pages/index.tsx +++ b/app/src/pages/index.tsx @@ -13,6 +13,9 @@ import { Button } from "@/components/ui/button"; const SERVICE_ID = "4fafc201-1fb5-459e-8fcc-c5c9c331914b"; const CHARACTERISTIC_ID = "beb5483e-36e1-4688-b7f5-ea07361b26a8"; +const SAMPLE_RATE = 44100; +const FRAMES_PER_BUFFER = 512; +const RECORD_SECONDS = 10; export default function Index() { const [devices, setDevices] = useState([]); @@ -60,6 +63,26 @@ export default function Index() { } } + async function sendAudioData(audioData: Uint8Array) { + const response = await fetch( + "https://bgkiorohiiofwtxnfvvo.supabase.co/functions/v1/process-audio", + { + method: "POST", + body: audioData.buffer, // Assuming audioData is a Uint8Array or similar binary data + headers: { + "Content-Type": "application/octet-stream", + }, + } + ); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const result = await response.json(); + console.log("Response from the backend:", result); + } + return (

Index

@@ -69,21 +92,26 @@ export default function Index() { )} */} {devices.map((device, index) => { - if (device.uuids?.includes("4fafc201-1fb5-459e-8fcc-c5c9c331914b")) { + if (device.device.name?.includes("ESP32")) { return ( + ); + } + })} + {connected &&

status: connected

} + {data &&

data: {data}

} + +
+ ); +} diff --git a/app/src/pages/setup.tsx b/app/src/pages/setup.tsx deleted file mode 100644 index b83240e..0000000 --- a/app/src/pages/setup.tsx +++ /dev/null @@ -1,28 +0,0 @@ -import { BleClient, numberToUUID } from "@capacitor-community/bluetooth-le"; - -// const HEART_RATE_SERVICE = numberToUUID(0x180d); - -export default function Setup() { - async function scan(): Promise { - try { - await BleClient.initialize(); - - await BleClient.requestLEScan({}, (result) => { - console.log("received new scan result", result); - }); - - setTimeout(async () => { - await BleClient.stopLEScan(); - console.log("stopped scanning"); - }, 5000); - } catch (error) { - console.error(error); - } - } - scan(); - return ( -
-

Setup

-
- ); -} diff --git a/supabase/functions/process-audio/index.ts b/supabase/functions/process-audio/index.ts index 10adabf..d81d126 100644 --- a/supabase/functions/process-audio/index.ts +++ b/supabase/functions/process-audio/index.ts @@ -1,6 +1,6 @@ import { serve } from "https://deno.land/std@0.170.0/http/server.ts"; import OpenAI, { toFile } from "https://deno.land/x/openai@v4.26.0/mod.ts"; - +import { decodeBase64 } from "https://deno.land/std@0.217.0/encoding/base64.ts"; import { corsHeaders } from "../common/cors.ts"; import { supabaseClient } from "../common/supabaseClient.ts"; @@ -49,13 +49,21 @@ const processAudio = async (req) => { apiKey: Deno.env.get("OPENAI_API_KEY"), }); + const body = await req.json(); + const { data } = body; + // console.log('json body', await req.json()); + console.log("**** Code version 0.0.2 ****"); + const audioData = decodeBase64(data); + // Read the binary data from the request body - const audioData = new Uint8Array(await req.arrayBuffer()); + // const audioData = new Uint8Array(await req.arrayBuffer()); + + // console.log("Audio data length:", audioData.length); // Create WAV header // Assuming 16000 Hz sample rate, 1 channel, and 32 bits per sample - const wavHeader = createWavHeader(audioData.length, 16000, 1, 32); + const wavHeader = createWavHeader(audioData.length, 8000, 1, 16); const wavFile = new Uint8Array(wavHeader.length + audioData.length); wavFile.set(wavHeader, 0); wavFile.set(audioData, wavHeader.length); From 925d6786ad99cb2dbd29d422045bb64e13aa9cc5 Mon Sep 17 00:00:00 2001 From: mkrupskis Date: Mon, 4 Mar 2024 14:39:58 -0800 Subject: [PATCH 04/12] Bluetooth --- app/src/pages/connect.tsx | 42 +++---- app/src/pages/index.tsx | 140 +++++++++++++++++++++- supabase/functions/common/cors.ts | 1 + supabase/functions/process-audio/index.ts | 5 +- 4 files changed, 156 insertions(+), 32 deletions(-) diff --git a/app/src/pages/connect.tsx b/app/src/pages/connect.tsx index 24b4474..5dc88d7 100644 --- a/app/src/pages/connect.tsx +++ b/app/src/pages/connect.tsx @@ -1,16 +1,10 @@ -import React, { useState, useEffect } from "react"; - -import { useSupabase, useSupabaseConfig } from "@/utils/useSupabaseConfig"; -import LoginForm from "@/components/LoginForm"; -import Chat from "@/components/Chat"; -import { CapacitorHttp, HttpResponse } from "@capacitor/core"; -import { useRouter } from "next/router"; -import { - BleClient, - ScanResult, - numberToUUID, -} from "@capacitor-community/bluetooth-le"; -import { Button } from "@/components/ui/button"; +import { useEffect, useState } from 'react'; + +import { Button } from '@/components/ui/button'; +import { useSupabase, useSupabaseConfig } from '@/utils/useSupabaseConfig'; +import { BleClient, ScanResult } from '@capacitor-community/bluetooth-le'; +import { CapacitorHttp, HttpResponse } from '@capacitor/core'; +import { useRouter } from 'next/router'; // const SERVICE_ID = "4fafc201-1fb5-459e-8fcc-c5c9c331914b"; // const CHARACTERISTIC_ID = "beb5483e-36e1-4688-b7f5-ea07361b26a8"; @@ -24,7 +18,7 @@ export default function Index() { const connect = async (deviceId: string) => { const device = await BleClient.requestDevice({ - services: ["4fafc201-1fb5-459e-8fcc-c5c9c331914b"], + services: ['4fafc201-1fb5-459e-8fcc-c5c9c331914b'], }); await BleClient.connect(device.deviceId); @@ -62,7 +56,7 @@ export default function Index() { setTimeout(async () => { await BleClient.stopLEScan(); - console.log("stopped scanning"); + console.log('stopped scanning'); }, 5000); } catch (error) { console.error(error); @@ -70,12 +64,12 @@ export default function Index() { } async function sendAudioData(audioData: Uint8Array) { - const data = Buffer.from(audioData).toString("base64"); + const data = Buffer.from(audioData).toString('base64'); const options = { - url: supabaseUrl + "/functions/v1/process-audio", + url: 'https://bgkiorohiiofwtxnfvvo.supabase.co/functions/v1/process-audio', headers: { - "Content-Type": "application/json", + 'Content-Type': 'application/json', }, data: { data: data }, }; @@ -87,7 +81,7 @@ export default function Index() { } const result = await response.data; - console.log("Response from the backend:", result); + console.log('Response from the backend:', result); } return ( @@ -99,21 +93,21 @@ export default function Index() { )} */} {devices.map((device, index) => { - if (device.device.name?.includes("ESP32")) { + if (device.device.name?.includes('ESP32')) { return ( ); diff --git a/app/src/pages/index.tsx b/app/src/pages/index.tsx index d7df913..2abfa15 100644 --- a/app/src/pages/index.tsx +++ b/app/src/pages/index.tsx @@ -1,13 +1,141 @@ -import Chat from '@/components/Chat'; -import LoginForm from '@/components/LoginForm'; -import { useSupabase } from '@/utils/useSupabaseConfig'; +import { useEffect, useState } from 'react'; + +import { Button } from '@/components/ui/button'; +import { useSupabase, useSupabaseConfig } from '@/utils/useSupabaseConfig'; +import { BleClient, ScanResult } from '@capacitor-community/bluetooth-le'; +import { CapacitorHttp, HttpResponse } from '@capacitor/core'; +import { useRouter } from 'next/router'; + +// const SERVICE_ID = "4fafc201-1fb5-459e-8fcc-c5c9c331914b"; +// const CHARACTERISTIC_ID = "beb5483e-36e1-4688-b7f5-ea07361b26a8"; +// const SAMPLE_RATE = 44100; +// const FRAMES_PER_BUFFER = 512; +// const RECORD_SECONDS = 10; export default function Index() { + const [devices, setDevices] = useState([]); + const { supabaseUrl, supabaseToken } = useSupabaseConfig(); + + const connect = async (deviceId: string) => { + const device = await BleClient.requestDevice({ + services: ['4fafc201-1fb5-459e-8fcc-c5c9c331914b'], + }); + + console.log('connecged'); + await BleClient.connect(device.deviceId); + + return device.deviceId; + }; + + const router = useRouter(); + const [loggedIn, setLoggedIn] = useState(false); + const [data, setData] = useState(null); + const [connected, setConnected] = useState(false); + const { user, supabaseClient } = useSupabase(); - if (!user || !supabaseClient) { - return ; + useEffect(() => { + scan(); + }, []); + + useEffect(() => { + if (user) { + setLoggedIn(true); + } else { + setLoggedIn(false); + } + }, [user]); + + async function scan(): Promise { + try { + await BleClient.initialize(); + + await BleClient.requestLEScan({}, (result: ScanResult) => { + setDevices((prev) => { + return [...prev, result]; + }); + }); + + setTimeout(async () => { + await BleClient.stopLEScan(); + console.log('stopped scanning'); + }, 5000); + } catch (error) { + console.error(error); + } } - return ; + async function sendAudioData(audioData: Uint8Array) { + const data = Buffer.from(audioData).toString('base64'); + console.log('Sending audio data to the backend:', data); + + const options = { + url: 'https://bgkiorohiiofwtxnfvvo.supabase.co/functions/v1/process-audio', + headers: { + 'Content-Type': 'application/json', + }, + data: { data: data }, + }; + + const response: HttpResponse = await CapacitorHttp.post(options); + + if (response.status !== 200) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const result = await response.data; + console.log('Response from the backend:', result); + } + + return ( +
+

Index

+ {/* {loggedIn && user ? ( + + ) : ( + + )} */} + {devices.map((device, index) => { + if (device.device.name?.includes('ESP32')) { + return ( + + ); + } + })} + {connected &&

status: connected

} + {data &&

data: {data}

} + +
+ ); } diff --git a/supabase/functions/common/cors.ts b/supabase/functions/common/cors.ts index 32a1cbe..88f49c9 100644 --- a/supabase/functions/common/cors.ts +++ b/supabase/functions/common/cors.ts @@ -3,3 +3,4 @@ export const corsHeaders = { "Access-Control-Allow-Headers": "authorization, x-client-info, apikey, content-type", "Access-Control-Max-Age": "3600", }; + \ No newline at end of file diff --git a/supabase/functions/process-audio/index.ts b/supabase/functions/process-audio/index.ts index e8ef43c..74fb0dc 100644 --- a/supabase/functions/process-audio/index.ts +++ b/supabase/functions/process-audio/index.ts @@ -1,8 +1,8 @@ import { serve } from "https://deno.land/std@0.170.0/http/server.ts"; import OpenAI, { toFile } from "https://deno.land/x/openai@v4.26.0/mod.ts"; import { decodeBase64 } from "https://deno.land/std@0.217.0/encoding/base64.ts"; -import { corsHeaders } from "../common/cors"; -import { supabaseClient } from "../common/supabaseClient"; +import { corsHeaders } from "../common/cors.ts"; +import { supabaseClient } from "../common/supabaseClient.ts"; import { multiParser } from 'https://deno.land/x/multiparser@0.114.0/mod.ts'; @@ -77,6 +77,7 @@ const processAudio = async (req: Request) => { const { data } = await req.json(); const audioData = decodeBase64(data); + console.log('Audio data:', audioData.length); // 1 Channel, 8000 sample rate, 16 bit depth const wavHeader = createWavHeader(audioData.length, 8000, 1, 16); const wavBytes = new Uint8Array(wavHeader.length + audioData.length); From 507395b7d2591e8296905ed736acb5ffd43b963a Mon Sep 17 00:00:00 2001 From: mkrupskis Date: Wed, 6 Mar 2024 14:07:54 -0800 Subject: [PATCH 05/12] bluetooth --- app/src/pages/connect.tsx | 8 ++++++-- app/src/pages/index.tsx | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/src/pages/connect.tsx b/app/src/pages/connect.tsx index 5dc88d7..94db8e9 100644 --- a/app/src/pages/connect.tsx +++ b/app/src/pages/connect.tsx @@ -66,8 +66,12 @@ export default function Index() { async function sendAudioData(audioData: Uint8Array) { const data = Buffer.from(audioData).toString('base64'); + if (!supabaseUrl) { + throw new Error('Supabase URL is not defined'); + } + const options = { - url: 'https://bgkiorohiiofwtxnfvvo.supabase.co/functions/v1/process-audio', + url: supabaseUrl + '/functions/v1/process-audio', headers: { 'Content-Type': 'application/json', }, @@ -99,7 +103,7 @@ export default function Index() { onClick={async () => { const deviceId = await connect(device.device.deviceId); - let bufferSize = 100000; + let bufferSize = 500000; let buffer = new Uint8Array(bufferSize); let count = 0; diff --git a/app/src/pages/index.tsx b/app/src/pages/index.tsx index 2abfa15..cc6cd29 100644 --- a/app/src/pages/index.tsx +++ b/app/src/pages/index.tsx @@ -70,7 +70,7 @@ export default function Index() { console.log('Sending audio data to the backend:', data); const options = { - url: 'https://bgkiorohiiofwtxnfvvo.supabase.co/functions/v1/process-audio', + url: '', headers: { 'Content-Type': 'application/json', }, From ef7542c28c2165e014c2ad19d5b8b749c1ca073b Mon Sep 17 00:00:00 2001 From: mkrupskis Date: Wed, 6 Mar 2024 14:08:56 -0800 Subject: [PATCH 06/12] update --- app/src/pages/index.tsx | 142 ++-------------------------- app/src/utils/useSupabaseConfig.tsx | 13 ++- 2 files changed, 17 insertions(+), 138 deletions(-) diff --git a/app/src/pages/index.tsx b/app/src/pages/index.tsx index cc6cd29..70b5bf7 100644 --- a/app/src/pages/index.tsx +++ b/app/src/pages/index.tsx @@ -1,141 +1,17 @@ -import { useEffect, useState } from 'react'; - -import { Button } from '@/components/ui/button'; -import { useSupabase, useSupabaseConfig } from '@/utils/useSupabaseConfig'; -import { BleClient, ScanResult } from '@capacitor-community/bluetooth-le'; -import { CapacitorHttp, HttpResponse } from '@capacitor/core'; -import { useRouter } from 'next/router'; - -// const SERVICE_ID = "4fafc201-1fb5-459e-8fcc-c5c9c331914b"; -// const CHARACTERISTIC_ID = "beb5483e-36e1-4688-b7f5-ea07361b26a8"; -// const SAMPLE_RATE = 44100; -// const FRAMES_PER_BUFFER = 512; -// const RECORD_SECONDS = 10; +import Chat from '@/components/Chat'; +import LoginForm from '@/components/LoginForm'; +import { useSupabase } from '@/utils/useSupabaseConfig'; export default function Index() { - const [devices, setDevices] = useState([]); - const { supabaseUrl, supabaseToken } = useSupabaseConfig(); - - const connect = async (deviceId: string) => { - const device = await BleClient.requestDevice({ - services: ['4fafc201-1fb5-459e-8fcc-c5c9c331914b'], - }); - - console.log('connecged'); - await BleClient.connect(device.deviceId); - - return device.deviceId; - }; - - const router = useRouter(); - const [loggedIn, setLoggedIn] = useState(false); - const [data, setData] = useState(null); - const [connected, setConnected] = useState(false); - - const { user, supabaseClient } = useSupabase(); - - useEffect(() => { - scan(); - }, []); - - useEffect(() => { - if (user) { - setLoggedIn(true); - } else { - setLoggedIn(false); - } - }, [user]); - - async function scan(): Promise { - try { - await BleClient.initialize(); + const { user, supabaseClient, loading } = useSupabase(); - await BleClient.requestLEScan({}, (result: ScanResult) => { - setDevices((prev) => { - return [...prev, result]; - }); - }); - - setTimeout(async () => { - await BleClient.stopLEScan(); - console.log('stopped scanning'); - }, 5000); - } catch (error) { - console.error(error); - } + if (loading) { + return null; } - async function sendAudioData(audioData: Uint8Array) { - const data = Buffer.from(audioData).toString('base64'); - console.log('Sending audio data to the backend:', data); - - const options = { - url: '', - headers: { - 'Content-Type': 'application/json', - }, - data: { data: data }, - }; - - const response: HttpResponse = await CapacitorHttp.post(options); - - if (response.status !== 200) { - throw new Error(`HTTP error! status: ${response.status}`); - } - - const result = await response.data; - console.log('Response from the backend:', result); + if (!user || !supabaseClient) { + return ; } - return ( -
-

Index

- {/* {loggedIn && user ? ( - - ) : ( - - )} */} - {devices.map((device, index) => { - if (device.device.name?.includes('ESP32')) { - return ( - - ); - } - })} - {connected &&

status: connected

} - {data &&

data: {data}

} - -
- ); + return ; } diff --git a/app/src/utils/useSupabaseConfig.tsx b/app/src/utils/useSupabaseConfig.tsx index a3c3ef3..3e4d1b1 100644 --- a/app/src/utils/useSupabaseConfig.tsx +++ b/app/src/utils/useSupabaseConfig.tsx @@ -5,6 +5,7 @@ import { useEffect, useState } from 'react'; export function useSupabaseConfig() { const [supabaseUrl, setSupabaseUrl] = useState(''); const [supabaseToken, setSupabaseToken] = useState(''); + const [loading, setLoading] = useState(true); useEffect(() => { async function fetchConfig() { @@ -23,6 +24,8 @@ export function useSupabaseConfig() { } catch (error) { // Handle any error that might occur during fetching console.error(error); + } finally { + setLoading(false); } } @@ -50,11 +53,11 @@ export function useSupabaseConfig() { } }; - return { supabaseUrl, supabaseToken, setSupabaseConfig }; + return { supabaseUrl, supabaseToken, setSupabaseConfig, loading }; } export function useSupabaseClient() { - const { supabaseUrl, supabaseToken } = useSupabaseConfig(); + const { supabaseUrl, supabaseToken, loading } = useSupabaseConfig(); const [supabaseClient, setSupabaseClient] = useState(); useEffect(() => { @@ -64,11 +67,11 @@ export function useSupabaseClient() { } }, [supabaseUrl, supabaseToken]); - return supabaseClient; + return { supabaseClient, loading }; } export function useSupabase() { - const supabaseClient = useSupabaseClient(); + const { supabaseClient, loading } = useSupabaseClient(); const [user, setUser] = useState(null); useEffect(() => { @@ -85,5 +88,5 @@ export function useSupabase() { }; }, [supabaseClient]); - return { user, supabaseClient }; + return { user, supabaseClient, loading }; } From f4f4395ab847b89b10834e4324f1bcfdfb26475d Mon Sep 17 00:00:00 2001 From: mkrupskis Date: Wed, 6 Mar 2024 14:14:50 -0800 Subject: [PATCH 07/12] Bluetooth Button --- app/src/components/Chat.tsx | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/src/components/Chat.tsx b/app/src/components/Chat.tsx index f35fdcf..e45eee5 100644 --- a/app/src/components/Chat.tsx +++ b/app/src/components/Chat.tsx @@ -1,5 +1,7 @@ import { SupabaseClient } from '@supabase/supabase-js'; import { useMutation, useQuery } from '@tanstack/react-query'; +import { Bluetooth } from 'lucide-react'; +import { useRouter } from 'next/router'; import { useEffect, useRef, useState } from 'react'; import { toast } from 'sonner'; import ChatLog, { Message } from './ChatLog'; @@ -9,6 +11,7 @@ import NewConversationButton from './NewConversationButton'; import PromptForm from './PromptForm'; import SideMenu from './SideMenu'; import { ThemeToggle } from './ThemeToggle'; +import { Button } from './ui/button'; export default function Chat({ supabaseClient, @@ -17,6 +20,7 @@ export default function Chat({ }) { const bottomRef = useRef(null); const textareaRef = useRef(null); + const router = useRouter(); const [entryData, setEntryData] = useState(''); const [messages, setMessages] = useState([]); @@ -156,6 +160,13 @@ export default function Chat({
+ { newConversation.mutate(); From 6942cc21780a5da4cc1a71a4f76df9ba0265304f Mon Sep 17 00:00:00 2001 From: mkrupskis Date: Wed, 6 Mar 2024 15:40:27 -0800 Subject: [PATCH 08/12] NavBar fixes --- app/src/components/Chat.tsx | 12 ++-- app/src/pages/connect.tsx | 106 ++++++++++++++++++++---------------- 2 files changed, 65 insertions(+), 53 deletions(-) diff --git a/app/src/components/Chat.tsx b/app/src/components/Chat.tsx index e45eee5..3450bb4 100644 --- a/app/src/components/Chat.tsx +++ b/app/src/components/Chat.tsx @@ -159,7 +159,11 @@ export default function Chat({
- + { + newConversation.mutate(); + }} + /> - { - newConversation.mutate(); - }} - /> +
diff --git a/app/src/pages/connect.tsx b/app/src/pages/connect.tsx index 94db8e9..7d6ca98 100644 --- a/app/src/pages/connect.tsx +++ b/app/src/pages/connect.tsx @@ -1,9 +1,13 @@ import { useEffect, useState } from 'react'; +import LogoutButton from '@/components/LogoutButton'; +import { NavMenu } from '@/components/NavMenu'; +import { ThemeToggle } from '@/components/ThemeToggle'; import { Button } from '@/components/ui/button'; import { useSupabase, useSupabaseConfig } from '@/utils/useSupabaseConfig'; import { BleClient, ScanResult } from '@capacitor-community/bluetooth-le'; import { CapacitorHttp, HttpResponse } from '@capacitor/core'; +import { Files } from 'lucide-react'; import { useRouter } from 'next/router'; // const SERVICE_ID = "4fafc201-1fb5-459e-8fcc-c5c9c331914b"; @@ -89,54 +93,62 @@ export default function Index() { } return ( -
-

Index

- {/* {loggedIn && user ? ( - - ) : ( - - )} */} - {devices.map((device, index) => { - if (device.device.name?.includes('ESP32')) { - return ( - + + + +
+
+ {devices.map((device, index) => { + if (device.device.name?.includes('ESP32')) { + return ( + - ); - } - })} - {connected &&

status: connected

} - {data &&

data: {data}

} - -
+ ); + + await sendAudioData(buffer); + }} + key={index} + > +

Device: {device.device.name + ''}

+

UUID: {device.uuids}

+ + ); + } + })} + + + ); } From 8140c3b268e779bcd3b7329cdf00118fdf87788d Mon Sep 17 00:00:00 2001 From: mkrupskis Date: Fri, 8 Mar 2024 16:37:18 -0800 Subject: [PATCH 09/12] try nrf --- app/src/pages/connect.tsx | 85 +++++++++++++---------- supabase/functions/process-audio/index.ts | 2 +- 2 files changed, 51 insertions(+), 36 deletions(-) diff --git a/app/src/pages/connect.tsx b/app/src/pages/connect.tsx index 7d6ca98..39c38f3 100644 --- a/app/src/pages/connect.tsx +++ b/app/src/pages/connect.tsx @@ -54,7 +54,11 @@ export default function Index() { await BleClient.requestLEScan({}, (result: ScanResult) => { setDevices((prev) => { - return [...prev, result]; + if (result.localName === 'ADeus') { + return [...prev, result]; + } else { + return prev; + } }); }); @@ -110,42 +114,53 @@ export default function Index() {
{devices.map((device, index) => { - if (device.device.name?.includes('ESP32')) { - return ( - - ); - } + } + ); + + // await sendAudioData(buffer); + }} + key={index} + > +

Device: {device.device.name + ''}

+

UUID: {device.uuids}

+ + ); })}
diff --git a/supabase/functions/process-audio/index.ts b/supabase/functions/process-audio/index.ts index 55acf65..d23f401 100644 --- a/supabase/functions/process-audio/index.ts +++ b/supabase/functions/process-audio/index.ts @@ -78,7 +78,7 @@ const processAudio = async (req: Request) => { console.log('Audio data:', audioData.length); // 1 Channel, 8000 sample rate, 16 bit depth - const wavHeader = createWavHeader(audioData.length, 8000, 1, 16); + const wavHeader = createWavHeader(audioData.length, 16000, 1, 16); const wavBytes = new Uint8Array(wavHeader.length + audioData.length); wavBytes.set(wavHeader, 0); wavBytes.set(audioData, wavHeader.length); From befb4cb246a77f82923132c7a31c5177e1142861 Mon Sep 17 00:00:00 2001 From: mkrupskis Date: Mon, 18 Mar 2024 22:31:10 -0700 Subject: [PATCH 10/12] small changes --- app/src/components/Chat.tsx | 7 +++- app/src/pages/connect.tsx | 47 +++++++++++++---------- scripts/wav_over_serial/main.py | 0 scripts/wav_over_serial/requirements.txt | 1 + supabase/functions/process-audio/index.ts | 1 + 5 files changed, 33 insertions(+), 23 deletions(-) create mode 100644 scripts/wav_over_serial/main.py create mode 100644 scripts/wav_over_serial/requirements.txt diff --git a/app/src/components/Chat.tsx b/app/src/components/Chat.tsx index e60d55f..3b894e0 100644 --- a/app/src/components/Chat.tsx +++ b/app/src/components/Chat.tsx @@ -1,6 +1,7 @@ import { SupabaseClient } from '@supabase/supabase-js'; import { useMutation, useQuery } from '@tanstack/react-query'; import { Bluetooth } from 'lucide-react'; +import Link from 'next/link'; import { useRouter } from 'next/router'; import { useEffect, useRef, useState } from 'react'; import { toast } from 'sonner'; @@ -164,11 +165,13 @@ export default function Chat({ }} /> diff --git a/app/src/pages/connect.tsx b/app/src/pages/connect.tsx index 39c38f3..f6ab479 100644 --- a/app/src/pages/connect.tsx +++ b/app/src/pages/connect.tsx @@ -8,6 +8,7 @@ import { useSupabase, useSupabaseConfig } from '@/utils/useSupabaseConfig'; import { BleClient, ScanResult } from '@capacitor-community/bluetooth-le'; import { CapacitorHttp, HttpResponse } from '@capacitor/core'; import { Files } from 'lucide-react'; +import Link from 'next/link'; import { useRouter } from 'next/router'; // const SERVICE_ID = "4fafc201-1fb5-459e-8fcc-c5c9c331914b"; @@ -21,12 +22,12 @@ export default function Index() { const { supabaseUrl, supabaseToken } = useSupabaseConfig(); const connect = async (deviceId: string) => { - const device = await BleClient.requestDevice({ - services: ['4fafc201-1fb5-459e-8fcc-c5c9c331914b'], - }); - await BleClient.connect(device.deviceId); + // const device = await BleClient.requestDevice({ + // services: ['4fafc201-1fb5-459e-8fcc-c5c9c331914b'], + // }); + await BleClient.connect(deviceId); - return device.deviceId; + return deviceId; }; const router = useRouter(); @@ -54,7 +55,7 @@ export default function Index() { await BleClient.requestLEScan({}, (result: ScanResult) => { setDevices((prev) => { - if (result.localName === 'ADeus') { + if (result.device.name === 'ADeus') { return [...prev, result]; } else { return prev; @@ -73,6 +74,7 @@ export default function Index() { async function sendAudioData(audioData: Uint8Array) { const data = Buffer.from(audioData).toString('base64'); + console.log('base64', data); if (!supabaseUrl) { throw new Error('Supabase URL is not defined'); @@ -96,20 +98,26 @@ export default function Index() { console.log('Response from the backend:', result); } + if (!supabaseClient) { + return
Supabase client not found
; + } + return ( <>
- +
@@ -117,43 +125,40 @@ export default function Index() { return (