diff --git a/.gitignore b/.gitignore index 5869894..6dd5598 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ /node_modules /.pnp .pnp.js +venv # testing /coverage diff --git a/app/android/app/capacitor.build.gradle b/app/android/app/capacitor.build.gradle index ac381a7..f3cf20c 100644 --- a/app/android/app/capacitor.build.gradle +++ b/app/android/app/capacitor.build.gradle @@ -9,6 +9,7 @@ android { apply from: "../capacitor-cordova-android-plugins/cordova.variables.gradle" dependencies { + implementation project(':capacitor-community-bluetooth-le') implementation project(':capacitor-preferences') } diff --git a/app/android/capacitor.settings.gradle b/app/android/capacitor.settings.gradle index a1c665a..d935a8a 100644 --- a/app/android/capacitor.settings.gradle +++ b/app/android/capacitor.settings.gradle @@ -2,5 +2,8 @@ include ':capacitor-android' project(':capacitor-android').projectDir = new File('../node_modules/@capacitor/android/capacitor') +include ':capacitor-community-bluetooth-le' +project(':capacitor-community-bluetooth-le').projectDir = new File('../node_modules/@capacitor-community/bluetooth-le/android') + include ':capacitor-preferences' project(':capacitor-preferences').projectDir = new File('../node_modules/@capacitor/preferences/android') diff --git a/app/capacitor.config.ts b/app/capacitor.config.ts index d084b43..bfc76a3 100644 --- a/app/capacitor.config.ts +++ b/app/capacitor.config.ts @@ -5,7 +5,12 @@ const config: CapacitorConfig = { appName: 'adeus', webDir: 'out', server: { - androidScheme: 'https', + androidScheme: 'https' + }, + plugins: { + CapacitorHttp: { + enabled: true, + }, }, }; diff --git a/app/ios/App/App.xcodeproj/project.pbxproj b/app/ios/App/App.xcodeproj/project.pbxproj index af318f5..b372be5 100644 --- a/app/ios/App/App.xcodeproj/project.pbxproj +++ b/app/ios/App/App.xcodeproj/project.pbxproj @@ -345,6 +345,7 @@ baseConfigurationReference = FC68EB0AF532CFC21C3344DD /* Pods-App.debug.xcconfig */; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_TEAM = 658U36Q86D; @@ -353,8 +354,9 @@ LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; MARKETING_VERSION = 1.0; OTHER_SWIFT_FLAGS = "$(inherited) \"-D\" \"COCOAPODS\" \"-DDEBUG\""; - PRODUCT_BUNDLE_IDENTIFIER = ai.adeus; + PRODUCT_BUNDLE_IDENTIFIER = app.mkrupskis; PRODUCT_NAME = "$(TARGET_NAME)"; + PROVISIONING_PROFILE_SPECIFIER = ""; SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; @@ -366,15 +368,17 @@ baseConfigurationReference = AF51FD2D460BCFE21FA515B2 /* Pods-App.release.xcconfig */; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = 658U36Q86D; + DEVELOPMENT_TEAM = Q892SK2RNM; INFOPLIST_FILE = App/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; MARKETING_VERSION = 1.0; - PRODUCT_BUNDLE_IDENTIFIER = ai.adeus; + PRODUCT_BUNDLE_IDENTIFIER = app.mkrupskis; PRODUCT_NAME = "$(TARGET_NAME)"; + PROVISIONING_PROFILE_SPECIFIER = ""; SWIFT_ACTIVE_COMPILATION_CONDITIONS = ""; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; diff --git a/app/ios/App/App/Info.plist b/app/ios/App/App/Info.plist index 5f0ed1c..7d9805a 100644 --- a/app/ios/App/App/Info.plist +++ b/app/ios/App/App/Info.plist @@ -5,7 +5,7 @@ CFBundleDevelopmentRegion en CFBundleDisplayName - adeus + adeus CFBundleExecutable $(EXECUTABLE_NAME) CFBundleIdentifier @@ -22,6 +22,10 @@ $(CURRENT_PROJECT_VERSION) LSRequiresIPhoneOS + NSBluetoothAlwaysUsageDescription + We require Bluetooth access to connect to nearby devices. + NSBluetoothPeripheralUsageDescription + We require Bluetooth access to connect to nearby devices. UILaunchStoryboardName LaunchScreen UIMainStoryboardFile diff --git a/app/ios/App/Podfile b/app/ios/App/Podfile index e403f39..cf00fc5 100644 --- a/app/ios/App/Podfile +++ b/app/ios/App/Podfile @@ -11,6 +11,7 @@ install! 'cocoapods', :disable_input_output_paths => true def capacitor_pods pod 'Capacitor', :path => '../../node_modules/@capacitor/ios' pod 'CapacitorCordova', :path => '../../node_modules/@capacitor/ios' + pod 'CapacitorCommunityBluetoothLe', :path => '../../node_modules/@capacitor-community/bluetooth-le' pod 'CapacitorPreferences', :path => '../../node_modules/@capacitor/preferences' end diff --git a/app/ios/App/Podfile.lock b/app/ios/App/Podfile.lock index 6cb3b9a..28f0abe 100644 --- a/app/ios/App/Podfile.lock +++ b/app/ios/App/Podfile.lock @@ -1,18 +1,23 @@ PODS: - Capacitor (5.6.0): - CapacitorCordova + - CapacitorCommunityBluetoothLe (3.1.1): + - Capacitor - CapacitorCordova (5.6.0) - CapacitorPreferences (5.0.7): - Capacitor DEPENDENCIES: - "Capacitor (from `../../node_modules/@capacitor/ios`)" + - "CapacitorCommunityBluetoothLe (from `../../node_modules/@capacitor-community/bluetooth-le`)" - "CapacitorCordova (from `../../node_modules/@capacitor/ios`)" - "CapacitorPreferences (from `../../node_modules/@capacitor/preferences`)" EXTERNAL SOURCES: Capacitor: :path: "../../node_modules/@capacitor/ios" + CapacitorCommunityBluetoothLe: + :path: "../../node_modules/@capacitor-community/bluetooth-le" CapacitorCordova: :path: "../../node_modules/@capacitor/ios" CapacitorPreferences: @@ -20,9 +25,10 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: Capacitor: ebfc16cdb8116d04c101686b080342872da42d43 + CapacitorCommunityBluetoothLe: 86ca83c89199336039bad94f45f8363114ae1464 CapacitorCordova: 931b48fcdbc9bc985fc2f16cec9f77c794a27729 CapacitorPreferences: 77ac427e98db83bace772455f8ba447430382c4c -PODFILE CHECKSUM: 769e120bf4dfe4ef1095b83775e36bafeeeb3cdd +PODFILE CHECKSUM: da5221e2db218790239ebdc591ce1525f4e8ad73 COCOAPODS: 1.15.2 diff --git a/app/package-lock.json b/app/package-lock.json index 3f6733a..b87651a 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -8,6 +8,7 @@ "name": "app", "version": "0.1.0", "dependencies": { + "@capacitor-community/bluetooth-le": "^3.1.1", "@capacitor/android": "^5.6.0", "@capacitor/core": "^5.6.0", "@capacitor/ios": "^5.6.0", @@ -88,6 +89,17 @@ "node": ">=6.9.0" } }, + "node_modules/@capacitor-community/bluetooth-le": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@capacitor-community/bluetooth-le/-/bluetooth-le-3.1.1.tgz", + "integrity": "sha512-ymKt9aEhkMD1ER8976bZ4JVywcmSnLAUjaL6SZp+pytp02yNxzqhvyxCw2gLGEMqjEXC7urL/RtUN5ELf2oZfA==", + "dependencies": { + "@types/web-bluetooth": "^0.0.16" + }, + "peerDependencies": { + "@capacitor/core": "^5.0.0" + } + }, "node_modules/@capacitor/android": { "version": "5.6.0", "resolved": "https://registry.npmjs.org/@capacitor/android/-/android-5.6.0.tgz", @@ -1674,6 +1686,11 @@ "integrity": "sha512-+OpjSaq85gvlZAYINyzKpLeiFkSC4EsC6IIiT6v6TLSU5k5U83fHGj9Lel8oKEXM0HqgrMVCjXPDPVICtxF7EQ==", "dev": true }, + "node_modules/@types/web-bluetooth": { + "version": "0.0.16", + "resolved": "https://registry.npmjs.org/@types/web-bluetooth/-/web-bluetooth-0.0.16.tgz", + "integrity": "sha512-oh8q2Zc32S6gd/j50GowEjKLoOVOwHP/bWVjKJInBwQqdOYMdPrf1oVlelTlyfFK3CKxL1uahMDAr+vy8T7yMQ==" + }, "node_modules/@types/ws": { "version": "8.5.10", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.10.tgz", diff --git a/app/package.json b/app/package.json index 9b9a4d4..fae502c 100644 --- a/app/package.json +++ b/app/package.json @@ -17,6 +17,7 @@ "android": "npm run app && npx cap open android" }, "dependencies": { + "@capacitor-community/bluetooth-le": "^3.1.1", "@capacitor/android": "^5.6.0", "@capacitor/core": "^5.6.0", "@capacitor/ios": "^5.6.0", diff --git a/app/src/components/Chat.tsx b/app/src/components/Chat.tsx index f49df13..8dd1606 100644 --- a/app/src/components/Chat.tsx +++ b/app/src/components/Chat.tsx @@ -1,5 +1,8 @@ import { SupabaseClient } from '@supabase/supabase-js'; import { useMutation, useQuery } from '@tanstack/react-query'; +import { Bluetooth } from 'lucide-react'; +import Link from 'next/link'; +import { useRouter } from 'next/router'; import { useEffect, useRef, useState } from 'react'; import { toast } from 'sonner'; import { useSupabaseConfig } from '../utils/useSupabaseConfig'; @@ -10,6 +13,7 @@ import NewConversationButton from './NewConversationButton'; import PromptForm from './PromptForm'; import SideMenu from './SideMenu'; import { ThemeToggle } from './ThemeToggle'; +import { Button } from './ui/button'; export default function Chat({ supabaseClient, @@ -18,6 +22,7 @@ export default function Chat({ }) { const bottomRef = useRef(null); const textareaRef = useRef(null); + const router = useRouter(); const [entryData, setEntryData] = useState(''); const [messages, setMessages] = useState([]); @@ -207,12 +212,21 @@ export default function Chat({
- { newConversation.mutate(); }} /> + +
diff --git a/app/src/pages/connect.tsx b/app/src/pages/connect.tsx new file mode 100644 index 0000000..f6ab479 --- /dev/null +++ b/app/src/pages/connect.tsx @@ -0,0 +1,174 @@ +import { useEffect, useState } from 'react'; + +import LogoutButton from '@/components/LogoutButton'; +import { NavMenu } from '@/components/NavMenu'; +import { ThemeToggle } from '@/components/ThemeToggle'; +import { Button } from '@/components/ui/button'; +import { useSupabase, useSupabaseConfig } from '@/utils/useSupabaseConfig'; +import { BleClient, ScanResult } from '@capacitor-community/bluetooth-le'; +import { CapacitorHttp, HttpResponse } from '@capacitor/core'; +import { Files } from 'lucide-react'; +import Link from 'next/link'; +import { useRouter } from 'next/router'; + +// const SERVICE_ID = "4fafc201-1fb5-459e-8fcc-c5c9c331914b"; +// const CHARACTERISTIC_ID = "beb5483e-36e1-4688-b7f5-ea07361b26a8"; +// const SAMPLE_RATE = 44100; +// const FRAMES_PER_BUFFER = 512; +// const RECORD_SECONDS = 10; + +export default function Index() { + const [devices, setDevices] = useState([]); + const { supabaseUrl, supabaseToken } = useSupabaseConfig(); + + const connect = async (deviceId: string) => { + // const device = await BleClient.requestDevice({ + // services: ['4fafc201-1fb5-459e-8fcc-c5c9c331914b'], + // }); + await BleClient.connect(deviceId); + + return deviceId; + }; + + const router = useRouter(); + const [loggedIn, setLoggedIn] = useState(false); + const [data, setData] = useState(null); + const [connected, setConnected] = useState(false); + + const { user, supabaseClient } = useSupabase(); + + useEffect(() => { + scan(); + }, []); + + useEffect(() => { + if (user) { + setLoggedIn(true); + } else { + setLoggedIn(false); + } + }, [user]); + + async function scan(): Promise { + try { + await BleClient.initialize(); + + await BleClient.requestLEScan({}, (result: ScanResult) => { + setDevices((prev) => { + if (result.device.name === 'ADeus') { + return [...prev, result]; + } else { + return prev; + } + }); + }); + + setTimeout(async () => { + await BleClient.stopLEScan(); + console.log('stopped scanning'); + }, 5000); + } catch (error) { + console.error(error); + } + } + + async function sendAudioData(audioData: Uint8Array) { + const data = Buffer.from(audioData).toString('base64'); + console.log('base64', data); + + if (!supabaseUrl) { + throw new Error('Supabase URL is not defined'); + } + + const options = { + url: supabaseUrl + '/functions/v1/process-audio', + headers: { + 'Content-Type': 'application/json', + }, + data: { data: data }, + }; + + const response: HttpResponse = await CapacitorHttp.post(options); + + if (response.status !== 200) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const result = await response.data; + console.log('Response from the backend:', result); + } + + if (!supabaseClient) { + return
Supabase client not found
; + } + + return ( + <> +
+
+ + + + + +
+
+ {devices.map((device, index) => { + return ( + + ); + })} + +
+ + ); +} diff --git a/scripts/wav_over_serial/main.py b/scripts/wav_over_serial/main.py new file mode 100644 index 0000000..43584f8 --- /dev/null +++ b/scripts/wav_over_serial/main.py @@ -0,0 +1,36 @@ +import serial +import wave +import time + +# Configure your serial port settings +SERIAL_PORT = '/dev/tty.usbmodem2101' # Update this to your serial port name +BAUD_RATE = 500000 +CHANNELS = 1 +SAMPLE_RATE = 16000 +SAMPLE_WIDTH = 2 # 16 bits = 2 bytes +RECORD_SECONDS = 10 # Modify as needed + +def main(): + ser = serial.Serial(SERIAL_PORT, BAUD_RATE, timeout=1) + wav_file = wave.open('output.wav', 'wb') + wav_file.setnchannels(CHANNELS) + wav_file.setsampwidth(SAMPLE_WIDTH) + wav_file.setframerate(SAMPLE_RATE) + + try: + print("Recording...") + start_time = time.time() + while (time.time() - start_time) < RECORD_SECONDS: + data = ser.read(SAMPLE_RATE * SAMPLE_WIDTH * 2) + print(len(data)) + if data: + wav_file.writeframes(data) + except KeyboardInterrupt: + print("Recording stopped.") + finally: + ser.close() + wav_file.close() + print("WAV file has been created.") + +if __name__ == "__main__": + main() diff --git a/scripts/wav_over_serial/requirements.txt b/scripts/wav_over_serial/requirements.txt new file mode 100644 index 0000000..4d1aaa2 --- /dev/null +++ b/scripts/wav_over_serial/requirements.txt @@ -0,0 +1 @@ +pyserial==3.5 \ No newline at end of file diff --git a/supabase/functions/common/cors.ts b/supabase/functions/common/cors.ts index 32a1cbe..88f49c9 100644 --- a/supabase/functions/common/cors.ts +++ b/supabase/functions/common/cors.ts @@ -3,3 +3,4 @@ export const corsHeaders = { "Access-Control-Allow-Headers": "authorization, x-client-info, apikey, content-type", "Access-Control-Max-Age": "3600", }; + \ No newline at end of file diff --git a/supabase/functions/process-audio/index.ts b/supabase/functions/process-audio/index.ts index 9c9d1f4..caa64f8 100644 --- a/supabase/functions/process-audio/index.ts +++ b/supabase/functions/process-audio/index.ts @@ -1,8 +1,49 @@ import { serve } from "https://deno.land/std@0.170.0/http/server.ts"; import OpenAI, { toFile } from "https://deno.land/x/openai@v4.26.0/mod.ts"; -import { multiParser } from 'https://deno.land/x/multiparser@0.114.0/mod.ts'; +import { decodeBase64 } from "https://deno.land/std@0.217.0/encoding/base64.ts"; import { corsHeaders } from "../common/cors.ts"; import { supabaseClient } from "../common/supabaseClient.ts"; +import { multiParser } from 'https://deno.land/x/multiparser@0.114.0/mod.ts'; + + +function createWavHeader(dataLength: number, sampleRate: number, numChannels: number, bitDepth: number) { + const headerLength = 44; // Fixed size for WAV header + const buffer = new ArrayBuffer(headerLength); + const view = new DataView(buffer); + + // Helper function to write a string to the DataView + function writeString(view: DataView, offset: number, value: string) { + for (let i = 0; i < value.length; i++) { + view.setUint8(offset + i, value.charCodeAt(i)); + } + } + + // Writes a 32-bit unsigned integer to the DataView + function writeUint32(view: DataView, offset: number, value) { + view.setUint32(offset, value, true); + } + + // Writes a 16-bit unsigned integer to the DataView + function writeUint16(view: DataView, offset: number, value) { + view.setUint16(offset, value, true); + } + + writeString(view, 0, "RIFF"); + writeUint32(view, 4, 36 + dataLength); + writeString(view, 8, "WAVE"); + writeString(view, 12, "fmt "); + writeUint32(view, 16, 16); + writeUint16(view, 20, 1); + writeUint16(view, 22, numChannels); + writeUint32(view, 24, sampleRate); + writeUint32(view, 28, sampleRate * numChannels * bitDepth / 8); + writeUint16(view, 32, numChannels * bitDepth / 8); + writeUint16(view, 34, bitDepth); + writeString(view, 36, "data"); + writeUint32(view, 40, dataLength); + + return new Uint8Array(buffer); +} const processAudio = async (req: Request) => { @@ -31,6 +72,17 @@ const processAudio = async (req: Request) => { const file = form.files.file; arrayBuffer = file.content.buffer; filenameTimestamp = file.filename || filenameTimestamp; + } else if (contentType.includes('application/json')) { + const { data } = await req.json(); + const audioData = decodeBase64(data); + + console.log('Audio data:', audioData.length); + // 1 Channel, 8000 sample rate, 16 bit depth + const wavHeader = createWavHeader(audioData.length, 8000, 1, 16); + const wavBytes = new Uint8Array(wavHeader.length + audioData.length); + wavBytes.set(wavHeader, 0); + wavBytes.set(audioData, wavHeader.length); + arrayBuffer = wavBytes.buffer; } else { arrayBuffer = await req.arrayBuffer(); } @@ -56,6 +108,7 @@ const processAudio = async (req: Request) => { (transcriptLowered.includes("thank") && transcriptLowered.includes("watch")) ) { + console.log("no transcript found"); return new Response(JSON.stringify({ message: "No transcript found." }), { headers: { ...corsHeaders, "Content-Type": "application/json" }, status: 200,