add-speech-to-text-in-chat (#182)

* add-speech-to-text-in-chat

* implement change request and fix CI error

frontend/app/chat/page.tsx

@@ -1,8 +1,8 @@
 "use client";
 import axios from "axios";
 import { redirect } from "next/navigation";
-import { useState } from "react";
+import { useEffect, useState } from "react";
-import { MdSettings } from "react-icons/md";
+import { MdMic, MdMicOff, MdSettings } from "react-icons/md";
 import Button from "../components/ui/Button";
 import Card from "../components/ui/Card";
 import Modal from "../components/ui/Modal";
@@ -17,14 +17,58 @@ export default function ChatPage() {
   const [temperature, setTemperature] = useState(0);
   const [maxTokens, setMaxTokens] = useState(500);
   const [isPending, setIsPending] = useState(false);
+  const [isListening, setIsListening] = useState(false);
   const { session } = useSupabase();
   if (session === null) {
     redirect("/login");
   }
 
+  useEffect(() => {
+    if (typeof window !== "undefined") {
+      const SpeechRecognition =
+        window.SpeechRecognition || window.webkitSpeechRecognition;
+
+      const mic = new SpeechRecognition();
+
+      mic.continuous = true;
+      mic.interimResults = false;
+      mic.lang = "en-US";
+
+      mic.onstart = () => {
+        console.log("Mics on");
+      };
+
+      mic.onend = () => {
+        console.log("Mics off");
+      };
+
+      mic.onerror = (event: SpeechRecognitionErrorEvent) => {
+        console.log(event.error);
+        setIsListening(false);
+      };
+
+      mic.onresult = (event: SpeechRecognitionEvent) => {
+        const interimTranscript =
+          event.results[event.results.length - 1][0].transcript;
+        setQuestion((prevQuestion) => prevQuestion + interimTranscript);
+      };
+
+      if (isListening) {
+        mic.start();
+      }
+
+      return () => {
+        if (mic) {
+          mic.stop();
+        }
+      };
+    }
+  }, [isListening]);
+
   const askQuestion = async () => {
     setHistory((hist) => [...hist, ["user", question]]);
     setIsPending(true);
+    setIsListening(false);
     const response = await axios.post(
       `${process.env.NEXT_PUBLIC_BACKEND_URL}/chat/`,
       {
@@ -45,6 +89,10 @@ export default function ChatPage() {
     setIsPending(false);
   };
 
+  const handleListen = () => {
+    setIsListening((prevIsListening) => !prevIsListening);
+  };
+
   return (
     <main className="min-h-screen w-full flex flex-col pt-32">
       <section className="flex flex-col justify-center items-center flex-1 gap-5 h-full">
@@ -74,6 +122,19 @@ export default function ChatPage() {
               <Button type="submit" isLoading={isPending}>
                 {isPending ? "Thinking..." : "Chat"}
               </Button>
+              {/* Mic Button */}
+              <Button
+                className="px-3"
+                variant={"tertiary"}
+                type="button"
+                onClick={handleListen}
+              >
+                {isListening ? (
+                  <MdMicOff className="text-2xl" />
+                ) : (
+                  <MdMic className="text-2xl" />
+                )}
+              </Button>
               {/* Settings Button */}
               <Modal
                 Trigger={

frontend/package.json

@@ -18,6 +18,7 @@
     "@supabase/auth-ui-react": "^0.4.2",
     "@supabase/auth-ui-shared": "^0.1.6",
     "@supabase/supabase-js": "^2.22.0",
+    "@types/dom-speech-recognition": "^0.0.1",
     "@types/node": "20.1.7",
     "@types/react": "18.2.6",
     "@types/react-dom": "18.2.4",

frontend/yarn.lock

@@ -547,6 +547,11 @@
   dependencies:
     "@types/ms" "*"
 
+"@types/dom-speech-recognition@^0.0.1":
+  version "0.0.1"
+  resolved "https://registry.yarnpkg.com/@types/dom-speech-recognition/-/dom-speech-recognition-0.0.1.tgz#e326761a04b4a49c0eec2ac7948afc1c6aa12baa"
+  integrity sha512-udCxb8DvjcDKfk1WTBzDsxFbLgYxmQGKrE/ricoMqHRNjSlSUCcamVTA5lIQqzY10mY5qCY0QDwBfFEwhfoDPw==
+
 "@types/hast@^2.0.0":
   version "2.3.4"
   resolved "https://registry.yarnpkg.com/@types/hast/-/hast-2.3.4.tgz#8aa5ef92c117d20d974a82bdfb6a648b08c0bafc"