Add speech synthesize from Gemini (#2404)

* Improve slim docker image example, clean up OpenaiChat provider

* Enhance event loop management for asynchronous generators

* Fix attribute " shutdown_default_executor" not found in old python versions

* asyncio file created with all async helpers

* Add speech synthesize from Gemini. You can use it without a account

g4f/Provider/needs_auth/Gemini.py

@@ -4,8 +4,10 @@ import os
 import json
 import random
 import re
+import base64
 
 from aiohttp import ClientSession, BaseConnector
+
 try:
     import nodriver
     has_nodriver = True
@@ -14,12 +16,13 @@ except ImportError:
 
 from ... import debug
 from ...typing import Messages, Cookies, ImageType, AsyncResult, AsyncIterator
-from ..base_provider import AsyncGeneratorProvider, BaseConversation
+from ..base_provider import AsyncGeneratorProvider, BaseConversation, SynthesizeData
 from ..helper import format_prompt, get_cookies
 from ...requests.raise_for_status import raise_for_status
 from ...requests.aiohttp import get_connector
 from ...errors import MissingAuthError
 from ...image import ImageResponse, to_bytes
+from ... import debug
 
 REQUEST_HEADERS = {
     "authority": "gemini.google.com",
@@ -54,6 +57,7 @@ class Gemini(AsyncGeneratorProvider):
     image_models = ["gemini"]
     default_vision_model = "gemini"
     models = ["gemini", "gemini-1.5-flash", "gemini-1.5-pro"]
+    synthesize_content_type = "audio/vnd.wav"
     _cookies: Cookies = None
     _snlm0e: str = None
     _sid: str = None
@@ -106,6 +110,7 @@ class Gemini(AsyncGeneratorProvider):
         prompt = format_prompt(messages) if conversation is None else messages[-1]["content"]
         cls._cookies = cookies or cls._cookies or get_cookies(".google.com", False, True)
         base_connector = get_connector(connector, proxy)
+
         async with ClientSession(
             headers=REQUEST_HEADERS,
             connector=base_connector
@@ -122,6 +127,7 @@ class Gemini(AsyncGeneratorProvider):
             if not cls._snlm0e:
                 raise RuntimeError("Invalid cookies. SNlM0e not found")
 
+            yield SynthesizeData(cls.__name__, {"text": messages[-1]["content"]})
             image_url = await cls.upload_image(base_connector, to_bytes(image), image_name) if image else None
 
             async with ClientSession(
@@ -198,6 +204,40 @@ class Gemini(AsyncGeneratorProvider):
                         except TypeError:
                             pass
 
+    @classmethod
+    async def synthesize(cls, params: dict, proxy: str = None) -> AsyncIterator[bytes]:
+        async with ClientSession(
+            cookies=cls._cookies,
+            headers=REQUEST_HEADERS,
+            connector=get_connector(proxy=proxy),
+        ) as session:
+            if not cls._snlm0e:
+                await cls.fetch_snlm0e(session, cls._cookies) if cls._cookies else None
+            if not cls._snlm0e:
+                async for chunk in cls.nodriver_login(proxy):
+                    debug.log(chunk)
+            inner_data = json.dumps([None, params["text"], "de-DE", None, 2])
+            async with session.post(
+                "https://gemini.google.com/_/BardChatUi/data/batchexecute",
+                data={
+                      "f.req": json.dumps([[["XqA3Ic", inner_data, None, "generic"]]]),
+                      "at": cls._snlm0e,
+                },
+                params={
+                    "rpcids": "XqA3Ic",
+                    "source-path": "/app/2704fb4aafcca926",
+                    "bl": "boq_assistant-bard-web-server_20241119.00_p1",
+                    "f.sid": "" if cls._sid is None else cls._sid,
+                    "hl": "de",
+                    "_reqid": random.randint(1111, 9999),
+                    "rt": "c"
+                },
+            ) as response:
+                await raise_for_status(response)
+                iter_base64_response = iter_filter_base64(response.content.iter_chunked(1024))
+                async for chunk in iter_base64_decode(iter_base64_response):
+                    yield chunk
+
     def build_request(
         prompt: str,
         language: str,
@@ -280,3 +320,27 @@ class Conversation(BaseConversation):
         self.conversation_id = conversation_id
         self.response_id = response_id
         self.choice_id = choice_id
+async def iter_filter_base64(response_iter: AsyncIterator[bytes]) -> AsyncIterator[bytes]:
+    search_for = b'[["wrb.fr","XqA3Ic","[\\"'
+    end_with = b'\\'
+    is_started = False
+    async for chunk in response_iter:
+        if is_started:
+            if end_with in chunk:
+                yield chunk.split(end_with, 1).pop(0)
+                break
+            else:
+                yield chunk
+        elif search_for in chunk:
+            is_started = True
+            yield chunk.split(search_for, 1).pop()
+        else:
+            raise RuntimeError(f"Response: {chunk}")
+
+async def iter_base64_decode(response_iter: AsyncIterator[bytes]) -> AsyncIterator[bytes]:
+    buffer = b""
+    async for chunk in response_iter:
+        chunk = buffer + chunk
+        rest = len(chunk) % 4
+        buffer = chunk[-rest:]
+        yield base64.b64decode(chunk[:-rest])

g4f/Provider/needs_auth/OpenaiChat.py

@@ -61,6 +61,7 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin):
     fallback_models = [default_model, "gpt-4", "gpt-4o", "gpt-4o-mini", "gpt-4o-canmore", "o1-preview", "o1-mini"]
     vision_models = fallback_models
     image_models = fallback_models
+    synthesize_content_type = "audio/mpeg"
 
     _api_key: str = None
     _headers: dict = None

g4f/api/__init__.py

@@ -26,6 +26,7 @@ from g4f.client.helper import filter_none
 from g4f.image import is_accepted_format, images_dir
 from g4f.typing import Messages
 from g4f.cookies import read_cookie_files
+from g4f.Provider import ProviderType, ProviderUtils, __providers__
 
 logger = logging.getLogger(__name__)
 
@@ -152,7 +153,9 @@ class Api:
             return HTMLResponse('g4f API: Go to '
                                 '<a href="/v1/models">models</a>, '
                                 '<a href="/v1/chat/completions">chat/completions</a>, or '
-                                '<a href="/v1/images/generate">images/generate</a>.')
+                                '<a href="/v1/images/generate">images/generate</a> <br><br>'
+                                'Open Swagger UI at: '
+                                '<a href="/docs">/docs</a>')
 
         @self.app.get("/v1/models")
         async def models():
@@ -290,6 +293,40 @@ class Api:
 
             return FileResponse(target, media_type=content_type)
 
+        @self.app.get("/providers")
+        async def providers():
+            model_list = [{
+                'id': provider.__name__,
+                'object': 'provider',
+                'created': 0,
+                'url': provider.url,
+                'label': getattr(provider, "label", None),
+            } for provider in __providers__ if provider.working]
+            return JSONResponse(model_list)
+
+        @self.app.get("/providers/{provider}")
+        async def providers_info(provider: str):
+            if provider not in ProviderUtils.convert:
+                return JSONResponse({"error": "The model does not exist."}, 404)
+            provider: ProviderType = ProviderUtils.convert[provider]
+            def safe_get_models(provider: ProviderType) -> list[str]:
+                try:
+                    return provider.get_models() if hasattr(provider, "get_models") else []
+                except:
+                    return []
+            provider_info = {
+                'id': provider.__name__,
+                'object': 'provider',
+                'created': 0,
+                'url': provider.url,
+                'label': getattr(provider, "label", None),
+                'models': safe_get_models(provider),
+                'image_models': getattr(provider, "image_models", []) or [],
+                'vision_models': [model for model in [getattr(provider, "default_vision_model", None)] if model],
+                'params': [*provider.get_parameters()] if hasattr(provider, "get_parameters") else []
+            }
+            return JSONResponse(provider_info)
+
 def format_exception(e: Exception, config: Union[ChatCompletionsConfig, ImageGenerationConfig], image: bool = False) -> str:
     last_provider = {} if not image else g4f.get_last_provider(True)
     provider = (AppConfig.image_provider if image else AppConfig.provider) if config.provider is None else config.provider

g4f/gui/client/index.html

@@ -191,6 +191,9 @@
             <button class="slide-systemPrompt">
                 <i class="fa-solid fa-angles-up"></i>
             </button>
+            <div class="media_player">
+                <i class="fa-regular fa-x"></i>
+            </div>
             <div class="toolbar">
                 <div id="input-count" class="">
                     <button class="hide-input">

g4f/gui/client/static/css/style.css

@@ -434,15 +434,28 @@ body {
     font-size: 12px;
 }
 
-.message audio {
+.media_player {
     display: none;
-    max-width: 400px;
 }
 
-.message audio.show {
+.media_player audio {
+    right: 28px;
+    position: absolute;
+    top: -4px;
+    z-index: 900;
+}
+
+.media_player.show {
     display: block;
 }
 
+.media_player .fa-x {
+    position: absolute;
+    right: 8px;
+    top: 8px;
+    z-index: 1000;
+}
+
 .count_total {
     font-size: 12px;
     padding-left: 25px;

g4f/gui/client/static/js/chat.v1.js

@@ -44,17 +44,18 @@ appStorage = window.localStorage || {
     removeItem: (key) => delete self[key],
     length: 0
 }
-
-const markdown = window.markdownit();
-const markdown_render = (content) => {
-    return markdown.render(content
-        .replaceAll(/<!-- generated images start -->|<!-- generated images end -->/gm, "")
-        .replaceAll(/<img data-prompt="[^>]+">/gm, "")
-    )
-        .replaceAll("<a href=", '<a target="_blank" href=')
-        .replaceAll('<code>', '<code class="language-plaintext">')
+let markdown_render = () => null;
+if (window.markdownit) {
+    const markdown = window.markdownit();
+    markdown_render = (content) => {
+        return markdown.render(content
+            .replaceAll(/<!-- generated images start -->|<!-- generated images end -->/gm, "")
+            .replaceAll(/<img data-prompt="[^>]+">/gm, "")
+        )
+            .replaceAll("<a href=", '<a target="_blank" href=')
+            .replaceAll('<code>', '<code class="language-plaintext">')
+    }
 }
-
 function filter_message(text) {
     return text.replaceAll(
         /<!-- generated images start -->[\s\S]+<!-- generated images end -->/gm, ""
@@ -135,10 +136,21 @@ const register_message_buttons = async () => {
         if (!("click" in el.dataset)) {
             el.dataset.click = "true";
             el.addEventListener("click", async () => {
-                const content_el = el.parentElement.parentElement;
-                const audio = content_el.querySelector("audio");
-                if (audio) {
-                    audio.classList.add("show");
+                const message_el = el.parentElement.parentElement.parentElement;
+                let audio;
+                if (message_el.dataset.synthesize_url) {
+                    el.classList.add("active");
+                    setTimeout(()=>el.classList.remove("active"), 2000);
+                    const media_player = document.querySelector(".media_player");
+                    if (!media_player.classList.contains("show")) {
+                        media_player.classList.add("show");
+                        audio = new Audio(message_el.dataset.synthesize_url);
+                        audio.controls = true;   
+                        media_player.appendChild(audio);
+                    } else {
+                        audio = media_player.querySelector("audio");
+                        audio.src = message_el.dataset.synthesize_url;
+                    }
                     audio.play();
                     return;
                 }
@@ -163,7 +175,7 @@ const register_message_buttons = async () => {
                 el.dataset.running = true;
                 el.classList.add("blink")
                 el.classList.add("active")
-                const message_el = content_el.parentElement;
+
                 let speechText = await get_message(window.conversation_id, message_el.dataset.index);
 
                 speechText = speechText.replaceAll(/([^0-9])\./gm, "$1.;");
@@ -351,6 +363,13 @@ stop_generating.addEventListener("click", async () => {
     await load_conversation(window.conversation_id, false);
 });
 
+document.querySelector(".media_player .fa-x").addEventListener("click", ()=>{
+    const media_player = document.querySelector(".media_player");
+    media_player.classList.remove("show");
+    const audio = document.querySelector(".media_player audio");
+    media_player.removeChild(audio);
+});
+
 const prepare_messages = (messages, message_index = -1) => {
     if (message_index >= 0) {
         messages = messages.filter((_, index) => message_index >= index);
@@ -726,17 +745,17 @@ const load_conversation = async (conversation_id, scroll=true) => {
                 ${item.provider.model ? ' with ' + item.provider.model : ''}
             </div>
         ` : "";
-        let audio = "";
+        let synthesize_params = {text: item.content}
+        let synthesize_provider = "Gemini";
         if (item.synthesize) {
-            const synthesize_params = (new URLSearchParams(item.synthesize.data)).toString();
-            audio = `
-                <audio controls preload="none">
-                    <source src="/backend-api/v2/synthesize/${item.synthesize.provider}?${synthesize_params}" type="audio/mpeg">
-                </audio> 
-            `;
+            synthesize_params = item.synthesize.data
+            synthesize_provider = item.synthesize.provider;
         }
+        synthesize_params = (new URLSearchParams(synthesize_params)).toString();
+        let synthesize_url = `/backend-api/v2/synthesize/${synthesize_provider}?${synthesize_params}`;
+
         elements += `
-            <div class="message${item.regenerate ? " regenerate": ""}" data-index="${i}">
+            <div class="message${item.regenerate ? " regenerate": ""}" data-index="${i}" data-synthesize_url="${synthesize_url}">
                 <div class="${item.role}">
                     ${item.role == "assistant" ? gpt_image : user_image}
                     <i class="fa-solid fa-xmark"></i>
@@ -748,7 +767,6 @@ const load_conversation = async (conversation_id, scroll=true) => {
                 <div class="content">
                     ${provider}
                     <div class="content_inner">${markdown_render(item.content)}</div>
-                    ${audio}
                     <div class="count">
                         ${count_words_and_tokens(item.content, next_provider?.model)}
                         <i class="fa-solid fa-volume-high"></i>

g4f/gui/server/api.py

@@ -140,13 +140,12 @@ class Api:
         }
 
     def _create_response_stream(self, kwargs: dict, conversation_id: str, provider: str, download_images: bool = True) -> Iterator:
-        if debug.logging:
-            debug.logs = []
-            print_callback = debug.log_handler
-            def log_handler(text: str):
-                debug.logs.append(text)
-                print_callback(text)
-            debug.log_handler = log_handler
+        debug.logs = []
+        print_callback = debug.log_handler
+        def log_handler(text: str):
+            debug.logs.append(text)
+            print_callback(text)
+        debug.log_handler = log_handler
         try:
             result = ChatCompletion.create(**kwargs)
             first = True

g4f/gui/server/backend.py

@@ -1,6 +1,8 @@
 import json
 import flask
 import os
+import logging
+import asyncio
 from flask import request, Flask
 from typing import Generator
 from werkzeug.utils import secure_filename
@@ -12,6 +14,8 @@ from g4f.errors import ProviderNotFoundError
 from g4f.cookies import get_cookies_dir
 from .api import Api
 
+logger = logging.getLogger(__name__)
+
 def safe_iter_generator(generator: Generator) -> Generator:
     start = next(generator)
     def iter_generator():
@@ -127,15 +131,17 @@ class Backend_Api(Api):
             return "Provider not found", 404
         if not hasattr(provider_handler, "synthesize"):
             return "Provider doesn't support synthesize", 500
-        try:
-            response_generator = provider_handler.synthesize({**request.args})
-            if hasattr(response_generator, "__aiter__"):
-                response_generator = to_sync_generator(response_generator)
-            response = flask.Response(safe_iter_generator(response_generator), content_type="audio/mpeg")
-            response.headers['Cache-Control'] = "max-age=604800"
-            return response
-        except Exception as e:
-            return f"{e.__class__.__name__}: {e}", 500
+        response_data = provider_handler.synthesize({**request.args})
+        if asyncio.iscoroutinefunction(provider_handler.synthesize):
+            response_data = asyncio.run(response_data)
+        else:
+            if hasattr(response_data, "__aiter__"):
+                response_data = to_sync_generator(response_data)
+            response_data = safe_iter_generator(response_data)
+        content_type = getattr(provider_handler, "synthesize_content_type", "application/octet-stream")
+        response = flask.Response(response_data, content_type=content_type)
+        response.headers['Cache-Control'] = "max-age=604800"
+        return response
 
     def get_provider_models(self, provider: str):
         api_key = None if request.authorization is None else request.authorization.token

g4f/providers/base_provider.py

@@ -66,11 +66,12 @@ class AbstractProvider(BaseProvider):
 
     @classmethod
     def get_parameters(cls) -> dict[str, Parameter]:
-        return signature(
+        return {name: parameter for name, parameter in signature(
             cls.create_async_generator if issubclass(cls, AsyncGeneratorProvider) else
             cls.create_async if issubclass(cls, AsyncProvider) else
             cls.create_completion
-        ).parameters
+        ).parameters.items() if name not in ["kwargs", "model", "messages"]
+            and (name != "stream" or cls.supports_stream)}
 
     @classmethod
     @property
@@ -90,8 +91,6 @@ class AbstractProvider(BaseProvider):
 
         args = ""
         for name, param in cls.get_parameters().items():
-            if name in ("self", "kwargs") or (name == "stream" and not cls.supports_stream):
-                continue
             args += f"\n    {name}"
             args += f": {get_type_name(param.annotation)}" if param.annotation is not Parameter.empty else ""
             default_value = f'"{param.default}"' if isinstance(param.default, str) else param.default