Untitled

import os
import asyncio
import base64
import io
import traceback
import cv2
import pyaudio
import PIL.Image
import mss
import argparse
from google import genai
from google.genai import types
from dotenv import load_dotenv
import subprocess
import sys
import time  # EKLE


game_process = {}
load_dotenv()
FORMAT = pyaudio.paInt16
CHANNELS = 1
SEND_SAMPLE_RATE = 16000
RECEIVE_SAMPLE_RATE = 24000
CHUNK_SIZE = 1024
camera_state = False
main = None

MODEL = "models/gemini-2.5-flash-live-preview"

DEFAULT_MODE = "all"

client = genai.Client(
    http_options={"api_version": "v1beta"},
    api_key=os.environ.get("GOOGLE_API_KEY"),
)


async def start_game(dosya_adi: str):
    try:
        python_path = sys.executable
        full_path = os.path.abspath(dosya_adi)
        working_dir = os.path.dirname(full_path)  # Çalışma dizini olarak klasörünü al

        proses = subprocess.Popen(
            [python_path, full_path],
            cwd=working_dir
        )

        game_process[dosya_adi] = proses
        print(f"{dosya_adi} başlatıldı. PID: {proses.pid}")
    except Exception as e:
        print(f"Hata oluştu: {e}")

async def stop_game(dosya_adi: str):
    proses = game_process.get(dosya_adi)
    if proses and proses.poll() is None:
        proses.terminate()
        print(f"{dosya_adi} durduruldu.")
        return f"Oyun durduruldu: {dosya_adi}"
    else:
        print(f"{dosya_adi} zaten çalışmıyor.")
        return f"Oyun zaten çalışmıyor: {dosya_adi}"

async def draw_game_start_stop(durum: bool):
    if durum:
        await main.stop_camera()
        print("oyun başladı")
        await start_game("test_projects/Games/AI-Virtual-Paint-main/Ai_virtual_painter.py")
        return "Çizim oyunu başlatıldı!"
    else:
        await stop_game("test_projects/Games/AI-Virtual-Paint-main/Ai_virtual_painter.py")
        print("oyun durdu")
        await main.start_camera()
        print("kamera başlatıldı")
        return "Çizim oyunu durduruldu!"


tools = [
    types.Tool(
        function_declarations=[
            types.FunctionDeclaration(
                name="draw_game_start_stop",
                description="Resim çizme oyununu başlatır veya durdurur.",
                parameters=genai.types.Schema(
                    type=genai.types.Type.OBJECT,
                    required=["durum"],
                    properties={
                        "durum": genai.types.Schema(
                            type=genai.types.Type.BOOLEAN,
                        ),
                    },
                ),
            ),
        ]
    ),
]

AVAILABLE_TOOLS = {
    "draw_game_start_stop": draw_game_start_stop,
}

CONFIG = types.LiveConnectConfig(
    response_modalities=[
        "AUDIO",
    ],
    media_resolution="MEDIA_RESOLUTION_MEDIUM",
    speech_config=types.SpeechConfig(
        language_code="tr-TR",
        voice_config=types.VoiceConfig(
            prebuilt_voice_config=types.PrebuiltVoiceConfig(voice_name="Puck")
        )
    ),
    context_window_compression=types.ContextWindowCompressionConfig(
        trigger_tokens=25600,
        sliding_window=types.SlidingWindow(target_tokens=12800),
    ),
    system_instruction=types.Content(
        parts=[types.Part.from_text(text=
            "Senin adın 'Jollyboo'. Artık küçük çocuklarla konuşan nazik, arkadaş canlısı ve neşeli bir karakter gibi davranacaksın."
        )],
        role="system"
    ),
    tools=tools,
)

pya = pyaudio.PyAudio()


class AudioLoop:
    def __init__(self, video_mode=DEFAULT_MODE):
        self.video_mode = video_mode

        self.audio_in_queue = None
        self.out_queue = None

        self.session = None

        self.send_text_task = None
        self.receive_audio_task = None
        self.play_audio_task = None

        self.is_talking = False
        self._talking_reset_task = None

        self.normal_face = cv2.imread("main_assets/faces/normal_face.png")
        self.talking_face = cv2.imread("main_assets/faces/talking_face.png")

        self.camera_state = True
        self.camera_stopped_event = asyncio.Event()

        self._suppress_mic_until = 0.0  # model ses çalarken mikrofona kilit
        self._playback_active = False


    async def start_camera(self):
        if not camera_state:
            asyncio.create_task(self.get_frames())

    async def stop_camera(self):
        global camera_state
        camera_state = False
        self.camera_stopped_event.clear()
        print("Kamera durması bekleniyor...")
        await self.camera_stopped_event.wait()
        print("Kamera durdu ve kaynak serbest.")

    async def send_text(self):
        while True:
            text = await asyncio.to_thread(
                input,
                "message > ",
            )
            if text.lower() == "q":
                break
            await self.session.send(input=text or ".", end_of_turn=True)

    @staticmethod
    def _get_frame(cap):
        ret, frame = cap.read()
        if not ret:
            return None
        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        img = PIL.Image.fromarray(frame_rgb)
        img.thumbnail([1024, 1024])

        image_io = io.BytesIO()
        img.save(image_io, format="jpeg")
        image_io.seek(0)

        mime_type = "image/jpeg"
        image_bytes = image_io.read()
        return {"mime_type": mime_type, "data": base64.b64encode(image_bytes).decode()}

    async def get_frames(self):
        print("[DEBUG] Kamera başlatılıyor...")
        cap = await asyncio.to_thread(cv2.VideoCapture, 0)
        global camera_state
        camera_state = True
        while True:
            frame = await asyncio.to_thread(self._get_frame, cap)
            if not camera_state:
                print("[DEBUG] Kamera durma isteği geldi.")
                break
            if frame is None:
                break
            await asyncio.sleep(1.0)
            await self.out_queue.put(frame)
        print("[DEBUG] Kamera kapatılıyor...")
        cap.release()
        self.camera_stopped_event.set()
        print("[LOG] Kamera kaynağı serbest bırakıldı.")

    @staticmethod
    def _get_screen():
        sct = mss.mss()
        monitor = sct.monitors[0]

        i = sct.grab(monitor)

        mime_type = "image/jpeg"
        image_bytes = mss.tools.to_png(i.rgb, i.size)
        img = PIL.Image.open(io.BytesIO(image_bytes))

        image_io = io.BytesIO()
        img.save(image_io, format="jpeg")
        image_io.seek(0)

        image_bytes = image_io.read()
        return {"mime_type": mime_type, "data": base64.b64encode(image_bytes).decode()}

    async def get_screen(self):
        while True:
            frame = await asyncio.to_thread(self._get_screen)
            if frame is None:
                break
            await asyncio.sleep(1.0)
            await self.out_queue.put(frame)

    async def send_realtime(self):
        while True:
            msg = await self.out_queue.get()
            await self.session.send(input=msg)

    async def listen_audio(self):
        mic_info = pya.get_default_input_device_info()
        self.audio_stream = await asyncio.to_thread(
            pya.open,
            format=FORMAT,
            channels=CHANNELS,
            rate=SEND_SAMPLE_RATE,
            input=True,
            input_device_index=mic_info["index"],
            frames_per_buffer=CHUNK_SIZE,
        )
        kwargs = {"exception_on_overflow": False} if __debug__ else {}
        while True:
            data = await asyncio.to_thread(self.audio_stream.read, CHUNK_SIZE, **kwargs)

            # ⭐ Model ses çalarken ve hemen sonrasında mikrofondan geleni YOLLAMA
            now = time.monotonic()
            if self.is_talking or now < self._suppress_mic_until:
                continue

            await self.out_queue.put({"data": data, "mime_type": "audio/pcm;rate=16000"})
    async def receive_audio(self):
        while True:
            turn = self.session.receive()
            async for response in turn:


                if data := response.data:
                    self.audio_in_queue.put_nowait(data)

                if text := response.text:
                    print("metin verisi geldi")
                    print(text, end="")

                result_message = ""
                if response.tool_call and response.tool_call.function_calls:
                    print(f"Model bir araç çağrısı önerdi.")
                    tool_output_results = []
                    for function_call in response.tool_call.function_calls:
                        function_name = function_call.name
                        args = function_call.args

                        print(f"\nModel bir fonksiyon çağırdı: {function_name} ({args})")

                        if function_name in AVAILABLE_TOOLS:
                            function_to_call = AVAILABLE_TOOLS[function_name]
                            try:
                                result_message = await function_to_call(**args)
                                print(f">> Araç çağrısı sonucu: {result_message}")

                                tool_output_results.append(
                                    types.FunctionResponse(
                                        name=function_name,
                                        response={"result": result_message},
                                        id=function_call.id
                                    )
                                )

                            except TypeError as e:
                                error_msg = f"Hata: {function_name} çağrılırken argüman hatası: {e}. Argümanlar: {args}"
                                print(f">> {error_msg}")
                                tool_output_results.append(
                                    types.FunctionResponse(
                                        name=function_name,
                                        response={"result": result_message},
                                        id=function_call.id
                                    )
                                )
                            except Exception as e:
                                error_msg = f"Hata: {function_name} çağrılırken bilinmeyen hata: {e}"
                                print(f">> {error_msg}")
                                tool_output_results.append(
                                    types.FunctionResponse(
                                        name=function_name,
                                        response={"result": result_message},
                                        id=function_call.id
                                    )
                                )
                        else:
                            error_msg = f"Hata: Tanımlı olmayan fonksiyon çağrısı: {function_name}"
                            print(f">> {error_msg}")
                            tool_output_results.append(
                                types.FunctionResponse(
                                    name=function_name,
                                    response={"result": result_message},
                                    id=function_call.id
                                )
                            )

                    if tool_output_results:
                        await self.session.send_tool_response(
                            function_responses=tool_output_results,
                        )
            while not self.audio_in_queue.empty():
                self.audio_in_queue.get_nowait()

    async def _reset_talking_flag_after_delay(self, delay=0.5):
        await asyncio.sleep(delay)
        self.is_talking = False

    async def play_audio(self):
        stream = await asyncio.to_thread(
            pya.open,
            format=FORMAT,
            channels=CHANNELS,
            rate=RECEIVE_SAMPLE_RATE,
            output=True,
        )
        while True:
            bytestream = await self.audio_in_queue.get()

            self.is_talking = True
            self._playback_active = True

            # varsa önceki reset görevini iptal et ve 0.5 sn sonra "konuşmuyor" bayrağını düşür
            if self._talking_reset_task is not None:
                self._talking_reset_task.cancel()
            self._talking_reset_task = asyncio.create_task(self._reset_talking_flag_after_delay())

            # ⭐ Ses çalarken ve hemen sonrasında mikrofonu kilitle (eko önleme penceresi)
            self._suppress_mic_until = time.monotonic() + 0.35  # 350ms tampon

            await asyncio.to_thread(stream.write, bytestream)

            # kısa bir güvenlik tamponu daha
            self._suppress_mic_until = time.monotonic() + 0.35
            self._playback_active = False
    async def show_face(self):
        while True:
            if self.is_talking:
                img = self.talking_face
            else:
                img = self.normal_face

            if img is not None:
                cv2.namedWindow("Yapay Zeka Yüzü", cv2.WND_PROP_FULLSCREEN)
                cv2.setWindowProperty("Yapay Zeka Yüzü", cv2.WND_PROP_FULLSCREEN, cv2.WND_PROP_FULLSCREEN)
                cv2.imshow("Yapay Zeka Yüzü", img)

            if cv2.waitKey(100) & 0xFF == ord('q'):
                break

            await asyncio.sleep(0.1)

        cv2.destroyAllWindows()

    async def run(self):
        try:
            async with (
                client.aio.live.connect(model=MODEL, config=CONFIG) as session,
                asyncio.TaskGroup() as tg,
            ):
                self.session = session

                self.audio_in_queue = asyncio.Queue()
                self.out_queue = asyncio.Queue(maxsize=5)

                send_text_task = tg.create_task(self.send_text())
                tg.create_task(self.send_realtime())
                tg.create_task(self.listen_audio())
                if self.video_mode == "camera":
                    tg.create_task(self.get_frames())
                elif self.video_mode == "screen":
                    tg.create_task(self.get_screen())
                elif self.video_mode == "all":
                    tg.create_task(self.get_frames())
                    tg.create_task(self.get_screen())
                tg.create_task(self.receive_audio())
                tg.create_task(self.play_audio())

                # Burada yüz gösterme görevini başlatıyoruz
                tg.create_task(self.show_face())

                await send_text_task
                raise asyncio.CancelledError("User requested exit")

        except asyncio.CancelledError:
            pass
        except ExceptionGroup as EG:
            self.audio_stream.close()
            traceback.print_exception(EG)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--mode",
        type=str,
        default=DEFAULT_MODE,
        help="pixels to stream from",
        choices=["camera", "screen",  "all","none"],
    )
    args = parser.parse_args()
    main = AudioLoop(video_mode=args.mode)
    asyncio.run(main.run())