Untitled

Anonymous
plain_text
01/19/2026 3:11 PM
22.1 KB
10
Indexable
using System;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.Android;

using LiveKit;
using LiveKit.Proto;
using RoomOptions = LiveKit.RoomOptions;

public class LiveKitCoachClient : MonoBehaviour
{
    [Header("Room")]
    public string roomName = "lobby";

    [Header("Token Provider (Sandbox)")]
    public LiveKitTokenClient tokenClient;

    [Header("Behavior")]
    public bool autoConnectOnStart = true;

    [Header("Audio Publish Settings")]
    public int micMaxBitrate = 64000;

    [Header("Optional UI")]
    public TMPro.TMP_Text statusText;

    [Header("Debug")]
    public bool verboseLog = true;
    public float statsLogIntervalSec = 1.0f;

    [Tooltip("Mic RMS threshold to consider 'voice present'")]
    public float voiceRmsThreshold = 0.01f;

    // Runtime
    private Room _room;
    private bool _isConnected = false;
    private bool _micPrepared = false;
    private bool _isTalking = false;

    private MicrophoneSource _micSource;
    private LocalAudioTrack _micTrack;

    // For local mic debug (Unity Microphone API)
    private string _micDevice;
    private AudioClip _micClip;
    private float[] _micSampleBuffer = new float[1024];

    // Keep ref to stop on cleanup
    private readonly List<RtcAudioSource> _rtcAudioSources = new();

    // Remote playback objects
    private readonly Dictionary<string, (GameObject go, AudioSource audioSource)> _remoteAudio = new();

    private Coroutine _statsLoop;

    private void Start()
    {
        LogStatus("Init");
        EnsureAudioListener();

        if (autoConnectOnStart)
            Connect();
    }

    // -------------------- PUBLIC API --------------------

    public void Connect()
    {
        if (_isConnected)
        {
            LogStatus("Already connected");
            return;
        }
        StartCoroutine(CoConnect());
    }

    public void Hangup()
    {
        DisconnectAndCleanup();
    }

    public void OnHoldStart()
    {
        if (!_isConnected || _room == null)
        {
            LogStatus("HoldStart: Not connected yet");
            return;
        }
        if (!_micPrepared || _micSource == null)
        {
            LogStatus("HoldStart: Mic not prepared yet");
            return;
        }
        if (_isTalking) return;

        LogStatus("Hold START -> micSource.Start()");
        try
        {
            // Start LiveKit capture
            _micSource.Start();
            _isTalking = true;

            // Start local mic debug capture (Unity side) so we can see RMS/position
            StartLocalMicDebug();
        }
        catch (Exception e)
        {
            Debug.LogError("[LiveKitCoachClient] micSource.Start EXCEPTION: " + e);
            LogStatus("Mic start exception (see console)");
        }
    }

    public void OnHoldEnd()
    {
        if (!_isConnected || _room == null) return;
        if (!_micPrepared || _micSource == null) return;
        if (!_isTalking) return;

        LogStatus("Hold END -> micSource.Stop()");
        try
        {
            _micSource.Stop();
        }
        catch (Exception e)
        {
            Debug.LogError("[LiveKitCoachClient] micSource.Stop EXCEPTION: " + e);
        }

        StopLocalMicDebug();
        _isTalking = false;
    }

    public void SendPing()
    {
        // If your agent listens to data packets, it can respond with pong.
        SendData("{\"type\":\"ping\",\"t\":" + DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() + "}");
    }

    public void SendData(string jsonText)
    {
        if (_room == null) return;
        _room.LocalParticipant.PublishData(System.Text.Encoding.UTF8.GetBytes(jsonText));
        if (verboseLog) Debug.Log("[LiveKitCoachClient] Data SENT: " + jsonText);
    }

    // -------------------- CONNECT FLOW --------------------

    private IEnumerator CoConnect()
    {
#if PLATFORM_ANDROID
        if (!Permission.HasUserAuthorizedPermission(Permission.Microphone))
        {
            LogStatus("Requesting mic permission...");
            Permission.RequestUserPermission(Permission.Microphone);
            yield return new WaitForSeconds(0.25f);
        }
#endif
        if (tokenClient == null)
        {
            Debug.LogError("[LiveKitCoachClient] tokenClient is null. Assign it in Inspector.");
            LogStatus("ERROR: tokenClient null");
            yield break;
        }

        // Your device log shows: NewAudioSource: 2 48000
        ForceUnityAudioConfig(48000, AudioSpeakerMode.Stereo);

        // 1) Token
        LiveKitTokenResponse tok = null;
        string err = null;

        LogStatus("Fetching token...");
        yield return tokenClient.FetchToken(
            roomName,
            r => tok = r,
            e => err = e
        );

        if (!string.IsNullOrEmpty(err))
        {
            Debug.LogError("[LiveKitCoachClient] Token error: " + err);
            LogStatus("Token error (see console)");
            yield break;
        }

        if (tok == null || string.IsNullOrEmpty(tok.serverUrl) || string.IsNullOrEmpty(tok.participantToken))
        {
            Debug.LogError("[LiveKitCoachClient] Token invalid/null.");
            LogStatus("Token invalid");
            yield break;
        }

        if (verboseLog)
            Debug.Log($"[LiveKitCoachClient] Sandbox details: url={tok.serverUrl}, room={tok.roomName}, name={tok.participantName}");

        // 2) Connect
        LogStatus("Connecting to LiveKit...");
        _room = new Room();

        _room.TrackSubscribed += OnTrackSubscribed;
        _room.TrackUnsubscribed += OnTrackUnsubscribed;
        _room.DataReceived += OnDataReceived;

        var options = new RoomOptions();
        var connect = _room.Connect(tok.serverUrl, tok.participantToken, options);
        yield return connect;

        if (connect.IsError)
        {
            Debug.LogError("[LiveKitCoachClient] Connect failed (connect.IsError=true).");
            LogStatus("Connect failed");
            yield break;
        }

        _isConnected = true;
        LogStatus("Connected ✅");

        if (verboseLog)
        {
            Debug.Log("[LiveKitCoachClient] Connected. Room=" + _room.Name);
            Debug.Log("[LiveKitCoachClient] Local identity=" + _room.LocalParticipant.Identity);
        }

        // Start periodic stats log
        if (_statsLoop != null) StopCoroutine(_statsLoop);
        _statsLoop = StartCoroutine(CoStatsLoop());

        // 3) Prepare mic track
        yield return PrepareMicrophoneTrack();

        // Optional: send a ping to see if agent responds via DataReceived (if implemented)
        // SendPing();
    }

    private IEnumerator PrepareMicrophoneTrack()
    {
        if (_room == null)
        {
            Debug.LogError("[LiveKitCoachClient] PrepareMic: room null");
            LogStatus("PrepareMic: room null");
            yield break;
        }

        if (Microphone.devices == null || Microphone.devices.Length == 0)
        {
            Debug.LogError("[LiveKitCoachClient] No microphone devices found.");
            LogStatus("No microphone device");
            yield break;
        }

        _micDevice = Microphone.devices[0];

        int minFreq, maxFreq;
        Microphone.GetDeviceCaps(_micDevice, out minFreq, out maxFreq);

        if (verboseLog)
        {
            Debug.Log($"[LiveKitCoachClient] Mic={_micDevice} caps: min={minFreq}, max={maxFreq} (0 means any)");
            Debug.Log($"[LiveKitCoachClient] AudioSettings: outputSampleRate={AudioSettings.outputSampleRate}, speakerMode={AudioSettings.speakerMode}");
        }

        // IMPORTANT: do NOT add AudioSource here (avoid multiple AudioSources warning)
        var go = new GameObject("unity-mic-source");

        _micSource = new MicrophoneSource(_micDevice, go);
        _rtcAudioSources.Add(_micSource);

        _micTrack = LocalAudioTrack.CreateAudioTrack("unity-mic-track", _micSource, _room);

        var pubOpts = new TrackPublishOptions();
        pubOpts.AudioEncoding = new AudioEncoding();
        pubOpts.AudioEncoding.MaxBitrate = (ulong)micMaxBitrate;
        pubOpts.Source = TrackSource.SourceMicrophone;

        LogStatus("Publishing mic track...");
        var publish = _room.LocalParticipant.PublishTrack(_micTrack, pubOpts);
        yield return publish;

        if (publish.IsError)
        {
            Debug.LogError("[LiveKitCoachClient] Publish mic track failed (publish.IsError=true).");
            LogStatus("Publish mic failed");
            yield break;
        }

        _micPrepared = true;
        LogStatus("Mic prepared ✅ (Hold to talk)");

        if (verboseLog)
            Debug.Log("[LiveKitCoachClient] Mic track published (muted until Hold).");
    }

    // -------------------- REMOTE AUDIO (TTS playback) --------------------

    private void OnTrackSubscribed(IRemoteTrack track, RemoteTrackPublication publication, RemoteParticipant participant)
    {
        if (track is RemoteAudioTrack audioTrack)
        {
            Debug.Log($"[LiveKitCoachClient] Remote AUDIO subscribed from={participant.Identity} sid={audioTrack.Sid}");

            var sid = audioTrack.Sid;

            // Avoid duplicates
            if (_remoteAudio.TryGetValue(sid, out var existing))
            {
                if (existing.audioSource != null) existing.audioSource.Stop();
                if (existing.go != null) Destroy(existing.go);
                _remoteAudio.Remove(sid);
            }

            var go = new GameObject("remote-audio-" + sid);
            var audioSource = go.AddComponent<AudioSource>();

            // Make sure audible
            audioSource.playOnAwake = true;
            audioSource.loop = false;
            audioSource.mute = false;
            audioSource.volume = 1f;
            audioSource.spatialBlend = 0f; // 2D
            audioSource.rolloffMode = AudioRolloffMode.Linear;

            // Bind track -> speaker
            var stream = new AudioStream(audioTrack, audioSource);

            _remoteAudio[sid] = (go, audioSource);

            LogStatus("Remote audio subscribed ✅ (agent is speaking?)");
            Debug.Log("[LiveKitCoachClient] Remote AudioSource created (2D, vol=1).");
        }
    }

    private void OnTrackUnsubscribed(IRemoteTrack track, RemoteTrackPublication publication, RemoteParticipant participant)
    {
        if (track is RemoteAudioTrack audioTrack)
        {
            var sid = audioTrack.Sid;
            Debug.Log($"[LiveKitCoachClient] Remote AUDIO unsubscribed sid={sid}");

            if (_remoteAudio.TryGetValue(sid, out var tuple))
            {
                if (tuple.audioSource != null) tuple.audioSource.Stop();
                if (tuple.go != null) Destroy(tuple.go);
                _remoteAudio.Remove(sid);
            }
        }
    }

    // -------------------- DATA --------------------

    private void OnDataReceived(byte[] data, Participant participant, DataPacketKind kind, string topic)
    {
        var str = System.Text.Encoding.UTF8.GetString(data);
        Debug.Log("[LiveKitCoachClient] Data RECEIVED from " + participant.Identity + ": " + str);
        LogStatus("Data received ✅ (agent alive)");
    }

    // -------------------- STATS LOOP --------------------

    private IEnumerator CoStatsLoop()
    {
        while (_room != null)
        {
            try
            {
                // Remote participants count
                string s =
                    $"State: connected={_isConnected}, " +
                    $"micPrepared={_micPrepared}, " +
                    $"talking={_isTalking}, " +
                    $"remoteAudioTracks={_remoteAudio.Count}";
                if (verboseLog) Debug.Log("[LiveKitCoachClient] " + s);
            }
            catch { }

            yield return new WaitForSeconds(statsLogIntervalSec);
        }
    }

    // -------------------- LOCAL MIC DEBUG (RMS + POSITION) --------------------

    private void StartLocalMicDebug()
    {
        if (string.IsNullOrEmpty(_micDevice)) return;

        // Start Unity mic (for debug only) at 48k stereo-ish pipeline (Unity mic is typically mono but clip API works)
        // Use looping 1 sec clip.
        int freq = 48000;
        _micClip = Microphone.Start(_micDevice, true, 1, freq);

        if (verboseLog)
            Debug.Log("[LiveKitCoachClient] LocalMicDebug START: device=" + _micDevice + " freq=" + freq);
    }

    private void StopLocalMicDebug()
    {
        if (string.IsNullOrEmpty(_micDevice)) return;

        try
        {
            if (Microphone.IsRecording(_micDevice))
                Microphone.End(_micDevice);
        }
        catch { }

        _micClip = null;

        if (verboseLog)
            Debug.Log("[LiveKitCoachClient] LocalMicDebug STOP");
    }

    private void Update()
    {
        // If holding, compute RMS every frame (cheap enough with 1024 samples)
        if (_isTalking && _micClip != null)
        {
            int pos = 0;
            try { pos = Microphone.GetPosition(_micDevice); } catch { }

            if (pos > 0)
            {
                // Read last 1024 samples (wrap safely)
                int sampleCount = _micSampleBuffer.Length;
                int start = Mathf.Max(0, pos - sampleCount);
                try
                {
                    _micClip.GetData(_micSampleBuffer, start);
                    float rms = ComputeRms(_micSampleBuffer);
                    if (verboseLog && Time.frameCount % 20 == 0)
                    {
                        Debug.Log($"[LiveKitCoachClient] MicDebug: pos={pos}, rms={rms:0.0000}, voice={(rms >= voiceRmsThreshold)}");
                    }
                }
                catch { /* ignore */ }
            }
        }
    }

    private float ComputeRms(float[] samples)
    {
        double sum = 0;
        for (int i = 0; i < samples.Length; i++)
        {
            float v = samples[i];
            sum += v * v;
        }
        return (float)Math.Sqrt(sum / samples.Length);
    }

    // -------------------- CLEANUP --------------------

    private void DisconnectAndCleanup()
    {
        LogStatus("Disconnecting...");

        StopLocalMicDebug();

        if (_micSource != null && _isTalking)
        {
            try { _micSource.Stop(); } catch { }
            _isTalking = false;
        }

        if (_room != null)
        {
            _room.Disconnect();
            _room = null;
        }

        if (_statsLoop != null)
        {
            StopCoroutine(_statsLoop);
            _statsLoop = null;
        }

        foreach (var s in _rtcAudioSources)
        {
            try { s.Stop(); } catch { }
        }
        _rtcAudioSources.Clear();

        foreach (var kv in _remoteAudio)
        {
            if (kv.Value.audioSource != null) kv.Value.audioSource.Stop();
            if (kv.Value.go != null) Destroy(kv.Value.go);
        }
        _remoteAudio.Clear();

        _isConnected = false;
        _micPrepared = false;

        LogStatus("Disconnected");
    }

    private void OnDestroy()
    {
        DisconnectAndCleanup();
    }

    // -------------------- HELPERS --------------------

    private void LogStatus(string msg)
    {
        Debug.Log("[LiveKitCoachClient] " + msg);
        if (statusText != null) statusText.text = msg;
    }

    private void EnsureAudioListener()
    {
        var listener = FindObjectOfType<AudioListener>();
        if (listener != null) return;

        var cam = Camera.main;
        if (cam != null)
        {
            cam.gameObject.AddComponent<AudioListener>();
            Debug.Log("[LiveKitCoachClient] Added AudioListener to Main Camera.");
        }
        else
        {
            Debug.LogWarning("[LiveKitCoachClient] No AudioListener found and no MainCamera. Remote audio may be silent.");
        }
    }

    private void ForceUnityAudioConfig(int sampleRate, AudioSpeakerMode mode)
    {
        var cfg = AudioSettings.GetConfiguration();
        cfg.sampleRate = sampleRate;
        cfg.speakerMode = mode;
        AudioSettings.Reset(cfg);

        AudioSettings.GetDSPBufferSize(out var dspBuf, out var dspNum);
        Debug.Log($"[LiveKitCoachClient] AudioSettings forced => sampleRate={AudioSettings.outputSampleRate}, speakerMode={AudioSettings.speakerMode}, dspBuf={dspBuf}, dspNum={dspNum}");
    }
}
Editor is loading...
Leave a Comment