Untitled
Anonymous
plain_text
01/19/2026 3:11 PM
22.1 KB
10
Indexable
using System;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.Android;
using LiveKit;
using LiveKit.Proto;
using RoomOptions = LiveKit.RoomOptions;
public class LiveKitCoachClient : MonoBehaviour
{
[Header("Room")]
public string roomName = "lobby";
[Header("Token Provider (Sandbox)")]
public LiveKitTokenClient tokenClient;
[Header("Behavior")]
public bool autoConnectOnStart = true;
[Header("Audio Publish Settings")]
public int micMaxBitrate = 64000;
[Header("Optional UI")]
public TMPro.TMP_Text statusText;
[Header("Debug")]
public bool verboseLog = true;
public float statsLogIntervalSec = 1.0f;
[Tooltip("Mic RMS threshold to consider 'voice present'")]
public float voiceRmsThreshold = 0.01f;
// Runtime
private Room _room;
private bool _isConnected = false;
private bool _micPrepared = false;
private bool _isTalking = false;
private MicrophoneSource _micSource;
private LocalAudioTrack _micTrack;
// For local mic debug (Unity Microphone API)
private string _micDevice;
private AudioClip _micClip;
private float[] _micSampleBuffer = new float[1024];
// Keep ref to stop on cleanup
private readonly List<RtcAudioSource> _rtcAudioSources = new();
// Remote playback objects
private readonly Dictionary<string, (GameObject go, AudioSource audioSource)> _remoteAudio = new();
private Coroutine _statsLoop;
private void Start()
{
LogStatus("Init");
EnsureAudioListener();
if (autoConnectOnStart)
Connect();
}
// -------------------- PUBLIC API --------------------
public void Connect()
{
if (_isConnected)
{
LogStatus("Already connected");
return;
}
StartCoroutine(CoConnect());
}
public void Hangup()
{
DisconnectAndCleanup();
}
public void OnHoldStart()
{
if (!_isConnected || _room == null)
{
LogStatus("HoldStart: Not connected yet");
return;
}
if (!_micPrepared || _micSource == null)
{
LogStatus("HoldStart: Mic not prepared yet");
return;
}
if (_isTalking) return;
LogStatus("Hold START -> micSource.Start()");
try
{
// Start LiveKit capture
_micSource.Start();
_isTalking = true;
// Start local mic debug capture (Unity side) so we can see RMS/position
StartLocalMicDebug();
}
catch (Exception e)
{
Debug.LogError("[LiveKitCoachClient] micSource.Start EXCEPTION: " + e);
LogStatus("Mic start exception (see console)");
}
}
public void OnHoldEnd()
{
if (!_isConnected || _room == null) return;
if (!_micPrepared || _micSource == null) return;
if (!_isTalking) return;
LogStatus("Hold END -> micSource.Stop()");
try
{
_micSource.Stop();
}
catch (Exception e)
{
Debug.LogError("[LiveKitCoachClient] micSource.Stop EXCEPTION: " + e);
}
StopLocalMicDebug();
_isTalking = false;
}
public void SendPing()
{
// If your agent listens to data packets, it can respond with pong.
SendData("{\"type\":\"ping\",\"t\":" + DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() + "}");
}
public void SendData(string jsonText)
{
if (_room == null) return;
_room.LocalParticipant.PublishData(System.Text.Encoding.UTF8.GetBytes(jsonText));
if (verboseLog) Debug.Log("[LiveKitCoachClient] Data SENT: " + jsonText);
}
// -------------------- CONNECT FLOW --------------------
private IEnumerator CoConnect()
{
#if PLATFORM_ANDROID
if (!Permission.HasUserAuthorizedPermission(Permission.Microphone))
{
LogStatus("Requesting mic permission...");
Permission.RequestUserPermission(Permission.Microphone);
yield return new WaitForSeconds(0.25f);
}
#endif
if (tokenClient == null)
{
Debug.LogError("[LiveKitCoachClient] tokenClient is null. Assign it in Inspector.");
LogStatus("ERROR: tokenClient null");
yield break;
}
// Your device log shows: NewAudioSource: 2 48000
ForceUnityAudioConfig(48000, AudioSpeakerMode.Stereo);
// 1) Token
LiveKitTokenResponse tok = null;
string err = null;
LogStatus("Fetching token...");
yield return tokenClient.FetchToken(
roomName,
r => tok = r,
e => err = e
);
if (!string.IsNullOrEmpty(err))
{
Debug.LogError("[LiveKitCoachClient] Token error: " + err);
LogStatus("Token error (see console)");
yield break;
}
if (tok == null || string.IsNullOrEmpty(tok.serverUrl) || string.IsNullOrEmpty(tok.participantToken))
{
Debug.LogError("[LiveKitCoachClient] Token invalid/null.");
LogStatus("Token invalid");
yield break;
}
if (verboseLog)
Debug.Log($"[LiveKitCoachClient] Sandbox details: url={tok.serverUrl}, room={tok.roomName}, name={tok.participantName}");
// 2) Connect
LogStatus("Connecting to LiveKit...");
_room = new Room();
_room.TrackSubscribed += OnTrackSubscribed;
_room.TrackUnsubscribed += OnTrackUnsubscribed;
_room.DataReceived += OnDataReceived;
var options = new RoomOptions();
var connect = _room.Connect(tok.serverUrl, tok.participantToken, options);
yield return connect;
if (connect.IsError)
{
Debug.LogError("[LiveKitCoachClient] Connect failed (connect.IsError=true).");
LogStatus("Connect failed");
yield break;
}
_isConnected = true;
LogStatus("Connected ✅");
if (verboseLog)
{
Debug.Log("[LiveKitCoachClient] Connected. Room=" + _room.Name);
Debug.Log("[LiveKitCoachClient] Local identity=" + _room.LocalParticipant.Identity);
}
// Start periodic stats log
if (_statsLoop != null) StopCoroutine(_statsLoop);
_statsLoop = StartCoroutine(CoStatsLoop());
// 3) Prepare mic track
yield return PrepareMicrophoneTrack();
// Optional: send a ping to see if agent responds via DataReceived (if implemented)
// SendPing();
}
private IEnumerator PrepareMicrophoneTrack()
{
if (_room == null)
{
Debug.LogError("[LiveKitCoachClient] PrepareMic: room null");
LogStatus("PrepareMic: room null");
yield break;
}
if (Microphone.devices == null || Microphone.devices.Length == 0)
{
Debug.LogError("[LiveKitCoachClient] No microphone devices found.");
LogStatus("No microphone device");
yield break;
}
_micDevice = Microphone.devices[0];
int minFreq, maxFreq;
Microphone.GetDeviceCaps(_micDevice, out minFreq, out maxFreq);
if (verboseLog)
{
Debug.Log($"[LiveKitCoachClient] Mic={_micDevice} caps: min={minFreq}, max={maxFreq} (0 means any)");
Debug.Log($"[LiveKitCoachClient] AudioSettings: outputSampleRate={AudioSettings.outputSampleRate}, speakerMode={AudioSettings.speakerMode}");
}
// IMPORTANT: do NOT add AudioSource here (avoid multiple AudioSources warning)
var go = new GameObject("unity-mic-source");
_micSource = new MicrophoneSource(_micDevice, go);
_rtcAudioSources.Add(_micSource);
_micTrack = LocalAudioTrack.CreateAudioTrack("unity-mic-track", _micSource, _room);
var pubOpts = new TrackPublishOptions();
pubOpts.AudioEncoding = new AudioEncoding();
pubOpts.AudioEncoding.MaxBitrate = (ulong)micMaxBitrate;
pubOpts.Source = TrackSource.SourceMicrophone;
LogStatus("Publishing mic track...");
var publish = _room.LocalParticipant.PublishTrack(_micTrack, pubOpts);
yield return publish;
if (publish.IsError)
{
Debug.LogError("[LiveKitCoachClient] Publish mic track failed (publish.IsError=true).");
LogStatus("Publish mic failed");
yield break;
}
_micPrepared = true;
LogStatus("Mic prepared ✅ (Hold to talk)");
if (verboseLog)
Debug.Log("[LiveKitCoachClient] Mic track published (muted until Hold).");
}
// -------------------- REMOTE AUDIO (TTS playback) --------------------
private void OnTrackSubscribed(IRemoteTrack track, RemoteTrackPublication publication, RemoteParticipant participant)
{
if (track is RemoteAudioTrack audioTrack)
{
Debug.Log($"[LiveKitCoachClient] Remote AUDIO subscribed from={participant.Identity} sid={audioTrack.Sid}");
var sid = audioTrack.Sid;
// Avoid duplicates
if (_remoteAudio.TryGetValue(sid, out var existing))
{
if (existing.audioSource != null) existing.audioSource.Stop();
if (existing.go != null) Destroy(existing.go);
_remoteAudio.Remove(sid);
}
var go = new GameObject("remote-audio-" + sid);
var audioSource = go.AddComponent<AudioSource>();
// Make sure audible
audioSource.playOnAwake = true;
audioSource.loop = false;
audioSource.mute = false;
audioSource.volume = 1f;
audioSource.spatialBlend = 0f; // 2D
audioSource.rolloffMode = AudioRolloffMode.Linear;
// Bind track -> speaker
var stream = new AudioStream(audioTrack, audioSource);
_remoteAudio[sid] = (go, audioSource);
LogStatus("Remote audio subscribed ✅ (agent is speaking?)");
Debug.Log("[LiveKitCoachClient] Remote AudioSource created (2D, vol=1).");
}
}
private void OnTrackUnsubscribed(IRemoteTrack track, RemoteTrackPublication publication, RemoteParticipant participant)
{
if (track is RemoteAudioTrack audioTrack)
{
var sid = audioTrack.Sid;
Debug.Log($"[LiveKitCoachClient] Remote AUDIO unsubscribed sid={sid}");
if (_remoteAudio.TryGetValue(sid, out var tuple))
{
if (tuple.audioSource != null) tuple.audioSource.Stop();
if (tuple.go != null) Destroy(tuple.go);
_remoteAudio.Remove(sid);
}
}
}
// -------------------- DATA --------------------
private void OnDataReceived(byte[] data, Participant participant, DataPacketKind kind, string topic)
{
var str = System.Text.Encoding.UTF8.GetString(data);
Debug.Log("[LiveKitCoachClient] Data RECEIVED from " + participant.Identity + ": " + str);
LogStatus("Data received ✅ (agent alive)");
}
// -------------------- STATS LOOP --------------------
private IEnumerator CoStatsLoop()
{
while (_room != null)
{
try
{
// Remote participants count
string s =
$"State: connected={_isConnected}, " +
$"micPrepared={_micPrepared}, " +
$"talking={_isTalking}, " +
$"remoteAudioTracks={_remoteAudio.Count}";
if (verboseLog) Debug.Log("[LiveKitCoachClient] " + s);
}
catch { }
yield return new WaitForSeconds(statsLogIntervalSec);
}
}
// -------------------- LOCAL MIC DEBUG (RMS + POSITION) --------------------
private void StartLocalMicDebug()
{
if (string.IsNullOrEmpty(_micDevice)) return;
// Start Unity mic (for debug only) at 48k stereo-ish pipeline (Unity mic is typically mono but clip API works)
// Use looping 1 sec clip.
int freq = 48000;
_micClip = Microphone.Start(_micDevice, true, 1, freq);
if (verboseLog)
Debug.Log("[LiveKitCoachClient] LocalMicDebug START: device=" + _micDevice + " freq=" + freq);
}
private void StopLocalMicDebug()
{
if (string.IsNullOrEmpty(_micDevice)) return;
try
{
if (Microphone.IsRecording(_micDevice))
Microphone.End(_micDevice);
}
catch { }
_micClip = null;
if (verboseLog)
Debug.Log("[LiveKitCoachClient] LocalMicDebug STOP");
}
private void Update()
{
// If holding, compute RMS every frame (cheap enough with 1024 samples)
if (_isTalking && _micClip != null)
{
int pos = 0;
try { pos = Microphone.GetPosition(_micDevice); } catch { }
if (pos > 0)
{
// Read last 1024 samples (wrap safely)
int sampleCount = _micSampleBuffer.Length;
int start = Mathf.Max(0, pos - sampleCount);
try
{
_micClip.GetData(_micSampleBuffer, start);
float rms = ComputeRms(_micSampleBuffer);
if (verboseLog && Time.frameCount % 20 == 0)
{
Debug.Log($"[LiveKitCoachClient] MicDebug: pos={pos}, rms={rms:0.0000}, voice={(rms >= voiceRmsThreshold)}");
}
}
catch { /* ignore */ }
}
}
}
private float ComputeRms(float[] samples)
{
double sum = 0;
for (int i = 0; i < samples.Length; i++)
{
float v = samples[i];
sum += v * v;
}
return (float)Math.Sqrt(sum / samples.Length);
}
// -------------------- CLEANUP --------------------
private void DisconnectAndCleanup()
{
LogStatus("Disconnecting...");
StopLocalMicDebug();
if (_micSource != null && _isTalking)
{
try { _micSource.Stop(); } catch { }
_isTalking = false;
}
if (_room != null)
{
_room.Disconnect();
_room = null;
}
if (_statsLoop != null)
{
StopCoroutine(_statsLoop);
_statsLoop = null;
}
foreach (var s in _rtcAudioSources)
{
try { s.Stop(); } catch { }
}
_rtcAudioSources.Clear();
foreach (var kv in _remoteAudio)
{
if (kv.Value.audioSource != null) kv.Value.audioSource.Stop();
if (kv.Value.go != null) Destroy(kv.Value.go);
}
_remoteAudio.Clear();
_isConnected = false;
_micPrepared = false;
LogStatus("Disconnected");
}
private void OnDestroy()
{
DisconnectAndCleanup();
}
// -------------------- HELPERS --------------------
private void LogStatus(string msg)
{
Debug.Log("[LiveKitCoachClient] " + msg);
if (statusText != null) statusText.text = msg;
}
private void EnsureAudioListener()
{
var listener = FindObjectOfType<AudioListener>();
if (listener != null) return;
var cam = Camera.main;
if (cam != null)
{
cam.gameObject.AddComponent<AudioListener>();
Debug.Log("[LiveKitCoachClient] Added AudioListener to Main Camera.");
}
else
{
Debug.LogWarning("[LiveKitCoachClient] No AudioListener found and no MainCamera. Remote audio may be silent.");
}
}
private void ForceUnityAudioConfig(int sampleRate, AudioSpeakerMode mode)
{
var cfg = AudioSettings.GetConfiguration();
cfg.sampleRate = sampleRate;
cfg.speakerMode = mode;
AudioSettings.Reset(cfg);
AudioSettings.GetDSPBufferSize(out var dspBuf, out var dspNum);
Debug.Log($"[LiveKitCoachClient] AudioSettings forced => sampleRate={AudioSettings.outputSampleRate}, speakerMode={AudioSettings.speakerMode}, dspBuf={dspBuf}, dspNum={dspNum}");
}
}
Editor is loading...
Leave a Comment