Untitled

# res://lib/dialogue/voice_gen_stream.gd
extends Node

signal opened(meta: Dictionary)
signal audio_chunk(pcm16: PackedByteArray, alignment: Dictionary, is_final: bool, meta: Dictionary)
signal final(meta: Dictionary)
signal error(msg: String)

var _ws: WebSocketPeer
var _connected := false
var _pending_meta: Dictionary = {}

var model_id := "eleven_turbo_v2_5"
var voice_id := "XBtMryMTdZsPZJXSeJyG"
var stability := 0.5
var similarity_boost := 0.5
var use_speaker_boost := false

const OUTPUT_FORMAT := "pcm_16000"

# runs when this Node is initialized
func _ready() -> void:
	_ws = WebSocketPeer.new()

# called once to open websocket stream
func open_stream(character_id: String, utterance_id: String, sync_alignment: bool = true) -> void:
	if Settings.eleven_api_key == "":
		push_error("error", "Missing ElevenLabs API key"); return

	# inactivity_timeout lets you idle longer if LLM pauses (default ~20s)
	var qs := "model_id=%s&output_format=%s" % [
		model_id, OUTPUT_FORMAT
	]
	var url := "wss://api.elevenlabs.io/v1/text-to-speech/%s/stream-input?%s" % [voice_id, qs]

	_pending_meta = {"character_id": character_id, "utterance_id": utterance_id}

	var err: int = _ws.connect_to_url(url)
	if err != OK:
		push_error("error", "WS connect_to_url failed: %s" % err)
		return
	_connected = true

# called every frame of the application
func _process(_dt: float) -> void:
	if not _connected:
		return
	
	_ws.poll()
	
	# Drain incoming packets
	while _ws.get_ready_state() == WebSocketPeer.STATE_OPEN and _ws.get_available_packet_count() > 0:
		var raw: PackedByteArray = _ws.get_packet()
		var s: String = raw.get_string_from_utf8()
		var parsed: Dictionary = JSON.parse_string(s)
		# Just to silence gdscript's type warnings
		var d: Dictionary = parsed

		# Log and emit server errors instead of silently closing later
		if d.has("error") or d.has("message") and String(d.message).begins_with("error"):
			print("PACKET EMBEDDED ERROR:", s)
			push_error("error", "Server error: %s" % s)
			continue
		
		# this never happens in my case because websocket DISCONNECTS after 1-2 generations
		var is_final: bool = d.has("isFinal") and d["isFinal"]
		
		# streams the audio to my audio player defined in audio.gd
		if d.has("audio"):
			var pcm16: PackedByteArray = Marshalls.base64_to_raw(String(d.audio))
			var align: Dictionary = {}
			if d.has("normalizedAlignment"):
				align = d.normalizedAlignment
			elif d.has("alignment"):
				align = d.alignment
			emit_signal("audio_chunk", pcm16, align, is_final, _pending_meta)
			if is_final:
				print("is final audio chunk")
				emit_signal("final", _pending_meta)
		elif is_final:
			emit_signal("final", _pending_meta)

	# Handle closure and report reason, code is always -1
	if _ws.get_ready_state() == WebSocketPeer.STATE_CLOSED:
		var code := _ws.get_close_code()
		var reason := _ws.get_close_reason()
		print("WEB SOCKET CLOSED code:", code, " reason:", reason)
		if _connected:
			_connected = false
			push_error("error", "WS closed: %s" % _ws.get_close_reason())

func send_text_fragment(text: String, utterance_i: int) -> void:
	if text == "" or _ws == null or _ws.get_ready_state() != WebSocketPeer.STATE_OPEN:
		return
	# get rid of empty space on the left
	text = text.lstrip(" \t\n\r")
	var msg: Dictionary = {"text": text, "flush": true}
	# if first message, send voice_settings and xi_api_key
	if utterance_i == 0:
		var init_msg := {
			"text": text,
			"voice_settings": {
				"stability": stability,
				"similarity_boost": similarity_boost,
				"use_speaker_boost": use_speaker_boost
			},
			"xi_api_key": Settings.eleven_api_key,
			"flush": true
		}
		print("Sending: " + JSON.stringify(init_msg))
		_ws.send_text(JSON.stringify(init_msg))
	else:
		print("Sending: " + JSON.stringify(msg))
		_ws.send_text(JSON.stringify(msg))

# never happens because websocket disconnects after 2-3 generations
func close_stream() -> void:
	print("closing stream")
	# This is how we tell ElevenLabs to close stream and flush buffer
	if _ws.get_ready_state() == WebSocketPeer.STATE_OPEN:
		_ws.send_text(JSON.stringify({"text": ""})) # required by ElevenLabs