Vendor OpenClaw source as Adolf fork baseline
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Adolf is a fork/vendored clone of github.com/openclaw/openclaw (v2026.6.11), free to diverge. Tree copied sans upstream .git; upstream remote added for future syncs. Node pinned to 24 (.nvmrc); engines already require >=22.19. Preserves docs/ARCHITECTURE.md. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LeqyaxJF2nbRXJtae2kNB2
This commit is contained in:
971
apps/ios/Sources/Voice/RealtimeTalkRelaySession.swift
Normal file
971
apps/ios/Sources/Voice/RealtimeTalkRelaySession.swift
Normal file
@@ -0,0 +1,971 @@
|
||||
import AVFAudio
|
||||
import Foundation
|
||||
import OpenClawChatUI
|
||||
import OpenClawKit
|
||||
import OpenClawProtocol
|
||||
import OSLog
|
||||
|
||||
private func makeRealtimeAudioTapBlock(
|
||||
inputSampleRate: Double,
|
||||
targetSampleRate: Double,
|
||||
onAudio: @escaping (Data, Double, Float) -> Void) -> AVAudioNodeTapBlock
|
||||
{
|
||||
{ buffer, _ in
|
||||
// This callback runs on Core Audio's realtime queue, not MainActor.
|
||||
let encoded = RealtimeTalkRelaySession.encodePCM16(
|
||||
buffer: buffer,
|
||||
inputSampleRate: inputSampleRate,
|
||||
targetSampleRate: targetSampleRate)
|
||||
guard !encoded.isEmpty else { return }
|
||||
let timestampMs = (ProcessInfo.processInfo.systemUptime * 1000).rounded()
|
||||
let rms = RealtimeTalkRelaySession.rmsLevel(buffer: buffer)
|
||||
onAudio(encoded, timestampMs, rms)
|
||||
}
|
||||
}
|
||||
|
||||
private actor RealtimeAudioSender {
|
||||
private let gateway: GatewayNodeSession
|
||||
private var relaySessionId: String?
|
||||
private var pendingSends = 0
|
||||
private let maxPendingSends = 4
|
||||
|
||||
init(gateway: GatewayNodeSession, relaySessionId: String) {
|
||||
self.gateway = gateway
|
||||
self.relaySessionId = relaySessionId
|
||||
}
|
||||
|
||||
func close() {
|
||||
self.relaySessionId = nil
|
||||
}
|
||||
|
||||
func send(_ data: Data, timestampMs: Double) async -> String? {
|
||||
guard let relaySessionId else { return nil }
|
||||
guard self.pendingSends < self.maxPendingSends else { return nil }
|
||||
self.pendingSends += 1
|
||||
defer { self.pendingSends -= 1 }
|
||||
let payload: [String: Any] = [
|
||||
"sessionId": relaySessionId,
|
||||
"audioBase64": data.base64EncodedString(),
|
||||
"timestamp": timestampMs,
|
||||
]
|
||||
do {
|
||||
_ = try await Self.requestJSON(
|
||||
gateway: self.gateway,
|
||||
method: "talk.session.appendAudio",
|
||||
payload: payload,
|
||||
decodeAs: TalkSessionOkResult.self,
|
||||
timeoutSeconds: 8)
|
||||
return nil
|
||||
} catch {
|
||||
return error.localizedDescription
|
||||
}
|
||||
}
|
||||
|
||||
private static func requestJSON<T: Decodable>(
|
||||
gateway: GatewayNodeSession,
|
||||
method: String,
|
||||
payload: [String: Any],
|
||||
decodeAs type: T.Type,
|
||||
timeoutSeconds: Int) async throws -> T
|
||||
{
|
||||
let data = try JSONSerialization.data(withJSONObject: payload)
|
||||
guard let json = String(data: data, encoding: .utf8) else {
|
||||
throw NSError(domain: "RealtimeTalkRelay", code: 4, userInfo: [
|
||||
NSLocalizedDescriptionKey: "Failed to encode \(method) payload",
|
||||
])
|
||||
}
|
||||
let response = try await gateway.request(
|
||||
method: method,
|
||||
paramsJSON: json,
|
||||
timeoutSeconds: timeoutSeconds)
|
||||
return try JSONDecoder().decode(type, from: response)
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
final class RealtimeTalkRelaySession {
|
||||
private static let agentControlToolName = "openclaw_agent_control"
|
||||
|
||||
struct Options {
|
||||
let sessionKey: String
|
||||
let provider: String?
|
||||
let model: String?
|
||||
let voice: String?
|
||||
}
|
||||
|
||||
private struct ToolCallStartResponse: Decodable {
|
||||
let runId: String?
|
||||
let idempotencyKey: String?
|
||||
}
|
||||
|
||||
private struct ChatCompletionResult {
|
||||
let text: String?
|
||||
let failed: Bool
|
||||
}
|
||||
|
||||
private enum StartupWaitResult {
|
||||
case ready
|
||||
case failed(TalkRuntimeIssue)
|
||||
case cancelled
|
||||
}
|
||||
|
||||
private nonisolated static let expectedInputEncoding = "pcm16"
|
||||
private nonisolated static let expectedOutputEncoding = "pcm16"
|
||||
private nonisolated static let defaultSampleRateHz = 24000
|
||||
private nonisolated static let audioFrameBufferSize: AVAudioFrameCount = 2048
|
||||
private nonisolated static let bargeInRmsThreshold: Float = 0.08
|
||||
private nonisolated static let bargeInCooldownMs: Double = 900
|
||||
private nonisolated static let minOutputBeforeBargeInMs: Double = 250
|
||||
private nonisolated static let startupReadyTimeoutSeconds = 12
|
||||
|
||||
private let gateway: GatewayNodeSession
|
||||
private let options: Options
|
||||
private let pcmPlayer: PCMStreamingAudioPlaying
|
||||
private let logger = Logger(subsystem: "ai.openclawfoundation.app", category: "RealtimeTalkRelay")
|
||||
private let onStatus: (String) -> Void
|
||||
private let onIssue: (TalkRuntimeIssue) -> Void
|
||||
private let onSpeakingChanged: (Bool) -> Void
|
||||
|
||||
private let audioEngine = AVAudioEngine()
|
||||
private var relaySessionId: String?
|
||||
private var hasReceivedReady = false
|
||||
private var hasReceivedFailure = false
|
||||
private var startupIssue: TalkRuntimeIssue?
|
||||
private var startupWaiter: CheckedContinuation<StartupWaitResult, Never>?
|
||||
private var pendingPreRelayEvents: [EventFrame] = []
|
||||
private var inputSampleRateHz = Double(RealtimeTalkRelaySession.defaultSampleRateHz)
|
||||
private var outputSampleRateHz = Double(RealtimeTalkRelaySession.defaultSampleRateHz)
|
||||
private var eventTask: Task<Void, Never>?
|
||||
private var outputTask: Task<Void, Never>?
|
||||
private var outputContinuation: AsyncThrowingStream<Data, Error>.Continuation?
|
||||
private var outputIdleTask: Task<Void, Never>?
|
||||
private var outputSessionId = 0
|
||||
private var pendingOutputChunks: [Data] = []
|
||||
private var pendingOutputDone = false
|
||||
private var audioSender: RealtimeAudioSender?
|
||||
private var isClosed = false
|
||||
private var isOutputPlaying = false
|
||||
private var outputStartedAtMs: Double?
|
||||
private var outputPlaybackExpectedEndMs: Double = 0
|
||||
private var lastBargeInAtMs: Double = 0
|
||||
private var micLogFrameCount = 0
|
||||
private var micLogByteCount = 0
|
||||
private var micLogMaxRms: Float = 0
|
||||
private var lastMicLogAtMs: Double = 0
|
||||
private var suppressedEchoFrameCount = 0
|
||||
private var suppressedEchoByteCount = 0
|
||||
private var suppressedEchoMaxRms: Float = 0
|
||||
private var lastSuppressedEchoLogAtMs: Double = 0
|
||||
private var outputAudioChunkCount = 0
|
||||
private var outputAudioByteCount = 0
|
||||
|
||||
init(
|
||||
gateway: GatewayNodeSession,
|
||||
options: Options,
|
||||
pcmPlayer: PCMStreamingAudioPlaying,
|
||||
onStatus: @escaping (String) -> Void,
|
||||
onIssue: @escaping (TalkRuntimeIssue) -> Void = { _ in },
|
||||
onSpeakingChanged: @escaping (Bool) -> Void)
|
||||
{
|
||||
self.gateway = gateway
|
||||
self.options = options
|
||||
self.pcmPlayer = pcmPlayer
|
||||
self.onStatus = onStatus
|
||||
self.onIssue = onIssue
|
||||
self.onSpeakingChanged = onSpeakingChanged
|
||||
}
|
||||
|
||||
func start() async throws {
|
||||
self.isClosed = false
|
||||
self.hasReceivedReady = false
|
||||
self.hasReceivedFailure = false
|
||||
self.startupIssue = nil
|
||||
self.startupWaiter = nil
|
||||
self.pendingPreRelayEvents.removeAll()
|
||||
self.onStatus("Connecting realtime…")
|
||||
let eventStream = await self.gateway.subscribeServerEvents(bufferingNewest: 200)
|
||||
self.startEventPump(stream: eventStream)
|
||||
do {
|
||||
let result = try await self.createRelaySession()
|
||||
guard let relaySessionId = result.relaysessionid?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!relaySessionId.isEmpty
|
||||
else {
|
||||
throw NSError(domain: "RealtimeTalkRelay", code: 1, userInfo: [
|
||||
NSLocalizedDescriptionKey: "Gateway did not return a realtime relay session",
|
||||
])
|
||||
}
|
||||
self.relaySessionId = relaySessionId
|
||||
self.audioSender = RealtimeAudioSender(gateway: self.gateway, relaySessionId: relaySessionId)
|
||||
self.configureAudioContract(result.audio)
|
||||
try self.startMicrophonePump()
|
||||
self.onStatus("Waiting for realtime…")
|
||||
await self.drainPendingPreRelayEvents()
|
||||
switch await self.waitForStartupResult(timeoutSeconds: Self.startupReadyTimeoutSeconds) {
|
||||
case .ready:
|
||||
return
|
||||
case let .failed(issue):
|
||||
self.close(sendClose: true)
|
||||
throw NSError(domain: "RealtimeTalkRelay", code: 6, userInfo: [
|
||||
NSLocalizedDescriptionKey: issue.displayMessage,
|
||||
])
|
||||
case .cancelled:
|
||||
return
|
||||
}
|
||||
} catch {
|
||||
let createdRelaySessionId = self.relaySessionId
|
||||
self.close(sendClose: false)
|
||||
if let createdRelaySessionId {
|
||||
await Self.closeRelaySession(gateway: self.gateway, relaySessionId: createdRelaySessionId)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
func stop() {
|
||||
self.close(sendClose: true)
|
||||
}
|
||||
|
||||
private func close(sendClose: Bool) {
|
||||
guard !self.isClosed else { return }
|
||||
self.isClosed = true
|
||||
self.finishStartupWait(.cancelled)
|
||||
self.stopMicrophonePump()
|
||||
self.eventTask?.cancel()
|
||||
self.eventTask = nil
|
||||
let audioSender = self.audioSender
|
||||
self.audioSender = nil
|
||||
Task { await audioSender?.close() }
|
||||
self.stopOutputPlayback()
|
||||
if sendClose, let relaySessionId = self.relaySessionId {
|
||||
Task { [gateway] in
|
||||
await Self.closeRelaySession(gateway: gateway, relaySessionId: relaySessionId)
|
||||
}
|
||||
}
|
||||
self.relaySessionId = nil
|
||||
self.onSpeakingChanged(false)
|
||||
}
|
||||
|
||||
private nonisolated static func closeRelaySession(
|
||||
gateway: GatewayNodeSession,
|
||||
relaySessionId: String) async
|
||||
{
|
||||
let payload = ["sessionId": relaySessionId]
|
||||
let data = try? JSONSerialization.data(withJSONObject: payload)
|
||||
let json = data.flatMap { String(data: $0, encoding: .utf8) }
|
||||
_ = try? await gateway.request(
|
||||
method: "talk.session.close",
|
||||
paramsJSON: json,
|
||||
timeoutSeconds: 8)
|
||||
}
|
||||
|
||||
func cancelOutput(reason: String = "user") {
|
||||
self.stopOutputPlayback()
|
||||
guard let relaySessionId else { return }
|
||||
Task { [gateway] in
|
||||
let payload: [String: Any] = [
|
||||
"sessionId": relaySessionId,
|
||||
"reason": reason,
|
||||
]
|
||||
let data = try? JSONSerialization.data(withJSONObject: payload)
|
||||
let json = data.flatMap { String(data: $0, encoding: .utf8) }
|
||||
_ = try? await gateway.request(
|
||||
method: "talk.session.cancelOutput",
|
||||
paramsJSON: json,
|
||||
timeoutSeconds: 8)
|
||||
}
|
||||
}
|
||||
|
||||
private func createRelaySession() async throws -> TalkSessionCreateResult {
|
||||
var payload: [String: Any] = [
|
||||
"sessionKey": self.options.sessionKey,
|
||||
"mode": "realtime",
|
||||
"transport": "gateway-relay",
|
||||
"brain": "agent-consult",
|
||||
]
|
||||
if let provider = self.nonEmpty(self.options.provider) {
|
||||
payload["provider"] = provider
|
||||
}
|
||||
if let model = self.nonEmpty(self.options.model) {
|
||||
payload["model"] = model
|
||||
}
|
||||
if let voice = self.nonEmpty(self.options.voice) {
|
||||
payload["voice"] = voice
|
||||
}
|
||||
let data = try JSONSerialization.data(withJSONObject: payload)
|
||||
guard let json = String(data: data, encoding: .utf8) else {
|
||||
throw NSError(domain: "RealtimeTalkRelay", code: 2, userInfo: [
|
||||
NSLocalizedDescriptionKey: "Failed to encode realtime relay request",
|
||||
])
|
||||
}
|
||||
let response = try await self.gateway.request(
|
||||
method: "talk.session.create",
|
||||
paramsJSON: json,
|
||||
timeoutSeconds: 20)
|
||||
return try JSONDecoder().decode(TalkSessionCreateResult.self, from: response)
|
||||
}
|
||||
|
||||
private func configureAudioContract(_ raw: AnyCodable?) {
|
||||
guard let audio = raw?.dictionaryValue else { return }
|
||||
let inputEncoding = audio["inputEncoding"]?.stringValue ?? Self.expectedInputEncoding
|
||||
let outputEncoding = audio["outputEncoding"]?.stringValue ?? Self.expectedOutputEncoding
|
||||
if inputEncoding != Self.expectedInputEncoding || outputEncoding != Self.expectedOutputEncoding {
|
||||
let message = "unexpected realtime relay audio contract input=\(inputEncoding) output=\(outputEncoding)"
|
||||
self.logger.warning("\(message, privacy: .public)")
|
||||
}
|
||||
self.inputSampleRateHz = audio["inputSampleRateHz"]?.doubleValue
|
||||
?? Double(Self.defaultSampleRateHz)
|
||||
self.outputSampleRateHz = audio["outputSampleRateHz"]?.doubleValue
|
||||
?? Double(Self.defaultSampleRateHz)
|
||||
}
|
||||
|
||||
private func startEventPump(stream: AsyncStream<EventFrame>) {
|
||||
self.eventTask?.cancel()
|
||||
self.eventTask = Task { [weak self] in
|
||||
for await event in stream {
|
||||
if Task.isCancelled { return }
|
||||
await self?.handleGatewayEvent(event)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func handleGatewayEvent(_ event: EventFrame) async {
|
||||
guard event.event == "talk.event",
|
||||
let payload = event.payload?.dictionaryValue
|
||||
else { return }
|
||||
guard let relaySessionId else {
|
||||
self.pendingPreRelayEvents.append(event)
|
||||
if self.pendingPreRelayEvents.count > 200 {
|
||||
self.pendingPreRelayEvents.removeFirst(self.pendingPreRelayEvents.count - 200)
|
||||
}
|
||||
return
|
||||
}
|
||||
if payload["relaySessionId"]?.stringValue != relaySessionId {
|
||||
return
|
||||
}
|
||||
guard let type = payload["type"]?.stringValue else { return }
|
||||
switch type {
|
||||
case "ready":
|
||||
self.hasReceivedReady = true
|
||||
self.finishStartupWait(.ready)
|
||||
self.onStatus("Listening (Realtime)")
|
||||
case "audio":
|
||||
guard let base64 = payload["audioBase64"]?.stringValue,
|
||||
let data = Data(base64Encoded: base64)
|
||||
else { return }
|
||||
self.recordOutputAudioChunk(byteCount: data.count)
|
||||
self.markOutputAudioStarted(byteCount: data.count, nowMs: ProcessInfo.processInfo.systemUptime * 1000)
|
||||
self.onSpeakingChanged(true)
|
||||
if self.outputContinuation == nil, self.outputTask != nil {
|
||||
self.pendingOutputChunks.append(data)
|
||||
return
|
||||
}
|
||||
self.ensureOutputPlaybackStarted()
|
||||
self.outputContinuation?.yield(data)
|
||||
case "audioDone":
|
||||
self.finishOutputPlaybackStream()
|
||||
case "clear":
|
||||
self.stopOutputPlayback()
|
||||
case "transcript":
|
||||
self.handleTranscriptEvent(payload)
|
||||
case "toolCall":
|
||||
await self.handleToolCall(payload)
|
||||
case "error":
|
||||
let message = payload["message"]?.stringValue ?? "Realtime failed"
|
||||
let issue = Self.issue(
|
||||
payload: payload,
|
||||
fallbackMessage: message,
|
||||
fallbackProvider: self.options.provider,
|
||||
fallbackModel: self.options.model)
|
||||
GatewayDiagnostics.log("talk realtime: error=\(Self.safeLogMessage(message))")
|
||||
self.hasReceivedFailure = true
|
||||
self.startupIssue = issue
|
||||
self.onIssue(issue)
|
||||
self.finishStartupWait(.failed(issue))
|
||||
self.onStatus(message)
|
||||
case "close":
|
||||
GatewayDiagnostics.log("talk realtime: close")
|
||||
if self.hasReceivedReady {
|
||||
self.onStatus("Ready")
|
||||
} else if !self.hasReceivedFailure {
|
||||
let issue = TalkRuntimeIssue(
|
||||
code: .realtimeUnavailable,
|
||||
message: "Realtime closed before it became ready.",
|
||||
provider: self.options.provider,
|
||||
model: self.options.model,
|
||||
transport: "gateway-relay",
|
||||
phase: "connect")
|
||||
self.onIssue(issue)
|
||||
self.startupIssue = issue
|
||||
self.finishStartupWait(.failed(issue))
|
||||
self.onStatus("Realtime failed before connecting")
|
||||
}
|
||||
self.close(sendClose: false)
|
||||
default:
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
private func waitForStartupResult(timeoutSeconds: Int) async -> StartupWaitResult {
|
||||
if self.isClosed { return .cancelled }
|
||||
if self.hasReceivedReady { return .ready }
|
||||
if let startupIssue { return .failed(startupIssue) }
|
||||
return await withCheckedContinuation { continuation in
|
||||
if self.isClosed {
|
||||
continuation.resume(returning: .cancelled)
|
||||
return
|
||||
}
|
||||
self.startupWaiter = continuation
|
||||
Task { [weak self] in
|
||||
try? await Task.sleep(nanoseconds: UInt64(max(0, timeoutSeconds)) * 1_000_000_000)
|
||||
await self?.timeoutStartupWaiterIfNeeded()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func drainPendingPreRelayEvents() async {
|
||||
let pendingEvents = self.pendingPreRelayEvents
|
||||
self.pendingPreRelayEvents.removeAll()
|
||||
for event in pendingEvents {
|
||||
await self.handleGatewayEvent(event)
|
||||
}
|
||||
}
|
||||
|
||||
private func finishStartupWait(_ result: StartupWaitResult) {
|
||||
guard let waiter = self.startupWaiter else { return }
|
||||
self.startupWaiter = nil
|
||||
waiter.resume(returning: result)
|
||||
}
|
||||
|
||||
private func timeoutStartupWaiterIfNeeded() {
|
||||
guard !self.isClosed, self.startupWaiter != nil, !self.hasReceivedReady, self.startupIssue == nil else {
|
||||
return
|
||||
}
|
||||
let issue = TalkRuntimeIssue(
|
||||
code: .realtimeUnavailable,
|
||||
message: "Realtime did not become ready in time.",
|
||||
provider: self.options.provider,
|
||||
model: self.options.model,
|
||||
transport: "gateway-relay",
|
||||
phase: "connect")
|
||||
self.hasReceivedFailure = true
|
||||
self.startupIssue = issue
|
||||
self.onIssue(issue)
|
||||
self.onStatus(issue.displayMessage)
|
||||
self.finishStartupWait(.failed(issue))
|
||||
}
|
||||
|
||||
private static func issue(
|
||||
payload: [String: AnyCodable],
|
||||
fallbackMessage: String,
|
||||
fallbackProvider: String?,
|
||||
fallbackModel: String?) -> TalkRuntimeIssue
|
||||
{
|
||||
let provider = payload["provider"]?.stringValue ?? fallbackProvider
|
||||
let model = payload["model"]?.stringValue ?? fallbackModel
|
||||
let transport = payload["transport"]?.stringValue ?? "gateway-relay"
|
||||
let phase = payload["phase"]?.stringValue
|
||||
return TalkRuntimeIssue.realtimeUnavailable(
|
||||
message: fallbackMessage,
|
||||
provider: provider,
|
||||
model: model,
|
||||
transport: transport,
|
||||
phase: phase)
|
||||
}
|
||||
|
||||
private func recordOutputAudioChunk(byteCount: Int) {
|
||||
self.outputAudioChunkCount += 1
|
||||
self.outputAudioByteCount += byteCount
|
||||
guard self.outputAudioChunkCount == 1 || self.outputAudioChunkCount % 20 == 0 else { return }
|
||||
GatewayDiagnostics.log(
|
||||
"talk realtime audio: chunks=\(self.outputAudioChunkCount) bytes=\(self.outputAudioByteCount)")
|
||||
}
|
||||
|
||||
private func markOutputAudioStarted(byteCount: Int, nowMs: Double) {
|
||||
if !self.isOutputPlaying {
|
||||
self.outputStartedAtMs = nowMs
|
||||
self.outputPlaybackExpectedEndMs = nowMs
|
||||
}
|
||||
self.isOutputPlaying = true
|
||||
let bytesPerSecond = max(1, self.outputSampleRateHz * Double(MemoryLayout<Int16>.size))
|
||||
let chunkDurationMs = (Double(byteCount) / bytesPerSecond) * 1000
|
||||
self.outputPlaybackExpectedEndMs = max(nowMs, self.outputPlaybackExpectedEndMs) + chunkDurationMs
|
||||
self.scheduleOutputPlaybackIdle(expectedEndMs: self.outputPlaybackExpectedEndMs)
|
||||
}
|
||||
|
||||
private func handleInputLevelDuringOutput(_ rms: Float, timestampMs: Double) {
|
||||
guard self.isOutputPlaying else { return }
|
||||
guard rms >= Self.bargeInRmsThreshold else { return }
|
||||
if let outputStartedAtMs,
|
||||
timestampMs - outputStartedAtMs < Self.minOutputBeforeBargeInMs
|
||||
{
|
||||
return
|
||||
}
|
||||
guard timestampMs - self.lastBargeInAtMs >= Self.bargeInCooldownMs else { return }
|
||||
self.lastBargeInAtMs = timestampMs
|
||||
self.cancelOutput(reason: "barge-in")
|
||||
}
|
||||
|
||||
private func handleTranscriptEvent(_ payload: [String: AnyCodable]) {
|
||||
let isFinal = payload["final"]?.boolValue == true
|
||||
let role = payload["role"]?.stringValue ?? ""
|
||||
let charCount = payload["text"]?.stringValue?.count ?? 0
|
||||
GatewayDiagnostics.log(
|
||||
"talk realtime transcript: role=\(role.isEmpty ? "unknown" : role) final=\(isFinal) chars=\(charCount)")
|
||||
guard isFinal else { return }
|
||||
if role == "user" {
|
||||
self.onStatus("Thinking…")
|
||||
} else if role == "assistant" {
|
||||
self.onStatus("Listening (Realtime)")
|
||||
}
|
||||
}
|
||||
|
||||
private func handleToolCall(_ payload: [String: AnyCodable]) async {
|
||||
guard let relaySessionId,
|
||||
let callId = payload["callId"]?.stringValue,
|
||||
let name = payload["name"]?.stringValue
|
||||
else { return }
|
||||
self.onStatus("Thinking…")
|
||||
do {
|
||||
if name == Self.agentControlToolName {
|
||||
try await self.handleAgentControlToolCall(
|
||||
callId: callId,
|
||||
relaySessionId: relaySessionId,
|
||||
args: payload["args"])
|
||||
return
|
||||
}
|
||||
let completionStream = await self.gateway.subscribeServerEvents(bufferingNewest: 200)
|
||||
let args = payload["args"]?.foundationValue ?? [:]
|
||||
let startPayload: [String: Any] = [
|
||||
"sessionKey": self.options.sessionKey,
|
||||
"callId": callId,
|
||||
"name": name,
|
||||
"args": args,
|
||||
"relaySessionId": relaySessionId,
|
||||
]
|
||||
let startResponse = try await self.requestJSON(
|
||||
method: "talk.client.toolCall",
|
||||
payload: startPayload,
|
||||
decodeAs: ToolCallStartResponse.self,
|
||||
timeoutSeconds: 30)
|
||||
guard let runId = startResponse.runId ?? startResponse.idempotencyKey else {
|
||||
throw NSError(domain: "RealtimeTalkRelay", code: 3, userInfo: [
|
||||
NSLocalizedDescriptionKey: "Realtime tool call did not return a run id",
|
||||
])
|
||||
}
|
||||
let completion = await self.waitForChatCompletion(
|
||||
runId: runId,
|
||||
stream: completionStream,
|
||||
timeoutSeconds: 120)
|
||||
let result: [String: Any] = completion.failed
|
||||
? ["error": "OpenClaw tool call failed"]
|
||||
: ["text": completion.text ?? "OpenClaw finished with no text."]
|
||||
try await self.submitToolResult(callId: callId, result: result)
|
||||
self.onStatus("Listening (Realtime)")
|
||||
} catch {
|
||||
try? await self.submitToolResult(callId: callId, result: [
|
||||
"error": error.localizedDescription,
|
||||
])
|
||||
self.onStatus("Listening (Realtime)")
|
||||
}
|
||||
}
|
||||
|
||||
private func handleAgentControlToolCall(
|
||||
callId: String,
|
||||
relaySessionId: String,
|
||||
args: AnyCodable?) async throws
|
||||
{
|
||||
let controlArgs = args?.dictionaryValue ?? [:]
|
||||
var payload: [String: Any] = [
|
||||
"sessionId": relaySessionId,
|
||||
"sessionKey": self.options.sessionKey,
|
||||
"text": controlArgs["text"]?.stringValue?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "status",
|
||||
]
|
||||
if let mode = controlArgs["mode"]?.stringValue?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!mode.isEmpty
|
||||
{
|
||||
payload["mode"] = mode
|
||||
}
|
||||
let response = try await self.requestJSON(
|
||||
method: "talk.session.steer",
|
||||
payload: payload,
|
||||
decodeAs: AnyCodable.self,
|
||||
timeoutSeconds: 30)
|
||||
let result = response.dictionaryValue?.mapValues(\.foundationValue) ?? [
|
||||
"result": response.foundationValue,
|
||||
]
|
||||
try await self.submitToolResult(callId: callId, result: result)
|
||||
self.onStatus("Listening (Realtime)")
|
||||
}
|
||||
|
||||
private func submitToolResult(callId: String, result: [String: Any]) async throws {
|
||||
guard let relaySessionId else { return }
|
||||
let payload: [String: Any] = [
|
||||
"sessionId": relaySessionId,
|
||||
"callId": callId,
|
||||
"result": result,
|
||||
]
|
||||
_ = try await self.requestJSON(
|
||||
method: "talk.session.submitToolResult",
|
||||
payload: payload,
|
||||
decodeAs: TalkSessionOkResult.self,
|
||||
timeoutSeconds: 30)
|
||||
}
|
||||
|
||||
private func waitForChatCompletion(
|
||||
runId: String,
|
||||
stream: AsyncStream<EventFrame>,
|
||||
timeoutSeconds: Int) async -> ChatCompletionResult
|
||||
{
|
||||
await withTaskGroup(of: ChatCompletionResult.self) { group in
|
||||
group.addTask {
|
||||
for await event in stream {
|
||||
if Task.isCancelled {
|
||||
return ChatCompletionResult(text: nil, failed: true)
|
||||
}
|
||||
guard event.event == "chat",
|
||||
let payload = event.payload,
|
||||
let chatEvent = try? GatewayPayloadDecoding.decode(
|
||||
payload,
|
||||
as: OpenClawChatEventPayload.self),
|
||||
chatEvent.runId == runId
|
||||
else { continue }
|
||||
if chatEvent.state == "final" {
|
||||
return ChatCompletionResult(
|
||||
text: OpenClawChatEventText.assistantText(from: chatEvent),
|
||||
failed: false)
|
||||
}
|
||||
if chatEvent.state == "aborted" || chatEvent.state == "error" {
|
||||
return ChatCompletionResult(text: nil, failed: true)
|
||||
}
|
||||
}
|
||||
return ChatCompletionResult(text: nil, failed: true)
|
||||
}
|
||||
group.addTask {
|
||||
try? await Task.sleep(nanoseconds: UInt64(timeoutSeconds) * 1_000_000_000)
|
||||
return ChatCompletionResult(text: nil, failed: true)
|
||||
}
|
||||
let result = await group.next() ?? ChatCompletionResult(text: nil, failed: true)
|
||||
group.cancelAll()
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
private func requestJSON<T: Decodable>(
|
||||
method: String,
|
||||
payload: [String: Any],
|
||||
decodeAs type: T.Type,
|
||||
timeoutSeconds: Int) async throws -> T
|
||||
{
|
||||
let data = try JSONSerialization.data(withJSONObject: payload)
|
||||
guard let json = String(data: data, encoding: .utf8) else {
|
||||
throw NSError(domain: "RealtimeTalkRelay", code: 4, userInfo: [
|
||||
NSLocalizedDescriptionKey: "Failed to encode \(method) payload",
|
||||
])
|
||||
}
|
||||
let response = try await self.gateway.request(
|
||||
method: method,
|
||||
paramsJSON: json,
|
||||
timeoutSeconds: timeoutSeconds)
|
||||
return try JSONDecoder().decode(type, from: response)
|
||||
}
|
||||
|
||||
private func startMicrophonePump() throws {
|
||||
self.stopMicrophonePump()
|
||||
let input = self.audioEngine.inputNode
|
||||
let format = input.inputFormat(forBus: 0)
|
||||
let targetSampleRate = self.inputSampleRateHz
|
||||
guard format.sampleRate > 0, format.channelCount > 0 else {
|
||||
throw NSError(domain: "RealtimeTalkRelay", code: 5, userInfo: [
|
||||
NSLocalizedDescriptionKey: "Invalid realtime audio input format",
|
||||
])
|
||||
}
|
||||
let tapBlock = makeRealtimeAudioTapBlock(
|
||||
inputSampleRate: format.sampleRate,
|
||||
targetSampleRate: targetSampleRate)
|
||||
{ [weak self, audioSender = self.audioSender] encoded, timestampMs, rms in
|
||||
guard let audioSender else { return }
|
||||
Task {
|
||||
let shouldSend = await MainActor.run { [weak self] in
|
||||
guard let self, !self.isClosed else { return false }
|
||||
self.recordMicrophoneFrame(byteCount: encoded.count, rms: rms, timestampMs: timestampMs)
|
||||
self.refreshOutputPlaybackState(timestampMs: timestampMs)
|
||||
if self.isOutputPlaying {
|
||||
if self.shouldSuppressMicrophoneDuringOutput() {
|
||||
self.recordSuppressedOutputEchoFrame(
|
||||
byteCount: encoded.count,
|
||||
rms: rms,
|
||||
timestampMs: timestampMs)
|
||||
return false
|
||||
}
|
||||
if rms >= Self.bargeInRmsThreshold {
|
||||
self.handleInputLevelDuringOutput(rms, timestampMs: timestampMs)
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
guard shouldSend else { return }
|
||||
guard let message = await audioSender.send(encoded, timestampMs: timestampMs) else { return }
|
||||
await MainActor.run { [weak self] in
|
||||
guard let self, !self.isClosed else { return }
|
||||
self.onStatus("Realtime audio failed: \(message)")
|
||||
}
|
||||
}
|
||||
}
|
||||
input.installTap(
|
||||
onBus: 0,
|
||||
bufferSize: Self.audioFrameBufferSize,
|
||||
format: format,
|
||||
block: tapBlock)
|
||||
self.audioEngine.prepare()
|
||||
try self.audioEngine.start()
|
||||
}
|
||||
|
||||
private func shouldSuppressMicrophoneDuringOutput() -> Bool {
|
||||
let outputs = AVAudioSession.sharedInstance().currentRoute.outputs
|
||||
// Built-in speaker output bleeds into the microphone even in voiceChat mode; keep the
|
||||
// realtime provider from treating its own speech as user input. Headsets keep barge-in.
|
||||
return outputs.contains { $0.portType == .builtInSpeaker }
|
||||
}
|
||||
|
||||
private func recordMicrophoneFrame(byteCount: Int, rms: Float, timestampMs: Double) {
|
||||
guard !self.isClosed else { return }
|
||||
self.micLogFrameCount += 1
|
||||
self.micLogByteCount += byteCount
|
||||
self.micLogMaxRms = max(self.micLogMaxRms, rms)
|
||||
guard timestampMs - self.lastMicLogAtMs >= 1000 else { return }
|
||||
self.lastMicLogAtMs = timestampMs
|
||||
let maxRms = String(format: "%.4f", Double(self.micLogMaxRms))
|
||||
GatewayDiagnostics.log(
|
||||
"talk realtime mic: buffers=\(self.micLogFrameCount) bytes=\(self.micLogByteCount) maxRms=\(maxRms)")
|
||||
self.micLogFrameCount = 0
|
||||
self.micLogByteCount = 0
|
||||
self.micLogMaxRms = 0
|
||||
}
|
||||
|
||||
private func recordSuppressedOutputEchoFrame(byteCount: Int, rms: Float, timestampMs: Double) {
|
||||
self.suppressedEchoFrameCount += 1
|
||||
self.suppressedEchoByteCount += byteCount
|
||||
self.suppressedEchoMaxRms = max(self.suppressedEchoMaxRms, rms)
|
||||
guard timestampMs - self.lastSuppressedEchoLogAtMs >= 1000 else { return }
|
||||
self.lastSuppressedEchoLogAtMs = timestampMs
|
||||
let maxRms = String(format: "%.4f", Double(self.suppressedEchoMaxRms))
|
||||
GatewayDiagnostics.log(
|
||||
"talk realtime mic suppressed during output: "
|
||||
+ "buffers=\(self.suppressedEchoFrameCount) "
|
||||
+ "bytes=\(self.suppressedEchoByteCount) maxRms=\(maxRms)")
|
||||
self.suppressedEchoFrameCount = 0
|
||||
self.suppressedEchoByteCount = 0
|
||||
self.suppressedEchoMaxRms = 0
|
||||
}
|
||||
|
||||
private func stopMicrophonePump() {
|
||||
self.audioEngine.inputNode.removeTap(onBus: 0)
|
||||
self.audioEngine.stop()
|
||||
}
|
||||
|
||||
private func ensureOutputPlaybackStarted() {
|
||||
guard self.outputContinuation == nil, self.outputTask == nil else { return }
|
||||
self.outputSessionId += 1
|
||||
let sessionId = self.outputSessionId
|
||||
let stream = AsyncThrowingStream<Data, Error> { continuation in
|
||||
self.outputContinuation = continuation
|
||||
}
|
||||
self.outputTask = Task { [weak self] in
|
||||
guard let self else { return }
|
||||
let result = await self.pcmPlayer.play(stream: stream, sampleRate: self.outputSampleRateHz)
|
||||
await MainActor.run {
|
||||
guard self.outputSessionId == sessionId else { return }
|
||||
self.outputTask = nil
|
||||
self.outputContinuation = nil
|
||||
if !result.finished, let interruptedAt = result.interruptedAt {
|
||||
self.logger.info("realtime output interrupted at \(interruptedAt, privacy: .public)s")
|
||||
}
|
||||
self.markOutputPlaybackFinished()
|
||||
self.startPendingOutputPlaybackIfNeeded()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func finishOutputPlaybackStream() {
|
||||
guard let continuation = self.outputContinuation else {
|
||||
if self.outputTask != nil, !self.pendingOutputChunks.isEmpty {
|
||||
self.pendingOutputDone = true
|
||||
}
|
||||
return
|
||||
}
|
||||
continuation.finish()
|
||||
self.outputContinuation = nil
|
||||
}
|
||||
|
||||
private func startPendingOutputPlaybackIfNeeded() {
|
||||
guard !self.pendingOutputChunks.isEmpty else {
|
||||
self.pendingOutputDone = false
|
||||
return
|
||||
}
|
||||
let chunks = self.pendingOutputChunks
|
||||
let shouldFinish = self.pendingOutputDone
|
||||
self.pendingOutputChunks = []
|
||||
self.pendingOutputDone = false
|
||||
self.ensureOutputPlaybackStarted()
|
||||
for chunk in chunks {
|
||||
self.markOutputAudioStarted(byteCount: chunk.count, nowMs: ProcessInfo.processInfo.systemUptime * 1000)
|
||||
self.onSpeakingChanged(true)
|
||||
self.outputContinuation?.yield(chunk)
|
||||
}
|
||||
if shouldFinish {
|
||||
self.finishOutputPlaybackStream()
|
||||
}
|
||||
}
|
||||
|
||||
private func scheduleOutputPlaybackIdle(expectedEndMs: Double) {
|
||||
self.outputIdleTask?.cancel()
|
||||
let nowMs = ProcessInfo.processInfo.systemUptime * 1000
|
||||
let idleDelayMs = max(350, expectedEndMs - nowMs + 500)
|
||||
self.outputIdleTask = Task { [weak self] in
|
||||
try? await Task.sleep(nanoseconds: UInt64(idleDelayMs * 1_000_000))
|
||||
guard !Task.isCancelled else { return }
|
||||
await MainActor.run { [weak self] in
|
||||
guard let self, !self.isClosed else { return }
|
||||
let nowMs = ProcessInfo.processInfo.systemUptime * 1000
|
||||
self.refreshOutputPlaybackState(timestampMs: nowMs, cancelIdleTask: false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func refreshOutputPlaybackState(timestampMs: Double, cancelIdleTask: Bool = true) {
|
||||
guard self.isOutputPlaying else { return }
|
||||
guard timestampMs >= self.outputPlaybackExpectedEndMs + 500 else { return }
|
||||
self.markOutputPlaybackFinished(cancelIdleTask: cancelIdleTask)
|
||||
}
|
||||
|
||||
private func markOutputPlaybackFinished(cancelIdleTask: Bool = true) {
|
||||
if cancelIdleTask {
|
||||
self.outputIdleTask?.cancel()
|
||||
self.outputIdleTask = nil
|
||||
}
|
||||
self.isOutputPlaying = false
|
||||
self.outputStartedAtMs = nil
|
||||
self.outputPlaybackExpectedEndMs = 0
|
||||
self.onSpeakingChanged(false)
|
||||
}
|
||||
|
||||
private func stopOutputPlayback() {
|
||||
self.outputSessionId += 1
|
||||
self.outputContinuation?.finish()
|
||||
self.outputContinuation = nil
|
||||
self.outputTask?.cancel()
|
||||
self.outputTask = nil
|
||||
self.outputIdleTask?.cancel()
|
||||
self.outputIdleTask = nil
|
||||
self.pendingOutputChunks = []
|
||||
self.pendingOutputDone = false
|
||||
_ = self.pcmPlayer.stop()
|
||||
self.isOutputPlaying = false
|
||||
self.outputStartedAtMs = nil
|
||||
self.outputPlaybackExpectedEndMs = 0
|
||||
self.onSpeakingChanged(false)
|
||||
}
|
||||
|
||||
fileprivate nonisolated static func encodePCM16(
|
||||
buffer: AVAudioPCMBuffer,
|
||||
inputSampleRate: Double,
|
||||
targetSampleRate: Double) -> Data
|
||||
{
|
||||
guard let channelData = buffer.floatChannelData,
|
||||
buffer.frameLength > 0,
|
||||
inputSampleRate > 0,
|
||||
targetSampleRate > 0
|
||||
else { return Data() }
|
||||
let frameCount = Int(buffer.frameLength)
|
||||
let channelCount = max(1, Int(buffer.format.channelCount))
|
||||
let outputCount = max(1, Int((Double(frameCount) * targetSampleRate / inputSampleRate).rounded(.down)))
|
||||
var data = Data(capacity: outputCount * MemoryLayout<Int16>.size)
|
||||
for index in 0..<outputCount {
|
||||
let sourcePosition = Double(index) * inputSampleRate / targetSampleRate
|
||||
let lower = min(frameCount - 1, Int(sourcePosition.rounded(.down)))
|
||||
let upper = min(frameCount - 1, lower + 1)
|
||||
let fraction = Float(sourcePosition - Double(lower))
|
||||
var mixed: Float = 0
|
||||
for channel in 0..<channelCount {
|
||||
let samples = channelData[channel]
|
||||
mixed += samples[lower] + ((samples[upper] - samples[lower]) * fraction)
|
||||
}
|
||||
let sample = max(-1, min(1, mixed / Float(channelCount)))
|
||||
var intSample = Int16((sample * Float(Int16.max)).rounded()).littleEndian
|
||||
withUnsafeBytes(of: &intSample) { data.append(contentsOf: $0) }
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
fileprivate nonisolated static func rmsLevel(buffer: AVAudioPCMBuffer) -> Float {
|
||||
guard let channelData = buffer.floatChannelData,
|
||||
buffer.frameLength > 0
|
||||
else { return 0 }
|
||||
let frameCount = Int(buffer.frameLength)
|
||||
let channelCount = max(1, Int(buffer.format.channelCount))
|
||||
var sumSquares: Float = 0
|
||||
var samples = 0
|
||||
for channel in 0..<channelCount {
|
||||
let values = channelData[channel]
|
||||
for index in 0..<frameCount {
|
||||
let sample = values[index]
|
||||
sumSquares += sample * sample
|
||||
samples += 1
|
||||
}
|
||||
}
|
||||
guard samples > 0 else { return 0 }
|
||||
return sqrt(sumSquares / Float(samples))
|
||||
}
|
||||
|
||||
private nonisolated static func safeLogMessage(_ value: String) -> String {
|
||||
let singleLine = value
|
||||
.replacingOccurrences(of: "\n", with: " ")
|
||||
.replacingOccurrences(of: "\r", with: " ")
|
||||
if singleLine.count <= 180 {
|
||||
return singleLine
|
||||
}
|
||||
return String(singleLine.prefix(180)) + "..."
|
||||
}
|
||||
|
||||
private func nonEmpty(_ value: String?) -> String? {
|
||||
let trimmed = value?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
return trimmed?.isEmpty == false ? trimmed : nil
|
||||
}
|
||||
}
|
||||
|
||||
extension RealtimeTalkRelaySession {
|
||||
func _test_setRelaySessionId(_ relaySessionId: String) {
|
||||
self.relaySessionId = relaySessionId
|
||||
}
|
||||
|
||||
func _test_handleGatewayEvent(_ event: EventFrame) async {
|
||||
await self.handleGatewayEvent(event)
|
||||
}
|
||||
|
||||
func _test_waitForStartupCancelled(timeoutSeconds: Int) async -> Bool {
|
||||
if case .cancelled = await self.waitForStartupResult(timeoutSeconds: timeoutSeconds) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func _test_startupReadyTimeoutSeconds() -> Int {
|
||||
Self.startupReadyTimeoutSeconds
|
||||
}
|
||||
|
||||
func _test_markOutputAudioStarted(nowMs: Double) {
|
||||
self.markOutputAudioStarted(byteCount: 4800, nowMs: nowMs)
|
||||
}
|
||||
|
||||
func _test_markOutputPlaybackFinished() {
|
||||
self.markOutputPlaybackFinished()
|
||||
}
|
||||
|
||||
func _test_outputStartedAtMs() -> Double? {
|
||||
self.outputStartedAtMs
|
||||
}
|
||||
|
||||
func _test_isOutputPlaying() -> Bool {
|
||||
self.isOutputPlaying
|
||||
}
|
||||
}
|
||||
34
apps/ios/Sources/Voice/TalkDefaults.swift
Normal file
34
apps/ios/Sources/Voice/TalkDefaults.swift
Normal file
@@ -0,0 +1,34 @@
|
||||
import AVFoundation
|
||||
import Foundation
|
||||
|
||||
enum TalkDefaults {
|
||||
static let silenceTimeoutMs = 900
|
||||
static let speakerphoneEnabledKey = "talk.speakerphone.enabled"
|
||||
static let speakerphoneEnabledByDefault = true
|
||||
|
||||
static func speakerphoneEnabled(defaults: UserDefaults = .standard) -> Bool {
|
||||
guard defaults.object(forKey: self.speakerphoneEnabledKey) != nil else {
|
||||
return self.speakerphoneEnabledByDefault
|
||||
}
|
||||
return defaults.bool(forKey: self.speakerphoneEnabledKey)
|
||||
}
|
||||
}
|
||||
|
||||
enum TalkAudioRoute {
|
||||
static func categoryOptions(speakerphoneEnabled: Bool) -> AVAudioSession.CategoryOptions {
|
||||
var options: AVAudioSession.CategoryOptions = [.allowBluetoothHFP, .allowBluetoothA2DP, .allowAirPlay]
|
||||
if speakerphoneEnabled {
|
||||
options.insert(.defaultToSpeaker)
|
||||
}
|
||||
return options
|
||||
}
|
||||
|
||||
static func shouldForceSpeaker(
|
||||
preferenceEnabled: Bool,
|
||||
outputPortTypes: [AVAudioSession.Port]) -> Bool
|
||||
{
|
||||
guard preferenceEnabled else { return false }
|
||||
guard !outputPortTypes.isEmpty else { return false }
|
||||
return outputPortTypes.allSatisfy { $0 == .builtInReceiver || $0 == .builtInSpeaker }
|
||||
}
|
||||
}
|
||||
63
apps/ios/Sources/Voice/TalkGatewayPermissionState.swift
Normal file
63
apps/ios/Sources/Voice/TalkGatewayPermissionState.swift
Normal file
@@ -0,0 +1,63 @@
|
||||
enum TalkGatewayPermissionState: Equatable {
|
||||
case unknown
|
||||
case ready
|
||||
case missingScope(String)
|
||||
case requestingUpgrade
|
||||
case upgradeRequested(requestId: String?)
|
||||
case requestFailed(String)
|
||||
case apiKeyMissing
|
||||
case loadFailed(String)
|
||||
|
||||
var statusLabel: String {
|
||||
switch self {
|
||||
case .unknown:
|
||||
"Not checked"
|
||||
case .ready:
|
||||
"Ready"
|
||||
case let .missingScope(scope):
|
||||
"Missing \(scope)"
|
||||
case .requestingUpgrade:
|
||||
"Requesting approval"
|
||||
case .upgradeRequested:
|
||||
"Approval requested"
|
||||
case .requestFailed:
|
||||
"Request failed"
|
||||
case .apiKeyMissing:
|
||||
"API key missing"
|
||||
case .loadFailed:
|
||||
"Load failed"
|
||||
}
|
||||
}
|
||||
|
||||
var requiresTalkPermissionAction: Bool {
|
||||
switch self {
|
||||
case .missingScope, .requestingUpgrade, .upgradeRequested, .requestFailed:
|
||||
true
|
||||
default:
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
var isApprovalRequestInProgress: Bool {
|
||||
switch self {
|
||||
case .requestingUpgrade, .upgradeRequested:
|
||||
true
|
||||
default:
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
var failureMessage: String? {
|
||||
if case let .requestFailed(message) = self {
|
||||
return message
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var requestId: String? {
|
||||
if case let .upgradeRequested(requestId) = self {
|
||||
return requestId
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
279
apps/ios/Sources/Voice/TalkGatewaySpeechClient.swift
Normal file
279
apps/ios/Sources/Voice/TalkGatewaySpeechClient.swift
Normal file
@@ -0,0 +1,279 @@
|
||||
import AVFoundation
|
||||
import Foundation
|
||||
import OpenClawKit
|
||||
import OpenClawProtocol
|
||||
import OSLog
|
||||
|
||||
struct TalkGatewaySpeechAudio: Equatable {
|
||||
enum PlaybackMode: Equatable {
|
||||
case pcm(sampleRate: Double)
|
||||
case buffered
|
||||
case unsupportedRaw(codec: String)
|
||||
}
|
||||
|
||||
let data: Data
|
||||
let provider: String
|
||||
let outputFormat: String?
|
||||
|
||||
var playbackMode: PlaybackMode {
|
||||
if let sampleRate = TalkTTSValidation.pcmSampleRate(from: self.outputFormat) {
|
||||
return .pcm(sampleRate: sampleRate)
|
||||
}
|
||||
let outputFormat = self.outputFormat?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
.lowercased()
|
||||
let isHeaderlessAudio = if let outputFormat {
|
||||
outputFormat.hasPrefix("raw-") ||
|
||||
outputFormat.hasPrefix("raw_") ||
|
||||
outputFormat == "pcm" ||
|
||||
outputFormat == "mulaw" ||
|
||||
outputFormat == "alaw" ||
|
||||
outputFormat.hasPrefix("mulaw_") ||
|
||||
outputFormat.hasPrefix("ulaw_") ||
|
||||
outputFormat.hasPrefix("alaw_")
|
||||
} else {
|
||||
false
|
||||
}
|
||||
if let outputFormat, isHeaderlessAudio {
|
||||
// talk.speak does not expose the sample rate needed to play headerless audio.
|
||||
// Keep these codecs out of AVAudioPlayer until that protocol metadata exists.
|
||||
return .unsupportedRaw(codec: outputFormat)
|
||||
}
|
||||
// Gateway providers return complete audio files (for example MP3, WAV, or FLAC).
|
||||
// AVAudioPlayer performs container detection; raw PCM keeps its sample-rate path above.
|
||||
return .buffered
|
||||
}
|
||||
}
|
||||
|
||||
struct TalkGatewaySpeechRequest {
|
||||
let text: String
|
||||
// These are talk.speak protocol inputs; each Gateway speech provider owns which overrides it honors.
|
||||
let voiceId: String?
|
||||
let modelId: String?
|
||||
let outputFormat: String?
|
||||
let directive: TalkDirective?
|
||||
}
|
||||
|
||||
@MainActor
|
||||
protocol TalkGatewaySpeechSynthesizing {
|
||||
func synthesize(_ request: TalkGatewaySpeechRequest) async throws -> TalkGatewaySpeechAudio
|
||||
}
|
||||
|
||||
@MainActor
|
||||
final class TalkGatewaySpeechClient: TalkGatewaySpeechSynthesizing {
|
||||
typealias Request = (_ method: String, _ paramsJSON: String?, _ timeoutSeconds: Int) async throws -> Data
|
||||
private static let requestTimeoutSeconds = 125
|
||||
|
||||
private let request: Request
|
||||
|
||||
init(gateway: GatewayNodeSession) {
|
||||
self.request = { method, paramsJSON, timeoutSeconds in
|
||||
try await gateway.request(
|
||||
method: method,
|
||||
paramsJSON: paramsJSON,
|
||||
timeoutSeconds: timeoutSeconds)
|
||||
}
|
||||
}
|
||||
|
||||
init(request: @escaping Request) {
|
||||
self.request = request
|
||||
}
|
||||
|
||||
func synthesize(_ speechRequest: TalkGatewaySpeechRequest) async throws -> TalkGatewaySpeechAudio {
|
||||
let directive = speechRequest.directive
|
||||
let params = TalkSpeakParams(
|
||||
text: speechRequest.text,
|
||||
voiceid: speechRequest.voiceId,
|
||||
modelid: speechRequest.modelId,
|
||||
outputformat: speechRequest.outputFormat,
|
||||
speed: directive?.speed,
|
||||
ratewpm: directive?.rateWPM,
|
||||
stability: directive?.stability,
|
||||
similarity: directive?.similarity,
|
||||
style: directive?.style,
|
||||
speakerboost: directive?.speakerBoost,
|
||||
seed: directive?.seed,
|
||||
normalize: directive?.normalize,
|
||||
language: directive?.language,
|
||||
latencytier: directive?.latencyTier)
|
||||
let paramsData = try JSONEncoder().encode(params)
|
||||
guard let paramsJSON = String(data: paramsData, encoding: .utf8) else {
|
||||
throw TalkGatewaySpeechError.invalidRequest
|
||||
}
|
||||
let responseData = try await request(
|
||||
"talk.speak",
|
||||
paramsJSON,
|
||||
Self.requestTimeoutSeconds)
|
||||
let response = try JSONDecoder().decode(TalkSpeakResult.self, from: responseData)
|
||||
guard let audioData = Data(base64Encoded: response.audiobase64), !audioData.isEmpty else {
|
||||
throw TalkGatewaySpeechError.emptyAudio
|
||||
}
|
||||
return TalkGatewaySpeechAudio(
|
||||
data: audioData,
|
||||
provider: response.provider,
|
||||
outputFormat: response.outputformat)
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
protocol TalkBufferedAudioPlaying {
|
||||
func play(data: Data) async -> StreamingPlaybackResult
|
||||
func stop() -> Double?
|
||||
}
|
||||
|
||||
@MainActor
|
||||
final class TalkBufferedAudioPlayer: NSObject, TalkBufferedAudioPlaying, @preconcurrency AVAudioPlayerDelegate {
|
||||
static let shared = TalkBufferedAudioPlayer()
|
||||
|
||||
private final class Playback: @unchecked Sendable {
|
||||
private let lock = NSLock()
|
||||
private var finished = false
|
||||
private var continuation: CheckedContinuation<StreamingPlaybackResult, Never>?
|
||||
private var watchdog: Task<Void, Never>?
|
||||
|
||||
func setContinuation(_ continuation: CheckedContinuation<StreamingPlaybackResult, Never>) {
|
||||
self.lock.lock()
|
||||
defer { self.lock.unlock() }
|
||||
self.continuation = continuation
|
||||
}
|
||||
|
||||
func setWatchdog(_ task: Task<Void, Never>?) {
|
||||
self.lock.lock()
|
||||
let old = self.watchdog
|
||||
self.watchdog = task
|
||||
self.lock.unlock()
|
||||
old?.cancel()
|
||||
}
|
||||
|
||||
func finish(_ result: StreamingPlaybackResult) {
|
||||
let continuation: CheckedContinuation<StreamingPlaybackResult, Never>?
|
||||
self.lock.lock()
|
||||
if self.finished {
|
||||
continuation = nil
|
||||
} else {
|
||||
self.finished = true
|
||||
continuation = self.continuation
|
||||
self.continuation = nil
|
||||
}
|
||||
self.lock.unlock()
|
||||
continuation?.resume(returning: result)
|
||||
}
|
||||
}
|
||||
|
||||
private let logger = Logger(subsystem: "ai.openclaw", category: "talk.tts")
|
||||
private var player: AVAudioPlayer?
|
||||
private var playback: Playback?
|
||||
|
||||
func play(data: Data) async -> StreamingPlaybackResult {
|
||||
self.stopInternal()
|
||||
|
||||
let playback = Playback()
|
||||
self.playback = playback
|
||||
return await withCheckedContinuation { continuation in
|
||||
playback.setContinuation(continuation)
|
||||
do {
|
||||
let player = try AVAudioPlayer(data: data)
|
||||
self.player = player
|
||||
player.delegate = self
|
||||
player.prepareToPlay()
|
||||
self.armWatchdog(playback: playback)
|
||||
if !player.play() {
|
||||
self.logger.error("talk buffered audio player refused to play")
|
||||
self.finish(playback: playback, result: .init(finished: false, interruptedAt: nil))
|
||||
}
|
||||
} catch {
|
||||
self.logger.error("talk buffered audio player failed: \(error.localizedDescription, privacy: .public)")
|
||||
self.finish(playback: playback, result: .init(finished: false, interruptedAt: nil))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stop() -> Double? {
|
||||
guard let player else { return nil }
|
||||
let interruptedAt = player.currentTime
|
||||
self.finish(
|
||||
playback: self.playback,
|
||||
result: .init(finished: false, interruptedAt: interruptedAt))
|
||||
return interruptedAt
|
||||
}
|
||||
|
||||
func audioPlayerDidFinishPlaying(_ player: AVAudioPlayer, successfully flag: Bool) {
|
||||
self.finish(
|
||||
playback: self.activePlayback(for: player),
|
||||
result: .init(finished: flag, interruptedAt: nil))
|
||||
}
|
||||
|
||||
func audioPlayerDecodeErrorDidOccur(_ player: AVAudioPlayer, error: (any Error)?) {
|
||||
let message = error?.localizedDescription ?? "unknown decode error"
|
||||
self.logger.error("talk buffered audio decode failed: \(message, privacy: .public)")
|
||||
self.finish(
|
||||
playback: self.activePlayback(for: player),
|
||||
result: .init(finished: false, interruptedAt: nil))
|
||||
}
|
||||
|
||||
private func activePlayback(for player: AVAudioPlayer) -> Playback? {
|
||||
// AVAudioPlayer can deliver callbacks after stop/replacement. Keep a stale
|
||||
// player from completing the current reply's continuation.
|
||||
guard self.player === player else { return nil }
|
||||
return self.playback
|
||||
}
|
||||
|
||||
private func stopInternal() {
|
||||
if let player, let playback {
|
||||
self.finish(
|
||||
playback: playback,
|
||||
result: .init(finished: false, interruptedAt: player.currentTime))
|
||||
return
|
||||
}
|
||||
self.player?.stop()
|
||||
self.player = nil
|
||||
}
|
||||
|
||||
private func finish(playback: Playback?, result: StreamingPlaybackResult) {
|
||||
guard let playback else { return }
|
||||
playback.setWatchdog(nil)
|
||||
playback.finish(result)
|
||||
|
||||
guard self.playback === playback else { return }
|
||||
self.playback = nil
|
||||
self.player?.stop()
|
||||
self.player = nil
|
||||
}
|
||||
|
||||
private func armWatchdog(playback: Playback) {
|
||||
playback.setWatchdog(Task { @MainActor [weak self] in
|
||||
guard let self else { return }
|
||||
try? await Task.sleep(nanoseconds: 650_000_000)
|
||||
guard !Task.isCancelled, self.playback === playback else { return }
|
||||
guard self.player?.isPlaying == true else {
|
||||
self.finish(
|
||||
playback: playback,
|
||||
result: .init(finished: false, interruptedAt: nil))
|
||||
return
|
||||
}
|
||||
|
||||
let duration = self.player?.duration ?? 0
|
||||
let timeoutSeconds = min(max(2.0, duration + 2.0), 5 * 60.0)
|
||||
try? await Task.sleep(nanoseconds: UInt64(timeoutSeconds * 1_000_000_000))
|
||||
guard !Task.isCancelled, self.playback === playback else { return }
|
||||
self.logger.error("talk buffered audio player watchdog completed unresolved playback")
|
||||
self.finish(
|
||||
playback: playback,
|
||||
result: .init(finished: false, interruptedAt: nil))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
private enum TalkGatewaySpeechError: LocalizedError {
|
||||
case invalidRequest
|
||||
case emptyAudio
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .invalidRequest:
|
||||
"Failed to encode talk.speak request"
|
||||
case .emptyAudio:
|
||||
"Gateway talk.speak returned empty audio"
|
||||
}
|
||||
}
|
||||
}
|
||||
531
apps/ios/Sources/Voice/TalkModeGatewayConfig.swift
Normal file
531
apps/ios/Sources/Voice/TalkModeGatewayConfig.swift
Normal file
@@ -0,0 +1,531 @@
|
||||
import Foundation
|
||||
import OpenClawKit
|
||||
|
||||
enum TalkModeExecutionMode: Equatable {
|
||||
case native
|
||||
case realtimeWebRTC
|
||||
case realtimeRelay
|
||||
}
|
||||
|
||||
struct TalkRuntimeIssue: Equatable {
|
||||
enum Code: String {
|
||||
case realtimeUnavailable = "realtime_unavailable"
|
||||
}
|
||||
|
||||
let code: Code
|
||||
let message: String
|
||||
let provider: String?
|
||||
let model: String?
|
||||
let transport: String?
|
||||
let phase: String?
|
||||
let occurredAt: Date
|
||||
|
||||
init(
|
||||
code: Code,
|
||||
message: String,
|
||||
provider: String? = nil,
|
||||
model: String? = nil,
|
||||
transport: String? = nil,
|
||||
phase: String? = nil,
|
||||
occurredAt: Date = Date())
|
||||
{
|
||||
self.code = code
|
||||
self.message = message.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
self.provider = provider?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
self.model = model?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
self.transport = transport?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
self.phase = phase?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
self.occurredAt = occurredAt
|
||||
}
|
||||
|
||||
var displayMessage: String {
|
||||
if !self.message.isEmpty { return self.message }
|
||||
return "Realtime voice did not start."
|
||||
}
|
||||
|
||||
var fallbackStatusText: String {
|
||||
"Listening (iOS Speech fallback)"
|
||||
}
|
||||
|
||||
var fallbackBannerTitle: String {
|
||||
"Using iOS Speech fallback"
|
||||
}
|
||||
|
||||
var fallbackBannerOwnerLabel: String {
|
||||
"Fallback active"
|
||||
}
|
||||
|
||||
var fallbackBannerMessage: String {
|
||||
"Realtime voice did not start. Talk is running with iOS speech recognition and TTS."
|
||||
}
|
||||
|
||||
var technicalDetails: String {
|
||||
var lines = [
|
||||
"code: \(code.rawValue)",
|
||||
"message: \(self.displayMessage)",
|
||||
]
|
||||
if let provider, !provider.isEmpty { lines.append("provider: \(provider)") }
|
||||
if let model, !model.isEmpty { lines.append("model: \(model)") }
|
||||
if let transport, !transport.isEmpty { lines.append("transport: \(transport)") }
|
||||
if let phase, !phase.isEmpty { lines.append("phase: \(phase)") }
|
||||
return lines.joined(separator: "\n")
|
||||
}
|
||||
|
||||
var diagnosticSummary: String {
|
||||
var parts = [displayMessage]
|
||||
if let provider, !provider.isEmpty { parts.append("provider: \(provider)") }
|
||||
if let model, !model.isEmpty { parts.append("model: \(model)") }
|
||||
if let transport, !transport.isEmpty { parts.append("transport: \(transport)") }
|
||||
if let phase, !phase.isEmpty { parts.append("phase: \(phase)") }
|
||||
return parts.joined(separator: " • ")
|
||||
}
|
||||
|
||||
static func realtimeUnavailable(
|
||||
message: String,
|
||||
provider: String? = nil,
|
||||
model: String? = nil,
|
||||
transport: String? = nil,
|
||||
phase: String? = nil) -> TalkRuntimeIssue
|
||||
{
|
||||
TalkRuntimeIssue(
|
||||
code: .realtimeUnavailable,
|
||||
message: message,
|
||||
provider: provider,
|
||||
model: model,
|
||||
transport: transport,
|
||||
phase: phase)
|
||||
}
|
||||
}
|
||||
|
||||
struct TalkVoiceModeDescriptor: Equatable {
|
||||
let title: String
|
||||
let subtitle: String?
|
||||
let providerId: String?
|
||||
let modelId: String?
|
||||
let voiceId: String?
|
||||
let transport: String?
|
||||
let isRealtime: Bool
|
||||
|
||||
var accessibilityValue: String {
|
||||
if let subtitle, !subtitle.isEmpty {
|
||||
return "\(self.title), \(subtitle)"
|
||||
}
|
||||
return self.title
|
||||
}
|
||||
}
|
||||
|
||||
enum TalkVoiceModeDescriptorBuilder {
|
||||
static func build(
|
||||
providerId: String,
|
||||
providerLabel: String,
|
||||
modelId: String?,
|
||||
voiceId: String?,
|
||||
transport: String?,
|
||||
isRealtime: Bool) -> TalkVoiceModeDescriptor
|
||||
{
|
||||
let normalizedProvider = providerId.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
||||
let trimmedModel = Self.trimmed(modelId)
|
||||
let trimmedVoice = Self.trimmed(voiceId)
|
||||
let trimmedTransport = Self.trimmed(transport)
|
||||
let title = if isRealtime, normalizedProvider == "openai", trimmedModel == "gpt-realtime-2" {
|
||||
"GPT Realtime 2.0"
|
||||
} else if isRealtime, normalizedProvider == "openai" {
|
||||
"OpenAI Realtime"
|
||||
} else if isRealtime {
|
||||
providerLabel.isEmpty ? "Realtime Voice" : providerLabel
|
||||
} else if normalizedProvider == "system" {
|
||||
"iOS System Voice"
|
||||
} else {
|
||||
providerLabel.isEmpty ? "Talk Voice" : providerLabel
|
||||
}
|
||||
|
||||
var details: [String] = []
|
||||
if isRealtime, normalizedProvider != "openai", !providerLabel.isEmpty, providerLabel != title {
|
||||
details.append(providerLabel)
|
||||
}
|
||||
if let trimmedTransport {
|
||||
details.append(Self.transportLabel(trimmedTransport))
|
||||
}
|
||||
if let trimmedModel, title != "GPT Realtime 2.0" || trimmedModel != "gpt-realtime-2" {
|
||||
details.append(trimmedModel)
|
||||
}
|
||||
if let trimmedVoice {
|
||||
details.append(Self.voiceLabel(trimmedVoice))
|
||||
}
|
||||
|
||||
return TalkVoiceModeDescriptor(
|
||||
title: title,
|
||||
subtitle: details.isEmpty ? nil : details.joined(separator: " • "),
|
||||
providerId: normalizedProvider.isEmpty ? nil : normalizedProvider,
|
||||
modelId: trimmedModel,
|
||||
voiceId: trimmedVoice,
|
||||
transport: trimmedTransport,
|
||||
isRealtime: isRealtime)
|
||||
}
|
||||
|
||||
private static func trimmed(_ value: String?) -> String? {
|
||||
let trimmed = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
return trimmed.isEmpty ? nil : trimmed
|
||||
}
|
||||
|
||||
private static func voiceLabel(_ voice: String) -> String {
|
||||
TalkModeRealtimeVoiceSelection.voices.contains(voice)
|
||||
? TalkModeRealtimeVoiceSelection.label(for: voice)
|
||||
: voice
|
||||
}
|
||||
|
||||
private static func transportLabel(_ transport: String) -> String {
|
||||
switch transport.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() {
|
||||
case "webrtc":
|
||||
"Native WebRTC"
|
||||
case "gateway-relay":
|
||||
"Gateway Relay"
|
||||
case "provider-websocket":
|
||||
"Provider WebSocket"
|
||||
case "managed-room":
|
||||
"Managed Room"
|
||||
case "native":
|
||||
"Native"
|
||||
case let value where !value.isEmpty:
|
||||
value
|
||||
default:
|
||||
"Native"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum TalkModeProviderSelection: String, CaseIterable, Identifiable {
|
||||
case gatewayDefault = "gateway"
|
||||
case nativeElevenLabs = "elevenlabs"
|
||||
case openAIRealtime = "openai-realtime"
|
||||
|
||||
static let storageKey = "talk.providerSelection"
|
||||
|
||||
var id: String {
|
||||
rawValue
|
||||
}
|
||||
|
||||
var label: String {
|
||||
switch self {
|
||||
case .gatewayDefault:
|
||||
"Gateway Default"
|
||||
case .nativeElevenLabs:
|
||||
"ElevenLabs"
|
||||
case .openAIRealtime:
|
||||
"Realtime-2 (OpenAI)"
|
||||
}
|
||||
}
|
||||
|
||||
static func resolved(_ raw: String?) -> TalkModeProviderSelection {
|
||||
let trimmed = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
return TalkModeProviderSelection(rawValue: trimmed) ?? .gatewayDefault
|
||||
}
|
||||
}
|
||||
|
||||
enum TalkModeRuntimeRoute: Equatable {
|
||||
case localElevenLabs
|
||||
case gatewayTalkSpeak
|
||||
case realtimeWebRTC
|
||||
case realtimeRelay
|
||||
|
||||
var usesRealtime: Bool {
|
||||
self == .realtimeRelay || self == .realtimeWebRTC
|
||||
}
|
||||
|
||||
var usesGatewayTalkSpeak: Bool {
|
||||
self == .gatewayTalkSpeak
|
||||
}
|
||||
|
||||
var gatewayOwnsCredentials: Bool {
|
||||
self != .localElevenLabs
|
||||
}
|
||||
}
|
||||
|
||||
struct TalkModeResolvedRouting: Equatable {
|
||||
let activeProvider: String
|
||||
let executionMode: TalkModeExecutionMode
|
||||
let realtimeProvider: String?
|
||||
let realtimeModelId: String?
|
||||
let route: TalkModeRuntimeRoute
|
||||
}
|
||||
|
||||
enum TalkModeRoutingResolver {
|
||||
static func resolve(
|
||||
parsed: TalkModeGatewayConfigState,
|
||||
providerSelection: TalkModeProviderSelection,
|
||||
defaultProvider: String,
|
||||
defaultRealtimeModelId: String) -> TalkModeResolvedRouting
|
||||
{
|
||||
var activeProvider = parsed.activeProvider
|
||||
var realtimeProvider = parsed.realtimeProvider
|
||||
var realtimeModelId = parsed.realtimeModelId
|
||||
let route: TalkModeRuntimeRoute
|
||||
|
||||
switch providerSelection {
|
||||
case .gatewayDefault:
|
||||
// Only an explicit realtime config selects the realtime transport. Other Gateway
|
||||
// speech providers stay native and synthesize through talk.speak.
|
||||
if parsed.executionMode == .realtimeWebRTC {
|
||||
route = .realtimeWebRTC
|
||||
} else if parsed.executionMode == .realtimeRelay {
|
||||
route = .realtimeRelay
|
||||
} else if Self.normalized(activeProvider) == Self.normalized(defaultProvider) {
|
||||
// Preserve the shipped local ElevenLabs path, including its streaming playback.
|
||||
route = .localElevenLabs
|
||||
} else {
|
||||
route = .gatewayTalkSpeak
|
||||
}
|
||||
case .nativeElevenLabs:
|
||||
activeProvider = defaultProvider
|
||||
route = .localElevenLabs
|
||||
case .openAIRealtime:
|
||||
activeProvider = "openai"
|
||||
realtimeProvider = "openai"
|
||||
realtimeModelId = defaultRealtimeModelId
|
||||
// Provider selection can replace provider details, but an explicit Gateway-owned
|
||||
// realtime route must remain on the Gateway (for example, Azure-backed OpenAI).
|
||||
route = parsed.openAIRequiresGatewayRealtimeTransport ? .realtimeRelay : .realtimeWebRTC
|
||||
}
|
||||
|
||||
return TalkModeResolvedRouting(
|
||||
activeProvider: activeProvider,
|
||||
executionMode: Self.executionMode(for: route),
|
||||
realtimeProvider: realtimeProvider,
|
||||
realtimeModelId: realtimeModelId,
|
||||
route: route)
|
||||
}
|
||||
|
||||
private static func executionMode(for route: TalkModeRuntimeRoute) -> TalkModeExecutionMode {
|
||||
switch route {
|
||||
case .localElevenLabs, .gatewayTalkSpeak:
|
||||
.native
|
||||
case .realtimeWebRTC:
|
||||
.realtimeWebRTC
|
||||
case .realtimeRelay:
|
||||
.realtimeRelay
|
||||
}
|
||||
}
|
||||
|
||||
private static func normalized(_ value: String) -> String {
|
||||
value.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
||||
}
|
||||
}
|
||||
|
||||
enum TalkModeRealtimeVoiceSelection {
|
||||
static let storageKey = "talk.realtime.voiceSelection"
|
||||
static let voices = [
|
||||
"alloy",
|
||||
"ash",
|
||||
"ballad",
|
||||
"coral",
|
||||
"echo",
|
||||
"sage",
|
||||
"shimmer",
|
||||
"verse",
|
||||
"marin",
|
||||
"cedar",
|
||||
]
|
||||
|
||||
static func resolvedOverride(_ raw: String?) -> String? {
|
||||
let trimmed = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
||||
guard !trimmed.isEmpty else { return nil }
|
||||
return Self.voices.contains(trimmed) ? trimmed : nil
|
||||
}
|
||||
|
||||
static func label(for voice: String) -> String {
|
||||
voice.prefix(1).uppercased() + String(voice.dropFirst())
|
||||
}
|
||||
}
|
||||
|
||||
struct TalkModeGatewayConfigState {
|
||||
let activeProvider: String
|
||||
let normalizedPayload: Bool
|
||||
let missingResolvedPayload: Bool
|
||||
let executionMode: TalkModeExecutionMode
|
||||
let requiresGatewayRealtimeTransport: Bool
|
||||
let openAIRequiresGatewayRealtimeTransport: Bool
|
||||
let defaultVoiceId: String?
|
||||
let voiceAliases: [String: String]
|
||||
let configuredModelId: String?
|
||||
let defaultModelId: String
|
||||
let defaultOutputFormat: String?
|
||||
let realtimeProvider: String?
|
||||
let realtimeModelId: String?
|
||||
let realtimeVoiceId: String?
|
||||
let rawConfigApiKey: String?
|
||||
let interruptOnSpeech: Bool?
|
||||
let silenceTimeoutMs: Int
|
||||
let speechLocaleID: String?
|
||||
}
|
||||
|
||||
enum TalkModeGatewayConfigParser {
|
||||
static func parse(
|
||||
config: [String: Any],
|
||||
defaultProvider: String,
|
||||
defaultModelIdFallback: String,
|
||||
defaultRealtimeModelIdFallback: String,
|
||||
defaultSilenceTimeoutMs: Int) -> TalkModeGatewayConfigState
|
||||
{
|
||||
let talk = TalkConfigParsing.bridgeFoundationDictionary(config["talk"] as? [String: Any])
|
||||
let selection = TalkConfigParsing.selectProviderConfig(
|
||||
talk,
|
||||
defaultProvider: defaultProvider,
|
||||
allowLegacyFallback: false)
|
||||
let activeProvider = selection?.provider ?? defaultProvider
|
||||
let activeConfig = selection?.config
|
||||
let voiceAliases: [String: String]
|
||||
if let aliases = activeConfig?["voiceAliases"]?.dictionaryValue {
|
||||
var resolved: [String: String] = [:]
|
||||
for (key, value) in aliases {
|
||||
guard let id = value.stringValue else { continue }
|
||||
let normalizedKey = key.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
||||
let trimmedId = id.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !normalizedKey.isEmpty, !trimmedId.isEmpty else { continue }
|
||||
resolved[normalizedKey] = trimmedId
|
||||
}
|
||||
voiceAliases = resolved
|
||||
} else {
|
||||
voiceAliases = [:]
|
||||
}
|
||||
let model = Self.firstString(activeConfig, keys: ["modelId", "model"])
|
||||
let defaultModelId = (model?.isEmpty == false) ? model! : defaultModelIdFallback
|
||||
let defaultVoiceId = Self.firstString(activeConfig, keys: ["voiceId", "voice"])
|
||||
let defaultOutputFormat = Self.firstString(activeConfig, keys: ["outputFormat"])
|
||||
let realtime = talk?["realtime"]?.dictionaryValue
|
||||
let realtimeProviders = realtime?["providers"]?.dictionaryValue
|
||||
let realtimeProvider = Self.firstString(realtime, keys: ["provider"])
|
||||
?? Self.singleRealtimeProviderId(realtimeProviders)
|
||||
let realtimeProviderConfig = Self.realtimeProviderConfig(
|
||||
providers: realtimeProviders,
|
||||
provider: realtimeProvider)
|
||||
let realtimeModel = Self.firstString(realtime, keys: ["model"])
|
||||
?? Self.firstString(realtimeProviderConfig, keys: ["model"])
|
||||
let realtimeModelId = realtimeModel ?? defaultRealtimeModelIdFallback
|
||||
let realtimeVoiceId = Self.firstString(realtime, keys: ["voice"])
|
||||
?? Self.firstString(realtimeProviderConfig, keys: ["voice"])
|
||||
let realtimeTransport = Self.firstString(realtime, keys: ["transport"])?.lowercased()
|
||||
let requiresGatewayRealtimeTransport = realtimeTransport == "gateway-relay"
|
||||
|| realtimeTransport == "provider-websocket"
|
||||
|| Self.usesAzureOpenAI(provider: realtimeProvider, config: realtimeProviderConfig)
|
||||
let openAIProviderConfig = Self.realtimeProviderConfig(
|
||||
providers: realtimeProviders,
|
||||
provider: "openai")
|
||||
let openAIRequiresGatewayRealtimeTransport = realtimeTransport == "gateway-relay"
|
||||
|| realtimeTransport == "provider-websocket"
|
||||
|| Self.usesAzureOpenAI(provider: "openai", config: openAIProviderConfig)
|
||||
let executionMode = Self.resolvedExecutionMode(
|
||||
realtime,
|
||||
requiresGatewayRealtimeTransport: requiresGatewayRealtimeTransport)
|
||||
let rawConfigApiKey = activeConfig?["apiKey"]?.stringValue?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let interruptOnSpeech = talk?["interruptOnSpeech"]?.boolValue
|
||||
let silenceTimeoutMs = TalkConfigParsing.resolvedSilenceTimeoutMs(
|
||||
talk,
|
||||
fallback: defaultSilenceTimeoutMs)
|
||||
let speechLocaleID = TalkConfigParsing.resolvedSpeechLocaleID(talk)
|
||||
|
||||
return TalkModeGatewayConfigState(
|
||||
activeProvider: activeProvider,
|
||||
normalizedPayload: selection?.normalizedPayload == true,
|
||||
missingResolvedPayload: talk != nil && selection == nil,
|
||||
executionMode: executionMode,
|
||||
requiresGatewayRealtimeTransport: requiresGatewayRealtimeTransport,
|
||||
openAIRequiresGatewayRealtimeTransport: openAIRequiresGatewayRealtimeTransport,
|
||||
defaultVoiceId: defaultVoiceId,
|
||||
voiceAliases: voiceAliases,
|
||||
configuredModelId: model,
|
||||
defaultModelId: defaultModelId,
|
||||
defaultOutputFormat: defaultOutputFormat,
|
||||
realtimeProvider: realtimeProvider,
|
||||
realtimeModelId: realtimeModelId,
|
||||
realtimeVoiceId: realtimeVoiceId,
|
||||
rawConfigApiKey: rawConfigApiKey,
|
||||
interruptOnSpeech: interruptOnSpeech,
|
||||
silenceTimeoutMs: silenceTimeoutMs,
|
||||
speechLocaleID: speechLocaleID)
|
||||
}
|
||||
|
||||
private static func firstString(_ config: [String: AnyCodable]?, keys: [String]) -> String? {
|
||||
guard let config else { return nil }
|
||||
for key in keys {
|
||||
let value = config[key]?.stringValue?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if value?.isEmpty == false {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
private static func resolvedExecutionMode(
|
||||
_ realtime: [String: AnyCodable]?,
|
||||
requiresGatewayRealtimeTransport: Bool) -> TalkModeExecutionMode
|
||||
{
|
||||
guard let realtime else { return .native }
|
||||
let mode = Self.firstString(realtime, keys: ["mode"])?.lowercased()
|
||||
let transport = Self.firstString(realtime, keys: ["transport"])?.lowercased()
|
||||
let provider = Self.firstString(realtime, keys: ["provider"])?.lowercased()
|
||||
?? Self.singleRealtimeProviderId(realtime["providers"]?.dictionaryValue)?.lowercased()
|
||||
let brain = Self.firstString(realtime, keys: ["brain"])?.lowercased()
|
||||
guard mode == "realtime" else {
|
||||
return .native
|
||||
}
|
||||
if brain != nil, brain != "agent-consult" {
|
||||
return .native
|
||||
}
|
||||
if requiresGatewayRealtimeTransport {
|
||||
return .realtimeRelay
|
||||
}
|
||||
switch transport {
|
||||
case "managed-room":
|
||||
return .native
|
||||
case "gateway-relay":
|
||||
return .realtimeRelay
|
||||
case "provider-websocket":
|
||||
return .realtimeRelay
|
||||
case "webrtc":
|
||||
if provider != "openai" {
|
||||
return .realtimeRelay
|
||||
}
|
||||
case nil:
|
||||
if provider != "openai" {
|
||||
return .realtimeRelay
|
||||
}
|
||||
default:
|
||||
return .realtimeRelay
|
||||
}
|
||||
return .realtimeWebRTC
|
||||
}
|
||||
|
||||
private static func usesAzureOpenAI(
|
||||
provider: String?,
|
||||
config: [String: AnyCodable]?) -> Bool
|
||||
{
|
||||
guard provider?.caseInsensitiveCompare("openai") == .orderedSame else { return false }
|
||||
return self.firstString(config, keys: ["azureEndpoint", "azureDeployment"]) != nil
|
||||
}
|
||||
|
||||
private static func singleRealtimeProviderId(_ providers: [String: AnyCodable]?) -> String? {
|
||||
guard let providers, providers.count == 1 else { return nil }
|
||||
let provider = providers.keys.first?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
return provider?.isEmpty == false ? provider : nil
|
||||
}
|
||||
|
||||
private static func realtimeProviderConfig(
|
||||
providers: [String: AnyCodable]?,
|
||||
provider: String?) -> [String: AnyCodable]?
|
||||
{
|
||||
guard let providers else { return nil }
|
||||
if let provider {
|
||||
if let exact = providers[provider]?.dictionaryValue {
|
||||
return exact
|
||||
}
|
||||
return providers.first { key, _ in
|
||||
key.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
.caseInsensitiveCompare(provider) == .orderedSame
|
||||
}?.value.dictionaryValue
|
||||
}
|
||||
if providers.count == 1 {
|
||||
return providers.values.first?.dictionaryValue
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
84
apps/ios/Sources/Voice/TalkModeManager+Permissions.swift
Normal file
84
apps/ios/Sources/Voice/TalkModeManager+Permissions.swift
Normal file
@@ -0,0 +1,84 @@
|
||||
import AVFAudio
|
||||
import Foundation
|
||||
import OpenClawKit
|
||||
import Speech
|
||||
|
||||
extension TalkModeManager {
|
||||
nonisolated static func requestMicrophonePermission() async -> Bool {
|
||||
switch AVAudioApplication.shared.recordPermission {
|
||||
case .granted:
|
||||
return true
|
||||
case .denied:
|
||||
return false
|
||||
case .undetermined:
|
||||
return await self.requestPermissionWithTimeout { completion in
|
||||
AVAudioApplication.requestRecordPermission(completionHandler: { ok in
|
||||
completion(ok)
|
||||
})
|
||||
}
|
||||
@unknown default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
nonisolated static func requestSpeechPermission() async -> Bool {
|
||||
let status = SFSpeechRecognizer.authorizationStatus()
|
||||
switch status {
|
||||
case .authorized:
|
||||
return true
|
||||
case .denied, .restricted:
|
||||
return false
|
||||
case .notDetermined:
|
||||
break
|
||||
@unknown default:
|
||||
return false
|
||||
}
|
||||
|
||||
return await self.requestPermissionWithTimeout { completion in
|
||||
SFSpeechRecognizer.requestAuthorization { authStatus in
|
||||
completion(authStatus == .authorized)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private nonisolated static func requestPermissionWithTimeout(
|
||||
_ operation: @escaping @Sendable (@escaping @Sendable (Bool) -> Void) -> Void) async -> Bool
|
||||
{
|
||||
do {
|
||||
return try await AsyncTimeout.withTimeout(
|
||||
seconds: 8,
|
||||
onTimeout: { NSError(domain: "TalkMode", code: 6, userInfo: [
|
||||
NSLocalizedDescriptionKey: "permission request timed out",
|
||||
]) },
|
||||
operation: {
|
||||
await withCheckedContinuation(isolation: nil) { cont in
|
||||
Task { @MainActor in
|
||||
operation { ok in
|
||||
cont.resume(returning: ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
static func permissionMessage(
|
||||
kind: String,
|
||||
status: SFSpeechRecognizerAuthorizationStatus) -> String
|
||||
{
|
||||
switch status {
|
||||
case .denied:
|
||||
return "\(kind) permission denied"
|
||||
case .restricted:
|
||||
return "\(kind) permission restricted"
|
||||
case .notDetermined:
|
||||
return "\(kind) permission not granted"
|
||||
case .authorized:
|
||||
return "\(kind) permission denied"
|
||||
@unknown default:
|
||||
return "\(kind) permission denied"
|
||||
}
|
||||
}
|
||||
}
|
||||
3468
apps/ios/Sources/Voice/TalkModeManager.swift
Normal file
3468
apps/ios/Sources/Voice/TalkModeManager.swift
Normal file
File diff suppressed because it is too large
Load Diff
178
apps/ios/Sources/Voice/TalkPermissionPromptView.swift
Normal file
178
apps/ios/Sources/Voice/TalkPermissionPromptView.swift
Normal file
@@ -0,0 +1,178 @@
|
||||
import SwiftUI
|
||||
|
||||
struct TalkPermissionPromptView: View {
|
||||
enum Style {
|
||||
case card
|
||||
case sheet
|
||||
}
|
||||
|
||||
@Environment(NodeAppModel.self) private var appModel
|
||||
|
||||
let style: Style
|
||||
var onPermissionReady: (() -> Void)?
|
||||
|
||||
private var state: TalkGatewayPermissionState {
|
||||
self.appModel.talkMode.gatewayTalkPermissionState
|
||||
}
|
||||
|
||||
private var requestIsPending: Bool {
|
||||
self.state.isApprovalRequestInProgress
|
||||
}
|
||||
|
||||
private var pollTaskKey: String {
|
||||
switch self.state {
|
||||
case .requestingUpgrade:
|
||||
"requesting"
|
||||
case let .upgradeRequested(requestId):
|
||||
"pending:\(requestId ?? "")"
|
||||
default:
|
||||
"idle:\(self.state.statusLabel)"
|
||||
}
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
VStack(alignment: .leading, spacing: self.style == .sheet ? 16 : 12) {
|
||||
HStack(alignment: .top, spacing: 12) {
|
||||
Image(systemName: self.iconSystemName)
|
||||
.font(OpenClawType.title3)
|
||||
.foregroundStyle(self.requestIsPending ? OpenClawBrand.warn : OpenClawBrand.accent)
|
||||
.frame(width: 28, height: 28)
|
||||
|
||||
VStack(alignment: .leading, spacing: 6) {
|
||||
Text(self.titleText)
|
||||
.font(self.style == .sheet ? OpenClawType.title3SemiBold : OpenClawType.headline)
|
||||
Text(self.messageText)
|
||||
.font(OpenClawType.footnote)
|
||||
.foregroundStyle(.secondary)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
}
|
||||
}
|
||||
|
||||
if let failureMessage = self.state.failureMessage {
|
||||
Label(failureMessage, systemImage: "exclamationmark.triangle.fill")
|
||||
.font(OpenClawType.footnote)
|
||||
.foregroundStyle(OpenClawBrand.danger)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
}
|
||||
|
||||
if let requestId = self.state.requestId {
|
||||
LabeledContent {
|
||||
Text(requestId)
|
||||
.font(OpenClawType.monoSmall)
|
||||
.foregroundStyle(.secondary)
|
||||
.textSelection(.enabled)
|
||||
} label: {
|
||||
Text("Request ID")
|
||||
.font(OpenClawType.footnoteSemiBold)
|
||||
}
|
||||
}
|
||||
|
||||
HStack(spacing: 10) {
|
||||
Button {
|
||||
self.appModel.requestTalkPermissionUpgrade()
|
||||
} label: {
|
||||
if case .requestingUpgrade = self.state {
|
||||
Label {
|
||||
Text("Sending...")
|
||||
.font(OpenClawType.subheadSemiBold)
|
||||
} icon: {
|
||||
ProgressView()
|
||||
}
|
||||
} else {
|
||||
Label(self.primaryButtonTitle, systemImage: self.primaryButtonSystemImage)
|
||||
.font(OpenClawType.subheadSemiBold)
|
||||
}
|
||||
}
|
||||
.buttonStyle(.borderedProminent)
|
||||
.disabled(self.state == .requestingUpgrade)
|
||||
|
||||
Button {
|
||||
Task { await self.appModel.talkMode.reloadConfig() }
|
||||
} label: {
|
||||
Label("Retry", systemImage: "arrow.triangle.2.circlepath")
|
||||
.font(OpenClawType.subheadSemiBold)
|
||||
}
|
||||
.buttonStyle(.bordered)
|
||||
}
|
||||
}
|
||||
.padding(self.style == .card || self.style == .sheet ? 16 : 0)
|
||||
.background {
|
||||
if self.style == .card || self.style == .sheet {
|
||||
RoundedRectangle(cornerRadius: OpenClawRadius.md, style: .continuous)
|
||||
.fill(.thinMaterial)
|
||||
}
|
||||
}
|
||||
.overlay {
|
||||
if self.style == .card || self.style == .sheet {
|
||||
RoundedRectangle(cornerRadius: OpenClawRadius.md, style: .continuous)
|
||||
.stroke(OpenClawBrand.accent.opacity(0.20), lineWidth: 1)
|
||||
}
|
||||
}
|
||||
.task(id: self.pollTaskKey) {
|
||||
guard self.requestIsPending else { return }
|
||||
await self.pollUntilReady()
|
||||
}
|
||||
.onChange(of: self.state) { _, newState in
|
||||
if newState == .ready {
|
||||
self.onPermissionReady?()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private var iconSystemName: String {
|
||||
switch self.state {
|
||||
case .requestingUpgrade:
|
||||
"paperplane.fill"
|
||||
case .upgradeRequested:
|
||||
"hourglass"
|
||||
case .requestFailed:
|
||||
"exclamationmark.triangle.fill"
|
||||
default:
|
||||
"key.fill"
|
||||
}
|
||||
}
|
||||
|
||||
private var titleText: String {
|
||||
switch self.state {
|
||||
case .requestingUpgrade:
|
||||
"Sending approval request"
|
||||
case .upgradeRequested:
|
||||
"Approval sent"
|
||||
case .requestFailed:
|
||||
"Could not request approval"
|
||||
default:
|
||||
"Enable Talk"
|
||||
}
|
||||
}
|
||||
|
||||
private var messageText: String {
|
||||
switch self.state {
|
||||
case .requestingUpgrade:
|
||||
"Sending a new pairing request to your gateway..."
|
||||
case .upgradeRequested:
|
||||
"Approve this request on your gateway. Talk will start automatically when approval lands."
|
||||
default:
|
||||
"This device needs gateway approval before Talk can use realtime voice. Audio will go directly from " +
|
||||
"this device to the voice provider."
|
||||
}
|
||||
}
|
||||
|
||||
private var primaryButtonTitle: String {
|
||||
self.requestIsPending ? "Request Again" : "Send Approval Request"
|
||||
}
|
||||
|
||||
private var primaryButtonSystemImage: String {
|
||||
self.requestIsPending ? "arrow.clockwise" : "paperplane.fill"
|
||||
}
|
||||
|
||||
private func pollUntilReady() async {
|
||||
while !Task.isCancelled {
|
||||
try? await Task.sleep(nanoseconds: 3_000_000_000)
|
||||
if Task.isCancelled { return }
|
||||
await self.appModel.pollTalkPermissionUpgrade()
|
||||
if !self.appModel.talkMode.gatewayTalkPermissionState.requiresTalkPermissionAction {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
97
apps/ios/Sources/Voice/TalkRealtimeClientSession.swift
Normal file
97
apps/ios/Sources/Voice/TalkRealtimeClientSession.swift
Normal file
@@ -0,0 +1,97 @@
|
||||
import Foundation
|
||||
|
||||
struct TalkRealtimeClientCreateParams: Encodable {
|
||||
var mode = "realtime"
|
||||
var provider: String?
|
||||
var transport = "webrtc"
|
||||
var brain = "agent-consult"
|
||||
var model: String?
|
||||
var voice: String?
|
||||
}
|
||||
|
||||
struct TalkRealtimeClientSession: Decodable {
|
||||
let provider: String
|
||||
let transport: String
|
||||
let clientSecret: String
|
||||
let offerUrl: String?
|
||||
let offerHeaders: [String: String]?
|
||||
let model: String?
|
||||
let voice: String?
|
||||
let expiresAt: Double?
|
||||
|
||||
var isWebRTC: Bool {
|
||||
self.transport.caseInsensitiveCompare("webrtc") == .orderedSame
|
||||
}
|
||||
}
|
||||
|
||||
struct TalkRealtimeToolCallResponse: Decodable {
|
||||
let runId: String?
|
||||
let idempotencyKey: String?
|
||||
}
|
||||
|
||||
struct TalkRealtimeServerEvent: Decodable {
|
||||
let type: String
|
||||
let error: TalkRealtimeServerError?
|
||||
let itemId: String?
|
||||
let item: TalkRealtimeServerItem?
|
||||
let callId: String?
|
||||
let name: String?
|
||||
let delta: String?
|
||||
let arguments: String?
|
||||
let transcript: String?
|
||||
let text: String?
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case type
|
||||
case error
|
||||
case itemId = "item_id"
|
||||
case item
|
||||
case callId = "call_id"
|
||||
case name
|
||||
case delta
|
||||
case arguments
|
||||
case transcript
|
||||
case text
|
||||
}
|
||||
|
||||
var resolvedItemId: String? {
|
||||
self.itemId ?? self.item?.id
|
||||
}
|
||||
|
||||
var resolvedCallId: String? {
|
||||
self.callId ?? self.item?.callId
|
||||
}
|
||||
|
||||
var resolvedName: String? {
|
||||
self.name ?? self.item?.name
|
||||
}
|
||||
|
||||
var resolvedArguments: String? {
|
||||
self.arguments ?? self.item?.arguments
|
||||
}
|
||||
|
||||
var isMaximumDurationError: Bool {
|
||||
guard self.type == "error", let message = self.error?.message?.lowercased() else { return false }
|
||||
return message.contains("session") && message.contains("maximum duration")
|
||||
}
|
||||
}
|
||||
|
||||
struct TalkRealtimeServerError: Decodable {
|
||||
let message: String?
|
||||
}
|
||||
|
||||
struct TalkRealtimeServerItem: Decodable {
|
||||
let id: String?
|
||||
let type: String?
|
||||
let callId: String?
|
||||
let name: String?
|
||||
let arguments: String?
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case id
|
||||
case type
|
||||
case callId = "call_id"
|
||||
case name
|
||||
case arguments
|
||||
}
|
||||
}
|
||||
1028
apps/ios/Sources/Voice/TalkRealtimeWebRTCSession.swift
Normal file
1028
apps/ios/Sources/Voice/TalkRealtimeWebRTCSession.swift
Normal file
File diff suppressed because it is too large
Load Diff
94
apps/ios/Sources/Voice/TalkSpeechLocale.swift
Normal file
94
apps/ios/Sources/Voice/TalkSpeechLocale.swift
Normal file
@@ -0,0 +1,94 @@
|
||||
import Foundation
|
||||
import OpenClawKit
|
||||
import Speech
|
||||
|
||||
enum TalkSpeechLocale {
|
||||
static let storageKey = "talk.speechLocale"
|
||||
static let automaticID = "auto"
|
||||
static let fallbackLocaleID = "en-US"
|
||||
|
||||
struct Option: Identifiable {
|
||||
let id: String
|
||||
let label: String
|
||||
}
|
||||
|
||||
static func supportedOptions(
|
||||
supportedLocales: Set<Locale> = SFSpeechRecognizer.supportedLocales()) -> [Option]
|
||||
{
|
||||
var seen = Set<String>()
|
||||
let dynamic: [Option] = supportedLocales
|
||||
.compactMap { locale in
|
||||
let id = self.canonicalID(locale.identifier)
|
||||
guard seen.insert(id).inserted else { return nil }
|
||||
return Option(id: id, label: self.friendlyName(for: locale))
|
||||
}
|
||||
.sorted { (lhs: Option, rhs: Option) in
|
||||
lhs.label.localizedCaseInsensitiveCompare(rhs.label) == .orderedAscending
|
||||
}
|
||||
return [Option(id: self.automaticID, label: "Automatic")] + dynamic
|
||||
}
|
||||
|
||||
static func resolvedLocaleID(
|
||||
localSelection: String?,
|
||||
gatewaySelection: String?,
|
||||
deviceLocaleID: String = Locale.autoupdatingCurrent.identifier,
|
||||
fallbackLocaleID: String = Self.fallbackLocaleID,
|
||||
supportedLocaleIDs: Set<String>) -> String?
|
||||
{
|
||||
TalkConfigParsing.resolvedSpeechRecognitionLocaleID(
|
||||
preferredLocaleIDs: [
|
||||
TalkConfigParsing.normalizedExplicitSpeechLocaleID(localSelection),
|
||||
TalkConfigParsing.normalizedExplicitSpeechLocaleID(gatewaySelection),
|
||||
deviceLocaleID,
|
||||
],
|
||||
fallbackLocaleID: fallbackLocaleID,
|
||||
supportedLocaleIDs: supportedLocaleIDs)
|
||||
}
|
||||
|
||||
static func makeRecognizer(
|
||||
localSelection: String?,
|
||||
gatewaySelection: String?,
|
||||
supportedLocales: Set<Locale> = SFSpeechRecognizer.supportedLocales()) -> (
|
||||
recognizer: SFSpeechRecognizer?,
|
||||
localeID: String?)
|
||||
{
|
||||
let supportedIDs = Set(supportedLocales.map(\.identifier))
|
||||
guard let localeID = self.resolvedLocaleID(
|
||||
localSelection: localSelection,
|
||||
gatewaySelection: gatewaySelection,
|
||||
supportedLocaleIDs: supportedIDs)
|
||||
else {
|
||||
let recognizer = SFSpeechRecognizer()
|
||||
return (recognizer, recognizer?.locale.identifier)
|
||||
}
|
||||
|
||||
if let recognizer = SFSpeechRecognizer(locale: Locale(identifier: localeID)) {
|
||||
return (recognizer, localeID)
|
||||
}
|
||||
|
||||
let recognizer = SFSpeechRecognizer()
|
||||
return (recognizer, recognizer?.locale.identifier)
|
||||
}
|
||||
|
||||
private static func canonicalID(_ raw: String) -> String {
|
||||
raw.replacingOccurrences(of: "_", with: "-")
|
||||
}
|
||||
|
||||
private static func friendlyName(for locale: Locale) -> String {
|
||||
let id = self.canonicalID(locale.identifier)
|
||||
let cleanLocale = Locale(identifier: id)
|
||||
if let langCode = cleanLocale.language.languageCode?.identifier,
|
||||
let lang = cleanLocale.localizedString(forLanguageCode: langCode),
|
||||
let regionCode = cleanLocale.region?.identifier,
|
||||
let region = cleanLocale.localizedString(forRegionCode: regionCode)
|
||||
{
|
||||
return "\(lang) (\(region))"
|
||||
}
|
||||
if let langCode = cleanLocale.language.languageCode?.identifier,
|
||||
let lang = cleanLocale.localizedString(forLanguageCode: langCode)
|
||||
{
|
||||
return lang
|
||||
}
|
||||
return cleanLocale.localizedString(forIdentifier: id) ?? id
|
||||
}
|
||||
}
|
||||
553
apps/ios/Sources/Voice/VoiceWakeManager.swift
Normal file
553
apps/ios/Sources/Voice/VoiceWakeManager.swift
Normal file
@@ -0,0 +1,553 @@
|
||||
import AVFAudio
|
||||
import Foundation
|
||||
import Observation
|
||||
import OpenClawKit
|
||||
import Speech
|
||||
import SwabbleKit
|
||||
|
||||
private func makeAudioTapEnqueueCallback(queue: AudioBufferQueue) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void {
|
||||
{ buffer, _ in
|
||||
// This callback is invoked on a realtime audio thread/queue. Keep it tiny and nonisolated.
|
||||
queue.enqueueCopy(of: buffer)
|
||||
}
|
||||
}
|
||||
|
||||
private final class AudioBufferQueue: @unchecked Sendable {
|
||||
private let lock = NSLock()
|
||||
private var buffers: [AVAudioPCMBuffer] = []
|
||||
|
||||
func enqueueCopy(of buffer: AVAudioPCMBuffer) {
|
||||
guard let copy = buffer.deepCopy() else { return }
|
||||
self.lock.lock()
|
||||
self.buffers.append(copy)
|
||||
self.lock.unlock()
|
||||
}
|
||||
|
||||
func drain() -> [AVAudioPCMBuffer] {
|
||||
self.lock.lock()
|
||||
let drained = self.buffers
|
||||
self.buffers.removeAll(keepingCapacity: true)
|
||||
self.lock.unlock()
|
||||
return drained
|
||||
}
|
||||
|
||||
func clear() {
|
||||
self.lock.lock()
|
||||
self.buffers.removeAll(keepingCapacity: false)
|
||||
self.lock.unlock()
|
||||
}
|
||||
}
|
||||
|
||||
private enum VoiceWakeAudioError: LocalizedError {
|
||||
case invalidInputFormat
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .invalidInputFormat:
|
||||
"Microphone input format unavailable"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extension AVAudioPCMBuffer {
|
||||
fileprivate func deepCopy() -> AVAudioPCMBuffer? {
|
||||
let format = self.format
|
||||
let frameLength = self.frameLength
|
||||
guard let copy = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frameLength) else {
|
||||
return nil
|
||||
}
|
||||
copy.frameLength = frameLength
|
||||
|
||||
if let src = self.floatChannelData, let dst = copy.floatChannelData {
|
||||
let channels = Int(format.channelCount)
|
||||
let frames = Int(frameLength)
|
||||
for ch in 0..<channels {
|
||||
dst[ch].update(from: src[ch], count: frames)
|
||||
}
|
||||
return copy
|
||||
}
|
||||
|
||||
if let src = self.int16ChannelData, let dst = copy.int16ChannelData {
|
||||
let channels = Int(format.channelCount)
|
||||
let frames = Int(frameLength)
|
||||
for ch in 0..<channels {
|
||||
dst[ch].update(from: src[ch], count: frames)
|
||||
}
|
||||
return copy
|
||||
}
|
||||
|
||||
if let src = self.int32ChannelData, let dst = copy.int32ChannelData {
|
||||
let channels = Int(format.channelCount)
|
||||
let frames = Int(frameLength)
|
||||
for ch in 0..<channels {
|
||||
dst[ch].update(from: src[ch], count: frames)
|
||||
}
|
||||
return copy
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
@Observable
|
||||
final class VoiceWakeManager: NSObject {
|
||||
var isEnabled: Bool = false
|
||||
var isListening: Bool = false
|
||||
var statusText: String = "Off"
|
||||
var triggerWords: [String] = VoiceWakePreferences.loadTriggerWords()
|
||||
var lastTriggeredCommand: String?
|
||||
|
||||
private let audioEngine = AVAudioEngine()
|
||||
private var speechRecognizer: SFSpeechRecognizer?
|
||||
private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest?
|
||||
private var recognitionTask: SFSpeechRecognitionTask?
|
||||
private var tapQueue: AudioBufferQueue?
|
||||
private var tapDrainTask: Task<Void, Never>?
|
||||
private var scheduledStartTask: Task<Void, Never>?
|
||||
private var isStarting: Bool = false
|
||||
private var isSuspendedForExternalAudio: Bool = false
|
||||
|
||||
private var lastDispatched: String?
|
||||
private var onCommand: (@Sendable (String) async -> Void)?
|
||||
private var userDefaultsObserver: NSObjectProtocol?
|
||||
private var suppressedByTalk: Bool = false
|
||||
|
||||
private let externalAudioResumeDelayNs: UInt64
|
||||
private let recognitionErrorRestartDelayNs: UInt64
|
||||
|
||||
override convenience init() {
|
||||
self.init(externalAudioResumeDelayNs: 350_000_000, recognitionErrorRestartDelayNs: 700_000_000)
|
||||
}
|
||||
|
||||
private init(externalAudioResumeDelayNs: UInt64, recognitionErrorRestartDelayNs: UInt64) {
|
||||
self.externalAudioResumeDelayNs = externalAudioResumeDelayNs
|
||||
self.recognitionErrorRestartDelayNs = recognitionErrorRestartDelayNs
|
||||
super.init()
|
||||
self.triggerWords = VoiceWakePreferences.loadTriggerWords()
|
||||
self.userDefaultsObserver = NotificationCenter.default.addObserver(
|
||||
forName: UserDefaults.didChangeNotification,
|
||||
object: UserDefaults.standard,
|
||||
queue: .main,
|
||||
using: { [weak self] _ in
|
||||
Task { @MainActor in
|
||||
self?.handleUserDefaultsDidChange()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@MainActor deinit {
|
||||
if let userDefaultsObserver = self.userDefaultsObserver {
|
||||
NotificationCenter.default.removeObserver(userDefaultsObserver)
|
||||
}
|
||||
}
|
||||
|
||||
var activeTriggerWords: [String] {
|
||||
VoiceWakePreferences.sanitizeTriggerWords(self.triggerWords)
|
||||
}
|
||||
|
||||
private func handleUserDefaultsDidChange() {
|
||||
let updated = VoiceWakePreferences.loadTriggerWords()
|
||||
if updated != self.triggerWords {
|
||||
self.triggerWords = updated
|
||||
}
|
||||
}
|
||||
|
||||
func configure(onCommand: @escaping @Sendable (String) async -> Void) {
|
||||
self.onCommand = onCommand
|
||||
}
|
||||
|
||||
func setEnabled(_ enabled: Bool) {
|
||||
self.isEnabled = enabled
|
||||
if enabled {
|
||||
self.scheduleStart()
|
||||
} else {
|
||||
self.stop()
|
||||
}
|
||||
}
|
||||
|
||||
func setSuppressedByTalk(_ suppressed: Bool) {
|
||||
self.suppressedByTalk = suppressed
|
||||
if suppressed {
|
||||
self.cancelScheduledStart()
|
||||
if self.isListening {
|
||||
self.isListening = false
|
||||
self.tearDownRecognitionPipeline()
|
||||
}
|
||||
if self.isEnabled {
|
||||
self.statusText = "Paused"
|
||||
}
|
||||
} else if self.isEnabled {
|
||||
self.scheduleStart()
|
||||
}
|
||||
}
|
||||
|
||||
private func scheduleStart(after delayNs: UInt64 = 0) {
|
||||
guard self.isEnabled else { return }
|
||||
|
||||
self.scheduledStartTask?.cancel()
|
||||
self.scheduledStartTask = Task { [weak self] in
|
||||
if delayNs > 0 {
|
||||
try? await Task.sleep(nanoseconds: delayNs)
|
||||
}
|
||||
guard !Task.isCancelled else { return }
|
||||
self?.scheduledStartTask = nil
|
||||
await self?.start()
|
||||
}
|
||||
}
|
||||
|
||||
private func cancelScheduledStart() {
|
||||
self.scheduledStartTask?.cancel()
|
||||
self.scheduledStartTask = nil
|
||||
}
|
||||
|
||||
func start() async {
|
||||
guard self.isEnabled else { return }
|
||||
if self.isListening { return }
|
||||
if self.isStarting { return }
|
||||
guard !self.isSuspendedForExternalAudio else {
|
||||
self.isListening = false
|
||||
self.statusText = "Paused"
|
||||
return
|
||||
}
|
||||
|
||||
self.isStarting = true
|
||||
defer { self.isStarting = false }
|
||||
|
||||
guard !self.suppressedByTalk else {
|
||||
self.isListening = false
|
||||
self.statusText = "Paused"
|
||||
return
|
||||
}
|
||||
|
||||
if ProcessInfo.processInfo.environment["SIMULATOR_DEVICE_NAME"] != nil ||
|
||||
ProcessInfo.processInfo.environment["SIMULATOR_UDID"] != nil
|
||||
{
|
||||
// The iOS Simulator’s audio stack is unreliable for long-running microphone capture.
|
||||
// (We’ve observed CoreAudio deadlocks after TCC permission prompts.)
|
||||
self.isListening = false
|
||||
self.statusText = "Voice Wake isn’t supported on Simulator"
|
||||
return
|
||||
}
|
||||
|
||||
self.statusText = "Requesting permissions…"
|
||||
|
||||
let micOk = await Self.requestMicrophonePermission()
|
||||
guard micOk else {
|
||||
self.statusText = Self.microphonePermissionMessage(kind: "Microphone")
|
||||
self.isListening = false
|
||||
return
|
||||
}
|
||||
|
||||
let speechOk = await Self.requestSpeechPermission()
|
||||
guard speechOk else {
|
||||
self.statusText = Self.permissionMessage(
|
||||
kind: "Speech recognition",
|
||||
status: SFSpeechRecognizer.authorizationStatus())
|
||||
self.isListening = false
|
||||
return
|
||||
}
|
||||
|
||||
self.speechRecognizer = SFSpeechRecognizer()
|
||||
guard self.speechRecognizer != nil else {
|
||||
self.statusText = "Speech recognizer unavailable"
|
||||
self.isListening = false
|
||||
return
|
||||
}
|
||||
|
||||
guard self.isEnabled, !self.suppressedByTalk, !self.isSuspendedForExternalAudio else {
|
||||
self.isListening = false
|
||||
self.statusText = self.isEnabled ? "Paused" : "Off"
|
||||
return
|
||||
}
|
||||
|
||||
do {
|
||||
try Self.configureAudioSession()
|
||||
try self.startRecognition()
|
||||
self.isListening = true
|
||||
self.statusText = "Listening"
|
||||
} catch {
|
||||
self.isListening = false
|
||||
self.tearDownRecognitionPipeline()
|
||||
self.statusText = "Start failed: \(error.localizedDescription)"
|
||||
}
|
||||
}
|
||||
|
||||
func stop() {
|
||||
self.isEnabled = false
|
||||
self.isListening = false
|
||||
self.statusText = "Off"
|
||||
self.isSuspendedForExternalAudio = false
|
||||
self.cancelScheduledStart()
|
||||
self.tearDownRecognitionPipeline()
|
||||
}
|
||||
|
||||
/// Temporarily releases the microphone so other subsystems (e.g. camera video capture) can record audio.
|
||||
/// Returns `true` when listening, starting, or a pending restart was active and was suspended.
|
||||
func suspendForExternalAudioCapture() -> Bool {
|
||||
let hadPendingStart = self.scheduledStartTask != nil
|
||||
self.cancelScheduledStart()
|
||||
guard self.isEnabled, self.isListening || self.isStarting || hadPendingStart else { return false }
|
||||
|
||||
self.isSuspendedForExternalAudio = true
|
||||
self.isListening = false
|
||||
self.statusText = "Paused"
|
||||
self.tearDownRecognitionPipeline()
|
||||
return true
|
||||
}
|
||||
|
||||
func resumeAfterExternalAudioCapture(wasSuspended: Bool) {
|
||||
guard wasSuspended else { return }
|
||||
self.isSuspendedForExternalAudio = false
|
||||
self.scheduleStart(after: self.externalAudioResumeDelayNs)
|
||||
}
|
||||
|
||||
private func startRecognition() throws {
|
||||
guard self.isEnabled, !self.suppressedByTalk, !self.isSuspendedForExternalAudio else { return }
|
||||
|
||||
self.recognitionTask?.cancel()
|
||||
self.recognitionTask = nil
|
||||
self.tapDrainTask?.cancel()
|
||||
self.tapDrainTask = nil
|
||||
self.tapQueue?.clear()
|
||||
self.tapQueue = nil
|
||||
|
||||
let request = SFSpeechAudioBufferRecognitionRequest()
|
||||
request.shouldReportPartialResults = true
|
||||
self.recognitionRequest = request
|
||||
|
||||
let inputNode = self.audioEngine.inputNode
|
||||
inputNode.removeTap(onBus: 0)
|
||||
|
||||
let recordingFormat = inputNode.outputFormat(forBus: 0)
|
||||
guard recordingFormat.sampleRate > 0, recordingFormat.channelCount > 0 else {
|
||||
throw VoiceWakeAudioError.invalidInputFormat
|
||||
}
|
||||
|
||||
let queue = AudioBufferQueue()
|
||||
self.tapQueue = queue
|
||||
let tapBlock: @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void = makeAudioTapEnqueueCallback(queue: queue)
|
||||
inputNode.installTap(
|
||||
onBus: 0,
|
||||
bufferSize: 1024,
|
||||
format: recordingFormat,
|
||||
block: tapBlock)
|
||||
|
||||
self.audioEngine.prepare()
|
||||
try self.audioEngine.start()
|
||||
|
||||
let handler = self.makeRecognitionResultHandler()
|
||||
self.recognitionTask = self.speechRecognizer?.recognitionTask(with: request, resultHandler: handler)
|
||||
|
||||
self.tapDrainTask = Task { [weak self] in
|
||||
guard let self, let queue = self.tapQueue else { return }
|
||||
while !Task.isCancelled {
|
||||
try? await Task.sleep(nanoseconds: 40_000_000)
|
||||
let drained = queue.drain()
|
||||
if drained.isEmpty { continue }
|
||||
for buf in drained {
|
||||
request.append(buf)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func tearDownRecognitionPipeline() {
|
||||
self.tapDrainTask?.cancel()
|
||||
self.tapDrainTask = nil
|
||||
self.tapQueue?.clear()
|
||||
self.tapQueue = nil
|
||||
|
||||
self.recognitionTask?.cancel()
|
||||
self.recognitionTask = nil
|
||||
self.recognitionRequest = nil
|
||||
|
||||
if self.audioEngine.isRunning {
|
||||
self.audioEngine.stop()
|
||||
}
|
||||
// A tap can be installed before AVAudioEngine.start() throws.
|
||||
self.audioEngine.inputNode.removeTap(onBus: 0)
|
||||
|
||||
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
|
||||
}
|
||||
|
||||
private nonisolated func makeRecognitionResultHandler() -> @Sendable (SFSpeechRecognitionResult?, Error?) -> Void {
|
||||
{ [weak self] result, error in
|
||||
let transcript = result?.bestTranscription.formattedString
|
||||
let segments = result.flatMap { result in
|
||||
transcript.map { WakeWordSpeechSegments.from(transcription: result.bestTranscription, transcript: $0) }
|
||||
} ?? []
|
||||
let errorText = error?.localizedDescription
|
||||
|
||||
Task { @MainActor in
|
||||
self?.handleRecognitionCallback(transcript: transcript, segments: segments, errorText: errorText)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func handleRecognitionCallback(transcript: String?, segments: [WakeWordSegment], errorText: String?) {
|
||||
if let errorText {
|
||||
self.statusText = "Recognizer error: \(errorText)"
|
||||
self.isListening = false
|
||||
|
||||
self.scheduleStart(after: self.recognitionErrorRestartDelayNs)
|
||||
return
|
||||
}
|
||||
|
||||
guard let transcript else { return }
|
||||
guard let cmd = self.extractCommand(from: transcript, segments: segments) else { return }
|
||||
|
||||
if cmd == self.lastDispatched { return }
|
||||
self.lastDispatched = cmd
|
||||
self.lastTriggeredCommand = cmd
|
||||
self.statusText = "Triggered"
|
||||
|
||||
Task { [weak self] in
|
||||
guard let self else { return }
|
||||
await self.onCommand?(cmd)
|
||||
await self.startIfEnabled()
|
||||
}
|
||||
}
|
||||
|
||||
private func startIfEnabled() async {
|
||||
self.scheduleStart()
|
||||
}
|
||||
|
||||
private func extractCommand(from transcript: String, segments: [WakeWordSegment]) -> String? {
|
||||
Self.extractCommand(from: transcript, segments: segments, triggers: self.activeTriggerWords)
|
||||
}
|
||||
|
||||
nonisolated static func extractCommand(
|
||||
from transcript: String,
|
||||
segments: [WakeWordSegment],
|
||||
triggers: [String],
|
||||
minPostTriggerGap: TimeInterval = 0.45) -> String?
|
||||
{
|
||||
let config = WakeWordGateConfig(triggers: triggers, minPostTriggerGap: minPostTriggerGap)
|
||||
return WakeWordGate.match(transcript: transcript, segments: segments, config: config)?.command
|
||||
}
|
||||
|
||||
private static func configureAudioSession() throws {
|
||||
let session = AVAudioSession.sharedInstance()
|
||||
try session.setCategory(.playAndRecord, mode: .measurement, options: [
|
||||
.duckOthers,
|
||||
.mixWithOthers,
|
||||
.allowBluetoothHFP,
|
||||
.defaultToSpeaker,
|
||||
])
|
||||
try session.setActive(true, options: [])
|
||||
}
|
||||
|
||||
private nonisolated static func requestMicrophonePermission() async -> Bool {
|
||||
switch AVAudioApplication.shared.recordPermission {
|
||||
case .granted:
|
||||
return true
|
||||
case .denied:
|
||||
return false
|
||||
case .undetermined:
|
||||
break
|
||||
@unknown default:
|
||||
return false
|
||||
}
|
||||
|
||||
return await self.requestPermissionWithTimeout { completion in
|
||||
AVAudioApplication.requestRecordPermission(completionHandler: completion)
|
||||
}
|
||||
}
|
||||
|
||||
private nonisolated static func microphonePermissionMessage(kind: String) -> String {
|
||||
let status = AVAudioApplication.shared.recordPermission
|
||||
return self.deniedByDefaultPermissionMessage(
|
||||
kind: kind,
|
||||
isUndetermined: status == .undetermined)
|
||||
}
|
||||
|
||||
private nonisolated static func requestSpeechPermission() async -> Bool {
|
||||
let status = SFSpeechRecognizer.authorizationStatus()
|
||||
switch status {
|
||||
case .authorized:
|
||||
return true
|
||||
case .denied, .restricted:
|
||||
return false
|
||||
case .notDetermined:
|
||||
break
|
||||
@unknown default:
|
||||
return false
|
||||
}
|
||||
|
||||
return await self.requestPermissionWithTimeout { completion in
|
||||
SFSpeechRecognizer.requestAuthorization { authStatus in
|
||||
completion(authStatus == .authorized)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private nonisolated static func requestPermissionWithTimeout(
|
||||
_ operation: @escaping @Sendable (@escaping @Sendable (Bool) -> Void) -> Void) async -> Bool
|
||||
{
|
||||
do {
|
||||
return try await AsyncTimeout.withTimeout(
|
||||
seconds: 8,
|
||||
onTimeout: { NSError(domain: "VoiceWake", code: 6, userInfo: [
|
||||
NSLocalizedDescriptionKey: "permission request timed out",
|
||||
]) },
|
||||
operation: {
|
||||
await withCheckedContinuation(isolation: nil) { cont in
|
||||
Task { @MainActor in
|
||||
operation { ok in
|
||||
cont.resume(returning: ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
private static func permissionMessage(
|
||||
kind: String,
|
||||
status: SFSpeechRecognizerAuthorizationStatus) -> String
|
||||
{
|
||||
switch status {
|
||||
case .denied:
|
||||
return "\(kind) permission denied"
|
||||
case .restricted:
|
||||
return "\(kind) permission restricted"
|
||||
case .notDetermined:
|
||||
return "\(kind) permission not granted"
|
||||
case .authorized:
|
||||
return "\(kind) permission denied"
|
||||
@unknown default:
|
||||
return "\(kind) permission denied"
|
||||
}
|
||||
}
|
||||
|
||||
private nonisolated static func deniedByDefaultPermissionMessage(kind: String, isUndetermined: Bool) -> String {
|
||||
if isUndetermined {
|
||||
return "\(kind) permission not granted"
|
||||
}
|
||||
return "\(kind) permission denied"
|
||||
}
|
||||
}
|
||||
|
||||
#if DEBUG
|
||||
extension VoiceWakeManager {
|
||||
static func _test_withoutRestartDelays() -> VoiceWakeManager {
|
||||
VoiceWakeManager(externalAudioResumeDelayNs: 0, recognitionErrorRestartDelayNs: 0)
|
||||
}
|
||||
|
||||
func _test_handleRecognitionCallback(transcript: String?, segments: [WakeWordSegment], errorText: String?) {
|
||||
self.handleRecognitionCallback(transcript: transcript, segments: segments, errorText: errorText)
|
||||
}
|
||||
|
||||
func _test_setStartInFlight(_ isStarting: Bool) {
|
||||
self.isStarting = isStarting
|
||||
}
|
||||
|
||||
func _test_waitForScheduledStart() async {
|
||||
let task = self.scheduledStartTask
|
||||
await task?.value
|
||||
}
|
||||
}
|
||||
#endif
|
||||
44
apps/ios/Sources/Voice/VoiceWakePreferences.swift
Normal file
44
apps/ios/Sources/Voice/VoiceWakePreferences.swift
Normal file
@@ -0,0 +1,44 @@
|
||||
import Foundation
|
||||
|
||||
enum VoiceWakePreferences {
|
||||
static let enabledKey = "voiceWake.enabled"
|
||||
static let triggerWordsKey = "voiceWake.triggerWords"
|
||||
|
||||
// Keep defaults aligned with the mac app.
|
||||
static let defaultTriggerWords: [String] = ["openclaw", "claude"]
|
||||
static let maxWords = 32
|
||||
static let maxWordLength = 64
|
||||
|
||||
static func decodeGatewayTriggers(from payloadJSON: String) -> [String]? {
|
||||
guard let data = payloadJSON.data(using: .utf8) else { return nil }
|
||||
return self.decodeGatewayTriggers(from: data)
|
||||
}
|
||||
|
||||
static func decodeGatewayTriggers(from data: Data) -> [String]? {
|
||||
struct Payload: Decodable { var triggers: [String] }
|
||||
guard let decoded = try? JSONDecoder().decode(Payload.self, from: data) else { return nil }
|
||||
return self.sanitizeTriggerWords(decoded.triggers)
|
||||
}
|
||||
|
||||
static func loadTriggerWords(defaults: UserDefaults = .standard) -> [String] {
|
||||
defaults.stringArray(forKey: self.triggerWordsKey) ?? self.defaultTriggerWords
|
||||
}
|
||||
|
||||
static func saveTriggerWords(_ words: [String], defaults: UserDefaults = .standard) {
|
||||
defaults.set(words, forKey: self.triggerWordsKey)
|
||||
}
|
||||
|
||||
static func sanitizeTriggerWords(_ words: [String]) -> [String] {
|
||||
let cleaned = words
|
||||
.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
|
||||
.filter { !$0.isEmpty }
|
||||
.prefix(Self.maxWords)
|
||||
.map { String($0.prefix(Self.maxWordLength)) }
|
||||
return cleaned.isEmpty ? Self.defaultTriggerWords : cleaned
|
||||
}
|
||||
|
||||
static func displayString(for words: [String]) -> String {
|
||||
let sanitized = self.sanitizeTriggerWords(words)
|
||||
return sanitized.joined(separator: ", ")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user