diff --git a/public/resource/sounds/1.wav b/public/resource/sounds/1.wav new file mode 100644 index 0000000..0415e94 Binary files /dev/null and b/public/resource/sounds/1.wav differ diff --git a/src/App.vue b/src/App.vue index 702a98f..d644811 100644 --- a/src/App.vue +++ b/src/App.vue @@ -30,18 +30,72 @@ const showQues=ref(false) const handleClick = () => { - talk(resourceMap.get('start')) - .then(() => { - startStreaming() - text.value="请简单阐述mysql的索引机制,请简单阐述mysql的索引机制请简单阐述mysql的索引机制请简单阐述mysql的索引机制请简单阐述mysql的索引机制" - Ques() - }) + startStreaming() + //const mouthOpen = 1; // 映射到 0~1 范围 + //handlePlayAction(resourceMap.get('test_wav')) + + //let voicePath=resourceMap.get('start') + // live2DSprite._model.("PARAM_Mouth_OpenY", volume) + //live2DSprite._model.playVoice({ voicePath, immediate: true }); + // talk(resourceMap.get('start')) + // .then(() => { + // startStreaming() + // text.value="请简单阐述mysql的索引机制,请简单阐述mysql的索引机制请简单阐述mysql的索引机制请简单阐述mysql的索引机制请简单阐述mysql的索引机制" + // Ques() + // }) }; const overClick = () => { app.ticker.stop() //stopStreaming() }; +const handlePlayAction = (voicePath) => { + console.log("接收到地址:"+voicePath); + live2DSprite.playVoice({ voicePath, immediate: true }) +}; + +const handleBlob = async (blobUrl) => { + // 获取 Blob 对象 + const response = await fetch(blobUrl); + const blob = await response.blob(); + let targetPath="1.wav" + // 创建下载链接 + const a = document.createElement('a'); + a.href = URL.createObjectURL(blob); + a.download = targetPath.split('/').pop(); // 提取文件名(如 "start.wav") + a.click(); + + // 释放内存 + URL.revokeObjectURL(a.href); + + + return handlePlayAction(targetPath); +}; + + +const talk = async (voicePath) => { + return new Promise((resolve, reject) => { + try { + const tempAudio = new Audio(voicePath); + tempAudio.onloadedmetadata = () => { + const duration = tempAudio.duration * 1000; // 转换为毫秒 + // 2. 播放语音 + live2DSprite.playVoice({ voicePath, immediate: true }); + // 3. 使用 setTimeout 模拟播放结束监听 + setTimeout(() => { + console.log("语音播放完成(估算时间)"); + resolve(); // 通知调用者播放完成 + }, duration); + }; + // 4. 处理加载错误 + tempAudio.onerror = () => { + reject(new Error("语音加载失败")); + }; + } catch (error) { + reject(error); // 捕获其他异常 + } + }); +}; function ansTran(delta){ live2DSprite.x += 8; @@ -54,8 +108,6 @@ const Ques=()=>{ app.ticker.stop() showQues.value=true }, 1000); - - } @@ -133,40 +185,12 @@ const modelInit=async (modelPath)=>{ } } - - onUnmounted(() => { console.log("释放实例") live2DSprite.destroy() }) -const talk = async (voicePath) => { - return new Promise((resolve, reject) => { - try { - const tempAudio = new Audio(voicePath); - tempAudio.onloadedmetadata = () => { - const duration = tempAudio.duration * 1000; // 转换为毫秒 - // 2. 播放语音 - live2DSprite.playVoice({ voicePath, immediate: true }); - // 3. 使用 setTimeout 模拟播放结束监听 - setTimeout(() => { - console.log("语音播放完成(估算时间)"); - resolve(); // 通知调用者播放完成 - }, duration); - }; - // 4. 处理加载错误 - tempAudio.onerror = () => { - reject(new Error("语音加载失败")); - }; - } catch (error) { - reject(error); // 捕获其他异常 - } - }); -}; - - - const isStreaming = ref(false); const status = ref(""); @@ -224,6 +248,7 @@ const handleError = (error) => { :is-streaming="isStreaming" @update-status="handleStatusUpdate" @streaming-error="handleError" + @audio-generated="handlePlayAction" />

{{ status }}

diff --git a/src/components/AudioStreamer.vue b/src/components/AudioStreamer.vue index cb97177..2368ce1 100644 --- a/src/components/AudioStreamer.vue +++ b/src/components/AudioStreamer.vue @@ -11,18 +11,26 @@ export default { props: { isStreaming: Boolean, }, - emits: ["update-status", "streaming-error"], + emits: ["update-status", "streaming-error","audio-generated"], setup(props, { emit }) { const status = ref(""); let audioContext = null; let mediaStream = null; let socket = null; let audioWorkletNode = null; - + let speechSynthesis = null; + let mediaSource = null; + let audioStream = null; + let audioElement = null; const SAMPLE_RATE = 16000; const CHANNELS = 1; - - // 统一资源清理方法 + const audioQueue = []; + let isAppending = false; + let receivedChunks = []; // 存储接收到的数据块 + let totalSize = 0; + let isReceiving = false; + let hasReceivedHeader = false; + let wavParams ={}; const cleanupResources = () => { if (audioWorkletNode) { audioWorkletNode.disconnect(); @@ -30,7 +38,7 @@ export default { audioWorkletNode = null; } if (mediaStream) { - mediaStream.getTracks().forEach(track => track.stop()); + mediaStream.getTracks().forEach((track) => track.stop()); mediaStream = null; } if (socket) { @@ -45,15 +53,99 @@ export default { audioContext.close().catch(() => {}); audioContext = null; } + if (speechSynthesis) { + speechSynthesis.cancel(); + speechSynthesis = null; + } + if (audioElement) { + audioElement.pause(); + audioElement.src = ""; + audioElement = null; + } + if (mediaSource) { + if (mediaSource.readyState === "open") { + mediaSource.endOfStream(); + } + mediaSource = null; + } + audioStream = null; + }; + + const processAudioQueue = () => { + if ( + audioQueue.length === 0 || + isAppending || + !mediaSource || + mediaSource.readyState !== "open" || + !audioStream + ) { + return; + } + + isAppending = true; + const audioData = audioQueue.shift(); + + try { + if (audioStream.updating) { + audioQueue.unshift(audioData); + isAppending = false; + return; + } + + audioStream.appendBuffer(audioData); + } catch (error) { + console.error("Error appending audio buffer:", error); + isAppending = false; + // 尝试恢复:重新创建 MediaSource 和音频元素 + setupAudioElement(); + processAudioQueue(); + } + }; + + const setupAudioElement = () => { + if (audioElement) { + audioElement.pause(); + audioElement.src = ""; + } + + mediaSource = new MediaSource(); + audioElement = document.createElement("audio"); + audioElement.src = URL.createObjectURL(mediaSource); + audioElement.play().catch((e) => { + console.error("Audio element play failed:", e); + }); + + mediaSource.addEventListener("sourceopen", () => { + try { + audioStream = mediaSource.addSourceBuffer("audio/mpeg"); + audioStream.addEventListener("updateend", () => { + isAppending = false; + processAudioQueue(); + }); + audioStream.addEventListener("error", (e) => { + console.error("SourceBuffer error:", e); + isAppending = false; + }); + } catch (error) { + console.error("Failed to add SourceBuffer:", error); + } + }); + + mediaSource.addEventListener("sourceended", () => { + console.log("MediaSource ended"); + }); }; const startStreaming = async () => { try { - // 先清理旧资源 cleanupResources(); + setupAudioElement(); - // 1. 初始化 WebSocket - socket = new WebSocket("ws://localhost:8000/ws"); + audioContext = new (window.AudioContext || window.webkitAudioContext)({ + sampleRate: SAMPLE_RATE, + }); + + socket = new WebSocket("ws://127.0.0.1:27004/ws/transcribe"); socket.binaryType = "arraybuffer"; socket.onopen = () => { @@ -69,32 +161,55 @@ export default { cleanupResources(); }; - // 2. 初始化 AudioContext(必须全新创建) - audioContext = new AudioContext({ sampleRate: SAMPLE_RATE }); + socket.onmessage = (event) => { + if (event.data instanceof ArrayBuffer) { + const chunk = new Uint8Array(event.data); + + //如果是第一个 chunk,检查 WAV 头 + // if (!hasReceivedHeader) { + // const header = chunk.slice(0, 4); + // const isRiff = header[0] === 0x52 && header[1] === 0x49 && header[2] === 0x46 && header[3] === 0x46; + // if (!isRiff) { + // console.error("错误:第一个 chunk 不是有效的 WAV 头!"); + // hasReceivedHeader=false + // return; + // } + // hasReceivedHeader = true; + // } + + receivedChunks.push(chunk); + totalSize += chunk.length; + } else { + const data = JSON.parse(event.data); + if (data.type === "EOF") { + wavParams = { + sampleRate: data.sampleRate, + numChannels: data.numChannels, + bitDepth: data.bitDepth, + }; + finalizeWavFile(); + } + } + }; - // 3. 获取麦克风输入 mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true }); const source = audioContext.createMediaStreamSource(mediaStream); - // 4. 加载 AudioWorklet 处理器 await audioContext.audioWorklet.addModule("/audio-processor.js"); - - // 5. 创建新的 AudioWorkletNode audioWorkletNode = new AudioWorkletNode(audioContext, "audio-processor"); - // 6. 监听音频数据并发送到 WebSocket audioWorkletNode.port.onmessage = (event) => { if (socket?.readyState === WebSocket.OPEN) { socket.send(event.data); } }; - // 7. 连接音频流 source.connect(audioWorkletNode); audioWorkletNode.connect(audioContext.destination); emit("update-status", "Streaming started"); } catch (error) { + console.error("Streaming error:", error); cleanupResources(); emit("streaming-error", error.message); } @@ -106,15 +221,89 @@ export default { }; watch( - () => props.isStreaming, - (newVal) => { - if (newVal) { - startStreaming(); - } else { - stopStreaming(); - } - }, - { immediate: true } + () => props.isStreaming, + (newVal) => { + if (newVal) { + startStreaming(); + } else { + stopStreaming(); + } + }, + { immediate: true } + ); + + function finalizeWavFile() { + if (!wavParams || receivedChunks.length === 0) return; + + const { sampleRate, numChannels, bitDepth } = wavParams; + const pcmData = new Uint8Array(totalSize); + let offset = 0; + + for (const chunk of receivedChunks) { + pcmData.set(chunk, offset); + offset += chunk.length; + } + + const wavHeader = generateWavHeader(sampleRate, numChannels, bitDepth, totalSize); + const wavData = new Uint8Array(wavHeader.length + totalSize); + wavData.set(wavHeader); + wavData.set(pcmData, wavHeader.length); + + const wavBlob = new Blob([wavData], { type: "audio/wav" }); + const audioUrl = URL.createObjectURL(wavBlob); + emit("audio-generated", audioUrl); + + // 清理 + receivedChunks = []; + totalSize = 0; + wavParams ={}; + } + + + const cleanupAudioData = () => { + receivedChunks = []; + totalSize = 0; + }; + + // 生成 WAV 头 + function generateWavHeader(sampleRate, numChannels, bitDepth, dataSize) { + const bytesPerSample = bitDepth / 8; + const blockAlign = numChannels * bytesPerSample; + const buffer = new ArrayBuffer(44); + const view = new DataView(buffer); + + // RIFF 头 + view.setUint32(0, 0x52494646, false); // "RIFF" + view.setUint32(4, 36 + dataSize, true); // 文件总大小 - 8 + view.setUint32(8, 0x57415645, false); // "WAVE" + + // fmt 子块 + view.setUint32(12, 0x666d7420, false); // "fmt " + view.setUint32(16, 16, true); // 子块大小(16 for PCM) + view.setUint16(20, 1, true); // 格式(1 = PCM) + view.setUint16(22, numChannels, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * blockAlign, true); // 字节率 + view.setUint16(32, blockAlign, true); + view.setUint16(34, bitDepth, true); + + // data 子块 + view.setUint32(36, 0x64617461, false); // "data" + view.setUint32(40, dataSize, true); // 音频数据大小 + + return new Uint8Array(buffer); + } + + watch( + () => props.isStreaming, + (newVal) => { + if (newVal) { + startStreaming(); + } else { + stopStreaming(); + } + }, + { immediate: true } ); onUnmounted(() => { @@ -124,4 +313,5 @@ export default { return { status }; }, }; + \ No newline at end of file diff --git a/src/utils/resource.js b/src/utils/resource.js index f88b65b..8b0c2e1 100644 --- a/src/utils/resource.js +++ b/src/utils/resource.js @@ -4,5 +4,6 @@ export const resourceMap = new Map([ ['start', '/resource/sounds/start.wav'], ['model', '/resource/models/UG/ugofficial.model3.json'], ['log', '/resource/img/log.png'], + ['test_wav', '/resource/sounds/1.wav'], // 其他资源... ]); \ No newline at end of file