484 lines
17 KiB
HTML
484 lines
17 KiB
HTML
<!DOCTYPE html>
|
|
<!--
|
|
VoiceStreamAI Client Interface
|
|
Real-time audio transcription using self-hosted Whisper and WebSocket
|
|
|
|
Contributor:
|
|
- Alessandro Saccoia - alessandro.saccoia@gmail.com
|
|
-->
|
|
<html lang="en">
|
|
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<title>Audio Stream to WebSocket Server</title>
|
|
<style>
|
|
body {
|
|
font-family: Arial, sans-serif;
|
|
margin: 0;
|
|
padding: 0;
|
|
background: #f4f4f4;
|
|
text-align: center;
|
|
}
|
|
|
|
h1 {
|
|
color: #333;
|
|
}
|
|
|
|
.controls {
|
|
margin: 20px auto;
|
|
padding: 10px;
|
|
width: 80%;
|
|
display: flex;
|
|
justify-content: space-around;
|
|
align-items: center;
|
|
}
|
|
|
|
.control-group {
|
|
display: flex;
|
|
flex-direction: column;
|
|
align-items: center;
|
|
}
|
|
|
|
.controls input,
|
|
.controls button,
|
|
.controls select {
|
|
padding: 8px;
|
|
margin: 5px;
|
|
border: 1px solid #ddd;
|
|
border-radius: 5px;
|
|
font-size: 0.9em;
|
|
}
|
|
|
|
#transcription {
|
|
margin: 20px auto;
|
|
border: 1px solid #ddd;
|
|
padding: 10px;
|
|
width: 80%;
|
|
height: 150px;
|
|
overflow-y: auto;
|
|
background: white;
|
|
}
|
|
|
|
.label {
|
|
font-size: 0.9em;
|
|
color: #555;
|
|
margin-bottom: 5px;
|
|
}
|
|
|
|
button {
|
|
cursor: pointer;
|
|
}
|
|
|
|
.buffering-strategy-panel {
|
|
margin-top: 10px;
|
|
}
|
|
|
|
/* ... existing styles ... */
|
|
.hidden {
|
|
display: none;
|
|
}
|
|
</style>
|
|
|
|
<style>
|
|
body {
|
|
margin: 0;
|
|
padding: 0;
|
|
/* background-color: #333;
|
|
color:white */
|
|
}
|
|
|
|
#mic_container {
|
|
display: flex;
|
|
width: 100%;
|
|
align-items: center;
|
|
justify-content: center;
|
|
}
|
|
|
|
#mic {
|
|
border: 1px solid #ddd;
|
|
border-radius: 4px;
|
|
margin-top: 1rem;
|
|
height: 300px;
|
|
width: 300px
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<body>
|
|
|
|
<div id="mic_container">
|
|
<div id="mic"></div>
|
|
|
|
</div>
|
|
|
|
<h1>VAD + Whisper + </h1>
|
|
<div class="controls">
|
|
<div class="control-group">
|
|
<label class="label" for="websocketAddress">WebSocket Address:</label>
|
|
<input type="text" id="websocketAddress" value="ws://localhost:8765">
|
|
</div>
|
|
<div class="control-group">
|
|
<label class="label" for="bufferingStrategySelect" onchange="toggleBufferingStrategyPanel()">Buffering
|
|
Strategy:</label>
|
|
<select id="bufferingStrategySelect">
|
|
<option value="silence_at_end_of_chunk" selected>Silence at End of Chunk</option>
|
|
</select>
|
|
</div>
|
|
<div class="silence_at_end_of_chunk_options_panel">
|
|
<div class="control-group">
|
|
<label class="label" for="chunk_length_seconds">Chunk Length (s):</label>
|
|
<input type="number" id="chunk_length_seconds" value="3" min="1">
|
|
</div>
|
|
<div class="control-group">
|
|
<label class="label" for="chunk_offset_seconds">Silence at the End of Chunk (s):</label>
|
|
<input type="number" id="chunk_offset_seconds" value="0.1" min="0">
|
|
</div>
|
|
</div>
|
|
<div class="control-group">
|
|
<label class="label" for="languageSelect">Language:</label>
|
|
<select id="languageSelect">
|
|
<option value="multilingual">Multilingual</option>
|
|
<option value="english">English</option>
|
|
<option value="italian">Italian</option>
|
|
<option value="spanish">Spanish</option>
|
|
<option value="french">French</option>
|
|
<option value="german">German</option>
|
|
<option value="chinese">Chinese</option>
|
|
<option value="arabic">Arabic</option>
|
|
<option value="portuguese">Portuguese</option>
|
|
<option value="russian">Russian</option>
|
|
<option value="japanese">Japanese</option>
|
|
<option value="dutch">Dutch</option>
|
|
<option value="korean">Korean</option>
|
|
<option value="hindi">Hindi</option>
|
|
<option value="turkish">Turkish</option>
|
|
<option value="swedish">Swedish</option>
|
|
<option value="norwegian">Norwegian</option>
|
|
<option value="danish">Danish</option>
|
|
<option value="polish">Polish</option>
|
|
<option value="finnish">Finnish</option>
|
|
<option value="thai">Thai</option>
|
|
<option value="czech">Czech</option>
|
|
<option value="hungarian">Hungarian</option>
|
|
<option value="greek">Greek</option>
|
|
</select>
|
|
</div>
|
|
<button id="connectButton">Connect</button>
|
|
</div>
|
|
<button id="startButton" disabled>Start Streaming</button>
|
|
<button id="stopButton" disabled>Stop Streaming</button>
|
|
<div id="transcription"></div>
|
|
<br />
|
|
<div>WebSocket: <span id="webSocketStatus">Not Connected</span></div>
|
|
<div>Detected Language: <span id="detected_language">Undefined</span></div>
|
|
<div>Last Processing Time: <span id="processing_time">Undefined</span></div>
|
|
|
|
<script type="module">
|
|
|
|
// Record plugin
|
|
|
|
import WaveSurfer from 'https://cdn.jsdelivr.net/npm/wavesurfer.js@7/dist/wavesurfer.esm.js'
|
|
import RecordPlugin from 'https://cdn.jsdelivr.net/npm/wavesurfer.js/dist/plugins/record.esm.js'
|
|
|
|
// 可视化
|
|
let wavesurfer, record;
|
|
|
|
// ws服务
|
|
let websocket;
|
|
let context;
|
|
let processor;
|
|
let globalStream;
|
|
|
|
const websocket_uri = 'ws://localhost:8765';
|
|
const bufferSize = 4096;
|
|
let isRecording = false;
|
|
let chunk_length_seconds, chunk_offset_seconds, language;
|
|
|
|
// Record button
|
|
const startButton = document.getElementById('startButton'),
|
|
stopButton = document.getElementById('stopButton'),
|
|
connectButton = document.getElementById('connectButton');
|
|
|
|
const createWaveSurfer = () => {
|
|
// Create an instance of WaveSurfer
|
|
if (wavesurfer) {
|
|
wavesurfer.destroy()
|
|
}
|
|
wavesurfer = WaveSurfer.create({
|
|
container: '#mic',
|
|
waveColor: 'rgb(200, 0, 200)',
|
|
progressColor: 'rgb(100, 0, 100)',
|
|
renderFunction: (channels, ctx) => {
|
|
const { width, height } = ctx.canvas
|
|
console.log(width, height)
|
|
const scale = channels[0].length / width
|
|
const step = 20
|
|
|
|
ctx.translate(0, height / 2)
|
|
ctx.strokeStyle = ctx.fillStyle
|
|
ctx.beginPath()
|
|
|
|
for (let i = 0; i < width; i += step * 2) {
|
|
const index = Math.floor(i * scale)
|
|
const value = Math.abs(channels[0][index])
|
|
let x = i
|
|
let y = value * height * 1.2
|
|
|
|
ctx.moveTo(x, 0)
|
|
ctx.lineTo(x, y)
|
|
ctx.arc(x + step / 2, y, step / 2, Math.PI, 0, true)
|
|
ctx.lineTo(x + step, 0)
|
|
|
|
x = x + step
|
|
y = -y
|
|
ctx.moveTo(x, 0)
|
|
ctx.lineTo(x, y)
|
|
ctx.arc(x + step / 2, y, step / 2, Math.PI, 0, false)
|
|
ctx.lineTo(x + step, 0)
|
|
}
|
|
|
|
ctx.stroke()
|
|
ctx.closePath()
|
|
},
|
|
// // Set a bar width
|
|
// barWidth: 20,
|
|
// // Optionally, specify the spacing between bars
|
|
// barGap: 4,
|
|
// // And the bar radius
|
|
// barRadius: 2,
|
|
})
|
|
|
|
// Initialize the Record plugin
|
|
record = wavesurfer.registerPlugin(RecordPlugin.create({ scrollingWaveform: false, renderRecordedAudio: false }))
|
|
// Render recorded audio
|
|
// recButton.textContent = 'Record'
|
|
|
|
}
|
|
|
|
startButton.addEventListener('click', e => {
|
|
record.startRecording()
|
|
|
|
if (isRecording) return;
|
|
isRecording = true;
|
|
|
|
const AudioContext = window.AudioContext || window.webkitAudioContext;
|
|
context = new AudioContext();
|
|
|
|
navigator.mediaDevices.getUserMedia({ audio: true }).then(stream => {
|
|
globalStream = stream;
|
|
const input = context.createMediaStreamSource(stream);
|
|
processor = context.createScriptProcessor(bufferSize, 1, 1);
|
|
processor.onaudioprocess = e => processAudio(e);
|
|
input.connect(processor);
|
|
processor.connect(context.destination);
|
|
|
|
sendAudioConfig();
|
|
}).catch(error => console.error('Error accessing microphone', error));
|
|
|
|
// Disable start button and enable stop button
|
|
startButton.disabled = true;
|
|
stopButton.disabled = false;
|
|
|
|
|
|
});
|
|
|
|
stopButton.addEventListener('click', e => {
|
|
if (!isRecording) return;
|
|
isRecording = false;
|
|
|
|
if (globalStream) {
|
|
globalStream.getTracks().forEach(track => track.stop());
|
|
}
|
|
if (processor) {
|
|
processor.disconnect();
|
|
processor = null;
|
|
}
|
|
if (context) {
|
|
context.close().then(() => context = null);
|
|
}
|
|
startButton.disabled = false;
|
|
stopButton.disabled = true;
|
|
|
|
// 可视化部分
|
|
if (record.isRecording() || record.isPaused()) {
|
|
record.stopRecording()
|
|
return
|
|
}
|
|
})
|
|
|
|
connectButton.addEventListener('click', e => {
|
|
initWebSocket()
|
|
})
|
|
|
|
// Initialize on page load
|
|
window.onload = () => {
|
|
initWebSocket()
|
|
createWaveSurfer();
|
|
};
|
|
|
|
function initWebSocket() {
|
|
const websocketAddress = document.getElementById('websocketAddress').value;
|
|
chunk_length_seconds = document.getElementById('chunk_length_seconds').value;
|
|
chunk_offset_seconds = document.getElementById('chunk_offset_seconds').value;
|
|
const selectedLanguage = document.getElementById('languageSelect').value;
|
|
language = selectedLanguage !== 'multilingual' ? selectedLanguage : null;
|
|
|
|
if (!websocketAddress) {
|
|
console.log("WebSocket address is required.");
|
|
return;
|
|
}
|
|
|
|
if (websocket) websocket.close()
|
|
|
|
websocket = new WebSocket(websocketAddress);
|
|
websocket.onopen = () => {
|
|
console.log("WebSocket connection established");
|
|
document.getElementById("webSocketStatus").textContent = 'Connected';
|
|
startButton.disabled = false;
|
|
};
|
|
websocket.onclose = event => {
|
|
console.log("WebSocket connection closed", event);
|
|
document.getElementById("webSocketStatus").textContent = 'Not Connected';
|
|
startButton.disabled = true;
|
|
stopButton.disabled = true;
|
|
};
|
|
websocket.onmessage = event => {
|
|
console.log("Message from server:", event.data);
|
|
const transcript_data = JSON.parse(event.data);
|
|
updateTranscription(transcript_data);
|
|
};
|
|
}
|
|
|
|
function updateTranscription(transcript_data) {
|
|
const transcriptionDiv = document.getElementById('transcription');
|
|
const languageDiv = document.getElementById('detected_language');
|
|
|
|
if (transcript_data['words'] && transcript_data['words'].length > 0) {
|
|
// Append words with color based on their probability
|
|
transcript_data['words'].forEach(wordData => {
|
|
const span = document.createElement('span');
|
|
const probability = wordData['probability'];
|
|
span.textContent = wordData['word'] + ' ';
|
|
|
|
// Set the color based on the probability
|
|
if (probability > 0.9) {
|
|
span.style.color = 'green';
|
|
} else if (probability > 0.6) {
|
|
span.style.color = 'orange';
|
|
} else {
|
|
span.style.color = 'red';
|
|
}
|
|
|
|
transcriptionDiv.appendChild(span);
|
|
});
|
|
|
|
// Add a new line at the end
|
|
transcriptionDiv.appendChild(document.createElement('br'));
|
|
} else {
|
|
// Fallback to plain text
|
|
transcriptionDiv.textContent += transcript_data['text'] + '\n';
|
|
}
|
|
|
|
// Update the language information
|
|
if (transcript_data['language'] && transcript_data['language_probability']) {
|
|
languageDiv.textContent = transcript_data['language'] + ' (' + transcript_data['language_probability'].toFixed(2) + ')';
|
|
}
|
|
|
|
// Update the processing time, if available
|
|
const processingTimeDiv = document.getElementById('processing_time');
|
|
if (transcript_data['processing_time']) {
|
|
processingTimeDiv.textContent = 'Processing time: ' + transcript_data['processing_time'].toFixed(2) + ' seconds';
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
function sendAudioConfig() {
|
|
let selectedStrategy = document.getElementById('bufferingStrategySelect').value;
|
|
let processingArgs = {};
|
|
|
|
if (selectedStrategy === 'silence_at_end_of_chunk') {
|
|
processingArgs = {
|
|
chunk_length_seconds: parseFloat(document.getElementById('chunk_length_seconds').value),
|
|
chunk_offset_seconds: parseFloat(document.getElementById('chunk_offset_seconds').value)
|
|
};
|
|
}
|
|
|
|
const audioConfig = {
|
|
type: 'config',
|
|
data: {
|
|
sampleRate: context.sampleRate,
|
|
bufferSize: bufferSize,
|
|
channels: 1, // Assuming mono channel
|
|
language: language,
|
|
processing_strategy: selectedStrategy,
|
|
processing_args: processingArgs
|
|
}
|
|
};
|
|
|
|
websocket.send(JSON.stringify(audioConfig));
|
|
}
|
|
|
|
function downsampleBuffer(buffer, inputSampleRate, outputSampleRate) {
|
|
if (inputSampleRate === outputSampleRate) {
|
|
return buffer;
|
|
}
|
|
var sampleRateRatio = inputSampleRate / outputSampleRate;
|
|
var newLength = Math.round(buffer.length / sampleRateRatio);
|
|
var result = new Float32Array(newLength);
|
|
var offsetResult = 0;
|
|
var offsetBuffer = 0;
|
|
while (offsetResult < result.length) {
|
|
var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
|
|
var accum = 0, count = 0;
|
|
for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
|
|
accum += buffer[i];
|
|
count++;
|
|
}
|
|
result[offsetResult] = accum / count;
|
|
offsetResult++;
|
|
offsetBuffer = nextOffsetBuffer;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function processAudio(e) {
|
|
const inputSampleRate = context.sampleRate;
|
|
const outputSampleRate = 16000; // Target sample rate
|
|
|
|
const left = e.inputBuffer.getChannelData(0);
|
|
const downsampledBuffer = downsampleBuffer(left, inputSampleRate, outputSampleRate);
|
|
const audioData = convertFloat32ToInt16(downsampledBuffer);
|
|
|
|
if (websocket && websocket.readyState === WebSocket.OPEN) {
|
|
websocket.send(audioData);
|
|
}
|
|
}
|
|
|
|
function convertFloat32ToInt16(buffer) {
|
|
let l = buffer.length;
|
|
const buf = new Int16Array(l);
|
|
while (l--) {
|
|
buf[l] = Math.min(1, buffer[l]) * 0x7FFF;
|
|
}
|
|
return buf.buffer;
|
|
}
|
|
|
|
|
|
function toggleBufferingStrategyPanel() {
|
|
var selectedStrategy = document.getElementById('bufferingStrategySelect').value;
|
|
if (selectedStrategy === 'silence_at_end_of_chunk') {
|
|
var panel = document.getElementById('silence_at_end_of_chunk_options_panel');
|
|
panel.classList.remove('hidden');
|
|
} else {
|
|
var panel = document.getElementById('silence_at_end_of_chunk_options_panel');
|
|
panel.classList.add('hidden');
|
|
}
|
|
}
|
|
|
|
|
|
</script>
|
|
</body>
|
|
|
|
</html> |