feat(audio): implement voice chat with Opus codec and streaming audio source

Add AudioStreamSource for continuous PCM playback. Refactor voice send/receive to encode with Opus and stream to a dedicated source instead of per-frame sources. Enable voice transmission over network.

include/Cubed/audio/audio_engine.hpp

@@ -3,6 +3,7 @@
 #include "Cubed/audio/audio_fade.hpp"
 #include "Cubed/audio/audio_recording.hpp"
 #include "Cubed/audio/audio_source.hpp"
+#include "Cubed/audio/audio_stream_source.hpp"
 #include "Cubed/audio/sound_manager.hpp"
 #include "Cubed/audio/source_pool.hpp"
 #include "Cubed/config.hpp"
@@ -70,6 +71,8 @@ private:
     bool m_underwater = false;
     float m_music_volume = 1.0f;
     float m_sfx_volume = 1.0f;
+
+    std::unique_ptr<AudioStreamSource> m_voice_source;
     std::unique_ptr<AudioFilter> m_low_pass_filter;
     std::unique_ptr<AudioEffect> m_underwater_effect;
     std::unique_ptr<AudioEffectSlot> m_underwater_slot;

include/Cubed/audio/audio_stream_source.hpp

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <al.h>
+#include <cstdint>
+#include <span>
+#include <vector>
+namespace Cubed {
+class AudioStreamSource {
+public:
+    static constexpr int NUM_BUFFERS = 4;
+    AudioStreamSource();
+    AudioStreamSource(const AudioStreamSource&) = delete;
+    AudioStreamSource(AudioStreamSource&&) = delete;
+    AudioStreamSource& operator=(const AudioStreamSource&) = delete;
+    AudioStreamSource& operator=(AudioStreamSource&&) = delete;
+    ~AudioStreamSource();
+
+    void push_pcm(std::span<const int16_t> pcm, ALsizei sample_rate);
+
+    void stop();
+    bool is_playing() const;
+
+private:
+    ALuint m_source = 0;
+    std::array<ALuint, NUM_BUFFERS> m_buffers{};
+    std::vector<ALuint> m_free_buffers;
+    bool m_playing = false;
+
+    void unqueue_processed();
+};
+} // namespace Cubed

src/CMakeLists.txt

@@ -87,4 +87,5 @@ target_sources(${PROJECT_NAME}
         tools/system_locate.cpp
         ui/chat_box.cpp
         audio/audio_recording.cpp
+        audio/audio_stream_source.cpp
 )

src/audio/audio_engine.cpp

@@ -23,10 +23,11 @@ AudioEngine::~AudioEngine() {
     m_bgm.reset();
     m_pool.reset();
     m_sounds.clear();
-
+    m_voice_source.reset();
     m_low_pass_filter.reset();
     m_underwater_effect.reset();
     m_underwater_slot.reset();
+
     opus_encoder_destroy(m_encoder);
     opus_decoder_destroy(m_decoder);
     alcMakeContextCurrent(nullptr);
@@ -118,6 +119,8 @@ void AudioEngine::init() {
     Logger::info("Audio Engine Init Success");
 
     m_init = true;
+
+    m_voice_source = std::make_unique<AudioStreamSource>();
 }
 
 void AudioEngine::play_bgm() { m_bgm->play(); }
@@ -254,27 +257,7 @@ void AudioEngine::set_client(std::weak_ptr<NetworkClient> client) {
 
 void AudioEngine::send_voice(
     const std::array<int16_t, AudioRecording::FRAME_SAMPLES>& pcm) {
-    {
-        AudioData data;
-        data.pcm = {pcm.begin(), pcm.end()};
-        data.channels = 1;
-        data.sample_rate = AudioRecording::SAMPLE_RATE;
-        auto* source = m_pool->acquire();
-        source->set_volume(m_sfx_volume);
-        if (!source) {
-            Logger::error("Source is Full");
-            return;
-        }
-        std::unique_ptr<AudioBuffer> buffer =
-            std::make_unique<AudioBuffer>(data);
 
-        if (m_efx_supported && m_underwater) {
-            source->set_filter(*m_low_pass_filter);
-            source->set_effect_slot(*m_underwater_slot);
-        }
-        source->play_2d(std::move(buffer));
-    }
-    /*
     std::array<uint8_t, OPUS_MAX_PACKET_SIZE> opus;
     int len = opus_encode(m_encoder, pcm.data(), AudioRecording::FRAME_SAMPLES,
                           opus.data(), opus.size());
@@ -283,7 +266,6 @@ void AudioEngine::send_voice(
         Logger::error("Opus encode failed: {}", opus_strerror(len));
         return;
     }
-    Logger::info("opus encode len={}", len);
     if (auto c = m_client.lock()) {
         Arena arena;
         auto msg = Arena::Create<VoiceMsg>(&arena);
@@ -296,37 +278,18 @@ void AudioEngine::send_voice(
         pos->set_z(p.z);
         c->send(make_packet(*msg));
     }
-        */
 }
 void AudioEngine::receive_voice(std::span<char> opus, const glm::vec3& pos) {
-    AudioData data;
+    std::array<int16_t, AudioRecording::FRAME_SAMPLES> pcm;
-    data.pcm.resize(AudioRecording::FRAME_SAMPLES);
+    int len =
-    data.channels = 1;
+        opus_decode(m_decoder, reinterpret_cast<const uint8_t*>(opus.data()),
-    data.sample_rate = AudioRecording::SAMPLE_RATE;
+                    opus.size(), pcm.data(), AudioRecording::FRAME_SAMPLES, 0);
-    int len = opus_decode(
-        m_decoder, reinterpret_cast<const uint8_t*>(opus.data()), opus.size(),
-        data.pcm.data(), AudioRecording::FRAME_SAMPLES, 0);
     if (len < 0) {
         Logger::error("Opus decode failed: {}", opus_strerror(len));
         return;
     }
-    Logger::info("decode samples={}", len);
+    m_voice_source->push_pcm(std::span(pcm.data(), len),
-    Logger::info("Receive Vocie and start play");
+                             AudioRecording::SAMPLE_RATE);
-    auto* source = m_pool->acquire();
-    source->set_volume(m_sfx_volume);
-    if (!source) {
-        Logger::error("Source is Full");
-        return;
-    }
-    for (int i = 0; i < 10; ++i)
-        Logger::info("recv {}", data.pcm[i]);
-    std::unique_ptr<AudioBuffer> buffer = std::make_unique<AudioBuffer>(data);
-
-    if (m_efx_supported && m_underwater) {
-        source->set_filter(*m_low_pass_filter);
-        source->set_effect_slot(*m_underwater_slot);
-    }
-    source->play_3d(std::move(buffer), pos);
 }
 
 AudioRecording& AudioEngine::audio_recording() { return m_recording; }

src/audio/audio_recording.cpp

@@ -60,9 +60,6 @@ void AudioRecording::update() {
     while (available >= FRAME_SAMPLES) {
         std::array<int16_t, FRAME_SAMPLES> pcm;
         alcCaptureSamples(m_capture, pcm.data(), FRAME_SAMPLES);
-        for (int i = 0; i < 20; i++) {
-            Logger::info("pcm[{}]={}", i, pcm[i]);
-        }
         send_voice(pcm);
         available -= FRAME_SAMPLES;
     }

src/audio/audio_stream_source.cpp

@@ -0,0 +1,71 @@
+#include "Cubed/audio/audio_stream_source.hpp"
+
+#include "Cubed/tools/log.hpp"
+namespace Cubed {
+AudioStreamSource::AudioStreamSource() {
+    alGenBuffers(NUM_BUFFERS, m_buffers.data());
+    alGenSources(1, &m_source);
+
+    alSourcei(m_source, AL_SOURCE_RELATIVE, AL_TRUE);
+    alSource3f(m_source, AL_POSITION, 0.0f, 0.0f, 0.0f);
+    m_free_buffers.assign(m_buffers.begin(), m_buffers.end());
+}
+
+AudioStreamSource::~AudioStreamSource() {
+    stop();
+    alDeleteSources(1, &m_source);
+    alDeleteBuffers(NUM_BUFFERS, m_buffers.data());
+}
+
+void AudioStreamSource::push_pcm(std::span<const int16_t> pcm,
+                                 ALsizei sample_rate) {
+    if (!m_source) {
+        return;
+    }
+    unqueue_processed();
+
+    if (m_free_buffers.empty()) {
+        Logger::warn("Queue is full, dropping frame");
+        return;
+    }
+
+    ALuint buf = m_free_buffers.back();
+    m_free_buffers.pop_back();
+
+    alBufferData(buf, AL_FORMAT_MONO16, pcm.data(),
+                 static_cast<ALsizei>(pcm.size() * sizeof(int16_t)),
+                 sample_rate);
+    alSourceQueueBuffers(m_source, 1, &buf);
+
+    ALint state;
+    alGetSourcei(m_source, AL_SOURCE_STATE, &state);
+    if (state != AL_PLAYING) {
+        alSourcePlay(m_source);
+        m_playing = true;
+    }
+}
+
+void AudioStreamSource::stop() {
+    if (!m_source) {
+        return;
+    }
+
+    alSourceStop(m_source);
+
+    alSourcei(m_source, AL_BUFFER, 0);
+    m_free_buffers.assign(m_buffers.begin(), m_buffers.end());
+    m_playing = false;
+}
+
+void AudioStreamSource::unqueue_processed() {
+    ALint processed = 0;
+    alGetSourcei(m_source, AL_BUFFERS_PROCESSED, &processed);
+
+    while (processed-- > 0) {
+        ALuint buf;
+        alSourceUnqueueBuffers(m_source, 1, &buf);
+        m_free_buffers.push_back(buf);
+    }
+}
+
+} // namespace Cubed