feat: add sensitive word filter for chat messages

Implement AC automaton-based filtering using a sensitive lexicon loaded from a JSON asset file. The filter is applied to both player names and chat messages before broadcasting, and can be toggled via the `sensitive_filter` config option.

include/Cubed/gameplay/server_world.hpp

@@ -10,6 +10,7 @@
 #include "Cubed/gameplay/server_player.hpp"
 #include "Cubed/tools/priority_thread_pool.hpp"
 #include "Cubed/tools/recent_queue.hpp"
+#include "Cubed/tools/sensitive_filter.hpp"
 #include "Cubed/tools/thread_pool.hpp"
 #include "Cubed/ui/color.hpp"
 #include "world/block_change.pb.h"
@@ -131,6 +132,8 @@ private:
 
     CaveCarver m_cave_carcer;
     RiverWorm m_river_worm;
+    bool m_enable_filter = true;
+    SensitiveFilter m_filter;
 
     std::jthread m_gen_thread;
     std::jthread m_server_thread;

include/Cubed/tools/sensitive_filter.hpp

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <nlohmann/json.hpp>
+namespace Cubed {
+class SensitiveFilter {
+public:
+    SensitiveFilter();
+    void load(const nlohmann::json& j);
+    std::string filter(std::string_view text);
+
+private:
+    void insert(const std::u32string& world);
+    void build();
+
+    struct Node {
+        std::unordered_map<char32_t, int> next;
+        int fail = 0;
+        size_t length = 0;
+        bool end = false;
+    };
+
+    std::vector<Node> trie{1};
+};
+} // namespace Cubed

src/CMakeLists.txt

@@ -88,4 +88,5 @@ target_sources(${PROJECT_NAME}
         ui/chat_box.cpp
         audio/audio_recording.cpp
         audio/audio_stream_source.cpp
+        tools/sensitive_filter.cpp
 )

src/gameplay/server_world.cpp

@@ -7,12 +7,14 @@
 #include "Cubed/tools/math_tools.hpp"
 #include "Cubed/tools/uuid.hpp"
 
+#include <nlohmann/json.hpp>
 #include <ranges>
 #include <utility>
 using namespace std::chrono;
 using namespace std::chrono_literals;
 using namespace google::protobuf;
-
+namespace fs = std::filesystem;
+using nlohmann::json;
 namespace Cubed {
 ServerWorld::ServerWorld(Config& config) : m_config(config) {}
 
@@ -198,6 +200,18 @@ void ServerWorld::init_world() {
 
     m_cave_carcer.init(ChunkGenerator::seed());
     m_river_worm.init(ChunkGenerator::seed());
+
+    m_enable_filter = m_config.get("sensitive_filter", true);
+    try {
+        fs::path path = std::format("{}SensitiveLexicon.json", ASSETS_PATH);
+        std::ifstream s{path};
+        json j = json::parse(s);
+        m_filter.load(j);
+    } catch (const std::exception& e) {
+        Logger::error("Load SensitiveLexicon.json Fail");
+        m_enable_filter = false;
+    }
+
     // m_chunks.reserve(MAX_DISTANCE * MAX_DISTANCE * 4);
     start_thread_pool();
 
@@ -934,9 +948,14 @@ void ServerWorld::boardcast_message(const std::string& name,
 
     Arena arena;
     auto msg = Arena::Create<ChatMsg>(&arena);
-
+    if (m_enable_filter) {
+        msg->set_msg(m_filter.filter(message));
+        msg->set_name(m_filter.filter(name));
+    } else {
         msg->set_msg(message);
         msg->set_name(name);
+    }
+
     msg->set_color(std::to_underlying(color));
     msg->set_system_msg(system_msg);
 

src/tools/sensitive_filter.cpp

@@ -0,0 +1,118 @@
+#include "Cubed/tools/sensitive_filter.hpp"
+
+#include <queue>
+#include <utf8cpp/utf8.h>
+using nlohmann::json;
+namespace Cubed {
+SensitiveFilter::SensitiveFilter() {
+
+};
+
+void SensitiveFilter::load(const nlohmann::json& j) {
+    trie.clear();
+    trie.emplace_back();
+    for (const auto& item : j["words"]) {
+
+        auto str = item.get<std::string>();
+        std::u32string word;
+        utf8::utf8to32(str.begin(), str.end(), std::back_inserter(word));
+        insert(word);
+    }
+    build();
+}
+std::string SensitiveFilter::filter(std::string_view str) {
+    if (!utf8::is_valid(str)) {
+        return {};
+    }
+    int state = 0;
+    std::vector<char32_t> text;
+
+    auto it = str.begin();
+
+    while (it != str.end()) {
+        text.push_back(utf8::next(it, str.end()));
+    }
+    std::vector<uint8_t> mask(text.size(), 0);
+    for (size_t i = 0; i < text.size(); i++) {
+        char32_t ch = text[i];
+
+        auto it = trie[state].next.find(ch);
+
+        while (state && it == trie[state].next.end()) {
+            state = trie[state].fail;
+            it = trie[state].next.find(ch);
+        }
+
+        if (it != trie[state].next.end()) {
+            state = it->second;
+        }
+
+        int t = state;
+
+        while (t) {
+            if (trie[t].end) {
+                size_t begin = i + 1 - trie[t].length;
+                for (size_t j = begin; j < begin + trie[t].length; j++)
+                    mask[j] = true;
+            }
+
+            t = trie[t].fail;
+        }
+    }
+    std::string out;
+    for (size_t i = 0; i < text.size();) {
+        if (mask[i]) {
+            out += "***";
+
+            while (i < text.size() && mask[i])
+                ++i;
+        } else {
+            // char32_t -> UTF8
+            utf8::append(text[i], std::back_inserter(out));
+            ++i;
+        }
+    }
+    return out;
+}
+
+void SensitiveFilter::insert(const std::u32string& word) {
+    int now = 0;
+    for (char32_t ch : word) {
+        auto it = trie[now].next.find(ch);
+        if (it == trie[now].next.end()) {
+            trie[now].next[ch] = trie.size();
+            trie.emplace_back();
+        }
+        now = trie[now].next[ch];
+    }
+    trie[now].end = true;
+    trie[now].length = word.size();
+}
+
+void SensitiveFilter::build() {
+    std::queue<int> q;
+
+    for (auto [ch, to] : trie[0].next) {
+        trie[to].fail = 0;
+        q.push(to);
+    }
+
+    while (!q.empty()) {
+        int u = q.front();
+        q.pop();
+
+        for (auto [ch, v] : trie[u].next) {
+            int f = trie[u].fail;
+
+            while (f && !trie[f].next.count(ch))
+                f = trie[f].fail;
+
+            if (trie[f].next.count(ch))
+                trie[v].fail = trie[f].next[ch];
+
+            q.push(v);
+        }
+    }
+}
+
+} // namespace Cubed