text_utils.hpp

Go to the documentation of this file.

// text_utils.hpp -- shared text conversion helpers.
//
// Bug #14, #15, #18 fix: previously each filesystem module had its own
// half-broken UTF-16->ASCII conversion (non-ASCII characters became '_')
// and an inconsistent sanitize_path_component (some replaced spaces, some
// silently dropped illegal characters like ':' '*' '\').
//
// This header centralises the logic and produces real UTF-8 output, so
// Italian/European/Asian filenames survive recovery instead of being
// reduced to underscores.
 
#pragma once
 
#include <cstdint>
#include <string>
 
namespace text_utils {
 
// Decode a UTF-16LE buffer of `byte_count` bytes into UTF-8.
// Stops at the first U+0000 code unit. Handles surrogate pairs.
inline std::string utf16le_to_utf8(const unsigned char* data, std::size_t byte_count) {
    std::string out;
    out.reserve(byte_count);
    auto read_unit = [&](std::size_t i) -> uint16_t {
        return static_cast<uint16_t>(data[i]) |
               (static_cast<uint16_t>(data[i + 1]) << 8);
    };
    auto append_utf8 = [&](uint32_t cp) {
        if (cp < 0x80) {
            out.push_back(static_cast<char>(cp));
        } else if (cp < 0x800) {
            out.push_back(static_cast<char>(0xC0 | (cp >> 6)));
            out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
        } else if (cp < 0x10000) {
            out.push_back(static_cast<char>(0xE0 | (cp >> 12)));
            out.push_back(static_cast<char>(0x80 | ((cp >> 6) & 0x3F)));
            out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
        } else if (cp <= 0x10FFFF) {
            out.push_back(static_cast<char>(0xF0 | (cp >> 18)));
            out.push_back(static_cast<char>(0x80 | ((cp >> 12) & 0x3F)));
            out.push_back(static_cast<char>(0x80 | ((cp >> 6) & 0x3F)));
            out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
        }
    };
    std::size_t i = 0;
    while (i + 1 < byte_count) {
        const uint16_t unit = read_unit(i);
        if (unit == 0) break;
        // High surrogate: combine with next low surrogate
        if (unit >= 0xD800 && unit <= 0xDBFF && i + 3 < byte_count) {
            const uint16_t low = read_unit(i + 2);
            if (low >= 0xDC00 && low <= 0xDFFF) {
                const uint32_t cp = 0x10000 +
                                    ((static_cast<uint32_t>(unit - 0xD800)) << 10) +
                                    static_cast<uint32_t>(low - 0xDC00);
                append_utf8(cp);
                i += 4;
                continue;
            }
        }
        // Lone low surrogate or normal BMP code point
        if (unit >= 0xDC00 && unit <= 0xDFFF) {
            append_utf8(0xFFFD); // replacement char for invalid surrogate
        } else {
            append_utf8(unit);
        }
        i += 2;
    }
    return out;
}
 
// Sanitize one path component for safe writing on any host filesystem.
// - Allows letters, digits, '.', '-', '_'
// - Allows printable UTF-8 multibyte sequences (so accented characters survive)
// - Replaces forbidden ASCII (/, \, :, *, ?, ", <, >, |, control chars) with '_'
// - Replaces whitespace with '_'
// - Returns "unnamed" if input is empty after sanitization
// - Trims trailing dots/spaces (Windows-illegal at end of path component)
inline std::string sanitize_component(const std::string& in) {
    // Trim trailing dots and spaces from the INPUT first (Windows rule).
    std::size_t end = in.size();
    while (end > 0 && (in[end - 1] == '.' || in[end - 1] == ' ' || in[end - 1] == '\t')) {
        --end;
    }
 
    std::string out;
    out.reserve(end);
    for (std::size_t i = 0; i < end; ) {
        const unsigned char c = static_cast<unsigned char>(in[i]);
 
        // ASCII branch
        if (c < 0x80) {
            // Forbidden filename characters across Windows + POSIX
            const bool forbidden =
                c < 0x20 || c == 0x7F ||
                c == '/' || c == '\\' || c == ':' || c == '*' ||
                c == '?' || c == '"' || c == '<' || c == '>' || c == '|';
            if (forbidden) {
                out.push_back('_');
            } else if (c == ' ' || c == '\t') {
                out.push_back('_');
            } else {
                out.push_back(static_cast<char>(c));
            }
            ++i;
            continue;
        }
 
        // UTF-8 multibyte: count expected length
        std::size_t expected = 0;
        if      ((c & 0xE0) == 0xC0) expected = 2;
        else if ((c & 0xF0) == 0xE0) expected = 3;
        else if ((c & 0xF8) == 0xF0) expected = 4;
        else {
            // invalid lead byte
            out.push_back('_');
            ++i;
            continue;
        }
        if (i + expected > end) {
            out.push_back('_');
            ++i;
            continue;
        }
        // Validate continuation bytes
        bool valid = true;
        for (std::size_t k = 1; k < expected; ++k) {
            if ((static_cast<unsigned char>(in[i + k]) & 0xC0) != 0x80) {
                valid = false;
                break;
            }
        }
        if (!valid) {
            out.push_back('_');
            ++i;
            continue;
        }
        // Append the whole UTF-8 sequence as-is
        for (std::size_t k = 0; k < expected; ++k) {
            out.push_back(in[i + k]);
        }
        i += expected;
    }
 
    if (out.empty()) out = "unnamed";
    return out;
}
 
} // namespace text_utils