Skip to content
API Documentation
BastionGuard™
hfs::anonymous_namespace{hfs_unicode.cpp} Namespace Reference

Functions

std::vector< uint32_t > utf16be_to_codepoints (const unsigned char *src, size_t code_units)
void append_utf8 (std::string &out, uint32_t cp)
std::string codepoints_to_utf8 (const std::vector< uint32_t > &cps)
uint64_t make_key (uint32_t base, uint32_t mark)
const std::unordered_map< uint64_t, uint32_t > & nfc_table ()
bool is_combining_mark (uint32_t cp)
std::vector< uint32_t > compose_nfc (const std::vector< uint32_t > &nfd)

Function Documentation

◆ append_utf8()

void hfs::anonymous_namespace{hfs_unicode.cpp}::append_utf8 ( std::string & out,
uint32_t cp )

Definition at line 59 of file hfs_unicode.cpp.

59 {
60 if (cp < 0x80) {
61 out.push_back(static_cast<char>(cp));
62 } else if (cp < 0x800) {
63 out.push_back(static_cast<char>(0xC0 | (cp >> 6)));
64 out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
65 } else if (cp < 0x10000) {
66 out.push_back(static_cast<char>(0xE0 | (cp >> 12)));
67 out.push_back(static_cast<char>(0x80 | ((cp >> 6) & 0x3F)));
68 out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
69 } else if (cp < 0x110000) {
70 out.push_back(static_cast<char>(0xF0 | (cp >> 18)));
71 out.push_back(static_cast<char>(0x80 | ((cp >> 12) & 0x3F)));
72 out.push_back(static_cast<char>(0x80 | ((cp >> 6) & 0x3F)));
73 out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
74 } else {
75 // Out of Unicode range -- replace with U+FFFD
76 out.push_back(static_cast<char>(0xEF));
77 out.push_back(static_cast<char>(0xBF));
78 out.push_back(static_cast<char>(0xBD));
79 }
80}

References append_utf8().

Referenced by append_utf8(), and codepoints_to_utf8().

◆ codepoints_to_utf8()

std::string hfs::anonymous_namespace{hfs_unicode.cpp}::codepoints_to_utf8 ( const std::vector< uint32_t > & cps)

Definition at line 82 of file hfs_unicode.cpp.

82 {
83 std::string out;
84 out.reserve(cps.size() * 2);
85 for (uint32_t cp : cps) append_utf8(out, cp);
86 return out;
87}
hfs::anonymous_namespace{hfs_unicode.cpp}::append_utf8
void append_utf8(std::string &out, uint32_t cp)
Definition hfs_unicode.cpp:59

References append_utf8(), and codepoints_to_utf8().

Referenced by codepoints_to_utf8().

◆ compose_nfc()

std::vector< uint32_t > hfs::anonymous_namespace{hfs_unicode.cpp}::compose_nfc ( const std::vector< uint32_t > & nfd)

Definition at line 216 of file hfs_unicode.cpp.

216 {
217 std::vector<uint32_t> out;
218 out.reserve(nfd.size());
219
220 const auto& tbl = nfc_table();
221 size_t i = 0;
222 while (i < nfd.size()) {
223 const uint32_t base = nfd[i];
224 // Look for a single combining mark following.
225 if (i + 1 < nfd.size() && is_combining_mark(nfd[i + 1])) {
226 const uint32_t mark = nfd[i + 1];
227 auto it = tbl.find(make_key(base, mark));
228 if (it != tbl.end()) {
229 out.push_back(it->second);
230 i += 2;
231 // Try to fold any further combining marks if they
232 // recompose with the new precomposed character.
233 while (i < nfd.size() && is_combining_mark(nfd[i])) {
234 auto it2 = tbl.find(make_key(out.back(), nfd[i]));
235 if (it2 == tbl.end()) {
236 out.push_back(nfd[i]);
237 ++i;
238 } else {
239 out.back() = it2->second;
240 ++i;
241 }
242 }
243 continue;
244 }
245 }
246 out.push_back(base);
247 ++i;
248 }
249 return out;
250}
hfs::anonymous_namespace{hfs_unicode.cpp}::make_key
uint64_t make_key(uint32_t base, uint32_t mark)
Definition hfs_unicode.cpp:105
hfs::anonymous_namespace{hfs_unicode.cpp}::is_combining_mark
bool is_combining_mark(uint32_t cp)
Definition hfs_unicode.cpp:209
hfs::anonymous_namespace{hfs_unicode.cpp}::nfc_table
const std::unordered_map< uint64_t, uint32_t > & nfc_table()
Definition hfs_unicode.cpp:109

References compose_nfc(), is_combining_mark(), make_key(), and nfc_table().

Referenced by compose_nfc().

◆ is_combining_mark()

bool hfs::anonymous_namespace{hfs_unicode.cpp}::is_combining_mark ( uint32_t cp)

Definition at line 209 of file hfs_unicode.cpp.

209 {
210 // Range of combining diacritical marks (U+0300 .. U+036F) covers
211 // every accent in our table. We don't need the full Unicode
212 // combining-class machinery for this.
213 return cp >= 0x0300 && cp <= 0x036F;
214}

References is_combining_mark().

Referenced by compose_nfc(), and is_combining_mark().

◆ make_key()

uint64_t hfs::anonymous_namespace{hfs_unicode.cpp}::make_key ( uint32_t base,
uint32_t mark )

Definition at line 105 of file hfs_unicode.cpp.

105 {
106 return (static_cast<uint64_t>(base) << 32) | mark;
107}

References make_key().

Referenced by compose_nfc(), make_key(), and nfc_table().

◆ nfc_table()

const std::unordered_map< uint64_t, uint32_t > & hfs::anonymous_namespace{hfs_unicode.cpp}::nfc_table ( )

Definition at line 109 of file hfs_unicode.cpp.

109 {
110 static const std::unordered_map<uint64_t, uint32_t> t = {
111 // ---- COMBINING GRAVE ACCENT (U+0300) ----
112 { make_key('A', 0x0300), 0x00C0 }, { make_key('a', 0x0300), 0x00E0 },
113 { make_key('E', 0x0300), 0x00C8 }, { make_key('e', 0x0300), 0x00E8 },
114 { make_key('I', 0x0300), 0x00CC }, { make_key('i', 0x0300), 0x00EC },
115 { make_key('O', 0x0300), 0x00D2 }, { make_key('o', 0x0300), 0x00F2 },
116 { make_key('U', 0x0300), 0x00D9 }, { make_key('u', 0x0300), 0x00F9 },
117
118 // ---- COMBINING ACUTE ACCENT (U+0301) ----
119 { make_key('A', 0x0301), 0x00C1 }, { make_key('a', 0x0301), 0x00E1 },
120 { make_key('E', 0x0301), 0x00C9 }, { make_key('e', 0x0301), 0x00E9 },
121 { make_key('I', 0x0301), 0x00CD }, { make_key('i', 0x0301), 0x00ED },
122 { make_key('O', 0x0301), 0x00D3 }, { make_key('o', 0x0301), 0x00F3 },
123 { make_key('U', 0x0301), 0x00DA }, { make_key('u', 0x0301), 0x00FA },
124 { make_key('Y', 0x0301), 0x00DD }, { make_key('y', 0x0301), 0x00FD },
125 { make_key('C', 0x0301), 0x0106 }, { make_key('c', 0x0301), 0x0107 },
126 { make_key('L', 0x0301), 0x0139 }, { make_key('l', 0x0301), 0x013A },
127 { make_key('N', 0x0301), 0x0143 }, { make_key('n', 0x0301), 0x0144 },
128 { make_key('R', 0x0301), 0x0154 }, { make_key('r', 0x0301), 0x0155 },
129 { make_key('S', 0x0301), 0x015A }, { make_key('s', 0x0301), 0x015B },
130 { make_key('Z', 0x0301), 0x0179 }, { make_key('z', 0x0301), 0x017A },
131
132 // ---- COMBINING CIRCUMFLEX (U+0302) ----
133 { make_key('A', 0x0302), 0x00C2 }, { make_key('a', 0x0302), 0x00E2 },
134 { make_key('E', 0x0302), 0x00CA }, { make_key('e', 0x0302), 0x00EA },
135 { make_key('I', 0x0302), 0x00CE }, { make_key('i', 0x0302), 0x00EE },
136 { make_key('O', 0x0302), 0x00D4 }, { make_key('o', 0x0302), 0x00F4 },
137 { make_key('U', 0x0302), 0x00DB }, { make_key('u', 0x0302), 0x00FB },
138 { make_key('C', 0x0302), 0x0108 }, { make_key('c', 0x0302), 0x0109 },
139 { make_key('G', 0x0302), 0x011C }, { make_key('g', 0x0302), 0x011D },
140 { make_key('H', 0x0302), 0x0124 }, { make_key('h', 0x0302), 0x0125 },
141 { make_key('J', 0x0302), 0x0134 }, { make_key('j', 0x0302), 0x0135 },
142 { make_key('S', 0x0302), 0x015C }, { make_key('s', 0x0302), 0x015D },
143 { make_key('W', 0x0302), 0x0174 }, { make_key('w', 0x0302), 0x0175 },
144 { make_key('Y', 0x0302), 0x0176 }, { make_key('y', 0x0302), 0x0177 },
145
146 // ---- COMBINING TILDE (U+0303) ----
147 { make_key('A', 0x0303), 0x00C3 }, { make_key('a', 0x0303), 0x00E3 },
148 { make_key('N', 0x0303), 0x00D1 }, { make_key('n', 0x0303), 0x00F1 },
149 { make_key('O', 0x0303), 0x00D5 }, { make_key('o', 0x0303), 0x00F5 },
150 { make_key('I', 0x0303), 0x0128 }, { make_key('i', 0x0303), 0x0129 },
151 { make_key('U', 0x0303), 0x0168 }, { make_key('u', 0x0303), 0x0169 },
152
153 // ---- COMBINING DIAERESIS (U+0308) ----
154 { make_key('A', 0x0308), 0x00C4 }, { make_key('a', 0x0308), 0x00E4 },
155 { make_key('E', 0x0308), 0x00CB }, { make_key('e', 0x0308), 0x00EB },
156 { make_key('I', 0x0308), 0x00CF }, { make_key('i', 0x0308), 0x00EF },
157 { make_key('O', 0x0308), 0x00D6 }, { make_key('o', 0x0308), 0x00F6 },
158 { make_key('U', 0x0308), 0x00DC }, { make_key('u', 0x0308), 0x00FC },
159 { make_key('Y', 0x0308), 0x0178 }, { make_key('y', 0x0308), 0x00FF },
160
161 // ---- COMBINING RING ABOVE (U+030A) ----
162 { make_key('A', 0x030A), 0x00C5 }, { make_key('a', 0x030A), 0x00E5 },
163 { make_key('U', 0x030A), 0x016E }, { make_key('u', 0x030A), 0x016F },
164
165 // ---- COMBINING CEDILLA (U+0327) ----
166 { make_key('C', 0x0327), 0x00C7 }, { make_key('c', 0x0327), 0x00E7 },
167 { make_key('G', 0x0327), 0x0122 }, { make_key('g', 0x0327), 0x0123 },
168 { make_key('K', 0x0327), 0x0136 }, { make_key('k', 0x0327), 0x0137 },
169 { make_key('L', 0x0327), 0x013B }, { make_key('l', 0x0327), 0x013C },
170 { make_key('N', 0x0327), 0x0145 }, { make_key('n', 0x0327), 0x0146 },
171 { make_key('R', 0x0327), 0x0156 }, { make_key('r', 0x0327), 0x0157 },
172 { make_key('S', 0x0327), 0x015E }, { make_key('s', 0x0327), 0x015F },
173 { make_key('T', 0x0327), 0x0162 }, { make_key('t', 0x0327), 0x0163 },
174
175 // ---- COMBINING MACRON (U+0304) ----
176 { make_key('A', 0x0304), 0x0100 }, { make_key('a', 0x0304), 0x0101 },
177 { make_key('E', 0x0304), 0x0112 }, { make_key('e', 0x0304), 0x0113 },
178 { make_key('I', 0x0304), 0x012A }, { make_key('i', 0x0304), 0x012B },
179 { make_key('O', 0x0304), 0x014C }, { make_key('o', 0x0304), 0x014D },
180 { make_key('U', 0x0304), 0x016A }, { make_key('u', 0x0304), 0x016B },
181
182 // ---- COMBINING CARON (U+030C, hacek) ----
183 { make_key('C', 0x030C), 0x010C }, { make_key('c', 0x030C), 0x010D },
184 { make_key('D', 0x030C), 0x010E }, { make_key('d', 0x030C), 0x010F },
185 { make_key('E', 0x030C), 0x011A }, { make_key('e', 0x030C), 0x011B },
186 { make_key('L', 0x030C), 0x013D }, { make_key('l', 0x030C), 0x013E },
187 { make_key('N', 0x030C), 0x0147 }, { make_key('n', 0x030C), 0x0148 },
188 { make_key('R', 0x030C), 0x0158 }, { make_key('r', 0x030C), 0x0159 },
189 { make_key('S', 0x030C), 0x0160 }, { make_key('s', 0x030C), 0x0161 },
190 { make_key('T', 0x030C), 0x0164 }, { make_key('t', 0x030C), 0x0165 },
191 { make_key('Z', 0x030C), 0x017D }, { make_key('z', 0x030C), 0x017E },
192
193 // ---- COMBINING OGONEK (U+0328) ----
194 { make_key('A', 0x0328), 0x0104 }, { make_key('a', 0x0328), 0x0105 },
195 { make_key('E', 0x0328), 0x0118 }, { make_key('e', 0x0328), 0x0119 },
196 { make_key('I', 0x0328), 0x012E }, { make_key('i', 0x0328), 0x012F },
197 { make_key('U', 0x0328), 0x0172 }, { make_key('u', 0x0328), 0x0173 },
198
199 // ---- COMBINING DOUBLE ACUTE (U+030B), Hungarian ----
200 { make_key('O', 0x030B), 0x0150 }, { make_key('o', 0x030B), 0x0151 },
201 { make_key('U', 0x030B), 0x0170 }, { make_key('u', 0x030B), 0x0171 },
202
203 // ---- COMBINING DOT ABOVE (U+0307) ----
204 { make_key('Z', 0x0307), 0x017B }, { make_key('z', 0x0307), 0x017C },
205 };
206 return t;
207}

References make_key(), and nfc_table().

Referenced by compose_nfc(), and nfc_table().

◆ utf16be_to_codepoints()

std::vector< uint32_t > hfs::anonymous_namespace{hfs_unicode.cpp}::utf16be_to_codepoints ( const unsigned char * src,
size_t code_units )

Definition at line 26 of file hfs_unicode.cpp.

27 {
28 std::vector<uint32_t> out;
29 out.reserve(code_units);
30 for (size_t i = 0; i < code_units; ++i) {
31 const unsigned char* p = src + i * 2;
32 const uint16_t u = static_cast<uint16_t>((p[0] << 8) | p[1]);
33 if (u >= 0xD800 && u <= 0xDBFF) {
34 // High surrogate, expect a low surrogate next
35 if (i + 1 < code_units) {
36 const unsigned char* q = src + (i + 1) * 2;
37 const uint16_t v = static_cast<uint16_t>((q[0] << 8) | q[1]);
38 if (v >= 0xDC00 && v <= 0xDFFF) {
39 const uint32_t cp = 0x10000u
40 + ((static_cast<uint32_t>(u - 0xD800) << 10)
41 | (v - 0xDC00));
42 out.push_back(cp);
43 ++i;
44 continue;
45 }
46 }
47 out.push_back(0xFFFD); // lone high surrogate
48 } else if (u >= 0xDC00 && u <= 0xDFFF) {
49 out.push_back(0xFFFD); // unexpected low surrogate
50 } else {
51 out.push_back(u);
52 }
53 }
54 return out;
55}

References utf16be_to_codepoints().

Referenced by utf16be_to_codepoints().