|
| 1 | +// Copyright (c) 2017-2022 Cloudflare, Inc. |
| 2 | +// Licensed under the Apache 2.0 license found in the LICENSE file or at: |
| 3 | +// https://opensource.org/licenses/Apache-2.0 |
| 4 | +// Copyright Joyent and Node contributors. All rights reserved. MIT license. |
| 5 | + |
| 6 | +#include "i18n.h" |
| 7 | + |
| 8 | +#include <workerd/jsg/exception.h> |
| 9 | + |
| 10 | +#include <unicode/putil.h> |
| 11 | +#include <unicode/timezone.h> |
| 12 | +#include <unicode/uchar.h> |
| 13 | +#include <unicode/uclean.h> |
| 14 | +#include <unicode/ucnv.h> |
| 15 | +#include <unicode/udata.h> |
| 16 | +#include <unicode/uidna.h> |
| 17 | +#include <unicode/ulocdata.h> |
| 18 | +#include <unicode/urename.h> |
| 19 | +#include <unicode/ustring.h> |
| 20 | +#include <unicode/utf16.h> |
| 21 | +#include <unicode/utf8.h> |
| 22 | +#include <unicode/utypes.h> |
| 23 | +#include <unicode/uvernum.h> |
| 24 | +#include <unicode/uversion.h> |
| 25 | + |
| 26 | +namespace workerd::api::node { |
| 27 | + |
| 28 | +namespace i18n { |
| 29 | + |
| 30 | +namespace { |
| 31 | + |
| 32 | +struct ConverterDisposer : public kj::Disposer { |
| 33 | + static const ConverterDisposer INSTANCE; |
| 34 | + void disposeImpl(void* pointer) const override { |
| 35 | + ucnv_close(reinterpret_cast<UConverter*>(pointer)); |
| 36 | + } |
| 37 | +}; |
| 38 | + |
| 39 | +const ConverterDisposer ConverterDisposer::INSTANCE; |
| 40 | + |
| 41 | +const char* getEncodingName(Encoding input) { |
| 42 | + switch (input) { |
| 43 | + case Encoding::ASCII: |
| 44 | + return "us-ascii"; |
| 45 | + case Encoding::LATIN1: |
| 46 | + return "iso8859-1"; |
| 47 | + case Encoding::UCS2: |
| 48 | + return "utf16le"; |
| 49 | + case Encoding::UTF8: |
| 50 | + return "utf-8"; |
| 51 | + default: |
| 52 | + KJ_UNREACHABLE; |
| 53 | + } |
| 54 | +} |
| 55 | + |
| 56 | +typedef kj::Maybe<kj::Array<kj::byte>> (*TranscodeImpl)(kj::ArrayPtr<kj::byte> source, |
| 57 | + Encoding fromEncoding, Encoding toEncoding); |
| 58 | + |
| 59 | +kj::Maybe<kj::Array<kj::byte>> TranscodeDefault(kj::ArrayPtr<kj::byte> source, |
| 60 | + Encoding fromEncoding, Encoding toEncoding) { |
| 61 | + Converter to(toEncoding); |
| 62 | + std::string substitude(to.minSize(), '?'); |
| 63 | + to.setSubstitudeChars(substitude); |
| 64 | + Converter from(fromEncoding); |
| 65 | + |
| 66 | + auto limit = source.size() + to.maxSize(); |
| 67 | + auto out = kj::heapArray<kj::byte>(limit); |
| 68 | + char* target = out.asChars().begin(); |
| 69 | + const char* source_ = source.asChars().begin(); |
| 70 | + UErrorCode status{}; |
| 71 | + ucnv_convertEx(to.conv(), from.conv(), &target, target + limit, &source_, source_ + source.size(), |
| 72 | + nullptr, nullptr, nullptr, nullptr, true, true, &status); |
| 73 | + if (U_SUCCESS(status)) { |
| 74 | + return out.slice(0, target - out.asChars().begin()).attach(kj::mv(out)); |
| 75 | + } |
| 76 | + |
| 77 | + return kj::none; |
| 78 | +} |
| 79 | + |
| 80 | +kj::Maybe<kj::Array<kj::byte>> TranscodeToUCS2(kj::ArrayPtr<kj::byte> source, Encoding fromEncoding, |
| 81 | + Encoding toEncoding) { |
| 82 | + UErrorCode status{}; |
| 83 | + const size_t length_in_chars = source.size() * sizeof(UChar); |
| 84 | + Converter from(fromEncoding); |
| 85 | + auto out = kj::heapArray<UChar>(source.size()); |
| 86 | + const auto source_ = source.asChars().begin(); |
| 87 | + ucnv_toUChars(from.conv(), out.begin(), length_in_chars, source_, source.size(), &status); |
| 88 | + if (U_SUCCESS(status)) { |
| 89 | + return out.asBytes().attach(kj::mv(out)); |
| 90 | + } |
| 91 | + return kj::none; |
| 92 | +} |
| 93 | + |
| 94 | +kj::Maybe<kj::Array<kj::byte>> TranscodeFromUCS2(kj::ArrayPtr<kj::byte> source, |
| 95 | + Encoding fromEncoding, Encoding toEncoding) { |
| 96 | + UErrorCode status{}; |
| 97 | + KJ_STACK_ARRAY(kj::byte, sourcebuf, 1024, 0, 1024); |
| 98 | + Converter to(toEncoding); |
| 99 | + std::string substitude(to.minSize(), '?'); |
| 100 | + to.setSubstitudeChars(substitude); |
| 101 | + |
| 102 | + const size_t length_in_chars = source.size() * sizeof(UChar); |
| 103 | + sourcebuf.copyFrom(source.slice(0, length_in_chars)); |
| 104 | + |
| 105 | + auto destbuf = kj::heapArray<kj::byte>(length_in_chars); |
| 106 | + const auto source_ = reinterpret_cast<const UChar*>(sourcebuf.asChars().begin()); |
| 107 | + auto len = ucnv_fromUChars(to.conv(), destbuf.asChars().begin(), length_in_chars, source_, |
| 108 | + length_in_chars, &status); |
| 109 | + |
| 110 | + if (U_SUCCESS(status)) { |
| 111 | + return destbuf.slice(0, len).attach(kj::mv(destbuf)); |
| 112 | + } |
| 113 | + |
| 114 | + return kj::none; |
| 115 | +} |
| 116 | + |
| 117 | +kj::Maybe<kj::Array<kj::byte>> TranscodeUcs2FromUtf8(kj::ArrayPtr<kj::byte> source, |
| 118 | + Encoding fromEncoding, Encoding toEncoding) { |
| 119 | + return kj::none; |
| 120 | +} |
| 121 | + |
| 122 | +kj::Maybe<kj::Array<kj::byte>> TranscodeUtf8FromUcs2(kj::ArrayPtr<kj::byte> source, |
| 123 | + Encoding fromEncoding, Encoding toEncoding) { |
| 124 | + return kj::none; |
| 125 | +} |
| 126 | + |
| 127 | +} // namespace |
| 128 | + |
| 129 | +Converter::Converter(Encoding encoding, kj::StringPtr substitude) { |
| 130 | + UErrorCode status = U_ZERO_ERROR; |
| 131 | + auto name = getEncodingName(encoding); |
| 132 | + auto conv = ucnv_open(name, &status); |
| 133 | + KJ_ASSERT(U_SUCCESS(status)); |
| 134 | + conv_ = kj::Own<UConverter>(conv, ConverterDisposer::INSTANCE); |
| 135 | + setSubstitudeChars(substitude); |
| 136 | +} |
| 137 | + |
| 138 | +Converter::Converter(UConverter* converter, kj::StringPtr substitude) |
| 139 | + : conv_(converter, ConverterDisposer::INSTANCE) { |
| 140 | + setSubstitudeChars(substitude); |
| 141 | +} |
| 142 | + |
| 143 | +UConverter* Converter::conv() const { |
| 144 | + return const_cast<UConverter*>(conv_.get()); |
| 145 | +} |
| 146 | + |
| 147 | +size_t Converter::maxSize() const { |
| 148 | + KJ_ASSERT_NONNULL(conv_.get()); |
| 149 | + return ucnv_getMaxCharSize(conv_.get()); |
| 150 | +} |
| 151 | + |
| 152 | +size_t Converter::minSize() const { |
| 153 | + KJ_ASSERT_NONNULL(conv_.get()); |
| 154 | + return ucnv_getMinCharSize(conv_.get()); |
| 155 | +} |
| 156 | + |
| 157 | +void Converter::reset() { |
| 158 | + KJ_ASSERT_NONNULL(conv_.get()); |
| 159 | + ucnv_reset(conv_.get()); |
| 160 | +} |
| 161 | + |
| 162 | +void Converter::setSubstitudeChars(kj::StringPtr sub) { |
| 163 | + KJ_ASSERT_NONNULL(conv_.get()); |
| 164 | + UErrorCode status = U_ZERO_ERROR; |
| 165 | + ucnv_setSubstChars(conv_.get(), sub.begin(), sub.size(), &status); |
| 166 | + KJ_ASSERT(U_SUCCESS(status)); |
| 167 | +} |
| 168 | + |
| 169 | +kj::Array<kj::byte> transcode(kj::ArrayPtr<kj::byte> source, Encoding fromEncoding, |
| 170 | + Encoding toEncoding) { |
| 171 | + TranscodeImpl transcode_function = &TranscodeDefault; |
| 172 | + switch (fromEncoding) { |
| 173 | + case Encoding::ASCII: |
| 174 | + case Encoding::LATIN1: |
| 175 | + if (toEncoding == Encoding::UCS2) { |
| 176 | + transcode_function = &TranscodeToUCS2; |
| 177 | + } |
| 178 | + break; |
| 179 | + case Encoding::UTF8: |
| 180 | + if (toEncoding == Encoding::UCS2) { |
| 181 | + transcode_function = &TranscodeUcs2FromUtf8; |
| 182 | + } |
| 183 | + break; |
| 184 | + case Encoding::UCS2: |
| 185 | + switch (toEncoding) { |
| 186 | + case Encoding::UCS2: |
| 187 | + transcode_function = &TranscodeDefault; |
| 188 | + break; |
| 189 | + case Encoding::UTF8: |
| 190 | + transcode_function = &TranscodeUtf8FromUcs2; |
| 191 | + break; |
| 192 | + default: |
| 193 | + transcode_function = &TranscodeFromUCS2; |
| 194 | + } |
| 195 | + default: |
| 196 | + KJ_UNREACHABLE; |
| 197 | + } |
| 198 | + |
| 199 | + return JSG_REQUIRE_NONNULL(transcode_function(source, fromEncoding, toEncoding), Error, |
| 200 | + "Unable to transcode buffer"); |
| 201 | +} |
| 202 | + |
| 203 | +} // namespace i18n |
| 204 | + |
| 205 | +} // namespace workerd::api::node |
0 commit comments