diff --git a/README.md b/README.md index 69c9a57..6b94583 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Features * Deals with basic auth for you, just provide username and password options * Simple service wrapper that allows you to easily put together REST API libraries * Transparently handle content-encoded responses (gzip, deflate) +* Transparently handle different content charsets via `iconv` API diff --git a/lib/restler.js b/lib/restler.js index 47d54a1..c301063 100644 --- a/lib/restler.js +++ b/lib/restler.js @@ -4,7 +4,8 @@ var sys = require('util'), url = require('url'), qs = require('querystring'), multipart = require('./multipartform'), - zlib = require('zlib'); + zlib = require('zlib'), + Iconv = require('iconv').Iconv; function mixin(target, source) { Object.keys(source).forEach(function(key) { @@ -123,6 +124,7 @@ mixin(Request.prototype, { return; } response.raw = body; + body = self._iconv(body, response); self._encode(body, response, function(err, body) { if (err) { self._fireError(err, response); @@ -142,6 +144,20 @@ mixin(Request.prototype, { callback(null, body); } }, + _iconv: function(body, response) { + var charset = response.headers['content-type']; + if (!charset) return body; + charset = charset.match(/\bcharset=(.+)(?:;|$)/i); + if (!charset) return body; + charset = charset[1].trim().toUpperCase(); + if (charset == 'UTF-8') return body; + try { + var iconv = new Iconv(charset, 'UTF-8//TRANSLIT//IGNORE'); + return iconv.convert(body); + } catch (err) { + return body; + } + }, _encode: function(body, response, callback) { var self = this; if (self.options.decoding == 'buffer') { diff --git a/package.json b/package.json index ae300eb..b55b0ac 100644 --- a/package.json +++ b/package.json @@ -6,5 +6,14 @@ "homepage": "https://github.com/danwrong/restler", "directories" : { "lib" : "./lib" }, "main" : "./lib/restler", - "engines": { "node": ">= 0.3.7" } + "engines": { "node": ">= 0.3.7" }, + "dependencies": { + "iconv": ">=1.0.0", + "zlib" : ">=1.0.0" + }, + "devDependencies": { + "nodeunit": ">=0.5.0", + "xml2js" : ">=0.1.0", + "yaml" : ">=0.2.0" + } } diff --git a/test/charsets/gb2312 b/test/charsets/gb2312 new file mode 100644 index 0000000..fa89831 --- /dev/null +++ b/test/charsets/gb2312 @@ -0,0 +1,17 @@ +百度一下，你就知道 + + +
搜索设置|登录注册
+

新 闻　网 页　贴 吧　知 道　MP3　图 片　视 频　地 图

输入法
+

空间　百科　hao123 | 更多>>

把百度设为主页把百度加入收藏夹

+

加入百度推广 | 搜索风云榜 | 关于百度 | About Baidu

©2012 Baidu 使用百度前必读 京ICP证030173号

+ + + + + + + + + + \ No newline at end of file diff --git a/test/charsets/gbk b/test/charsets/gbk new file mode 100644 index 0000000..aef576e --- /dev/null +++ b/test/charsets/gbk @@ -0,0 +1,1234 @@ + + + + + + + + + + + + + + +亿起发，中国效果营销第一联盟 + + + + + + + + + + + + + + + + + + +
+ + + +
+ 新手上路 + 网站主帮助 + 广告主帮助 +
关注亿起发：sina微博腾讯微博
+
+ +
+ + +
+
+ + +
+ 首页 + 广告主 + 网站主 + 购物客 + 掘金链 + 广告活动 + 积分计划 + 开放平台
+
+
+ + +
+
+ + + + + + + + + + + + + + + +
+ +
+ + +
+

亿起发优势

+
+ 网站主优势 + 平台优势 + 广告主优势 +
+
+ +
+

星级网站主

+
+
    + +
  • + + 搜狗 + +
    • + +
  • + + 2345导航 + +
    • + +
  • + + 豆瓣 + +
    • + +
  • + + 360 + +
    • + +
  • + + 网易 + +
    • + +
  • + + 易购网 + +
    • + +
+
+
+ +
+

媒体报道

+
+ +
+
+ +
+

合作感言

+
+
+

秦小站从一个不懂做站的新手到现在拥有几万IP流量的站长，在与亿起发合作过程中感受到了亿起发工作人员的热忱、专业和严谨，经过双方不断的优化努力，收入不断攀升。祝亿起发联盟越办越好!

+
+
+

Highwell亿起发作为中国效果营销第一联盟，我们非常荣幸能成为亿起发的合作伙伴，也希望今后能携手亿起发共同发展！

+
+
+

笨阿牛亿起发积分非常赞，现在兑换了移动充值卡，我要多赚佣金多攒积分，兑换京东礼品卡。

+
+
+
+ +
+

联系我们

+
+

广告主合作：010-58794151-6358

网站主合作：010-58793980-8806

电子邮箱：客服邮箱投诉邮箱

+ +
+
+
+
+ +
+ +
+
+
+
    + +
    • + +
    • + +
    • + +
    • + +
    • + +
    • + +
    • + +
    • + +
    • + +
    • + +
    • + +
    • + +
+
+ +
+
+ +
+

积分奖励

积分是什么？
+
+
    + + + +
  • 奖品图
    • +
  • 奖品图
    • +
  • 奖品图
    • +
  • 奖品图
    • + +
+

兑换动态

+
    + +
  • 会员to***** + 成功兑换移动充值卡100元
    • + +
  • 会员to***** + 成功兑换移动充值卡100元
    • + +
  • 会员to***** + 成功兑换移动充值卡100元
    • + +
+
+
+
+
+ +
+ +
+

亿起发公告

更多订阅
+ + +
+ +
+

广告主动态

更多订阅
+ + +
+
+
+ +
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+
+ +
+ +
+
+
+
+
+
+
+
+ +
+ + + + + +
+ +
+

优惠券

+
+
    + + + +
  • + +
    +
    +

    100-15

    + +
    +
    +

    免费

    + +

    去领取

    + +
    +
    +
    +
    + +
    • + + + +
  • + +
    +
    +

    200-30

    + +
    +
    +

    免费

    + +

    去领取

    + +
    +
    +
    +
    + +
    • + + + +
  • + +
    +
    +

    50-10

    + +
    +
    +

    免费

    + +

    去领取

    + +
    +
    +
    +
    + +
    • + + +
  • + + +
    +
    +

    500-200

    + +
    +
    +

    免费

    + +

    去领取

    + +
    +
    +
    +
    + +
    • + + + +
  • + +
    +
    +

    99-10

    + +
    +
    +

    免费

    + +

    去领取

    + +
    +
    +
    +
    + +
    • + + + +
  • + +
    +
    +

    100-15

    + +
    +
    +

    免费

    + +

    去领取

    + +
    +
    +
    +
    + +
    • + + + +
  • + +
    +
    +

    68-5

    + +
    +
    +

    免费

    + +

    去领取

    + +
    +
    +
    +
    + +
    • + + +
  • + + +
    +
    +

    80-20

    + +
    +
    +

    免费

    + +

    去领取

    + +
    +
    +
    +
    + +
    • + +
    +
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+
+
+ + +
+ +
+ 关于我们|联系我们|诚聘英才|服务条款|合作伙伴|网站地图 +
+

Copyright©2004-2010 亿起发联盟效果营销平台 [京ICP备10001024号] 网络运营：北京亿玛在线科技有限公司

+ +

+
+ + +
+
+
+

什么是RSS？RSS订阅

+
+
+
+

在线阅读器直接订阅：

+

+

+ + + 如果还没有RSS阅读器，请立即下载 +

+
+

+
+ + diff --git a/test/charsets/iso-8859-1 b/test/charsets/iso-8859-1 new file mode 100644 index 0000000..fc8a742 --- /dev/null +++ b/test/charsets/iso-8859-1 @@ -0,0 +1,418 @@ + + + + +ZEDO Advertising Technology Partner + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
  + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + + +
+
+ +
+ + + + + + + + + +
+ + + + + +
+
+ + + + + + + + +
Publisher + solutions, tools, and services
+
+ + + + + + + + +
ZEDO + in the News
+ + + + + + + + + + + + + + + + + + + +
Overcoming + Corporate Culture Challenges + + + + +
September + 8, 2011
Interview: + 'Innovations change user expirence' + + + + +
September + 6, 2011
Interview: + 'What drives managers at work' + + + +
September + 5, 2011
+
+ + + 		+ + + + + + + + + + +
+
+ + + + + + + + + + + +
Introducing...
+ + + + + + + + + + + +
Click + to buy guaranteed placements on newspaper + sites.
  Know More
+
+
+ + + + + + + + +
Make money below the fold!
+ If you still don't believe you can monetize + content below the fold, let us show you otherwise. +

Analytics for advertisers have never been + better. They want quality brand-safe content + and ads that are viewed not hidden below + the fold. This presents a challenge to publishers + especially if they have long pages. But + publishers don't worry - below the fold + can be monetized at good CPMs using ZEDO + technology. Advertisers using tools like + DoubleVerify will be able assure themselves + their ads have been seen -- even below the + fold.

+

Get in touch with our sales representative today!
+ +
+

+
+
+ +
+		 + + + + + + + + + + + + + + + + + +
+ +
+
+
+ +

"I am not exaggerating when I say I have never met a more professional and efficient group of people. I just sat next to the CEO of AdTech on the plane and told him there was no way we抎 leave ZEDO. Your reporting, ease of use, ranking, and geo-targeting are industry leading and I抳e used them all, DART, Atlas, etc. Reporting is especially well done, this is a true asset to ZEDO as much as your human capital."
+ - Peter S. Shin
VP, Business Development
Mobissimo

+

"Zedo has reliable customer service. You can be certain that the support team will reply promptly to any questions posed."
+ - Cheza Estacion
fomerly of ABS-CBN

+

"I am very happy with your amazing support team. They are all very professional and responsive and help us with any issue we have."
+ - Naama Miller
Director of Publisher Relations
XTEND Media

+

"ZEDO has been a great partner year after year, working with us to develop high-impact Rich Media formats so we have something unique to offer our clients. Their support team backs us up 24/7."
+ - Josh Peerless
Senior Interactive Developer, Corporate Marketing
CBS Interactive

+ +

"ZEDO is always there when you need them...they listen/they respond/they act....they're a perfect ad serving partner"
+ - Roary Wilder
CEO
AdGroups

+

"ZEDO [is] a leading provider of ad serving solutions. Their function set, service, and uptime is in my opinion unparalleled. ZEDO will offer just about any functionality you can think of or expect from an ad serving provider at rates that you will find very hard to beat. I have worked with ZEDO for several years and have had nothing but excellent service from them."
+ - Jess Dolgin
JMedia

+

"ZEDO is light years ahead of any other ad serving company as far as serving our needs."
+ - Major Customer in the Newspaper Space

+

"[The ZEDO Implementation team's] help has been invaluable, and helped enable a smooth transition for us."
+ - Shawn Verdin
Client Services Coordinator
iMedia Connection

+

"I think ZEDO is the perfect ad server for any site or company looking to make money from unsold ad inventory."
+ - Major Customer in the Newspaper Space

+

"We find ZEDO to have the best reporting of all ad servers that our team uses on behalf of our clients, being both fast and flexible, allowing us to drill down into data from multiple views."
+ - Doucette Media

+

"I love all the great development work Zedo does and the highly responsive support team that is available 24/7."
+ - Dorian Kim
Senior Ad Operations Manager
DogTime Media, Inc

+

"ZEDO's ad ops team is experienced with our ad server and works with us to make sure they understand our specific needs... we never have to worry about being short on trafficking resources."
+ - Kelly Bolyard
Vice President
Paddock Publications

+

"I would recommend ZEDO to anyone, your teams are always so helpful."
+ - Colleen Harrison
Account Manager
Cyberstrobe Marketing

+
+
+
+ +
+
+ + + + + + + + + + + + + + +
+ + + + + + +
+ + + + + + + + + + + + +
+
+
+
Publishers + | Advertisers + | Ad Networks + | About Us | News + | Contact Us | + Privacy Statement + | Opt + Out | Anti-Spyware + | Blog + | Contact Sales
+ + +
© 2000- + + , ZEDO, Inc.
+ 		 
+ + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/charsets/shift_jis b/test/charsets/shift_jis new file mode 100644 index 0000000..57dc923 --- /dev/null +++ b/test/charsets/shift_jis @@ -0,0 +1,104 @@ + + + + + + + + + + +俀偪傖傫偹傞宖帵斅傊傛偆偙偦 + + + + + + +
+ + +
+僯儏乕僗乥僿僢僪儔僀儞乥僥儗價棑乥夁嫀儘僌憅屔 + +
+ +
+ +
+ + + + + + +
+ + \ No newline at end of file diff --git a/test/charsets/windows-1251 b/test/charsets/windows-1251 new file mode 100644 index 0000000..c81b86c --- /dev/null +++ b/test/charsets/windows-1251 @@ -0,0 +1,324 @@ + + + + + + + + + + + + +念狃� 镱驵腩忄螯 + + + + + + + + + + + + + +
+
+ +
+
+ +
+ + + +
+
+
+
+
+
+
+
+
+ + +
+
+
+ + + +
+
+
+ +
+
+
+
+
+
+ +
+

率铐蜞牝� � 箜桠屦襦朦眍� 耩邃耱忸 潆� 觐祆箜桕圉梃 � 镱桉赅 膻溴�,
觐蝾瘥� 彐邃礤忭� 镱朦珞�� 溴��蜿� 扈腚桀眍� 麇腩忮�.

+ +

填戾眚嚯�� 疱汨耱疣鲨�

+ +
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +

� 麇� 镱祛驽� 率铐蜞牝�?

+
    +
  • 袜轵� 膻溴�, � 觐蝾瘥扈 蔓 觐沅�-腓犷 篦桦桉�, 疣犷蜞腓 桦� 铗潲踵腓.
    • +
  • 隅磬螯 犷朦 � 膻���, 觐蝾瘥� 锣� 铌痼驵, � 磬轵� 眍恹� 漯箸彘.
    • +
  • 埋邈溧 铖蜞忄螯�� � 觐眚嚓蝈 � 蝈扈, 牝� 锣� 漕痤�.
    • +
+
+
+ + + + +
+
+
+
+
+
+ + +
+
+
+
+ + + + + \ No newline at end of file diff --git a/test/charsets/windows-1252 b/test/charsets/windows-1252 new file mode 100644 index 0000000..c77d4c8 --- /dev/null +++ b/test/charsets/windows-1252 @@ -0,0 +1,30 @@ + + + + + ForumFree - La pi� grande comunit� di forum gratis + + + + +
Accedi
PasswordRecupera password

Registrati subito!

+

E-mail

+

Nome

+

Password

+

accetto i termini e le condizioni autorizzando il trattamento dei miei dati personali ai sensi dell'informativa
acconsento al trattamento dei dati per l'invio di comunicazioni commerciali di cui all'art. 2.B dell'informativa

+

Il tuo forum. Gratis.

  • In un attimo. ForumFree ti permette di creare un forum o un blog in pochi semplici passi.
  • Migliaia di forum. Accedi a tutti i forum con una sola registrazione. Le tue impostazioni personali rimangono le stesse.
  • La tua grafica. Puoi utilizzare uno dei temi creati dai nostri utenti, oppure creare il tuo stile personalizzato.
  • Facile e sicuro. Puoi scegliere chi potr� accedere o scrivere sul tuo forum in modo molto veloce ed efficace.
+

Assistenza

Dubbi o problemi con il tuo forum? Tranquillo! Il servizio ha una grande raccolta di guide e un forum di supporto ufficiale in cui potrai ricevere assistenza.

Motori di ricerca

I nostri forum si trovano nelle prime posizioni in tutti i principali motori di ricerca e ricevono giornalmente migliaia di visite.

Album

Carica le immagini direttamente sul tuo album. Potrai allegarle ai messaggi e mostrarle ai tuoi amici in pochi passi.

Skin

Personalizza il tuo forum in un attimo. Centinaia di temi grafici sono gi酄 pronti. Puoi chiedere consiglio agli altri utenti sullo Skin Lab, la nostra community ufficiale di grafica.

+ + \ No newline at end of file diff --git a/test/restler.js b/test/restler.js index 74cab6f..e487eec 100644 --- a/test/restler.js +++ b/test/restler.js @@ -1,8 +1,12 @@ -var rest = require('../lib/restler'); -var http = require('http'); -var sys = require('util'); -var zlib = require('zlib'); +var rest = require('../lib/restler'); +var http = require('http'); +var sys = require('util'); +var zlib = require('zlib'); +var path = require('path'); +var fs = require('fs'); +var crypto = require('crypto'); + var p = sys.inspect; var port = 9000; @@ -470,11 +474,54 @@ module.exports['Deserialization'] = { }).on('fail', function() { test.ok(false, 'should not have got here'); }); - }, + } + +}; +function md5(data) { + return crypto.createHash('md5').update(data).digest('hex'); +} + +function charsetsResponse(request, response) { + var charset = request.url.substr(1); + response.writeHead(200, { + 'content-type': 'text/plain; charset=' + charset, + 'content-encoding': 'gzip' + }); + fs.createReadStream(path.join(__dirname, charsetsDir, charset)).pipe(zlib.createGzip()).pipe(response); +} + +module.exports['Charsets'] = { + setUp: setup(charsetsResponse), + tearDown: teardown() +}; + +var charsetsDir = 'charsets'; +var charsetCases = { + /** + * key - the name of the charset of tested file found in directory with the same name + * value - md5 hash of properly utf8-iconv'erted file. + */ + 'iso-8859-1' : '8c7fbd6fb81d089573540bf0bdf82cc6', + 'gb2312' : 'ab788473ee3b5f5fff5eba4ca6172834', + 'windows-1251' : 'ab90f5e5333149acbfd58441cfe69d70', + 'shift_jis' : '3b93ee3382ed73ec6d064a2ce852a50a', + 'windows-1252' : 'b9bd334aeb238eb104628168cb011351', + 'gbk' : '01329db97a6a202ecffaf95d4f77a18d' }; +for (var charset in charsetCases) { + (function(charset, hash) { + module.exports['Charsets']['Should correctly convert charset ' + charset] = function(test) { + rest.get(host + '/' + charset).on('complete', function(data) { + test.equal(md5(Buffer(data, 'utf8')), hash, 'hashes should match'); + test.done(); + }); + }; + })(charset, charsetCases[charset]); +} + function redirectResponse(request, response) { if (request.url == '/redirected') {