handle different content charsets

mgreschke · Jan 8, 2012 · 4fe9cc4 · 4fe9cc4
1 parent 8b70a1a
commit 4fe9cc4
Show file tree

Hide file tree

Showing 10 changed files with 2,207 additions and 7 deletions.
diff --git a/README.md b/README.md
@@ -22,6 +22,7 @@ Features
 * Deals with basic auth for you, just provide username and password options
 * Simple service wrapper that allows you to easily put together REST API libraries
 * Transparently handle content-encoded responses (gzip, deflate)
+* Transparently handle different content charsets via `iconv`
 
 
 API

diff --git a/lib/restler.js b/lib/restler.js
@@ -4,7 +4,8 @@ var sys       = require('util'),
     url       = require('url'),
     qs        = require('querystring'),
     multipart = require('./multipartform'),
-    zlib      = require('zlib');
+    zlib      = require('zlib'),
+    Iconv     = require('iconv').Iconv;
 
 function mixin(target, source) {
   Object.keys(source).forEach(function(key) {
@@ -123,6 +124,7 @@ mixin(Request.prototype, {
             return;
           }
           response.raw = body;
+          body = self._iconv(body, response);
           self._encode(body, response, function(err, body) {
             if (err) {
               self._fireError(err, response);
@@ -142,6 +144,20 @@ mixin(Request.prototype, {
       callback(null, body);
     }
   },
+  _iconv: function(body, response) {
+    var charset = response.headers['content-type'];
+    if (!charset) return body;
+    charset = charset.match(/\bcharset=(.+)(?:;|$)/i);
+    if (!charset) return body;
+    charset = charset[1].trim().toUpperCase();
+    if (charset == 'UTF-8') return body;
+    try {
+      var iconv = new Iconv(charset, 'UTF-8//TRANSLIT//IGNORE');
+      return iconv.convert(body);
+    } catch (err) {
+      return body;
+    }
+  },
   _encode: function(body, response, callback) {
     var self = this;
     if (self.options.decoding == 'buffer') {

diff --git a/package.json b/package.json
@@ -6,5 +6,14 @@
     "homepage": "https://github.com/danwrong/restler",
     "directories" : { "lib" : "./lib" },
     "main" : "./lib/restler",
-    "engines": { "node": ">= 0.3.7" }
+    "engines": { "node": ">= 0.3.7" },
+    "dependencies": {
+        "iconv": ">=1.0.0",
+        "zlib" : ">=1.0.0"
+    },
+    "devDependencies": {
+        "nodeunit": ">=0.5.0",
+        "xml2js"  : ">=0.1.0",
+        "yaml"    : ">=0.2.0"
+    }
 }
diff --git a/test/charsets/gb2312 b/test/charsets/gb2312
@@ -0,0 +1,17 @@
+<!doctype html><html><head><meta http-equiv="Content-Type" content="text/html;charset=gb2312"><title>�ٶ�һ�£����֪��      </title><style>html{overflow-y:auto}body{font:12px arial;text-align:center;background:#fff}body,p,form,ul,li{margin:0;padding:0;list-style:none}body,form,#fm{position:relative}td{text-align:left}img{border:0}a{color:#00c}a:active{color:#f60}#u{color:#999;padding:4px 10px 5px 0;text-align:right}#u a{margin:0 5px}#u .reg{margin:0}#m{width:680px;margin:0 auto;}#nv a,#nv b,.btn,#lk{font-size:14px}#fm{padding-left:90px;text-align:left}input{border:0;padding:0}#nv{height:19px;font-size:16px;margin:0 0 4px;text-align:left;text-indent:117px}.s_ipt_wr{width:418px;height:30px;display:inline-block;margin-right:5px;background:url(http://www.baidu.com/img/i-1.0.0.png) no-repeat -304px 0;border:1px solid #b6b6b6;border-color:#9a9a9a #cdcdcd #cdcdcd #9a9a9a;vertical-align:top}.s_ipt{width:405px;height:22px;font:16px/22px arial;margin:5px 0 0 7px;background:#fff;outline:none}.s_btn{width:95px;height:32px;padding-top:2px\9;font-size:14px;background:#ddd url(http://www.baidu.com/img/i-1.0.0.png);cursor:pointer}.s_btn_h{background-position:-100px 0}.s_btn_wr{width:97px;height:34px;display:inline-block;background:url(http://www.baidu.com/img/i-1.0.0.png) no-repeat -202px 0;*position:relative;z-index:0;vertical-align:top}#lg img{vertical-align:top;margin-bottom:3px}#lk{margin:33px 0}#lk span{font:14px "����"}#lm{height:60px}#lh{margin:16px 0 5px;word-spacing:3px}.tools{position:absolute;top:-4px;*top:10px;right:-13px;}#mHolder{width:62px;position:relative;z-index:296;display:none}#mCon{height:18px;line-height:18px;position:absolute;cursor:pointer;padding:0 18px 0 0;background:url(http://www.baidu.com/img/bg-1.0.0.gif) no-repeat right -134px;background-position:right -136px\9}#mCon span{color:#00c;cursor:default;display:block}#mCon .hw{text-decoration:underline;cursor:pointer}#mMenu a{width:100%;height:100%;display:block;line-height:22px;text-indent:6px;text-decoration:none;filter:none\9}#mMenu,#user ul{box-shadow:1px 1px 2px #ccc;-moz-box-shadow:1px 1px 2px #ccc;-webkit-box-shadow:1px 1px 2px #ccc;filter: progid:DXImageTransform.Microsoft.Shadow(Strength=2, Direction=135, Color="#cccccc")\9;}#mMenu{width:56px;border:1px solid #9b9b9b;list-style:none;position:absolute;right:7px;top:28px;display:none;background:#fff}#mMenu a:hover{background:#ebebeb}#mMenu .ln{height:1px;background:#ebebeb;overflow:hidden;font-size:1px;line-height:1px;margin-top:-1px}#cp,#cp a{color:#77c}#seth{display:none;behavior:url(#default#homepage)}#setf{display:none}</style>
+<script type="text/javascript">function h(obj){obj.style.behavior='url(#default#homepage)';var a = obj.setHomePage('http://www.baidu.com/');}</script></head>
+
+<body><div id="u"><a href="http://www.baidu.com/gaoji/preferences.html" name="tj_setting">��������</a>|<a href="http://passport.baidu.com/?login&tpl=mn" name="tj_login">��¼</a><a href="https://passport.baidu.com/?reg&tpl=mn" name="tj_reg" class="reg">ע��</a></div>
+<div id="m"><p id="lg"><img src="http://www.baidu.com/img/baidu_sylogo1.gif" width="270" height="129" usemap="#mp"><map name="mp"><area shape="rect" coords="40,25,230,95" href="http://hi.baidu.com/baidu/" target="_blank" title="��˽��� �ٶȵĿռ�" ></map></p><p id="nv"><a href="http://news.baidu.com">��&nbsp;��</a>��<b>��&nbsp;ҳ</b>��<a href="http://tieba.baidu.com">��&nbsp;��</a>��<a href="http://zhidao.baidu.com">֪&nbsp;��</a>��<a href="http://mp3.baidu.com">MP3</a>��<a href="http://image.baidu.com">ͼ&nbsp;Ƭ</a>��<a href="http://video.baidu.com">��&nbsp;Ƶ</a>��<a href="http://map.baidu.com">��&nbsp;ͼ</a></p><div id="fm"><form name="f" action="/s"><span class="s_ipt_wr"><input type="text" name="wd" id="kw" maxlength="100" class="s_ipt"></span><input type="hidden" name="rsv_bp" value="0"><input type="hidden" name="rsv_spt" value="3"><span class="s_btn_wr"><input type="submit" value="�ٶ�һ��" id="su" class="s_btn" onmousedown="this.className='s_btn s_btn_h'" onmouseout="this.className='s_btn'"></span></form><span class="tools"><span id="mHolder"><div id="mCon"><span>���뷨</span></div></span></span><ul id="mMenu"><li><a href="#" name="ime_hw">��д</a></li><li><a href="#" name="ime_py">ƴ��</a></li><li class="ln"></li><li><a href="#" name="ime_cl">�ر�</a></li></ul></div>
+<p id="lk"><a href="http://hi.baidu.com">�ռ�</a>��<a href="http://baike.baidu.com">�ٿ�</a>��<a href="http://www.hao123.com">hao123</a><span> | <a href="/more/">����&gt;&gt;</a></span></p><p id="lm"></p><p><a id="seth" onClick="h(this)" href="http://utility.baidu.com/traf/click.php?id=215&url=http://www.baidu.com" onmousedown="return ns_c({'fm':'behs','tab':'homepage','pos':0})">�Ѱٶ���Ϊ��ҳ</a><a id="setf" onClick="fa(this)" href="javascript:void(0)" onmousedown="return ns_c({'fm':'behs','tab':'favorites','pos':0})">�Ѱٶȼ����ղؼ�</a></p>
+<p id="lh"><a href="http://e.baidu.com/?refer=888">����ٶ��ƹ�</a> | <a href="http://top.baidu.com">�������ư�</a> | <a href="http://home.baidu.com">���ڰٶ�</a> | <a href="http://ir.baidu.com">About Baidu</a></p><p id="cp">&copy;2012 Baidu <a href="/duty/">ʹ�ðٶ�ǰ�ض�</a> <a href="http://www.miibeian.gov.cn" target="_blank">��ICP֤030173��</a> <img src="http://gimg.baidu.com/img/gs.gif"></p></div></body>
+
+<script>var bdUser = null;var w=window,d=document,n=navigator,k=d.f.wd,a=d.getElementById("nv").getElementsByTagName("a"),isIE=n.userAgent.indexOf("MSIE")!=-1&&!window.opera;for(var i=0;i<a.length;i++){a[i].onclick=function(){if(k.value.length>0){var o=this,h=o.href,q=encodeURIComponent(k.value);if(h.indexOf("q=")!=-1){o.href=h.replace(/q=[^&\x24]*/,"q="+q)}else{this.href+="?q="+q}}}};(function(){if(/q=([^&]+)/.test(location.search)){k.value=decodeURIComponent(RegExp["\x241"])}})();if(n.cookieEnabled&&!/sug?=0/.test(d.cookie)){d.write('<script src=http://www.baidu.com/cache/sug/js/bdsug-1.1.js><\/script>')};function addEV(o, e, f){if(w.attachEvent){o.attachEvent("on" + e, f);}else if(w.addEventListener){ o.addEventListener(e, f, false);}}function G(id){return d.getElementById(id);}function ns_c(q){var p = encodeURIComponent(window.document.location.href), sQ = '', sV = '', mu='', img = window["BD_PS_C" + (new Date()).getTime()] = new Image();for (v in q) {sV = q[v];sQ += v + "=" + sV + "&";} mu= "&mu=" + p ;img.src = "http://nsclick.baidu.com/v.gif?pid=201&pj=www&" + sQ + "path="+p+"&t="+new Date().getTime();return true;}if(/\bbdime=[12]/.test(d.cookie)){document.write('<script src=http://www.baidu.com/cache/ime/js/openime-1.0.0.js><\/script>');}(function(){var user = G("user"), userMenu = G("userMenu");if(user && userMenu){user.onmouseover = function(){userMenu.style.display = "block"};user.onmouseout = function(){userMenu.style.display = "none"};}})();(function(){var u = G("u").getElementsByTagName("a"), nv = G("nv").getElementsByTagName("a"), lk = G("lk").getElementsByTagName("a"), un = "";var tj_nv = ["news","tieba","zhidao","mp3","img","video","map"];var tj_lk = ["hi","baike","hao123","more"];un = bdUser == null ? "" : bdUser;function _addTJ(obj){addEV(obj, "mousedown", function(e){var e = e || window.event;var target = e.target || e.srcElement;ns_c({'fm':'behs','tab':target.name||'tj_user','un':encodeURIComponent(un)});});}for(var i = 0; i < u.length; i++){_addTJ(u[i]);}for(var i = 0; i < nv.length; i++){nv[i].name = 'tj_' + tj_nv[i];_addTJ(nv[i]);}for(var i = 0; i < lk.length; i++){lk[i].name = 'tj_' + tj_lk[i];_addTJ(lk[i]);}})();addEV(w,"load",function(){k.focus()});w.onunload=function(){};</script>
+
+
+
+</html>
+
+<script type="text/javascript" src="http://www.baidu.com/cache/hps/js/hps-1.4.js"></script>
+
+<!--2d1a628f7e6b9aeb-->