utf-8 aware truncate method, codings removed

Rakefile

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 task :default => %w(test:spec)
 
 def shrink_js(t)
@@ -70,7 +69,7 @@ end
 desc 'Generate documentation'
 task :doc => 'doc/api/index.html'
 file 'doc/api/index.html' => FileList['**/*.rb'] do |f|
-  sh "rdoc -o doc/api --title 'Ōlelo Source Documentation' --inline-source --format=html #{f.prerequisites.join(' ')}"
+  sh "rdoc -o doc/api --title 'Olelo Source Documentation' --inline-source --format=html #{f.prerequisites.join(' ')}"
 end
 
 namespace :notes do

config.ru

@@ -1,5 +1,4 @@
 #!/usr/bin/env rackup
-# -*- coding: utf-8 -*-
 
 path = ::File.expand_path(::File.dirname(__FILE__))
 $: << ::File.join(path, 'lib')
@@ -70,6 +69,12 @@ end
 
 use Rack::MethodOverride
 use Rack::CommonLogger, LoggerOutput.new(logger)
+
+if ''.respond_to? :encoding
+  require 'rack/force_encoding'
+  use Rack::ForceEncoding
+end
+
 run Olelo::Application.new(nil, :logger => logger)
 
 logger.info "Olelo started in #{timer.stop.elapsed_ms}ms (#{Olelo::Config.production? ? 'Production' : 'Development'} mode)"

lib/olelo.rb

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 require 'logger'
 require 'cgi'
 require 'digest/md5'

lib/olelo/application.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 module Olelo
   # Main class of the application
   class Application

lib/olelo/attributeeditor.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 module Olelo
   module AttributeEditor
     def self.included(base)

lib/olelo/config.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 module Olelo
   class Config
     include Enumerable

lib/olelo/extensions.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 class Module
   # Generate accessor method with question mark
   def attr_reader?(*attrs)
@@ -133,6 +132,21 @@ def try(name, *args)
 end
 
 class String
+  # Check if string is valid text encoding
+  if ''.respond_to?(:encoding)
+    def valid_text_encoding?
+      Encoding.compatible?('', self)
+    end
+  else
+    require 'iconv'
+    def valid_text_encoding?
+      Iconv.conv('utf-8', 'utf-8', self)
+      true
+    rescue
+      false
+    end
+  end
+
   # Unindent string
   def unindent
     result = ''
@@ -171,11 +185,6 @@ def cleanpath
     names.join('/')
   end
 
-  # Truncate string and add omission
-  def truncate(max, omission = '...')
-    (length > max ? self[0...max] + omission : self)
-  end
-
   # Concatenate path components
   def /(name)
     "#{self}/#{name}".cleanpath

lib/olelo/fs.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 module Olelo
   class DirectoryFS
     def initialize(dir)

lib/olelo/helper.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 module Olelo
   module BlockHelper
     def blocks
@@ -46,8 +45,8 @@ def pagination(path, last_page, page_nr, opts = {})
       if last_page > 0
         li = []
         if page_nr > 0
-          li << %{<a href="#{escape_html absolute_path(path, opts.merge(:page => 0))}">«</a>}
-          li << %{<a href="#{escape_html absolute_path(path, opts.merge(:page => page_nr - 1))}">‹</a>}
+          li << %{<a href="#{escape_html absolute_path(path, opts.merge(:page => 0))}">&#171;</a>}
+          li << %{<a href="#{escape_html absolute_path(path, opts.merge(:page => page_nr - 1))}">&#8249;</a>}
         end
         min = page_nr - 3
         max = page_nr + 3
@@ -58,18 +57,18 @@ def pagination(path, last_page, page_nr, opts = {})
         end
         max = [max, last_page].min
         min = [min, 0].max
-        li << '…' if min != 0
+        li << '&#8230;' if min != 0
         (min..max).each do |i|
           if i == page_nr
             li << %{<a class="current" href="#">#{i + 1}</a>}
           else
             li << %{<a href="#{escape_html absolute_path(path, opts.merge(:page => i))}">#{i + 1}</a>}
           end
         end
-        li << '…' if max != last_page
+        li << '&#8230;' if max != last_page
         if page_nr < last_page
-          li << %{<a href="#{escape_html absolute_path(path, opts.merge(:page => page_nr + 1))}">›</a>}
-          li << %{<a href="#{escape_html absolute_path(path, opts.merge(:page => last_page))}">»</a>}
+          li << %{<a href="#{escape_html absolute_path(path, opts.merge(:page => page_nr + 1))}">&#8250;</a>}
+          li << %{<a href="#{escape_html absolute_path(path, opts.merge(:page => last_page))}">&#187;</a>}
         end
         '<ul class="pagination">' + li.map {|x| "<li>#{x}</li>"}.join + '</ul>'
       end
@@ -121,18 +120,13 @@ def action_path(path, action)
     end
 
     def edit_content(page)
-      return params[:content] if params[:content]
-      if page.content.respond_to?(:encoding)
-        return :error_binary.t(:page => page.title, :type => "#{page.mime.comment} (#{page.mime})") if page.content.encoding != __ENCODING__
+      if params[:content]
+        params[:content]
+      elsif !page.content.valid_text_encoding?
+        :error_binary.t(:page => page.title, :type => "#{page.mime.comment} (#{page.mime})")
       else
-        begin
-          require 'iconv'
-          Iconv.conv('utf-8', 'utf-8', page.content)
-        rescue
-          return :error_binary.t(:page => page.title, :type => "#{page.mime.comment} (#{page.mime})")
-        end
+        params[:pos] ? page.content[params[:pos].to_i, params[:len].to_i].to_s : page.content
       end
-      params[:pos] ? page.content[params[:pos].to_i, params[:len].to_i].to_s : page.content
     end
   end
 

lib/olelo/hooks.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 module Olelo
   module Hooks
     def self.included(base)

lib/olelo/i18n.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 module Olelo
   module I18n
     @locale = nil

lib/olelo/page.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 module Olelo
   class Version
     attr_reader :id, :author, :date, :comment, :parents

lib/olelo/patch.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 module Olelo
   class PatchParser
     include Util

lib/olelo/plugin.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 module Olelo
   # Olelo plugin system
   class Plugin

lib/olelo/routing.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 module Olelo
   module Routing
     def self.included(base)
@@ -18,17 +17,12 @@ def call!(env)
       @response = Rack::Response.new
       @params = @original_params = @request.params.with_indifferent_access
 
-      # Interpret everything as utf-8
-      if ''.respond_to? :encoding
-        encode(@env)
-        encode(@params)
-      end
-
       catch(:forward) do
         with_hooks(:request) { perform! }
         status, header, body = response.finish
         return [status, header, request.head? ? [] : body]
       end
+
       @app ? @app.call(env) : error!(NotFound.new(@request.path_info))
     end
 
@@ -50,25 +44,6 @@ def forward
 
     private
 
-    # Interpret everything as utf-8
-    def encode(x)
-      case x
-      when Hash
-        x.each { |k,v| x[k] = encode(v) }
-      when Array
-        x.each_with_index {|v,i| x[i] = encode(v) }
-      when String
-        if x.encoding != __ENCODING__
-          x = x.dup if x.frozen?
-          x.force_encoding(__ENCODING__)
-        else
-          x
-        end
-      else
-        x
-      end
-    end
-
     def error!(ex)
       response.status = Rack::Utils.status_code(ex.try(:status) || :internal_server_error)
       response.body   = [ex.message]

lib/olelo/templates.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 module Olelo
   module Templates
     HAML_OPTIONS = { :format => :xhtml, :attr_wrapper  => '"', :ugly => true, :escape_html => true }

lib/olelo/timer.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 module Olelo
   class Timer
     def initialize

lib/olelo/user.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 module Olelo
   class AuthenticationError < RuntimeError
   end

lib/olelo/util.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 module Olelo
   class NotFound < NameError
     def initialize(id)
@@ -64,32 +63,39 @@ def check
 
     # Like CGI.escape but escapes space not as +
     def escape(s)
+      s = s.to_s
       s.gsub(/([^a-zA-Z0-9_.-]+)/) do
         '%' + $1.unpack('H2' * $1.bytesize).join('%').upcase
       end
     end
 
     # Like CGI.unescape but does not unescape +
-    def unescape(s)
-      if s.respond_to? :encoding
+    if ''.respond_to? :encoding
+      def unescape(s)
+        s = s.to_s
         enc = s.encoding
         s.gsub(/((?:%[0-9a-fA-F]{2})+)/) do
           [$1.delete('%')].pack('H*').force_encoding(enc)
         end
-      else
-        s.gsub(/((?:%[0-9a-fA-F]{2})+)/) do
+      end
+    else
+      def unescape(s)
+        s.to_s.gsub(/((?:%[0-9a-fA-F]{2})+)/) do
           [$1.delete('%')].pack('H*')
         end
       end
     end
 
-    def unescape_backslash(s)
-      if s.respond_to? :encoding
+    if ''.respond_to? :encoding
+      def unescape_backslash(s)
+        s = s.to_s
         enc = s.encoding
         s.gsub(/\\([0-7]{3})/) { $1.to_i(8).chr.force_encoding(enc) }.
           gsub(/\\x([\da-f]{2})/i) { $1.to_i(16).chr.force_encoding(enc) }
-      else
-        s.gsub(/\\([0-7]{3})/) { $1.to_i(8).chr }.
+      end
+    else
+      def unescape_backslash(s)
+        s.to_s.gsub(/\\([0-7]{3})/) { $1.to_i(8).chr }.
           gsub(/\\x([\da-f]{2})/i) { $1.to_i(16).chr }
       end
     end
@@ -125,5 +131,24 @@ def build_query(params)
         end
       end.join('&')
     end
+
+    # Truncate string and add omission
+    if ''.respond_to?(:encoding)
+      def truncate(s, max, omission = '...')
+        s = s.to_s
+        (s.length > max ? s[0...max] + omission : s)
+      end
+    else
+      def truncate(s, max, omission = '...')
+        s = s.to_s
+        if s.length > max
+          max += 1 until max >= s.length || s[0...max].valid_text_encoding?
+          s[0...max] + omission
+        else
+          s
+        end
+      end
+    end
+
   end
 end

lib/rack/degrade_mime_type.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 module Rack
   class DegradeMimeType
     def initialize(app)

lib/rack/force_encoding.rb

@@ -0,0 +1,39 @@
+module Rack
+  class ForceEncoding
+    def initialize(app, encoding = Encoding::UTF_8)
+      @app = app
+      @encoding = encoding
+    end
+
+    def call(env)
+      request = Rack::Request.new(env)
+      encode(env)
+      encode(request.params)
+      @app.call(env)
+    end
+
+    private
+
+    def encode(x)
+      case x
+      when Hash
+        x.each { |k,v| x[k] = encode(v) }
+      when Array
+        x.each_with_index {|v,i| x[i] = encode(v) }
+      when String
+        # Try to force encoding and revert to old encoding if this doesn't work
+        encoding = x.encoding
+        if encoding != @encoding
+          x = x.dup if x.frozen?
+          x.force_encoding(@encoding)
+          x.force_encoding(encoding) if !x.valid_encoding?
+          x
+        else
+          x
+        end
+      else
+        x
+      end
+    end
+  end
+end

lib/rack/patches.rb

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 require 'rack'
 require 'rack/session/abstract/id'
 require 'socket'

plugins/editor/enhanced_edit.rb

@@ -47,7 +47,7 @@ class Olelo::Application
 
         # Read in binary mode and fix encoding afterwards
         patch = IO.popen("diff -u '#{original.path}' '#{new.path}'", 'rb') {|io| io.read }
-        patch.force_encoding(__ENCODING__) if patch.respond_to? :force_encoding
+        patch.force_encoding(Encoding::UTF_8) if patch.respond_to? :force_encoding
         @patch = PatchParser.parse(patch, PatchFormatter.new).html
 
 	halt render(:edit)

plugins/engine/tree.rb

@@ -36,8 +36,8 @@ def output(context)
         %td.link
           %a{:href=>page_path(child), :class => classes}= child.name
         %td!= date(child.version.date)
-        %td= child.version.author.name.truncate(30)
-        %td= child.version.comment.truncate(30)
+        %td= truncate(child.version.author.name, 30)
+        %td= truncate(child.version.comment, 30)
         %td.actions
           %a.action-edit{:href=>action_path(child, :edit), :title => :edit.t}= :edit.t
           %a.action-history{:href=>action_path(child, :history), :title => :history.t}= :history.t