Added charset support, centralized UTF-8 usage (#305)

* Added charset support, centralized UTF-8 usage

* Code style, buffer methods with charsets

* assure remote charset isn't null

Author: ISQ-GTT

src/main/java/com/hierynomus/sshj/backport/Jdk7HttpProxySocket.java

@@ -18,7 +18,8 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.*;
-import java.nio.charset.Charset;
+
+import net.schmizz.sshj.common.IOUtils;
 
 public class Jdk7HttpProxySocket extends Socket {
 
@@ -48,7 +49,7 @@ private void connectHttpProxy(SocketAddress endpoint, int timeout) throws IOExce
         }
         InetSocketAddress isa = (InetSocketAddress) endpoint;
         String httpConnect = "CONNECT " + isa.getHostName() + ":" + isa.getPort() + " HTTP/1.0\n\n";
-        getOutputStream().write(httpConnect.getBytes(Charset.forName("UTF-8")));
+        getOutputStream().write(httpConnect.getBytes(IOUtils.UTF8));
         checkAndFlushProxyResponse();
     }
 
@@ -61,7 +62,7 @@ private void checkAndFlushProxyResponse()throws IOException {
             throw new SocketException("Empty response from proxy");
         }
 
-        String proxyResponse = new String(tmpBuffer, 0, len, "UTF-8");
+        String proxyResponse = new String(tmpBuffer, 0, len, IOUtils.UTF8);
 
         // Expecting HTTP/1.x 200 OK
         if (proxyResponse.contains("200")) {

src/main/java/net/schmizz/sshj/SSHClient.java

@@ -16,6 +16,7 @@
 package net.schmizz.sshj;
 
 import net.schmizz.sshj.common.Factory;
+import net.schmizz.sshj.common.IOUtils;
 import net.schmizz.sshj.common.LoggerFactory;
 import net.schmizz.sshj.common.SSHException;
 import net.schmizz.sshj.common.SecurityUtils;
@@ -61,6 +62,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.net.ServerSocket;
+import java.nio.charset.Charset;
 import java.security.KeyPair;
 import java.security.PublicKey;
 import java.util.*;
@@ -128,6 +130,9 @@ public class SSHClient
 
     private final List<LocalPortForwarder> forwarders = new ArrayList<LocalPortForwarder>();
 
+    /** character set of the remote machine */
+    protected Charset remoteCharset = IOUtils.UTF8;
+
     /** Default constructor. Initializes this object using {@link DefaultConfig}. */
     public SSHClient() {
         this(new DefaultConfig());
@@ -440,6 +445,15 @@ public Connection getConnection() {
         return conn;
     }
 
+    /**
+     * Returns the character set used to communicate with the remote machine for certain strings (like paths).
+     *
+     * @return remote character set
+     */
+    public Charset getRemoteCharset() {
+        return remoteCharset;
+    }
+
     /** @return a {@link RemotePortForwarder} that allows requesting remote forwarding over this connection. */
     public RemotePortForwarder getRemotePortForwarder() {
         synchronized (conn) {
@@ -708,12 +722,22 @@ public void rekey()
         doKex();
     }
 
+    /**
+     * Sets the character set used to communicate with the remote machine for certain strings (like paths)
+     *
+     * @param remoteCharset
+     *        remote character set or {@code null} for default
+     */
+    public void setRemoteCharset(Charset remoteCharset) {
+        this.remoteCharset = remoteCharset != null ? remoteCharset : IOUtils.UTF8;
+    }
+
     @Override
     public Session startSession()
             throws ConnectionException, TransportException {
         checkConnected();
         checkAuthenticated();
-        final SessionChannel sess = new SessionChannel(conn);
+        final SessionChannel sess = new SessionChannel(conn, remoteCharset);
         sess.open();
         return sess;
     }

src/main/java/net/schmizz/sshj/common/Buffer.java

@@ -15,8 +15,8 @@
  */
 package net.schmizz.sshj.common;
 
-import java.io.UnsupportedEncodingException;
 import java.math.BigInteger;
+import java.nio.charset.Charset;
 import java.security.GeneralSecurityException;
 import java.security.PublicKey;
 import java.util.Arrays;
@@ -361,24 +361,31 @@ public T putUInt64(long uint64) {
     /**
      * Reads an SSH string
      *
+     * @param cs the charset to use for decoding
+     *
      * @return the string as a Java {@code String}
      */
-    public String readString()
+    public String readString(Charset cs)
             throws BufferException {
         int len = readUInt32AsInt();
         if (len < 0 || len > 32768)
             throw new BufferException("Bad item length: " + len);
         ensureAvailable(len);
-        String s;
-        try {
-            s = new String(data, rpos, len, "UTF-8");
-        } catch (UnsupportedEncodingException e) {
-            throw new SSHRuntimeException(e);
-        }
+        String s = new String(data, rpos, len, cs);
         rpos += len;
         return s;
     }
 
+    /**
+     * Reads an SSH string using {@code UTF8}
+     *
+     * @return the string as a Java {@code String}
+     */
+    public String readString()
+            throws BufferException {
+        return readString(IOUtils.UTF8);
+    }
+
     /**
      * Reads an SSH string
      *
@@ -397,8 +404,12 @@ public T putString(byte[] str, int offset, int len) {
         return putBytes(str, offset, len);
     }
 
+    public T putString(String string, Charset cs) {
+        return putString(string.getBytes(cs));
+    }
+
     public T putString(String string) {
-        return putString(string.getBytes(IOUtils.UTF8));
+        return putString(string, IOUtils.UTF8);
     }
 
     /**

src/main/java/net/schmizz/sshj/connection/channel/AbstractChannel.java

@@ -26,6 +26,7 @@
 
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.nio.charset.Charset;
 import java.util.LinkedList;
 import java.util.Queue;
 import java.util.concurrent.TimeUnit;
@@ -51,6 +52,8 @@ public abstract class AbstractChannel
     private final int id;
     /** Remote recipient ID */
     private int recipient;
+    /** Remote character set */
+    private final Charset remoteCharset;
 
     private boolean eof = false;
 
@@ -78,12 +81,16 @@ public abstract class AbstractChannel
     private volatile boolean autoExpand = false;
 
     protected AbstractChannel(Connection conn, String type) {
+        this(conn, type, null);
+    }
+    protected AbstractChannel(Connection conn, String type, Charset remoteCharset) {
         this.conn = conn;
         this.loggerFactory = conn.getTransport().getConfig().getLoggerFactory();
         this.type = type;
         this.log = loggerFactory.getLogger(getClass());
         this.trans = conn.getTransport();
 
+        this.remoteCharset = remoteCharset != null ? remoteCharset : IOUtils.UTF8;
         id = conn.nextID();
 
         lwin = new Window.Local(conn.getWindowSize(), conn.getMaxPacketSize(), loggerFactory);
@@ -135,6 +142,11 @@ public int getRecipient() {
         return recipient;
     }
 
+    @Override
+    public Charset getRemoteCharset() {
+        return remoteCharset;
+    }
+
     @Override
     public int getRemoteMaxPacketSize() {
         return rwin.getMaxPacketSize();

src/main/java/net/schmizz/sshj/connection/channel/Channel.java

@@ -24,6 +24,7 @@
 import java.io.Closeable;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.nio.charset.Charset;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -123,6 +124,9 @@ interface Forwarded extends Channel {
      */
     int getRecipient();
 
+    /** @return the character set used to communicate with the remote machine for certain strings (like paths). */
+    Charset getRemoteCharset();
+
     /**
      * @return the maximum packet size as specified by the remote end.
      */

src/main/java/net/schmizz/sshj/connection/channel/direct/AbstractDirectChannel.java

@@ -25,6 +25,7 @@
 import net.schmizz.sshj.connection.channel.OpenFailException;
 import net.schmizz.sshj.transport.TransportException;
 
+import java.nio.charset.Charset;
 import java.util.concurrent.TimeUnit;
 
 /** Base class for direct channels whose open is initated by the client. */
@@ -41,6 +42,15 @@ protected AbstractDirectChannel(Connection conn, String type) {
         conn.attach(this);
     }
 
+    protected AbstractDirectChannel(Connection conn, String type, Charset remoteCharset) {
+        super(conn, type, remoteCharset);
+
+        /*
+         * We expect to receive channel open confirmation/rejection and want to be able to next this packet.
+         */
+        conn.attach(this);
+    }
+
     @Override
     public void open()
             throws ConnectionException, TransportException {

src/main/java/net/schmizz/sshj/connection/channel/direct/SessionChannel.java

@@ -22,6 +22,7 @@
 import net.schmizz.sshj.transport.TransportException;
 
 import java.io.InputStream;
+import java.nio.charset.Charset;
 import java.util.Collections;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
@@ -47,6 +48,10 @@ public SessionChannel(Connection conn) {
         super(conn, "session");
     }
 
+    public SessionChannel(Connection conn, Charset remoteCharset) {
+        super(conn, "session", remoteCharset);
+    }
+
     @Override
     public void allocateDefaultPTY()
             throws ConnectionException, TransportException {
@@ -93,7 +98,7 @@ public Command exec(String command)
             throws ConnectionException, TransportException {
         checkReuse();
         log.debug("Will request to exec `{}`", command);
-        sendChannelRequest("exec", true, new Buffer.PlainBuffer().putString(command))
+        sendChannelRequest("exec", true, new Buffer.PlainBuffer().putString(command, getRemoteCharset()))
                 .await(conn.getTimeoutMs(), TimeUnit.MILLISECONDS);
         usedUp = true;
         return this;

src/main/java/net/schmizz/sshj/sftp/RemoteDirectory.java

@@ -42,7 +42,7 @@ public List<RemoteResourceInfo> scan(RemoteResourceFilter filter)
                 case NAME:
                     final int count = res.readUInt32AsInt();
                     for (int i = 0; i < count; i++) {
-                        final String name = res.readString();
+                        final String name = res.readString(requester.sub.getRemoteCharset());
                         res.readString(); // long name - IGNORED - shdve never been in the protocol
                         final FileAttributes attrs = res.readFileAttributes();
                         final PathComponents comps = requester.getPathHelper().getComponents(path, name);