diff --git a/Lib/readline.py b/Lib/readline.py index 92eb87bab..2296a5404 100644 --- a/Lib/readline.py +++ b/Lib/readline.py @@ -14,10 +14,11 @@ 'set_history_length', 'set_pre_input_hook', 'set_startup_hook', 'write_history_file'] -try: - _reader = sys._jy_interpreter.reader +try: + _console = sys._jy_console + _reader = _console.reader except AttributeError: - raise ImportError("Cannot access JLineConsole") + raise ImportError("Cannot access JLineConsole reader") _history_list = None @@ -38,7 +39,7 @@ def _setup_history(): # modify the history (ipython uses the function # remove_history_item to mutate the history relatively frequently) global _history_list - + history = _reader.history try: history_list_field = history.class.getDeclaredField("history") @@ -68,7 +69,7 @@ def get_line_buffer(): def insert_text(string): _reader.putString(string) - + def read_init_file(filename=None): warn("read_init_file: %s" % (filename,), NotImplementedWarning, "module", 2) @@ -128,8 +129,8 @@ def redisplay(): _reader.redrawLine() def set_startup_hook(function=None): - sys._jy_interpreter.startupHook = function - + _console.startupHook = function + def set_pre_input_hook(function=None): warn("set_pre_input_hook %s" % (function,), NotImplementedWarning, stacklevel=2) @@ -161,7 +162,7 @@ def complete_handler(buffer, cursor, candidates): return start _reader.addCompletor(complete_handler) - + def get_completer(): return _completer_function diff --git a/build.xml b/build.xml index 9c5a9205a..701109ac3 100644 --- a/build.xml +++ b/build.xml @@ -936,6 +936,7 @@ The readme text for the next release will be like: + diff --git a/registry b/registry index 2109253eb..9c92643c3 100644 --- a/registry +++ b/registry @@ -31,15 +31,16 @@ python.packages.directories = java.ext.dirs #python.verbose = message # Jython ships with a JLine console (http://jline.sourceforge.net/) -# out of the box. Setting this to the name of a different console class, -# new console features can be enabled. Readline support is such an -# example: +# out of the box. +python.console=org.python.util.JLineConsole +# To activate explicitly the featureless Jython console, choose: +#python.console=org.python.core.PlainConsole +# By setting this to the name of a different console class, +# new console features can be enabled. For example: #python.console=org.python.util.ReadlineConsole -#python.console.readlinelib=JavaReadline -# To activate the legacy Jython console: -#python.console=org.python.util.InteractiveConsole +#python.console.readlinelib=GnuReadline -# Setting this to a valid codec name will cause the console to use a +# Setting this to a valid (Java) codec name will cause the console to use a # different encoding when reading commands from the console. #python.console.encoding = cp850 diff --git a/src/org/python/core/Console.java b/src/org/python/core/Console.java new file mode 100644 index 000000000..c8172d915 --- /dev/null +++ b/src/org/python/core/Console.java @@ -0,0 +1,60 @@ +// Copyright (c) 2013 Jython Developers +package org.python.core; + +import java.io.EOFException; +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * A class named in configuration as the value of python.console must implement this + * interface, and provide a constructor with a single String argument, to be acceptable + * during initialization of the interpreter. The argument to the constructor names the encoding in + * use on the console. Such a class may provide line editing and history recall to an interactive + * console. A default implementation (that does not provide any such facilities) is available as + * {@link PlainConsole}. + */ +public interface Console { + + /** + * Complete initialization and (optionally) install a stream object with line-editing as the + * replacement for System.in. + * + * @throws IOException in case of failure related to i/o + */ + public void install() throws IOException; + + /** + * Uninstall the Console (if possible). A Console that installs a replacement for + * System.in should put back the original value. + * + * @throws UnsupportedOperationException if the Console cannot be uninstalled + */ + public void uninstall() throws UnsupportedOperationException; + + /** + * Write a prompt and read a line from standard input. The returned line does not include the + * trailing newline. When the user enters the EOF key sequence, an EOFException should be + * raised. The built-in function raw_input calls this method on the installed + * console. + * + * @param prompt to output before reading a line + * @return the line read in (encoded as bytes) + * @throws IOException in case of failure related to i/o + * @throws EOFException when the user enters the EOF key sequence + */ + public ByteBuffer raw_input(CharSequence prompt) throws IOException, EOFException; + + /** + * Write a prompt and read a line from standard input. The returned line does not include the + * trailing newline. When the user enters the EOF key sequence, an EOFException should be + * raised. The Py3k built-in function input calls this method on the installed + * console. + * + * @param prompt to output before reading a line + * @return the line read in + * @throws IOException in case of failure related to i/o + * @throws EOFException when the user enters the EOF key sequence + */ + public CharSequence input(CharSequence prompt) throws IOException, EOFException; + +} diff --git a/src/org/python/core/PlainConsole.java b/src/org/python/core/PlainConsole.java new file mode 100644 index 000000000..09fb458ba --- /dev/null +++ b/src/org/python/core/PlainConsole.java @@ -0,0 +1,106 @@ +// Copyright (c) 2013 Jython Developers +package org.python.core; + +import java.io.BufferedReader; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.IllegalCharsetNameException; +import java.nio.charset.UnsupportedCharsetException; + +/** + * A base class for classes that can install a console wrapper for a specific console-handling + * library. The Jython command-line application, when it detects that the console is an interactive + * session, chooses and installs a class named in registry item python.console, and + * implementing interface {@link Console}. PlainConsole may be selected by the user + * through that registry, and is the default console when the selected one fails to load. It will + * also be installed by the Jython command-line application when in non-interactive mode. + *

+ * Unlike some consoles, PlainConsole does not install a replacement for + * System.in or use a native library. It prompts on System.out and reads + * from System.in (wrapped with the console encoding). + */ +public class PlainConsole implements Console { + + /** Encoding to use for line input. */ + public final String encoding; + + /** Encoding to use for line input as a Charset. */ + public final Charset encodingCharset; + + /** BufferedReader used by {@link #input(CharSequence)} */ + private BufferedReader reader; + + /** + * Construct an instance of the console class specifying the character encoding. This encoding + * must be one supported by the JVM. The PlainConsole does not replace System.in, + * and does not add any line-editing capability to what is standard for your OS console. + * + * @param encoding name of a supported encoding or null for + * Charset.defaultCharset() + */ + public PlainConsole(String encoding) throws IllegalCharsetNameException, + UnsupportedCharsetException { + if (encoding == null) { + encoding = Charset.defaultCharset().name(); + } + this.encoding = encoding; + encodingCharset = Charset.forName(encoding); + } + + @Override + public void install() { + // Create a Reader with the right character encoding + reader = new BufferedReader(new InputStreamReader(System.in, encodingCharset)); + } + + /** + * A PlainConsole may be uninstalled. This method assumes any sub-class may not be + * uninstalled. Sub-classes that permit themselves to be uninstalled must override (and + * not call) this method. + * + * @throws UnsupportedOperationException unless this class is exactly PlainConsole + */ + @Override + public void uninstall() throws UnsupportedOperationException { + Class myClass = this.getClass(); + if (myClass != PlainConsole.class) { + throw new UnsupportedOperationException(myClass.getSimpleName() + + " console may not be uninstalled."); + } + } + + /** + * {@inheritDoc} + *

+ * The base implementation calls {@link #input(CharSequence)} and applies the console encoding + * to obtain the bytes. This may be a surprise. Line-editing consoles necessarily operate in + * terms of characters rather than bytes, and therefore support a direct implementation of + * input. + */ + @Override + public ByteBuffer raw_input(CharSequence prompt) throws IOException, EOFException { + CharSequence line = input(prompt); + return encodingCharset.encode(CharBuffer.wrap(line)); + } + + // The base implementation simply uses System.out and System.in. + @Override + public CharSequence input(CharSequence prompt) throws IOException, EOFException { + + // Issue the prompt with no newline + System.out.print(prompt); + + // Get the line from the console via java.io + String line = reader.readLine(); + if (line == null) { + throw new EOFException(); + } else { + return line; + } + } + +} diff --git a/src/org/python/core/Py.java b/src/org/python/core/Py.java index 99e691ed8..d4682a429 100644 --- a/src/org/python/core/Py.java +++ b/src/org/python/core/Py.java @@ -3,6 +3,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileDescriptor; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -17,14 +18,17 @@ import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; +import java.util.ArrayList; import java.util.Calendar; +import java.util.List; import java.util.Set; -import org.python.antlr.base.mod; import jnr.constants.Constant; import jnr.constants.platform.Errno; -import java.util.ArrayList; -import java.util.List; +import jnr.posix.POSIX; +import jnr.posix.POSIXFactory; + +import org.python.antlr.base.mod; import org.python.core.adapter.ClassicPyObjectAdapter; import org.python.core.adapter.ExtensiblePyObjectAdapter; import org.python.modules.posix.PosixModule; @@ -1386,6 +1390,73 @@ public static void setFrame(PyFrame f) { getThreadState().frame = f; } + /** + * The handler for interactive consoles, set by {@link #installConsole(Console)} and accessed by + * {@link #getConsole()}. + */ + private static Console console; + + /** + * Get the Jython Console (used for input(), raw_input(), etc.) as + * constructed and set by {@link PySystemState} initialization. + * + * @return the Jython Console + */ + public static Console getConsole() { + if (console == null) { + // We really shouldn't ask for a console before PySystemState initialization but ... + try { + // ... something foolproof that we can supersede. + installConsole(new PlainConsole("ascii")); + } catch (Exception e) { + // This really, really shouldn't happen + throw Py.RuntimeError("Could not create fall-back PlainConsole: " + e); + } + } + return console; + } + + /** + * Install the provided Console, first uninstalling any current one. The Jython Console is used + * for raw_input() etc., and may provide line-editing and history recall at the + * prompt. A Console may replace System.in with its line-editing input method. + * + * @param console The new Console object + * @throws UnsupportedOperationException if some prior Console refuses to uninstall + * @throws IOException if {@link Console#install()} raises it + */ + public static void installConsole(Console console) throws UnsupportedOperationException, + IOException { + if (Py.console != null) { + // Some Console class already installed: may be able to uninstall + Py.console.uninstall(); + Py.console = null; + } + + // Install the specified Console + console.install(); + Py.console = console; + + // Cause sys (if it exists) to export the console handler that was installed + if (Py.defaultSystemState != null) { + Py.defaultSystemState.__setattr__("_jy_console", Py.java2py(console)); + } + } + + /** + * Check (using the {@link POSIX} library) whether we are in an interactive environment. Amongst + * other things, this affects the type of console that may be legitimately installed during + * system initialisation. + * + * @return + */ + public static boolean isInteractive() { + // Decide if System.in is interactive + POSIX posix = POSIXFactory.getPOSIX(); + FileDescriptor in = FileDescriptor.in; + return posix.isatty(in); + } + /* A collection of functions for implementing the print statement */ public static StdoutWrapper stderr; static StdoutWrapper stdout; diff --git a/src/org/python/core/PySystemState.java b/src/org/python/core/PySystemState.java index 74ebc51a2..3dab313fc 100644 --- a/src/org/python/core/PySystemState.java +++ b/src/org/python/core/PySystemState.java @@ -2,7 +2,6 @@ package org.python.core; import java.io.BufferedReader; -import java.io.Console; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -11,6 +10,8 @@ import java.lang.ref.Reference; import java.lang.ref.ReferenceQueue; import java.lang.ref.WeakReference; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URL; import java.net.URLDecoder; @@ -40,7 +41,6 @@ import org.python.expose.ExposedGet; import org.python.expose.ExposedType; import org.python.modules.Setup; -import org.python.modules.zipimport.zipimporter; import org.python.util.Generic; /** @@ -199,7 +199,7 @@ public PySystemState() { meta_path = new PyList(); path_hooks = new PyList(); path_hooks.append(new JavaImporter()); - path_hooks.append(zipimporter.TYPE); + path_hooks.append(org.python.modules.zipimport.zipimporter.TYPE); path_hooks.append(ClasspathPyImporter.TYPE); path_importer_cache = new PyDictionary(); @@ -879,17 +879,25 @@ public static synchronized PySystemState doInitialize(Properties preProperties, if (jarFileName != null) { standalone = isStandalone(jarFileName); } + // initialize the Jython registry initRegistry(preProperties, postProperties, standalone, jarFileName); + // other initializations initBuiltins(registry); initStaticFields(); + // Initialize the path (and add system defaults) defaultPath = initPath(registry, standalone, jarFileName); defaultArgv = initArgv(argv); defaultExecutable = initExecutable(registry); + // Set up the known Java packages initPackages(registry); + + // Condition the console + initConsole(registry); + // Finish up standard Python initialization... Py.defaultSystemState = new PySystemState(); Py.setSystemState(Py.defaultSystemState); @@ -897,10 +905,16 @@ public static synchronized PySystemState doInitialize(Properties preProperties, Py.defaultSystemState.setClassLoader(classLoader); } Py.initClassExceptions(getDefaultBuiltins()); + // defaultSystemState can't init its own encoding, see its constructor Py.defaultSystemState.initEncoding(); + // Make sure that Exception classes have been loaded new PySyntaxError("", 1, 1, "", ""); + + // Cause sys to export the console handler that was installed + Py.defaultSystemState.__setattr__("_jy_console", Py.java2py(Py.getConsole())); + return Py.defaultSystemState; } @@ -1022,6 +1036,86 @@ private static PyObject initExecutable(Properties props) { return new PyString(executableFile.getPath()); } + /** + * Wrap standard input with a customised console handler specified in the property + * python.console in the supplied property set, which in practice is the + * fully-initialised Jython {@link #registry}. The value of python.console is the + * name of a class that implements {@link org.python.core.Console}. An instance is constructed + * with the value of python.console.encoding, and the console + * System.in returns characters in that encoding. After the call, the console + * object may be accessed via {@link Py#getConsole()}. + * + * @param props containing (or not) python.console + */ + private static void initConsole(Properties props) { + // At this stage python.console.encoding is always defined (but null=default) + String encoding = props.getProperty(PYTHON_CONSOLE_ENCODING); + // The console type is chosen by this registry entry: + String consoleName = props.getProperty("python.console", "").trim(); + // And must be of type ... + final Class consoleType = Console.class; + + if (consoleName.length() > 0 && Py.isInteractive()) { + try { + // Load the class specified as the console + Class consoleClass = Class.forName(consoleName); + + // Ensure it can be cast to the interface type of all consoles + if (! consoleType.isAssignableFrom(consoleClass)) { + throw new ClassCastException(); + } + + // Construct an instance + Constructor consoleConstructor = consoleClass.getConstructor(String.class); + Object consoleObject = consoleConstructor.newInstance(encoding); + Console console = consoleType.cast(consoleObject); + + // Replace System.in with stream this console manufactures + Py.installConsole(console); + return; + + } catch (NoClassDefFoundError e) { + writeConsoleWarning(consoleName, "not found"); + } catch (ClassCastException e) { + writeConsoleWarning(consoleName, "does not implement " + consoleType); + } catch (NoSuchMethodException e) { + writeConsoleWarning(consoleName, "has no constructor from String"); + } catch (InvocationTargetException e) { + writeConsoleWarning(consoleName, e.getCause().toString()); + } catch (Exception e) { + writeConsoleWarning(consoleName, e.toString()); + } + } + + // No special console required, or requested installation failed somehow + try { + // Default is a plain console + Py.installConsole(new PlainConsole(encoding)); + return; + } catch (Exception e) { + /* + * May end up here if prior console won't uninstall: but then at least we have a + * console. Or it may be an unsupported encoding, in which case Py.getConsole() will try + * "ascii" + */ + writeConsoleWarning(consoleName, e.toString()); + } + } + + /** + * Convenience method wrapping {@link Py#writeWarning(String, String)} to issue a warning + * message something like: + * "console: Failed to load 'org.python.util.ReadlineConsole': msg.". It's only a warning + * because the interpreter will fall back to a plain console, but it is useful to know exactly + * why it didn't work. + * + * @param consoleName console class name we're trying to initialise + * @param msg specific cause of the failure + */ + private static void writeConsoleWarning(String consoleName, String msg) { + Py.writeWarning("console", "Failed to install '" + consoleName + "': " + msg + "."); + } + private static void addBuiltin(String name) { String classname; String modname; @@ -1456,6 +1550,7 @@ public PyObject __call__(PyObject arg) { } } + @Override public PyObject __call__(PyObject arg1, PyObject arg2, PyObject arg3) { switch (index) { case 30: @@ -1478,10 +1573,12 @@ class PyAttributeDeleted extends PyObject { private PyAttributeDeleted() {} + @Override public String toString() { return ""; } + @Override public Object __tojava__(Class c) { if (c == PyObject.class) { return this; diff --git a/src/org/python/core/__builtin__.java b/src/org/python/core/__builtin__.java index 815b73056..3393a3083 100644 --- a/src/org/python/core/__builtin__.java +++ b/src/org/python/core/__builtin__.java @@ -4,18 +4,19 @@ */ package org.python.core; +import java.io.EOFException; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.HashMap; +import java.nio.ByteBuffer; import java.util.Iterator; import java.util.Map; import org.python.antlr.base.mod; import org.python.core.util.RelativeFile; - +import org.python.core.util.StringUtil; import org.python.modules._functools._functools; class BuiltinFunctions extends PyBuiltinFunctionSet { @@ -1010,6 +1011,14 @@ private static PyString readline(PyObject file) { } } + /** + * Companion to raw_input built-in function used when the interactive interpreter + * is directed to a file. + * + * @param prompt to issue at console before read + * @param file a file-like object to read from + * @return line of text from the file (encoded as bytes values compatible with PyString) + */ public static String raw_input(PyObject prompt, PyObject file) { PyObject stdout = Py.getSystemState().stdout; if (stdout instanceof PyAttributeDeleted) { @@ -1027,16 +1036,32 @@ public static String raw_input(PyObject prompt, PyObject file) { return data; } + /** + * Implementation of raw_input(prompt) built-in function using the console + * directly. + * + * @param prompt to issue at console before read + * @return line of text from console (encoded as bytes values compatible with PyString) + */ public static String raw_input(PyObject prompt) { - PyObject stdin = Py.getSystemState().stdin; - if (stdin instanceof PyAttributeDeleted) { - throw Py.RuntimeError("[raw_]input: lost sys.stdin"); + try { + Console console = Py.getConsole(); + ByteBuffer buf = console.raw_input(prompt.toString()); + return StringUtil.fromBytes(buf); + } catch (EOFException eof) { + throw Py.EOFError("raw_input()"); + } catch (IOException ioe) { + throw Py.IOError(ioe); } - return raw_input(prompt, stdin); } + /** + * Implementation of raw_input() built-in function using the console directly. + * + * @return line of text from console (encoded as bytes values compatible with PyString) + */ public static String raw_input() { - return raw_input(new PyString("")); + return raw_input(Py.EmptyString); } public static PyObject reduce(PyObject f, PyObject l, PyObject z) { diff --git a/src/org/python/util/ConsoleStream.java b/src/org/python/util/ConsoleStream.java new file mode 100644 index 000000000..0f4132ff9 --- /dev/null +++ b/src/org/python/util/ConsoleStream.java @@ -0,0 +1,242 @@ +// Copyright (c) 2013 Jython Developers +package org.python.util; + +import java.io.EOFException; +import java.io.FilterInputStream; +import java.io.IOException; +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; + +/** + * This class is intended to replace System.in for use with console libraries that + * provide a line-oriented input mechanism. The console libraries provide a method to get the next + * line from the console as a String. Particular sub-classes should wrap this character-oriented + * method in a definition of {@link #getLine()}. + *

+ * The libraries JLine and Java Readline have both been used to give Jython line-recall, editing and + * a line history preserved between sessions. Both deal with the console encoding internally, and + * interact with the user in terms of a buffer of characters. Our need in Jython is to access a + * byte-stream encoding the characters, with line-endings, since it is the text layer of the Python + * io stack, whether we are using the io module or file built-in, that + * should deal with encoding. + */ +public abstract class ConsoleStream extends FilterInputStream { + + /** + * Enumeration used to specify whether an end-of-line should be added or replaced at the end of + * each line read. LEAVE means process the line exactly as the library returns it; ADD means + * always add an end-of-line; and REPLACE means strip any final '\n', '\r', or '\r\n' and add an + * end-of-line. The end-of-line to add is specified as a String in the constructor. + */ + public enum EOLPolicy { + LEAVE, ADD, REPLACE + }; + + /** The {@link EOLPolicy} specified in the constructor. */ + protected final EOLPolicy eolPolicy; + /** The end-of-line String specified in the constructor. */ + protected final String eol; + /** The end-of-line String specified in the constructor. */ + protected final Charset encoding; + /** Bytes decoded from the last line read. */ + private ByteBuffer buf; + /** Empty buffer */ + protected static final ByteBuffer EMPTY_BUF = ByteBuffer.allocate(0); + /** Platform-defined end-of-line for convenience */ + protected static final String LINE_SEPARATOR = System.getProperty("line.separator"); + + /** + * Create a wrapper configured with end-of-line handling that matches the specific console + * library being wrapped, and a character encoding matching the expectations of the client. + * Since this is an abstract class, this constructor will be called as the first action of the + * library-specific concrete class. The end-of-line policy can be chosen from LEAVE + * (do not modify the line), ADD (always append eol, and + * REPLACE (remove a trailing '\n', '\r', or '\r\n' provided by the library, then + * add eol). + * + * @param encoding to use to encode the buffered characters + * @param eolPolicy choice of how to treat an end-of-line marker + * @param eol the end-of-line to use when eolPolicy is not LEAVE + */ + ConsoleStream(Charset encoding, EOLPolicy eolPolicy, String eol) { + + // Wrap original System.in so StreamIO.isatty() will find it reflectively + super(System.in); + + // But our real input comes from (re-)encoding the console line + this.encoding = encoding; + this.eolPolicy = eolPolicy; + this.eol = eol != null ? eol : LINE_SEPARATOR; + + // The logic is simpler if we always supply a buffer + buf = EMPTY_BUF; + } + + /** + * Get one line of input from the console. Override this method with the actions specific to the + * library in use. + * + * @return Line entered by user + * @throws IOException in case of an error + * @throws EOFException if the library recognises an end-of-file condition + */ + protected abstract CharSequence getLine() throws IOException, EOFException; + + /** + * Get a line of text from the console and re-encode it using the console encoding to bytes that + * will be returned from this InputStream in subsequent read operations. + * + * @throws IOException + * @throws EOFException + */ + private void fillBuffer() throws IOException, EOFException { + + // In case we exit on an exception ... + buf = EMPTY_BUF; + + // Bring in another line + CharSequence line = getLine(); + CharBuffer cb = CharBuffer.allocate(line.length() + eol.length()); + cb.append(line); + + // Apply the EOL policy + switch (eolPolicy) { + + case LEAVE: + // Do nothing + break; + + case ADD: + // Always add eol + cb.append(eol); + break; + + case REPLACE: + // Strip '\n', '\r', or '\r\n' and add eol + int n = cb.position() - 1; + if (n >= 0 && cb.charAt(n) == '\n') { + n -= 1; + } + if (n >= 0 && cb.charAt(n) == '\r') { + n -= 1; + } + cb.position(n + 1); + cb.append(eol); + break; + } + + // Prepare to read + cb.flip(); + + // Make this line into a new buffer of encoded bytes + if (cb.hasRemaining()) { + buf = encoding.encode(cb); // includes a flip() + } + } + + /** + * Reads the next byte of data from the buffered input line. + * + * The byte is returned as an int in the range 0 to 255. If no byte is available because the end + * of the stream has been recognised, the value -1 is returned. This method blocks until input + * data is available, the end of the stream is detected, or an exception is thrown. Normally, an + * empty line results in an encoded end-of-line being returned. + */ + @Override + public int read() throws IOException { + + try { + // Do we need to refill? + while (!buf.hasRemaining()) { + fillBuffer(); + } + return buf.get() & 0xff; + } catch (EOFException e) { + // End of file condition recognised (e.g. ctrl-D, ctrl-Z) + return -1; + } + } + + /** + * Reads up to len bytes of data from this input stream into an array of bytes. If len is not + * zero, the method blocks until some input is available; otherwise, no bytes are read and 0 is + * returned. This implementation calls {@link #fillBuffer()} at most once to get a line of + * characters from the console using {@link #getLine()}, and encodes them as bytes to be read + * back from the stream. + */ + @Override + public int read(byte[] b, int off, int len) throws IOException, EOFException { + + if (off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException(); + + } else { + try { + if (len > 0) { + // Do we need to refill? (Not if zero bytes demanded.) + int n = buf.remaining(); + if (n <= 0) { + fillBuffer(); + n = buf.remaining(); + } + + // Deliver all there is, or all that's wanted, whichever is less. + len = n < len ? n : len; + buf.get(b, off, len); + } + return len; + + } catch (EOFException e) { + // Thrown from getLine + return -1; + } + } + } + + /** + * Skip forward n bytes within the current encoded line. A call to skip will not + * result in reading a new line with {@link #getLine()}. + */ + @Override + public long skip(long n) throws IOException { + long r = buf.remaining(); + if (n > r) { + n = r; + } + buf.position(buf.position() + (int)n); + return n; + } + + /** The number of bytes left unread in the current encoded line. */ + @Override + public int available() throws IOException { + return buf.remaining(); + } + + /** + * If possible, restore the standard System.in. Override this if necessary to + * perform close down actions on the console library, then call super.close(). + */ + @Override + public void close() throws IOException { + // Restore original System.in + System.setIn(in); + } + + /** Mark is not supported. */ + @Override + public synchronized void mark(int readlimit) {} + + /** Mark is not supported. */ + @Override + public synchronized void reset() throws IOException {} + + /** Mark is not supported. */ + @Override + public boolean markSupported() { + return false; + } + +} diff --git a/src/org/python/util/JLineConsole.java b/src/org/python/util/JLineConsole.java index 555118d8e..636b612cb 100644 --- a/src/org/python/util/JLineConsole.java +++ b/src/org/python/util/JLineConsole.java @@ -1,14 +1,16 @@ -/* Copyright (c) Jython Developers */ +// Copyright (c) 2013 Jython Developers package org.python.util; +import java.io.EOFException; import java.io.File; import java.io.FileDescriptor; import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileOutputStream; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStreamWriter; +import java.io.PrintWriter; import java.io.Writer; import java.util.Arrays; import java.util.List; @@ -18,21 +20,22 @@ import jline.WindowsTerminal; import jnr.constants.platform.Errno; -import org.python.core.Py; +import org.python.core.PlainConsole; import org.python.core.PyObject; /** * This class uses JLine to provide readline like * functionality to its console without requiring native readline support. */ -public class JLineConsole extends InteractiveConsole { +public class JLineConsole extends PlainConsole { /** Main interface to JLine. */ - protected ConsoleReader reader; + public ConsoleReader reader; - /** Set by readline.set_startup_hook */ + /** Callable object set by readline.set_startup_hook. */ protected PyObject startup_hook; + /** Not currently set by readline.set_pre_input_hook. Why not? */ protected PyObject pre_input_hook; /** Whether reader is a WindowsTerminal. */ @@ -48,109 +51,212 @@ public class JLineConsole extends InteractiveConsole { private static final List SUSPENDED_STRERRORS = Arrays.asList( Errno.EINTR.description(), Errno.EIO.description()); - public JLineConsole() { - this(null); + /** + * Construct an instance of the console class specifying the character encoding. This encoding + * must be one supported by the JVM. + *

+ * Most of the initialisation is deferred to the {@link #install()} method so that any prior + * console can uninstall itself before we change system console settings and + * System.in. + * + * @param encoding name of a supported encoding or null for + * Charset.defaultCharset() + */ + public JLineConsole(String encoding) { + /* + * Super-class needs the encoding in order to re-encode the characters that + * jline.ConsoleReader.readLine() has decoded. + */ + super(encoding); + /* + * Communicate the specified encoding to JLine. jline.ConsoleReader.readLine() edits a line + * of characters, decoded from stdin. + */ + System.setProperty("jline.WindowsTerminal.input.encoding", this.encoding); + System.setProperty("input.encoding", this.encoding); + // ... not "jline.UnixTerminal.input.encoding" as you might think, not even in JLine2 } - public JLineConsole(PyObject locals) { - this(locals, CONSOLE_FILENAME); + /** + * {@inheritDoc} + *

+ * This implementation overrides that by setting System.in to a + * FilterInputStream object that wraps JLine. + */ + @Override + public void install() { + Terminal.setupTerminal(); + + String userHomeSpec = System.getProperty("user.home", "."); + + // Configure a ConsoleReader (the object that does most of the line editing). try { - File historyFile = new File(System.getProperty("user.home"), ".jline-jython.history"); - reader.getHistory().setHistoryFile(historyFile); - } catch (IOException e) { - // oh well, no history from file - } - } + /* + * Wrap System.out in the specified encoding. jline.ConsoleReader.readLine() echoes the + * line through this Writer. + */ + Writer out = new PrintWriter(new OutputStreamWriter(System.out, encoding)); - public JLineConsole(PyObject locals, String filename) { - super(locals, filename, true); + // Get the key bindings (built in ones treat TAB Pythonically). + InputStream bindings = getBindings(userHomeSpec, getClass().getClassLoader()); - // Disable JLine's unicode handling so it yields raw bytes - System.setProperty("jline.UnixTerminal.input.encoding", "ISO-8859-1"); - System.setProperty("jline.WindowsTerminal.input.encoding", "ISO-8859-1"); + // Create the reader as unbuffered as possible + InputStream in = new FileInputStream(FileDescriptor.in); + reader = new ConsoleReader(in, out, bindings); - Terminal.setupTerminal(); - try { - InputStream input = new FileInputStream(FileDescriptor.in); - // Raw bytes in, so raw bytes out - Writer output = - new OutputStreamWriter(new FileOutputStream(FileDescriptor.out), "ISO-8859-1"); - reader = new ConsoleReader(input, output, getBindings()); + // We find the bell too noisy reader.setBellEnabled(false); + } catch (IOException e) { throw new RuntimeException(e); } + // Access and load (if possible) the line history. + try { + File historyFile = new File(userHomeSpec, ".jline-jython.history"); + reader.getHistory().setHistoryFile(historyFile); + } catch (IOException e) { + // oh well, no history from file + } + + // Check for OS type windows = reader.getTerminal() instanceof WindowsTerminal; + + // Replace System.in + FilterInputStream wrapper = new Stream(); + System.setIn(wrapper); } + // Inherited raw_input() is adequate: calls input() + /** - * Return the JLine bindings file. - * - * This handles loading the user's custom key bindings (normally JLine does) so it can fall back - * to Jython's (which disable tab completion) when the user's are not available. - * - * @return an InputStream of the JLine bindings file. + * {@inheritDoc} + *

+ * This console implements input using JLine to handle the prompt and data entry, + * so that the cursor may be correctly handled in relation to the prompt string. */ - protected InputStream getBindings() { - String userBindings = - new File(System.getProperty("user.home"), ".jlinebindings.properties") - .getAbsolutePath(); - File bindingsFile = new File(System.getProperty("jline.keybindings", userBindings)); + @Override + public CharSequence input(CharSequence prompt) throws IOException, EOFException { + // Get the line from the console via the library + String line = readerReadLine(prompt.toString()); + if (line == null) { + throw new EOFException(); + } else { + return line; + } + } - try { - if (bindingsFile.isFile()) { - try { - return new FileInputStream(bindingsFile); - } catch (FileNotFoundException fnfe) { - // Shouldn't really ever happen - fnfe.printStackTrace(); - } + /** + * Class to wrap the line-oriented interface to JLine with an InputStream that can replace + * System.in. + */ + private class Stream extends ConsoleStream { + + /** Create a System.in replacement with JLine that adds system-specific line endings */ + Stream() { + super(encodingCharset, EOLPolicy.ADD, LINE_SEPARATOR); + } + + @Override + protected CharSequence getLine() throws IOException, EOFException { + + // Get a line and hope to be done. Suppress any remembered prompt. + String line = readerReadLine(""); + + if (!isEOF(line)) { + return line; + } else { + // null or ctrl-z on Windows indicates EOF + throw new EOFException(); } - } catch (SecurityException se) { - // continue } - return getClass().getResourceAsStream("jline-keybindings.properties"); } - @Override - public String raw_input(PyObject prompt) { - String line = null; - String promptString = prompt.toString(); + /** + * Wrapper on reader.readLine(prompt) that deals with retries (on Unix) when the user enters + * cvtrl-Z to background Jython, the brings it back to the foreground. The inherited + * implementation says this is necessary and effective on BSD Unix. + * + * @param prompt to display + * @return line of text read in + * @throws IOException if an error occurs (other than an end of suspension) + * @throws EOFException if an EOF is detected + */ + private String readerReadLine(String prompt) throws IOException, EOFException { + + // We must be prepared to try repeatedly since the read may be interrupted. while (true) { + try { + // If there's a hook, call it if (startup_hook != null) { - try { - startup_hook.__call__(); - } catch (Exception ex) { - System.err.println(ex); - } + startup_hook.__call__(); } - line = reader.readLine(promptString); - break; + // Get a line and hope to be done. + String line = reader.readLine(prompt); + return line; + } catch (IOException ioe) { + // Something went wrong, or we were interrupted (seems only BSD throws this) if (!fromSuspend(ioe)) { - throw Py.IOError(ioe); - } + // The interruption is not the result of (the end of) a ctrl-Z suspension + throw ioe; - // Hopefully an IOException caused by ctrl-z (seems only BSD throws this). - // Must reset jline to continue - try { - reader.getTerminal().initializeTerminal(); - } catch (Exception e) { - throw Py.IOError(e.getMessage()); + } else { + // The interruption seems to be (return from) a ctrl-Z suspension: + try { + // Must reset JLine and continue (not repeating the prompt) + reader.getTerminal().initializeTerminal(); + prompt = ""; + } catch (Exception e) { + // Do our best to say what went wrong + throw new IOException("Failed to re-initialize JLine: " + e.getMessage()); + } } - // Don't redisplay the prompt - promptString = ""; } } - if (isEOF(line)) { - throw Py.EOFError(""); + } + + /** + * Return the JLine bindings file. + * + * This handles loading the user's custom key bindings (normally JLine does) so it can fall back + * to Jython's (which disable tab completion) when the user's are not available. + * + * @return an InputStream of the JLine bindings file. + */ + protected static InputStream getBindings(String userHomeSpec, ClassLoader loader) { + + // The key bindings file may be specified explicitly + String bindingsFileSpec = System.getProperty("jline.keybindings"); + File bindingsFile; + + if (bindingsFileSpec != null) { + // Bindings file explicitly specified + bindingsFile = new File(bindingsFileSpec); + } else { + // Otherwise try ~/.jlinebindings.properties + bindingsFile = new File(userHomeSpec, ".jlinebindings.properties"); + } + + // See if that file really exists (and can be read) + try { + if (bindingsFile.isFile()) { + try { + return new FileInputStream(bindingsFile); + } catch (FileNotFoundException fnfe) { + // Shouldn't really ever happen + fnfe.printStackTrace(); + } + } + } catch (SecurityException se) { + // continue } - return line; + // User/specific key bindings could not be read: use the ones from the class path or jar. + return loader.getResourceAsStream("org/python/util/jline-keybindings.properties"); } /** diff --git a/src/org/python/util/ReadlineConsole.java b/src/org/python/util/ReadlineConsole.java index 23bc18230..cc16436a1 100644 --- a/src/org/python/util/ReadlineConsole.java +++ b/src/org/python/util/ReadlineConsole.java @@ -1,72 +1,208 @@ -// Copyright (c) Corporation for National Research Initiatives +// Copyright (c) 2013 Jython Developers package org.python.util; import java.io.EOFException; +import java.io.FilterInputStream; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; import org.gnu.readline.Readline; import org.gnu.readline.ReadlineLibrary; - -import org.python.core.Py; -import org.python.core.PyException; -import org.python.core.PyObject; -import org.python.core.PySystemState; +import org.python.core.PlainConsole; /** - * Uses: Java Readline

- * - * Based on CPython-1.5.2's code module - * + * Uses: Java Readline to provide readline like + * functionality to its console through native readline support (either GNU Readline or Editline). */ -public class ReadlineConsole extends InteractiveConsole { +public class ReadlineConsole extends PlainConsole { - public String filename; + /** + * Construct an instance of the console class specifying the character encoding. This encoding + * must be one supported by the JVM. The particular backing library loaded will be as specified + * by registry item python.console.readlinelib, or "Editline" by default. + *

+ * Most of the initialisation is deferred to the {@link #install()} method so that any prior + * console can uninstall itself before we change system console settings and + * System.in. + * + * @param encoding name of a supported encoding or null for + * Charset.defaultCharset() + */ + public ReadlineConsole(String encoding) { + super(encoding); + /* + * Load the chosen native library. If it's not there, raise UnsatisfiedLinkError. We cannot + * fall back to Readline's Java mode since it reads from System.in, which would be pointless + * ... and fatal once we have replaced System.in with a wrapper on Readline. + */ + String backingLib = System.getProperty("python.console.readlinelib", "Editline"); + Readline.load(ReadlineLibrary.byName(backingLib)); - public ReadlineConsole() { - this(null, CONSOLE_FILENAME); + /* + * The following is necessary to compensate for (a possible thinking error in) Readline's + * handling of the bytes returned from the library, and of the prompt. + */ + String name = encodingCharset.name(); + if (name.equals("ISO-8859-1") || name.equals("US-ASCII")) { + // Indicate that Readline's Latin fixation will work for this encoding + latin1 = null; + } else { + // We'll need the bytes-to-pointcode mapping + latin1 = Charset.forName("ISO-8859-1"); + } } - public ReadlineConsole(PyObject locals) { - this(locals, CONSOLE_FILENAME); - } + /** + * {@inheritDoc} + *

+ * This implementation overrides that by setting System.in to a + * FilterInputStream object that wraps the configured console library. + */ + @Override + public void install() { - public ReadlineConsole(PyObject locals, String filename) { - super(locals, filename, true); - String backingLib = PySystemState.registry.getProperty("python.console.readlinelib", - "Editline"); - try { - Readline.load(ReadlineLibrary.byName(backingLib)); - } catch(RuntimeException e) { - // Silently ignore errors during load of the native library. - // Will use a pure java fallback. - } + // Complete the initialisation Readline.initReadline("jython"); try { // Force rebind of tab to insert a tab instead of complete Readline.parseAndBind("tab: tab-insert"); - } - catch (UnsupportedOperationException uoe) { + } catch (UnsupportedOperationException uoe) { // parseAndBind not supported by this readline } + + // Replace System.in + FilterInputStream wrapper = new Stream(); + System.setIn(wrapper); } /** - * Write a prompt and read a line. - * - * The returned line does not include the trailing newline. When the user - * enters the EOF key sequence, EOFError is raised. - * - * This subclass implements the functionality using JavaReadline. + * {@inheritDoc} + *

+ * This console implements input using the configured library to handle the prompt + * and data entry, so that the cursor may be correctly handled in relation to the prompt string. */ - public String raw_input(PyObject prompt) { - try { - String line = Readline.readline(prompt == null ? "" : prompt.toString()); - return (line == null ? "" : line); - } catch(EOFException eofe) { - throw new PyException(Py.EOFError); - } catch(IOException ioe) { - throw new PyException(Py.IOError); + @Override + public ByteBuffer raw_input(CharSequence prompt) throws IOException { + // If Readline.readline really returned the line as typed, we could simply use: + // return line==null ? "" : line; + // Compensate for Readline.readline prompt handling + prompt = preEncode(prompt); + // Get the line from the console via the library + String line = Readline.readline(prompt.toString()); + return postDecodeToBuffer(line); + } + + /** + * {@inheritDoc} + *

+ * This console implements input using the configured library to handle the prompt + * and data entry, so that the cursor may be correctly handled in relation to the prompt string. + */ + @Override + public CharSequence input(CharSequence prompt) throws IOException, EOFException { + // Compensate for Readline.readline prompt handling + prompt = preEncode(prompt); + // Get the line from the console via the library + String line = Readline.readline(prompt.toString()); + // If Readline.readline really returned the line as typed, next would have been: + // return line==null ? "" : line; + return postDecode(line); + } + + /** + * Class to wrap the line-oriented interface to Readline with an InputStream that can replace + * System.in. + */ + protected class Stream extends ConsoleStream { + + /** Create a System.in replacement with Readline that adds Unix-like line endings */ + Stream() { + super(encodingCharset, EOLPolicy.ADD, LINE_SEPARATOR); + } + + @Override + protected CharSequence getLine() throws IOException, EOFException { + // The Py3k input method does exactly what we want + return input(""); + } + } + + /** + * Encode a prompt to bytes in the console encoding and represent these bytes as the point codes + * of a Java String. The actual GNU readline function expects a prompt string that is C char + * array in the console encoding, but the wrapper Readline.readline acts as if this + * encoding is always Latin-1. This transformation compensates by encoding correctly then + * representing those bytes as point codes. + * + * @param prompt to display via Readline.readline + * @return encoded form of prompt + */ + private CharSequence preEncode(CharSequence prompt) { + if (prompt == null || prompt.length() == 0) { + return ""; + } else if (latin1 == null) { + // Encoding is such that readline does the right thing + return prompt; + } else { + // Compensate for readline prompt handling + CharBuffer cb = CharBuffer.wrap(prompt); + ByteBuffer bb = encodingCharset.encode(cb); + return latin1.decode(bb).toString(); + } + } + + /** + * Decode the bytes argument (a return from code>Readline.readline) to the String + * actually entered at the console. The actual GNU readline function returns a C char array in + * the console encoding, but the wrapper Readline.readline acts as if this encoding + * is always Latin-1, and on this basis it gives us a Java String whose point codes are the + * encoded bytes. This method gets the bytes back, then decodes them correctly to a String. + * + * @param bytes encoded line (or null for an empty line) + * @return bytes recovered from the argument + */ + private CharSequence postDecode(String line) { + if (line == null) { + // Library returns null for an empty line + return ""; + } else if (latin1 == null) { + // Readline's assumed Latin-1 encoding will have produced the correct result + return line; + } else { + // We have to transcode the line + CharBuffer cb = CharBuffer.wrap(line); + ByteBuffer bb = latin1.encode(cb); + return encodingCharset.decode(bb).toString(); + } + } + + /** + * Decode the line (a return from code>Readline.readline) to bytes in the console + * encoding. The actual GNU readline function returns a C char array in the console encoding, + * but the wrapper Readline.readline acts as if this encoding is always Latin-1, + * and on this basis it gives us a Java String whose point codes are the encoded bytes. This + * method gets the bytes back. + * + * @param bytes encoded line (or null for an empty line) + * @return bytes recovered from the argument + */ + private ByteBuffer postDecodeToBuffer(String line) { + if (line == null) { + // Library returns null for an empty line + return ConsoleStream.EMPTY_BUF; + } else if (latin1 == null) { + // Readline's assumed Latin-1 encoding will have produced the correct result + return encodingCharset.encode(line); + } else { + // We have to transcode the line + CharBuffer cb = CharBuffer.wrap(line); + return latin1.encode(cb); } } + + private final Charset latin1; + } diff --git a/src/org/python/util/jython.java b/src/org/python/util/jython.java index 952dd4b2c..1f304df18 100644 --- a/src/org/python/util/jython.java +++ b/src/org/python/util/jython.java @@ -2,6 +2,7 @@ package org.python.util; import java.io.File; +import java.io.FileDescriptor; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; @@ -14,6 +15,9 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipFile; +import jnr.posix.POSIX; +import jnr.posix.POSIXFactory; + import org.python.Version; import org.python.core.CodeFlag; import org.python.core.CompileMode; @@ -87,7 +91,7 @@ public class jython { * root of the JAR archive. Note that the __name__ is set to the base name of the JAR file and * not to "__main__" (for historic reasons). This method do NOT handle exceptions. the caller * SHOULD handle any (Py)Exceptions thrown by the code. - * + * * @param filename The path to the filename to run. */ public static void runJar(String filename) { @@ -201,8 +205,17 @@ public static void run(String[] args) { System.exit(exitcode); } + // Get system properties (or empty set if we're prevented from accessing them) + Properties preProperties = PySystemState.getBaseProperties(); + + // Decide if System.in is interactive + if (!opts.fixInteractive || opts.interactive) { + // The options suggest System.in is interactive: but only if isatty() agrees + opts.interactive = Py.isInteractive(); + } + // Setup the basic python system state from these options - PySystemState.initialize(PySystemState.getBaseProperties(), opts.properties, opts.argv); + PySystemState.initialize(preProperties, opts.properties, opts.argv); PySystemState systemState = Py.getSystemState(); PyList warnoptions = new PyList(); @@ -216,17 +229,12 @@ public static void run(String[] args) { imp.load("warnings"); } - // Decide if stdin is interactive - if (!opts.fixInteractive || opts.interactive) { - opts.interactive = ((PyFile)Py.defaultSystemState.stdin).isatty(); - if (!opts.interactive) { - systemState.ps1 = systemState.ps2 = Py.EmptyString; - } - } - // Now create an interpreter - InteractiveConsole interp = newInterpreter(opts.interactive); - systemState.__setattr__("_jy_interpreter", Py.java2py(interp)); + if (!opts.interactive) { + // Not (really) interactive, so do not use console prompts + systemState.ps1 = systemState.ps2 = Py.EmptyString; + } + InteractiveConsole interp = new InteractiveConsole(); // Print banner and copyright information (or not) if (opts.interactive && opts.notice && !opts.runModule) { @@ -381,30 +389,6 @@ public static void run(String[] args) { interp.cleanup(); } - /** - * Returns a new python interpreter using the InteractiveConsole subclass from the - * python.console registry key. - *

- * When stdin is interactive the default is {@link JLineConsole}. Otherwise the featureless - * {@link InteractiveConsole} is always used as alternative consoles cause unexpected behavior - * with the std file streams. - */ - private static InteractiveConsole newInterpreter(boolean interactiveStdin) { - if (!interactiveStdin) { - return new InteractiveConsole(); - } - - String interpClass = PySystemState.registry.getProperty("python.console", ""); - if (interpClass.length() > 0) { - try { - return (InteractiveConsole)Class.forName(interpClass).newInstance(); - } catch (Throwable t) { - // fall through - } - } - return new JLineConsole(); - } - /** * Run any finalizations on the current interpreter in preparation for a SytemRestart. */ diff --git a/tests/java/javatests/Issue1972.java b/tests/java/javatests/Issue1972.java index 2d1eee2d3..2a88c1ede 100644 --- a/tests/java/javatests/Issue1972.java +++ b/tests/java/javatests/Issue1972.java @@ -14,6 +14,7 @@ import java.util.List; import java.util.Properties; import java.util.concurrent.LinkedBlockingQueue; +import java.util.regex.Pattern; import org.junit.After; import org.junit.Test; @@ -26,9 +27,7 @@ * debugging of the subprocess. *

* This test passes in Jython 2.5.2 and 2.5.4rc1. The test {@link #jythonReadline()} fails with - * Jython 2.5.3. The test will fail the first time it is run on a clean build, or after switching - * Jython versions (JAR files). This is because it monitors stderr from the subprocess and does not - * expect the messages the cache manager produces on a first run. + * Jython 2.5.3. *

* The bulk of this program is designed to be run as JUnit tests, but it also has a * {@link #main(String[])} method that echoes System.in onto System.out @@ -46,7 +45,13 @@ public class Issue1972 { static int DEBUG_PORT = 0; // 8000 or 0 /** Control the amount of output to the console: 0, 1 or 2. */ - static int VERBOSE = 2; + static int VERBOSE = 0; + + /** Lines in stdout (as regular expressions) to ignore when checking subprocess output. */ + static String[] STDOUT_IGNORE = {"^Listening for transport dt_socket"}; + + /** Lines in stderr (as regular expressions) to ignore when checking subprocess output. */ + static String[] STDERR_IGNORE = {"^Jython 2", "^\\*sys-package-mgr"}; /** * Extra JVM options used when debugging is enabled. DEBUG_PORT will be substituted @@ -81,7 +86,7 @@ public void afterEachTest() { /** * Check that on this system we know how to launch and read the error output from a subprocess. - * + * * @throws IOException */ @Test @@ -103,7 +108,7 @@ public void readStderr() throws Exception { /** * Check that on this system we know how to launch and read standard output from a subprocess. - * + * * @throws IOException */ @Test @@ -122,7 +127,7 @@ public void readStdout() throws Exception { /** * Check that on this system we know how to launch, write to and read from a subprocess. - * + * * @throws IOException */ @Test @@ -152,7 +157,7 @@ public void echoStdin() throws Exception { * System.in in the subprocess, which of course writes hex to * System.out but that data is not received back in the parent process until * System.out.println() is called in the subprocess. - * + * * @throws IOException */ @Test @@ -176,7 +181,7 @@ public void echoStdinAsHex() throws Exception { /** * Test reading back from Jython subprocess with program on command-line. - * + * * @throws Exception */ @Test @@ -195,43 +200,42 @@ public void jythonSubprocess() throws Exception { /** * Discover what is handling the "console" when the program is on the command line only. - * + * * @throws Exception */ @Test - public void jythonNonInteractiveConsole() throws Exception { + public void jythonNonInteractive() throws Exception { announceTest(VERBOSE, "jythonNonInteractiveConsole"); // Run Jython enquiry about console as -c program setProcJava("org.python.util.jython", "-c", - "import sys; print type(sys._jy_interpreter).__name__; print sys.stdin.isatty()"); + "import sys; print type(sys._jy_console).__name__; print sys.stdin.isatty()"); proc.waitFor(); outputAsStrings(VERBOSE, inFromProc, errFromProc); checkErrFromProc(); - checkInFromProc("InteractiveConsole", "False"); + checkInFromProc("PlainConsole", "False"); } /** * Discover what is handling the "console" when the program is entered interactively at the * Jython prompt. - * + * * @throws Exception */ @Test - public void jythonInteractiveConsole() throws Exception { + public void jythonInteractive() throws Exception { announceTest(VERBOSE, "jythonInteractiveConsole"); // Run Jython with simple actions at the command prompt setProcJava( // - "-Dpython.console=org.python.util.InteractiveConsole", // "-Dpython.home=" + pythonHome, // "org.python.util.jython"); writeToProc("12+3\n"); writeToProc("import sys\n"); - writeToProc("print type(sys._jy_interpreter).__name__\n"); + writeToProc("print type(sys._jy_console).__name__\n"); writeToProc("print sys.stdin.isatty()\n"); toProc.close(); proc.waitFor(); @@ -239,13 +243,13 @@ public void jythonInteractiveConsole() throws Exception { outputAsStrings(VERBOSE, inFromProc, errFromProc); checkErrFromProc(""); // stderr produces one empty line. Why? - checkInFromProc("15", "InteractiveConsole", "False"); + checkInFromProc("15", "PlainConsole", "False"); } /** * Discover what is handling the "console" when the program is entered interactively at the * Jython prompt, and we try to force use of JLine (which fails). - * + * * @throws Exception */ @Test @@ -260,7 +264,7 @@ public void jythonJLineConsole() throws Exception { writeToProc("12+3\n"); writeToProc("import sys\n"); - writeToProc("print type(sys._jy_interpreter).__name__\n"); + writeToProc("print type(sys._jy_console).__name__\n"); writeToProc("print sys.stdin.isatty()\n"); toProc.close(); proc.waitFor(); @@ -269,13 +273,13 @@ public void jythonJLineConsole() throws Exception { checkErrFromProc(""); // stderr produces one empty line. Why? - // Although we asked for it, a subprocess doesn't get JLine, and isatty() is false - checkInFromProc("15", "InteractiveConsole", "False"); + // We can specify JLineConsole, but isatty() is not fooled. + checkInFromProc("15", "PlainConsole", "False"); } /** * Test writing to and reading back from Jython subprocess with echo program on command-line. - * + * * @throws Exception */ @Test @@ -284,6 +288,9 @@ public void jythonReadline() throws Exception { // Run Jython simple readline programme setProcJava( // + "-Dpython.console=org.python.util.JLineConsole", // + // "-Dpython.console.interactive=True", // + "-Dpython.home=" + pythonHome, // "org.python.util.jython", // "-c", // "import sys; sys.stdout.write(sys.stdin.readline()); sys.stdout.flush();" // @@ -338,7 +345,7 @@ public void jythonReadline() throws Exception { * echo the characters as hexadecimal * * - * + * * @param args * @throws IOException */ @@ -373,7 +380,7 @@ public static void main(String args[]) throws IOException { /** * Invoke the java command with the given arguments. The class path will be the same as this * programme's class path (as in the property java.class.path). - * + * * @param args further arguments to the program run * @return the running process * @throws IOException @@ -413,7 +420,7 @@ static Process startJavaProcess(String... args) throws IOException { * programme's class path (as in the property java.class.path). After the call, * {@link #proc} references the running process and {@link #inFromProc} and {@link #errFromProc} * are handling the stdout and stderr of the subprocess. - * + * * @param args further arguments to the program run * @throws IOException */ @@ -427,7 +434,7 @@ private void setProcJava(String... args) throws IOException { /** * Write this string into the stdin of the subprocess. The platform default * encoding will be used. - * + * * @param s to write * @throws IOException */ @@ -441,14 +448,15 @@ private void writeToProc(String s) throws IOException { * {@link #escape(byte[])} transormation has been applied, are expected to be equal to the * strings supplied, optionally after {@link #escapedSeparator} has been added to the expected * strings. - * + * * @param message identifies the queue in error message * @param addSeparator if true, system-defined line separator expected * @param queue to be compared + * @param toIgnore patterns defining lines to ignore while processing * @param expected lines of text (given without line separators) */ private void checkFromProc(String message, boolean addSeparator, LineQueue queue, - String... expected) { + List toIgnore, String... expected) { if (addSeparator) { // Each expected string must be treated as if the lineSeparator were appended @@ -470,29 +478,73 @@ private void checkFromProc(String message, boolean addSeparator, LineQueue queue // Count through the results, stopping when either results or expected strings run out int count = 0; for (String r : results) { - if (count < expected.length) { - assertEquals(message, expected[count++], r); - } else { + if (count >= expected.length) { break; + } else if (!matchesAnyOf(r, toIgnore)) { + assertEquals(message, expected[count++], r); } } assertEquals(message, expected.length, results.size()); } + /** Compiled regular expressions for the lines to ignore (on stdout). */ + private static List stdoutIgnore; + + /** Compiled regular expressions for the lines to ignore (on stderr). */ + private static List stderrIgnore; + + /** If not already done, compile the regular expressions we need. */ + private static void compileToIgnore() { + if (stdoutIgnore == null || stderrIgnore == null) { + // Compile the lines to ignore to Pattern objects + stdoutIgnore = compileAll(STDOUT_IGNORE); + stderrIgnore = compileAll(STDERR_IGNORE); + } + } + + /** If not already done, compile one set of regular expressions to patterns. */ + private static List compileAll(String[] regex) { + List result = new LinkedList(); + if (regex != null) { + for (String s : regex) { + Pattern p = Pattern.compile(s); + result.add(p); + } + } + return result; + } + + /** + * Compute whether a string matches any of a set of strings. + * + * @param s the string in question + * @param patterns to check against + * @return + */ + private static boolean matchesAnyOf(String s, List patterns) { + for (Pattern p : patterns) { + if (p.matcher(s).matches()) { + return true; + } + } + return false; + } + /** * Check lines of {@link #inFromProc} against expected text. - * + * * @param addSeparator if true, system-defined line separator expected * @param expected lines of text (given without line separators) */ private void checkInFromProc(boolean addSeparator, String... expected) { - checkFromProc("subprocess stdout", addSeparator, inFromProc, expected); + compileToIgnore(); // Make sure we have the matcher patterns + checkFromProc("subprocess stdout", addSeparator, inFromProc, stdoutIgnore, expected); } /** * Check lines of {@link #inFromProc} against expected text. Lines from the subprocess are * expected to be equal to those supplied after {@link #escapedSeparator} has been added. - * + * * @param expected lines of text (given without line separators) */ private void checkInFromProc(String... expected) { @@ -501,18 +553,19 @@ private void checkInFromProc(String... expected) { /** * Check lines of {@link #errFromProc} against expected text. - * + * * @param addSeparator if true, system-defined line separator expected * @param expected lines of text (given without line separators) */ private void checkErrFromProc(boolean addSeparator, String... expected) { - checkFromProc("subprocess stderr", addSeparator, errFromProc, expected); + compileToIgnore(); // Make sure we have the matcher patterns + checkFromProc("subprocess stderr", addSeparator, errFromProc, stderrIgnore, expected); } /** * Check lines of {@link #errFromProc} against expected text. Lines from the subprocess are * expected to be equal to those supplied after {@link #escapedSeparator} has been added. - * + * * @param expected lines of text (given without line separators) */ private void checkErrFromProc(String... expected) { @@ -521,7 +574,7 @@ private void checkErrFromProc(String... expected) { /** * Brevity for announcing tests on the console when that is used to dump values. - * + * * @param verbose if <1 suppress output * @param name of test */ @@ -533,7 +586,7 @@ static void announceTest(int verbose, String name) { /** * Output is System.out the formatted strings representing lines from a subprocess stdout. - * + * * @param verbose if <2 suppress output * @param inFromProc lines received from the stdout of a subprocess */ @@ -546,7 +599,7 @@ static void outputAsStrings(int verbose, LineQueue inFromProc) { /** * Output is System.out the formatted strings representing lines from a subprocess stdout, and * if there are any, from stderr. - * + * * @param verbose if <2 suppress output * @param inFromProc lines received from the stdout of a subprocess * @param errFromProc lines received from the stderr of a subprocess @@ -559,7 +612,7 @@ static void outputAsStrings(int verbose, LineQueue inFromProc, LineQueue errFrom /** * Output is System.out a hex dump of lines from a subprocess stdout. - * + * * @param verbose if <2 suppress output * @param inFromProc lines received from the stdout of a subprocess */ @@ -572,7 +625,7 @@ static void outputAsHexDump(int verbose, LineQueue inFromProc) { /** * Output is System.out a hex dump of lines from a subprocess stdout, and if there are any, from * stderr. - * + * * @param verbose if <2 suppress output * @param inFromProc lines received from the stdout of a subprocess * @param errFromProc lines received from the stderr of a subprocess @@ -586,7 +639,7 @@ static void outputAsHexDump(int verbose, LineQueue inFromProc, LineQueue errFrom /** * Output is System.out the formatted strings representing lines from a subprocess stdout, and * if there are any, from stderr. - * + * * @param stdout to output labelled "Output stream:" * @param stderr to output labelled "Error stream:" unless an empty list or null */ @@ -612,7 +665,7 @@ private static void outputStreams(List stdout, List stderr) { /** * Helper to format one line of string output hex-escaping non-ASCII characters. - * + * * @param sb to overwrite with the line of dump output * @param bb from which to take the bytes */ @@ -646,7 +699,7 @@ private static void stringDump(StringBuilder sb, ByteBuffer bb) { /** * Convert bytes (interpreted as ASCII) to String where the non-ascii characters are escaped. - * + * * @param b * @return */ @@ -676,7 +729,7 @@ static class LineQueue extends LinkedBlockingQueue { /** * Wrap a stream in the reader and immediately begin reading it. - * + * * @param in */ LineQueue(InputStream in) { @@ -701,7 +754,7 @@ public void run() { /** * Scan every byte read from the input and squirrel them away in buffers, one per line, * where lines are delimited by \r, \n or \r\n.. - * + * * @throws IOException */ private void runScribe() throws IOException { @@ -754,7 +807,7 @@ private void emitBuffer() { /** * Return the contents of the queue as a list of escaped strings, interpreting the bytes as * ASCII. - * + * * @return contents as strings */ public List asStrings() { @@ -774,7 +827,7 @@ public List asStrings() { /** * Return a hex dump the contents of the object as a list of strings - * + * * @return dump as strings */ public List asHexDump() { @@ -799,7 +852,7 @@ public List asHexDump() { /** * Helper to format one line of hex dump output up to a maximum number of bytes. - * + * * @param sb to overwrite with the line of dump output * @param bb from which to take the bytes * @param n number of bytes to take (up to len) diff --git a/tests/java/org/python/util/jythonTest.java b/tests/java/org/python/util/jythonTest.java index 1cacea30f..1d5fadfbb 100644 --- a/tests/java/org/python/util/jythonTest.java +++ b/tests/java/org/python/util/jythonTest.java @@ -1,88 +1,44 @@ package org.python.util; -import java.lang.reflect.Method; -import java.util.Properties; +import static org.junit.Assert.*; -import org.python.core.PySystemState; - -import junit.framework.TestCase; +import org.junit.Test; +import org.python.core.Console; +import org.python.core.PlainConsole; +import org.python.core.Py; /** - * Tests for creating the right interactive console. + * Tests of creating and getting the right interactive console. + *

+ * System initialisation is a one-time thing normally, and the embedding of a console handler + * similarly, so it is difficult to test more than one console choice in a single executable. For + * this reason, there are two programs like this one: one that follows the native preference for a + * {@link PlainConsole} and this one that induces selection of a {@link JLineConsole}. Other + * features of the JLine console (such as access history) could be tested here. But the test + * Lib/test/test_readline.py does this fairly well, although it has to be run manually. + *

+ * Automated testing of the console seems impossible since, in a scripted context (e.g. as a + * subprocess or under Ant) Jython is no longer interactive. To run it at the prompt, suggested + * idiom is (all one line): + * + * 

+ * java -cp build/exposed;build/classes;extlibs/* -Dpython.home=dist
+ *              org.junit.runner.JUnitCore org.python.util.jythonTest
+ *
*/ -public class jythonTest extends TestCase { - - private static final String PYTHON_CONSOLE = "python.console"; - - private Properties _originalRegistry; - - @Override - protected void setUp() throws Exception { - _originalRegistry = PySystemState.registry; - Properties registry; - if (_originalRegistry != null) { - registry = new Properties(_originalRegistry); - } else { - registry = new Properties(); - } - PySystemState.registry = registry; - } - - @Override - protected void tearDown() throws Exception { - PySystemState.registry = _originalRegistry; - } - - /** - * test the default behavior - * - * @throws Exception - */ - public void testNewInterpreter() throws Exception { - assertEquals(JLineConsole.class, invokeNewInterpreter(true).getClass()); - } +public class jythonTest { - /** - * test registry override - * - * @throws Exception - */ - public void testNewInterpreter_registry() throws Exception { - PySystemState.registry.setProperty(PYTHON_CONSOLE, "org.python.util.InteractiveConsole"); - assertEquals(InteractiveConsole.class, invokeNewInterpreter(true).getClass()); - } - - /** - * test fallback in case of an invalid registry value - * - * @throws Exception - */ - public void testNewInterpreter_unknown() throws Exception { - PySystemState.registry.setProperty(PYTHON_CONSOLE, "foo.bar.NoConsole"); - assertEquals(JLineConsole.class, invokeNewInterpreter(true).getClass()); - } - - /** - * test non-interactive fallback to legacy console - * - * @throws Exception - */ - public void testNewInterpreter_NonInteractive() throws Exception { - assertEquals(InteractiveConsole.class, invokeNewInterpreter(false).getClass()); - } + private static String[] commands = {"-c", "import sys; print type(sys._jy_console)"}; /** - * Invoke the private static method 'newInterpreter(boolean)' on jython.class - * - * @throws Exception + * Test that the default behaviour is to provide a JLineConsole. If CALL_RUN is true, it fails + * under Ant (or Eclipse) as the console is not then recognised to be interactive. */ - private InteractiveConsole invokeNewInterpreter(boolean interactiveStdin) throws Exception { - Method method = jython.class.getDeclaredMethod("newInterpreter", Boolean.TYPE); - assertNotNull(method); - method.setAccessible(true); - Object result = method.invoke(null, interactiveStdin); - assertNotNull(result); - assertTrue(result instanceof InteractiveConsole); - return (InteractiveConsole)result; + @Test + public void testDefaultConsole() { + // This path only if you changed it to run manually + jython.run(commands); + Console console = Py.getConsole(); + assertEquals(JLineConsole.class, console.getClass()); } }