From jython-checkins at python.org  Sat May 11 21:32:24 2013
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 11 May 2013 21:32:24 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_regression_in_PyBufferT?=
 =?utf-8?q?est_due_to_tightening_bytearray=2Eextend_API=2E?=
Message-ID: <3b7JMr2ShzzMf8@mail.python.org>

http://hg.python.org/jython/rev/96e2e620b894
changeset:   7107:96e2e620b894
parent:      7104:efbbc2b78186
user:        Jeff Allen <ja...py at farowl.co.uk>
date:        Sat Mar 23 11:20:47 2013 +0000
summary:
  Fix regression in PyBufferTest due to tightening bytearray.extend API.

files:
  tests/java/org/python/core/PyBufferTest.java |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/tests/java/org/python/core/PyBufferTest.java b/tests/java/org/python/core/PyBufferTest.java
--- a/tests/java/org/python/core/PyBufferTest.java
+++ b/tests/java/org/python/core/PyBufferTest.java
@@ -850,7 +850,7 @@
             // Size-changing access should succeed
             try {
                 PyByteArray sub = ((PyByteArray)subject);
-                sub.bytearray_extend(Py.One);
+                sub.bytearray_append(Py.Zero);
                 sub.del(sub.__len__() - 1);
             } catch (Exception e) {
                 fail("bytearray unexpectedly locked");

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat May 11 21:32:26 2013
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 11 May 2013 21:32:26 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_A_Java_test_program_that_de?=
 =?utf-8?q?monstrates_Issue_=231972_readline=28=29_hangs_in_a?=
Message-ID: <3b7JMt118pz7Ljh@mail.python.org>

http://hg.python.org/jython/rev/1ac6acf8736e
changeset:   7108:1ac6acf8736e
user:        Jeff Allen <ja...py at farowl.co.uk>
date:        Sat May 11 18:29:19 2013 +0100
summary:
  A Java test program that demonstrates Issue #1972 readline() hangs in a subprocess.
I found this a good way to investigate the issue as I can debug more easily, and examine exactly what bytes are being received by the main and sub-processes. This test fails in 2.5.3 and passes in 2.5.2, 2.5.4rc1 and 2.7b1+. Needs JUnit4.

files:
  tests/java/javatests/Issue1972.java |  845 ++++++++++++++++
  1 files changed, 845 insertions(+), 0 deletions(-)


diff --git a/tests/java/javatests/Issue1972.java b/tests/java/javatests/Issue1972.java
new file mode 100644
--- /dev/null
+++ b/tests/java/javatests/Issue1972.java
@@ -0,0 +1,845 @@
+// Copyright (c)2013 Jython Developers
+package javatests;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.junit.After;
+import org.junit.Test;
+
+/**
+ * Tests investigating issues with readline() first raised in Jython Issue #1972. these involve
+ * sub-process input and output through the console streams. Although the console streams are used,
+ * the JLine console handler is not engaged, as the test {@link #jythonJLineConsole()} verifies. You
+ * could run this as a straight JUnit test, or in various debugging configurations, including remote
+ * debugging of the subprocess.
+ * <p>
+ * This test passes in Jython 2.5.2 and 2.5.4rc1. The test {@link #jythonReadline()} fails with
+ * Jython 2.5.3. The test will fail the first time it is run on a clean build, or after switching
+ * Jython versions (JAR files). This is because it monitors stderr from the subprocess and does not
+ * expect the messages the cache manager produces on a first run.
+ * <p>
+ * The bulk of this program is designed to be run as JUnit tests, but it also has a
+ * {@link #main(String[])} method that echoes <code>System.in</code> onto <code>System.out</code>
+ * either as byte data (characters effectively) or as hexadecimal. The early tests run this as a
+ * subprocess to establish exactly what bytes, in particular exactly what line endings, are received
+ * and emitted by a simple Java subprocess. The later tests run Jython as the subprocess, executing
+ * various command line programs and interactive console commands.
+ * <p>
+ * This was developed on Windows. It tries to abstract away the particular choice of line separator,
+ * so it will run on other platforms, but it hasn't been tested that this was successful.
+ */
+public class Issue1972 {
+
+    /** Set to non-zero port number to enable subprocess debugging in selected tests. */
+    static int DEBUG_PORT = 0; // 8000 or 0
+
+    /** Control the amount of output to the console: 0, 1 or 2. */
+    static int VERBOSE = 2;
+
+    /**
+     * Extra JVM options used when debugging is enabled. <code>DEBUG_PORT</code> will be substituted
+     * for the <code>%d</code> marker in a <code>String.format</code> call. The debugger must attach
+     * to the application after it is launched by this test programme.
+     * */
+    static final String DEBUG_OPTS =
+            "-agentlib:jdwp=transport=dt_socket,server=y,address=%d,suspend=y";
+
+    /** Subprocess created by {@link #setProcJava(String...)} */
+    private Process proc = null;
+
+    /** The <code>stdin</code> of the subprocess as a writable stream. */
+    private OutputStream toProc;
+
+    /** A queue handling the <code>stdout</code> of the subprocess. */
+    private LineQueue inFromProc;
+    /** A queue handling the <code>stderr</code> of the subprocess. */
+    private LineQueue errFromProc;
+
+    @After
+    public void afterEachTest() {
+        if (proc != null) {
+            proc.destroy();
+        }
+        inFromProc = errFromProc = null;
+    }
+
+    static final Properties props = System.getProperties();
+    static final String lineSeparator = props.getProperty("line.separator");
+    static final String pythonHome = props.getProperty("python.home");
+
+    /**
+     * Check that on this system we know how to launch and read the error output from a subprocess.
+     * 
+     * @throws IOException
+     */
+    @Test
+    public void readStderr() throws Exception {
+        announceTest(VERBOSE, "readStderr");
+
+        // Run java -version, which outputs on System.err
+        setProcJava("-version");
+        proc.waitFor();
+
+        // Dump to console
+        outputAsHexDump(VERBOSE, inFromProc, errFromProc);
+
+        assertEquals("Unexpected text in stdout", 0, inFromProc.size());
+        assertTrue("No text output to stderr", errFromProc.size() > 0);
+        String res = errFromProc.asStrings().get(0);
+        assertTrue("stderr should mention version", res.contains("version"));
+    }
+
+    /**
+     * Check that on this system we know how to launch and read standard output from a subprocess.
+     * 
+     * @throws IOException
+     */
+    @Test
+    public void readStdout() throws Exception {
+        announceTest(VERBOSE, "readStdout");
+
+        // Run the main of this class
+        setProcJava(this.getClass().getName());
+        proc.waitFor();
+
+        outputAsHexDump(VERBOSE, inFromProc, errFromProc);
+
+        checkErrFromProc();
+        checkInFromProc("Hello");
+    }
+
+    /**
+     * Check that on this system we know how to launch, write to and read from a subprocess.
+     * 
+     * @throws IOException
+     */
+    @Test
+    public void echoStdin() throws Exception {
+        announceTest(VERBOSE, "echoStdin");
+
+        // Run the main of this class as an echo programme
+        setProcJava(this.getClass().getName(), "echo");
+
+        writeToProc("spam");
+        writeToProc("spam\r");
+        writeToProc("spam\r\n");
+        toProc.close();
+        proc.waitFor();
+
+        outputAsHexDump(VERBOSE, inFromProc, errFromProc);
+
+        checkErrFromProc();
+        checkInFromProc(false, "Hello\\r\\n", "spamspam\\r", "spam\\r\\n");
+    }
+
+    /**
+     * Check that on this system line endings are received as expected by a subprocess.
+     * <p>
+     * <b>Observation from Windows 7 x64:</b> data is written to the subprocess once
+     * <code>flush()</code> is called on the output stream. It can be read from
+     * <code>System.in</code> in the subprocess, which of course writes hex to
+     * <code>System.out</code> but that data is not received back in the parent process until
+     * <code>System.out.println()</code> is called in the subprocess.
+     * 
+     * @throws IOException
+     */
+    @Test
+    public void echoStdinAsHex() throws Exception {
+        announceTest(VERBOSE, "echoStdinAsHex");
+
+        // Run the main of this class as an echo programme
+        setProcJava(this.getClass().getName(), "hex");
+
+        writeToProc("a\r");
+        writeToProc("b\n");
+        writeToProc("c\r\n");
+        toProc.close();
+        proc.waitFor();
+
+        outputAsStrings(VERBOSE, inFromProc, errFromProc);
+
+        checkErrFromProc();
+        checkInFromProc("Hello", " 61", " 0d", " 62", " 0a", " 63", " 0d", " 0a");
+    }
+
+    /**
+     * Test reading back from Jython subprocess with program on command-line.
+     * 
+     * @throws Exception
+     */
+    @Test
+    public void jythonSubprocess() throws Exception {
+        announceTest(VERBOSE, "jythonSubprocess");
+
+        // Run Jython hello programme
+        setProcJava("org.python.util.jython", "-c", "print 'Hello'");
+        proc.waitFor();
+
+        outputAsHexDump(VERBOSE, inFromProc, errFromProc);
+
+        checkErrFromProc();
+        checkInFromProc("Hello");
+    }
+
+    /**
+     * Discover what is handling the "console" when the program is on the command line only.
+     * 
+     * @throws Exception
+     */
+    @Test
+    public void jythonNonInteractiveConsole() throws Exception {
+        announceTest(VERBOSE, "jythonNonInteractiveConsole");
+
+        // Run Jython enquiry about console as -c program
+        setProcJava("org.python.util.jython", "-c",
+                "import sys; print type(sys._jy_interpreter).__name__; print sys.stdin.isatty()");
+        proc.waitFor();
+
+        outputAsStrings(VERBOSE, inFromProc, errFromProc);
+
+        checkErrFromProc();
+        checkInFromProc("InteractiveConsole", "False");
+    }
+
+    /**
+     * Discover what is handling the "console" when the program is entered interactively at the
+     * Jython prompt.
+     * 
+     * @throws Exception
+     */
+    @Test
+    public void jythonInteractiveConsole() throws Exception {
+        announceTest(VERBOSE, "jythonInteractiveConsole");
+
+        // Run Jython with simple actions at the command prompt
+        setProcJava(    //
+                "-Dpython.console=org.python.util.InteractiveConsole", //
+                "-Dpython.home=" + pythonHome, //
+                "org.python.util.jython");
+
+        writeToProc("12+3\n");
+        writeToProc("import sys\n");
+        writeToProc("print type(sys._jy_interpreter).__name__\n");
+        writeToProc("print sys.stdin.isatty()\n");
+        toProc.close();
+        proc.waitFor();
+
+        outputAsStrings(VERBOSE, inFromProc, errFromProc);
+
+        checkErrFromProc("");   // stderr produces one empty line. Why?
+        checkInFromProc("15", "InteractiveConsole", "False");
+    }
+
+    /**
+     * Discover what is handling the "console" when the program is entered interactively at the
+     * Jython prompt, and we try to force use of JLine (which fails).
+     * 
+     * @throws Exception
+     */
+    @Test
+    public void jythonJLineConsole() throws Exception {
+        announceTest(VERBOSE, "jythonJLineConsole");
+
+        // Run Jython with simple actions at the command prompt
+        setProcJava(    //
+                "-Dpython.console=org.python.util.JLineConsole", //
+                "-Dpython.home=" + pythonHome, //
+                "org.python.util.jython");
+
+        writeToProc("12+3\n");
+        writeToProc("import sys\n");
+        writeToProc("print type(sys._jy_interpreter).__name__\n");
+        writeToProc("print sys.stdin.isatty()\n");
+        toProc.close();
+        proc.waitFor();
+
+        outputAsStrings(VERBOSE, inFromProc, errFromProc);
+
+        checkErrFromProc("");   // stderr produces one empty line. Why?
+
+        // Although we asked for it, a subprocess doesn't get JLine, and isatty() is false
+        checkInFromProc("15", "InteractiveConsole", "False");
+    }
+
+    /**
+     * Test writing to and reading back from Jython subprocess with echo program on command-line.
+     * 
+     * @throws Exception
+     */
+    @Test
+    public void jythonReadline() throws Exception {
+        announceTest(VERBOSE, "jythonReadline");
+
+        // Run Jython simple readline programme
+        setProcJava( //
+                "org.python.util.jython", //
+                "-c", //
+                "import sys; sys.stdout.write(sys.stdin.readline()); sys.stdout.flush();" //
+        );
+
+        // Discard first output (banner or debugging sign-on)
+        inFromProc.clear();
+        errFromProc.clear();
+
+        // Force lines in until something comes out or it breaks
+        String spamString = "spam" + lineSeparator;
+        byte[] spam = (spamString).getBytes();
+        int count, limit = 9000;
+        for (count = 0; count <= limit; count += spam.length) {
+            toProc.write(spam);
+            toProc.flush();
+            // Give the sub-process a chance the first time and the last
+            if (count == 0 || count + spam.length > limit) {
+                Thread.sleep(10000);
+            }
+            // If anything came back, we're done
+            if (inFromProc.size() > 0) {
+                break;
+            }
+            if (errFromProc.size() > 0) {
+                break;
+            }
+        }
+
+        if (VERBOSE >= 1) {
+            System.out.println(String.format("  count = %4d", count));
+        }
+
+        toProc.close();
+        proc.waitFor();
+        outputAsHexDump(VERBOSE, inFromProc, errFromProc);
+
+        assertTrue("Subprocess did not respond promptly to first line", count == 0);
+        checkInFromProc("spam");
+    }
+
+    /**
+     * A main program that certain tests in the module will use as a subprocess. If an argument is
+     * given it means:
+     * <table>
+     * <tr>
+     * <td>"echo"</td>
+     * <td>echo the characters received as text</td>
+     * </tr>
+     * <tr>
+     * <td>"hex"</td>
+     * <td>echo the characters as hexadecimal</td>
+     * </tr>
+     * </table>
+     * 
+     * @param args
+     * @throws IOException
+     */
+    public static void main(String args[]) throws IOException {
+
+        System.out.println("Hello");
+
+        if (args.length > 0) {
+            String arg = args[0];
+
+            if ("echo".equals(arg)) {
+                int c;
+                while ((c = System.in.read()) != -1) {
+                    System.out.write(c);
+                    System.out.flush();
+                }
+
+            } else if ("hex".equals(arg)) {
+                int c;
+                while ((c = System.in.read()) != -1) {
+                    System.out.printf(" %02x", c);
+                    System.out.println();
+                }
+
+            } else {
+                System.err.println("Huh?");
+            }
+        }
+
+    }
+
+    /**
+     * Invoke the java command with the given arguments. The class path will be the same as this
+     * programme's class path (as in the property <code>java.class.path</code>).
+     * 
+     * @param args further arguments to the program run
+     * @return the running process
+     * @throws IOException
+     */
+    static Process startJavaProcess(String... args) throws IOException {
+
+        // Prepare arguments for the java command
+        String javaClassPath = props.getProperty("java.class.path");
+
+        List<String> cmd = new ArrayList<String>();
+        cmd.add("java");
+        cmd.add("-classpath");
+        cmd.add(javaClassPath);
+        if (DEBUG_PORT > 0) {
+            cmd.add(String.format(DEBUG_OPTS, DEBUG_PORT));
+        }
+        for (String arg : args) {
+            cmd.add(arg);
+        }
+
+        // Create the factory for the external process with the given command
+        ProcessBuilder pb = new ProcessBuilder();
+        pb.command(cmd);
+
+        // If you want to check environment variables, it looks like this:
+        /*
+         * Map<String, String> env = pb.environment(); for (Map.Entry<String, String> entry :
+         * env.entrySet()) { System.out.println(entry); }
+         */
+
+        // Actually create the external process and return the Java object representing it
+        return pb.start();
+    }
+
+    /**
+     * Invoke the java command with the given arguments. The class path will be the same as this
+     * programme's class path (as in the property <code>java.class.path</code>). After the call,
+     * {@link #proc} references the running process and {@link #inFromProc} and {@link #errFromProc}
+     * are handling the <code>stdout</code> and <code>stderr</code> of the subprocess.
+     * 
+     * @param args further arguments to the program run
+     * @throws IOException
+     */
+    private void setProcJava(String... args) throws IOException {
+        proc = startJavaProcess(args);
+        inFromProc = new LineQueue(proc.getInputStream());
+        errFromProc = new LineQueue(proc.getErrorStream());
+        toProc = proc.getOutputStream();
+    }
+
+    /**
+     * Write this string into the <code>stdin</code> of the subprocess. The platform default
+     * encoding will be used.
+     * 
+     * @param s to write
+     * @throws IOException
+     */
+    private void writeToProc(String s) throws IOException {
+        toProc.write(s.getBytes());
+        toProc.flush();
+    }
+
+    /**
+     * Check lines of {@link #queue} against expected text. Lines from the subprocess, after the
+     * {@link #escape(byte[])} transormation has been applied, are expected to be equal to the
+     * strings supplied, optionally after {@link #escapedSeparator} has been added to the expected
+     * strings.
+     * 
+     * @param message identifies the queue in error message
+     * @param addSeparator if true, system-defined line separator expected
+     * @param queue to be compared
+     * @param expected lines of text (given without line separators)
+     */
+    private void checkFromProc(String message, boolean addSeparator, LineQueue queue,
+            String... expected) {
+
+        if (addSeparator) {
+            // Each expected string must be treated as if the lineSeparator were appended
+            String escapedSeparator = "";
+            try {
+                escapedSeparator = escape(lineSeparator.getBytes("US-ASCII"));
+            } catch (UnsupportedEncodingException e) {
+                fail("Could not encode line separator as ASCII"); // Never happens
+            }
+            // ... so append one
+            for (int i = 0; i < expected.length; i++) {
+                expected[i] += escapedSeparator;
+            }
+        }
+
+        // Get the escaped form of the byte buffers in the queue
+        List<String> results = queue.asStrings();
+
+        // Count through the results, stopping when either results or expected strings run out
+        int count = 0;
+        for (String r : results) {
+            if (count < expected.length) {
+                assertEquals(message, expected[count++], r);
+            } else {
+                break;
+            }
+        }
+        assertEquals(message, expected.length, results.size());
+    }
+
+    /**
+     * Check lines of {@link #inFromProc} against expected text.
+     * 
+     * @param addSeparator if true, system-defined line separator expected
+     * @param expected lines of text (given without line separators)
+     */
+    private void checkInFromProc(boolean addSeparator, String... expected) {
+        checkFromProc("subprocess stdout", addSeparator, inFromProc, expected);
+    }
+
+    /**
+     * Check lines of {@link #inFromProc} against expected text. Lines from the subprocess are
+     * expected to be equal to those supplied after {@link #escapedSeparator} has been added.
+     * 
+     * @param expected lines of text (given without line separators)
+     */
+    private void checkInFromProc(String... expected) {
+        checkInFromProc(true, expected);
+    }
+
+    /**
+     * Check lines of {@link #errFromProc} against expected text.
+     * 
+     * @param addSeparator if true, system-defined line separator expected
+     * @param expected lines of text (given without line separators)
+     */
+    private void checkErrFromProc(boolean addSeparator, String... expected) {
+        checkFromProc("subprocess stderr", addSeparator, errFromProc, expected);
+    }
+
+    /**
+     * Check lines of {@link #errFromProc} against expected text. Lines from the subprocess are
+     * expected to be equal to those supplied after {@link #escapedSeparator} has been added.
+     * 
+     * @param expected lines of text (given without line separators)
+     */
+    private void checkErrFromProc(String... expected) {
+        checkErrFromProc(true, expected);
+    }
+
+    /**
+     * Brevity for announcing tests on the console when that is used to dump values.
+     * 
+     * @param verbose if <1 suppress output
+     * @param name of test
+     */
+    static void announceTest(int verbose, String name) {
+        if (verbose >= 1) {
+            System.out.println(String.format("------- Test: %-40s -------", name));
+        }
+    }
+
+    /**
+     * Output is System.out the formatted strings representing lines from a subprocess stdout.
+     * 
+     * @param verbose if <2 suppress output
+     * @param inFromProc lines received from the stdout of a subprocess
+     */
+    static void outputAsStrings(int verbose, LineQueue inFromProc) {
+        if (verbose >= 2) {
+            outputStreams(inFromProc.asStrings(), null);
+        }
+    }
+
+    /**
+     * Output is System.out the formatted strings representing lines from a subprocess stdout, and
+     * if there are any, from stderr.
+     * 
+     * @param verbose if <2 suppress output
+     * @param inFromProc lines received from the stdout of a subprocess
+     * @param errFromProc lines received from the stderr of a subprocess
+     */
+    static void outputAsStrings(int verbose, LineQueue inFromProc, LineQueue errFromProc) {
+        if (verbose >= 2) {
+            outputStreams(inFromProc.asStrings(), errFromProc.asStrings());
+        }
+    }
+
+    /**
+     * Output is System.out a hex dump of lines from a subprocess stdout.
+     * 
+     * @param verbose if <2 suppress output
+     * @param inFromProc lines received from the stdout of a subprocess
+     */
+    static void outputAsHexDump(int verbose, LineQueue inFromProc) {
+        if (verbose >= 2) {
+            outputStreams(inFromProc.asHexDump(), null);
+        }
+    }
+
+    /**
+     * Output is System.out a hex dump of lines from a subprocess stdout, and if there are any, from
+     * stderr.
+     * 
+     * @param verbose if <2 suppress output
+     * @param inFromProc lines received from the stdout of a subprocess
+     * @param errFromProc lines received from the stderr of a subprocess
+     */
+    static void outputAsHexDump(int verbose, LineQueue inFromProc, LineQueue errFromProc) {
+        if (verbose >= 2) {
+            outputStreams(inFromProc.asHexDump(), errFromProc.asHexDump());
+        }
+    }
+
+    /**
+     * Output is System.out the formatted strings representing lines from a subprocess stdout, and
+     * if there are any, from stderr.
+     * 
+     * @param stdout to output labelled "Output stream:"
+     * @param stderr to output labelled "Error stream:" unless an empty list or null
+     */
+    private static void outputStreams(List<String> stdout, List<String> stderr) {
+
+        PrintStream out = System.out;
+
+        out.println("Output stream:");
+        for (String line : stdout) {
+            out.println(line);
+        }
+
+        if (stderr != null && stderr.size() > 0) {
+            out.println("Error stream:");
+            for (String line : stderr) {
+                out.println(line);
+            }
+        }
+    }
+
+    private static final String ESC_CHARS = "\r\n\t\\\b\f";
+    private static final String[] ESC_STRINGS = {"\\r", "\\n", "\\t", "\\\\", "\\b", "\\f"};
+
+    /**
+     * Helper to format one line of string output hex-escaping non-ASCII characters.
+     * 
+     * @param sb to overwrite with the line of dump output
+     * @param bb from which to take the bytes
+     */
+    private static void stringDump(StringBuilder sb, ByteBuffer bb) {
+
+        // Reset the string buffer
+        sb.setLength(0);
+        int n = bb.remaining();
+
+        for (int i = 0; i < n; i++) {
+            // Read byte as character code (old-style ascii mindset at work here)
+            char c = (char)(0xff & bb.get());
+
+            // Check for C-style escape characters
+            int j = ESC_CHARS.indexOf(c);
+            if (j >= 0) {
+                // Use replacement escape sequence
+                sb.append(ESC_STRINGS[j]);
+
+            } else if (c < ' ' || c > 126) {
+                // Some non-printing character that doesn't have an escape
+                sb.append(String.format("\\x%02x", c));
+
+            } else {
+                // A safe character
+                sb.append(c);
+            }
+        }
+
+    }
+
+    /**
+     * Convert bytes (interpreted as ASCII) to String where the non-ascii characters are escaped.
+     * 
+     * @param b
+     * @return
+     */
+    public static String escape(byte[] b) {
+        StringBuilder sb = new StringBuilder(100);
+        ByteBuffer bb = ByteBuffer.wrap(b);
+        stringDump(sb, bb);
+        return sb.toString();
+    }
+
+    /**
+     * Wrapper for an InputStream that creates a thread to read it in the background into a queue of
+     * ByteBuffer objects. Line endings (\r, \n or \r\n) are preserved. This is used in the tests to
+     * see exactly what a subprocess produces, without blocking the subprocess as it writes. The
+     * data are available as a hexadecimal dump (a bit like <code>od</code>) and as string, assuming
+     * a UTF-8 encoding, or some subset like ASCII.
+     */
+    static class LineQueue extends LinkedBlockingQueue<ByteBuffer> {
+
+        static final int BUFFER_SIZE = 1024;
+
+        private InputStream in;
+        ByteBuffer buf;
+        boolean seenCR;
+
+        Thread scribe;
+
+        /**
+         * Wrap a stream in the reader and immediately begin reading it.
+         * 
+         * @param in
+         */
+        LineQueue(InputStream in) {
+            this.in = in;
+
+            scribe = new Thread() {
+
+                @Override
+                public void run() {
+                    try {
+                        runScribe();
+                    } catch (IOException e) {
+                        e.printStackTrace();
+                    }
+                }
+
+            };
+            // Set the scribe thread off filling buffers
+            scribe.start();
+        }
+
+        /**
+         * Scan every byte read from the input and squirrel them away in buffers, one per line,
+         * where lines are delimited by \r, \n or \r\n..
+         * 
+         * @throws IOException
+         */
+        private void runScribe() throws IOException {
+            int c;
+            newBuffer();
+
+            while ((c = in.read()) != -1) {
+
+                byte b = (byte)c;
+
+                if (c == '\n') {
+                    // This is always the end of a line
+                    buf.put(b);
+                    emitBuffer();
+                    newBuffer();
+
+                } else if (seenCR) {
+                    // The line ended just before the new character
+                    emitBuffer();
+                    newBuffer();
+                    buf.put(b);
+
+                } else if (c == '\r') {
+                    // This may be the end of a line (if next is not '\n')
+                    buf.put(b);
+                    seenCR = true;
+
+                } else {
+                    // Not the end of a line, just accumulate
+                    buf.put(b);
+                }
+            }
+
+            // Emit a partial line if there is one.
+            if (buf.position() > 0) {
+                emitBuffer();
+            }
+        }
+
+        private void newBuffer() {
+            buf = ByteBuffer.allocate(BUFFER_SIZE);
+            seenCR = false;
+        }
+
+        private void emitBuffer() {
+            buf.flip();
+            add(buf);
+        }
+
+        /**
+         * Return the contents of the queue as a list of escaped strings, interpreting the bytes as
+         * ASCII.
+         * 
+         * @return contents as strings
+         */
+        public List<String> asStrings() {
+
+            // Make strings here:
+            StringBuilder sb = new StringBuilder(100);
+
+            // Build a list of decoded buffers
+            List<String> list = new LinkedList<String>();
+            for (ByteBuffer bb : this) {
+                stringDump(sb, bb);
+                list.add(sb.toString());
+                bb.rewind();
+            }
+            return list;
+        }
+
+        /**
+         * Return a hex dump the contents of the object as a list of strings
+         * 
+         * @return dump as strings
+         */
+        public List<String> asHexDump() {
+            final int LEN = 16;
+            StringBuilder sb = new StringBuilder(4 * LEN + 20);
+            // Build a list of dumped buffer rows
+            List<String> list = new LinkedList<String>();
+            for (ByteBuffer bb : this) {
+                int n;
+                while ((n = bb.remaining()) >= LEN) {
+                    hexDump(sb, bb, n, LEN);
+                    list.add(sb.toString());
+                }
+                if (n > 0) {
+                    hexDump(sb, bb, n, LEN);
+                    list.add(sb.toString());
+                }
+                bb.rewind();
+            }
+            return list;
+        }
+
+        /**
+         * Helper to format one line of hex dump output up to a maximum number of bytes.
+         * 
+         * @param sb to overwrite with the line of dump output
+         * @param bb from which to take the bytes
+         * @param n number of bytes to take (up to <code>len</code>)
+         * @param len maximum number of bytes to take
+         */
+        private static void hexDump(StringBuilder sb, ByteBuffer bb, int n, int len) {
+
+            // Reset the string buffer
+            sb.setLength(0);
+
+            // Impose the limit
+            if (n > len) {
+                n = len;
+            }
+
+            // The data on this row of output start here in the ByteBuffer
+            bb.mark();
+            sb.append(String.format("%4d: ", bb.position()));
+
+            // Output n of them
+            for (int i = 0; i < n; i++) {
+                sb.append(String.format(" %02x", bb.get()));
+            }
+
+            // And make it up to the proper width
+            for (int i = n; i < len; i++) {
+                sb.append("   ");
+            }
+
+            // Now go back to the start of the row and output printable characters
+            bb.reset();
+            sb.append("|");
+            for (int i = 0; i < n; i++) {
+                char c = (char)(0xff & bb.get());
+                if (c < ' ' || c > 126) {
+                    c = '.';
+                }
+                sb.append(c);
+            }
+        }
+    }
+
+}

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat May 11 21:32:27 2013
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 11 May 2013 21:32:27 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython_=28merge_default_-=3E_default?=
	=?utf-8?q?=29=3A_Merge?=
Message-ID: <3b7JMv6k03z7LjP@mail.python.org>

http://hg.python.org/jython/rev/aa079dc20555
changeset:   7109:aa079dc20555
parent:      7106:11776cd9765b
parent:      7108:1ac6acf8736e
user:        Jeff Allen <ja...py at farowl.co.uk>
date:        Sat May 11 19:28:25 2013 +0100
summary:
  Merge

files:
  tests/java/javatests/Issue1972.java          |  845 ++++++++++
  tests/java/org/python/core/PyBufferTest.java |    2 +-
  2 files changed, 846 insertions(+), 1 deletions(-)


diff --git a/tests/java/javatests/Issue1972.java b/tests/java/javatests/Issue1972.java
new file mode 100644
--- /dev/null
+++ b/tests/java/javatests/Issue1972.java
@@ -0,0 +1,845 @@
+// Copyright (c)2013 Jython Developers
+package javatests;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.junit.After;
+import org.junit.Test;
+
+/**
+ * Tests investigating issues with readline() first raised in Jython Issue #1972. these involve
+ * sub-process input and output through the console streams. Although the console streams are used,
+ * the JLine console handler is not engaged, as the test {@link #jythonJLineConsole()} verifies. You
+ * could run this as a straight JUnit test, or in various debugging configurations, including remote
+ * debugging of the subprocess.
+ * <p>
+ * This test passes in Jython 2.5.2 and 2.5.4rc1. The test {@link #jythonReadline()} fails with
+ * Jython 2.5.3. The test will fail the first time it is run on a clean build, or after switching
+ * Jython versions (JAR files). This is because it monitors stderr from the subprocess and does not
+ * expect the messages the cache manager produces on a first run.
+ * <p>
+ * The bulk of this program is designed to be run as JUnit tests, but it also has a
+ * {@link #main(String[])} method that echoes <code>System.in</code> onto <code>System.out</code>
+ * either as byte data (characters effectively) or as hexadecimal. The early tests run this as a
+ * subprocess to establish exactly what bytes, in particular exactly what line endings, are received
+ * and emitted by a simple Java subprocess. The later tests run Jython as the subprocess, executing
+ * various command line programs and interactive console commands.
+ * <p>
+ * This was developed on Windows. It tries to abstract away the particular choice of line separator,
+ * so it will run on other platforms, but it hasn't been tested that this was successful.
+ */
+public class Issue1972 {
+
+    /** Set to non-zero port number to enable subprocess debugging in selected tests. */
+    static int DEBUG_PORT = 0; // 8000 or 0
+
+    /** Control the amount of output to the console: 0, 1 or 2. */
+    static int VERBOSE = 2;
+
+    /**
+     * Extra JVM options used when debugging is enabled. <code>DEBUG_PORT</code> will be substituted
+     * for the <code>%d</code> marker in a <code>String.format</code> call. The debugger must attach
+     * to the application after it is launched by this test programme.
+     * */
+    static final String DEBUG_OPTS =
+            "-agentlib:jdwp=transport=dt_socket,server=y,address=%d,suspend=y";
+
+    /** Subprocess created by {@link #setProcJava(String...)} */
+    private Process proc = null;
+
+    /** The <code>stdin</code> of the subprocess as a writable stream. */
+    private OutputStream toProc;
+
+    /** A queue handling the <code>stdout</code> of the subprocess. */
+    private LineQueue inFromProc;
+    /** A queue handling the <code>stderr</code> of the subprocess. */
+    private LineQueue errFromProc;
+
+    @After
+    public void afterEachTest() {
+        if (proc != null) {
+            proc.destroy();
+        }
+        inFromProc = errFromProc = null;
+    }
+
+    static final Properties props = System.getProperties();
+    static final String lineSeparator = props.getProperty("line.separator");
+    static final String pythonHome = props.getProperty("python.home");
+
+    /**
+     * Check that on this system we know how to launch and read the error output from a subprocess.
+     * 
+     * @throws IOException
+     */
+    @Test
+    public void readStderr() throws Exception {
+        announceTest(VERBOSE, "readStderr");
+
+        // Run java -version, which outputs on System.err
+        setProcJava("-version");
+        proc.waitFor();
+
+        // Dump to console
+        outputAsHexDump(VERBOSE, inFromProc, errFromProc);
+
+        assertEquals("Unexpected text in stdout", 0, inFromProc.size());
+        assertTrue("No text output to stderr", errFromProc.size() > 0);
+        String res = errFromProc.asStrings().get(0);
+        assertTrue("stderr should mention version", res.contains("version"));
+    }
+
+    /**
+     * Check that on this system we know how to launch and read standard output from a subprocess.
+     * 
+     * @throws IOException
+     */
+    @Test
+    public void readStdout() throws Exception {
+        announceTest(VERBOSE, "readStdout");
+
+        // Run the main of this class
+        setProcJava(this.getClass().getName());
+        proc.waitFor();
+
+        outputAsHexDump(VERBOSE, inFromProc, errFromProc);
+
+        checkErrFromProc();
+        checkInFromProc("Hello");
+    }
+
+    /**
+     * Check that on this system we know how to launch, write to and read from a subprocess.
+     * 
+     * @throws IOException
+     */
+    @Test
+    public void echoStdin() throws Exception {
+        announceTest(VERBOSE, "echoStdin");
+
+        // Run the main of this class as an echo programme
+        setProcJava(this.getClass().getName(), "echo");
+
+        writeToProc("spam");
+        writeToProc("spam\r");
+        writeToProc("spam\r\n");
+        toProc.close();
+        proc.waitFor();
+
+        outputAsHexDump(VERBOSE, inFromProc, errFromProc);
+
+        checkErrFromProc();
+        checkInFromProc(false, "Hello\\r\\n", "spamspam\\r", "spam\\r\\n");
+    }
+
+    /**
+     * Check that on this system line endings are received as expected by a subprocess.
+     * <p>
+     * <b>Observation from Windows 7 x64:</b> data is written to the subprocess once
+     * <code>flush()</code> is called on the output stream. It can be read from
+     * <code>System.in</code> in the subprocess, which of course writes hex to
+     * <code>System.out</code> but that data is not received back in the parent process until
+     * <code>System.out.println()</code> is called in the subprocess.
+     * 
+     * @throws IOException
+     */
+    @Test
+    public void echoStdinAsHex() throws Exception {
+        announceTest(VERBOSE, "echoStdinAsHex");
+
+        // Run the main of this class as an echo programme
+        setProcJava(this.getClass().getName(), "hex");
+
+        writeToProc("a\r");
+        writeToProc("b\n");
+        writeToProc("c\r\n");
+        toProc.close();
+        proc.waitFor();
+
+        outputAsStrings(VERBOSE, inFromProc, errFromProc);
+
+        checkErrFromProc();
+        checkInFromProc("Hello", " 61", " 0d", " 62", " 0a", " 63", " 0d", " 0a");
+    }
+
+    /**
+     * Test reading back from Jython subprocess with program on command-line.
+     * 
+     * @throws Exception
+     */
+    @Test
+    public void jythonSubprocess() throws Exception {
+        announceTest(VERBOSE, "jythonSubprocess");
+
+        // Run Jython hello programme
+        setProcJava("org.python.util.jython", "-c", "print 'Hello'");
+        proc.waitFor();
+
+        outputAsHexDump(VERBOSE, inFromProc, errFromProc);
+
+        checkErrFromProc();
+        checkInFromProc("Hello");
+    }
+
+    /**
+     * Discover what is handling the "console" when the program is on the command line only.
+     * 
+     * @throws Exception
+     */
+    @Test
+    public void jythonNonInteractiveConsole() throws Exception {
+        announceTest(VERBOSE, "jythonNonInteractiveConsole");
+
+        // Run Jython enquiry about console as -c program
+        setProcJava("org.python.util.jython", "-c",
+                "import sys; print type(sys._jy_interpreter).__name__; print sys.stdin.isatty()");
+        proc.waitFor();
+
+        outputAsStrings(VERBOSE, inFromProc, errFromProc);
+
+        checkErrFromProc();
+        checkInFromProc("InteractiveConsole", "False");
+    }
+
+    /**
+     * Discover what is handling the "console" when the program is entered interactively at the
+     * Jython prompt.
+     * 
+     * @throws Exception
+     */
+    @Test
+    public void jythonInteractiveConsole() throws Exception {
+        announceTest(VERBOSE, "jythonInteractiveConsole");
+
+        // Run Jython with simple actions at the command prompt
+        setProcJava(    //
+                "-Dpython.console=org.python.util.InteractiveConsole", //
+                "-Dpython.home=" + pythonHome, //
+                "org.python.util.jython");
+
+        writeToProc("12+3\n");
+        writeToProc("import sys\n");
+        writeToProc("print type(sys._jy_interpreter).__name__\n");
+        writeToProc("print sys.stdin.isatty()\n");
+        toProc.close();
+        proc.waitFor();
+
+        outputAsStrings(VERBOSE, inFromProc, errFromProc);
+
+        checkErrFromProc("");   // stderr produces one empty line. Why?
+        checkInFromProc("15", "InteractiveConsole", "False");
+    }
+
+    /**
+     * Discover what is handling the "console" when the program is entered interactively at the
+     * Jython prompt, and we try to force use of JLine (which fails).
+     * 
+     * @throws Exception
+     */
+    @Test
+    public void jythonJLineConsole() throws Exception {
+        announceTest(VERBOSE, "jythonJLineConsole");
+
+        // Run Jython with simple actions at the command prompt
+        setProcJava(    //
+                "-Dpython.console=org.python.util.JLineConsole", //
+                "-Dpython.home=" + pythonHome, //
+                "org.python.util.jython");
+
+        writeToProc("12+3\n");
+        writeToProc("import sys\n");
+        writeToProc("print type(sys._jy_interpreter).__name__\n");
+        writeToProc("print sys.stdin.isatty()\n");
+        toProc.close();
+        proc.waitFor();
+
+        outputAsStrings(VERBOSE, inFromProc, errFromProc);
+
+        checkErrFromProc("");   // stderr produces one empty line. Why?
+
+        // Although we asked for it, a subprocess doesn't get JLine, and isatty() is false
+        checkInFromProc("15", "InteractiveConsole", "False");
+    }
+
+    /**
+     * Test writing to and reading back from Jython subprocess with echo program on command-line.
+     * 
+     * @throws Exception
+     */
+    @Test
+    public void jythonReadline() throws Exception {
+        announceTest(VERBOSE, "jythonReadline");
+
+        // Run Jython simple readline programme
+        setProcJava( //
+                "org.python.util.jython", //
+                "-c", //
+                "import sys; sys.stdout.write(sys.stdin.readline()); sys.stdout.flush();" //
+        );
+
+        // Discard first output (banner or debugging sign-on)
+        inFromProc.clear();
+        errFromProc.clear();
+
+        // Force lines in until something comes out or it breaks
+        String spamString = "spam" + lineSeparator;
+        byte[] spam = (spamString).getBytes();
+        int count, limit = 9000;
+        for (count = 0; count <= limit; count += spam.length) {
+            toProc.write(spam);
+            toProc.flush();
+            // Give the sub-process a chance the first time and the last
+            if (count == 0 || count + spam.length > limit) {
+                Thread.sleep(10000);
+            }
+            // If anything came back, we're done
+            if (inFromProc.size() > 0) {
+                break;
+            }
+            if (errFromProc.size() > 0) {
+                break;
+            }
+        }
+
+        if (VERBOSE >= 1) {
+            System.out.println(String.format("  count = %4d", count));
+        }
+
+        toProc.close();
+        proc.waitFor();
+        outputAsHexDump(VERBOSE, inFromProc, errFromProc);
+
+        assertTrue("Subprocess did not respond promptly to first line", count == 0);
+        checkInFromProc("spam");
+    }
+
+    /**
+     * A main program that certain tests in the module will use as a subprocess. If an argument is
+     * given it means:
+     * <table>
+     * <tr>
+     * <td>"echo"</td>
+     * <td>echo the characters received as text</td>
+     * </tr>
+     * <tr>
+     * <td>"hex"</td>
+     * <td>echo the characters as hexadecimal</td>
+     * </tr>
+     * </table>
+     * 
+     * @param args
+     * @throws IOException
+     */
+    public static void main(String args[]) throws IOException {
+
+        System.out.println("Hello");
+
+        if (args.length > 0) {
+            String arg = args[0];
+
+            if ("echo".equals(arg)) {
+                int c;
+                while ((c = System.in.read()) != -1) {
+                    System.out.write(c);
+                    System.out.flush();
+                }
+
+            } else if ("hex".equals(arg)) {
+                int c;
+                while ((c = System.in.read()) != -1) {
+                    System.out.printf(" %02x", c);
+                    System.out.println();
+                }
+
+            } else {
+                System.err.println("Huh?");
+            }
+        }
+
+    }
+
+    /**
+     * Invoke the java command with the given arguments. The class path will be the same as this
+     * programme's class path (as in the property <code>java.class.path</code>).
+     * 
+     * @param args further arguments to the program run
+     * @return the running process
+     * @throws IOException
+     */
+    static Process startJavaProcess(String... args) throws IOException {
+
+        // Prepare arguments for the java command
+        String javaClassPath = props.getProperty("java.class.path");
+
+        List<String> cmd = new ArrayList<String>();
+        cmd.add("java");
+        cmd.add("-classpath");
+        cmd.add(javaClassPath);
+        if (DEBUG_PORT > 0) {
+            cmd.add(String.format(DEBUG_OPTS, DEBUG_PORT));
+        }
+        for (String arg : args) {
+            cmd.add(arg);
+        }
+
+        // Create the factory for the external process with the given command
+        ProcessBuilder pb = new ProcessBuilder();
+        pb.command(cmd);
+
+        // If you want to check environment variables, it looks like this:
+        /*
+         * Map<String, String> env = pb.environment(); for (Map.Entry<String, String> entry :
+         * env.entrySet()) { System.out.println(entry); }
+         */
+
+        // Actually create the external process and return the Java object representing it
+        return pb.start();
+    }
+
+    /**
+     * Invoke the java command with the given arguments. The class path will be the same as this
+     * programme's class path (as in the property <code>java.class.path</code>). After the call,
+     * {@link #proc} references the running process and {@link #inFromProc} and {@link #errFromProc}
+     * are handling the <code>stdout</code> and <code>stderr</code> of the subprocess.
+     * 
+     * @param args further arguments to the program run
+     * @throws IOException
+     */
+    private void setProcJava(String... args) throws IOException {
+        proc = startJavaProcess(args);
+        inFromProc = new LineQueue(proc.getInputStream());
+        errFromProc = new LineQueue(proc.getErrorStream());
+        toProc = proc.getOutputStream();
+    }
+
+    /**
+     * Write this string into the <code>stdin</code> of the subprocess. The platform default
+     * encoding will be used.
+     * 
+     * @param s to write
+     * @throws IOException
+     */
+    private void writeToProc(String s) throws IOException {
+        toProc.write(s.getBytes());
+        toProc.flush();
+    }
+
+    /**
+     * Check lines of {@link #queue} against expected text. Lines from the subprocess, after the
+     * {@link #escape(byte[])} transormation has been applied, are expected to be equal to the
+     * strings supplied, optionally after {@link #escapedSeparator} has been added to the expected
+     * strings.
+     * 
+     * @param message identifies the queue in error message
+     * @param addSeparator if true, system-defined line separator expected
+     * @param queue to be compared
+     * @param expected lines of text (given without line separators)
+     */
+    private void checkFromProc(String message, boolean addSeparator, LineQueue queue,
+            String... expected) {
+
+        if (addSeparator) {
+            // Each expected string must be treated as if the lineSeparator were appended
+            String escapedSeparator = "";
+            try {
+                escapedSeparator = escape(lineSeparator.getBytes("US-ASCII"));
+            } catch (UnsupportedEncodingException e) {
+                fail("Could not encode line separator as ASCII"); // Never happens
+            }
+            // ... so append one
+            for (int i = 0; i < expected.length; i++) {
+                expected[i] += escapedSeparator;
+            }
+        }
+
+        // Get the escaped form of the byte buffers in the queue
+        List<String> results = queue.asStrings();
+
+        // Count through the results, stopping when either results or expected strings run out
+        int count = 0;
+        for (String r : results) {
+            if (count < expected.length) {
+                assertEquals(message, expected[count++], r);
+            } else {
+                break;
+            }
+        }
+        assertEquals(message, expected.length, results.size());
+    }
+
+    /**
+     * Check lines of {@link #inFromProc} against expected text.
+     * 
+     * @param addSeparator if true, system-defined line separator expected
+     * @param expected lines of text (given without line separators)
+     */
+    private void checkInFromProc(boolean addSeparator, String... expected) {
+        checkFromProc("subprocess stdout", addSeparator, inFromProc, expected);
+    }
+
+    /**
+     * Check lines of {@link #inFromProc} against expected text. Lines from the subprocess are
+     * expected to be equal to those supplied after {@link #escapedSeparator} has been added.
+     * 
+     * @param expected lines of text (given without line separators)
+     */
+    private void checkInFromProc(String... expected) {
+        checkInFromProc(true, expected);
+    }
+
+    /**
+     * Check lines of {@link #errFromProc} against expected text.
+     * 
+     * @param addSeparator if true, system-defined line separator expected
+     * @param expected lines of text (given without line separators)
+     */
+    private void checkErrFromProc(boolean addSeparator, String... expected) {
+        checkFromProc("subprocess stderr", addSeparator, errFromProc, expected);
+    }
+
+    /**
+     * Check lines of {@link #errFromProc} against expected text. Lines from the subprocess are
+     * expected to be equal to those supplied after {@link #escapedSeparator} has been added.
+     * 
+     * @param expected lines of text (given without line separators)
+     */
+    private void checkErrFromProc(String... expected) {
+        checkErrFromProc(true, expected);
+    }
+
+    /**
+     * Brevity for announcing tests on the console when that is used to dump values.
+     * 
+     * @param verbose if <1 suppress output
+     * @param name of test
+     */
+    static void announceTest(int verbose, String name) {
+        if (verbose >= 1) {
+            System.out.println(String.format("------- Test: %-40s -------", name));
+        }
+    }
+
+    /**
+     * Output is System.out the formatted strings representing lines from a subprocess stdout.
+     * 
+     * @param verbose if <2 suppress output
+     * @param inFromProc lines received from the stdout of a subprocess
+     */
+    static void outputAsStrings(int verbose, LineQueue inFromProc) {
+        if (verbose >= 2) {
+            outputStreams(inFromProc.asStrings(), null);
+        }
+    }
+
+    /**
+     * Output is System.out the formatted strings representing lines from a subprocess stdout, and
+     * if there are any, from stderr.
+     * 
+     * @param verbose if <2 suppress output
+     * @param inFromProc lines received from the stdout of a subprocess
+     * @param errFromProc lines received from the stderr of a subprocess
+     */
+    static void outputAsStrings(int verbose, LineQueue inFromProc, LineQueue errFromProc) {
+        if (verbose >= 2) {
+            outputStreams(inFromProc.asStrings(), errFromProc.asStrings());
+        }
+    }
+
+    /**
+     * Output is System.out a hex dump of lines from a subprocess stdout.
+     * 
+     * @param verbose if <2 suppress output
+     * @param inFromProc lines received from the stdout of a subprocess
+     */
+    static void outputAsHexDump(int verbose, LineQueue inFromProc) {
+        if (verbose >= 2) {
+            outputStreams(inFromProc.asHexDump(), null);
+        }
+    }
+
+    /**
+     * Output is System.out a hex dump of lines from a subprocess stdout, and if there are any, from
+     * stderr.
+     * 
+     * @param verbose if <2 suppress output
+     * @param inFromProc lines received from the stdout of a subprocess
+     * @param errFromProc lines received from the stderr of a subprocess
+     */
+    static void outputAsHexDump(int verbose, LineQueue inFromProc, LineQueue errFromProc) {
+        if (verbose >= 2) {
+            outputStreams(inFromProc.asHexDump(), errFromProc.asHexDump());
+        }
+    }
+
+    /**
+     * Output is System.out the formatted strings representing lines from a subprocess stdout, and
+     * if there are any, from stderr.
+     * 
+     * @param stdout to output labelled "Output stream:"
+     * @param stderr to output labelled "Error stream:" unless an empty list or null
+     */
+    private static void outputStreams(List<String> stdout, List<String> stderr) {
+
+        PrintStream out = System.out;
+
+        out.println("Output stream:");
+        for (String line : stdout) {
+            out.println(line);
+        }
+
+        if (stderr != null && stderr.size() > 0) {
+            out.println("Error stream:");
+            for (String line : stderr) {
+                out.println(line);
+            }
+        }
+    }
+
+    private static final String ESC_CHARS = "\r\n\t\\\b\f";
+    private static final String[] ESC_STRINGS = {"\\r", "\\n", "\\t", "\\\\", "\\b", "\\f"};
+
+    /**
+     * Helper to format one line of string output hex-escaping non-ASCII characters.
+     * 
+     * @param sb to overwrite with the line of dump output
+     * @param bb from which to take the bytes
+     */
+    private static void stringDump(StringBuilder sb, ByteBuffer bb) {
+
+        // Reset the string buffer
+        sb.setLength(0);
+        int n = bb.remaining();
+
+        for (int i = 0; i < n; i++) {
+            // Read byte as character code (old-style ascii mindset at work here)
+            char c = (char)(0xff & bb.get());
+
+            // Check for C-style escape characters
+            int j = ESC_CHARS.indexOf(c);
+            if (j >= 0) {
+                // Use replacement escape sequence
+                sb.append(ESC_STRINGS[j]);
+
+            } else if (c < ' ' || c > 126) {
+                // Some non-printing character that doesn't have an escape
+                sb.append(String.format("\\x%02x", c));
+
+            } else {
+                // A safe character
+                sb.append(c);
+            }
+        }
+
+    }
+
+    /**
+     * Convert bytes (interpreted as ASCII) to String where the non-ascii characters are escaped.
+     * 
+     * @param b
+     * @return
+     */
+    public static String escape(byte[] b) {
+        StringBuilder sb = new StringBuilder(100);
+        ByteBuffer bb = ByteBuffer.wrap(b);
+        stringDump(sb, bb);
+        return sb.toString();
+    }
+
+    /**
+     * Wrapper for an InputStream that creates a thread to read it in the background into a queue of
+     * ByteBuffer objects. Line endings (\r, \n or \r\n) are preserved. This is used in the tests to
+     * see exactly what a subprocess produces, without blocking the subprocess as it writes. The
+     * data are available as a hexadecimal dump (a bit like <code>od</code>) and as string, assuming
+     * a UTF-8 encoding, or some subset like ASCII.
+     */
+    static class LineQueue extends LinkedBlockingQueue<ByteBuffer> {
+
+        static final int BUFFER_SIZE = 1024;
+
+        private InputStream in;
+        ByteBuffer buf;
+        boolean seenCR;
+
+        Thread scribe;
+
+        /**
+         * Wrap a stream in the reader and immediately begin reading it.
+         * 
+         * @param in
+         */
+        LineQueue(InputStream in) {
+            this.in = in;
+
+            scribe = new Thread() {
+
+                @Override
+                public void run() {
+                    try {
+                        runScribe();
+                    } catch (IOException e) {
+                        e.printStackTrace();
+                    }
+                }
+
+            };
+            // Set the scribe thread off filling buffers
+            scribe.start();
+        }
+
+        /**
+         * Scan every byte read from the input and squirrel them away in buffers, one per line,
+         * where lines are delimited by \r, \n or \r\n..
+         * 
+         * @throws IOException
+         */
+        private void runScribe() throws IOException {
+            int c;
+            newBuffer();
+
+            while ((c = in.read()) != -1) {
+
+                byte b = (byte)c;
+
+                if (c == '\n') {
+                    // This is always the end of a line
+                    buf.put(b);
+                    emitBuffer();
+                    newBuffer();
+
+                } else if (seenCR) {
+                    // The line ended just before the new character
+                    emitBuffer();
+                    newBuffer();
+                    buf.put(b);
+
+                } else if (c == '\r') {
+                    // This may be the end of a line (if next is not '\n')
+                    buf.put(b);
+                    seenCR = true;
+
+                } else {
+                    // Not the end of a line, just accumulate
+                    buf.put(b);
+                }
+            }
+
+            // Emit a partial line if there is one.
+            if (buf.position() > 0) {
+                emitBuffer();
+            }
+        }
+
+        private void newBuffer() {
+            buf = ByteBuffer.allocate(BUFFER_SIZE);
+            seenCR = false;
+        }
+
+        private void emitBuffer() {
+            buf.flip();
+            add(buf);
+        }
+
+        /**
+         * Return the contents of the queue as a list of escaped strings, interpreting the bytes as
+         * ASCII.
+         * 
+         * @return contents as strings
+         */
+        public List<String> asStrings() {
+
+            // Make strings here:
+            StringBuilder sb = new StringBuilder(100);
+
+            // Build a list of decoded buffers
+            List<String> list = new LinkedList<String>();
+            for (ByteBuffer bb : this) {
+                stringDump(sb, bb);
+                list.add(sb.toString());
+                bb.rewind();
+            }
+            return list;
+        }
+
+        /**
+         * Return a hex dump the contents of the object as a list of strings
+         * 
+         * @return dump as strings
+         */
+        public List<String> asHexDump() {
+            final int LEN = 16;
+            StringBuilder sb = new StringBuilder(4 * LEN + 20);
+            // Build a list of dumped buffer rows
+            List<String> list = new LinkedList<String>();
+            for (ByteBuffer bb : this) {
+                int n;
+                while ((n = bb.remaining()) >= LEN) {
+                    hexDump(sb, bb, n, LEN);
+                    list.add(sb.toString());
+                }
+                if (n > 0) {
+                    hexDump(sb, bb, n, LEN);
+                    list.add(sb.toString());
+                }
+                bb.rewind();
+            }
+            return list;
+        }
+
+        /**
+         * Helper to format one line of hex dump output up to a maximum number of bytes.
+         * 
+         * @param sb to overwrite with the line of dump output
+         * @param bb from which to take the bytes
+         * @param n number of bytes to take (up to <code>len</code>)
+         * @param len maximum number of bytes to take
+         */
+        private static void hexDump(StringBuilder sb, ByteBuffer bb, int n, int len) {
+
+            // Reset the string buffer
+            sb.setLength(0);
+
+            // Impose the limit
+            if (n > len) {
+                n = len;
+            }
+
+            // The data on this row of output start here in the ByteBuffer
+            bb.mark();
+            sb.append(String.format("%4d: ", bb.position()));
+
+            // Output n of them
+            for (int i = 0; i < n; i++) {
+                sb.append(String.format(" %02x", bb.get()));
+            }
+
+            // And make it up to the proper width
+            for (int i = n; i < len; i++) {
+                sb.append("   ");
+            }
+
+            // Now go back to the start of the row and output printable characters
+            bb.reset();
+            sb.append("|");
+            for (int i = 0; i < n; i++) {
+                char c = (char)(0xff & bb.get());
+                if (c < ' ' || c > 126) {
+                    c = '.';
+                }
+                sb.append(c);
+            }
+        }
+    }
+
+}
diff --git a/tests/java/org/python/core/PyBufferTest.java b/tests/java/org/python/core/PyBufferTest.java
--- a/tests/java/org/python/core/PyBufferTest.java
+++ b/tests/java/org/python/core/PyBufferTest.java
@@ -850,7 +850,7 @@
             // Size-changing access should succeed
             try {
                 PyByteArray sub = ((PyByteArray)subject);
-                sub.bytearray_extend(Py.One);
+                sub.bytearray_append(Py.Zero);
                 sub.del(sub.__len__() - 1);
             } catch (Exception e) {
                 fail("bytearray unexpectedly locked");

-- 
Repository URL: http://hg.python.org/jython