From jython-checkins at python.org Sun Sep 23 09:57:16 2018 From: jython-checkins at python.org (jeff.allen) Date: Sun, 23 Sep 2018 13:57:16 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Factor_out_some_pre-Python?= =?utf-8?q?_runtime_support_and_implement_PYTHONINSPECT=2E?= Message-ID: <20180923135716.1.011CE6BBD2EAA63C@mg.python.org> https://hg.python.org/jython/rev/561eb05e0ace changeset: 8182:561eb05e0ace user: Jeff Allen date: Sat Sep 22 13:55:44 2018 +0100 summary: Factor out some pre-Python runtime support and implement PYTHONINSPECT. This change implements Py.getenv for access to os.environ, and uses it to so that PYTHONINSPECT may trigger an interactive session sfter a script. At the same time, we separate out some utilities that work before the Python type system is ready. files: Lib/test/test_cmd_line.py | 23 +- src/org/python/core/PrePy.java | 230 +++++++++++++ src/org/python/core/Py.java | 222 +++-------- src/org/python/core/PySystemState.java | 2 +- src/org/python/util/jython.java | 82 +--- 5 files changed, 335 insertions(+), 224 deletions(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -146,8 +146,9 @@ self.assertEqual(err.splitlines().count(b'Unknown option: -a'), 1) self.assertEqual(b'', out) - def test_jython_startup(self): - # Test that the file designated by JYTHONSTARTUP is executed when interactive + def test_python_startup(self): + # Test that the file designated by [PJ]YTHONSTARTUP is executed when interactive. + # Note: this test depends on the -i option forcing Python to treat stdin as interactive. filename = test.test_support.TESTFN self.addCleanup(test.test_support.unlink, filename) with open(filename, "w") as script: @@ -167,6 +168,24 @@ else: check('-i', PYTHONSTARTUP=filename) + @unittest.skipUnless(test.test_support.is_jython, "Requires write to sys.flags.inspect") + def test_python_inspect(self): + # Test that PYTHONINSPECT set during a script causes an interactive session to start. + # Note: this test depends on the -i option forcing Python to treat stdin as interactive, + # and on Jython permitting manipulation of sys.flags.inspect (which CPython won't) + # so that PYTHONINSPECT can have some effect. + filename = test.test_support.TESTFN + self.addCleanup(test.test_support.unlink, filename) + with open(filename, "w") as script: + print >>script, "import sys, os" + print >>script, "sys.flags.inspect = False" + print >>script, "os.environ['PYTHONINSPECT'] = 'whatever'" + print >>script, "print os.environ['PYTHONINSPECT']" + expected = ['whatever', '>>> '] + result = assert_python_ok('-i', filename) + self.assertListEqual(expected, result[1].splitlines()) + + def test_main(): test.test_support.run_unittest(CmdLineTest) test.test_support.reap_children() diff --git a/src/org/python/core/PrePy.java b/src/org/python/core/PrePy.java new file mode 100644 --- /dev/null +++ b/src/org/python/core/PrePy.java @@ -0,0 +1,230 @@ +package org.python.core; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.JarURLConnection; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLConnection; +import java.security.AccessControlException; +import java.util.Properties; + +import jnr.posix.util.Platform; + +/** + * This class is part of the Jython run-time system, and contains only "pre-Python" data and methods + * that may safely be used before the type system is ready. The Jython type system springs into + * existence in response to a program's first use of any {@code PyObject}, for example when creating + * the first interpreter. When preparing an application (from the command line options, say) for + * creation of the first interpreter, it useful to defer type system creation until pre-Python + * configuration is complete. See PEP 432 for further rationale. + *

+ * Creation of the type system may happen as a side effect of referring using (almost) any object + * from a class that statically refers to a {@code PyObject}, for example {@code Py} or + * {@code PySystemState}. The present class is intended to hold utility methods and configuration + * useful in the pre-Python phase. + */ +// Do not refer to any PyObject, Py export or PySystemState in this class. +public class PrePy { + + /** + * Get the System properties if we are allowed to. Configuration values set via + * {@code -Dprop=value} to the java command will be found here. If a security manager prevents + * access, we will return a new (empty) object instead. + * + * @return {@code System} properties or a new {@code Properties} object + */ + public static Properties getSystemProperties() { + try { + return System.getProperties(); + } catch (AccessControlException ace) { + return new Properties(); + } + } + + /** Return {@code true} iff the console is accessible through System.console(). */ + public static boolean haveConsole() { + try { + return System.console() != null; + } catch (SecurityException se) { + return false; + } + } + + /** + * Check whether an input stream is interactive. This emulates CPython + * {@code Py_FdIsInteractive} within the constraints of pure Java. + * + * The input stream is considered ``interactive'' if either + *

    + *
  1. it is {@code System.in} and {@code System.console()} is not {@code null}, or
    2. + *
  2. the {@code -i} flag was given ({@link Options#interactive}={@code true}), and the + * filename associated with it is {@code null} or {@code""} or {@code "???"}.
    3. + *
+ * + * @param fp stream (tested only for {@code System.in}) + * @param filename + * @return true iff thought to be interactive + */ + public static boolean isInteractive(InputStream fp, String filename) { + if (fp == System.in && haveConsole()) { + return true; + } else if (!Options.interactive) { + return false; + } else { + return filename == null || filename.equals("") || filename.equals("???"); + } + } + + /** + * Infers the usual Jython executable name from the position of the jar-file returned by + * {@link #getJarFileName()} by replacing the file name with "bin/jython". This is intended as + * an easy fallback for cases where {@code sys.executable} is {@code None} due to direct + * launching via the java executable. + *

+ * Note that this does not necessarily return the actual executable, but instead infers the + * place where it is usually expected to be. Use {@code sys.executable} to get the actual + * executable (may be {@code None}. + * + * @return usual Jython-executable as absolute path + */ + public static String getDefaultExecutableName() { + return getDefaultBinDir() + File.separator + + (Platform.IS_WINDOWS ? "jython.exe" : "jython"); + } + + /** + * Infers the usual Jython bin-dir from the position of the jar-file returned by + * {@link #getJarFileName()} byr replacing the file name with "bin". This is intended as an easy + * fallback for cases where {@code sys.executable} is {@code null} due to direct launching via + * the java executable. + *

+ * Note that this does not necessarily return the actual bin-directory, but instead infers the + * place where it is usually expected to be. + * + * @return usual Jython bin-dir as absolute path + */ + public static String getDefaultBinDir() { + String jar = _getJarFileName(); + return jar.substring(0, jar.lastIndexOf(File.separatorChar) + 1) + "bin"; + } + + /** + * Utility-method to obtain the name (including absolute path) of the currently used + * jython-jar-file. Usually this is jython.jar, but can also be jython-dev.jar or + * jython-standalone.jar or something custom. + * + * @return the full name of the jar file containing this class, null if not + * available. + */ + public static String getJarFileName() { + String jar = _getJarFileName(); + return jar; + } + + /** + * Utility-method to obtain the name (including absolute path) of the currently used + * jython-jar-file. Usually this is jython.jar, but can also be jython-dev.jar or + * jython-standalone.jar or something custom. + * + * @return the full name of the jar file containing this class, null if not + * available. + */ + public static String _getJarFileName() { + Class thisClass = Py.class; + String fullClassName = thisClass.getName(); + String className = fullClassName.substring(fullClassName.lastIndexOf(".") + 1); + URL url = thisClass.getResource(className + ".class"); + return getJarFileNameFromURL(url); + } + + /** + * Return the path in the file system (as a string) of a JAR located by a URL. Three protocols + * are supported, Java JAR-file protocol, and two JBoss protocols "vfs" and "vfszip". + *

+ * The JAR-file protocol URL, which must be a {@code jar:file:} reference to a contained element + * (that is, it has a "!/" part) is able to identify an actual JAR in a file system that may + * then be opened using {@code jarFile = new JarFile(jarFileName)}. The path to the JAR is + * returned. If the JAR is accessed by another mechanism ({@code http:} say) this will fail. + *

+ * The JBoss URL must be a reference to exactly + * {@code vfs:/org/python/core/PySystemState.class}, or the same thing using the + * {@code vfszip:} protocol, where <JAR> stands for the absolute path to the Jython JAR in + * VFS. There is no "!/" marker: in JBoss VFS a JAR is treated just like a directory and can no + * longer be opened as a JAR. The method essentially just swaps a VFS protocol for the Java + * {@code file:} protocol. The path returned will be correct only if this naive swap is valid. + * + * @param url into the JAR + * @return the file path or {@code null} in the event of a detectable error + */ + public static String getJarFileNameFromURL(URL url) { + URI fileURI = null; + try { + switch (url == null ? "" : url.getProtocol()) { + + case "jar": + // url is jar:file:/some/path/some.jar!/package/with/A.class + if (Platform.IS_WINDOWS) { + // ... or jar:file://host/some/path/some.jar!/package/with/A.class + // ... or jar:file:////host/some/path/some.jar!/package/with/A.class + url = tweakWindowsFileURL(url); + } + URLConnection c = url.openConnection(); + fileURI = ((JarURLConnection) c).getJarFileURL().toURI(); + break; + + case "vfs": + case "vfszip": + // path is /some/path/some-jython.jar/org/python/core/PySystemState.class + String path = url.getPath(); + final String target = ".jar/" + Py.class.getName().replace('.', '/'); + int jarIndex = path.indexOf(target); + if (jarIndex > 0) { + // path contains the target class in a JAR, so make a file URL for it + fileURI = new URL("file:" + path.substring(0, jarIndex + 4)).toURI(); + } + break; + + default: + // Unknown protocol or url==null: fileURI = null + break; + } + } catch (IOException | URISyntaxException | IllegalArgumentException e) { + // Handler cannot open connection or URL is malformed some way: fileURI = null + } + + // The JAR file is now identified in fileURI but needs decoding to a file + return fileURI == null ? null : new File(fileURI).toString(); + } + + /** + * If the argument is a {@code jar:file:} or {@code file:} URL, compensate for a bug in Java's + * construction of URLs affecting {@code java.io.File} and {@code java.net.URLConnection} on + * Windows. This is a helper for {@link #getJarFileNameFromURL(URL)}. + *

+ * This bug bites when a JAR file is at a (Windows) UNC location, and a {@code jar:file:} URL is + * derived from {@code Class.getResource()} as it is in {@link #_getJarFileName()}. When URL is + * supplied to {@link #getJarFileNameFromURL(URL)}, the bug leads to a URI that falsely treats a + * server as an "authority". It subsequently causes an {@code IllegalArgumentException} with the + * message "URI has an authority component" when we try to construct a File. See + * {@link https://bugs.java.com/view_bug.do?bug_id=6360233} ("won't fix"). + * + * @param url Possibly malformed URL + * @return corrected URL + */ + private static URL tweakWindowsFileURL(URL url) throws MalformedURLException { + String urlstr = url.toString(); + int fileIndex = urlstr.indexOf("file://"); // 7 chars + if (fileIndex >= 0) { + // Intended UNC path. If there is no slash following these two, insert "/" here: + int insert = fileIndex + 7; + if (urlstr.length() > insert && urlstr.charAt(insert) != '/') { + url = new URL(urlstr.substring(0, insert) + "//" + urlstr.substring(insert)); + } + } + return url; + } +} diff --git a/src/org/python/core/Py.java b/src/org/python/core/Py.java --- a/src/org/python/core/Py.java +++ b/src/org/python/core/Py.java @@ -16,12 +16,6 @@ import java.io.StreamCorruptedException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import java.net.JarURLConnection; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.net.URLConnection; import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; @@ -43,7 +37,7 @@ import jnr.posix.POSIXFactory; import jnr.posix.util.Platform; -public final class Py { +public final class Py extends PrePy { static class SingletonResolver implements Serializable { @@ -790,6 +784,71 @@ return list; } + /** + * Get the environment variables from {@code os.environ}. Keys and values should be + * {@code PyString}s in the file system encoding, and it may be a {@code dict} but nothing can + * be guaranteed. (Note that in the case of multiple interpreters, the target is in the current + * interpreter's copy of {@code os}.) + * + * @return {@code os.environ} + */ + private static PyObject getEnvironment() { + PyObject os = imp.importName("os", true); + PyObject environ = os.__getattr__("environ"); + return environ; + } + + /** The same as {@code getenv(name, null)}. See {@link #getenv(PyString, PyString)}. */ + public static PyString getenv(PyString name) { + return getenv(name, null); + } + + /** + * Get the value of the environment variable named from {@code os.environ} or return the given + * default value. Empty string values are treated as undefined for this purpose. + * + * @param name of the environment variable. + * @param defaultValue to return if {@code key} is not defined (may be {@code null}. + * @return the corresponding value or defaultValue. + */ + public static PyString getenv(PyString name, PyString defaultValue) { + try { + PyObject value = getEnvironment().__finditem__(name); + if (value == null) { + return defaultValue; + } else { + return value.__str__(); + } + } catch (PyException e) { + // Something is fishy about os.environ, so the name is not defined. + return defaultValue; + } + } + + /** The same as {@code getenv(name, null)}. See {@link #getenv(String, String)}. */ + public static String getenv(String name) { + return getenv(name, null); + } + + /** + * Get the value of the environment variable named from {@code os.environ} or return the given + * default value. This is a convenience wrapper on {@link #getenv(PyString, PyString)} which + * takes care of the fact that environment variables are FS-encoded. + * + * @param name to access in the environment. + * @param defaultValue to return if {@code key} is not defined. + * @return the corresponding value or defaultValue. + */ + public static String getenv(String name, String defaultValue) { + PyString value = getenv(newUnicode(name), null); + if (value == null) { + return defaultValue; + } else { + // Environment variables are FS-encoded byte strings + return fileSystemDecode(value); + } + } + public static PyStringMap newStringMap() { return new PyStringMap(); } @@ -2568,155 +2627,6 @@ return objs.toArray(Py.EmptyObjects); } - /** - * Infers the usual Jython executable name from the position of the jar-file returned by - * {@link #getJarFileName()} by replacing the file name with "bin/jython". This is intended as - * an easy fallback for cases where {@code sys.executable} is {@code None} due to direct - * launching via the java executable. - *

- * Note that this does not necessarily return the actual executable, but instead infers the - * place where it is usually expected to be. Use {@code sys.executable} to get the actual - * executable (may be {@code None}. - * - * @return usual Jython-executable as absolute path - */ - public static String getDefaultExecutableName() { - return getDefaultBinDir() + File.separator - + (Platform.IS_WINDOWS ? "jython.exe" : "jython"); - } - - /** - * Infers the usual Jython bin-dir from the position of the jar-file returned by - * {@link #getJarFileName()} byr replacing the file name with "bin". This is intended as an easy - * fallback for cases where {@code sys.executable} is {@code null} due to direct launching via - * the java executable. - *

- * Note that this does not necessarily return the actual bin-directory, but instead infers the - * place where it is usually expected to be. - * - * @return usual Jython bin-dir as absolute path - */ - public static String getDefaultBinDir() { - String jar = _getJarFileName(); - return jar.substring(0, jar.lastIndexOf(File.separatorChar) + 1) + "bin"; - } - - /** - * Utility-method to obtain the name (including absolute path) of the currently used - * jython-jar-file. Usually this is jython.jar, but can also be jython-dev.jar or - * jython-standalone.jar or something custom. - * - * @return the full name of the jar file containing this class, null if not - * available. - */ - public static String getJarFileName() { - String jar = _getJarFileName(); - return jar; - } - - /** - * Utility-method to obtain the name (including absolute path) of the currently used - * jython-jar-file. Usually this is jython.jar, but can also be jython-dev.jar or - * jython-standalone.jar or something custom. - * - * @return the full name of the jar file containing this class, null if not - * available. - */ - public static String _getJarFileName() { - Class thisClass = Py.class; - String fullClassName = thisClass.getName(); - String className = fullClassName.substring(fullClassName.lastIndexOf(".") + 1); - URL url = thisClass.getResource(className + ".class"); - return getJarFileNameFromURL(url); - } - - /** - * Return the path in the file system (as a string) of a JAR located by a URL. Three protocols - * are supported, Java JAR-file protocol, and two JBoss protocols "vfs" and "vfszip". - *

- * The JAR-file protocol URL, which must be a {@code jar:file:} reference to a contained element - * (that is, it has a "!/" part) is able to identify an actual JAR in a file system that may - * then be opened using {@code jarFile = new JarFile(jarFileName)}. The path to the JAR is - * returned. If the JAR is accessed by another mechanism ({@code http:} say) this will fail. - *

- * The JBoss URL must be a reference to exactly - * {@code vfs:/org/python/core/PySystemState.class}, or the same thing using the - * {@code vfszip:} protocol, where <JAR> stands for the absolute path to the Jython JAR in - * VFS. There is no "!/" marker: in JBoss VFS a JAR is treated just like a directory and can no - * longer be opened as a JAR. The method essentially just swaps a VFS protocol for the Java - * {@code file:} protocol. The path returned will be correct only if this naive swap is valid. - * - * @param url into the JAR - * @return the file path or {@code null} in the event of a detectable error - */ - public static String getJarFileNameFromURL(URL url) { - URI fileURI = null; - try { - switch (url == null ? "" : url.getProtocol()) { - - case "jar": - // url is jar:file:/some/path/some.jar!/package/with/A.class - if (Platform.IS_WINDOWS) { - // ... or jar:file://host/some/path/some.jar!/package/with/A.class - // ... or jar:file:////host/some/path/some.jar!/package/with/A.class - url = tweakWindowsFileURL(url); - } - URLConnection c = url.openConnection(); - fileURI = ((JarURLConnection) c).getJarFileURL().toURI(); - break; - - case "vfs": - case "vfszip": - // path is /some/path/some-jython.jar/org/python/core/PySystemState.class - String path = url.getPath(); - final String target = ".jar/" + Py.class.getName().replace('.', '/'); - int jarIndex = path.indexOf(target); - if (jarIndex > 0) { - // path contains the target class in a JAR, so make a file URL for it - fileURI = new URL("file:" + path.substring(0, jarIndex + 4)).toURI(); - } - break; - - default: - // Unknown protocol or url==null: fileURI = null - break; - } - } catch (IOException | URISyntaxException | IllegalArgumentException e) { - // Handler cannot open connection or URL is malformed some way: fileURI = null - } - - // The JAR file is now identified in fileURI but needs decoding to a file - return fileURI == null ? null : new File(fileURI).toString(); - } - - /** - * If the argument is a {@code jar:file:} or {@code file:} URL, compensate for a bug in Java's - * construction of URLs affecting {@code java.io.File} and {@code java.net.URLConnection} on - * Windows. This is a helper for {@link #getJarFileNameFromURL(URL)}. - *

- * This bug bites when a JAR file is at a (Windows) UNC location, and a {@code jar:file:} URL is - * derived from {@code Class.getResource()} as it is in {@link #_getJarFileName()}. When URL is - * supplied to {@link #getJarFileNameFromURL(URL)}, the bug leads to a URI that falsely treats a - * server as an "authority". It subsequently causes an {@code IllegalArgumentException} with the - * message "URI has an authority component" when we try to construct a File. See - * {@link https://bugs.java.com/view_bug.do?bug_id=6360233} ("won't fix"). - * - * @param url Possibly malformed URL - * @return corrected URL - */ - private static URL tweakWindowsFileURL(URL url) throws MalformedURLException { - String urlstr = url.toString(); - int fileIndex = urlstr.indexOf("file://"); // 7 chars - if (fileIndex >= 0) { - // Intended UNC path. If there is no slash following these two, insert "/" here: - int insert = fileIndex + 7; - if (urlstr.length() > insert && urlstr.charAt(insert) != '/') { - url = new URL(urlstr.substring(0, insert) + "//" + urlstr.substring(insert)); - } - } - return url; - } - //------------------------constructor-section--------------------------- static class py2JyClassCacheItem { List> interfaces; diff --git a/src/org/python/core/PySystemState.java b/src/org/python/core/PySystemState.java --- a/src/org/python/core/PySystemState.java +++ b/src/org/python/core/PySystemState.java @@ -896,7 +896,7 @@ * Command-line definitions {@code -Dkey=value}) * * - * ... preProperties also contain ... + * ... preProperties also contains ... * Environment variables via {@link org.python.util.jython} * * diff --git a/src/org/python/util/jython.java b/src/org/python/util/jython.java --- a/src/org/python/util/jython.java +++ b/src/org/python/util/jython.java @@ -7,7 +7,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; -import java.security.AccessControlException; import java.util.LinkedList; import java.util.List; import java.util.Properties; @@ -19,6 +18,7 @@ import org.python.core.CompileMode; import org.python.core.CompilerFlags; import org.python.core.Options; +import org.python.core.PrePy; import org.python.core.Py; import org.python.core.PyCode; import org.python.core.PyException; @@ -30,7 +30,6 @@ import org.python.core.PyStringMap; import org.python.core.PySystemState; import org.python.core.imp; -import org.python.modules.thread.thread; public class jython { @@ -297,7 +296,7 @@ // Following CPython PyRun_AnyFileExFlags here, blindly, concerning null name. filename = filename != null ? filename : "???"; // Run the contents in the interpreter - if (isInteractive(fp, filename)) { + if (PrePy.isInteractive(fp, filename)) { // __file__ not defined interp.interact(null, new PyFile(fp)); } else { @@ -399,7 +398,7 @@ } // Get system properties (or empty set if we're prevented from accessing them) - Properties preProperties = getSystemProperties(); + Properties preProperties = PrePy.getSystemProperties(); addDefaultsFromEnvironment(preProperties); // Treat the apparent filename "-" as no filename @@ -409,14 +408,14 @@ } // Sense whether the console is interactive, or we have been told to consider it so. - boolean stdinIsInteractive = isInteractive(System.in, null); + boolean stdinIsInteractive = PrePy.isInteractive(System.in, null); // Shorthand boolean haveScript = opts.command != null || opts.filename != null || opts.module != null; if (Options.inspect || !haveScript) { // We'll be going interactive eventually. condition an interactive console. - if (haveConsole()) { + if (PrePy.haveConsole()) { // Set the default console type if nothing else has addDefault(preProperties, "python.console", PYTHON_CONSOLE_CLASS); } @@ -554,9 +553,9 @@ * Check this environment variable at the end, to give programs the opportunity to set it * from Python. */ - // XXX: Java does not let us set environment variables but we could have our own os.environ. if (!Options.inspect) { - Options.inspect = getenv("PYTHONINSPECT") != null; + // If set from Python, the value will be in os.environ, not Java System.getenv. + Options.inspect = Py.getenv("PYTHONINSPECT", "").length() > 0; } if (Options.inspect && stdinIsInteractive && haveScript) { @@ -627,20 +626,22 @@ } } - /** The same as {@code getenv(name, null)} */ + /** The same as {@code getenv(name, null)}. */ private static String getenv(String name) { return getenv(name, null); } /** - * Get the value of an environment variable, if we are allowed to and it is defined; otherwise, - * return the chosen default value. An empty string value is treated as undefined. We are - * allowed to access the environment variable if if the JVM security environment permits and if - * {@link Options#ignore_environment} is {@code false}, which it is by default. It may be set by - * the -E flag given to the launcher, or by program action, or once it turns out we do not have - * permission (saving further work). + * Get the value of an environment variable, respecting {@link Options#ignore_environment} (the + * -E option), or return the given default if the variable is undefined or the security + * environment prevents access. An empty string value from the environment is treated as + * undefined. + *

+ * This accesses the read-only Java copy of the system environment directly, not + * {@code os.environ} so that it is safe to use before Python types are available. * - * @param name to access in the environment. + * @param name to access in the environment (if allowed by + * {@link Options#ignore_environment}=={@code false}). * @param defaultValue to return if {@code name} is not defined or "" or access is forbidden. * @return the corresponding value or defaultValue. */ @@ -672,55 +673,6 @@ } /** - * Check whether an input stream is interactive. This emulates CPython - * {@code Py_FdIsInteractive} within the constraints of pure Java. - * - * The input stream is considered ``interactive'' if either - *

    - *
  1. it is {@code System.in} and {@code System.console()} is not {@code null}, or
    2. - *
  2. the {@code -i} flag was given ({@link Options#interactive}={@code true}), and the - * filename associated with it is {@code null} or {@code""} or {@code "???"}.
    3. - *
- * - * @param fp stream (tested only for {@code System.in}) - * @param filename - * @return true iff thought to be interactive - */ - private static boolean isInteractive(InputStream fp, String filename) { - if (fp == System.in && haveConsole()) { - return true; - } else if (!Options.interactive) { - return false; - } else { - return filename == null || filename.equals("") || filename.equals("???"); - } - } - - /** Return {@code true} iff the console is accessible through System.console(). */ - private static boolean haveConsole() { - try { - return System.console() != null; - } catch (SecurityException se) { - return false; - } - } - - /** - * Get the System properties if we are allowed to. Configuration values set via - * {@code -Dprop=value} to the java command will be found here. If a security manager prevents - * access, we will return a new (empty) object instead. - * - * @return {@code System} properties or a new {@code Properties} object - */ - private static Properties getSystemProperties() { - try { - return System.getProperties(); - } catch (AccessControlException ace) { - return new Properties(); - } - } - - /** * Append strings to a PyList as {@code bytes/str} objects. These might come from the command * line, or any source with the possibility of non-ascii values. * -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Sep 23 09:57:15 2018 From: jython-checkins at python.org (jeff.allen) Date: Sun, 23 Sep 2018 13:57:15 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Retire_SystemRestart=2E?= Message-ID: <20180923135715.1.8DF794D80A3B887E@mg.python.org> https://hg.python.org/jython/rev/32a639ba08ff changeset: 8181:32a639ba08ff user: Jeff Allen date: Thu Sep 20 20:59:52 2018 +0100 summary: Retire SystemRestart. This feature ("experimental" since 2008) has been broken in 2.7.x since 2014. As implemented (when fixed), it still poses technical problems, including an incompatibility with multiple interpreters (sys modules). See Jython-dev. files: Lib/threading.py | 4 - src/org/python/core/FunctionThread.java | 4 +- src/org/python/core/PyBaseCode.java | 22 +- src/org/python/core/PySystemState.java | 15 -- src/org/python/core/PyTableCode.java | 23 +- src/org/python/modules/Setup.java | 1 - src/org/python/modules/_systemrestart.java | 29 ---- src/org/python/modules/thread/thread.java | 16 +- src/org/python/util/jython.java | 72 ++------- tests/java/org/python/util/jythonTest.java | 2 +- tests/java/org/python/util/jythonTestPlain.java | 4 +- 11 files changed, 45 insertions(+), 147 deletions(-) diff --git a/Lib/threading.py b/Lib/threading.py --- a/Lib/threading.py +++ b/Lib/threading.py @@ -222,10 +222,6 @@ self.run() except SystemExit: pass - except InterruptedException: - # Quiet InterruptedExceptions if they're caused by system restart - if not _sys._shouldRestart: - raise except: # If sys.stderr is no more (most likely from interpreter # shutdown) use self.__stderr. Otherwise still use sys (as in diff --git a/src/org/python/core/FunctionThread.java b/src/org/python/core/FunctionThread.java --- a/src/org/python/core/FunctionThread.java +++ b/src/org/python/core/FunctionThread.java @@ -2,8 +2,6 @@ import java.util.concurrent.atomic.AtomicInteger; -import org.python.modules._systemrestart; - public class FunctionThread extends Thread { private final PyObject func; @@ -24,7 +22,7 @@ try { func.__call__(args); } catch (PyException exc) { - if (exc.match(Py.SystemExit) || exc.match(_systemrestart.SystemRestart)) { + if (exc.match(Py.SystemExit)) { return; } Py.stderr.println("Unhandled exception in thread started by " + func); diff --git a/src/org/python/core/PyBaseCode.java b/src/org/python/core/PyBaseCode.java --- a/src/org/python/core/PyBaseCode.java +++ b/src/org/python/core/PyBaseCode.java @@ -4,7 +4,6 @@ */ package org.python.core; -import org.python.modules._systemrestart; import com.google.common.base.CharMatcher; public abstract class PyBaseCode extends PyCode { @@ -103,21 +102,16 @@ ts.exception = previous_exception; ts.frame = ts.frame.f_back; - - // Check for interruption, which is used for restarting the interpreter - // on Jython - if (ts.getSystemState()._systemRestart && Thread.currentThread().isInterrupted()) { - throw new PyException(_systemrestart.SystemRestart); - } return ret; } public PyObject call(ThreadState state, PyObject globals, PyObject[] defaults, PyObject closure) { - if (co_argcount != 0 || varargs || varkwargs) + if (co_argcount != 0 || varargs || varkwargs) { return call(state, Py.EmptyObjects, Py.NoKeywords, globals, defaults, closure); + } PyFrame frame = new PyFrame(this, globals); if (co_flags.isFlagSet(CodeFlag.CO_GENERATOR)) { return new PyGenerator(frame, closure); @@ -128,9 +122,10 @@ public PyObject call(ThreadState state, PyObject arg1, PyObject globals, PyObject[] defaults, PyObject closure) { - if (co_argcount != 1 || varargs || varkwargs) + if (co_argcount != 1 || varargs || varkwargs) { return call(state, new PyObject[] {arg1}, Py.NoKeywords, globals, defaults, closure); + } PyFrame frame = new PyFrame(this, globals); frame.f_fastlocals[0] = arg1; if (co_flags.isFlagSet(CodeFlag.CO_GENERATOR)) { @@ -142,9 +137,10 @@ public PyObject call(ThreadState state, PyObject arg1, PyObject arg2, PyObject globals, PyObject[] defaults, PyObject closure) { - if (co_argcount != 2 || varargs || varkwargs) + if (co_argcount != 2 || varargs || varkwargs) { return call(state, new PyObject[] {arg1, arg2}, Py.NoKeywords, globals, defaults, closure); + } PyFrame frame = new PyFrame(this, globals); frame.f_fastlocals[0] = arg1; frame.f_fastlocals[1] = arg2; @@ -158,9 +154,10 @@ PyObject globals, PyObject[] defaults, PyObject closure) { - if (co_argcount != 3 || varargs || varkwargs) + if (co_argcount != 3 || varargs || varkwargs) { return call(state, new PyObject[] {arg1, arg2, arg3}, Py.NoKeywords, globals, defaults, closure); + } PyFrame frame = new PyFrame(this, globals); frame.f_fastlocals[0] = arg1; frame.f_fastlocals[1] = arg2; @@ -175,9 +172,10 @@ public PyObject call(ThreadState state, PyObject arg1, PyObject arg2, PyObject arg3, PyObject arg4, PyObject globals, PyObject[] defaults, PyObject closure) { - if (co_argcount != 4 || varargs || varkwargs) + if (co_argcount != 4 || varargs || varkwargs) { return call(state, new PyObject[]{arg1, arg2, arg3, arg4}, Py.NoKeywords, globals, defaults, closure); + } PyFrame frame = new PyFrame(this, globals); frame.f_fastlocals[0] = arg1; frame.f_fastlocals[1] = arg2; diff --git a/src/org/python/core/PySystemState.java b/src/org/python/core/PySystemState.java --- a/src/org/python/core/PySystemState.java +++ b/src/org/python/core/PySystemState.java @@ -188,9 +188,6 @@ private codecs.CodecState codecState; - /** true when a SystemRestart is triggered. */ - public boolean _systemRestart = false; - /** Whether bytecode should be written to disk on import. */ public boolean dont_write_bytecode = false; @@ -1244,18 +1241,6 @@ return Py.defaultSystemState; } - /** - * Reset the global static {@code PySytemState} so that a subsequent call to - * {@link #initialize()} will re-create the state. This is only really necessary in the context - * of a system restart, but is harmless when shutting down. Using Python after this call is - * likely to result in an implicit full static initialisation, or fail badly. - */ - public static void undoInitialize() { - Py.defaultSystemState = null; - registry = null; - initialized = false; - } - private static PyVersionInfo getVersionInfo() { String s; if (Version.PY_RELEASE_LEVEL == 0x0A) { diff --git a/src/org/python/core/PyTableCode.java b/src/org/python/core/PyTableCode.java --- a/src/org/python/core/PyTableCode.java +++ b/src/org/python/core/PyTableCode.java @@ -5,9 +5,6 @@ * An implementation of PyCode where the actual executable content * is stored as a PyFunctionTable instance and an integer index. */ - -import org.python.modules._systemrestart; - @Untraversable public class PyTableCode extends PyBaseCode { @@ -69,15 +66,18 @@ @Override public PyObject __dir__() { PyString members[] = new PyString[__members__.length]; - for (int i = 0; i < __members__.length; i++) + for (int i = 0; i < __members__.length; i++) { members[i] = new PyString(__members__[i]); + } return new PyList(members); } private void throwReadonly(String name) { - for (int i = 0; i < __members__.length; i++) - if (__members__[i] == name) + for (int i = 0; i < __members__.length; i++) { + if (__members__[i] == name) { throw Py.TypeError("readonly attribute"); + } + } throw Py.AttributeError(name); } @@ -93,7 +93,9 @@ } private static PyTuple toPyStringTuple(String[] ar) { - if (ar == null) return Py.EmptyTuple; + if (ar == null) { + return Py.EmptyTuple; + } int sz = ar.length; PyString[] pystr = new PyString[sz]; for (int i = 0; i < sz; i++) { @@ -207,14 +209,7 @@ // Restore previously defined exception ts.exception = previous_exception; - ts.frame = ts.frame.f_back; - - // Check for interruption, which is used for restarting the interpreter - // on Jython - if (ts.getSystemState()._systemRestart && Thread.currentThread().isInterrupted()) { - throw new PyException(_systemrestart.SystemRestart); - } return ret; } diff --git a/src/org/python/modules/Setup.java b/src/org/python/modules/Setup.java --- a/src/org/python/modules/Setup.java +++ b/src/org/python/modules/Setup.java @@ -41,7 +41,6 @@ "_py_compile", "_random:org.python.modules.random.RandomModule", "_sre", - "_systemrestart", "_threading:org.python.modules._threading._threading", "_weakref:org.python.modules._weakref.WeakrefModule", "array:org.python.modules.ArrayModule", diff --git a/src/org/python/modules/_systemrestart.java b/src/org/python/modules/_systemrestart.java deleted file mode 100644 --- a/src/org/python/modules/_systemrestart.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.python.modules; - -import org.python.core.ClassDictInit; -import org.python.core.Py; -import org.python.core.PyException; -import org.python.core.PyObject; -import org.python.core.PyStringMap; - -public class _systemrestart implements ClassDictInit { - /** - * Jython-specific exception for restarting the interpreter. Currently - * supported only by jython.java, when executing a file (i.e, - * non-interactive mode). - * - * WARNING: This is highly *experimental* and subject to change. - */ - public static PyObject SystemRestart; - - public static void classDictInit(PyObject dict) { - SystemRestart = Py.makeClass( - "_systemrestart.SystemRestart", Py.BaseException, - new PyStringMap() {{ - __setitem__("__doc__", - Py.newString("Request to restart the interpreter. " + - "(Jython-specific)")); - }}); - dict.__delitem__("classDictInit"); - } -} diff --git a/src/org/python/modules/thread/thread.java b/src/org/python/modules/thread/thread.java --- a/src/org/python/modules/thread/thread.java +++ b/src/org/python/modules/thread/thread.java @@ -61,17 +61,9 @@ } /** - * Interrupts all running threads spawned by the thread module. - * - * This works in conjunction with:
  • - * {@link org.python.core.PyTableCode#call}: checks for the interrupted - * status of the current thread and raise a SystemRestart exception if a - * interruption is detected.
    • - *
  • {@link FunctionThread#run()}: exits the current thread when a - * SystemRestart exception is not caught.
    • - * - * Thus, it is possible that this doesn't make all running threads to stop, - * if SystemRestart exception is caught. + * Interrupt all running threads spawned by the thread module. This works in conjunction with: + * {@link org.python.core.PyTableCode#call}, which checks for the interrupted status of the + * current thread, and {@link FunctionThread#run()}, which exits the current thread. */ public static void interruptAllThreads() { group.interrupt(); @@ -92,7 +84,7 @@ public static long get_ident() { return Thread.currentThread().getId(); } - + public static long stack_size(PyObject[] args) { switch (args.length) { case 0: diff --git a/src/org/python/util/jython.java b/src/org/python/util/jython.java --- a/src/org/python/util/jython.java +++ b/src/org/python/util/jython.java @@ -30,14 +30,13 @@ import org.python.core.PyStringMap; import org.python.core.PySystemState; import org.python.core.imp; -import org.python.modules._systemrestart; import org.python.modules.thread.thread; public class jython { /** Exit status: must have {@code OK.ordinal()==0} */ private enum Status { - OK, ERROR, NOT_RUN, SHOULD_RESTART, NO_FILE + OK, ERROR, NOT_RUN, NO_FILE } // An instance of this class will provide the console (python.console) by default. @@ -163,11 +162,14 @@ return Status.OK.ordinal(); } - public static void main(String[] args) { - Status status; - do { - status = run(args); - } while (status == Status.SHOULD_RESTART); + /** Now equivalent to {@link #main(String[])}, which is to be preferred. */ + @Deprecated + public static void run(String[] args) { + main(args); + } + + /** Exit Jython with status (converted to an integer). */ + private static void exit(Status status) { System.exit(status.ordinal()); } @@ -379,20 +381,19 @@ * {@code (args)} are arguments to the program. * * @param args arguments to the program. - * @return status indicating outcome. */ - public static Status run(String[] args) { + public static void main(String[] args) { // Parse the command line options CommandLineOptions opts = CommandLineOptions.parse(args); switch (opts.action) { case VERSION: System.err.printf("Jython %s\n", Version.PY_VERSION); - return Status.OK; + exit(Status.OK); case HELP: - return usage(Status.OK); + exit(usage(Status.OK)); case ERROR: System.err.println(opts.message); - return usage(Status.ERROR); + exit(usage(Status.ERROR)); case RUN: // Let's run some Python! ... } @@ -543,15 +544,10 @@ } catch (PyException pye) { // Whatever the mode of execution an uncaught PyException lands here. - if (pye.match(_systemrestart.SystemRestart)) { - // Leave ourselves a note to restart. - sts = Status.SHOULD_RESTART; - } else { - // If pye was SystemExit *and* Options.inspect==false, this will exit the JVM: - Py.printException(pye); - // It was an exception other than SystemExit or Options.inspect==true. - sts = Status.ERROR; - } + // If pye was SystemExit *and* Options.inspect==false, this will exit the JVM: + Py.printException(pye); + // It was an exception other than SystemExit or Options.inspect==true. + sts = Status.ERROR; } /* @@ -581,41 +577,9 @@ } } - if (sts == Status.SHOULD_RESTART) { - // Shut down *all* threads and sockets - shutdownInterpreter(); - // ..reset the state... - // XXX seems unnecessary given we will call PySystemState.initialize() - // Py.setSystemState(new PySystemState()); - // ...and start again - } - // Shut down in a tidy way interp.cleanup(); - PySystemState.undoInitialize(); - - return sts; - } - - /** - * Run any finalizations on the current interpreter in preparation for a SytemRestart. - */ - public static void shutdownInterpreter() { - PySystemState sys = Py.getSystemState(); - // Signal to threading.py to modify response to InterruptedException: - sys._systemRestart = true; - // Interrupt (stop?) all the active threads in the jython-threads group. - thread.interruptAllThreads(); - // Close all sockets not already covered by Thread.interrupt (e.g. pre-nio sockets) - try { - PyObject socket = sys.modules.__finditem__("socket"); - if (socket != null) { - // XXX Fossil code. Raises AttributeError as _closeActiveSockets has been deleted. - socket.__getattr__("_closeActiveSockets").__call__(); - } - } catch (PyException pye) { - // don't worry about errors: we're shutting down - } + exit(sts); } /** diff --git a/tests/java/org/python/util/jythonTest.java b/tests/java/org/python/util/jythonTest.java --- a/tests/java/org/python/util/jythonTest.java +++ b/tests/java/org/python/util/jythonTest.java @@ -35,7 +35,7 @@ */ @Test public void testDefaultConsole() { - jython.run(commands); + jython.main(commands); Console console = Py.getConsole(); assertEquals(JLineConsole.class, console.getClass()); } diff --git a/tests/java/org/python/util/jythonTestPlain.java b/tests/java/org/python/util/jythonTestPlain.java --- a/tests/java/org/python/util/jythonTestPlain.java +++ b/tests/java/org/python/util/jythonTestPlain.java @@ -39,7 +39,7 @@ public void testFallbackConsole() { System.out.println("testFallbackConsole"); System.getProperties().setProperty(PYTHON_CONSOLE, "org.python.util.InteractiveConsole"); - jython.run(commands); + jython.main(commands); Console console = Py.getConsole(); assertEquals(PlainConsole.class, console.getClass()); } @@ -55,7 +55,7 @@ PythonInterpreter interp = new PythonInterpreter(); // Now replace it Py.installConsole(new JLineConsole(null)); - jython.run(commands); + jython.main(commands); Console console = Py.getConsole(); assertEquals(JLineConsole.class, console.getClass()); interp.cleanup(); -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Sep 23 09:57:16 2018 From: jython-checkins at python.org (jeff.allen) Date: Sun, 23 Sep 2018 13:57:16 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Minor_clean-up_in_jython?= =?utf-8?q?=2Ejava_and_fix_--version_option_=28is_-V_not_-v=29=2E?= Message-ID: <20180923135716.1.893146011BAE66ED@mg.python.org> https://hg.python.org/jython/rev/3ca459532036 changeset: 8183:3ca459532036 user: Jeff Allen date: Sat Sep 22 20:39:30 2018 +0100 summary: Minor clean-up in jython.java and fix --version option (is -V not -v). files: src/org/python/util/jython.java | 28 ++++++++++++++------ 1 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/org/python/util/jython.java b/src/org/python/util/jython.java --- a/src/org/python/util/jython.java +++ b/src/org/python/util/jython.java @@ -16,7 +16,6 @@ import org.python.Version; import org.python.core.BytecodeLoader; import org.python.core.CompileMode; -import org.python.core.CompilerFlags; import org.python.core.Options; import org.python.core.PrePy; import org.python.core.Py; @@ -60,10 +59,10 @@ + "-Dprop=v : Set the property `prop' to value `v'\n" + "-E : ignore environment variables (such as JYTHONPATH)\n" + "-h : print this help message and exit (also --help)\n" - + "-i : inspect interactively after running script; forces a prompt even" + + "-i : inspect interactively after running script; forces a prompt even\n" + " if stdin does not appear to be a terminal; also PYTHONINSPECT=x\n" - + "-jar jar : program read from __run__.py in jar file (deprecated).\n" - + " Use PEP 338 support for jar file as argument (runs __main__.py).\n" + + "-jar jar : program read from __run__.py in jar file. Deprecated: instead,\n" + + " name the archive as the file argument (runs __main__.py).\n" + "-m mod : run library module as a script (terminates option list)\n" // + "-O : optimize generated bytecode (a tad; also PYTHONOPTIMIZE=x)\n" // + "-OO : remove doc-strings in addition to the -O optimizations\n" @@ -89,7 +88,7 @@ + "JYTHONSTARTUP: file executed on interactive startup (no default)\n" + "JYTHONPATH : '" + File.pathSeparator + "'-separated list of directories prefixed to the default module\n" - + " search path. The result is sys.path.\n" + + " search path. The result is sys.path.\n" + "PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr."; // @formatter:on @@ -658,6 +657,7 @@ return defaultValue; } + /** Non-fatal error message when ignoring unsupported option (usually one valid for CPython). */ private static void optionNotSupported(char option) { printError("Option -%c is not supported", option); } @@ -692,26 +692,37 @@ */ static class CommandLineOptions { + /** Possible actions to take after processing the options. */ enum Action { RUN, ERROR, HELP, VERSION }; + /** The action to take after processing the options. */ Action action = Action.RUN; + /** Set informatively when {@link #action}{@code ==ERROR}. */ String message = ""; + /** Argument to the -c option. */ String command; + /** First argument that is not an option (therefore the executable file). */ String filename; + /** Argument to the -m option. */ String module; + /** -h or --help option. */ boolean help = false; + /** -V or --version option. */ boolean version = false; + /** -jar option. */ boolean jar = false; + /** Collects definitions made with the -D option directly to Jython (not java -D). */ Properties properties = new Properties(); + /** Arguments after the first non-option, therefore arguments to the executable program. */ List argv = new LinkedList(); + /** Arguments collected from succesive -W options. */ List warnoptions = new LinkedList(); - CompilerFlags cf = new CompilerFlags(); /** Valid single character options. ':' means expect an argument following. */ // XJD are extra to CPython. X and J are sanctioned while D is potentially a clash. @@ -720,10 +731,9 @@ /** Valid long-name options. */ static final char JAR_OPTION = '\u2615'; static final OptionScanner.LongSpec[] PROGRAM_LONG_OPTS = { - // new OptionScanner.LongSpec("-", OptionScanner.DONE), new OptionScanner.LongSpec("--", OptionScanner.DONE), new OptionScanner.LongSpec("--help", 'h'), - new OptionScanner.LongSpec("--version", 'v'), + new OptionScanner.LongSpec("--version", 'V'), new OptionScanner.LongSpec("-jar", JAR_OPTION, true), // Yes, just one dash. }; @@ -944,7 +954,7 @@ /** * Helper for option {@code -Dprop=v}. This is potentially a clash with Python: work around - * for luncher misplacement of -J-D...? + * for launcher misplacement of -J-D...? */ private void optionD(OptionScanner scanner) throws SecurityException { String[] kv = scanner.getWholeArgument().split("=", 2); -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Sep 23 09:57:15 2018 From: jython-checkins at python.org (jeff.allen) Date: Sun, 23 Sep 2018 13:57:15 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Re-write_main_program_logi?= =?utf-8?q?c_to_be_more_like_the_CPython_main=2Ec_=28partial_=232686=29?= Message-ID: <20180923135715.1.8365F60141602BDD@mg.python.org> https://hg.python.org/jython/rev/9a4866ea2f91 changeset: 8180:9a4866ea2f91 user: Jeff Allen date: Thu Sep 20 10:53:00 2018 +0100 summary: Re-write main program logic to be more like the CPython main.c (partial #2686) This change improves supportability and conformance. It carefully stitches together existing Jython fragments using a main program logic ported from CPython 2.7.15. Many a small bug or divergence has been ironed out. Some small changes were needed outside org.python.util (and a lot of comments). We pass test_cmd_line_script unmodified and test_cmd_line (updated from CPython) skipping only -R support. Support for the -jar option is restored (but deprecated as PEP 338 appears sufficient). Support for SystemRestart (broken since 2.7.0) is restored only to be removed shortly -- it can't wholly work and hadn't been missed. files: Lib/test/test_cmd_line.py | 129 +- Lib/test/test_cmd_line_script.py | 224 - Lib/threading.py | 5 +- NEWS | 6 + registry | 8 + src/org/python/core/Options.java | 57 +- src/org/python/core/Py.java | 61 +- src/org/python/core/PySystemState.java | 93 +- src/org/python/util/OptionScanner.java | 207 + src/org/python/util/PythonInterpreter.java | 19 +- src/org/python/util/jython.java | 1354 ++++++--- 11 files changed, 1349 insertions(+), 814 deletions(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -1,37 +1,32 @@ -import os -import test.test_support, unittest +# Tests invocation of the interpreter with various command line arguments +# All tests are executed with environment variables ignored +# See test_cmd_line_script.py for testing of script execution + +import test.test_support import sys -import popen2 -import subprocess +import unittest +from test.script_helper import ( + assert_python_ok, assert_python_failure, spawn_python, kill_python, + python_exit_code +) + class CmdLineTest(unittest.TestCase): @classmethod def tearDownClass(cls): if test.test_support.is_jython: - # GC is not immediate, so if Popen.__del__ may be delayed. + # GC is not immediate, so Popen.__del__ may be delayed. # Try to force any Popen.__del__ errors within scope of test. from test_weakref import extra_collect extra_collect() - def start_python(self, cmd_line): - outfp, infp = popen2.popen4('"%s" %s' % (sys.executable, cmd_line)) - infp.close() - data = outfp.read() - outfp.close() - # try to cleanup the child so we don't appear to leak when running - # with regrtest -R. This should be a no-op on Windows. - popen2._cleanup() - return data + def start_python(self, *args): + p = spawn_python(*args) + return kill_python(p) def exit_code(self, *args): - cmd_line = [sys.executable] - cmd_line.extend(args) - devnull = open(os.devnull, 'w') - result = subprocess.call(cmd_line, stdout=devnull, - stderr=subprocess.STDOUT) - devnull.close() - return result + return python_exit_code(*args) def test_directories(self): self.assertNotEqual(self.exit_code('.'), 0) @@ -40,10 +35,7 @@ def verify_valid_flag(self, cmd_line): data = self.start_python(cmd_line) self.assertTrue(data == '' or data.endswith('\n')) - self.assertTrue('Traceback' not in data) - - def test_environment(self): - self.verify_valid_flag('-E') + self.assertNotIn('Traceback', data) def test_optimize(self): self.verify_valid_flag('-O') @@ -59,14 +51,12 @@ self.verify_valid_flag('-S') def test_usage(self): - self.assertTrue('usage' in self.start_python('-h')) + self.assertIn('usage', self.start_python('-h')) def test_version(self): - prefix = 'J' if test.test_support.is_jython else 'P' - version = prefix + 'ython %d.%d' % sys.version_info[:2] - start = self.start_python('-V') - self.assertTrue(start.startswith(version), - "%s does not start with %s" % (start, version)) + prefix = 'Jython' if test.test_support.is_jython else 'Python' + version = (prefix + ' %d.%d') % sys.version_info[:2] + self.assertTrue(self.start_python('-V').startswith(version)) def test_run_module(self): # Test expected operation of the '-m' switch @@ -86,6 +76,17 @@ self.exit_code('-m', 'timeit', '-n', '1'), 0) + def test_run_module_bug1764407(self): + # -m and -i need to play well together + # Runs the timeit module and checks the __main__ + # namespace has been populated appropriately + p = spawn_python('-i', '-m', 'timeit', '-n', '1') + p.stdin.write('Timer\n') + p.stdin.write('exit()\n') + data = kill_python(p) + self.assertTrue(data.startswith('1 loop')) + self.assertIn('__main__.Timer', data) + def test_run_code(self): # Test expected operation of the '-c' switch # Switch needs an argument @@ -99,6 +100,72 @@ self.exit_code('-c', 'pass'), 0) + @unittest.skipIf(test.test_support.is_jython, + "Hash randomisation is not supported in Jython.") + def test_hash_randomization(self): + # Verify that -R enables hash randomization: + self.verify_valid_flag('-R') + hashes = [] + for i in range(2): + code = 'print(hash("spam"))' + data = self.start_python('-R', '-c', code) + hashes.append(data) + self.assertNotEqual(hashes[0], hashes[1]) + + # Verify that sys.flags contains hash_randomization + code = 'import sys; print sys.flags' + data = self.start_python('-R', '-c', code) + self.assertTrue('hash_randomization=1' in data) + + def test_del___main__(self): + # Issue #15001: PyRun_SimpleFileExFlags() did crash because it kept a + # borrowed reference to the dict of __main__ module and later modify + # the dict whereas the module was destroyed + filename = test.test_support.TESTFN + self.addCleanup(test.test_support.unlink, filename) + with open(filename, "w") as script: + print >>script, "import sys" + print >>script, "del sys.modules['__main__']" + assert_python_ok(filename) + + def test_unknown_options(self): + rc, out, err = assert_python_failure('-E', '-z') + self.assertIn(b'Unknown option: -z', err) + self.assertEqual(err.splitlines().count(b'Unknown option: -z'), 1) + self.assertEqual(b'', out) + # Add "without='-E'" to prevent _assert_python to append -E + # to env_vars and change the output of stderr + rc, out, err = assert_python_failure('-z', without='-E') + self.assertIn(b'Unknown option: -z', err) + self.assertEqual(err.splitlines().count(b'Unknown option: -z'), 1) + self.assertEqual(b'', out) + rc, out, err = assert_python_failure('-a', '-z', without='-E') + self.assertIn(b'Unknown option: -a', err) + # only the first unknown option is reported + self.assertNotIn(b'Unknown option: -z', err) + self.assertEqual(err.splitlines().count(b'Unknown option: -a'), 1) + self.assertEqual(b'', out) + + def test_jython_startup(self): + # Test that the file designated by JYTHONSTARTUP is executed when interactive + filename = test.test_support.TESTFN + self.addCleanup(test.test_support.unlink, filename) + with open(filename, "w") as script: + print >>script, "print 6*7" + print >>script, "print 'Ni!'" + expected = ['42', 'Ni!'] + def check(*args, **kwargs): + result = assert_python_ok(*args, **kwargs) + self.assertListEqual(expected, result[1].splitlines()) + if test.test_support.is_jython: + # Jython produces a prompt before exit, but not CPython. Hard to say who is correct. + expected.append('>>> ') + # The Jython way is to set a registry item python.startup + check('-i', '-J-Dpython.startup={}'.format(filename)) + # But a JYTHONSTARTUP environment variable is also supported + check('-i', JYTHONSTARTUP=filename) + else: + check('-i', PYTHONSTARTUP=filename) def test_main(): test.test_support.run_unittest(CmdLineTest) diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py deleted file mode 100644 --- a/Lib/test/test_cmd_line_script.py +++ /dev/null @@ -1,224 +0,0 @@ -# Tests command line execution of scripts - -import unittest -import os -import os.path -import test.test_support -from test.script_helper import (run_python, - temp_dir, make_script, compile_script, - make_pkg, make_zip_script, make_zip_pkg) -from test.test_support import is_jython - -verbose = test.test_support.verbose - - -test_source = """\ -# Script may be run with optimisation enabled, so don't rely on assert -# statements being executed -def assertEqual(lhs, rhs): - if lhs != rhs: - raise AssertionError('%r != %r' % (lhs, rhs)) -def assertIdentical(lhs, rhs): - if lhs is not rhs: - raise AssertionError('%r is not %r' % (lhs, rhs)) -# Check basic code execution -result = ['Top level assignment'] -def f(): - result.append('Lower level reference') -f() -assertEqual(result, ['Top level assignment', 'Lower level reference']) -# Check population of magic variables -assertEqual(__name__, '__main__') -print '__file__==%r' % __file__ -print '__package__==%r' % __package__ -# Check the sys module -import sys -assertIdentical(globals(), sys.modules[__name__].__dict__) -print 'sys.argv[0]==%r' % sys.argv[0] -""" - -def _make_test_script(script_dir, script_basename, source=test_source): - return make_script(script_dir, script_basename, source) - -def _make_test_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, - source=test_source, depth=1): - return make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, - source, depth) - -# There's no easy way to pass the script directory in to get -# -m to work (avoiding that is the whole point of making -# directories and zipfiles executable!) -# So we fake it for testing purposes with a custom launch script -launch_source = """\ -import sys, os.path, runpy -sys.path.insert(0, %s) -runpy._run_module_as_main(%r) -""" - -def _make_launch_script(script_dir, script_basename, module_name, path=None): - if path is None: - path = "os.path.dirname(__file__)" - else: - path = repr(path) - source = launch_source % (path, module_name) - return make_script(script_dir, script_basename, source) - -class CmdLineTest(unittest.TestCase): - def _check_script(self, script_name, expected_file, - expected_argv0, expected_package, - *cmd_line_switches): - run_args = cmd_line_switches + (script_name,) - exit_code, data = run_python(*run_args) - if verbose: - print 'Output from test script %r:' % script_name - print data - self.assertEqual(exit_code, 0) - printed_file = '__file__==%r' % str(expected_file) - printed_argv0 = 'sys.argv[0]==%r' % str(expected_argv0) - printed_package = '__package__==%r' % (str(expected_package) if expected_package is not None else expected_package) - if verbose: - print 'Expected output:' - print printed_file - print printed_package - print printed_argv0 - self.assertIn(printed_file, data) - self.assertIn(printed_package, data) - self.assertIn(printed_argv0, data) - - def _check_import_error(self, script_name, expected_msg, - *cmd_line_switches): - run_args = cmd_line_switches + (script_name,) - exit_code, data = run_python(*run_args) - if verbose: - print 'Output from test script %r:' % script_name - print data - print 'Expected output: %r' % expected_msg - self.assertIn(expected_msg, data) - - def test_basic_script(self): - with temp_dir() as script_dir: - script_name = _make_test_script(script_dir, 'script') - self._check_script(script_name, script_name, script_name, None) - - @unittest.skipIf(is_jython, "FIXME: not working in Jython") - def test_script_compiled(self): - with temp_dir() as script_dir: - script_name = _make_test_script(script_dir, 'script') - compiled_name = compile_script(script_name) - os.remove(script_name) - self._check_script(compiled_name, compiled_name, compiled_name, None) - - @unittest.skipIf(is_jython, "FIXME: not working in Jython") - def test_directory(self): - with temp_dir() as script_dir: - script_name = _make_test_script(script_dir, '__main__') - self._check_script(script_dir, script_name, script_dir, '') - - @unittest.skipIf(is_jython, "FIXME: not working in Jython") - def test_directory_compiled(self): - with temp_dir() as script_dir: - script_name = _make_test_script(script_dir, '__main__') - compiled_name = compile_script(script_name) - os.remove(script_name) - self._check_script(script_dir, compiled_name, script_dir, '') - - @unittest.skipIf(is_jython, "FIXME: not working in Jython") - def test_directory_error(self): - with temp_dir() as script_dir: - msg = "can't find '__main__' module in %r" % script_dir - self._check_import_error(script_dir, msg) - - @unittest.skipIf(is_jython, "FIXME: not working in Jython") - def test_zipfile(self): - with temp_dir() as script_dir: - script_name = _make_test_script(script_dir, '__main__') - zip_name, run_name = make_zip_script(script_dir, 'test_zip', script_name) - self._check_script(zip_name, run_name, zip_name, '') - - @unittest.skipIf(is_jython, "FIXME: not working in Jython") - def test_zipfile_compiled(self): - with temp_dir() as script_dir: - script_name = _make_test_script(script_dir, '__main__') - compiled_name = compile_script(script_name) - zip_name, run_name = make_zip_script(script_dir, 'test_zip', compiled_name) - self._check_script(zip_name, run_name, zip_name, '') - - @unittest.skipIf(is_jython, "FIXME: not working in Jython") - def test_zipfile_error(self): - with temp_dir() as script_dir: - script_name = _make_test_script(script_dir, 'not_main') - zip_name, run_name = make_zip_script(script_dir, 'test_zip', script_name) - msg = "can't find '__main__' module in %r" % zip_name - self._check_import_error(zip_name, msg) - - def test_module_in_package(self): - with temp_dir() as script_dir: - pkg_dir = os.path.join(script_dir, 'test_pkg') - make_pkg(pkg_dir) - script_name = _make_test_script(pkg_dir, 'script') - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script') - self._check_script(launch_name, script_name, script_name, 'test_pkg') - - @unittest.skipIf(is_jython, "FIXME: not working in Jython") - def test_module_in_package_in_zipfile(self): - with temp_dir() as script_dir: - zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script') - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script', zip_name) - self._check_script(launch_name, run_name, run_name, 'test_pkg') - - @unittest.skipIf(is_jython, "FIXME: not working in Jython") - def test_module_in_subpackage_in_zipfile(self): - with temp_dir() as script_dir: - zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script', depth=2) - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.test_pkg.script', zip_name) - self._check_script(launch_name, run_name, run_name, 'test_pkg.test_pkg') - - def test_package(self): - with temp_dir() as script_dir: - pkg_dir = os.path.join(script_dir, 'test_pkg') - make_pkg(pkg_dir) - script_name = _make_test_script(pkg_dir, '__main__') - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') - self._check_script(launch_name, script_name, - script_name, 'test_pkg') - - @unittest.skipIf(is_jython, "FIXME: not working in Jython") - def test_package_compiled(self): - with temp_dir() as script_dir: - pkg_dir = os.path.join(script_dir, 'test_pkg') - make_pkg(pkg_dir) - script_name = _make_test_script(pkg_dir, '__main__') - compiled_name = compile_script(script_name) - os.remove(script_name) - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') - self._check_script(launch_name, compiled_name, - compiled_name, 'test_pkg') - - def test_package_error(self): - with temp_dir() as script_dir: - pkg_dir = os.path.join(script_dir, 'test_pkg') - make_pkg(pkg_dir) - msg = ("'test_pkg' is a package and cannot " - "be directly executed") - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') - self._check_import_error(launch_name, msg) - - def test_package_recursion(self): - with temp_dir() as script_dir: - pkg_dir = os.path.join(script_dir, 'test_pkg') - make_pkg(pkg_dir) - main_dir = os.path.join(pkg_dir, '__main__') - make_pkg(main_dir) - msg = ("Cannot use package as __main__ module; " - "'test_pkg' is a package and cannot " - "be directly executed") - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') - self._check_import_error(launch_name, msg) - - -def test_main(): - test.test_support.run_unittest(CmdLineTest) - test.test_support.reap_children() - -if __name__ == '__main__': - test_main() diff --git a/Lib/threading.py b/Lib/threading.py --- a/Lib/threading.py +++ b/Lib/threading.py @@ -223,9 +223,8 @@ except SystemExit: pass except InterruptedException: - # Quiet InterruptedExceptions if they're caused by - # _systemrestart - if not jython.shouldRestart: + # Quiet InterruptedExceptions if they're caused by system restart + if not _sys._shouldRestart: raise except: # If sys.stderr is no more (most likely from interpreter diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -21,6 +21,12 @@ - [ 2650 ] Detail message is not set on PyException from PythonInterpreter - [ 2403 ] VerifyError when implementing interfaces containing default methods (Java 8) + New Features + - The main program behaves more like CPython in many small ways, including a more correct + treatment of the -i option. This simplifies support, and may also make it unnecessary for + users to work around differences from CPython. + - python.startup registry property (and JYTHONSTARTUP environment variable) added. + Jython 2.7.2a1 Bugs fixed - [ 2632 ] Handle unicode data appropriately in csv module diff --git a/registry b/registry --- a/registry +++ b/registry @@ -58,6 +58,14 @@ # behaviour. python.options.caseok = false +# Setting this non-empty will drop the interpreter into an interactive session at the end of +# execution, like adding the -i flag (roughly) or setting the environment variable PYTHONINSPECT +# during execution. +#python.inspect = true + +# Setting this to a file name will cause that file to be run at the start of each interactive +# session (but not when dropping in with the -i flag in after a script has run). +#python.startup = jython-startup.py # Use this registry entry to control the list of builtin modules; you # can add, remove, or override builtin modules. The value for this diff --git a/src/org/python/core/Options.java b/src/org/python/core/Options.java --- a/src/org/python/core/Options.java +++ b/src/org/python/core/Options.java @@ -41,12 +41,29 @@ public static boolean respectJavaAccessibility = true; /** - * When false the site.py will not be imported. This is only - * honored by the command line main class. + * When {@code false} the site.py will not be imported. This may be set by the + * command line main class ({@code -S} option) or from the registry and is checked in + * {@link org.python.util.PythonInterpreter}. + * + * @see #no_site */ public static boolean importSite = true; /** + * When {@code true} the {@code site.py} was not imported. This is may be set by the command + * line main class ({@code -S} option) or from the registry. However, in Jython 2, + * {@code no_site} is simply the opposite of {@link #importSite}, as the interpreter starts up, + * provided for compatibility with the standard Python {@code sys.flags}. Actual control over + * the import of the site module in Jython 2, when necessary from Java, is accomplished through + * {@link #importSite}. + */ + /* + * This should be the standard Python way to control import of the site module. Unfortunately, + * importSite is quite old and we cannot rule out use by applications. Correct in Jython 3. + */ + public static boolean no_site = false; + + /** * Set verbosity to Py.ERROR, Py.WARNING, Py.MESSAGE, Py.COMMENT, or * Py.DEBUG for varying levels of informative messages from Jython. Normally * this option is set from the command line. @@ -54,6 +71,21 @@ public static int verbose = Py.MESSAGE; /** + * Set by the {@code -i} option to the interpreter command, to ask for an interactive session to + * start after the script ends. It also allows certain streams to be considered interactive when + * {@code isatty} is not available. + */ + public static boolean interactive = false; + + /** + * When a script given on the command line finishes, start an interactive interpreter. It is set + * {@code true} by the {@code -i} option on the command-line, or programmatically from the + * script, and reset to {@code false} just before the interactive session starts. (This session + * only actually starts if the console is interactive.) + */ + public static boolean inspect = false; + + /** * A directory where the dynamically generated classes are written. Nothing is * ever read from here, it is only for debugging purposes. */ @@ -78,25 +110,27 @@ /** Whether -3 (py3k warnings) was enabled via the command line. */ public static boolean py3k_warning = false; - + /** Whether -B (don't write bytecode on import) was enabled via the command line. */ public static boolean dont_write_bytecode = false; /** Whether -E (ignore environment) was enabled via the command line. */ public static boolean ignore_environment = false; - //XXX: place holder, not implemented yet. + /** + * Whether -s (don't add user site directory to {@code sys.path}) was on the command line. The + * implementation is mostly in the {@code site} module. + */ public static boolean no_user_site = false; - //XXX: place holder, not implemented yet. - public static boolean no_site = false; - //XXX: place holder public static int bytes_warning = 0; - // Corresponds to -O (Python bytecode optimization), -OO (remove docstrings) - // flags in CPython; it's not clear how Jython should expose its optimization, - // but this is user visible as of 2.7. + /** + * Corresponds to -O (Python bytecode optimization), -OO (remove docstrings) flags in CPython. + * Jython processes the option and makes it visible as of 2.7, but there is no change of + * behaviour in the current version. + */ public static int optimize = 0; /** @@ -201,7 +235,8 @@ } Options.sreCacheSpec = getStringOption("sre.cachespec", Options.sreCacheSpec); - + Options.inspect |= getStringOption("inspect", "").length() > 0; Options.importSite = getBooleanOption("import.site", Options.importSite); + Options.no_site = !Options.importSite; } } diff --git a/src/org/python/core/Py.java b/src/org/python/core/Py.java --- a/src/org/python/core/Py.java +++ b/src/org/python/core/Py.java @@ -278,26 +278,50 @@ static void maybeSystemExit(PyException exc) { if (exc.match(Py.SystemExit)) { + // No actual exit here if Options.interactive (-i flag) is in force. + handleSystemExit(exc); + } + } + + /** + * Exit the process, if {@value Options#inspect}{@code ==false}, cleaning up the system state. + * This exception (normally SystemExit) determines the message, if any, and the + * {@code System.exit} status. + * + * @param exc supplies the message or exit status + */ + static void handleSystemExit(PyException exc) { + if (!Options.inspect) { PyObject value = exc.value; if (PyException.isExceptionInstance(exc.value)) { value = value.__findattr__("code"); } - Py.getSystemState().callExitFunc(); + + // Decide exit status and produce message while Jython still works + int exitStatus; if (value instanceof PyInteger) { - System.exit(((PyInteger) value).getValue()); + exitStatus = ((PyInteger) value).getValue(); } else { if (value != Py.None) { try { Py.println(value); - System.exit(1); + exitStatus = 1; } catch (Throwable t) { - // continue + exitStatus = 0; } + } else { + exitStatus = 0; } - System.exit(0); } + + // Shut down Jython + PySystemState sys = Py.getSystemState(); + sys.callExitFunc(); + sys.close(); + System.exit(exitStatus); } } + public static PyObject StopIteration; public static PyException StopIteration(String message) { @@ -1188,15 +1212,35 @@ return str; } - /* Display a PyException and stack trace */ + /** + * Display an exception and stack trace through + * {@link #printException(Throwable, PyFrame, PyObject)}. + * + * @param t to display + */ public static void printException(Throwable t) { printException(t, null, null); } + /** + * Display an exception and stack trace through + * {@link #printException(Throwable, PyFrame, PyObject)}. + * + * @param t to display + * @param f frame at which to start the stack trace + */ public static void printException(Throwable t, PyFrame f) { printException(t, f, null); } + /** + * Display an exception and stack trace. If the exception was {@link Py#SystemExit} and + * {@link Options#inspect}{@code ==false}, this will exit the JVM. + * + * @param t to display + * @param f frame at which to start the stack trace + * @param file output onto this stream or {@link Py#stderr} if {@code null} + */ public static synchronized void printException(Throwable t, PyFrame f, PyObject file) { StdoutWrapper stderr = Py.stderr; @@ -1218,6 +1262,7 @@ PyException exc = Py.JavaError(t); + // Act on SystemExit here. maybeSystemExit(exc); setException(exc, f); @@ -1772,6 +1817,8 @@ + "You can use the -S option or python.import.site=false to not import the site module"; public static boolean importSiteIfSelected() { + // Ensure sys.flags.no_site actually reflects what happened. (See docs of these two.) + Options.no_site = !Options.importSite; if (Options.importSite) { try { // Ensure site-packages are available @@ -2670,7 +2717,7 @@ return url; } -//------------------------contructor-section--------------------------- +//------------------------constructor-section--------------------------- static class py2JyClassCacheItem { List> interfaces; List pyClasses; diff --git a/src/org/python/core/PySystemState.java b/src/org/python/core/PySystemState.java --- a/src/org/python/core/PySystemState.java +++ b/src/org/python/core/PySystemState.java @@ -233,7 +233,7 @@ currentWorkingDir = new File("").getAbsolutePath(); dont_write_bytecode = Options.dont_write_bytecode; - py3kwarning = Options.py3k_warning; + py3kwarning = Options.py3k_warning; // XXX why here if static? // Set up the initial standard ins and outs String mode = Options.unbuffered ? "b" : ""; int buffering = Options.unbuffered ? 0 : 1; @@ -746,6 +746,15 @@ this.classLoader = classLoader; } + /** + * Work out the root directory of the installation of Jython. Sources for this information are + * quite diverse. {@code python.home} will take precedence if set in either + * {@code postProperties} or {@code preProperties}, {@code install.root} in + * {@code preProperties}, in that order. After this, we search the class path for a JAR, or + * nagigate from the JAR deduced by from the class path, or finally {@code jarFileName}. + *

    + * We also set by side-effect: {@link #defaultPlatform} from {@code java.version}. + */ private static String findRoot(Properties preProperties, Properties postProperties, String jarFileName) { String root = null; @@ -793,6 +802,7 @@ } } + /** Set {@link #defaultPlatform} by examination of the {@code java.version} JVM property. */ private static void determinePlatform(Properties props) { String version = props.getProperty("java.version"); if (version == null) { @@ -869,6 +879,54 @@ } } + /** + * Install the first argument as the application-wide {@link #registry} (a + * {@code java.util.Properties} object), merge values from system and local (or user) properties + * files, and finally allow values from {@code postProperties} to override. Usually the first + * argument is the {@code System.getProperties()}, if were allowed to access it, and therefore + * represents definitions made on the command-line. The net precedence order is: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
    SourceFilled by
    postPropertiesCustom {@link JythonInitializer}
    prePropertiesCommand-line definitions {@code -Dkey=value})
    ... preProperties also contain ...Environment variables via {@link org.python.util.jython}
    [user.home]/.jythonUser-specific registry file
    [python.home]/registryInstallation-wide registry file
    Environmental inferencee.g. {@code locale} command for console encoding
    + *

    + * We call {@link Options#setFromRegistry()} to translate certain final values to + * application-wide controls. By side-effect, set {@link #prefix} and {@link #exec_prefix} from + * {@link #findRoot(Properties, Properties, String)}. If it has not been set otherwise, a + * default value for python.console.encoding is derived from the OS environment, via + * {@link #getConsoleEncoding(Properties)}. + * + * @param preProperties initial registry + * @param postProperties overriding values + * @param standalone default {@code python.cachedir.skip} to true (if not otherwise defined) + * @param jarFileName as a clue to the location of the installation + */ private static void initRegistry(Properties preProperties, Properties postProperties, boolean standalone, String jarFileName) { if (registry != null) { @@ -901,6 +959,7 @@ PySystemState.exec_prefix = Py.fileSystemEncode(exec_prefix); } try { + // XXX: Respect or ignore Options.ignore_environment? String jythonpath = System.getenv("JYTHONPATH"); if (jythonpath != null) { registry.setProperty("python.path", jythonpath); @@ -924,7 +983,7 @@ * python.io.encoding is dubious. */ if (!registry.containsKey(PYTHON_CONSOLE_ENCODING)) { - registry.put(PYTHON_CONSOLE_ENCODING, getConsoleEncoding()); + registry.put(PYTHON_CONSOLE_ENCODING, getConsoleEncoding(registry)); } // Set up options from registry @@ -935,17 +994,19 @@ * Try to determine the console encoding from the platform, if necessary using a sub-process to * enquire. If everything fails, assume UTF-8. * + * @param props in which to look for clues (normally the Jython registry) * @return the console encoding (and never {@code null}) */ - private static String getConsoleEncoding() { + private static String getConsoleEncoding(Properties props) { // From Java 8 onwards, the answer may already be to hand in the registry: - String encoding = System.getProperty("sun.stdout.encoding"); + String encoding = props.getProperty("sun.stdout.encoding"); + String os = props.getProperty("os.name"); if (encoding != null) { return encoding; - } else if (System.getProperty("os.name").startsWith("Windows")) { + } else if (os != null && os.startsWith("Windows")) { // Go via the Windows code page built-in command "chcp". String output = getCommandResult("cmd", "/c", "chcp"); /* @@ -972,8 +1033,8 @@ } /** - * Merge the contents of a property file into the registry without overriding any values already - * set there. + * Merge the contents of a property file into the registry, but existing entries with the same + * key take precedence. * * @param file */ @@ -1162,12 +1223,16 @@ // Condition the console initConsole(registry); - // Finish up standard Python initialization... + /* + * Create the first interpreter (which is also the first instance of the sys module) and + * cache it as the default state. + */ Py.defaultSystemState = new PySystemState(); Py.setSystemState(Py.defaultSystemState); if (classLoader != null) { Py.defaultSystemState.setClassLoader(classLoader); } + Py.initClassExceptions(getDefaultBuiltins()); // Make sure that Exception classes have been loaded @@ -1179,6 +1244,18 @@ return Py.defaultSystemState; } + /** + * Reset the global static {@code PySytemState} so that a subsequent call to + * {@link #initialize()} will re-create the state. This is only really necessary in the context + * of a system restart, but is harmless when shutting down. Using Python after this call is + * likely to result in an implicit full static initialisation, or fail badly. + */ + public static void undoInitialize() { + Py.defaultSystemState = null; + registry = null; + initialized = false; + } + private static PyVersionInfo getVersionInfo() { String s; if (Version.PY_RELEASE_LEVEL == 0x0A) { diff --git a/src/org/python/util/OptionScanner.java b/src/org/python/util/OptionScanner.java new file mode 100644 --- /dev/null +++ b/src/org/python/util/OptionScanner.java @@ -0,0 +1,207 @@ +package org.python.util; + +/** + * A somewhat general-purpose scanner for command options, based on CPython {@code getopt.c}. + */ +class OptionScanner { + + /** Valid options. ':' means expect an argument following. */ + private final String programOpts; // e.g. in Python "3bBc:dEhiJm:OQ:RsStuUvVW:xX?"; + /** Index in argv of the arg currently being processed (or about to be started). */ + private int argIndex = 0; + /** Character index within the current element of argv (of the next option to process). */ + private int optIndex = 0; + /** Option argument (where present for returned option). */ + private String optarg = null; + /** Error message (after returning {@link #ERROR}. */ + private String message = ""; + /** Original argv passed at reset */ + private String[] args; + /** Return to indicate argument processing is over. */ + static final char DONE = '\uffff'; + /** Return to indicate option was not recognised. */ + static final char ERROR = '\ufffe'; + /** Return to indicate the next argument is a free-standing argument. */ + static final char ARGUMENT = '\ufffd'; + + /** + * Class representing an argument of the long type, where the whole program argument represents + * one option, e.g. "--help" or "-version". Such options must start with a '-'. The client + * supplies an array of {@code LongSpec} objects to the constructor to define the valid cases. + * Long options are recognised before single-letter options are looked for. Note that "-" itself + * is treated as a long option (even though it is quite short), returning + * {@link OptionScanner#ERROR} if not explicitly defined as a {@code LongSpec}. + */ + static class LongSpec { + + final String key; + final char returnValue; + final boolean hasArgument; + + /** + * Define that the long argument should return a given char value in calls to + * {@link OptionScanner#getOption()}, and whether or not an option argument should appear + * following it on the command line. This character value need not be the same as any + * single-character option, and may be {@link OptionScanner#DONE} (typically for the key + * {@code "--"}. + * + * @param key to match + * @param returnValue to return when that matches + * @param hasArgument an argument to the option is expected to follow + */ + public LongSpec(String key, char returnValue, boolean hasArgument) { + this.key = key; + this.returnValue = returnValue; + this.hasArgument = hasArgument; + } + + /** The same as {@code LongSpec(key, returnValue, false)}. */ + public LongSpec(String key, char returnValue) { + this(key, returnValue, false); + } + } + + private final LongSpec[] longSpec; + + /** + * Create the scanner from command-line arguments, and information about the valid options. + * + * @param args command-line arguments (which must not change during scanning) + * @param programOpts the one-letter options (with : indicating an option argument + * @param longSpec table of long options (like --help) + */ + OptionScanner(String[] args, String programOpts, LongSpec[] longSpec) { + this.args = args; + this.programOpts = programOpts; + this.longSpec = longSpec; + } + + /** + * Get the next option (as a character), or return a code designating successful or erroneous + * completion. + * + * @return next option from command line: the actual character or a code. + */ + char getOption() { + message = ""; + String arg; + optarg = null; + + if (argIndex >= args.length) { + // Option processing is complete + return DONE; + } else { + // We are currently processing: + arg = args[argIndex]; + if (optIndex == 0) { + // And we're at the start of it. + if (!arg.startsWith("-") || arg.length() <= 1) { + // Non-option program argument e.g. "-" or file name. Note no ++argIndex. + return ARGUMENT; + } else if (longSpec != null) { + // Test for "whole arg" special cases + for (LongSpec spec : longSpec) { + if (spec.key.equals(arg)) { + if (spec.hasArgument) { + // Argument to option should be in next arg + if (++argIndex < args.length) { + optarg = args[argIndex]; + } else { + // There wasn't a next arg. + return error("Argument expected for the %s option", arg); + } + } + // And the next processing will be in the next arg + ++argIndex; + return spec.returnValue; + } + } + // No match: fall through. + } + // arg is one or more single character options. Continue after the '-'. + optIndex = 1; + } + } + + // We are in arg=argv[argvIndex] at the character to examine is at optIndex. + assert argIndex < args.length; + assert optIndex > 0; + assert optIndex < arg.length(); + + char option = arg.charAt(optIndex++); + if (optIndex >= arg.length()) { + // The option was at the end of the arg, so the next action uses the next arg. + ++argIndex; + optIndex = 0; + } + + // Look up the option character in the list of allowable ones + int ptr; + if ((ptr = programOpts.indexOf(option)) < 0 || option == ':') { + if (arg.length() <= 2) { + return error("Unknown option: -%c", option); + } else { + // Might be unrecognised long arg, or a one letter option in a group. + return error("Unknown option: -%c or '%s'", option, arg); + } + } + + // Is the option marked as expecting an argument? + if (++ptr < programOpts.length() && programOpts.charAt(ptr) == ':') { + /* + * The option's argument is the rest of the current argv[argvIndex][optIndex:]. If the + * option is the last character of arg, argvIndex has already moved on and optIndex==0, + * so this statement is still true, except that argvIndex may have moved beyond the end + * of the array. + */ + if (argIndex < args.length) { + optarg = args[argIndex].substring(optIndex); + // And the next processing will be in the next arg + ++argIndex; + optIndex = 0; + } else { + // We were looking for an argument but there wasn't one. + return error("Argument expected for the -%c option", option); + } + } + + return option; + } + + /** Get the argument of the previously returned option or {@code null} if none. */ + String getOptionArgument() { + return optarg; + } + + /** + * Get a whole argument (not the argument of an option), for use after {@code ARGUMENT} was + * returned. This advances the internal state to the next argument. + */ + String getWholeArgument() { + return args[argIndex++]; + } + + /** + * Peek at a whole argument (not the argument of an option), for use after {@code ARGUMENT} was + * returned. This does not advance the internal state to the next argument. + */ + String peekWholeArgument() { + return args[argIndex]; + } + + /** Number of arguments that remain unprocessed from the original array. */ + int countRemainingArguments() { + return args.length - argIndex; + } + + /** Get the error message (when we previously returned {@link #ERROR}. */ + String getMessage() { + return message; + } + + /** Set the error message as {@code String.format(message, args)} and return {@link #ERROR}. */ + char error(String message, Object... args) { + this.message = String.format(message, args); + return ERROR; + } +} diff --git a/src/org/python/util/PythonInterpreter.java b/src/org/python/util/PythonInterpreter.java --- a/src/org/python/util/PythonInterpreter.java +++ b/src/org/python/util/PythonInterpreter.java @@ -6,6 +6,7 @@ import java.util.Properties; import org.python.antlr.base.mod; +import org.python.core.CodeFlag; import org.python.core.CompileMode; import org.python.core.CompilerFlags; import org.python.core.imp; @@ -96,21 +97,19 @@ protected PythonInterpreter(PyObject dict, PySystemState systemState, boolean useThreadLocalState) { - if (dict == null) { - dict = Py.newStringMap(); - } - globals = dict; - if (systemState == null) { - systemState = Py.getSystemState(); - } - this.systemState = systemState; + globals = dict != null ? dict : Py.newStringMap(); + this.systemState = systemState != null ? systemState : Py.getSystemState(); setSystemState(); this.useThreadLocalState = useThreadLocalState; if (!useThreadLocalState) { - PyModule module = new PyModule("__main__", dict); - systemState.modules.__setitem__("__main__", module); + PyModule module = new PyModule("__main__", globals); + this.systemState.modules.__setitem__("__main__", module); + } + + if (Options.Qnew) { + cflags.setFlag(CodeFlag.CO_FUTURE_DIVISION); } Py.importSiteIfSelected(); diff --git a/src/org/python/util/jython.java b/src/org/python/util/jython.java --- a/src/org/python/util/jython.java +++ b/src/org/python/util/jython.java @@ -6,16 +6,18 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.Arrays; +import java.io.PrintStream; +import java.security.AccessControlException; +import java.util.LinkedList; import java.util.List; import java.util.Properties; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import org.python.Version; -import org.python.core.CodeFlag; +import org.python.core.BytecodeLoader; import org.python.core.CompileMode; +import org.python.core.CompilerFlags; import org.python.core.Options; import org.python.core.Py; import org.python.core.PyCode; @@ -28,24 +30,30 @@ import org.python.core.PyStringMap; import org.python.core.PySystemState; import org.python.core.imp; -import org.python.core.util.RelativeFile; import org.python.modules._systemrestart; -import org.python.modules.posix.PosixModule; import org.python.modules.thread.thread; public class jython { + /** Exit status: must have {@code OK.ordinal()==0} */ + private enum Status { + OK, ERROR, NOT_RUN, SHOULD_RESTART, NO_FILE + } + // An instance of this class will provide the console (python.console) by default. private static final String PYTHON_CONSOLE_CLASS = "org.python.util.JLineConsole"; private static final String COPYRIGHT = "Type \"help\", \"copyright\", \"credits\" or \"license\" for more information."; + /** The message output when reporting command-line errors and when asked for help. */ static final String usageHeader = "usage: jython [option] ... [-c cmd | -m mod | file | -] [arg] ...\n"; - private static final String usage = usageHeader - + "Options and arguments:\n" + /** The message additional to {@link #usageHeader} output when asked for help. */ + // @formatter:off + static final String usageBody = + "Options and arguments:\n" // + "(and corresponding environment variables):\n" + "-B : don't write bytecode files on import\n" // + "also PYTHONDONTWRITEBYTECODE=x\n" + @@ -54,10 +62,10 @@ + "-Dprop=v : Set the property `prop' to value `v'\n" + "-E : ignore environment variables (such as JYTHONPATH)\n" + "-h : print this help message and exit (also --help)\n" - + "-i : inspect interactively after running script\n" - // + ", (also PYTHONINSPECT=x)\n" - + " and force prompts, even if stdin does not appear to be a terminal\n" - + "-jar jar : program read from __run__.py in jar file\n" + + "-i : inspect interactively after running script; forces a prompt even" + + " if stdin does not appear to be a terminal; also PYTHONINSPECT=x\n" + + "-jar jar : program read from __run__.py in jar file (deprecated).\n" + + " Use PEP 338 support for jar file as argument (runs __main__.py).\n" + "-m mod : run library module as a script (terminates option list)\n" // + "-O : optimize generated bytecode (a tad; also PYTHONOPTIMIZE=x)\n" // + "-OO : remove doc-strings in addition to the -O optimizations\n" @@ -80,22 +88,42 @@ + "- : program read from stdin (default; interactive mode if a tty)\n" + "arg ... : arguments passed to program in sys.argv[1:]\n" + "\n" + "Other environment variables:\n" // - + "JYTHONPATH: '" + File.pathSeparator + + "JYTHONSTARTUP: file executed on interactive startup (no default)\n" + + "JYTHONPATH : '" + File.pathSeparator + "'-separated list of directories prefixed to the default module\n" + " search path. The result is sys.path.\n" + "PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr."; + // @formatter:on - public static boolean shouldRestart; + /** + * Print a full usage message onto {@code System.out} or a brief usage message onto + * {@code System.err}. + * + * @param status if {@code == 0} full help version on {@code System.out}. + * @return the status given as the argument. + */ + static Status usage(Status status) { + boolean fullHelp = (status == Status.OK); + PrintStream f = fullHelp ? System.out : System.err; + f.printf(usageHeader); + if (fullHelp) { + f.printf(usageBody); + } else { + f.println("Try 'jython -h' for more information."); + } + return status; + } /** * Runs a JAR file, by executing the code found in the file __run__.py, which should be in the - * root of the JAR archive. Note that the __name__ is set to the base name of the JAR file and - * not to "__main__" (for historic reasons). This method do NOT handle exceptions. the caller - * SHOULD handle any (Py)Exceptions thrown by the code. + * root of the JAR archive. Note that {@code __name__} is set to the base name of the JAR file + * and not to "__main__" (for historical reasons). This method does not handle + * exceptions. the caller should handle any {@code (Py)Exceptions} thrown by the code. * * @param filename The path to the filename to run. + * @return {@code 0} on normal termination (otherwise throws {@code PyException}). */ - public static void runJar(String filename) { + public static int runJar(String filename) { // TBD: this is kind of gross because a local called `zipfile' just magically // shows up in the module's globals. Either `zipfile' should be called // `__zipfile__' or (preferably, IMO), __run__.py should be imported and a main() @@ -103,378 +131,494 @@ // argument. // // Probably have to keep this code around for backwards compatibility (?) - try { - ZipFile zip = new ZipFile(filename); + try (ZipFile zip = new ZipFile(filename)) { ZipEntry runit = zip.getEntry("__run__.py"); if (runit == null) { - throw Py.ValueError("jar file missing '__run__.py'"); + throw Py.ValueError("can't find '__run__.py' in '" + filename + "'"); + } + + /* + * Stripping the stuff before the last File.separator fixes Bug #931129 by keeping + * illegal characters out of the generated proxy class name. Mostly. + */ + int beginIndex = filename.lastIndexOf(File.separator); + if (beginIndex >= 0) { + filename = filename.substring(beginIndex + 1); } PyStringMap locals = Py.newStringMap(); - - // Stripping the stuff before the last File.separator fixes Bug #931129 by - // keeping illegal characters out of the generated proxy class name - int beginIndex; - if ((beginIndex = filename.lastIndexOf(File.separator)) != -1) { - filename = filename.substring(beginIndex + 1); - } - - locals.__setitem__("__name__", new PyString(filename)); + locals.__setitem__("__name__", Py.fileSystemEncode(filename)); locals.__setitem__("zipfile", Py.java2py(zip)); - InputStream file = zip.getInputStream(runit); - PyCode code; - try { - code = Py.compile(file, "__run__", CompileMode.exec); - } finally { - file.close(); - } + InputStream file = zip.getInputStream(runit); // closed when zip is closed + + PyCode code = Py.compile(file, "__run__", CompileMode.exec); Py.runCode(code, locals, locals); + } catch (IOException e) { throw Py.IOError(e); } + + return Status.OK.ordinal(); } public static void main(String[] args) { + Status status; do { - shouldRestart = false; - run(args); - } while (shouldRestart); + status = run(args); + } while (status == Status.SHOULD_RESTART); + System.exit(status.ordinal()); } - private static List warnOptionsFromEnv() { - ArrayList opts = new ArrayList(); - - try { - String envVar = System.getenv("PYTHONWARNINGS"); - if (envVar == null) { - return opts; - } - - for (String opt : envVar.split(",")) { - opt = opt.trim(); - if (opt.length() == 0) { - continue; - } - opts.add(opt); + /** + * Append options from the environment variable {@code PYTHONWARNINGS}, respecting the -E + * option. + * + * @param opts the list to which the options are appended. + */ + private static void addWarnOptionsFromEnv(PyList opts) { + String envVar = getenv("PYTHONWARNINGS", ""); + for (String opt : envVar.split(",")) { + opt = opt.trim(); + if (opt.length() > 0) { + opts.add(Py.fileSystemEncode(opt)); } - } catch (SecurityException e) { - // Continue - } - - return opts; - } - - private static List validWarnActions = Arrays.asList("error", "ignore", "always", - "default", "module", "once"); - - private static void addWarnings(List from, PyList to) { - outerLoop : for (String opt : from) { - String action = opt.split(":")[0]; - for (String validWarnAction : validWarnActions) { - if (validWarnAction.startsWith(action)) { - if (opt.contains(":")) { - to.append(Py.newString(validWarnAction + opt.substring(opt.indexOf(":")))); - } else { - to.append(Py.newString(validWarnAction)); - } - continue outerLoop; - } - } - System.err.println(String.format("Invalid -W option ignored: invalid action: '%s'", - action)); } } - private static void runModule(InteractiveConsole interp, String moduleName) { - runModule(interp, moduleName, false); + /** + * Attempt to run a module as the {@code __main__} module, via a call to + * {@code runpy._run_module_as_main}. Exceptions raised by the imported module, including + * {@code SystemExit}, if not handled by {@code runpy} itself, will propagate out of this + * method. Note that if {@code runpy} cannot import the module it calls {@code sys.exit} with a + * message, which will raise {@code SystemExit} from this method. + * + * @param moduleName to run + * @param set_argv0 replace {@code sys.argv[0]} with the file name of the module source + * {@code runpy._run_module_as_main} option. + * @return {@code Status.OK} on normal termination. + */ + private static Status runModule(String moduleName, boolean set_argv0) { + // PEP 338 - Execute module as a script + PyObject runpy = imp.importName("runpy", true); + PyObject runmodule = runpy.__findattr__("_run_module_as_main"); + // May raise SystemExit (with message) + runmodule.__call__(Py.fileSystemEncode(moduleName), Py.newBoolean(set_argv0)); + return Status.OK; + } + + /** + * Attempt to treat a file as a source of imports, import a module {@code __main__}, and run it. + * If the file is suitable (e.g. it's a directory or a ZIP archive) the method places the file + * first on {@code sys.path}, so that {@code __main__} and its packaged dependencies may be + * found in it. This permits a zip file containing Python source to be run when given as a first + * argument on the command line. It may be that the file is not of a type that can be imported, + * in which case the return indicates this. + * + * @param archive to run (FS encoded name) + * @return {@code Status.OK} or {@code Status.NOT_RUN} (if the file was not an archive). + */ + private static Status runMainFromImporter(PyString archive) { + PyObject importer = imp.getImporter(archive); + if (!(importer instanceof PyNullImporter)) { + // filename is usable as an import source, so put it in sys.path[0] and import __main__ + Py.getSystemState().path.insert(0, archive); + return runModule("__main__", false); + } + return Status.NOT_RUN; } - private static void runModule(InteractiveConsole interp, String moduleName, boolean set_argv0) { - // PEP 338 - Execute module as a script - try { - PyObject runpy = imp.importName("runpy", true); - PyObject runmodule = runpy.__findattr__("_run_module_as_main"); - runmodule.__call__(Py.fileSystemEncode(moduleName), Py.newBoolean(set_argv0)); - } catch (Throwable t) { - Py.printException(t); - interp.cleanup(); - System.exit(-1); + /** + * Execute the stream {@code fp} as a file, in the given interpreter, as {@code __main__}. The + * file name provided must correspond to the stream. In particular, the file name extension + * "$py.class" will cause the stream to be interpreted as compiled code. For streams that are + * not really files, this name may be a conventional one like {@code ""}, however this + * method will not treat a console stream as interactive. + * + * @param fp Python source code + * @param filename appears FS-encoded in variable {@code __file__} and in error messages + * @param interp to do the work + * @return {@code Status.OK} on normal termination. + */ + // This is roughly equivalent to CPython PyRun_SimpleFileExFlags + private static Status runSimpleFile(InputStream fp, String filename, PythonInterpreter interp) { + + // Reflect the current name in the module's __file__, compare PyRun_SimpleFileExFlags. + final String __file__ = "__file__"; + PyObject globals = interp.globals; + PyObject previousFilename = globals.__finditem__(__file__); + if (previousFilename == null) { + globals.__setitem__(__file__, + // Note that __file__ is widely expected to be encoded bytes + Py.fileSystemEncode(filename)); + } + + // Allow for already-compiled target, but for us it's a $py.class not a .pyc. + if (filename.endsWith("$py.class")) { + // Jython compiled file. + String name = filename.substring(0, filename.length() - 6); // = - ".class" + try { + byte[] codeBytes = imp.readCode(filename, fp, false, imp.NO_MTIME); + File file = new File(filename); + PyCode code = BytecodeLoader.makeCode(name, codeBytes, file.getParent()); + interp.exec(code); + } catch (IOException e) { + throw Py.IOError(e); + } + } else { + // Assume Python source file: run in the interpreter + interp.execfile(fp, filename); + } + + // Delete __file__ variable, previously non-existent. Compare PyRun_SimpleFileExFlags. + if (previousFilename == null) { + globals.__delitem__(__file__); + } + return Status.OK; + } + + /** + * Execute the stream {@code fp} in the given interpreter. If {@code fp} refers to a stream + * associated with an interactive device (console or terminal input), execute Python source + * statements from the stream in the interpreter as {@code __main__}. Otherwise, the file name + * provided must correspond to the stream, as in + * {@link #runSimpleFile(InputStream, String, PythonInterpreter)}. + * + * @param fp Python source code + * @param filename the name of the file or {@code null} meaning "???" + * @param interp to do the work + * @return {@code Status.OK} on normal termination. + */ + // This is roughly equivalent to CPython PyRun_AnyFileExFlags + private static Status runStream(InputStream fp, String filename, InteractiveConsole interp) { + // Following CPython PyRun_AnyFileExFlags here, blindly, concerning null name. + filename = filename != null ? filename : "???"; + // Run the contents in the interpreter + if (isInteractive(fp, filename)) { + // __file__ not defined + interp.interact(null, new PyFile(fp)); + } else { + // __file__ will be defined + runSimpleFile(fp, filename, interp); + } + return Status.OK; + } + + /** + * Attempt to open the named file and execute it in the interpreter as {@code __main__}, as in + * {@link #runStream(InputStream, String, InteractiveConsole)}. This may raise a Python + * exception, including {@code SystemExit}. If the file cannot be opened, or using it throws a + * Java {@code IOException} that is not converted to a {@code PyException} (i.e. not within the + * executing code), it is reported via {@link #printError(String, Object...)}, and reflected in + * the return status. If the file can be opened, its parent directory will be inserted at + * {@code sys.argv[0]}. + * + * @param filename the name of the file or {@code null} meaning "???" + * @param interp to do the work + * @return {@code Status.OK} on normal termination, {@code Status.NO_FILE} if the file cannot be + * read, or {@code Status.ERROR} on other {@code IOException}s. + */ + private static Status runFile(String filename, InteractiveConsole interp) { + File file = new File(filename); + try (InputStream is = new FileInputStream(file)) { + String parent = file.getAbsoluteFile().getParent(); + interp.getSystemState().path.insert(0, Py.fileSystemEncode(parent)); + // May raise exceptions, (including SystemExit) + return runStream(is, filename, interp); + } catch (FileNotFoundException fnfe) { + // Couldn't open it. No point in going interactive, even if -i given. + printError("can't open file '%s': %s", filename, fnfe); + return Status.NO_FILE; + } catch (IOException ioe) { + // This may happen on the automatically-generated close() + printError("error closing '%s': %s", filename, ioe); + return Status.ERROR; } } - private static boolean runMainFromImporter(InteractiveConsole interp, String filename) { - // Support http://bugs.python.org/issue1739468 - Allow interpreter to execute a zip file or directory - PyString argv0 = Py.fileSystemEncode(filename); - PyObject importer = imp.getImporter(argv0); - if (!(importer instanceof PyNullImporter)) { - /* argv0 is usable as an import source, so - put it in sys.path[0] and import __main__ */ - Py.getSystemState().path.insert(0, argv0); - runModule(interp, "__main__", true); - return true; + /** + * Attempt to execute the file named in the registry entry {@code python.startup}, which may + * also have been set via the environment variable {@code JYTHONSTARTUP}. This may raise a + * Python exception, including {@code SystemExit} that propagates to the caller. If the file + * cannot be opened, or using it throws a Java {@code IOException} that is not converted to a + * {@code PyException} (i.e. not within the executing code), it is reported via + * {@link #printError(String, Object...)}. + * + * @param interp to do the work + */ + private static void runStartupFile(InteractiveConsole interp) { + String filename = PySystemState.registry.getProperty("python.startup", null); + if (filename != null) { + try (InputStream fp = new FileInputStream(filename)) { + // May raise exceptions, (including SystemExit) + // RunStreamOrThrow(fp, filename, interp); + runSimpleFile(fp, filename, interp); + } catch (FileNotFoundException fnfe) { + // Couldn't open it. No point in going interactive, even if -i given. + printError("Could not open startup file '%s'", filename); + } catch (IOException ioe) { + // This may happen on the automatically-generated close() + printError("error closing '%s': %s", filename, ioe); + } } - return false; } - public static void run(String[] args) { - + /** + * Main Jython program, following the structure and logic of CPython {@code main.c} to produce + * the same behaviour. The argument to the method is the argument list supplied after the class + * name in the {@code java} command. Arguments up to the executable script name are options for + * Jython; arguments after the executable script are supplied in {@code sys.argv}. "Executable + * script" here means a Python source file name, a module name (following the {@code -m} + * option), a literal command (following the {@code -c} option), or a JAR file name (following + * the {@code -jar} option). As a special case of the file name, "-" is allowed, meaning take + * the script from standard input. + *

    + * The main difference for the caller stems from a difference between C and Java: in C, the + * argument list {@code (argv)} begins with the program name, while in Java all elements of + * {@code (args)} are arguments to the program. + * + * @param args arguments to the program. + * @return status indicating outcome. + */ + public static Status run(String[] args) { // Parse the command line options - CommandLineOptions opts = new CommandLineOptions(); - if (!opts.parse(args)) { - if (opts.version) { - System.err.println("Jython " + Version.PY_VERSION); - System.exit(0); - } - if (opts.help) { - System.err.println(usage); - } else if (!opts.runCommand && !opts.runModule) { - System.err.print(usageHeader); - System.err.println("Try `jython -h' for more information."); - } - - int exitcode = opts.help ? 0 : -1; - System.exit(exitcode); + CommandLineOptions opts = CommandLineOptions.parse(args); + switch (opts.action) { + case VERSION: + System.err.printf("Jython %s\n", Version.PY_VERSION); + return Status.OK; + case HELP: + return usage(Status.OK); + case ERROR: + System.err.println(opts.message); + return usage(Status.ERROR); + case RUN: + // Let's run some Python! ... } // Get system properties (or empty set if we're prevented from accessing them) - Properties preProperties = PySystemState.getBaseProperties(); + Properties preProperties = getSystemProperties(); + addDefaultsFromEnvironment(preProperties); - // Read environment variable PYTHONIOENCODING into properties (registry) - String pythonIoEncoding = getenv("PYTHONIOENCODING"); - if (pythonIoEncoding != null) { - String[] spec = splitString(pythonIoEncoding, ':', 2); - // Note that if encoding or errors is blank (=null), the registry value wins. - addDefault(preProperties, PySystemState.PYTHON_IO_ENCODING, spec[0]); - addDefault(preProperties, PySystemState.PYTHON_IO_ERRORS, spec[1]); + // Treat the apparent filename "-" as no filename + boolean haveDash = "-".equals(opts.filename); + if (haveDash) { + opts.filename = null; } - // If/when we interact with standard input, will we use a line-editing console? - boolean stdinIsInteractive = Py.isInteractive(); + // Sense whether the console is interactive, or we have been told to consider it so. + boolean stdinIsInteractive = isInteractive(System.in, null); - // Decide if System.in is interactive - if (!opts.fixInteractive || opts.interactive) { - // The options suggest System.in is interactive: but only if isatty() agrees - opts.interactive = stdinIsInteractive; - if (opts.interactive) { + // Shorthand + boolean haveScript = opts.command != null || opts.filename != null || opts.module != null; + + if (Options.inspect || !haveScript) { + // We'll be going interactive eventually. condition an interactive console. + if (haveConsole()) { // Set the default console type if nothing else has addDefault(preProperties, "python.console", PYTHON_CONSOLE_CLASS); } } - // Setup the basic python system state from these options - PySystemState.initialize(preProperties, opts.properties, opts.argv); - PySystemState systemState = Py.getSystemState(); + /* + * Set up the executable-wide state from the options, environment and registry, and create + * the first instance of a sys module. We try to leave to this initialisation the things + * necessary to an embedded interpreter, and to do in the present method only that which + * belongs only to command line application. + * + * (Jython partitions system and interpreter state differently from modern CPython, which is + * able explicitly to create a PyInterpreterState first, after which sys and the thread + * state are created to hang from it.) + */ + // The Jython type system will spring into existence here. This may take a while. + PySystemState.initialize(preProperties, opts.properties); + // Now we can use PyObjects safely. + PySystemState sys = Py.getSystemState(); - PyList warnoptions = new PyList(); - addWarnings(opts.warnoptions, warnoptions); - if (!Options.ignore_environment) { - addWarnings(warnOptionsFromEnv(), warnoptions); - } - systemState.setWarnoptions(warnoptions); - - // Make sure warnings module is loaded if there are warning options - // Not sure this is needed, but test_warnings.py expects this - if (warnoptions.size() > 0) { + /* + * Jython initialisation does not load the "warnings" module. Rather we defer it to here, + * where we may safely prepare sys.warnoptions from the -W arguments and the contents of + * PYTHONWARNINGS (compare PEP 432). + */ + addFSEncoded(opts.warnoptions, sys.warnoptions); + addWarnOptionsFromEnv(sys.warnoptions); + if (!sys.warnoptions.isEmpty()) { + // The warnings module validates (and may complain about) the warning options. imp.load("warnings"); } - // Now create an interpreter - if (!opts.interactive) { - // Not (really) interactive, so do not use console prompts - systemState.ps1 = systemState.ps2 = Py.EmptyString; - } + /* + * Create the interpreter that we will use as a name space in which to execute the script or + * interactive session. We run site.py as part of interpreter initialisation (as CPython). + */ InteractiveConsole interp = new InteractiveConsole(); - // Print banner and copyright information (or not) - if (opts.interactive && opts.notice && !opts.runModule) { + if (Options.verbose > Py.MESSAGE || (!haveScript && stdinIsInteractive)) { + // Verbose or going interactive immediately: produce sign on messages. System.err.println(InteractiveConsole.getDefaultBanner()); - } - - if (Py.importSiteIfSelected()) { - if (opts.interactive && opts.notice && !opts.runModule) { + if (Options.importSite) { System.err.println(COPYRIGHT); } } - if (opts.division != null) { - if ("old".equals(opts.division)) { - Options.division_warning = 0; - } else if ("warn".equals(opts.division)) { - Options.division_warning = 1; - } else if ("warnall".equals(opts.division)) { - Options.division_warning = 2; - } else if ("new".equals(opts.division)) { - Options.Qnew = true; - interp.cflags.setFlag(CodeFlag.CO_FUTURE_DIVISION); + /* + * We currently have sys.argv = PySystemState.defaultArgv = ['']. Python has a special use + * for sys.argv[0] according to the source of the script (-m, -c, etc.), but the rest of it + * comes from the unparsed part of the command line. + */ + addFSEncoded(opts.argv, sys.argv); + + /* + * At last, we are ready to execute something. This has two parts: execute the script or + * console and (if we didn't execute the console) optionally start an interactive console + * session. The sys.path needs to be prepared in a slightly different way for each case. + */ + Status sts = Status.NOT_RUN; + + try { + if (opts.command != null) { + // The script is an immediate command -c "..." + sys.argv.set(0, Py.newString("-c")); + sys.path.insert(0, Py.EmptyString); + interp.exec(opts.command); + sts = Status.OK; + + } else if (opts.module != null) { + // The script is a module + sys.argv.set(0, Py.newString("-m")); + sts = runModule(opts.module, true); + + } else if (opts.filename != null) { + // The script is designated by file (or directory) name. + PyString pyFileName = Py.fileSystemEncode(opts.filename); + sys.argv.set(0, pyFileName); + + if (opts.jar) { + // The filename was given with the -jar option. + sys.path.insert(0, pyFileName); + runJar(opts.filename); + sts = Status.OK; + + } else { + /* + * The filename was given as the leading argument after the options. Our first + * approach is to treat it as an archive (or directory) in which to find a + * __main__.py (as per PEP 338). The handler for this inserts the module at + * sys.path[0] if it runs. It may raise exceptions, but only SystemExit as runpy + * deals with the others. + */ + sts = runMainFromImporter(pyFileName); + + if (sts == Status.NOT_RUN) { + /* + * The filename was not a suitable source for import, so treat it as a file + * to execute. The handler for this inserts the parent of the file at + * sys.path[0]. + */ + sts = runFile(opts.filename, interp); + // If we really had no script, do not go interactive at the end. + haveScript = sts != Status.NO_FILE; + } + } + + } else { // filename == null + // There is no script. (No argument or it was "-".) + if (haveDash) { + sys.argv.set(0, Py.newString('-')); + } + sys.path.insert(0, Py.EmptyString); + + // Genuinely interactive, or just interpreting piped instructions? + if (stdinIsInteractive) { + // If genuinely interactive, SystemExit should mean exit the application. + Options.inspect = false; + // If genuinely interactive, run a start-up file if one is specified. + runStartupFile(interp); + } + + // Run from console: exceptions other than SystemExit are handled in the REPL. + sts = runStream(System.in, "", interp); + } + + } catch (PyException pye) { + // Whatever the mode of execution an uncaught PyException lands here. + if (pye.match(_systemrestart.SystemRestart)) { + // Leave ourselves a note to restart. + sts = Status.SHOULD_RESTART; + } else { + // If pye was SystemExit *and* Options.inspect==false, this will exit the JVM: + Py.printException(pye); + // It was an exception other than SystemExit or Options.inspect==true. + sts = Status.ERROR; } } - // was there a filename on the command line? - if (opts.filename != null) { - if (runMainFromImporter(interp, opts.filename)) { - interp.cleanup(); - return; - } + /* + * Check this environment variable at the end, to give programs the opportunity to set it + * from Python. + */ + // XXX: Java does not let us set environment variables but we could have our own os.environ. + if (!Options.inspect) { + Options.inspect = getenv("PYTHONINSPECT") != null; + } - String path; + if (Options.inspect && stdinIsInteractive && haveScript) { + /* + * The inspect flag is set (-i option) so we've been asked to end with an interactive + * session: the console is interactive, and we have just executed some kind of script + * (i.e. it wasn't already an interactive session). + */ try { - path = new File(opts.filename).getCanonicalFile().getParent(); - } catch (IOException ioe) { - path = new File(opts.filename).getAbsoluteFile().getParent(); - } - if (path == null) { - path = ""; - } - Py.getSystemState().path.insert(0, Py.fileSystemEncode(path)); - if (opts.jar) { - try { - runJar(opts.filename); - } catch (Throwable t) { - Py.printException(t); - System.exit(-1); - } - } else if (opts.filename.equals("-")) { - try { - interp.globals.__setitem__(new PyString("__file__"), new PyString("")); - interp.execfile(System.in, ""); - } catch (Throwable t) { - Py.printException(t); - } - } else { - try { - interp.globals.__setitem__(new PyString("__file__"), - // Note that __file__ is widely expected to be encoded bytes - Py.fileSystemEncode(opts.filename)); - FileInputStream file; - try { - file = new FileInputStream(new RelativeFile(opts.filename)); - } catch (FileNotFoundException e) { - throw Py.IOError(e); - } - try { - boolean isInteractive = false; - try { - isInteractive = PosixModule.getPOSIX().isatty(file.getFD()); - } catch (SecurityException ex) {} - if (isInteractive) { - opts.interactive = true; - interp.interact(null, new PyFile(file)); - return; - } else { - interp.execfile(file, opts.filename); - } - } finally { - file.close(); - } - } catch (Throwable t) { - if (t instanceof PyException - && ((PyException)t).match(_systemrestart.SystemRestart)) { - // Shutdown this instance... - shouldRestart = true; - shutdownInterpreter(); - interp.cleanup(); - // ..reset the state... - Py.setSystemState(new PySystemState()); - // ...and start again - return; - } else { - Py.printException(t); - interp.cleanup(); - System.exit(-1); - } - } - } - } else { - // if there was no file name on the command line, then "" is the first element - // on sys.path. This is here because if there /was/ a filename on the c.l., - // and say the -i option was given, sys.path[0] will have gotten filled in - // with the dir of the argument filename. - Py.getSystemState().path.insert(0, Py.EmptyString); - - if (opts.command != null) { - try { - interp.exec(opts.command); - } catch (Throwable t) { - Py.printException(t); - System.exit(1); - } - } - - if (opts.moduleName != null) { - runModule(interp, opts.moduleName); - interp.cleanup(); - return; + // Ensure that this time SystemExit means exit. + Options.inspect = false; + // Run from console: exceptions other than SystemExit are handled in the REPL. + sts = runStream(System.in, "", interp); + } catch (PyException pye) { + // Exception from the execution of Python code. + Py.printException(pye); // SystemExit will exit the JVM here. + sts = Status.ERROR; } } - if (opts.fixInteractive || (opts.filename == null && opts.command == null)) { - // Go interactive with the console: the parser needs to know the encoding. - String encoding = Py.getConsole().getEncoding(); - // Run the interpreter interactively - try { - interp.cflags.encoding = encoding; - if (!opts.interactive) { - // Don't print prompts. http://bugs.jython.org/issue2325 - interp._interact(null, null); - } - else { - interp.interact(null, null); - } - } catch (Throwable t) { - Py.printException(t); - } + if (sts == Status.SHOULD_RESTART) { + // Shut down *all* threads and sockets + shutdownInterpreter(); + // ..reset the state... + // XXX seems unnecessary given we will call PySystemState.initialize() + // Py.setSystemState(new PySystemState()); + // ...and start again } + // Shut down in a tidy way interp.cleanup(); + PySystemState.undoInitialize(); + + return sts; } /** * Run any finalizations on the current interpreter in preparation for a SytemRestart. */ public static void shutdownInterpreter() { - // Stop all the active threads and signal the SystemRestart + PySystemState sys = Py.getSystemState(); + // Signal to threading.py to modify response to InterruptedException: + sys._systemRestart = true; + // Interrupt (stop?) all the active threads in the jython-threads group. thread.interruptAllThreads(); - Py.getSystemState()._systemRestart = true; - // Close all sockets -- not all of their operations are stopped by - // Thread.interrupt (in particular pre-nio sockets) + // Close all sockets not already covered by Thread.interrupt (e.g. pre-nio sockets) try { - imp.load("socket").__findattr__("_closeActiveSockets").__call__(); + PyObject socket = sys.modules.__finditem__("socket"); + if (socket != null) { + // XXX Fossil code. Raises AttributeError as _closeActiveSockets has been deleted. + socket.__getattr__("_closeActiveSockets").__call__(); + } } catch (PyException pye) { - // continue + // don't worry about errors: we're shutting down } } /** - * Return an array of trimmed strings by splitting the argument at each occurrence of a - * separator character. (Helper for configuration variable processing.) Segments of zero length - * after trimming emerge as null. If there are more than the specified number of - * segments the last element of the array contains all of the source string after the - * (n-1)th occurrence of sep. - * - * @param spec to split - * @param sep character on which to split - * @param n number of parts to split into - * @return n-element array of strings (or nulls) - */ - private static String[] splitString(String spec, char sep, int n) { - String[] list = new String[n]; - int p = 0, i = 0, L = spec.length(); - while (p < L) { - int c = spec.indexOf(sep, p); - if (c < 0 || i >= n - 1) { - // No more seps, or no more space: i.th piece is the rest of spec. - c = L; - } - String s = spec.substring(p, c).trim(); - list[i++] = (s.length() > 0) ? s : null; - p = c + 1; - } - return list; - } - - /** * If the key is not currently present and the passed value is not null, sets the * key to the value in the given Properties object. Thus, * it provides a default value for a subsequent getProperty(). @@ -495,244 +639,414 @@ } /** - * Get the value of an environment variable, if we are allowed to and it exists; otherwise - * return null. We are allowed to access the environment variable if the -E flag - * was not given and the application has permission to read environment variables. The -E flag - * is reflected in {@link Options#ignore_environment}, and will be set automatically if it turns - * out we do not have permission. + * Provides default registry entries from particular supported environment variables, obtained + * by calls to {@link #getenv(String)}. If a corresponding entry already exists in the + * properties passed, it takes precedence. * - * @param varname name to access in the environment - * @return the value or null. + * @param registry to be (possibly) updated */ - private static String getenv(String varname) { + private static void addDefaultsFromEnvironment(Properties registry) { + // Runs at the start of each (wholly) interactive session. + addDefault(registry, "python.startup", getenv("JYTHONSTARTUP")); + // Go interactive after script. (PYTHONINSPECT because Python scripts may set it.) + addDefault(registry, "python.inspect", getenv("PYTHONINSPECT")); + + // Read environment variable PYTHONIOENCODING into properties (registry) + String pythonIoEncoding = getenv("PYTHONIOENCODING"); + if (pythonIoEncoding != null) { + String[] spec = pythonIoEncoding.split(":", 2); + // Note that if encoding or errors is blank (=null), the registry value wins. + addDefault(registry, PySystemState.PYTHON_IO_ENCODING, spec[0]); + if (spec.length > 1) { + addDefault(registry, PySystemState.PYTHON_IO_ERRORS, spec[1]); + } + } + } + + /** The same as {@code getenv(name, null)} */ + private static String getenv(String name) { + return getenv(name, null); + } + + /** + * Get the value of an environment variable, if we are allowed to and it is defined; otherwise, + * return the chosen default value. An empty string value is treated as undefined. We are + * allowed to access the environment variable if if the JVM security environment permits and if + * {@link Options#ignore_environment} is {@code false}, which it is by default. It may be set by + * the -E flag given to the launcher, or by program action, or once it turns out we do not have + * permission (saving further work). + * + * @param name to access in the environment. + * @param defaultValue to return if {@code name} is not defined or "" or access is forbidden. + * @return the corresponding value or defaultValue. + */ + private static String getenv(String name, String defaultValue) { if (!Options.ignore_environment) { try { - return System.getenv(varname); + String value = System.getenv(name); + return (value != null && value.length() > 0) ? value : defaultValue; } catch (SecurityException e) { // We're not allowed to access them after all Options.ignore_environment = true; } } - return null; + return defaultValue; + } + + private static void optionNotSupported(char option) { + printError("Option -%c is not supported", option); + } + + /** + * Print {@code "jython: "} on {@code System.err} as one line. + * + * @param format suitable to use in {@code String.format(format, args)} + * @param args zero or more args + */ + private static void printError(String format, Object... args) { + System.err.println(String.format("jython: " + format, args)); } -} - -class CommandLineOptions { - - public String filename; - public boolean jar, notice; - public boolean runCommand, runModule; - /** True unless a file, module, jar, or command argument awaits execution. */ - public boolean interactive = true; - /** Eventually go interactive: reflects the -i ("inspect") flag. */ - public boolean fixInteractive = false; - public boolean help, version; - public String[] argv; - public Properties properties; - public String command; - public List warnoptions = Generic.list(); - public String encoding; - public String division; - public String moduleName; - - public CommandLineOptions() { - filename = null; - jar = false; - notice = true; - runModule = false; - properties = new Properties(); - help = version = false; + /** + * Check whether an input stream is interactive. This emulates CPython + * {@code Py_FdIsInteractive} within the constraints of pure Java. + * + * The input stream is considered ``interactive'' if either + *

      + *
    1. it is {@code System.in} and {@code System.console()} is not {@code null}, or
      2. + *
    2. the {@code -i} flag was given ({@link Options#interactive}={@code true}), and the + * filename associated with it is {@code null} or {@code""} or {@code "???"}.
      3. + *
    + * + * @param fp stream (tested only for {@code System.in}) + * @param filename + * @return true iff thought to be interactive + */ + private static boolean isInteractive(InputStream fp, String filename) { + if (fp == System.in && haveConsole()) { + return true; + } else if (!Options.interactive) { + return false; + } else { + return filename == null || filename.equals("") || filename.equals("???"); + } } - public void setProperty(String key, String value) { - properties.put(key, value); + /** Return {@code true} iff the console is accessible through System.console(). */ + private static boolean haveConsole() { try { - System.setProperty(key, value); - } catch (SecurityException e) { - // continue + return System.console() != null; + } catch (SecurityException se) { + return false; + } + } + + /** + * Get the System properties if we are allowed to. Configuration values set via + * {@code -Dprop=value} to the java command will be found here. If a security manager prevents + * access, we will return a new (empty) object instead. + * + * @return {@code System} properties or a new {@code Properties} object + */ + private static Properties getSystemProperties() { + try { + return System.getProperties(); + } catch (AccessControlException ace) { + return new Properties(); + } + } + + /** + * Append strings to a PyList as {@code bytes/str} objects. These might come from the command + * line, or any source with the possibility of non-ascii values. + * + * @param source of {@code String}s + * @param destination list + */ + private static void addFSEncoded(Iterable source, PyList destination) { + for (String s : source) { + destination.add(Py.fileSystemEncode(s)); } } - private boolean argumentExpected(String arg) { - System.err.println("Argument expected for the " + arg + " option"); - return false; - } + /** + * Class providing a parser for Jython command line options. Many of the allowable options set + * values directly in the static {@link Options} as the parser runs, while others set values in + * (an instance) of this class. + */ + static class CommandLineOptions { + + enum Action { + RUN, ERROR, HELP, VERSION + }; - public boolean parse(String[] args) { - int index = 0; + Action action = Action.RUN; + String message = ""; + + String command; + String filename; + String module; + + boolean help = false; + boolean version = false; + boolean jar = false; + + Properties properties = new Properties(); - while (index < args.length && args[index].startsWith("-")) { - String arg = args[index]; - if (arg.equals("-h") || arg.equals("-?") || arg.equals("--help")) { - help = true; - return false; - } else if (arg.equals("-V") || arg.equals("--version")) { - version = true; - return false; - } else if (arg.equals("-")) { - if (!fixInteractive) { - interactive = false; - } - filename = "-"; - } else if (arg.equals("-i")) { - fixInteractive = true; - interactive = true; - } else if (arg.equals("-jar")) { - jar = true; - if (!fixInteractive) { - interactive = false; - } - } else if (arg.equals("-u")) { - Options.unbuffered = true; - } else if (arg.equals("-v")) { - Options.verbose++; - } else if (arg.equals("-vv")) { - Options.verbose += 2; - } else if (arg.equals("-vvv")) { - Options.verbose += 3; - } else if (arg.equals("-s")) { - Options.no_user_site = true; - } else if (arg.equals("-S")) { - Options.importSite = false; - } else if (arg.equals("-B")) { - Options.dont_write_bytecode = true; - } else if (arg.startsWith("-c")) { - runCommand = true; - if (arg.length() > 2) { - command = arg.substring(2); - } else if ((index + 1) < args.length) { - command = args[++index]; - } else { - return argumentExpected(arg); - } - if (!fixInteractive) { - interactive = false; - } - index++; - break; - } else if (arg.startsWith("-W")) { - if (arg.length() > 2) { - warnoptions.add(arg.substring(2)); - } else if ((index + 1) < args.length) { - warnoptions.add(args[++index]); - } else { - return argumentExpected(arg); - } - } else if (arg.equals("-E")) { - // -E (ignore environment variables) - Options.ignore_environment = true; - } else if (arg.startsWith("-D")) { - String key = null; - String value = null; - int equals = arg.indexOf("="); - if (equals == -1) { - String arg2 = args[++index]; - key = arg.substring(2, arg.length()); - value = arg2; - } else { - key = arg.substring(2, equals); - value = arg.substring(equals + 1, arg.length()); - } - setProperty(key, value); - } else if (arg.startsWith("-Q")) { - if (arg.length() > 2) { - division = arg.substring(2); - } else { - division = args[++index]; - } - } else if (arg.startsWith("-m")) { - runModule = true; - if (arg.length() > 2) { - moduleName = arg.substring(2); - } else if ((index + 1) < args.length) { - moduleName = args[++index]; - } else { - return argumentExpected(arg); - } - if (!fixInteractive) { - interactive = false; - } + List argv = new LinkedList(); + List warnoptions = new LinkedList(); + CompilerFlags cf = new CompilerFlags(); + + /** Valid single character options. ':' means expect an argument following. */ + // XJD are extra to CPython. X and J are sanctioned while D is potentially a clash. + static final String PROGRAM_OPTS = "3bBc:dEhim:OQ:RsStuUvVW:x?" + "XJD"; - index++; - int n = args.length - index + 1; - argv = new String[n]; - argv[0] = moduleName; - for (int i = 1; index < args.length; i++, index++) { - argv[i] = args[index]; - } - return true; - } else if (arg.startsWith("-3")) { - Options.py3k_warning = true; - } else { - String opt = args[index]; - if (opt.startsWith("--")) { - opt = opt.substring(2); - } else if (opt.startsWith("-")) { - opt = opt.substring(1); - } - System.err.println("Unknown option: " + opt); - return false; - } - index += 1; - } - notice = interactive; - if (filename == null && index < args.length && command == null) { - filename = args[index++]; - if (!fixInteractive) { - interactive = false; - } - notice = false; - } - if (command != null) { - notice = false; + /** Valid long-name options. */ + static final char JAR_OPTION = '\u2615'; + static final OptionScanner.LongSpec[] PROGRAM_LONG_OPTS = { + // new OptionScanner.LongSpec("-", OptionScanner.DONE), + new OptionScanner.LongSpec("--", OptionScanner.DONE), + new OptionScanner.LongSpec("--help", 'h'), + new OptionScanner.LongSpec("--version", 'v'), + new OptionScanner.LongSpec("-jar", JAR_OPTION, true), // Yes, just one dash. + }; + + /** + * Parse the arguments into the static {@link Options} and a returned instance of this + * class. + * + * @param args from program invocation. + * @return + */ + static CommandLineOptions parse(String args[]) { + CommandLineOptions opts = new CommandLineOptions(); + opts._parse(args); + return opts; } - int n = args.length - index + 1; - - /* Exceptionally we allow -J-Dcpython_cmd=... also postpone the filename. - * E.g. the Linux launcher allows this already on launcher level for all - * -J flags, while the Windows launcher does not. - * - * Todo: Resolve this discrepancy! - * - * This is required to use cpython_cmd property in context of pip, e.g. - * pip install --global-option="-J-Dcpython_cmd=python" - * For details about the cpython_cmd property, look into - * org.python.compiler.Module.loadPyBytecode source. - */ - int cpython_cmd_pos = -1; - for (int i = index; i < args.length; i++) { - if (args[i].startsWith("-J-Dcpython_cmd=")) { - cpython_cmd_pos = i; - System.setProperty("cpython_cmd", args[i].substring(16)); - n--; - break; + /** Parser implementation. Do not call this twice on the same instance. */ + private void _parse(String args[]) { + // Create a scanner with the right tables for Python/Jython + OptionScanner scanner = new OptionScanner(args, PROGRAM_OPTS, PROGRAM_LONG_OPTS); + _parse(scanner, args); + if (action == Action.RUN) { + // Squirrel away the unprocessed arguments + while (scanner.countRemainingArguments() > 0) { + argv.add(scanner.getWholeArgument()); + } } } - argv = new String[n]; - if (filename != null) { - argv[0] = filename; - } else if (command != null) { - argv[0] = "-c"; - } else { - argv[0] = ""; - } + /** + * Parse options into object state, until we encounter the first argument. This is a helper + * to {@link #_parse(String[])}. + */ + private void _parse(OptionScanner scanner, String args[]) { + char c; + /* + * The default action is RUN, taken when we all the options have been processed, and + * either we have run out of arguments (we'll start an interactive session) or + * encountered a non-option argument, which ought to name the file to execute. + * Executable options (like -m and -c) cause a return with action==RUN from their case. + * Any errors, and some special options like --help and -V, return set some other action + * than RUN, ending the loop. + */ + while (action == Action.RUN && (c = scanner.getOption()) != OptionScanner.DONE) { + + switch (c) { + /* + * The first 4 cases all terminate the options in with a RUN action, meaning + * that this option defines the executable script and the arguments following + * will be passed to the script. + */ + case 'c': + /* + * -c is the last option; following arguments that look like options are + * left for the command to interpret. + */ + command = scanner.getOptionArgument() + "\n"; + return; + + case 'm': + /* + * -m is the last option; following arguments that look like options are + * left for the module to interpret. + */ + module = scanner.getOptionArgument(); + return; + + case JAR_OPTION: + /* + * -jar is the last option; following arguments that look like options are + * left for __run__.py to interpret. + */ + jar = true; + filename = scanner.getOptionArgument(); + return; + + case OptionScanner.ARGUMENT: + /* + * This should be a file name (or "-", meaning stdin); following arguments + * that look like options are left for the code it contains to interpret. + */ + filename = scanner.getWholeArgument(); + return; + + // Options that don't terminate option processing (mostly). + + case 'b': + case 'd': + optionNotSupported(c); + break; + + case '3': + Options.py3k_warning = true; + if (Options.division_warning == 0) { + Options.division_warning = 1; + } + break; + + case 'Q': + switch (scanner.getOptionArgument()) { + case "old": + Options.division_warning = 0; + break; + case "warn": + Options.division_warning = 1; + break; + case "warnall": + Options.division_warning = 2; + break; + case "new": + Options.Qnew = true; + default: + error("-Q option should be `-Qold', " + + "`-Qwarn', `-Qwarnall', or `-Qnew' only"); + } + break; + + case 'i': + Options.inspect = Options.interactive = true; + break; - if (cpython_cmd_pos == -1) { - for (int i = 1; i < n; i++, index++) { - argv[i] = args[index]; - } - } else { - for (int i = 1; i < n; i++, index++) { - if (index == cpython_cmd_pos) { - index++; + case 'O': + Options.optimize++; + break; + + case 'B': + Options.dont_write_bytecode = true; + break; + + case 's': + Options.no_user_site = true; + break; + + case 'S': + Options.no_site = true; + Options.importSite = false; + break; + + case 'E': + Options.ignore_environment = true; + break; + + case 't': + optionNotSupported(c); + // Py_TabcheckFlag++; + break; + + case 'u': + Options.unbuffered = true; + break; + + case 'v': + Options.verbose++; + break; + + case 'x': + optionNotSupported(c); + // skipfirstline = true; + break; + + // case 'X': reserved for implementation-specific arguments + + case 'U': + optionNotSupported(c); + // Py_UnicodeFlag++; + break; + + case 'W': + warnoptions.add(scanner.getOptionArgument()); + break; + + case 'R': + optionNotSupported(c); + break; + + case 'D': + // Definitions made on the command line: -Dprop=v + try { + optionD(scanner); + } catch (SecurityException e) { + // Prevented by security policy. + } + break; + + // Options that terminate option processing with something other than RUN. + + case 'h': + case '?': + action = Action.HELP; + break; + + case 'V': + action = Action.VERSION; + break; + + case 'J': + /* + * This shouldn't happen because the launcher should have recognised this + * and converted it to an option or argument to the java command. If it + * shows up here, maybe it was supplied outside the loader or the context + * has confused the launcher. + */ + error("-J is only valid when using the Jython launcher. " + + "In a complex command, put the -J options early."); + break; + + case OptionScanner.ERROR: + error(scanner.getMessage()); + break; + + default: + // Acceptable to the scanner, but missing from the case statement? + error("parser did not recognise option -%c \\u%04x", c, c); + break; } - argv[i] = args[index]; } } - return true; + /** + * Helper for option {@code -Dprop=v}. This is potentially a clash with Python: work around + * for luncher misplacement of -J-D...? + */ + private void optionD(OptionScanner scanner) throws SecurityException { + String[] kv = scanner.getWholeArgument().split("=", 2); + String prop = kv[0].trim(); + if (kv.length > 1) { + properties.put(prop, kv[1]); + } else { + properties.put(prop, ""); + } + } + + /** + * Set the error message as {@code String.format(message, args)} and set the action to + * {@link Action#ERROR}. + */ + private void error(String message, Object... args) { + this.message = args.length == 0 ? message : String.format(message, args); + action = Action.ERROR; + } } } -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Sep 23 09:57:17 2018 From: jython-checkins at python.org (jeff.allen) Date: Sun, 23 Sep 2018 13:57:17 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Unify_approach_to_isIntera?= =?utf-8?b?Y3RpdmUoKS4=?= Message-ID: <20180923135717.1.7044C23B0C018BEE@mg.python.org> https://hg.python.org/jython/rev/fbe8e11c24c8 changeset: 8184:fbe8e11c24c8 user: Jeff Allen date: Sun Sep 23 09:02:44 2018 +0100 summary: Unify approach to isInteractive(). This change combines the emulation of CPython Py_FdIsInteractive, evolved in the rework of jython.main, with the launcher-based detection from #2325. The simple test there "echo 'print 1' | dist\bin\jython" still produces output identical with CPython's. files: src/org/python/core/PrePy.java | 51 ++++++++++++- src/org/python/core/Py.java | 36 --------- src/org/python/core/PySystemState.java | 11 +- src/org/python/util/PyServlet.java | 3 +- 4 files changed, 50 insertions(+), 51 deletions(-) diff --git a/src/org/python/core/PrePy.java b/src/org/python/core/PrePy.java --- a/src/org/python/core/PrePy.java +++ b/src/org/python/core/PrePy.java @@ -45,6 +45,44 @@ } } + /** + * Get a System property if it is defined, not null, and we are allowed to access it, otherwise + * return the given default. + * + * @param key of the entry to return + * @param defaultValue to return if null or disallowed + * @return property value or given default + */ + public static String getSystemProperty(String key, String defaultValue) { + try { + String value = System.getProperty(key, null); + return value != null ? value : defaultValue; + } catch (AccessControlException ace) { + return defaultValue; + } + } + + /** + * Determine whether standard input is an interactive stream. If the Java system property + * {@code python.launcher.tty} is defined and equal to {@code true} or {@code false}, then that + * provides the result. This property is normally supplied by the launcher. In the absence of + * this certainty, we use {@link #haveConsole()}. + * + * @return true if (we think) standard input is an interactive stream + */ + public static boolean isInteractive() { + // python.launcher.tty is authoritative; see http://bugs.jython.org/issue2325 + String tty = getSystemProperty("python.launcher.tty", ""); + if (tty.equalsIgnoreCase("true")) { + return true; + } else if (tty.equalsIgnoreCase("false")) { + return false; + } else { + // See if we have access to System.console() + return haveConsole(); + } + } + /** Return {@code true} iff the console is accessible through System.console(). */ public static boolean haveConsole() { try { @@ -56,11 +94,10 @@ /** * Check whether an input stream is interactive. This emulates CPython - * {@code Py_FdIsInteractive} within the constraints of pure Java. - * - * The input stream is considered ``interactive'' if either + * {@code Py_FdIsInteractive} within the constraints of pure Java. The input stream is + * considered ``interactive'' if either *
      - *
    1. it is {@code System.in} and {@code System.console()} is not {@code null}, or
      2. + *
    2. it is {@code System.in} and {@link #isInteractive()} is {@code true}, or
      3. *
    3. the {@code -i} flag was given ({@link Options#interactive}={@code true}), and the * filename associated with it is {@code null} or {@code""} or {@code "???"}.
      4. *
    @@ -70,7 +107,7 @@ * @return true iff thought to be interactive */ public static boolean isInteractive(InputStream fp, String filename) { - if (fp == System.in && haveConsole()) { + if (fp == System.in && isInteractive()) { return true; } else if (!Options.interactive) { return false; @@ -133,8 +170,8 @@ * @return the full name of the jar file containing this class, null if not * available. */ - public static String _getJarFileName() { - Class thisClass = Py.class; + private static String _getJarFileName() { + Class thisClass = PrePy.class; String fullClassName = thisClass.getName(); String className = fullClassName.substring(fullClassName.lastIndexOf(".") + 1); URL url = thisClass.getResource(className + ".class"); diff --git a/src/org/python/core/Py.java b/src/org/python/core/Py.java --- a/src/org/python/core/Py.java +++ b/src/org/python/core/Py.java @@ -1831,40 +1831,6 @@ } } - /** - * Determine whether standard input is an interactive stream. This is not the same as - * deciding whether the interpreter is or should be in interactive mode. Amongst other things, - * this affects the type of console that may be legitimately installed during system - * initialisation. - *

    - * If the Java system property {@code python.launcher.tty} is defined and equal to {@code true} - * or {@code false}, then that provides the result. This property is normally supplied by the - * launcher. In the absence of this certainty, we try to find outusing {@code isatty()} in the - * Posix emulation library. Note that the result may vary according to whether a - * jnr-posix native library is found along java.library.path, or the - * pure Java fall-back is used. - * - * @return true if (we think) standard input is an interactive stream - */ - public static boolean isInteractive() { - String tty = System.getProperty("python.launcher.tty"); - if (tty != null) { - // python.launcher.tty is authoritative; see http://bugs.jython.org/issue2325 - tty = tty.toLowerCase(); - if (tty.equals("true")) { - return true; - } else if (tty.equals("false")) { - return false; - } - } - // Base decision on whether System.in is interactive according to OS - try { - POSIX posix = POSIXFactory.getPOSIX(); - return posix.isatty(FileDescriptor.in); - } catch (SecurityException ex) {} - return false; - } - private static final String IMPORT_SITE_ERROR = "" + "Cannot import site module and its dependencies: %s\n" + "Determine if the following attributes are correct:\n" // @@ -2809,7 +2775,6 @@ * @param args constructor-arguments * @return a new instance of the desired class */ - @SuppressWarnings("unchecked") public static T newJ(PyModule module, Class jcls, Object... args) { PyObject cls = module.__getattr__(jcls.getSimpleName().intern()); return newJ(cls, jcls, args); @@ -2834,7 +2799,6 @@ * @param args constructor-arguments * @return a new instance of the desired class */ - @SuppressWarnings("unchecked") public static T newJ(PyModule module, Class jcls, String[] keywords, Object... args) { PyObject cls = module.__getattr__(jcls.getSimpleName().intern()); return newJ(cls, jcls, keywords, args); diff --git a/src/org/python/core/PySystemState.java b/src/org/python/core/PySystemState.java --- a/src/org/python/core/PySystemState.java +++ b/src/org/python/core/PySystemState.java @@ -1064,11 +1064,8 @@ } public static Properties getBaseProperties() { - try { - return System.getProperties(); - } catch (AccessControlException ace) { - return new Properties(); - } + // Moved to PrePy since does not depend on PyObject). Retain in 2.7.x for compatibility. + return PrePy.getSystemProperties(); } public static synchronized void initialize() { @@ -1096,7 +1093,7 @@ return; } if (preProperties == null) { - preProperties = getBaseProperties(); + preProperties = PrePy.getSystemProperties(); } if (postProperties == null) { postProperties = new Properties(); @@ -1197,7 +1194,7 @@ initialized = true; Py.setAdapter(adapter); boolean standalone = false; - String jarFileName = Py._getJarFileName(); + String jarFileName = Py.getJarFileName(); if (jarFileName != null) { standalone = isStandalone(jarFileName); } diff --git a/src/org/python/util/PyServlet.java b/src/org/python/util/PyServlet.java --- a/src/org/python/util/PyServlet.java +++ b/src/org/python/util/PyServlet.java @@ -15,6 +15,7 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; +import org.python.core.PrePy; import org.python.core.Py; import org.python.core.PyException; import org.python.core.PyObject; @@ -98,7 +99,7 @@ protected static void init(Properties props, ServletContext context) { String rootPath = getRootPath(context); context.setAttribute(INIT_ATTR, true); - Properties baseProps = PySystemState.getBaseProperties(); + Properties baseProps = PrePy.getSystemProperties(); // Context parameters Enumeration e = context.getInitParameterNames(); while (e.hasMoreElements()) { -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Wed Sep 26 14:31:17 2018 From: jython-checkins at python.org (jeff.allen) Date: Wed, 26 Sep 2018 18:31:17 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_handling_of_-Dprop=3Dv?= =?utf-8?q?_in_Jython_main=28=29_options_parser=2E?= Message-ID: <20180926183117.1.D2F831B779D96625@mg.python.org> https://hg.python.org/jython/rev/abd1592dfee4 changeset: 8185:abd1592dfee4 user: Jeff Allen date: Tue Sep 25 19:05:56 2018 +0100 summary: Fix handling of -Dprop=v in Jython main() options parser. files: src/org/python/util/OptionScanner.java | 1 + src/org/python/util/jython.java | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/org/python/util/OptionScanner.java b/src/org/python/util/OptionScanner.java --- a/src/org/python/util/OptionScanner.java +++ b/src/org/python/util/OptionScanner.java @@ -178,6 +178,7 @@ * returned. This advances the internal state to the next argument. */ String getWholeArgument() { + optIndex = 0; return args[argIndex++]; } diff --git a/src/org/python/util/jython.java b/src/org/python/util/jython.java --- a/src/org/python/util/jython.java +++ b/src/org/python/util/jython.java @@ -725,8 +725,8 @@ List warnoptions = new LinkedList(); /** Valid single character options. ':' means expect an argument following. */ - // XJD are extra to CPython. X and J are sanctioned while D is potentially a clash. - static final String PROGRAM_OPTS = "3bBc:dEhim:OQ:RsStuUvVW:x?" + "XJD"; + // XJD are extra to CPython. X and J are sanctioned while D may one day clash. + static final String PROGRAM_OPTS = "3bBc:dEhim:OQ:RsStuUvVW:x?" + "XJD:"; /** Valid long-name options. */ static final char JAR_OPTION = '\u2615'; @@ -953,11 +953,12 @@ } /** - * Helper for option {@code -Dprop=v}. This is potentially a clash with Python: work around - * for launcher misplacement of -J-D...? + * Helper for option {@code -Dprop=v}, adding to the "post-properties". (This is a + * clash-in-waiting with Python.) The effect is slightly different from {@code -J-Dprop=v}, + * which contributes to the "pre-properties". */ private void optionD(OptionScanner scanner) throws SecurityException { - String[] kv = scanner.getWholeArgument().split("=", 2); + String[] kv = scanner.getOptionArgument().split("=", 2); String prop = kv[0].trim(); if (kv.length > 1) { properties.put(prop, kv[1]); -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Thu Sep 27 03:20:31 2018 From: jython-checkins at python.org (jeff.allen) Date: Thu, 27 Sep 2018 07:20:31 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Remove_-E_from_jython=2Epy?= =?utf-8?q?_=28fixes_=232707=29_and_improve_--print_on_Linux?= Message-ID: <20180927072031.1.639B01AA8B55F19D@mg.python.org> https://hg.python.org/jython/rev/38824a8816a8 changeset: 8186:38824a8816a8 user: Jeff Allen date: Wed Sep 26 22:47:02 2018 +0100 summary: Remove -E from jython.py (fixes #2707) and improve --print on Linux Adds quoting and escaping to the output from the launcher when --print is specified, lost in a previous change, sufficient for test_jython_launcher to pass. See #2686. The shebang line of jython.py was not valid on Linux where the entire line after the first space is treated as one argument, thus python2.7 is all we can have after usr/bin/env. See #2707. files: src/shell/jython.py | 39 +++++++++++++++++++++++++++----- 1 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/shell/jython.py b/src/shell/jython.py --- a/src/shell/jython.py +++ b/src/shell/jython.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 -E +#!/usr/bin/env python2.7 # -*- coding: utf-8 -*- # Launch script for Jython. It may be run directly (note the shebang line), but @@ -13,10 +13,8 @@ # Developers' Guide). import glob -import inspect import os import os.path -import pipes import shlex import subprocess import sys @@ -219,8 +217,8 @@ # Frozen. Let it go with the executable path. bytes_path = sys.executable else: - # Not frozen. Any object defined in this file will do. - bytes_path = inspect.getfile(JythonCommand) + # Not frozen. Use the __file__ of this module.. + bytes_path = __file__ # Python 2 thinks in bytes. Carefully normalise in Unicode. path = os.path.realpath(bytes_path.decode(ENCODING)) try: @@ -499,6 +497,33 @@ opts = shlex.split(opts) return decode_list(opts) +def maybe_quote(s): + """ Enclose the string argument in single quotes if it looks like it needs it. + Spaces and quotes will trigger; single quotes in the argument are escaped. + This is only used to compose the --print output so need only satisfy shlex. + """ + NEED_QUOTE = u" \t\"\\'" + clean = True + for c in s: + if c in NEED_QUOTE: + clean = False + break + if clean: return s + # Something needs quoting or escaping. + QUOTE = u"'" + ESC = u"\\" + arg = [QUOTE] + for c in s: + if c == QUOTE: + arg.append(QUOTE) + arg.append(ESC) + arg.append(QUOTE) + elif c == ESC: + arg.append(ESC) + arg.append(c) + arg.append(QUOTE) + return ''.join(arg) + def main(sys_args): # The entire program must work in Unicode sys_args = decode_list(sys_args) @@ -532,7 +557,9 @@ # Normally used for a byte strings but Python is tolerant :) command_line = subprocess.list2cmdline(command) else: - # Just concatenate with spaces + # Transform any element that seems to need quotes + command = map(maybe_quote, command) + # Now concatenate with spaces command_line = u" ".join(command) # It is possible the Unicode cannot be encoded for the console enc = sys.stdout.encoding or 'ascii' -- Repository URL: https://hg.python.org/jython