[Jython-checkins] jython: Enabled org.python.compiler.Module.loadPyBytecode to automatically run CPython
stefan.richthofer
jython-checkins at python.org
Mon Feb 27 12:24:29 EST 2017
https://hg.python.org/jython/rev/03f4808038f8
changeset: 8037:03f4808038f8
user: Stefan Richthofer <stefan.richthofer at gmx.de>
date: Mon Feb 27 18:24:04 2017 +0100
summary:
Enabled org.python.compiler.Module.loadPyBytecode to automatically run CPython 2.7 bytecode compilation, if a CPython 2.7 command is provided in property 'cpython_cmd'. Took care that this can be passed through pip e.g. using --global-option='-J-Dcpython_cmd=python'; note that pip launches separate Jython processes. Without that option, running e.g. pip install sympy is tedious, because it will prompt you several times to provide yet another pyc-file.
files:
NEWS | 4 +
src/org/python/compiler/Module.java | 93 ++++++++++++++--
src/org/python/util/jython.java | 35 ++++++-
3 files changed, 119 insertions(+), 13 deletions(-)
diff --git a/NEWS b/NEWS
--- a/NEWS
+++ b/NEWS
@@ -61,6 +61,10 @@
- [ 1767 ] Rich comparisons
New Features
+ - Recognize cpython_cmd property to automatically build CPython bytecode for oversized
+ functions (e.g. jython -J-Dcpython_cmd=python). This is especially convenient when
+ installing things like SymPy via pip; it would frequently prompt you to provide yet
+ another pyc-file. Now just run: pip install --global-option="-J-Dcpython_cmd=python" sympy
- SymPy is now workable. (However it runs somewhat slow; some profiling will be required.)
- Updated ucnhash to support name lookup for Unicode 9.0 (like in u'\N{name}').
- Jython doc-entries in Java-code (__doc__foo) now accept anything that implements
diff --git a/src/org/python/compiler/Module.java b/src/org/python/compiler/Module.java
--- a/src/org/python/compiler/Module.java
+++ b/src/org/python/compiler/Module.java
@@ -10,6 +10,8 @@
import java.io.ByteArrayOutputStream;
import java.io.ObjectOutputStream;
import java.io.File;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.Hashtable;
@@ -712,8 +714,20 @@
module.mainCode = main;
}
- private static PyBytecode loadPyBytecode(String filename) throws RuntimeException
+ private static PyBytecode loadPyBytecode(String filename, boolean try_cpython)
+ throws RuntimeException
{
+ String cpython_cmd_msg =
+ "\n\nAlternatively provide proper CPython 2.7 execute command via"+
+ "\ncpython_cmd property, e.g. call "+
+ "\n jython -J-Dcpython_cmd=python"+
+ "\nor if running pip on Jython:"+
+ "\n pip install --global-option=\"-J-Dcpython_cmd=python\" <package>";
+ String large_method_msg = "\nEncountered too large method code in \n"+filename+"\n";
+ String please_provide_msg =
+ "\nPlease provide a CPython 2.7 bytecode file (.pyc) to proceed, e.g. run"+
+ "\npython -m py_compile "+filename+"\nand try again.";
+
String pyc_filename = filename+"c";
File pyc_file = new File(pyc_filename);
if (pyc_file.exists()) {
@@ -726,11 +740,10 @@
// (bts[5]<< 8) & 0x0000FF00 |
// (bts[4]<< 0) & 0x000000FF;
if (magic != 62211) { // check Python 2.7 bytecode
- throw new RuntimeException("Encountered too large method code in \n"+filename+
+ throw new RuntimeException(large_method_msg+
"\n"+pyc_filename+
"\ncontains wrong bytecode version, not CPython 2.7 bytecode."+
- "\nPlease provide a CPython 2.7 bytecode file (.pyc) to proceed, e.g. run"+
- "\npython -m py_compile "+filename+"\nand try again.");
+ please_provide_msg);
}
_marshal.Unmarshaller un = new _marshal.Unmarshaller(f);
PyObject code = un.load();
@@ -738,15 +751,73 @@
if (code instanceof PyBytecode) {
return (PyBytecode) code;
}
- throw new RuntimeException("Encountered too large method code in \n"+filename+
+ throw new RuntimeException(large_method_msg+
"\n"+pyc_filename+
"\ncontains invalid bytecode."+
- "\nPlease provide a CPython 2.7 bytecode file (.pyc) to proceed, e.g. run"+
- "\npython -m py_compile "+filename+"\nand try again.");
+ please_provide_msg);
} else {
- throw new RuntimeException("Encountered too large method code in \n"+filename+
- "\nPlease provide a CPython 2.7 bytecode file (.pyc) to proceed, e.g. run"+
- "\npython -m py_compile "+filename+"\nand try again.");
+ String CPython_command = System.getProperty("cpython_cmd");
+ if (try_cpython && CPython_command != null) {
+ // check version...
+ String command_ver = CPython_command+" --version";
+ String command = CPython_command+" -m py_compile "+filename;
+ String tried_create_pyc_msg = "\nTried to create pyc-file by executing\n"+
+ command+"\nThis failed because of\n";
+ Exception exc = null;
+ int result = 0;
+ try {
+ Process p = Runtime.getRuntime().exec(command_ver);
+ // Python 2.7 writes version to error-stream for some reason:
+ BufferedReader br = new BufferedReader(new InputStreamReader(p.getErrorStream()));
+ String cp_version = br.readLine();
+ while (br.readLine() != null) {}
+ br.close();
+ if (cp_version == null) {
+ // Also try input-stream as fallback, just in case...
+ br = new BufferedReader(new InputStreamReader(p.getInputStream()));
+ cp_version = br.readLine();
+ while (br.readLine() != null) {}
+ br.close();
+ }
+ result = p.waitFor();
+ if (!cp_version.startsWith("Python 2.7.")) {
+ throw new RuntimeException(large_method_msg+
+ tried_create_pyc_msg+
+ "wrong Python version: "+cp_version+"."+
+ "\nRequired is Python 2.7.x.\n"+
+ please_provide_msg+cpython_cmd_msg);
+ }
+ }
+ catch (InterruptedException ie) {
+ exc = ie;
+ }
+ catch (IOException ioe) {
+ exc = ioe;
+ }
+ if (exc == null && result == 0) {
+ try {
+ Process p = Runtime.getRuntime().exec(command);
+ result = p.waitFor();
+ if (result == 0) {
+ return loadPyBytecode(filename, false);
+ }
+ }
+ catch (InterruptedException ie) {
+ exc = ie;
+ }
+ catch (IOException ioe) {
+ exc = ioe;
+ }
+ }
+ String exc_msg = large_method_msg+
+ tried_create_pyc_msg+
+ (exc != null ? exc.toString() : "bad return: "+result)+".\n"+
+ please_provide_msg+cpython_cmd_msg;
+ throw exc != null ? new RuntimeException(exc_msg, exc) : new RuntimeException(exc_msg);
+ } else {
+ throw new RuntimeException(large_method_msg+
+ please_provide_msg+cpython_cmd_msg);
+ }
}
}
@@ -841,7 +912,7 @@
module.write(ostream);
} catch (RuntimeException re) {
if (re.getMessage() != null && re.getMessage().equals("Method code too large!")) {
- PyBytecode btcode = loadPyBytecode(filename);
+ PyBytecode btcode = loadPyBytecode(filename, true);
int thresh = 22000;
// No idea, how to determine at this point if a method is oversized, so we just try
// a threshold regarding Python code-length, while JVM restriction is actually about
diff --git a/src/org/python/util/jython.java b/src/org/python/util/jython.java
--- a/src/org/python/util/jython.java
+++ b/src/org/python/util/jython.java
@@ -682,6 +682,28 @@
}
int n = args.length - index + 1;
+
+ /* Exceptionally we allow -J-Dcpython_cmd=... also postpone the filename.
+ * E.g. the Linux launcher allows this already on launcher level for all
+ * -J flags, while the Windows launcher does not.
+ *
+ * Todo: Resolve this discrepancy!
+ *
+ * This is required to use cpython_cmd property in context of pip, e.g.
+ * pip install --global-option="-J-Dcpython_cmd=python" <package>
+ * For details about the cpython_cmd property, look into
+ * org.python.compiler.Module.loadPyBytecode source.
+ */
+ int cpython_cmd_pos = -1;
+ for (int i = index; i < args.length; i++) {
+ if (args[i].startsWith("-J-Dcpython_cmd=")) {
+ cpython_cmd_pos = i;
+ System.setProperty("cpython_cmd", args[i].substring(16));
+ n--;
+ break;
+ }
+ }
+
argv = new String[n];
if (filename != null) {
argv[0] = filename;
@@ -691,8 +713,17 @@
argv[0] = "";
}
- for (int i = 1; i < n; i++, index++) {
- argv[i] = args[index];
+ if (cpython_cmd_pos == -1) {
+ for (int i = 1; i < n; i++, index++) {
+ argv[i] = args[index];
+ }
+ } else {
+ for (int i = 1; i < n; i++, index++) {
+ if (index == cpython_cmd_pos) {
+ index++;
+ }
+ argv[i] = args[index];
+ }
}
return true;
--
Repository URL: https://hg.python.org/jython
More information about the Jython-checkins
mailing list