[Jython-checkins] jython: Fix star-import on Java 9 for selected modules, partially addresses #2362.
jeff.allen
jython-checkins at python.org
Sun Nov 25 03:08:35 EST 2018
https://hg.python.org/jython/rev/f662c11b87a0
changeset: 8198:f662c11b87a0
user: Jeff Allen <ja.py at farowl.co.uk>
date: Sun Nov 25 07:24:37 2018 +0000
summary:
Fix star-import on Java 9 for selected modules, partially addresses #2362.
CachedJarsPackageManager now creates PyJavaPackage objects with class lists,
for packages in modules, as we already do for JARs. This is used, when the
"jrt:" filesystem exists (Java 9), from SysPackageManager::findAllPackages().
Some refactoring and a lot of new comment is included, for intelligibility in
this and further work.
No cache file is created at present, and we process only a configured list of
modules, both shortcomings we may address in further work.
An update of asm to 7.0 is necessary to parse the accessibility information
from Java runtime classes without instantiating them.
files:
Lib/test/import_nonexistent.py | 3 +
Lib/test/import_star_from_java.py | 13 +-
Lib/test/test_import_jy.py | 3 -
build.gradle | 7 +-
build.xml | 18 +-
extlibs/asm-5.2.jar | Bin
extlibs/asm-7.0.jar | Bin
extlibs/asm-commons-5.2.jar | Bin
extlibs/asm-commons-7.0.jar | Bin
extlibs/asm-util-5.2.jar | Bin
extlibs/asm-util-7.0.jar | Bin
registry | 9 +-
src/org/python/core/PyJavaPackage.java | 36 +-
src/org/python/core/packagecache/CachedJarsPackageManager.java | 431 +++++++--
src/org/python/core/packagecache/PackageManager.java | 62 +-
src/org/python/core/packagecache/PathPackageManager.java | 14 +-
src/org/python/core/packagecache/SysPackageManager.java | 82 +-
17 files changed, 477 insertions(+), 201 deletions(-)
diff --git a/Lib/test/import_nonexistent.py b/Lib/test/import_nonexistent.py
--- a/Lib/test/import_nonexistent.py
+++ b/Lib/test/import_nonexistent.py
@@ -1,3 +1,6 @@
+# Test material for:
+# SecurityManagerTest.test_nonexistent_import_with_security (test_java_integration.py)
+
try:
import nonexistent_module
except ImportError:
diff --git a/Lib/test/import_star_from_java.py b/Lib/test/import_star_from_java.py
--- a/Lib/test/import_star_from_java.py
+++ b/Lib/test/import_star_from_java.py
@@ -1,4 +1,13 @@
-from java.util.regex import *
-
+# Test material for ImpTestCase.test_import_star (test_import_jy.py)
+#
+from java.util.regex import * # Module: java.base
p = Pattern.compile("foo")
assert p.flags() == 0
+
+from java.sql import * # Module: java.sql
+d = Date(1541492230300L)
+assert str(d) == '2018-11-06'
+
+from java.awt import * # Module: java.desktop
+assert Color(255,0,255) == Color.MAGENTA
+
diff --git a/Lib/test/test_import_jy.py b/Lib/test/test_import_jy.py
--- a/Lib/test/test_import_jy.py
+++ b/Lib/test/test_import_jy.py
@@ -172,9 +172,6 @@
# causes a stack overflow if the bug occurs
self.assertRaises(Exception, getattr, anygui, 'abc')
- @unittest.skipIf(test_support.get_java_version() >= (9,),
- "Fails on Java 9+. See b.j.o. issue #2362") # FIXME
- # Probably related to Java modules: ensure also works outside java.base
def test_import_star(self):
self.assertEquals(0, subprocess.call(
[sys.executable, test_support.findfile("import_star_from_java.py")]))
diff --git a/build.gradle b/build.gradle
--- a/build.gradle
+++ b/build.gradle
@@ -149,10 +149,9 @@
implementation group: 'org.apache.commons', name: 'commons-compress', version: '1.14'
- // pin to ASM 5.2 until we upgrade compilation
- implementation group: 'org.ow2.asm', name: 'asm', version: '5.2'
- implementation group: 'org.ow2.asm', name: 'asm-commons', version: '5.2'
- implementation group: 'org.ow2.asm', name: 'asm-util', version: '5.2'
+ implementation group: 'org.ow2.asm', name: 'asm', version: '7.0'
+ implementation group: 'org.ow2.asm', name: 'asm-commons', version: '7.0'
+ implementation group: 'org.ow2.asm', name: 'asm-util', version: '7.0'
api group: 'com.google.guava', name: 'guava', version: '22.0-android'
implementation group: 'com.ibm.icu', name: 'icu4j', version: '59.1'
diff --git a/build.xml b/build.xml
--- a/build.xml
+++ b/build.xml
@@ -148,10 +148,9 @@
<pathelement path="${extlibs.dir}/antlr-3.1.3.jar" />
<pathelement path="${extlibs.dir}/stringtemplate-3.2.1.jar" />
<pathelement path="${extlibs.dir}/commons-compress-1.14.jar"/>
- <!-- pin to ASM 5.2 until we upgrade compilation -->
- <pathelement path="${extlibs.dir}/asm-5.2.jar" />
- <pathelement path="${extlibs.dir}/asm-commons-5.2.jar" />
- <pathelement path="${extlibs.dir}/asm-util-5.2.jar" />
+ <pathelement path="${extlibs.dir}/asm-7.0.jar" />
+ <pathelement path="${extlibs.dir}/asm-commons-7.0.jar" />
+ <pathelement path="${extlibs.dir}/asm-util-7.0.jar" />
<pathelement path="${extlibs.dir}/guava-22.0-android.jar" />
<pathelement path="${extlibs.dir}/icu4j-59_1.jar" />
<pathelement path="${extlibs.dir}/jffi-1.2.16.jar"/>
@@ -174,8 +173,8 @@
<path id="test.classpath">
<path refid="main.classpath"/>
- <pathelement path="${extlibs.dir}/asm-commons-5.2.jar" />
- <pathelement path="${extlibs.dir}/asm-util-5.2.jar" />
+ <pathelement path="${extlibs.dir}/asm-commons-7.0.jar" />
+ <pathelement path="${extlibs.dir}/asm-util-7.0.jar" />
<pathelement path="${extlibs.dir}/junit-4.10.jar" />
<pathelement path="${exposed.dir}" />
<pathelement path="${compile.dir}" />
@@ -551,10 +550,9 @@
<!-- pin to Antlr 3.1.3 until we upgrade parsing -->
<zipfileset src="extlibs/antlr-runtime-3.1.3.jar"/>
<rule pattern="org.antlr.runtime.**" result="org.python.antlr.runtime. at 1"/>
- <!-- pin to ASM 5.0.3 until we upgrade compilation -->
- <zipfileset src="extlibs/asm-5.2.jar"/>
- <zipfileset src="extlibs/asm-commons-5.2.jar"/>
- <zipfileset src="extlibs/asm-util-5.2.jar"/>
+ <zipfileset src="extlibs/asm-7.0.jar"/>
+ <zipfileset src="extlibs/asm-commons-7.0.jar"/>
+ <zipfileset src="extlibs/asm-util-7.0.jar"/>
<rule pattern="org.objectweb.asm.**" result="org.python.objectweb.asm. at 1"/>
<zipfileset src="extlibs/bcpkix-jdk15on-1.57.jar" excludes="META-INF/**"/>
<rule pattern="org.bouncycastle.**" result="org.python.bouncycastle. at 1"/>
diff --git a/extlibs/asm-5.2.jar b/extlibs/asm-5.2.jar
deleted file mode 100644
index aea11818d5181162d58cb2c206cd2243f9357b30..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/asm-7.0.jar b/extlibs/asm-7.0.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..2cf38f46167192e6d65c4906c77cd7e3dce5598d
GIT binary patch
[stripped]
diff --git a/extlibs/asm-commons-5.2.jar b/extlibs/asm-commons-5.2.jar
deleted file mode 100644
index cdd2e45c99bdeceafb8cb78a0fa195a2cdb53a37..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/asm-commons-7.0.jar b/extlibs/asm-commons-7.0.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..ba0de6cd49dc34cb2ea5423b1a16f1122a44bdca
GIT binary patch
[stripped]
diff --git a/extlibs/asm-util-5.2.jar b/extlibs/asm-util-5.2.jar
deleted file mode 100644
index 686c3f0d04245fc1ed2d6459d2ad265033e0d62d..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/asm-util-7.0.jar b/extlibs/asm-util-7.0.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..47717e5d18f9b3a6df1300c9872541b3c895d2bc
GIT binary patch
[stripped]
diff --git a/registry b/registry
--- a/registry
+++ b/registry
@@ -23,9 +23,14 @@
# Properties to check for initializing and updating the package cache
# Values shown here are those hard-coded in Jython's cache manager.
# Treat JARs on the classpath and (before Java 9) in the JRE as a source of Python packages.
-#python.packages.paths = java.class.path, sun.boot.class.path
+#python.packages.paths = java.class.path, sun.boot.class.path # before Java 9
+#python.packages.paths = java.class.path # from Java 9
# Treat installed optional (Java) packages as source of Python packages (before Java 9)
-#python.packages.directories = java.ext.dirs
+#python.packages.directories = java.ext.dirs # before Java 9
+#python.packages.directories # undefined from Java 9
+# Treat the following Java modules as sources of of Python packages (from Java 9)
+#python.packages.modules = java.base, java.desktop, java.logging, java.se, java.sql, java.xml
+
# Set verbosity to error, warning, message, comment, or debug
# for varying levels of informative messages from Jython. Normally
diff --git a/src/org/python/core/PyJavaPackage.java b/src/org/python/core/PyJavaPackage.java
--- a/src/org/python/core/PyJavaPackage.java
+++ b/src/org/python/core/PyJavaPackage.java
@@ -5,7 +5,7 @@
import org.python.core.packagecache.PackageManager;
-import java.util.StringTokenizer;
+import java.util.Collection;
/** A representation of java package. */
public class PyJavaPackage extends PyObject implements Traverseproc {
@@ -48,13 +48,25 @@
return addPackage(name, null);
}
+ /**
+ * From a dotted package name {@code a.b.c} interpreted relative to this package {@code t},
+ * ensure that {@code a} is in the dictionary of {@code t} and then recursively process the
+ * remainder {@code b.c} relative to {@code a}, finally returning the {@link #PyJavaPackage} of
+ * {@code t.a.b.c}. In the case where the initial package name is just {@code a}, no dots, the
+ * method simply ensures {@code a} is entered in {@code t}, and returns the
+ * {@link PyJavaPackage} of {@code t.a}.
+ *
+ * @param name a package name
+ * @param jarfile becomes the {@code __file__} attribute
+ * @return the {@link PyJavaPackage} of the package named
+ */
public PyJavaPackage addPackage(String name, String jarfile) {
int dot = name.indexOf('.');
String firstName = name;
- String lastName = null;
+ String remainder = null;
if (dot != -1) {
firstName = name.substring(0, dot);
- lastName = name.substring(dot + 1, name.length());
+ remainder = name.substring(dot + 1, name.length());
}
firstName = firstName.intern();
PyJavaPackage p = (PyJavaPackage) __dict__.__finditem__(firstName);
@@ -63,13 +75,12 @@
p = new PyJavaPackage(pname, __mgr__, jarfile);
__dict__.__setitem__(firstName, p);
} else {
- // this code is ok here, because this is not needed for
- // a top level package
+ // this code is ok here, because this is not needed for a top level package
if (jarfile == null || !jarfile.equals(p.__file__)) {
p.__file__ = null;
}
}
- return lastName != null ? p.addPackage(lastName, jarfile) : p;
+ return remainder != null ? p.addPackage(remainder, jarfile) : p;
}
public PyObject addClass(String name, Class<?> c) {
@@ -79,15 +90,14 @@
}
/**
- * Add statically known classes.
+ * Add the classes named to this package, but with only a placeholder value. These names are
+ * statically known, typically from processing JAR files on the class path.
*
- * @param classes their names as comma-separated string
+ * @param classes the names as strings
*/
- public void addPlaceholders(String classes) {
- StringTokenizer tok = new StringTokenizer(classes, ",@");
- while (tok.hasMoreTokens()) {
- String p = tok.nextToken();
- String name = p.trim().intern();
+ public void addPlaceholders(Collection<String> classes) {
+ for (String name : classes) {
+ name = name.intern();
if (clsSet.__finditem__(name) == null) {
clsSet.__setitem__(name, Py.One);
}
diff --git a/src/org/python/core/packagecache/CachedJarsPackageManager.java b/src/org/python/core/packagecache/CachedJarsPackageManager.java
--- a/src/org/python/core/packagecache/CachedJarsPackageManager.java
+++ b/src/org/python/core/packagecache/CachedJarsPackageManager.java
@@ -4,6 +4,7 @@
package org.python.core.packagecache;
import org.python.core.Options;
+import org.python.core.PyJavaPackage;
import org.python.util.Generic;
import java.io.BufferedInputStream;
@@ -19,7 +20,15 @@
import java.lang.reflect.Modifier;
import java.net.URL;
import java.net.URLConnection;
+import java.nio.file.FileVisitResult;
+import java.nio.file.FileVisitor;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.BasicFileAttributes;
import java.security.AccessControlException;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -69,7 +78,7 @@
*
* @param name class/pkg name
* @param pkg if true, name refers to a pkg
- * @return true if name must be filtered out
+ * @return {@code true} if name should be filtered out
*/
protected boolean filterByName(String name, boolean pkg) {
return name.indexOf('$') != -1;
@@ -78,101 +87,138 @@
/**
* Filter class by access perms helper method - hook. The default implementation is used by
* {@link #addJarToPackages} in order to filter out non-public classes. Should be used or
- * overridden by derived classes too. Also to be used in {@link #doDir}. Access perms can be
- * read with {@link #checkAccess}.
+ * overridden by derived classes too. Also to be used in {@link #doDir}. Access permissions can
+ * be read with {@link #checkAccess}.
*
* @param name class name
* @param acc class access permissions as int
- * @return true if name must be filtered out
+ * @return {@code true} if name should be filtered out
*/
protected boolean filterByAccess(String name, int acc) {
return (acc & Modifier.PUBLIC) != Modifier.PUBLIC;
}
+ /**
+ * Set {@code true} whenever the cache index is modified (needs ultimately to be re-written).
+ */
private boolean indexModified;
- private Map<String, JarXEntry> jarfiles;
+ /**
+ * Map from some source of class definitions to the name of a file used to cache lists of
+ * classes in that source. {@code null} if cache is not operating. (This source must have a time
+ * last modified so we may check that the cache is valid).
+ */
+ private Map<String, JarXEntry> index;
- private static String listToString(List<String> list) {
- int n = list.size();
- StringBuilder ret = new StringBuilder();
- for (int i = 0; i < n; i++) {
- ret.append(list.get(i));
- if (i < n - 1) {
- ret.append(",");
- }
- }
- return ret.toString();
- }
+ /**
+ * Process one entry from a JAR/ZIP, and if the entry is a (qualifying) Java class, add its name
+ * to that package's entry in the map passed in. A class is added only if
+ * {@link #filterByName(String, boolean)} returns {@code true}. We add it to the accessible or
+ * inaccessible list according to whether {@link #filterByAccess(String, int)} returns
+ * {@code true} for the name and permissions of that class.
+ *
+ * @param zipPackages map (to update) from package name to class names by accessibility
+ * @param entry possibly representing a class
+ * @param zip the JAR or ZIP to which the entry belongs
+ * @throws IOException
+ */
+ private void addZipEntry(Map<String, ClassList> zipPackages, ZipEntry entry, ZipInputStream zip)
+ throws IOException {
- /** Add a single class from zipFile to zipPackages. Only add valid, public classes. */
- private void addZipEntry(Map<String, List<String>[]> zipPackages, ZipEntry entry,
- ZipInputStream zip) throws IOException {
String name = entry.getName();
// System.err.println("entry: "+name);
- if (!name.endsWith(".class")) {
- return;
- }
+ if (name.endsWith(".class")) {
- char sep = '/';
- int breakPoint = name.lastIndexOf(sep);
- if (breakPoint == -1) {
- breakPoint = name.lastIndexOf('\\');
- sep = '\\';
- }
+ // Split off the bare class name
+ char sep = '/';
+ int slash = name.lastIndexOf(sep);
+ if (slash == -1) {
+ if ((slash = name.lastIndexOf('\\')) >= 0) {
+ sep = '\\'; // Shouldn't be necessary according to standards, but is.
+ }
+ }
+ String className = name.substring(slash + 1, name.length() - 6);
- String packageName;
- if (breakPoint == -1) {
- packageName = "";
- } else {
- packageName = name.substring(0, breakPoint).replace(sep, '.');
- }
-
- String className = name.substring(breakPoint + 1, name.length() - 6);
+ // Check acceptable name: in practice, this is used to ignore inner classes.
+ if (!filterByName(className, false)) {
+ // File this class by name against the package
+ String packageName = slash == -1 ? "" : name.substring(0, slash).replace(sep, '.');
+ ClassList classes = zipPackages.get(packageName);
- if (filterByName(className, false)) {
- return;
- }
+ if (classes == null) {
+ // It wasn't in the map so add it
+ classes = new ClassList();
+ zipPackages.put(packageName, classes);
+ }
- List<String>[] vec = zipPackages.get(packageName);
- if (vec == null) {
- vec = createGenericStringListArray();
- zipPackages.put(packageName, vec);
- }
- int access = checkAccess(zip);
- if ((access != -1) && !filterByAccess(name, access)) {
- vec[0].add(className);
- } else {
- vec[1].add(className);
+ // Put the class on the right list
+ int access = checkAccess(zip);
+ if ((access != -1) && !filterByAccess(name, access)) {
+ classes.accessible.add(className);
+ } else {
+ classes.inaccessible.add(className);
+ }
+ }
}
}
- @SuppressWarnings("unchecked")
- private List<String>[] createGenericStringListArray() {
- return new List[] {Generic.list(), Generic.list()};
+ /** Used when representing the classes in a particular package. */
+ private static class ClassList {
+
+ /** Class names of accessible classes */
+ List<String> accessible = new ArrayList<String>();
+ /** Class names of inaccessible classes */
+ List<String> inaccessible = new ArrayList<String>();
+
+ /** Retrieve the two lists in "cache file" format {@code A,B,C[@D,E]}. */
+ @Override
+ public String toString() {
+ StringBuilder buf = new StringBuilder();
+ appendList(buf, accessible);
+ if (inaccessible.size() > 0) {
+ buf.append('@');
+ appendList(buf, inaccessible);
+ }
+ return buf.toString();
+ }
+
+ private static void appendList(StringBuilder buf, List<String> names) {
+ if (names.size() > 0) {
+ for (String n : names) {
+ buf.append(n).append(',');
+ }
+ // Deal with surplus comma
+ buf.deleteCharAt(buf.length() - 1);
+ }
+ }
}
- /** Extract all of the packages in a single jar file */
+ /**
+ * Detect all of the packages in an open JAR/ZIP file, that contain any classes for which
+ * {@link #filterByName(String, boolean)} returns {@code true}. For each such package, list the
+ * (surviving) classes in two lists: accessible and inaccessible, which is judged according to
+ * {@link #filterByAccess(String, int)}. The returned map is from package to these two lists,
+ * now comma-separated lists, with an '@' between them.
+ *
+ * @param jarin an open JAR/ZIP file
+ * @return map from package name to comma/@-separated list of class names
+ * @throws IOException
+ */
private Map<String, String> getZipPackages(InputStream jarin) throws IOException {
- Map<String, List<String>[]> zipPackages = Generic.map();
+ Map<String, ClassList> zipPackages = Generic.map();
ZipInputStream zip = new ZipInputStream(jarin);
+ ZipEntry entry;
- ZipEntry entry;
while ((entry = zip.getNextEntry()) != null) {
addZipEntry(zipPackages, entry, zip);
zip.closeEntry();
}
- // Turn each vector into a comma-separated String
+ // Turn each ClassList into a comma/@-separated String
Map<String, String> transformed = Generic.map();
- for (Entry<String, List<String>[]> kv : zipPackages.entrySet()) {
- List<String>[] vec = kv.getValue();
- String classes = listToString(vec[0]);
- if (vec[1].size() > 0) {
- classes += '@' + listToString(vec[1]);
- }
- transformed.put(kv.getKey(), classes);
+ for (Entry<String, ClassList> kv : zipPackages.entrySet()) {
+ transformed.put(kv.getKey(), kv.getValue().toString());
}
return transformed;
@@ -206,30 +252,46 @@
}
/**
- * Gathers classes info from jar specified by File jarfile. Eventually just using previously
- * cached info. Eventually updated info is (re-)cached if param cache is true. Persistent cache
- * storage access goes through inOpenCacheFile() and outCreateCacheFile().
+ * Gathers package and class lists from a jar specified by a {@code File}. Eventually just using
+ * previously cached info. Eventually updated info is (re-)cached if param cache is true.
+ * Persistent cache storage access goes through inOpenCacheFile() and outCreateCacheFile().
*/
public void addJarToPackages(File jarfile, boolean cache) {
addJarToPackages(null, jarfile, cache);
}
- private void addJarToPackages(URL jarurl, File jarfile, boolean cache) {
+ /**
+ * Create (or ensure we have) a {@link PyJavaPackage}, for each package in a jar specified by a
+ * file or URL, descending from {@link PackageManager#topLevelPackage} in this
+ * {@link PackageManager} instance. Ensure that the class list in each package is updated with
+ * the classes this JAR supplies to it. This information may be from a previously cached account
+ * of the JAR, if the last-modified time of the JAR matches a cached value. Otherwise, it will
+ * be obtained by inspecting the JAR, and a new cache will be written (if requested). Eventually
+ * updated info is (re-)cached if param cache is true. Persistent cache storage access goes
+ * through {@link #inOpenCacheFile(String)} and {@link #outCreateCacheFile(JarXEntry, boolean)}.
+ *
+ * @param jarurl identifying the JAR if {@code jarfile} is {@code null}
+ * @param jarfile identifying the JAR
+ * @param writeCache re-write the cache if it was out of date (and caching is active).
+ */
+ private void addJarToPackages(URL jarurl, File jarfile, boolean writeCache) {
try {
- boolean caching = this.jarfiles != null;
-
+ boolean readCache = this.index != null;
+ writeCache &= readCache;
URLConnection jarconn = null;
boolean localfile = true;
if (jarfile == null) {
+ // We were not given a File, so the URL must be reliable (but maybe not a file)
jarconn = jarurl.openConnection();
+ // The following comment may be out of date. Also 2 reasons or just the bug?
/*
* This is necessary because 'file:' url-connections always return 0 through
* getLastModified (bug?). And in order to handle localfiles (from urls too)
* uniformly.
*/
if (jarconn.getURL().getProtocol().equals("file")) {
- // ??pending: need to use java2 URLDecoder.decode?
+ // Although given as a URL, this *is* a file.
String jarfilename = jarurl.getFile();
jarfilename = jarfilename.replace('/', File.separatorChar);
jarfile = new File(jarfilename);
@@ -238,10 +300,12 @@
}
}
+ // Claimed JAR does not exist. Silently ignore.
if (localfile && !jarfile.exists()) {
return;
}
+ // The classes to discover are in a local JAR file. They go in this map:
Map<String, String> zipPackages = null;
long mtime = 0;
@@ -249,8 +313,9 @@
JarXEntry entry = null;
boolean brandNew = false;
- if (caching) {
+ if (readCache) {
+ // Get the name and last modified time of the actual JAR on disk.
if (localfile) {
mtime = jarfile.lastModified();
jarcanon = jarfile.getCanonicalPath();
@@ -259,11 +324,14 @@
jarcanon = jarurl.toString();
}
- entry = this.jarfiles.get(jarcanon);
+ // The canonical name is our key in the (in memory) index to the cache file.
+ entry = this.index.get(jarcanon);
- if ((entry == null || !(new File(entry.cachefile).exists())) && cache) {
+ if (writeCache && (entry == null || !(new File(entry.cachefile).exists()))) {
+ // We intend to use a cache but there is no valid existing file.
comment("processing new jar, '" + jarcanon + "'");
+ // Create a base-name for the cache file
String jarname;
if (localfile) {
jarname = jarfile.getName();
@@ -276,22 +344,28 @@
}
jarname = jarname.substring(0, jarname.length() - 4);
+ // Create a new (or replacement) index entry. Remember to create the file.
entry = new JarXEntry(jarname);
- this.jarfiles.put(jarcanon, entry);
-
+ this.index.put(jarcanon, entry);
brandNew = true;
}
- if (mtime != 0 && entry != null && entry.mtime == mtime) {
+ // If the source has a date and the cache matches, create the map we need from it.
+ if (entry != null && mtime != 0 && entry.mtime == mtime) {
zipPackages = readCacheFile(entry, jarcanon);
}
+ }
- }
+ /*
+ * At this point, we will have the package map from the cache if it was valid, and a
+ * cache is in use generally in this package manager.
+ */
if (zipPackages == null) {
- caching = caching && cache;
+ // We'll have to read the actual JAR file
- if (caching) {
+ if (writeCache) {
+ // Update the index entry for the cache file we shall eventually write.
this.indexModified = true;
if (entry.mtime != 0) {
comment("processing modified jar, '" + jarcanon + "'");
@@ -299,37 +373,53 @@
entry.mtime = mtime;
}
+ // Create the package-to-class mapping from whatever stream.
InputStream jarin = null;
try {
if (jarconn == null) {
jarin = new BufferedInputStream(new FileInputStream(jarfile));
} else {
+ // We were given a URL originally so use that.
jarin = jarconn.getInputStream();
}
zipPackages = getZipPackages(jarin);
+
} finally {
if (jarin != null) {
jarin.close();
}
}
- if (caching) {
+ if (writeCache) {
+ // Write what we created out to a cache file (new or updated)
writeCacheFile(entry, jarcanon, zipPackages, brandNew);
}
}
+ /*
+ * We now have the package map we need (from a cache or by construction). Now create or
+ * update corresponding package objects with the discovered classes (named, but not as
+ * PyObjects).
+ */
addPackages(zipPackages, jarcanon);
+
} catch (IOException ioe) {
- // silently skip any bad directories
+ // Skip the bad JAR with a message
warning("skipping bad jar, '"
+ (jarfile != null ? jarfile.toString() : jarurl.toString()) + "'");
}
-
}
- private void addPackages(Map<String, String> zipPackages, String jarfile) {
- for (Entry<String, String> entry : zipPackages.entrySet()) {
+ /**
+ * From a map of package name to comma/@-separated list of classes therein, relating to a
+ * particular JAR file, create a {@link PyJavaPackage} for each package.
+ *
+ * @param packageToClasses source of mappings
+ * @param jarfile becomes the __file__ attribute of the {@link PyJavaPackage}
+ */
+ private void addPackages(Map<String, String> packageToClasses, String jarfile) {
+ for (Entry<String, String> entry : packageToClasses.entrySet()) {
String pkg = entry.getKey();
String classes = entry.getValue();
@@ -343,8 +433,8 @@
}
/**
- * Read in cache file storing package info for a single jar. Return null and delete this
- * cachefile if it is invalid.
+ * Read in cache file storing package info for a single jar. Return null and delete this cache
+ * file if it is invalid.
*/
@SuppressWarnings("empty-statement")
private Map<String, String> readCacheFile(JarXEntry entry, String jarcanon) {
@@ -398,7 +488,14 @@
}
}
- /** Write a cache file storing package info for a single jar. */
+ /**
+ * Write a cache file storing package info for a single JAR. *
+ *
+ * @param entry in {@link #index} corresponding to the JAR
+ * @param jarcanon canonical name of the JAR (used as key into {@link #index})
+ * @param zipPackages a map from package name to class names for that pacjkage in the JAR
+ * @param brandNew if the cache file must be created (vs. being re-written)
+ */
private void writeCacheFile(JarXEntry entry, String jarcanon, Map<String, String> zipPackages,
boolean brandNew) {
DataOutputStream ostream = null;
@@ -430,6 +527,80 @@
}
}
+ /** Scan a module from the modular JVM, creating package objects. */
+ protected void addModule(Path modulePath) {
+ try {
+ Map<String, String> packages = getModularPackages(modulePath);
+ addPackages(packages, modulePath.toUri().toString());
+ } catch (IOException ioe) {
+ warning("skipping bad module, '" + modulePath + "'" + ioe);
+ }
+ }
+
+ /**
+ * Detect all of the packages in a single module, that contain any classes for which
+ * {@link #filterByName(String, boolean)} returns {@code true}. For each such package, list the
+ * (surviving) classes in two lists: accessible and inaccessible, which is judged according to
+ * {@link #filterByAccess(String, int)}. The returned map is from package to these two lists,
+ * now comma-separated lists, with an '@' between them.
+ */
+ private Map<String, String> getModularPackages(Path modulePath) throws IOException {
+
+ final Map<String, ClassList> modPackages = Generic.map();
+
+ FileVisitor<Path> visitor = new SimpleFileVisitor<Path>() {
+
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
+ throws IOException {
+ //System.out.println(" visitFile:" + file);
+
+ // file has at least 4 parts /modules/[module]/ ... /[name].class
+ int n = file.getNameCount();
+ // Apply name and access tests and conditionally add to modPackages
+ String fileName = file.getFileName().toString();
+ if (fileName.endsWith(".class") && n > 3) {
+ // Split off the bare class name
+ String className = fileName.substring(0, fileName.length() - 6);
+
+ // Check acceptable name: in practice, this is used to ignore inner classes.
+ if (!filterByName(className, false)) {
+ // File this class by name against the package
+ String packageName = file.subpath(2, n - 1).toString().replace('/', '.');
+ ClassList classes = modPackages.get(packageName);
+
+ if (classes == null) {
+ // It wasn't in the map so add it
+ classes = new ClassList();
+ modPackages.put(packageName, classes);
+ }
+
+ // Put the class on the right list
+ try (InputStream c = Files.newInputStream(file, StandardOpenOption.READ)) {
+ int access = checkAccess(c);
+ if ((access != -1) && !filterByAccess(fileName, access)) {
+ classes.accessible.add(className);
+ } else {
+ classes.inaccessible.add(className);
+ }
+ }
+ }
+ }
+ return FileVisitResult.CONTINUE;
+ }
+ };
+
+ Files.walkFileTree(modulePath, visitor);
+
+ // Turn each ClassList into a comma/@-separated String
+ Map<String, String> transformed = Generic.map();
+ for (Entry<String, ClassList> kv : modPackages.entrySet()) {
+ transformed.put(kv.getKey(), kv.getValue().toString());
+ }
+
+ return transformed;
+ }
+
/**
* Split up a string into several chunks based on a certain size
*
@@ -459,12 +630,12 @@
}
/**
- * Initializes cache. Eventually reads back cache index. Index persistent storage is accessed
- * through inOpenIndex().
+ * Initialise the cache by reading the index from storage, through {@link #inOpenIndex()}, or by
+ * creating a new empty one.
*/
protected void initCache() {
this.indexModified = false;
- this.jarfiles = Generic.map();
+ this.index = Generic.map();
DataInputStream istream = null;
try {
@@ -478,7 +649,7 @@
String jarcanon = istream.readUTF();
String cachefile = istream.readUTF();
long mtime = istream.readLong();
- this.jarfiles.put(jarcanon, new JarXEntry(cachefile, mtime));
+ this.index.put(jarcanon, new JarXEntry(cachefile, mtime));
}
} catch (EOFException eof) {
// ignore
@@ -496,9 +667,14 @@
}
}
- /** Write back cache index. Index persistent storage is accessed through outOpenIndex(). */
+ /**
+ * Write back cache <b>index</b>. The index is a mapping from each source of class definitions
+ * to the file where the cache for that source is held. This list is accessed through
+ * outOpenIndex().
+ */
public void saveCache() {
- if (jarfiles == null || !indexModified) {
+
+ if (index == null || !indexModified) {
return;
}
@@ -509,7 +685,7 @@
DataOutputStream ostream = null;
try {
ostream = outOpenIndex();
- for (Entry<String, JarXEntry> entry : jarfiles.entrySet()) {
+ for (Entry<String, JarXEntry> entry : index.entrySet()) {
String jarcanon = entry.getKey();
JarXEntry xentry = entry.getValue();
ostream.writeUTF(jarcanon);
@@ -531,10 +707,15 @@
// hooks for changing cache storage
- /** To pass a cachefile id by ref. And for internal use. See outCreateCacheFile. */
+ /**
+ * Class of object used to represent a cache file and last modification time, internally and to
+ * {@link CachedJarsPackageManager#outCreateCacheFile}. When caching, a {@code JarXEntry} is
+ * created for each JAR processed for classes, and corresponds to a file in the package cache
+ * directory. The name is based on the name of the JAR.
+ */
public static class JarXEntry extends Object {
- /** cachefile id */
+ /** Specifies the actual cache file once that is created or opened. */
public String cachefile;
public long mtime;
@@ -551,8 +732,8 @@
}
/**
- * Open cache index for reading from persistent storage - hook. Must Return null if this is
- * absent. This default implementation is part of the off-the-shelf local file-system cache
+ * Open cache index for reading from persistent storage – hook. Must Return null if this
+ * is absent. This default implementation is part of the off-the-shelf local file-system cache
* implementation. Can be overridden.
*/
protected DataInputStream inOpenIndex() throws IOException {
@@ -566,8 +747,9 @@
}
/**
- * Open cache index for writing back to persistent storage - hook. This default implementation
- * is part of the off-the-shelf local file-system cache implementation. Can be overridden.
+ * Open cache index for writing back to persistent storage – hook. This default
+ * implementation is part of the off-the-shelf local file-system cache implementation. Can be
+ * overridden.
*/
protected DataOutputStream outOpenIndex() throws IOException {
File indexFile = new File(this.cachedir, "packages.idx");
@@ -576,8 +758,8 @@
}
/**
- * Open cache file for reading from persistent storage - hook. This default implementation is
- * part of the off-the-shelf local file-system cache implementation. Can be overridden.
+ * Open a particular cache file for reading from persistent storage. This default implementation
+ * is part of the off-the-shelf local file-system cache implementation. Can be overridden.
*/
protected DataInputStream inOpenCacheFile(String cachefile) throws IOException {
return new DataInputStream(new BufferedInputStream(new FileInputStream(cachefile)));
@@ -592,40 +774,43 @@
}
/**
- * Create/open cache file for rewriting back to persistent storage - hook. If create is false,
- * cache file is supposed to exist and must be opened for rewriting, entry.cachefile is a valid
- * cachefile id. If create is true, cache file must be created. entry.cachefile is a flat
- * jarname to be used to produce a valid cachefile id (to be put back in entry.cachefile on
- * exit). This default implementation is part of the off-the-shelf local file-system cache
- * implementation. Can be overridden.
+ * Create/open cache file for rewriting back to persistent storage – hook. If
+ * {@code create} is {@code false}, the cache file is supposed to exist at
+ * {@code entry.cachefile} and will be opened for rewriting. If {@code create} is {@code true},
+ * {@code entry.cachefile} is the base name (e.g. JAR or module name) for a cache to be created,
+ * and the full name will be put in {@code entry.cachefile} on exit. This default implementation
+ * is part of the off-the-shelf local file-system cache implementation. It may be overridden to
+ * provide a different cache medium and use of {@code entry.cachefile}.
+ *
+ * @param entry cache file description
+ * @param create new or use existing file named in {@code entry.cachefile}
+ * @return stream on which to represent the package to class-list textually
+ * @throws IOException
*/
protected DataOutputStream outCreateCacheFile(JarXEntry entry, boolean create)
throws IOException {
- File cachefile = null;
+ File file;
if (create) {
- int index = 1;
- String suffix = "";
+ // Create a new cache file with a name based on the initial value
String jarname = entry.cachefile;
- while (true) {
- cachefile = new File(this.cachedir, jarname + suffix + ".pkc");
- // System.err.println("try cachefile: "+cachefile);
- if (!cachefile.exists()) {
- break;
- }
- suffix = "$" + index;
- index += 1;
+ file = new File(this.cachedir, jarname + ".pkc");
+ for (int index = 1; file.exists(); index++) {
+ // That name is in use: make up another one.
+ file = new File(this.cachedir, jarname + "$" + index + ".pkc");
}
- entry.cachefile = cachefile.getCanonicalPath();
+ entry.cachefile = file.getCanonicalPath();
+
} else {
- cachefile = new File(entry.cachefile);
+ // Use an existing cache file named in the entry
+ file = new File(entry.cachefile);
}
- return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(cachefile)));
+ return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file)));
}
- /** for default cache (local fs based) implementation */
+ /** Directory in which cache files are stored. */
private File cachedir;
/**
diff --git a/src/org/python/core/packagecache/PackageManager.java b/src/org/python/core/packagecache/PackageManager.java
--- a/src/org/python/core/packagecache/PackageManager.java
+++ b/src/org/python/core/packagecache/PackageManager.java
@@ -13,12 +13,17 @@
import org.python.core.PyStringMap;
import java.io.IOException;
+import java.io.InputStream;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.StringTokenizer;
/**
* Abstract package manager.
*/
public abstract class PackageManager extends Object {
+ /** Nominal top-level package of all (Java) packages, containing "java", "com", "org", etc.. */
public PyJavaPackage topLevelPackage;
public PackageManager() {
@@ -117,6 +122,16 @@
return list1;
}
+ /**
+ * Given the (dotted) name of a package, find the {@link PyJavaPackage} corresponding, by
+ * navigating from the {@link #topLevelPackage}, successively applying
+ * {@link PyObject#__findattr__(String)}. This in fact drives the creation of
+ * {@link PyJavaPackage} objects since it indirectly calls
+ * {@link #packageExists(String, String)}.
+ *
+ * @param name (dotted) package name
+ * @return the package named
+ */
public PyObject lookupName(String name) {
PyObject top = this.topLevelPackage;
do {
@@ -139,13 +154,15 @@
}
/**
- * Creates package/updates statically known classes info. Uses
- * {@link PyJavaPackage#addPackage(java.lang.String, java.lang.String) },
+ * Create (or ensure we have) a {@link PyJavaPackage} for the named package and add to it the
+ * names of classes mentioned here. These classes are added as "place holders" only, so they
+ * become members of it, without being instantiated. This method relies on
+ * {@link PyJavaPackage#addPackage(java.lang.String, java.lang.String)} and
* {@link PyJavaPackage#addPlaceholders}.
*
* @param name package name
- * @param classes comma-separated string
- * @param jarfile involved jarfile; can be null
+ * @param classes comma or @-sign separated string
+ * @param jarfile involved; can be null
* @return created/updated package
*/
public PyJavaPackage makeJavaPackage(String name, String classes, String jarfile) {
@@ -153,14 +170,11 @@
if (name.length() != 0) {
p = p.addPackage(name, jarfile);
}
-
- if (classes != null) {
- p.addPlaceholders(classes);
- }
-
+ p.addPlaceholders(split(classes, ",@"));
return p;
}
+ /** ASM visitor class supporting {@link #checkAccess(InputStream)}. */
private static class AccessVisitor extends ClassVisitor {
private int class_access;
@@ -181,10 +195,10 @@
}
/**
- * Check that a given stream is a valid Java .class file. And return its access permissions as
+ * Check that a given stream is a valid Java .class file, and return its access permissions as
* an int.
*/
- static protected int checkAccess(java.io.InputStream cstream) throws IOException {
+ static protected int checkAccess(InputStream cstream) throws IOException {
try {
ClassReader reader = new ClassReader(cstream);
AccessVisitor visitor = new AccessVisitor();
@@ -195,4 +209,30 @@
}
}
+ /**
+ * Helper to split a textual list into a list. The semantics are basically those of
+ * {@code StringTokenizer}, followed by {@code String.trim()} and duplicate removal.
+ *
+ * @param target compound string to split into tokens ({@code null} treated as "".
+ * @param sep characters, any of which will be treated as a token separator
+ * @return set of tokens trimmed of white space
+ */
+ protected static Set<String> split(String target, String sep) {
+ Set<String> result = new LinkedHashSet<>();
+ if (target != null) {
+ StringTokenizer tok = new StringTokenizer(target, sep);
+ while (tok.hasMoreTokens()) {
+ String entry = tok.nextToken().trim();
+ if (entry.length() > 0) {
+ result.add(entry);
+ }
+ }
+ }
+ return result;
+ }
+
+ /** Equivalent to {@code split(target, ",")}. See {@link #split(String, String)}. */
+ protected static Set<String> split(String target) {
+ return split(target, ",");
+ }
}
diff --git a/src/org/python/core/packagecache/PathPackageManager.java b/src/org/python/core/packagecache/PathPackageManager.java
--- a/src/org/python/core/packagecache/PathPackageManager.java
+++ b/src/org/python/core/packagecache/PathPackageManager.java
@@ -30,10 +30,12 @@
}
/**
- * Helper for {@link #packageExists(java.lang.String,java.lang.String)}. Scans for package
- * pkg.name the directories in path.
+ * Helper for {@link #packageExists(String,String)}. Scans the directories in the given path for
+ * package pkg.name. A directory with a matching name is considered to define a package if it
+ * contains no Python (source or compiled), or contains a Java .class file (not compiled from
+ * Python).
*/
- protected boolean packageExists(PyList path, String pkg, String name) {
+ protected static boolean packageExists(PyList path, String pkg, String name) {
String child = pkg.replace('.', File.separatorChar) + File.separator + name;
for (int i = 0; i < path.__len__(); i++) {
@@ -46,9 +48,9 @@
try {
if (f.isDirectory() && imp.caseok(f, name)) {
/*
- * Figure out if we have a directory a mixture of python and java or just an
- * empty directory (which means Java) or a directory with only Python source
- * (which means Python).
+ * f is a directory matching the package name. This directory is considered to
+ * define a package if it contains no Python (source or compiled), or contains a
+ * Java .class file (not compiled from Python).
*/
PackageExistsFileFilter m = new PackageExistsFileFilter();
f.listFiles(m);
diff --git a/src/org/python/core/packagecache/SysPackageManager.java b/src/org/python/core/packagecache/SysPackageManager.java
--- a/src/org/python/core/packagecache/SysPackageManager.java
+++ b/src/org/python/core/packagecache/SysPackageManager.java
@@ -9,7 +9,13 @@
import org.python.core.PySystemState;
import java.io.File;
+import java.net.URI;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import java.nio.file.ProviderNotFoundException;
import java.util.Properties;
+import java.util.Set;
import java.util.StringTokenizer;
/**
@@ -58,7 +64,13 @@
addJarDir(jdir, cache, cache);
}
- /** Index the contents of every JAR or ZIP in a directory. */
+ /**
+ * Index the contents of every JAR or ZIP in a directory.
+ *
+ * @param jdir direcory containing some JAR or ZIP files
+ * @param cache
+ * @param saveCache
+ */
private void addJarDir(String jdir, boolean cache, boolean saveCache) {
File file = new File(jdir);
@@ -89,6 +101,7 @@
while (tok.hasMoreTokens()) {
// ??pending: do jvms trim? how is interpreted entry=""?
String entry = tok.nextToken();
+ // Use the extra flag to defer writing out the cache.
addJarDir(entry, true, false);
}
}
@@ -100,23 +113,40 @@
* @param registry
*/
private void findAllPackages(Properties registry) {
+
+ String defaultPaths = "java.class.path";
+ String defaultDirectories = "";
+ String defaultModules = "java.base,java.desktop,java.logging,java.se,java.sql,java.xml";
+
+ try {
+ // Support for the modular JVM (particular packages).
+ // XXX This may not be our final approach: maybe enumerate all the packages instead?
+ Set<String> modules =
+ split(registry.getProperty("python.packages.modules", defaultModules));
+ FileSystem jrtfs = FileSystems.getFileSystem(URI.create("jrt:/"));
+ for (String moduleName : modules) {
+ Path modulePath = jrtfs.getPath("/modules/" + moduleName);
+ addModule(modulePath);
+ }
+ } catch (ProviderNotFoundException e) {
+ // Running on a JVM before Java 9: add boot class path and optional extensions.
+ defaultPaths = "java.class.path,sun.boot.class.path";
+ defaultDirectories = "java.ext.dirs";
+ }
+
/*
- * python.packages.directories defines a sequence of property names. Each property name is a
- * path string. The default setting causes directories and JARs on the classpath and in the
- * JRE (before Java 9) to be sources of Python packages.
+ * python.packages.paths defines a sequence of property names. Each property is a path
+ * string. The default setting causes directories and JARs on the classpath and in the JRE
+ * (before Java 9) to be sources of Python packages.
*/
- String defaultPaths = "java.class.path,sun.boot.class.path";
- String paths = registry.getProperty("python.packages.paths", defaultPaths);
- StringTokenizer tok = new StringTokenizer(paths, ",");
- while (tok.hasMoreTokens()) {
- // Each property name is a path string containing directories
- String entry = tok.nextToken().trim();
- String tmp = registry.getProperty(entry);
- if (tmp == null) {
- continue;
+ Set<String> paths = split(registry.getProperty("python.packages.paths", defaultPaths));
+ for (String name : paths) {
+ // Each property is a path string containing directories
+ String path = registry.getProperty(name);
+ if (path != null) {
+ // Each path may be a mixture of directory and JAR specifiers (source of packages)
+ addClassPath(path);
}
- // Each path may be a mixture of directory and JAR specifiers (source of packages)
- addClassPath(tmp);
}
/*
@@ -125,17 +155,15 @@
* that are to be a source of Python packages. By default, these directories are those where
* the JVM stores its optional packages as JARs (a mechanism withdrawn in Java 9).
*/
- String directories = registry.getProperty("python.packages.directories", "java.ext.dirs");
- tok = new StringTokenizer(directories, ",");
- while (tok.hasMoreTokens()) {
- // Each property name is a path string containing directories
- String entry = tok.nextToken().trim();
- String tmp = registry.getProperty(entry);
- if (tmp == null) {
- continue;
+ Set<String> directories =
+ split(registry.getProperty("python.packages.directories", defaultDirectories));
+ for (String name : directories) {
+ // Each property defines a path string containing directories
+ String path = registry.getProperty(name);
+ if (path != null) {
+ // Add the JAR/ZIP archives in those directories to the search path for packages
+ addJarPath(path);
}
- // Add the JAR/ZIP archives found in those directories to the search path for packages
- addJarPath(tmp);
}
/*
@@ -194,8 +222,8 @@
return true;
}
- PySystemState system = Py.getSystemState();
- if (system.getClassLoader() == null && packageExists(Py.getSystemState().path, pkg, name)) {
+ PySystemState sys = Py.getSystemState();
+ if (sys.getClassLoader() == null && packageExists(sys.path, pkg, name)) {
return true;
} else {
return false;
--
Repository URL: https://hg.python.org/jython
More information about the Jython-checkins
mailing list