[Jython-checkins] jython: Scan the whole Java runtime filesystem for packages (fixes #2362).
jeff.allen
jython-checkins at python.org
Sat Jan 12 08:26:19 EST 2019
https://hg.python.org/jython/rev/7fd811df6d33
changeset: 8218:7fd811df6d33
user: Jeff Allen <ja.py at farowl.co.uk>
date: Sat Jan 12 12:23:05 2019 +0000
summary:
Scan the whole Java runtime filesystem for packages (fixes #2362).
It does not seem necessary to provide the user with a way to filter modules to
be included. A fuller implementation later should allow for a module path.
files:
registry | 17 +-
src/org/python/core/packagecache/CachedJarsPackageManager.java | 49 ++++--
src/org/python/core/packagecache/PathPackageManager.java | 4 +-
src/org/python/core/packagecache/SysPackageManager.java | 79 +++++++--
4 files changed, 98 insertions(+), 51 deletions(-)
diff --git a/registry b/registry
--- a/registry
+++ b/registry
@@ -10,10 +10,8 @@
#python.path = d:\\python20\\lib
# Set the directory to use for caches (currently just package information)
-# This directory should be writable by the user
-# If this is an absolute path it is used as given
-# Otherwise it is interpreted relative to sys.prefix
-# (typically the directory of this file)
+# This directory should be writable by the user. If this is an absolute path it is used as given,
+# otherwise it is interpreted relative to sys.prefix (typically the directory of this file).
python.cachedir = cachedir
# Setting this property to true disables the package scan for the cachedir.
@@ -22,15 +20,12 @@
# Properties to check for initializing and updating the package cache
# Values shown here are those hard-coded in Jython's cache manager.
-# Treat JARs on the classpath and (before Java 9) in the JRE as a source of Python packages.
-#python.packages.paths = java.class.path, sun.boot.class.path # before Java 9
+# Treat JARs on the classpath and (up to Java 8) in the JRE as a source of Python packages.
+#python.packages.paths = java.class.path, sun.boot.class.path # up to Java 8
#python.packages.paths = java.class.path # from Java 9
# Treat installed optional (Java) packages as source of Python packages (before Java 9)
-#python.packages.directories = java.ext.dirs # before Java 9
+#python.packages.directories = java.ext.dirs # up to Java 8
#python.packages.directories # undefined from Java 9
-# Treat the following Java modules as sources of of Python packages (from Java 9)
-#python.packages.modules = java.base, java.desktop, java.logging, java.se, java.sql, java.xml
-
# Set verbosity to error, warning, message, comment, or debug
# for varying levels of informative messages from Jython. Normally
@@ -39,7 +34,7 @@
# Jython ships with a JLine console (http://jline.sourceforge.net/) out of the
# box. This is selected by default in the Jython command-line application
-# (org.python.util,jython) if you do not define python.console to be another
+# (org.python.util.jython) if you do not define python.console to be another
# class on the command line. Alternatively, you could set python.console here,
# but be aware that this will also affect the console in applications that
# embed a PythonInterpreter, or use Jython as a JSR-223 script engine.
diff --git a/src/org/python/core/packagecache/CachedJarsPackageManager.java b/src/org/python/core/packagecache/CachedJarsPackageManager.java
--- a/src/org/python/core/packagecache/CachedJarsPackageManager.java
+++ b/src/org/python/core/packagecache/CachedJarsPackageManager.java
@@ -261,14 +261,18 @@
}
/**
- * Create (or ensure we have) a {@link PyJavaPackage}, for each package in a jar specified by a
- * file or URL, descending from {@link PackageManager#topLevelPackage} in this
- * {@link PackageManager} instance. Ensure that the class list in each package is updated with
- * the classes this JAR supplies to it. This information may be from a previously cached account
- * of the JAR, if the last-modified time of the JAR matches a cached value. Otherwise, it will
- * be obtained by inspecting the JAR, and a new cache will be written (if requested). Eventually
- * updated info is (re-)cached if param cache is true. Persistent cache storage access goes
- * through {@link #inOpenCacheFile(String)} and {@link #outCreateCacheFile(JarXEntry, boolean)}.
+ * Create (or ensure we have) a {@link PyJavaPackage}, descending from
+ * {@link PackageManager#topLevelPackage} in this {@link PackageManager} instance, for each
+ * package in a jar specified by a file or URL. Ensure that the class list in each package is
+ * updated with the classes this JAR supplies to it.
+ *
+ * The information concerning packages in the JAR and the classes they contain, may be read from
+ * from a previously cached account of the JAR, if the last-modified time of the JAR matches a
+ * cached value. If it is not read from a cache, it will be obtained by inspecting the JAR, and
+ * a new cache will be written (if requested).
+ *
+ * Access to persistent cache storage goes through {@link #inOpenCacheFile(String)} and
+ * {@link #outCreateCacheFile(JarXEntry, boolean)}.
*
* @param jarurl identifying the JAR if {@code jarfile} is {@code null}
* @param jarfile identifying the JAR
@@ -276,13 +280,17 @@
*/
private void addJarToPackages(URL jarurl, File jarfile, boolean writeCache) {
try {
+ // We try to read the cache (for this jar) if caching is in operation.
boolean readCache = this.index != null;
+ // We write a cache if caching is in operation AND writing has been requested.
writeCache &= readCache;
+
URLConnection jarconn = null;
boolean localfile = true;
+ // If a local JAR file was not given directly in jarfile, try to find one from the URL.
if (jarfile == null) {
- // We were not given a File, so the URL must be reliable (but maybe not a file)
+ // We were not given a File, so the URL must be reliable (but may not be a file)
jarconn = jarurl.openConnection();
// The following comment may be out of date. Also 2 reasons or just the bug?
/*
@@ -296,12 +304,13 @@
jarfilename = jarfilename.replace('/', File.separatorChar);
jarfile = new File(jarfilename);
} else {
+ // We can't find a local file
localfile = false;
}
}
- // Claimed JAR does not exist. Silently ignore.
if (localfile && !jarfile.exists()) {
+ // Local JAR file claimed or deduced does not exist. Silently ignore.
return;
}
@@ -527,9 +536,10 @@
}
}
- /** Scan a module from the modular JVM, creating package objects. */
- protected void addModule(Path modulePath) {
+ /** Scan a Java module, creating package objects. */
+ protected void addModuleToPackages(Path modulePath) {
try {
+ comment("reading packages from " + modulePath);
Map<String, String> packages = getModularPackages(modulePath);
addPackages(packages, modulePath.toUri().toString());
} catch (IOException ioe) {
@@ -543,9 +553,14 @@
* (surviving) classes in two lists: accessible and inaccessible, which is judged according to
* {@link #filterByAccess(String, int)}. The returned map is from package to these two lists,
* now comma-separated lists, with an '@' between them.
+ *
+ * @param modulePath up to and including the name of the module
+ * @return map from packages to classes
+ * @throws IOException
*/
private Map<String, String> getModularPackages(Path modulePath) throws IOException {
+ final int M = modulePath.getNameCount();
final Map<String, ClassList> modPackages = Generic.map();
FileVisitor<Path> visitor = new SimpleFileVisitor<Path>() {
@@ -555,18 +570,18 @@
throws IOException {
//System.out.println(" visitFile:" + file);
- // file has at least 4 parts /modules/[module]/ ... /[name].class
+ // file starts with modulePath, then has package & class: / ... /[name].class
int n = file.getNameCount();
// Apply name and access tests and conditionally add to modPackages
String fileName = file.getFileName().toString();
- if (fileName.endsWith(".class") && n > 3) {
+ if (fileName.endsWith(".class") && n > M + 1) {
// Split off the bare class name
String className = fileName.substring(0, fileName.length() - 6);
// Check acceptable name: in practice, this is used to ignore inner classes.
if (!filterByName(className, false)) {
- // File this class by name against the package
- String packageName = file.subpath(2, n - 1).toString().replace('/', '.');
+ // Parts M to n-1 define the package of this class
+ String packageName = file.subpath(M, n - 1).toString().replace('/', '.');
ClassList classes = modPackages.get(packageName);
if (classes == null) {
@@ -575,7 +590,7 @@
modPackages.put(packageName, classes);
}
- // Put the class on the right list
+ // Put the class on the accessible or inaccessible list
try (InputStream c = Files.newInputStream(file, StandardOpenOption.READ)) {
int access = checkAccess(c);
if ((access != -1) && !filterByAccess(fileName, access)) {
diff --git a/src/org/python/core/packagecache/PathPackageManager.java b/src/org/python/core/packagecache/PathPackageManager.java
--- a/src/org/python/core/packagecache/PathPackageManager.java
+++ b/src/org/python/core/packagecache/PathPackageManager.java
@@ -191,8 +191,8 @@
}
/**
- * Scan a path that may be a mixture of directory and JAR specifiers, and within each path entry
- * index the packages. Calls {@link #addDirectory} if a path entry refers to a dir,
+ * Scan a Java class-path that may be a mixture of directory and JAR specifiers, and within each
+ * path entry index the packages. Calls {@link #addDirectory} if a path entry refers to a dir,
* {@link #addJarToPackages(java.io.File, boolean)} with param cache true if the path entry
* refers to a jar.
*/
diff --git a/src/org/python/core/packagecache/SysPackageManager.java b/src/org/python/core/packagecache/SysPackageManager.java
--- a/src/org/python/core/packagecache/SysPackageManager.java
+++ b/src/org/python/core/packagecache/SysPackageManager.java
@@ -9,11 +9,17 @@
import org.python.core.PySystemState;
import java.io.File;
+import java.io.IOException;
import java.net.URI;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
+import java.nio.file.FileVisitResult;
+import java.nio.file.FileVisitor;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.ProviderNotFoundException;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
import java.util.Properties;
import java.util.Set;
import java.util.StringTokenizer;
@@ -67,7 +73,7 @@
/**
* Index the contents of every JAR or ZIP in a directory.
*
- * @param jdir direcory containing some JAR or ZIP files
+ * @param jdir directory containing some JAR or ZIP files
* @param cache
* @param saveCache
*/
@@ -107,30 +113,61 @@
}
/**
+ * Index the packages in every module in a directory. Entries in the directory that are not modules
+ * (do not contain a {@code module-info.class}) are ignored. Only modules exploded on the file system of this path are (currently) supported,
+ * and at the time of writing, we only use this method on the {@code jrt:} file system.
+ *
+ * @param moduleDir directory containing some modules
+ */
+ private void addModuleDir(final Path moduleDir) {
+ try {
+ // Walk the directory tree with this visitor
+ FileVisitor<Path> visitor = new SimpleFileVisitor<Path>() {
+
+ @Override
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
+ // System.out.println(dir);
+ if (dir.equals(moduleDir)) {
+ // Ignore this, it's just the root)
+ } else if (Files.exists(dir.resolve("module-info.class"))) {
+ // dir is a module: scan packages from it.
+ addModuleToPackages(dir);
+ return FileVisitResult.SKIP_SUBTREE;
+ }
+ return FileVisitResult.CONTINUE;
+ }
+ };
+
+ Files.walkFileTree(moduleDir, visitor);
+
+ } catch (IOException e) {
+ warning("error enumerating Java modules in " + moduleDir + ": " + e.getMessage());
+ }
+ }
+
+ /**
* Walk the packages found in paths specified indirectly through the given {@code Properties}
- * object, which in practice is the Jython registry.
+ * object.
*
- * @param registry
+ * @param registry in practice, the Jython registry
*/
private void findAllPackages(Properties registry) {
- String defaultPaths = "java.class.path";
- String defaultDirectories = "";
- String defaultModules = "java.base,java.desktop,java.logging,java.se,java.sql,java.xml";
-
+ /*
+ * Packages in the Java runtime environment are enumerated in the jrt file system (from Java
+ * 9 onwards), or in JARs and directories designated by the properties
+ * sun.boot.class.path and java.ext.dirs (up to Java 8).
+ */
+ String defaultClassPaths, defaultDirectories;
try {
// Support for the modular JVM (particular packages).
- // XXX This may not be our final approach: maybe enumerate all the packages instead?
- Set<String> modules =
- split(registry.getProperty("python.packages.modules", defaultModules));
FileSystem jrtfs = FileSystems.getFileSystem(URI.create("jrt:/"));
- for (String moduleName : modules) {
- Path modulePath = jrtfs.getPath("/modules/" + moduleName);
- addModule(modulePath);
- }
+ addModuleDir(jrtfs.getPath("/modules/"));
+ defaultClassPaths = "java.class.path";
+ defaultDirectories = "";
} catch (ProviderNotFoundException e) {
// Running on a JVM before Java 9: add boot class path and optional extensions.
- defaultPaths = "java.class.path,sun.boot.class.path";
+ defaultClassPaths = "java.class.path,sun.boot.class.path";
defaultDirectories = "java.ext.dirs";
}
@@ -139,13 +176,13 @@
* string. The default setting causes directories and JARs on the classpath and in the JRE
* (before Java 9) to be sources of Python packages.
*/
- Set<String> paths = split(registry.getProperty("python.packages.paths", defaultPaths));
- for (String name : paths) {
- // Each property is a path string containing directories
- String path = registry.getProperty(name);
- if (path != null) {
+ Set<String> cps = split(registry.getProperty("python.packages.paths", defaultClassPaths));
+ for (String name : cps) {
+ // Each property is a class-path string containing JARS and directories
+ String classPath = registry.getProperty(name);
+ if (classPath != null) {
// Each path may be a mixture of directory and JAR specifiers (source of packages)
- addClassPath(path);
+ addClassPath(classPath);
}
}
--
Repository URL: https://hg.python.org/jython
More information about the Jython-checkins
mailing list