From jython-checkins at python.org Sun Jan 26 15:59:24 2020 From: jython-checkins at python.org (jeff.allen) Date: Sun, 26 Jan 2020 20:59:24 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Restrict_accessibility_of_?= =?utf-8?q?compiled_files_=28fixes_=232044_again=29=2E?= Message-ID: <20200126205924.1.B57D393E71BB885F@mg.python.org> https://hg.python.org/jython/rev/7e917a237b7a changeset: 8321:7e917a237b7a user: Jeff Allen date: Sun Jan 26 15:03:50 2020 +0000 summary: Restrict accessibility of compiled files (fixes #2044 again). CVE-2013-2027 points out that Jython may be run with umask 0, and then files cached will be world-writable affecting later sessions. #2044 claimed this fixed by other work, but this change fixes the permissions explicitly in the compiler and package manager. files: NEWS | 1 + src/org/python/core/imp.java | 4 +- src/org/python/core/packagecache/CachedJarsPackageManager.java | 4 +- src/org/python/core/util/FileUtil.java | 32 ++++++++++ 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -8,6 +8,7 @@ Jython 2.7.2b3 Bugs fixed + - [ 2044 ] CVE-2013-2027 Current umask sets privileges of class files and cache - [ 2834 ] Import of Java classes is not thread safe - [ 2820 ] Import fails with UnicodeDecodeError if sys.path contains invalid UTF-8 bytes - [ 2826 ] Unicode hex string decode failure diff --git a/src/org/python/core/imp.java b/src/org/python/core/imp.java --- a/src/org/python/core/imp.java +++ b/src/org/python/core/imp.java @@ -480,12 +480,12 @@ if (man != null) { man.checkWrite(compiledFilename); } - fop = new FileOutputStream(compiledFilename); + fop = new FileOutputStream(FileUtil.makePrivateRW(compiledFilename)); fop.write(compiledSource); fop.close(); return compiledFilename; } catch (IOException | SecurityException exc) { - // If we can't write the cache file, just logger and continue + // If we can't write the cache file, just log and continue logger.log(Level.FINE, "Unable to write to source cache file ''{0}'' due to {1}", new Object[] {compiledFilename, exc}); return null; diff --git a/src/org/python/core/packagecache/CachedJarsPackageManager.java b/src/org/python/core/packagecache/CachedJarsPackageManager.java --- a/src/org/python/core/packagecache/CachedJarsPackageManager.java +++ b/src/org/python/core/packagecache/CachedJarsPackageManager.java @@ -5,6 +5,7 @@ import org.python.core.Options; import org.python.core.PyJavaPackage; +import org.python.core.util.FileUtil; import org.python.util.Generic; import java.io.BufferedInputStream; @@ -773,7 +774,7 @@ * overridden. */ protected DataOutputStream outOpenIndex() throws IOException { - File indexFile = new File(this.cachedir, "packages.idx"); + File indexFile = FileUtil.makePrivateRW(new File(this.cachedir, "packages.idx")); FileOutputStream ostream = new FileOutputStream(indexFile); return new DataOutputStream(new BufferedOutputStream(ostream)); } @@ -821,6 +822,7 @@ // That name is in use: make up another one. file = new File(this.cachedir, jarname + "$" + index + ".pkc"); } + file = FileUtil.makePrivateRW(file); entry.cachefile = file.getCanonicalPath(); } else { diff --git a/src/org/python/core/util/FileUtil.java b/src/org/python/core/util/FileUtil.java --- a/src/org/python/core/util/FileUtil.java +++ b/src/org/python/core/util/FileUtil.java @@ -2,6 +2,7 @@ package org.python.core.util; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -90,4 +91,35 @@ } return out.toByteArray(); } + + /** + * Create the named file (if necessary) and give just the owner read-write access. + * + * @param filename to create/control + * @return {@code File} object for subsequent open + * @throws IOException + */ + public static File makePrivateRW(String filename) throws IOException { + return makePrivateRW(new File(filename)); + } + + /** + * Create the identified file (if necessary) and give just the owner read-write access. + * + * @param file to create/control + * @return {@code File} object for subsequent open + * @throws IOException + */ + public static File makePrivateRW(File file) throws IOException { + file.createNewFile(); + // Remove permissions for all + file.setReadable(false, false); + file.setWritable(false, false); + file.setExecutable(false, false); + // Add permissions for owner + file.setReadable(true); + file.setWritable(true); + return file; + } + } -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Jan 26 15:59:22 2020 From: jython-checkins at python.org (jeff.allen) Date: Sun, 26 Jan 2020 20:59:22 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Improved_=28failing=29_Con?= =?utf-8?q?currentTypeTest?= Message-ID: <20200126205922.1.220E069B751EAF2D@mg.python.org> https://hg.python.org/jython/rev/20162088f4f8 changeset: 8319:20162088f4f8 user: Jeff Allen date: Thu Jan 23 15:12:48 2020 +0000 summary: Improved (failing) ConcurrentTypeTest Part of the investigation of #2834, adds use of Thread.sleep to some of the test scripts, access to which fails quite frequently. The case testSeparateLoader2() does not produce failures because incorrect behaviour is currently not tested. files: src/org/python/core/packagecache/SysPackageManager.java | 6 +- tests/java/org/python/core/ConcurrentTypeTest.java | 166 +++++++-- 2 files changed, 122 insertions(+), 50 deletions(-) diff --git a/src/org/python/core/packagecache/SysPackageManager.java b/src/org/python/core/packagecache/SysPackageManager.java --- a/src/org/python/core/packagecache/SysPackageManager.java +++ b/src/org/python/core/packagecache/SysPackageManager.java @@ -240,9 +240,9 @@ doDir(this.searchPath, ret, jpkg, instantiate, exclpkgs); - PySystemState system = Py.getSystemState(); - if (system.getClassLoader() == null) { - doDir(system.path, ret, jpkg, instantiate, exclpkgs); + PySystemState sys = Py.getSystemState(); + if (sys.getClassLoader() == null) { + doDir(sys.path, ret, jpkg, instantiate, exclpkgs); } return merge(basic, ret); diff --git a/tests/java/org/python/core/ConcurrentTypeTest.java b/tests/java/org/python/core/ConcurrentTypeTest.java --- a/tests/java/org/python/core/ConcurrentTypeTest.java +++ b/tests/java/org/python/core/ConcurrentTypeTest.java @@ -5,8 +5,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNotNull; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -43,7 +43,7 @@ */ public class ConcurrentTypeTest { - private static int RUNNERS = 30; + private static int RUNNERS = 100; static { // Do not need site.py for test: makes more complicated in IDE. @@ -52,20 +52,30 @@ private abstract static class ScriptRunner implements Runnable { - final String script; + final int instance; final Thread thread; + String script; + PyCode code; final PyStringMap globals = Py.newStringMap(); /** Sub-class constructor must assign the configured interpreter. */ - protected PythonInterpreter interp; + PythonInterpreter interp; - ScriptRunner(String script) { + ScriptRunner(int instance) { + this.instance = instance; + this.thread = new Thread(this); + this.globals.__setitem__("instance", Py.newInteger(instance)); + } + + /** Set the interpreter and compile the script. */ + void setInterp(PythonInterpreter interp, String script) { + this.interp = interp; this.script = script; - this.thread = new Thread(this); + this.code = interp.compile(script); } @Override public void run() { - interp.exec(script); + interp.exec(code); } } @@ -79,7 +89,7 @@ // Make all the runners in advance. List runners = new ArrayList<>(RUNNERS); for (int i = 0; i < RUNNERS; i++) { - runners.add(new SharedStateRunner(javaImportScript)); + runners.add(new SharedStateRunner(javaImportScript, i)); } // Start the runners then let all of them finish (or fail). @@ -87,9 +97,8 @@ // Check status of every thread for (SharedStateRunner r : runners) { - PyObject status = r.globals.__finditem__("status"); - assertTrue("status not set to an int", status instanceof PyInteger); - assertEquals(((PyInteger) status).asInt(), 1); + PyObject e = r.globals.__finditem__("e"); + assertEquals("Runner " + r.instance, null, e); } } @@ -99,30 +108,39 @@ */ //@formatter:off static final String javaImportScript = String.join("\n", new String[] { - "from javax.swing.text.Utilities import *", - "try:", - " f = getNextWord", - " status = 1", - "except Exception:", - " status = 0" + "try:", + " from javax.swing.text.Utilities import *", + " f = getNextWord", // May raise NameError + "except Exception as e:", + " pass" }); //@formatter:on /** - * Each instance of this type has its own interpreter, but they all share the same (default) - * {@code PySystemState} + * Each instance of this type has its own {@link PythonInterpreter}, but they all share the same + * (default) {@code PySystemState} */ private static class SharedStateRunner extends ScriptRunner { - SharedStateRunner(String script) { - super(script); - this.interp = new PythonInterpreter(globals); + SharedStateRunner(String script, int instance) { + super(instance); + setInterp(new PythonInterpreter(globals), script); } } /** * Test concurrency when importing the same Java class where the interpreters all have their own - * {@code PySystemState}. + * {@code PySystemState}. At version 2.7.2b2, this test occasionally fails with a + * {@code NameError}, when the member {@code getNextWord} imported from + * {@code javax.swing.text.Utilities} is not found. The cause is thought to be a race between + * threads in {@code SysPackageManager}. It is necessary to run this test in the shell with + * something like:
+     * PS jython-trunk> do {
+     * >>   java -cp "build\exposed;build\classes;extlibs\*" org.junit.runner.JUnitCore org.python.core.ConcurrentTypeTest
+     * >> } while ($LastExitCode -eq 0)
+     *
Expect a hundred iterations or more before you see + * {@code "name 'getNextWord' is not defined"}. Suppress other tests in the suite. + * */ @Test public void testSeparateState() { @@ -130,7 +148,7 @@ // Make all the runners in advance. List runners = new ArrayList<>(RUNNERS); for (int i = 0; i < RUNNERS; i++) { - runners.add(new SeparateStateRunner(javaImportScript)); + runners.add(new SeparateStateRunner(javaImportScript, i)); } // Start the runners then let all of them finish (or fail). @@ -138,24 +156,67 @@ // Check status of every thread for (SeparateStateRunner r : runners) { - PyObject status = r.globals.__finditem__("status"); - assertTrue("status not set to an int", status instanceof PyInteger); - assertEquals(((PyInteger) status).asInt(), 1); + PyObject e = r.globals.__finditem__("e"); + assertEquals("Runner " + r.instance, null, e); } } /** - * Each instance of this type has its own {@code PySystemState}, as well as its own interpreter. + * Script to import all names from a Java class for {@link #testSharedState()} and + * {@link #testSeparateState()}. At version 2.7.2b2, this test frequently (not every time) fails + * with an {@code AttributeError}, when member {@code sleep} of {@code java.lang.Thread} is not + * found for the call. + */ + //@formatter:off + static final String javaImportScript2 = String.join("\n", new String[] { + "try:", + " from java.lang import Thread", + " Thread.sleep(instance*10)", // May raise AttributeError + " from javax.swing.text.Utilities import *", + " f = getNextWord", // May raise NameError + "except Exception as e:", + " pass" + }); + //@formatter:on + + /** + * Test concurrency when importing the same Java class where the interpreters all have their own + * {@code PySystemState}. It differs from {@link #testSeparateState()} in importing + * {@code Thread} add calling {@code Thread.sleep}. (The original idea was to provoke + * switching.) At version 2.7.2b2, this test fails about one time in 5 with an AttributeError, + * when the member {@code Thread.sleep} is not found. + */ + @Test + public void testSeparateState2() { + + // Make all the runners in advance. + List runners = new ArrayList<>(RUNNERS); + for (int i = 0; i < RUNNERS; i++) { + runners.add(new SeparateStateRunner(javaImportScript2, i)); + } + + // Start the runners then let all of them finish (or fail). + awaitAll(runners); + + // Check status of every thread + for (SeparateStateRunner r : runners) { + PyObject e = r.globals.__finditem__("e"); + assertEquals("Runner " + r.instance, null, e); + } + } + + /** + * Each instance of this type has its own {@code PySystemState}, as well as its own + * {@link PythonInterpreter}. */ private static class SeparateStateRunner extends ScriptRunner { final PySystemState sys = new PySystemState(); - SeparateStateRunner(String script) { - super(script); - this.interp = new PythonInterpreter(globals, sys); + SeparateStateRunner(String script, int instance) { + super(instance); + setInterp(new PythonInterpreter(globals, sys), script); } - } /** @@ -171,15 +232,18 @@ // Make all the runners in advance, primed with the same script. List runners = new ArrayList<>(RUNNERS); for (int i = 0; i < RUNNERS; i++) { - runners.add(new SeparateLoaderRunner(loaderScript, fileManager.newClassLoader())); + runners.add(new SeparateLoaderRunner(loaderScript, i, fileManager.newClassLoader())); } // Start the runners then let all of them finish (or fail). awaitAll(runners); - // Check status of every thread + // Check result of every thread for (SeparateLoaderRunner r : runners) { + PyObject e = r.globals.__finditem__("e"); + assertEquals("Runner " + r.instance, null, e); PyObject staticConstant = r.globals.__finditem__("staticConstant"); + assertNotNull(staticConstant); assertEquals(staticConstant.asInt(), 42); PyObject x = r.globals.__finditem__("x"); assertEquals(x.asInt(), 42); @@ -204,13 +268,16 @@ //@formatter:on /** - * Script to import all names from a Java class conjured from thin air (via class loader), usaed + * Script to import all names from a Java class conjured from thin air (via class loader), used * by {@link #testSeparateLoader()}. */ //@formatter:off static final String loaderScript = String.join("\n", new String[] { - "from thin.air.Foo import *", - "x = staticMethod()" + "try:", + " from thin.air.Foo import *", + " x = staticMethod()", + "except Exception as e:", + " pass" }); //@formatter:on @@ -228,17 +295,19 @@ // Make all the runners in advance, primed with the same script. List runners = new ArrayList<>(RUNNERS); for (int i = 0; i < RUNNERS; i++) { - runners.add(new SeparateLoaderRunner(loaderScript2, fileManager.newClassLoader())); + runners.add(new SeparateLoaderRunner(loaderScript2, i, fileManager.newClassLoader())); } // Start the runners then let all of them finish (or fail). awaitAll(runners); - // Check status of every thread + // Check result of every thread Set classes = new HashSet<>(); Set types = new HashSet<>(); for (SeparateLoaderRunner r : runners) { + PyObject e = r.globals.__finditem__("e"); + assertEquals("Runner " + r.instance, null, e); PyObject m = r.globals.__finditem__("m"); assertEquals(m.toString(), "forty-two"); PyObject x = r.globals.__finditem__("x"); @@ -260,25 +329,28 @@ */ //@formatter:off static final String loaderScript2 = String.join("\n", new String[] { - "from thin.air import Foo", - "f = Foo()", - "m = f.member", - "x = f.method()" + "try:", + " from thin.air import Foo", + " f = Foo()", + " m = f.member", + " x = f.method()", + "except Exception as e:", + " pass" }); //@formatter:on /** * Each instance of this type has its own {@code ClassLoader}, as well as its own {@code sys} - * module and interpreter. + * module and {@link PythonInterpreter}. */ private static class SeparateLoaderRunner extends ScriptRunner { final PySystemState sys = new PySystemState(); - SeparateLoaderRunner(String script, ClassLoader classLoader) { - super(script); + SeparateLoaderRunner(String script, int instance, ClassLoader classLoader) { + super(instance); sys.setClassLoader(classLoader); - this.interp = new PythonInterpreter(globals, sys); + setInterp(new PythonInterpreter(globals, sys), script); } } -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Jan 26 15:59:23 2020 From: jython-checkins at python.org (jeff.allen) Date: Sun, 26 Jan 2020 20:59:23 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Re-work_the_publication_of?= =?utf-8?q?_incomplete_PyTypes_=28fixes_=232834=29?= Message-ID: <20200126205923.1.17DABEFF379B955B@mg.python.org> https://hg.python.org/jython/rev/44280f7e1854 changeset: 8320:44280f7e1854 user: Jeff Allen date: Sun Jan 26 00:17:18 2020 +0000 summary: Re-work the publication of incomplete PyTypes (fixes #2834) It is necessary to allow PyType.fromClass() to return incompletely consructed types when building dependent PyTypes in a single thread. This change intends they should not escape to other threads. We use a RuntimeException to return a PyType without binding it to its class prematurely. Succinct debug methods are left in the code (disabled). We've had to write these several times to address similar bugs. files: NEWS | 1 + src/org/python/core/PyType.java | 140 ++++++++++++++++--- 2 files changed, 116 insertions(+), 25 deletions(-) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -8,6 +8,7 @@ Jython 2.7.2b3 Bugs fixed + - [ 2834 ] Import of Java classes is not thread safe - [ 2820 ] Import fails with UnicodeDecodeError if sys.path contains invalid UTF-8 bytes - [ 2826 ] Unicode hex string decode failure - [ 2836 ] Java Swing library works only in interactive jython session diff --git a/src/org/python/core/PyType.java b/src/org/python/core/PyType.java --- a/src/org/python/core/PyType.java +++ b/src/org/python/core/PyType.java @@ -5,8 +5,8 @@ import java.lang.ref.Reference; import java.lang.ref.ReferenceQueue; import java.lang.ref.WeakReference; +import java.util.ArrayList; import java.util.Arrays; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.IdentityHashMap; @@ -14,8 +14,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; import java.util.concurrent.atomic.AtomicReferenceArray; import org.python.antlr.ast.cmpopType; @@ -194,7 +192,7 @@ */ private static final class Registry { - /** Mapping of Java classes to their PyTypes, not yet in {@link #classToType}. */ + /** Mapping of Java classes to their PyTypes, under construction. */ private static final Map, PyType> classToNewType = new IdentityHashMap<>(); /** Mapping of Java classes to their TypeBuilders, until these are used. */ @@ -204,16 +202,36 @@ static ClassValue classToType = new ClassValue() { @Override - protected PyType computeValue(Class c) { + protected PyType computeValue(Class c) throws IncompleteType { synchronized (Registry.class) { // Competing threads will block and this thread will win the return. - resolveType(c); - return classToNewType.remove(c); + return resolveType(c); } } }; /** + * An exception used internally to signal a result from {@code classToType.get()} when the + * type is still under construction. + */ + private static class IncompleteType extends RuntimeException { + + /** The incompletely constructed {@code PyType}. */ + final PyType type; + + IncompleteType(PyType type) { + this.type = type; + } + } + + /** + * Mapping of Java classes to their PyTypes that have been thrown as {@link IncompleteType} + * exceptions. That action causes the {@code computeValue()} in progress to be re-tried, so + * we have to have the answer ready. + */ + private static final Map, PyType> classToThrownType = new IdentityHashMap<>(); + + /** * A list of PyObject sub-classes, instances of which are used in the type * system itself. We require to reference their PyTypes before Java static * initialisation can produce a builder, because that initialisation itself depends on @@ -245,7 +263,7 @@ * {@link #resolveType(Class)} is called reentrantly, while attempting to type some other * class. */ - private static boolean topLevel = true; + private static int depth = 0; /** Java types awaiting processing of their inner classes. */ private static Set needsInners = new HashSet<>(); @@ -270,58 +288,87 @@ } /** - * Place the PyType for the given target Java class in {@link #classToNewType} - * if it is not there already. This supports {@link PyType#fromClass(Class)} in the case - * where a PyType has not already been published. See there for caveats. If the - * making of a type (it will be a PyJavaType) might require the processing of - * inner classes, {@link PyType#fromClass(Class)} is called recursively for each. + * Return the PyType for the given target Java class. During the execution of + * this method, {@link #classToNewType} will hold any incompletely constructed + * {@link PyType}s, and a second attempt to resolve the same class will throw an + * {@link IncompleteType}, holding the incomplete {@code PyType}. This supports + * {@link PyType#fromClass(Class)} in the case where a PyType has not already + * been published. See there for caveats. If the making of a type (it will be a + * PyJavaType) might require the processing of inner classes, + * {@link PyType#fromClass(Class)} is called recursively for each. *

* The caller guarantees that this thread holds the lock on the registry. * * @param c for which a PyType is to be created + * @throws IncompleteType to signal an incompletely constructed result */ - static void resolveType(Class c) { + static PyType resolveType(Class c) throws IncompleteType { + // log("resolve", c); PyType type = classToNewType.get(c); if (type != null) { - // The type for c was "waiting" in classToNewType (some reentrant calls land here). - return; + /* + * The type for c is still under construction, so we cannot return it normally, + * which would publish it immediately to other threads. The client must be our + * thread, calling re-entrantly, so we sneak it a reference in an exception, and + * remember we have done this. + */ + classToThrownType.put(c, type); + // log("> ", type); + throw new IncompleteType(type); - } else if (!topLevel) { + } else if ((type = classToThrownType.remove(c)) != null) { + /* + * The type for c has been fully constructed, but was ignored because an exception + * was raised between the call to computeValue() and the return. . + */ + + } else if (depth > 0) { /* * This is a nested call about a class c we haven't seen before. Create (or choose * an existing) type for c. */ + depth += 1; addFromClass(c); + depth -= 1; + type = classToNewType.remove(c); } else { /* * This is a top-level call about a class c we haven't seen before. Create (or * choose an existing) type for c. (In rare circumstances a thread that saw the * cache miss will arrive here after some other thread populates classToNewType. The - * duplicate will be discarded and this must be harmless. + * duplicate will be discarded, but the implementation must ensure this is harmless. */ assert needsInners.isEmpty(); try { // Signal to further invocations that they are nested. - topLevel = false; + depth = 1; // Create (or choose an existing) type for c, accumulating classes in // needsInners. addFromClass(c); - // Process inner classes too, if necessary. (This invalidates needsIners.) + // Process inner classes too, if necessary. (This invalidates needsInners.) if (!needsInners.isEmpty()) { processInners(); } } finally { // Guarantee subsequent calls are top-level and needsInners is empty. - topLevel = true; + depth = 0; needsInners.clear(); } + type = classToNewType.remove(c); } + + /* + * Return the PyType we made for c, which is now complete. (We may still be part way + * through making others.) + */ + // log("+-->", type); + return type; } /** @@ -338,7 +385,7 @@ PyJavaType[] ni = needsInners.toArray(new PyJavaType[needsInners.size()]); // Ensure calls to fromClass are top-level. - topLevel = true; + depth = 0; needsInners.clear(); for (PyJavaType javaType : ni) { @@ -504,6 +551,40 @@ } return missing; } + + // -------- Debugging for the registry -------- + private static void log(String where, Class c) { + String name = abbr(c.getName()); + System.err.printf("%s%s: %s %s thr=%s\n", pad(), where, name, names(classToNewType), + names(classToThrownType)); + // logger.log(Level.INFO, "{0}{1}: {2}", new Object[] {pad, where, name}); + } + + private static void log(String kind, PyType result) { + String r = result.toString(); + System.err.printf("%s%s %s %s thr=%s\n", pad(), kind, r, names(classToNewType), + names(classToThrownType)); + } + + /** For logging formatting. */ + private static final String PAD = " "; + + private static String abbr(String name) { + return name.replace("java.lang.", "j.l.").replace("org.python.core.", ""); + } + + private static String pad() { + int d = Math.min(Math.max(2 * depth, 0), PAD.length()); + return PAD.substring(0, d); + } + + private static List names(Map, PyType> map) { + ArrayList names = new ArrayList<>(map.size()); + for (Class k : map.keySet()) { + names.add(abbr(k.getName())); + } + return names; + } } /** @@ -2050,10 +2131,18 @@ * @param c for which the corresponding PyType is to be found * @return the PyType found or created */ - // It would be nice to give a less complicated guarantee about when the PyType is complete. public static PyType fromClass(Class c) { - // Look up or create a Python type for c in the registry. - return Registry.classToType.get(c); + try { + // Look up or create a Python type for c in the registry. + return Registry.classToType.get(c); + } catch (Registry.IncompleteType it) { + /* + * This *only* happens when called recursively during type construction, and therefore + * we can assume the caller is prepared to receive an incompletely constructed PyType as + * the answer. * + */ + return it.type; + } } @ExposedMethod(doc = BuiltinDocs.type___getattribute___doc) @@ -2715,6 +2804,7 @@ table.compareAndSet(index, entry, new MethodCacheEntry(versionTag, name, where[0], value)); } + return value; } -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Jan 26 15:59:25 2020 From: jython-checkins at python.org (jeff.allen) Date: Sun, 26 Jan 2020 20:59:25 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Update_netty_JARs_to_4=2E1?= =?utf-8?q?=2E45_=28issue_=232828_and_CVE-2019-16869=29?= Message-ID: <20200126205925.1.238B406D45A19616@mg.python.org> https://hg.python.org/jython/rev/ccd1215b3d0e changeset: 8322:ccd1215b3d0e user: Jeff Allen date: Sun Jan 26 18:32:58 2020 +0000 summary: Update netty JARs to 4.1.45 (issue #2828 and CVE-2019-16869) files: NEWS | 1 + build.gradle | 12 ++-- build.xml | 24 +++++----- extlibs/netty-buffer-4.1.24.Final.jar | Bin extlibs/netty-buffer-4.1.45.Final.jar | Bin extlibs/netty-codec-4.1.24.Final.jar | Bin extlibs/netty-codec-4.1.45.Final.jar | Bin extlibs/netty-common-4.1.24.Final.jar | Bin extlibs/netty-common-4.1.45.Final.jar | Bin extlibs/netty-handler-4.1.24.Final.jar | Bin extlibs/netty-handler-4.1.45.Final.jar | Bin extlibs/netty-resolver-4.1.24.Final.jar | Bin extlibs/netty-resolver-4.1.45.Final.jar | Bin extlibs/netty-transport-4.1.24.Final.jar | Bin extlibs/netty-transport-4.1.45.Final.jar | Bin 15 files changed, 19 insertions(+), 18 deletions(-) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -8,6 +8,7 @@ Jython 2.7.2b3 Bugs fixed + - [ 2828 ] Update netty JARs to 4.1.45 - [ 2044 ] CVE-2013-2027 Current umask sets privileges of class files and cache - [ 2834 ] Import of Java classes is not thread safe - [ 2820 ] Import fails with UnicodeDecodeError if sys.path contains invalid UTF-8 bytes diff --git a/build.gradle b/build.gradle --- a/build.gradle +++ b/build.gradle @@ -182,12 +182,12 @@ implementation 'jline:jline:2.14.5' - implementation 'io.netty:netty-buffer:4.1.24.Final' - implementation 'io.netty:netty-codec:4.1.24.Final' - implementation 'io.netty:netty-common:4.1.24.Final' - implementation 'io.netty:netty-handler:4.1.24.Final' - implementation 'io.netty:netty-resolver:4.1.24.Final' - implementation 'io.netty:netty-transport:4.1.24.Final' + implementation 'io.netty:netty-buffer:4.1.45.Final' + implementation 'io.netty:netty-codec:4.1.45.Final' + implementation 'io.netty:netty-common:4.1.45.Final' + implementation 'io.netty:netty-handler:4.1.45.Final' + implementation 'io.netty:netty-resolver:4.1.45.Final' + implementation 'io.netty:netty-transport:4.1.45.Final' // Used implicitly in the Ant build, must be explicit here implementation 'org.apache.ant:ant:1.9.7' diff --git a/build.xml b/build.xml --- a/build.xml +++ b/build.xml @@ -260,12 +260,12 @@ - - - - - - + + + + + + @@ -843,17 +843,17 @@ - + - + - + - + - + - + diff --git a/extlibs/netty-buffer-4.1.24.Final.jar b/extlibs/netty-buffer-4.1.24.Final.jar deleted file mode 100644 index c309e99ca606f1cccbd5ab11dcd0f898b14a7857..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 GIT binary patch [stripped] diff --git a/extlibs/netty-buffer-4.1.45.Final.jar b/extlibs/netty-buffer-4.1.45.Final.jar new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..45d9ff28a5bc57fd59ff95981de506dbd41a1580 GIT binary patch [stripped] diff --git a/extlibs/netty-codec-4.1.24.Final.jar b/extlibs/netty-codec-4.1.24.Final.jar deleted file mode 100644 index 6ca8e3da4601f3efe2d42686d7fca85bd9e73032..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 GIT binary patch [stripped] diff --git a/extlibs/netty-codec-4.1.45.Final.jar b/extlibs/netty-codec-4.1.45.Final.jar new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..e8378e7667755c582de8927a91202967f2e5a79f GIT binary patch [stripped] diff --git a/extlibs/netty-common-4.1.24.Final.jar b/extlibs/netty-common-4.1.24.Final.jar deleted file mode 100644 index a5f89c8bf5b5bdac7396a610c480f379c3462049..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 GIT binary patch [stripped] diff --git a/extlibs/netty-common-4.1.45.Final.jar b/extlibs/netty-common-4.1.45.Final.jar new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..038f1f72c45a2bf2bcad46a288a6d93c814a58b4 GIT binary patch [stripped] diff --git a/extlibs/netty-handler-4.1.24.Final.jar b/extlibs/netty-handler-4.1.24.Final.jar deleted file mode 100644 index e3e90ce2e140d02af44e3f1a8823ab1cb91bcfd2..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 GIT binary patch [stripped] diff --git a/extlibs/netty-handler-4.1.45.Final.jar b/extlibs/netty-handler-4.1.45.Final.jar new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..ef3d012e7c17dbd59361631554246878cfa592e9 GIT binary patch [stripped] diff --git a/extlibs/netty-resolver-4.1.24.Final.jar b/extlibs/netty-resolver-4.1.24.Final.jar deleted file mode 100644 index fcbab834e3147319b489403e82aac7d0e8c0960f..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 GIT binary patch [stripped] diff --git a/extlibs/netty-resolver-4.1.45.Final.jar b/extlibs/netty-resolver-4.1.45.Final.jar new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..defcf0483208a89c6932d19cbf46cd4a3d2cda08 GIT binary patch [stripped] diff --git a/extlibs/netty-transport-4.1.24.Final.jar b/extlibs/netty-transport-4.1.24.Final.jar deleted file mode 100644 index 63ad9edfc2052b1c41533b46d121e927c6f1e39f..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 GIT binary patch [stripped] diff --git a/extlibs/netty-transport-4.1.45.Final.jar b/extlibs/netty-transport-4.1.45.Final.jar new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..2a16b1cedf59c4f87a2af1a230c85305d3ade766 GIT binary patch [stripped] -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Thu Jan 30 03:15:28 2020 From: jython-checkins at python.org (jeff.allen) Date: Thu, 30 Jan 2020 08:15:28 +0000 Subject: [Jython-checkins] =?utf-8?b?anl0aG9uOiBDcmVhdGUgYSBfX21haW5fXyBt?= =?utf-8?q?odule_and_set_=5F=5Fname=5F=5F_in_JSR-223_=28fixes_=232846=29?= =?utf-8?q?=2E?= Message-ID: <20200130081528.1.E78AA5154F78E161@mg.python.org> https://hg.python.org/jython/rev/58d45a33c32f changeset: 8323:58d45a33c32f user: Jeff Allen date: Thu Jan 30 08:02:32 2020 +0000 summary: Create a __main__ module and set __name__ in JSR-223 (fixes #2846). This change causes the default engine context in a PyScriptEngine to be created as a module __main__, and __name__ to be defined, comparable to the environment found when executing a Python program. The thread-local name space concept is otherwise undisturbed. org.python.jsr223.ScriptEngineTest cases are modified correspondingly, and testThreadLocalBindings is fixed to test isolation within a thread. files: NEWS | 1 + src/org/python/util/PythonInterpreter.java | 6 +- tests/java/org/python/jsr223/ScriptEngineTest.java | 63 +++++++-- 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -8,6 +8,7 @@ Jython 2.7.2b3 Bugs fixed + - [ 2846 ] Main module __name __ is not "__main__" under Java Scripting API - [ 2828 ] Update netty JARs to 4.1.45 - [ 2044 ] CVE-2013-2027 Current umask sets privileges of class files and cache - [ 2834 ] Import of Java classes is not thread safe diff --git a/src/org/python/util/PythonInterpreter.java b/src/org/python/util/PythonInterpreter.java --- a/src/org/python/util/PythonInterpreter.java +++ b/src/org/python/util/PythonInterpreter.java @@ -101,10 +101,8 @@ setSystemState(); this.useThreadLocalState = useThreadLocalState; - if (!useThreadLocalState) { - PyModule module = new PyModule("__main__", globals); - this.systemState.modules.__setitem__("__main__", module); - } + PyModule module = new PyModule("__main__", globals); + this.systemState.modules.__setitem__("__main__", module); if (Options.Qnew) { cflags.setFlag(CodeFlag.CO_FUTURE_DIVISION); diff --git a/tests/java/org/python/jsr223/ScriptEngineTest.java b/tests/java/org/python/jsr223/ScriptEngineTest.java --- a/tests/java/org/python/jsr223/ScriptEngineTest.java +++ b/tests/java/org/python/jsr223/ScriptEngineTest.java @@ -15,12 +15,13 @@ import javax.script.ScriptException; import javax.script.SimpleScriptContext; -import junit.framework.TestCase; - import org.junit.Assert; import org.python.core.Options; +import org.python.core.PyList; import org.python.core.PyString; +import junit.framework.TestCase; + public class ScriptEngineTest extends TestCase { public void testEvalString() throws ScriptException { @@ -160,25 +161,37 @@ assertNull(pythonEngine.get("x")); } - class ThreadLocalBindingsTest implements Runnable { + static class ThreadLocalBindingsTest implements Runnable { ScriptEngine engine; + int value; Object x; + Object name; Throwable exception; - public ThreadLocalBindingsTest(ScriptEngine engine) { + public ThreadLocalBindingsTest(ScriptEngine engine, int value) { this.engine = engine; + this.value = value; } + //@formatter:off + static final String script = String.join("\n", new String[] { + "try:", + " a", + "except NameError:", + " pass", + "else:", + " raise Exception('a is defined', a)"}); + //@formatter:on + @Override public void run() { try { Bindings bindings = engine.createBindings(); - assertNull(engine.eval( - "try: a\nexcept NameError: pass\nelse: raise Exception('a is defined', a)", - bindings)); - bindings.put("x", -7); + assertNull(engine.eval(script, bindings)); + bindings.put("x", value); x = engine.eval("x", bindings); + name = engine.eval("__name__", bindings); } catch (Throwable e) { e.printStackTrace(); exception = e; @@ -186,6 +199,13 @@ } } + /** + * Test that, with the use of a {@code Bindings} argument to {@code ScriptEngine.eval}, the + * interpreter is presented with a distinct name space, whether in a thread or not, and + * that __name__ == "__main__" in the engine-scoped name space (only). + * + * @throws Exception + */ public void testThreadLocalBindings() throws ScriptException, InterruptedException { ScriptEngineManager manager = new ScriptEngineManager(); ScriptEngine pythonEngine = manager.getEngineByName("python"); @@ -193,15 +213,27 @@ pythonEngine.put("a", 42); pythonEngine.put("x", 15); - ThreadLocalBindingsTest test = new ThreadLocalBindingsTest(pythonEngine); - Thread thread = new Thread(test); - thread.run(); - thread.join(); + // Examine name space of the engine with Bindings + ThreadLocalBindingsTest test = new ThreadLocalBindingsTest(pythonEngine, -7); + test.run(); // This does not start a thread assertNull(test.exception); assertEquals(-7, test.x); + assertEquals("__builtin__", test.name); + + // Examine name space of the engine with Bindings and in a thread + ThreadLocalBindingsTest test2 = new ThreadLocalBindingsTest(pythonEngine, -22); + Thread thread = new Thread(test2); + thread.start(); // This *does* start a thread + thread.join(); + assertNull(test2.exception); + assertEquals(-22, test2.x); + assertEquals("__builtin__", test2.name); + + // Test name space of the pythonEngine without Bindings is unaffected assertEquals(15, pythonEngine.get("x")); assertNull(pythonEngine.eval("del x")); assertNull(pythonEngine.get("x")); + assertEquals("__main__", pythonEngine.eval("__name__")); } public void testInvoke() throws ScriptException, NoSuchMethodException { @@ -295,9 +327,10 @@ ScriptEngine pythonEngine = manager.getEngineByName("python"); pythonEngine.eval("a = 4"); pythonEngine.eval("b = 'hi'"); - assertEquals( - "['__builtins__', 'a', 'b']", - pythonEngine.eval("repr(sorted((item for item in locals())))")); + PyList locals = (PyList) pythonEngine.eval("sorted((item for item in locals()))"); + Assert.assertTrue(locals.contains("a")); + Assert.assertTrue(locals.contains("b")); + Assert.assertTrue(locals.contains("__name__")); } public void testScope_lookup() throws ScriptException { -- Repository URL: https://hg.python.org/jython