From jython-checkins at python.org Sun Mar 2 21:24:45 2014 From: jython-checkins at python.org (jeff.allen) Date: Sun, 2 Mar 2014 21:24:45 +0100 (CET) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Remove_test_of_symbolic_lin?= =?utf-8?q?ks_on_Windows_=28os=2Esymlink_not_available=29=2E?= Message-ID: <3fcbZ55dbyz7LjQ@mail.python.org> http://hg.python.org/jython/rev/586159611003 changeset: 7189:586159611003 user: Jeff Allen date: Thu Feb 27 08:41:36 2014 +0000 summary: Remove test of symbolic links on Windows (os.symlink not available). files: Lib/test/test_chdir.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_chdir.py b/Lib/test/test_chdir.py --- a/Lib/test/test_chdir.py +++ b/Lib/test/test_chdir.py @@ -715,6 +715,8 @@ SymlinkTestCase] if WINDOWS: tests.append(WindowsChdirTestCase) + tests.remove(SymlinkTestCase) # os.symlink ... Availability: Unix. + if test_support.is_jython: tests.extend((ImportJavaClassTestCase, ImportJarTestCase)) -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Sun Mar 2 21:24:46 2014 From: jython-checkins at python.org (jeff.allen) Date: Sun, 2 Mar 2014 21:24:46 +0100 (CET) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Tweak_test=5Fpopen2_and_tes?= =?utf-8?q?t=5Fcmd=5Fline_to_own_their_Popen=2E=5F=5Fdel=5F=5F_failures=2E?= Message-ID: <3fcbZ66vGfz7Lk3@mail.python.org> http://hg.python.org/jython/rev/f238f769301a changeset: 7190:f238f769301a user: Jeff Allen date: Sat Mar 01 23:22:35 2014 +0000 summary: Tweak test_popen2 and test_cmd_line to own their Popen.__del__ failures. Force GC to run during these tests so that failures they cause appear during the tests, rather than randomly later. files: Lib/test/test_cmd_line.py | 9 +++++++++ Lib/test/test_popen2.py | 10 +++++++++- 2 files changed, 18 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -5,6 +5,15 @@ import subprocess class CmdLineTest(unittest.TestCase): + + @classmethod + def tearDownClass(cls): + if test.test_support.is_jython: + # GC is not immediate, so if Popen.__del__ may be delayed. + # Try to force any Popen.__del__ errors within scope of test. + from test_weakref import extra_collect + extra_collect() + def start_python(self, cmd_line): outfp, infp = popen2.popen4('"%s" %s' % (sys.executable, cmd_line)) infp.close() diff --git a/Lib/test/test_popen2.py b/Lib/test/test_popen2.py --- a/Lib/test/test_popen2.py +++ b/Lib/test/test_popen2.py @@ -12,7 +12,7 @@ import unittest import popen2 -from test.test_support import run_unittest, reap_children +from test.test_support import run_unittest, reap_children, is_jython if sys.platform[:4] == 'beos' or sys.platform[:6] == 'atheos': # Locks get messed up or something. Generally we're supposed @@ -59,6 +59,14 @@ self.assertFalse(subprocess._active, "subprocess._active not empty") reap_children() + @classmethod + def tearDownClass(cls): + if is_jython: + # GC is not immediate, so if Popen.__del__ may be delayed. + # Try to force Popen.__del__ within scope of test. + from test_weakref import extra_collect + extra_collect() + def validate_output(self, teststr, expected_out, r, w, e=None): w.write(teststr) w.close() -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Sun Mar 2 21:24:48 2014 From: jython-checkins at python.org (jeff.allen) Date: Sun, 2 Mar 2014 21:24:48 +0100 (CET) Subject: [Jython-checkins] =?utf-8?b?anl0aG9uOiBGaXggUG9wZW4uX19kZWxfXyBm?= =?utf-8?q?ailures_that_occur_during_GC=2E?= Message-ID: <3fcbZ81Pscz7Lk3@mail.python.org> http://hg.python.org/jython/rev/78efc2f8c832 changeset: 7191:78efc2f8c832 user: Jeff Allen date: Sun Mar 02 19:19:13 2014 +0000 summary: Fix Popen.__del__ failures that occur during GC. Provides Jython version of subprocess.Popen._internal_poll method. files: Lib/subprocess.py | 13 +++++++++++++ 1 files changed, 13 insertions(+), 0 deletions(-) diff --git a/Lib/subprocess.py b/Lib/subprocess.py --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -1353,6 +1353,19 @@ pass return self.returncode + def _internal_poll(self, _deadstate=None): + """Check if child process has terminated. Returns returncode + attribute. Called by __del__.""" + if self.returncode is None: + try: + self.returncode = self._process.exitValue() + except java.lang.IllegalThreadStateException: + # The child process is not ready to return status, so None os still right. + pass + except java.io.IOException: + # Child has exited but returncode lost? + self.returncode = _deadstate + return self.returncode def wait(self): """Wait for child process to terminate. Returns returncode -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Sat Mar 15 10:00:31 2014 From: jython-checkins at python.org (jeff.allen) Date: Sat, 15 Mar 2014 10:00:31 +0100 (CET) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Improve_os=2Echdir_for_Wind?= =?utf-8?q?ows=2C_partially_addressing_=232117?= Message-ID: <3fmHmb29jwzP0J@mail.python.org> http://hg.python.org/jython/rev/f553f6a07fb5 changeset: 7192:f553f6a07fb5 user: Jeff Allen date: Sat Mar 08 17:49:48 2014 +0000 summary: Improve os.chdir for Windows, partially addressing #2117 Corrects ntpath.abspath based on CPython 2.7.6 but using PySystemState.getPath; adds logic to maximise use of native wit behind java.io.File for Windows. files: Lib/ntpath.py | 26 +- Lib/test/test_chdir.py | 58 ++++- src/org/python/core/PySystemState.java | 144 ++++++++++++- 3 files changed, 194 insertions(+), 34 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -452,23 +452,25 @@ from nt import _getfullpathname except ImportError: # not running on Windows - mock up something sensible - import java.io.File - from org.python.core.Py import newString def abspath(path): """Return the absolute version of a path.""" - if not isabs(path): + try: if isinstance(path, unicode): - cwd = os.getcwdu() + if path: + path = sys.getPath(path) + else: + # Empty path must return current working directory + path = os.getcwdu() else: - cwd = os.getcwd() - path = join(cwd, path) - if not splitunc(path)[0] and not splitdrive(path)[0]: - # cwd lacks a UNC mount point, so it should have a drive - # letter (but lacks one): determine it - canon_path = newString(java.io.File(path).getCanonicalPath()) - drive = splitdrive(canon_path)[0] - path = join(drive, path) + if path: + path = sys.getPath(path).encode('latin-1') + else: + # Empty path must return current working directory + path = os.getcwd() + except EnvironmentError: + pass # Bad path - return unchanged. + return normpath(path) else: # use native Windows method on Windows diff --git a/Lib/test/test_chdir.py b/Lib/test/test_chdir.py --- a/Lib/test/test_chdir.py +++ b/Lib/test/test_chdir.py @@ -194,11 +194,39 @@ self.assertEqual(os.getcwd(), os.path.realpath(dos_name)) def test_windows_getcwd_ensures_drive_letter(self): - drive = os.path.splitdrive(self.subdir)[0] - os.chdir('\\') + # subdir is in the TEMP directory, usually on C:, while the + # current working directory could be (for the sake of comments) + # D:\HOME . TEMP and HOME stand for arbitrarily long relative paths. + + # Check chdir to \ occurs without change of drive letter. + drive0, sub0 = os.path.splitdrive(os.getcwd()) # d:, HOME + os.chdir('\\') # d:\ + self.assertEqual(os.path.normcase(os.getcwd()), + os.path.normcase(os.path.join(drive0, '\\'))) + + # Check chdir to HOME occurs without change of drive letter. + os.chdir(sub0) # d:\HOME + self.assertEqual(os.path.normcase(os.getcwd()), + os.path.normcase(os.path.join(drive0, sub0))) + + # Check chdir to path with drive letter, changes drive in cwd. + drive, sub = os.path.splitdrive(self.subdir) # c:, TEMP\Program Files + os.chdir(self.subdir) # c:\TEMP\Program Files + self.assertEqual(os.path.normcase(os.getcwd()), + os.path.normcase(os.path.join(drive, sub))) + + # Check chdir to \ occurs without change of drive letter (again). + os.chdir('\\') # c:\ self.assertEqual(os.path.normcase(os.getcwd()), os.path.normcase(os.path.join(drive, '\\'))) + if drive.upper() != drive0.upper(): + # Check chdir to (different) original drive takes us to previous directory too. + # You only get this test if the temp directory and cwd are on different drives. + os.chdir(drive0) # d:\HOME + self.assertEqual(os.path.normcase(os.getcwd()), + os.path.normcase(os.path.join(drive0, sub0))) + def test_windows_chdir_slash_isabs(self): drive = os.path.splitdrive(os.getcwd())[0] os.chdir('/') @@ -702,17 +730,19 @@ def test_main(): - tests = [ChdirTestCase, - ImportTestCase, - ImportPackageTestCase, - ZipimportTestCase, - PyCompileTestCase, - ExecfileTestCase, - ExecfileTracebackTestCase, - ListdirTestCase, - DirsTestCase, - FilesTestCase, - SymlinkTestCase] + tests = [ + ChdirTestCase, + ImportTestCase, + ImportPackageTestCase, + ZipimportTestCase, + PyCompileTestCase, + ExecfileTestCase, + ExecfileTracebackTestCase, + ListdirTestCase, + DirsTestCase, + FilesTestCase, + SymlinkTestCase + ] if WINDOWS: tests.append(WindowsChdirTestCase) tests.remove(SymlinkTestCase) # os.symlink ... Availability: Unix. @@ -720,8 +750,10 @@ if test_support.is_jython: tests.extend((ImportJavaClassTestCase, ImportJarTestCase)) + if test_support.is_resource_enabled('subprocess'): tests.append(SubprocessTestCase) + test_support.run_unittest(*tests) diff --git a/src/org/python/core/PySystemState.java b/src/org/python/core/PySystemState.java --- a/src/org/python/core/PySystemState.java +++ b/src/org/python/core/PySystemState.java @@ -546,22 +546,148 @@ } private static String getPath(PySystemState sys, String path) { - if (path == null) { - return path; + if (path != null) { + path = getFile(sys, path).getAbsolutePath(); } + return path; + } + /** + * Resolve a path, returning a {@link File}, taking the current working directory into account. + * + * @param path a path String + * @return a resolved File + */ + public File getFile(String path) { + return getFile(this, path); + } + + /** + * Resolve a path, returning a {@link File}, taking the current working directory of the + * specified PySystemState into account. Use of a static here is a + * trick to avoid getting the current state if the path is absolute. (Noted that this may be + * needless optimisation.) + * + * @param sys a PySystemState or null meaning the current one + * @param path a path String + * @return a resolved File + */ + private static File getFile(PySystemState sys, String path) { File file = new File(path); - // Python considers r'\Jython25' and '/Jython25' abspaths on Windows, unlike - // java.io.File - if (!file.isAbsolute() - && (!Platform.IS_WINDOWS || !(path.startsWith("\\") || path.startsWith("/")))) { + if (!file.isAbsolute()) { + // path meaning depends on the current working directory if (sys == null) { sys = Py.getSystemState(); } - file = new File(sys.getCurrentWorkingDir(), path); + String cwd = sys.getCurrentWorkingDir(); + if (Platform.IS_WINDOWS) { + // Form absolute reference (with mysterious Windows semantics) + file = getWindowsFile(cwd, path); + } else { + // Form absolute reference (with single slash) + file = new File(cwd, path); + } } - // This needs to be performed always to trim trailing backslashes on Windows - return file.getPath(); + return file; + } + + /** + * Resolve a relative path against the supplied current working directory or Windows environment + * working directory for any drive specified in the path. and return a file object. Essentially + * equivalent to os.path.join, but the work is done by {@link File}. The intention is that + * calling {@link File#getAbsolutePath()} should return the corresponding absolute path. + *

+ * Note: in the context where we use this method, path is already known not to be + * absolute, and cwd is assumed to be absolute. + * + * @param cwd current working directory (of some {@link PySystemState}) + * @param path to resolve + * @return specifier of the intended file + */ + private static File getWindowsFile(String cwd, String path) { + // Assumptions: cwd is absolute and path is not absolute + + // Start by getting the slashes the right (wrong) way round. + if (path.indexOf('/') >= 0) { + path = path.replace('/', '\\'); + } + + // Does path start with a drive letter? + char d = driveLetter(path); + if (d != 0) { + if (d == driveLetter(cwd)) { + /* + * path specifies the same drive letter as in the cwd of this PySystemState. Let + * File interpret the rest of the path relative to cwd as parent. + */ + return new File(cwd, path.substring(2)); + } else { + // Let File resolve the specified drive against the process environment. + return new File(path); + } + + } else if (path.startsWith("\\")) { + // path replaces the file part of the cwd. (Method assumes path is not UNC.) + if (driveLetter(cwd) != 0) { + // cwd has a drive letter + return new File(cwd.substring(0, 2), path); + } else { + // cwd has no drive letter, so should be a UNC path \\host\share\directory\etc + return new File(uncShare(cwd), path); + } + + } else { + // path is relative to the cwd of this PySystemState. + return new File(cwd, path); + } + } + + /** + * Return the Windows drive letter from the start of the path, upper case, or 0 if + * the path does not start X: where X is a letter. + * + * @param path to examine + * @return drive letter or char 0 if no drive letter + */ + private static char driveLetter(String path) { + if (path.length() >= 2 && path.charAt(1) == ':') { + // Looks like both strings start with a drive letter + char pathDrive = path.charAt(0); + if (Character.isLetter(pathDrive)) { + return Character.toUpperCase(pathDrive); + } + } + return (char)0; + } + + /** + * Return the Windows UNC share name from the start of the path, or null if the + * path is not of Windows UNC type. The path has to be formed with Windows-backslashes: + * slashes '/' are not accepted as a substitute here. + * + * @param path to examine + * @return share name or null + */ + private static String uncShare(String path) { + int n = path.length(); + // Has to accommodate at least \\A (3 chars) + if (n >= 5 && path.startsWith("\\\\")) { + // Look for the separator backslash A\B + int p = path.indexOf('\\', 2); + // Has to be at least index 3 (path begins \\A) and 2 more characters left \B + if (p >= 3 && n > p + 2) { + // Look for directory backslash that ends the share name + int dir = path.indexOf('\\', p + 1); + if (dir < 0) { + // path has the form \\A\B (is just the share name) + return path; + } else if (dir > p + 1) { + // path has the form \\A\B\C + return path.substring(0, dir); + } + } + } + return null; } public void callExitFunc() throws PyIgnoreMethodTag { -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Sat Mar 15 10:00:32 2014 From: jython-checkins at python.org (jeff.allen) Date: Sat, 15 Mar 2014 10:00:32 +0100 (CET) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_indexer_JUnit_test_fail?= =?utf-8?q?ures_on_Windows=2E?= Message-ID: <3fmHmc5dRBz7Lk4@mail.python.org> http://hg.python.org/jython/rev/1753dd61195c changeset: 7193:1753dd61195c user: Jeff Allen date: Fri Mar 14 22:59:40 2014 +0000 summary: Fix indexer JUnit test failures on Windows. Removes extensive hard-coding of "/" as file path separator in favour of java.io.File to supply normalisation to platform. Also adds constant (in TestBase) to activate the logging built into the Indexer. files: build.xml | 2 + src/org/python/indexer/AstCache.java | 31 +++-- src/org/python/indexer/Indexer.java | 14 +- src/org/python/indexer/Util.java | 28 ++-- tests/java/org/python/indexer/TestBase.java | 57 +++++++-- 5 files changed, 86 insertions(+), 46 deletions(-) diff --git a/build.xml b/build.xml --- a/build.xml +++ b/build.xml @@ -269,6 +269,8 @@ deprecation = '${deprecation}' debug = '${debug}' nowarn = '${nowarn}' + test = '${test}' + test.source.dir = '${test.source.dir}' --- properties (used for full-build only) --- checkout.dir = '${checkout.dir}' javahl.dir = '${javahl.dir}' diff --git a/src/org/python/indexer/AstCache.java b/src/org/python/indexer/AstCache.java --- a/src/org/python/indexer/AstCache.java +++ b/src/org/python/indexer/AstCache.java @@ -4,14 +4,6 @@ */ package org.python.indexer; -import org.antlr.runtime.ANTLRFileStream; -import org.antlr.runtime.ANTLRStringStream; -import org.antlr.runtime.CharStream; -import org.antlr.runtime.RecognitionException; -import org.python.antlr.AnalyzingParser; -import org.python.antlr.base.mod; -import org.python.indexer.ast.NModule; - import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -24,13 +16,22 @@ import java.util.logging.Level; import java.util.logging.Logger; +import org.antlr.runtime.ANTLRFileStream; +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.runtime.CharStream; +import org.antlr.runtime.RecognitionException; +import org.python.antlr.AnalyzingParser; +import org.python.antlr.base.mod; +import org.python.indexer.ast.NModule; + /** * Provides a factory for python source ASTs. Maintains configurable on-disk and * in-memory caches to avoid re-parsing files during analysis. */ public class AstCache { - public static final String CACHE_DIR = Util.getSystemTempDir() + "jython/ast_cache/"; + public static final String CACHE_DIR = (new File(Util.getSystemTempDir(), "jython/ast_cache")) + .getAbsolutePath() + File.separator; private static final Logger LOG = Logger.getLogger(AstCache.class.getCanonicalName()); @@ -86,7 +87,9 @@ * @throws Exception if anything unexpected occurs */ public NModule getAST(String path) throws Exception { - if (path == null) throw new IllegalArgumentException("null path"); + if (path == null) { + throw new IllegalArgumentException("null path"); + } return fetch(path); } @@ -98,8 +101,12 @@ * @param contents the source to parse */ public NModule getAST(String path, String contents) throws Exception { - if (path == null) throw new IllegalArgumentException("null path"); - if (contents == null) throw new IllegalArgumentException("null contents"); + if (path == null) { + throw new IllegalArgumentException("null path"); + } + if (contents == null) { + throw new IllegalArgumentException("null contents"); + } // Cache stores null value if the parse failed. if (cache.containsKey(path)) { diff --git a/src/org/python/indexer/Indexer.java b/src/org/python/indexer/Indexer.java --- a/src/org/python/indexer/Indexer.java +++ b/src/org/python/indexer/Indexer.java @@ -132,7 +132,7 @@ if (logger == null) { throw new IllegalArgumentException("null logger param"); } - logger = logger; + this.logger = logger; } public Logger getLogger() { @@ -179,10 +179,12 @@ } throw new IndexingException(cause); } - if (msg == null) + if (msg == null) { msg = ""; - if (cause == null) + } + if (cause == null) { cause = new Exception(); + } logger.log(Level.WARNING, msg, cause); } @@ -243,7 +245,7 @@ } public boolean isLibFile(String file) { - if (file.startsWith("/")) { + if (file.startsWith(File.separator)) { return true; } if (path != null) { @@ -682,7 +684,7 @@ if (modname.endsWith(".py")) { modname = modname.substring(0, modname.length() - 3); } - String modpath = modname.replace('.', '/'); + String modpath = modname.replace('.', File.separatorChar); // A nasty hack to avoid e.g. python2.5 becoming python2/5. // Should generalize this for directory components containing '.'. @@ -909,7 +911,7 @@ public List getLoadedFiles() { List files = new ArrayList(); for (String file : moduleTable.keySet()) { - if (file.startsWith("/")) { + if (file.startsWith(File.separator)) { files.add(file); } } diff --git a/src/org/python/indexer/Util.java b/src/org/python/indexer/Util.java --- a/src/org/python/indexer/Util.java +++ b/src/org/python/indexer/Util.java @@ -19,7 +19,10 @@ public class Util { private static final String UTF_8 = "UTF-8"; - + private static final char SEPCHAR = File.separatorChar; + private static final String SEP = File.separator; + private static final String INIT_PY = "__init__.py"; + private static final String SEP_INIT_PY = SEP + INIT_PY; private static int gensymCount = -1; public static String gensym(String base) { @@ -29,11 +32,10 @@ public static String getSystemTempDir() { String tmp = System.getProperty("java.io.tmpdir"); - String sep = System.getProperty("file.separator"); - if (tmp.endsWith(sep)) { + if (tmp.endsWith(SEP)) { return tmp; } - return tmp + sep; + return tmp + SEP; } /** @@ -62,15 +64,15 @@ * @return null if {@code file} is not somewhere under the load path */ public static String moduleQname(String file) { - boolean initpy = file.endsWith("/__init__.py"); + boolean initpy = file.endsWith(SEP_INIT_PY); if (initpy) { - file = file.substring(0, file.length() - "/__init__.py".length()); + file = file.substring(0, file.length() - SEP_INIT_PY.length()); } else if (file.endsWith(".py")) { file = file.substring(0, file.length() - ".py".length()); } for (String root : Indexer.idx.getLoadPath()) { if (file.startsWith(root)) { - return file.substring(root.length()).replace('/', '.'); + return file.substring(root.length()).replace(SEPCHAR, '.'); } } return null; @@ -102,7 +104,7 @@ throw new IllegalStateException("failed assertion: " + path); } String fname = f.getName(); - if (fname.equals("__init__.py")) { + if (fname.equals(INIT_PY)) { return f.getParentFile().getName(); } return fname.substring(0, fname.lastIndexOf('.')); @@ -113,10 +115,10 @@ } public static File joinPath(String dir, String file) { - if (dir.endsWith("/")) { + if (dir.endsWith(SEP)) { return new File(dir + file); } - return new File(dir + "/" + file); + return new File(dir + SEP + file); } public static void writeFile(String path, String contents) throws Exception { @@ -189,15 +191,15 @@ /** * Return absolute path for {@code path}. - * Make sure path ends with "/" if it's a directory. + * Make sure path ends with SEP if it's a directory. * Does _not_ resolve symlinks, since the caller may need to play * symlink tricks to produce the desired paths for loaded modules. */ public static String canonicalize(String path) { File f = new File(path); path = f.getAbsolutePath(); - if (f.isDirectory() && !path.endsWith("/")) { - return path + "/"; + if (f.isDirectory() && !path.endsWith(SEP)) { + return path + SEP; } return path; } diff --git a/tests/java/org/python/indexer/TestBase.java b/tests/java/org/python/indexer/TestBase.java --- a/tests/java/org/python/indexer/TestBase.java +++ b/tests/java/org/python/indexer/TestBase.java @@ -4,33 +4,57 @@ */ package org.python.indexer; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStreamReader; +import java.util.logging.ConsoleHandler; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.logging.SimpleFormatter; + import junit.framework.TestCase; -import org.python.indexer.Def; -import org.python.indexer.Ref; -import org.python.indexer.ast.NNode; import org.python.indexer.types.NType; import org.python.indexer.types.NUnknownType; -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.InputStreamReader; - /** * Test utilities for {@link IndexerTest}. */ public class TestBase extends TestCase { - static protected final String TEST_SOURCE_DIR; + // Set this to control logging to the console from the Indexer (mostly at FINER level). + static protected final Level LOGGING_LEVEL = Level.OFF; + static protected final String TEST_DATA_DIR; static protected final String TEST_LIB_DIR; static { - TEST_SOURCE_DIR = - System.getProperties().getProperty("python.test.source.dir") - + "/org/python/indexer/"; - TEST_DATA_DIR = TEST_SOURCE_DIR + "data/"; - TEST_LIB_DIR = System.getProperties().getProperty("python.home") + "/Lib/"; + /* + * Locate cardinal directories in a way that insulates us from the vagueries of the + * environment, Ant, IDE and OS. + */ + String home = System.getProperty("python.home", "dist"); + String test = System.getProperty("python.test.source.dir", "tests/java"); + File source = new File(test, "org/python/indexer"); // corrects to \ where needed. + + // Program actually uses strings, with a trailing slash + TEST_DATA_DIR = (new File(source, "data")).getAbsolutePath() + File.separator; + TEST_LIB_DIR = (new File(home, "Lib")).getAbsolutePath() + File.separator; + + // Give the logger used by Indexer an outlet + setUpLogging(); + } + + // Define a handler for the logger to use + static private void setUpLogging() { + // Enable tracing of the operation of the Indexer onto the console + Logger indexerLogger = Logger.getLogger(Indexer.class.getCanonicalName()); + Handler logHandler = new ConsoleHandler(); + logHandler.setFormatter(new SimpleFormatter()); + logHandler.setLevel(Level.FINEST); + indexerLogger.addHandler(logHandler); } protected Indexer idx; @@ -41,6 +65,7 @@ @Override protected void setUp() throws Exception { idx = new Indexer(); + idx.getLogger().setLevel(LOGGING_LEVEL); idx.enableAggressiveAssertions(true); idx.setProjectDir(TEST_DATA_DIR); AstCache.get().clearDiskCache(); @@ -108,13 +133,15 @@ * @throws IllegalArgumentException if the {@code n}th occurrence does not exist */ protected int nthIndexOf(String s, String find, int n) { - if (n <= 0) + if (n <= 0) { throw new IllegalArgumentException(); + } int index = -1; for (int i = 0; i < n; i++) { index = s.indexOf(find, index == -1 ? 0 : index + 1); - if (index == -1) + if (index == -1) { throw new IllegalArgumentException(); + } } return index; } -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Mon Mar 17 00:22:18 2014 From: jython-checkins at python.org (jeff.allen) Date: Mon, 17 Mar 2014 00:22:18 +0100 (CET) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Console_changes_to_make_inc?= =?utf-8?q?omplete_lines_emerge_promptly=2E?= Message-ID: <3fnGrV1Ytgz7LjR@mail.python.org> http://hg.python.org/jython/rev/ceb8045c1adf changeset: 7194:ceb8045c1adf user: Jeff Allen date: Sun Mar 16 18:18:44 2014 +0000 summary: Console changes to make incomplete lines emerge promptly. ... And emerge again when they are in fact a prompt. Issue #2089 refers, but this isn't enough to resolve it. files: src/org/python/util/ConsoleOutputStream.java | 67 +++------ src/org/python/util/JLineConsole.java | 25 ++- src/org/python/util/ReadlineConsole.java | 11 +- 3 files changed, 47 insertions(+), 56 deletions(-) diff --git a/src/org/python/util/ConsoleOutputStream.java b/src/org/python/util/ConsoleOutputStream.java --- a/src/org/python/util/ConsoleOutputStream.java +++ b/src/org/python/util/ConsoleOutputStream.java @@ -10,11 +10,11 @@ /** * This class may be used to wrap and replace System.out so that the console handling * library (JLine or Java Readline) can treat an incomplete line (one without a newline) as a prompt - * string, and so know the true position of the cursor. It achieves this by buffering bytes written - * from upstream until a newline arrives, or a defined capacity is reached. This interferes - * necessarily with upstream expectations about flush(). If client code requests the - * partial line as a prompt, that action empties the buffer. In that case, the client (which is the - * console object) is responsible for making the prompt emerge on the real console. + * string, and so know the true position of the cursor. It achieves this by keeping a copy of bytes + * that pass through from from upstream to the true System.out, until either a newline + * arrives, or a defined capacity (typically the console width) is reached. If client code requests + * the partial line as a prompt, that action also empties the buffer. In that case, the client + * (which is the console object) is responsible for making the prompt emerge on the real console. */ public class ConsoleOutputStream extends FilterOutputStream { @@ -22,8 +22,8 @@ protected ByteBuffer buf; /** - * Create a wrapper on an OutputStream that holds back the last incomplete line - * written to it (as bytes), in case it is needed as a console prompt. + * Create a wrapper on an OutputStream that holds a copy of the last incomplete + * line written to it (as bytes), in case it is needed as a console prompt. * * @param out the stream wrapped (normally System.out) * @param promptCapacity maximum number of bytes to buffer @@ -34,54 +34,37 @@ } /** - * This write method buffers up the bytes until there is a complete line, then this is written - * all at once either to the wrapped stream, or by the client as a console prompt. + * This write method steals a copy of each byte in a buffer while passing it on to the wrapped + * stream. The buffer is reset by each newline, when it overflows, or by the client when it is + * taken as a console prompt in {@link #getPrompt(Charset)}. */ @Override public void write(int b) throws IOException { buf.put((byte)b); + out.write(b); if (b == '\n' || buf.remaining() == 0) { - // Send these bytes downstream - writeBufOut(); + // Empty the prompt buffer + buf.position(0); } } - /** - * This class does not flush on request, only at line endings. - */ @Override public void flush() throws IOException { - // Flush disabled until we are ready to write the buffer out + // Flush passed on to wrapped System.out + out.flush(); + } + + @Override + public void close() throws IOException { + super.close(); // ... with a flush + out.close(); } /** - * Ensure bytes stored in the buffer are written (and flushed), then close downstream. - */ - @Override - public void close() throws IOException { - writeBufOut(); - super.close(); - } - - /** - * Write the stored bytes downstream, with a flush following. We do this when an end-of-line - * byte is written, since the buffer contents cannot then be intended as a prompt, and when we - * run out of space. - * - * @throws IOException - */ - private void writeBufOut() throws IOException { - // Could flip before and compact after, but it's really not necessary. - out.write(buf.array(), 0, buf.position()); - buf.position(0); - out.flush(); - } - - /** - * Return the stored bytes encoded as characters instead of sending them downstream. Whatever is - * in the buffer at the point this method is called will be returned, decoded as a CharSequence - * (from which a String can easily be got) and the buffer left empty. The expectation is that - * the characters will be issued by the caller as a prompt. + * Return the stored bytes encoded as characters. Whatever is in the buffer at the point this + * method is called will be returned, decoded as a CharSequence (from which a String can easily + * be got) and the buffer left empty. The expectation is that the characters will be issued by + * the caller as a prompt. * * @param encoding with which to decode the bytes * @return the decoded prompt diff --git a/src/org/python/util/JLineConsole.java b/src/org/python/util/JLineConsole.java --- a/src/org/python/util/JLineConsole.java +++ b/src/org/python/util/JLineConsole.java @@ -40,8 +40,6 @@ private boolean windows; /** The ctrl-z character String. */ protected static final String CTRL_Z = "\u001a"; - /** The longest we expect a console prompt to be (in encoded bytes) */ - public static final int MAX_PROMPT = 512; /** Stream wrapping System.out in order to capture the last prompt. */ private ConsoleOutputStream outWrapper; @@ -99,14 +97,6 @@ */ Writer out = new PrintWriter(new OutputStreamWriter(System.out, encoding)); - /* - * Everybody else, using sys.stdout or java.lang.System.out gets to write on a special - * PrintStream that keeps the last incomplete line in case it turns out to be a console - * prompt. - */ - outWrapper = new ConsoleOutputStream(System.out, MAX_PROMPT); - System.setOut(new PrintStream(outWrapper, true, encoding)); - // Get the key bindings (built in ones treat TAB Pythonically). InputStream bindings = getBindings(userHomeSpec, getClass().getClassLoader()); @@ -117,6 +107,14 @@ // We find the bell too noisy reader.setBellEnabled(false); + /* + * Everybody else, using sys.stdout or java.lang.System.out gets to write on a special + * PrintStream that keeps the last incomplete line in case it turns out to be a console + * prompt. + */ + outWrapper = new ConsoleOutputStream(System.out, reader.getTermwidth()); + System.setOut(new PrintStream(outWrapper, true, encoding)); + } catch (IOException e) { throw new RuntimeException(e); } @@ -186,7 +184,12 @@ if (startup_hook != null) { startup_hook.__call__(); } - // Get a line and hope to be done. + + // Send the cursor to the start of the line (no prompt, empty buffer). + reader.setDefaultPrompt(null); + reader.redrawLine(); + + // The prompt is whatever was already on the line. String line = reader.readLine(prompt); return line; diff --git a/src/org/python/util/ReadlineConsole.java b/src/org/python/util/ReadlineConsole.java --- a/src/org/python/util/ReadlineConsole.java +++ b/src/org/python/util/ReadlineConsole.java @@ -23,6 +23,8 @@ public static final int MAX_PROMPT = 512; /** Stream wrapping System.out in order to capture the last prompt. */ private ConsoleOutputStream outWrapper; + /** Original System.out as sometimes we have to sneak one out. */ + private PrintStream originalSystemOut; /** * Construct an instance of the console class specifying the character encoding. This encoding @@ -65,7 +67,7 @@ *

* This implementation overrides that by setting System.in to a * FilterInputStream object that wraps the configured console library, and wraps - * System.out in a stream that captures the prompt. + * System.out in a stream that captures the prompt so the Readline library may re-use it. */ @Override public void install() { @@ -81,11 +83,12 @@ } /* - * Wrap System.out in a special PrintStream that keeps the last incomplete line in case it + * Wrap System.out in a special PrintStream that keeps a copy of the last incomplete line in case it * turns out to be a console prompt. */ try { - outWrapper = new ConsoleOutputStream(System.out, MAX_PROMPT); + originalSystemOut = System.out; + outWrapper = new ConsoleOutputStream(originalSystemOut, MAX_PROMPT); System.setOut(new PrintStream(outWrapper, true, encoding)); } catch (IOException e) { throw new RuntimeException(e); @@ -109,6 +112,8 @@ @Override protected CharSequence getLine() throws IOException, EOFException { + // Send cursor to start of line. + originalSystemOut.print('\r'); // The prompt is the current partial output line. CharSequence prompt = outWrapper.getPrompt(encodingCharset).toString(); // Compensate for Readline.readline prompt handling -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Mon Mar 17 00:22:19 2014 From: jython-checkins at python.org (jeff.allen) Date: Mon, 17 Mar 2014 00:22:19 +0100 (CET) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixes_issue_=232089_failure?= =?utf-8?q?_to_flush_sys=2Estdout_when_sys=2Eexit_called=2E?= Message-ID: <3fnGrW30ZGz7Ljk@mail.python.org> http://hg.python.org/jython/rev/aa042b69bdda changeset: 7195:aa042b69bdda user: Jeff Allen date: Sun Mar 16 22:45:44 2014 +0000 summary: Fixes issue #2089 failure to flush sys.stdout when sys.exit called. Ensures that StdoutWrapper.flushLine() always calls flush(). files: NEWS | 5 +++-- src/org/python/core/StdoutWrapper.java | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -19,9 +19,10 @@ - [ 2075 ] Incorrect padding for hex format strings - [ 2082 ] Unexpected (Pdb) prompt during regression tests - [ 2083 ] os.unlink() can delete directories + - [ 2089 ] sys.stdout not flushed after sys.exit New Features - - Command line option -E (ignore environment variables) - - Environment variable PYTHONIOENCODING, and corresponding registry items + - Command line option -E (ignore environment variables) + - Environment variable PYTHONIOENCODING, and corresponding registry items Jython 2.7b1 Bugs Fixed diff --git a/src/org/python/core/StdoutWrapper.java b/src/org/python/core/StdoutWrapper.java --- a/src/org/python/core/StdoutWrapper.java +++ b/src/org/python/core/StdoutWrapper.java @@ -84,8 +84,8 @@ PyFile file = (PyFile) out; if (file.softspace) { file.write("\n"); - file.flush(); } + file.flush(); file.softspace = false; } else { PyObject ss = out.__findattr__("softspace"); -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Wed Mar 19 00:24:17 2014 From: jython-checkins at python.org (jeff.allen) Date: Wed, 19 Mar 2014 00:24:17 +0100 (CET) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixes_test=5Fargparse_failu?= =?utf-8?q?res_in_regrtest_with_-j_option_=28as_under_ant=29=2E?= Message-ID: <3fpVns68wwz7LjV@mail.python.org> http://hg.python.org/jython/rev/28a49481ddb0 changeset: 7196:28a49481ddb0 user: Jeff Allen date: Tue Mar 18 22:42:54 2014 +0000 summary: Fixes test_argparse failures in regrtest with -j option (as under ant). This change adds custom __eq__ and __ne__ comparators to junit_xml.Tee, so that it reports equality with the wrapped sys.stdout, during tests of argparse for "-" argument. files: Lib/test/junit_xml.py | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diff --git a/Lib/test/junit_xml.py b/Lib/test/junit_xml.py --- a/Lib/test/junit_xml.py +++ b/Lib/test/junit_xml.py @@ -168,6 +168,14 @@ StringIO.flush(self) self.stream.flush() + # Equalities to ensure that Tee(stream) == stream + + def __eq__(self, other): + return self is other or self.stream == other + + def __ne__(self, other): + return self is not other and self.stream != other + def write_testsuite_xml(stream, tests, errors, failures, skipped, name, took): """Write the XML header ()""" -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Sat Mar 22 22:41:33 2014 From: jython-checkins at python.org (jeff.allen) Date: Sat, 22 Mar 2014 22:41:33 +0100 (CET) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Drop_support_for_Java_6=2E_?= =?utf-8?q?Minimum_target_is_now_Java_7=2E?= Message-ID: <3frwKT1BPdz7LjV@mail.python.org> http://hg.python.org/jython/rev/68e2098b8f12 changeset: 7197:68e2098b8f12 user: Jeff Allen date: Sat Mar 22 20:49:07 2014 +0000 summary: Drop support for Java 6. Minimum target is now Java 7. Provision for invokedynamic is no longer an exception to the norm. files: build.xml | 16 +------ src/org/python/compiler/InvokedynamicCodeCompiler.java | 24 ---------- 2 files changed, 1 insertions(+), 39 deletions(-) diff --git a/build.xml b/build.xml --- a/build.xml +++ b/build.xml @@ -93,18 +93,7 @@ - - - - - + @@ -263,7 +252,6 @@ templates.lazy = '${templates.lazy}' python.lib = '${python.lib}' build.compiler = '${build.compiler}' - jython.use.invokedynamic = '${jython.use.invokedynamic}' jdk.target.version = '${jdk.target.version}' jdk.source.version = '${jdk.source.version}' deprecation = '${deprecation}' @@ -492,8 +480,6 @@ - - diff --git a/src/org/python/compiler/InvokedynamicCodeCompiler.java b/src/org/python/compiler/InvokedynamicCodeCompiler.java deleted file mode 100644 --- a/src/org/python/compiler/InvokedynamicCodeCompiler.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.python.compiler; - -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; -import java.lang.invoke.MutableCallSite; - -/** - * - * @author shashank - */ -public class InvokedynamicCodeCompiler extends CodeCompiler { - - public InvokedynamicCodeCompiler(Module module, boolean print_results) { - super(module, print_results); - } - - // just use indy functions to make sure the build process is proper. - public void foo(){ - Class mhClazz = MethodHandles.class; - MethodType target = null; - new MutableCallSite(target); - } - -} -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Mon Mar 24 04:13:31 2014 From: jython-checkins at python.org (frank.wierzbicki) Date: Mon, 24 Mar 2014 04:13:31 +0100 (CET) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Prepare_for_release=2E?= Message-ID: <3fsgf30CDjz7Ljg@mail.python.org> http://hg.python.org/jython/rev/92b054125b28 changeset: 7198:92b054125b28 user: Frank Wierzbicki date: Mon Mar 24 03:13:25 2014 +0000 summary: Prepare for release. files: README.txt | 19 +++++++++---------- build.xml | 10 +++++----- src/org/python/core/imp.java | 2 +- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/README.txt b/README.txt --- a/README.txt +++ b/README.txt @@ -1,23 +1,22 @@ -Welcome to Jython 2.7b1 +Welcome to Jython 2.7b2 ======================= -This is the first beta release of the 2.7 version of Jython. Thanks to +This is the second beta release of the 2.7 version of Jython. Thanks to Adconion Media Group (http://www.adconion.com/) for sponsoring this release. Thanks to all who contribute to Jython. -Jython 2.7b1 brings us up to language level compatibility with the 2.7 version +Jython 2.7b2 brings us up to language level compatibility with the 2.7 version of CPython. We have focused largely on CPython compatibility, and so this release of Jython can run more pure Python apps then any previous release. -Please see the NEWS file for detailed release notes. Some notable new features -in this release are: a bytearray implementation, a buffer api, memoryview, a -bz2 module. +Please see the NEWS file for detailed release notes. This is primarily a bugfix +release, with numerous improvements, including much improvement on Windows +support. -As the first beta release, this marks a change in the focus of development -from adding features to bug fixing and on getting more of the test suite -running properly. +As a beta release we are concentrating on bug fixing and stabilizion for a +production release. Please see the NEWS file for detailed release notes. -The release was compiled on Ubuntu with JDK 7 and requires JDK 6 to run. +The release was compiled on OSXX with JDK 7 and requires JDK 7 to run. Please try this out and report any bugs at http://bugs.jython.org. diff --git a/build.xml b/build.xml --- a/build.xml +++ b/build.xml @@ -52,9 +52,9 @@ oracle.jar=C:/workspace/HEAD/for_development/bisdevsrv28/jboss/server/infra/lib/ojdbc14.jar #informix.jar=${basedir}/../externals/external-jars/ifxjdbc.jar -# - option for javac (build.compiler=modern is a global option to use standard jdk 1.5/1.6/1.7) +# - option for javac (build.compiler=modern is a global option to use standard jdk 1.7/1.8) #build.compiler=modern -#jdk.target.version=1.6 +#jdk.target.version=1.7 #debug=false #deprecation=off @@ -84,15 +84,15 @@ - - + + - + diff --git a/src/org/python/core/imp.java b/src/org/python/core/imp.java --- a/src/org/python/core/imp.java +++ b/src/org/python/core/imp.java @@ -26,7 +26,7 @@ private static final String UNKNOWN_SOURCEFILE = ""; - private static final int APIVersion = 33; + private static final int APIVersion = 34; public static final int NO_MTIME = -1; -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Mon Mar 24 04:48:02 2014 From: jython-checkins at python.org (frank.wierzbicki) Date: Mon, 24 Mar 2014 04:48:02 +0100 (CET) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Added_tag_v2=2E7b2_for_chan?= =?utf-8?q?geset_92b054125b28?= Message-ID: <3fshPt6q7Xz7LjP@mail.python.org> http://hg.python.org/jython/rev/15b97f9efd14 changeset: 7199:15b97f9efd14 user: Frank Wierzbicki date: Mon Mar 24 03:47:49 2014 +0000 summary: Added tag v2.7b2 for changeset 92b054125b28 files: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -65,3 +65,4 @@ 3d2dbae23c5292b7f31ac4c92fa6d1afd8bd8eb2 v2.5.3 f5b12dc4ff970c9594c99fc895772958c4a669a0 v2.5.4rc1 8a556c4cc2810912e4ef277d53668ffc9d9f5772 v2.7b1 +92b054125b28a17b2e4b64ea6ffa416555a52725 v2.7b2 -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Mon Mar 24 15:33:53 2014 From: jython-checkins at python.org (frank.wierzbicki) Date: Mon, 24 Mar 2014 15:33:53 +0100 (CET) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_spelling_error=2E?= Message-ID: <3fsyl50r2tz7LjS@mail.python.org> http://hg.python.org/jython/rev/3fe8544d3d59 changeset: 7200:3fe8544d3d59 user: Frank Wierzbicki date: Mon Mar 24 14:33:44 2014 +0000 summary: Fix spelling error. files: README.txt | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/README.txt b/README.txt --- a/README.txt +++ b/README.txt @@ -17,6 +17,6 @@ Please see the NEWS file for detailed release notes. -The release was compiled on OSXX with JDK 7 and requires JDK 7 to run. +The release was compiled on OSX with JDK 7 and requires JDK 7 to run. Please try this out and report any bugs at http://bugs.jython.org. -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Mon Mar 24 15:37:02 2014 From: jython-checkins at python.org (frank.wierzbicki) Date: Mon, 24 Mar 2014 15:37:02 +0100 (CET) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Added_tag_v2=2E7b2_for_chan?= =?utf-8?q?geset_3fe8544d3d59?= Message-ID: <3fsypk02Y4z7Ll0@mail.python.org> http://hg.python.org/jython/rev/dbbe922d89e9 changeset: 7201:dbbe922d89e9 user: Frank Wierzbicki date: Mon Mar 24 14:36:50 2014 +0000 summary: Added tag v2.7b2 for changeset 3fe8544d3d59 files: .hgtags | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -66,3 +66,5 @@ f5b12dc4ff970c9594c99fc895772958c4a669a0 v2.5.4rc1 8a556c4cc2810912e4ef277d53668ffc9d9f5772 v2.7b1 92b054125b28a17b2e4b64ea6ffa416555a52725 v2.7b2 +92b054125b28a17b2e4b64ea6ffa416555a52725 v2.7b2 +3fe8544d3d592372da99103fc3720a36b949e57b v2.7b2 -- Repository URL: http://hg.python.org/jython