[Python-checkins] cpython: Issue #13146: Writing a pyc file is now atomic under POSIX.

[antoine.pitrou]

http://hg.python.org/cpython/rev/c16063765d3a
changeset:   72960:c16063765d3a
user:        Antoine Pitrou <solipsis at pitrou.net>
date:        Mon Oct 17 19:28:44 2011 +0200
summary:
  Issue #13146: Writing a pyc file is now atomic under POSIX.

files:
  Lib/importlib/_bootstrap.py |  26 ++++++++++++-
  Misc/NEWS                   |   2 +
  Python/import.c             |  47 +++++++++++++++++++-----
  3 files changed, 62 insertions(+), 13 deletions(-)


diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -80,6 +80,27 @@
             return _path_join(_os.getcwd(), path)
 
 
+def _write_atomic(path, data):
+    """Best-effort function to write data to a path atomically."""
+    if not sys.platform.startswith('win'):
+        # On POSIX-like platforms, renaming is atomic
+        path_tmp = path + '.tmp'
+        try:
+            fd = _os.open(path_tmp, _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY)
+            with _io.FileIO(fd, 'wb') as file:
+                file.write(data)
+            _os.rename(path_tmp, path)
+        except OSError:
+            try:
+                _os.unlink(path_tmp)
+            except OSError:
+                pass
+            raise
+    else:
+        with _io.FileIO(path, 'wb') as file:
+            file.write(data)
+
+
 def _wrap(new, old):
     """Simple substitute for functools.wraps."""
     for replace in ['__module__', '__name__', '__doc__']:
@@ -494,9 +515,8 @@
                 else:
                     raise
         try:
-            with _io.FileIO(path, 'wb') as file:
-                file.write(data)
-        except IOError as exc:
+            _write_atomic(path, data)
+        except OSError as exc:
             # Don't worry if you can't write bytecode.
             if exc.errno == errno.EACCES:
                 return
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,8 @@
 Core and Builtins
 -----------------
 
+- Issue #13146: Writing a pyc file is now atomic under POSIX.
+
 - Issue #7833: Extension modules built using distutils on Windows will no
   longer include a "manifest" to prevent them failing at import time in some
   embedded situations.
diff --git a/Python/import.c b/Python/import.c
--- a/Python/import.c
+++ b/Python/import.c
@@ -1284,7 +1284,8 @@
 #ifdef MS_WINDOWS
     int fd;
 #else
-    PyObject *cpathbytes;
+    PyObject *cpathbytes, *cpathbytes_tmp;
+    Py_ssize_t cpathbytes_len;
 #endif
     PyObject *dirname;
     Py_UCS4 *dirsep;
@@ -1345,13 +1346,25 @@
     else
         fp = NULL;
 #else
+    /* Under POSIX, we first write to a tmp file and then take advantage
+       of atomic renaming. */
     cpathbytes = PyUnicode_EncodeFSDefault(cpathname);
     if (cpathbytes == NULL) {
         PyErr_Clear();
         return;
     }
-
-    fp = open_exclusive(PyBytes_AS_STRING(cpathbytes), mode);
+    cpathbytes_len = PyBytes_GET_SIZE(cpathbytes);
+    cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 4);
+    if (cpathbytes_tmp == NULL) {
+        Py_DECREF(cpathbytes);
+        PyErr_Clear();
+        return;
+    }
+    memcpy(PyBytes_AS_STRING(cpathbytes_tmp), PyBytes_AS_STRING(cpathbytes),
+           cpathbytes_len);
+    memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, ".tmp", 4);
+
+    fp = open_exclusive(PyBytes_AS_STRING(cpathbytes_tmp), mode);
 #endif
     if (fp == NULL) {
         if (Py_VerboseFlag)
@@ -1359,6 +1372,7 @@
                 "# can't create %R\n", cpathname);
 #ifndef MS_WINDOWS
         Py_DECREF(cpathbytes);
+        Py_DECREF(cpathbytes_tmp);
 #endif
         return;
     }
@@ -1366,6 +1380,11 @@
     /* First write a 0 for mtime */
     PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION);
     PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION);
+    fflush(fp);
+    /* Now write the true mtime */
+    fseek(fp, 4L, 0);
+    assert(mtime < LONG_MAX);
+    PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION);
     if (fflush(fp) != 0 || ferror(fp)) {
         if (Py_VerboseFlag)
             PySys_FormatStderr("# can't write %R\n", cpathname);
@@ -1374,20 +1393,28 @@
 #ifdef MS_WINDOWS
         (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname));
 #else
-        (void) unlink(PyBytes_AS_STRING(cpathbytes));
+        (void) unlink(PyBytes_AS_STRING(cpathbytes_tmp));
         Py_DECREF(cpathbytes);
+        Py_DECREF(cpathbytes_tmp);
 #endif
         return;
     }
+    fclose(fp);
+    /* Under POSIX, do an atomic rename */
 #ifndef MS_WINDOWS
+    if (rename(PyBytes_AS_STRING(cpathbytes_tmp),
+               PyBytes_AS_STRING(cpathbytes))) {
+        if (Py_VerboseFlag)
+            PySys_FormatStderr("# can't write %R\n", cpathname);
+        /* Don't keep tmp file */
+        unlink(PyBytes_AS_STRING(cpathbytes_tmp));
+        Py_DECREF(cpathbytes);
+        Py_DECREF(cpathbytes_tmp);
+        return;
+    }
     Py_DECREF(cpathbytes);
+    Py_DECREF(cpathbytes_tmp);
 #endif
-    /* Now write the true mtime */
-    fseek(fp, 4L, 0);
-    assert(mtime < LONG_MAX);
-    PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION);
-    fflush(fp);
-    fclose(fp);
     if (Py_VerboseFlag)
         PySys_FormatStderr("# wrote %R\n", cpathname);
 }

-- 
Repository URL: http://hg.python.org/cpython