[Python-checkins] bpo-35596: Use unchecked PYCs for the embeddable distro to avoid zipimport restrictions (GH-11465)

[Steve Dower]

https://github.com/python/cpython/commit/872bd2b57ce8e4ea7a54acb3934222c0e4e7276b
commit: 872bd2b57ce8e4ea7a54acb3934222c0e4e7276b
branch: master
author: Steve Dower <steve.dower at microsoft.com>
committer: GitHub <noreply at github.com>
date: 2019-01-08T02:38:01-08:00
summary:

bpo-35596: Use unchecked PYCs for the embeddable distro to avoid zipimport restrictions (GH-11465)

Also adds extra steps to the CI build for Windows on Azure Pipelines to validate that the various layouts at least execute.

files:
A .azure-pipelines/windows-layout-steps.yml
A Misc/NEWS.d/next/Windows/2019-01-08-13-56-01.bpo-35596.oFvhcm.rst
M .azure-pipelines/ci.yml
M PC/layout/main.py

diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
index 49a7bb6232aa..78075bcfc147 100644
--- a/.azure-pipelines/ci.yml
+++ b/.azure-pipelines/ci.yml
@@ -134,3 +134,13 @@ jobs:
 
   steps:
   - template: ./windows-steps.yml
+
+  - template: ./windows-layout-steps.yml
+    parameters:
+      kind: nuget
+  - template: ./windows-layout-steps.yml
+    parameters:
+      kind: embed
+  - template: ./windows-layout-steps.yml
+    parameters:
+      kind: appx
diff --git a/.azure-pipelines/windows-layout-steps.yml b/.azure-pipelines/windows-layout-steps.yml
new file mode 100644
index 000000000000..62e5259375f5
--- /dev/null
+++ b/.azure-pipelines/windows-layout-steps.yml
@@ -0,0 +1,11 @@
+parameters:
+  kind: nuget
+  extraOpts: --precompile
+
+steps:
+- script: .\python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Py_IntDir)\layout-tmp-${{ parameters['kind'] }}-$(arch)" --copy "$(Py_OutDir)\layout-${{ parameters['kind'] }}-$(arch)" ${{ parameters['extraOpts'] }} --preset-${{ parameters['kind'] }} --include-tests
+  displayName: Create ${{ parameters['kind'] }} layout
+
+- script: .\python.exe -m test.pythoninfo
+  workingDirectory: $(Py_OutDir)\layout-${{ parameters['kind'] }}-$(arch)
+  displayName: Show layout info (${{ parameters['kind'] }})
diff --git a/Misc/NEWS.d/next/Windows/2019-01-08-13-56-01.bpo-35596.oFvhcm.rst b/Misc/NEWS.d/next/Windows/2019-01-08-13-56-01.bpo-35596.oFvhcm.rst
new file mode 100644
index 000000000000..db4d8fa420d6
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-01-08-13-56-01.bpo-35596.oFvhcm.rst
@@ -0,0 +1,2 @@
+Use unchecked PYCs for the embeddable distro to avoid zipimport
+restrictions.
diff --git a/PC/layout/main.py b/PC/layout/main.py
index 7eaf201d532e..d372fe50df32 100644
--- a/PC/layout/main.py
+++ b/PC/layout/main.py
@@ -240,12 +240,18 @@ def _c(d):
             yield "DLLs/{}".format(ns.include_cat.name), ns.include_cat
 
 
-def _compile_one_py(src, dest, name, optimize):
+def _compile_one_py(src, dest, name, optimize, checked=True):
     import py_compile
 
     if dest is not None:
         dest = str(dest)
 
+    mode = (
+        py_compile.PycInvalidationMode.CHECKED_HASH
+        if checked
+        else py_compile.PycInvalidationMode.UNCHECKED_HASH
+    )
+
     try:
         return Path(
             py_compile.compile(
@@ -254,7 +260,7 @@ def _compile_one_py(src, dest, name, optimize):
                 str(name),
                 doraise=True,
                 optimize=optimize,
-                invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
+                invalidation_mode=mode,
             )
         )
     except py_compile.PyCompileError:
@@ -262,16 +268,16 @@ def _compile_one_py(src, dest, name, optimize):
         return None
 
 
-def _py_temp_compile(src, ns, dest_dir=None):
+def _py_temp_compile(src, ns, dest_dir=None, checked=True):
     if not ns.precompile or src not in PY_FILES or src.parent in DATA_DIRS:
         return None
 
     dest = (dest_dir or ns.temp) / (src.stem + ".py")
-    return _compile_one_py(src, dest.with_suffix(".pyc"), dest, optimize=2)
+    return _compile_one_py(src, dest.with_suffix(".pyc"), dest, optimize=2, checked=checked)
 
 
-def _write_to_zip(zf, dest, src, ns):
-    pyc = _py_temp_compile(src, ns)
+def _write_to_zip(zf, dest, src, ns, checked=True):
+    pyc = _py_temp_compile(src, ns, checked=checked)
     if pyc:
         try:
             zf.write(str(pyc), dest.with_suffix(".pyc"))
@@ -321,7 +327,7 @@ def generate_source_files(ns):
         ns.temp.mkdir(parents=True, exist_ok=True)
         with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
             for dest, src in get_lib_layout(ns):
-                _write_to_zip(zf, dest, src, ns)
+                _write_to_zip(zf, dest, src, ns, checked=False)
 
     if ns.include_underpth:
         log_info("Generating {} in {}", PYTHON_PTH_NAME, ns.temp)