[Python-checkins] bpo-35683: Improve Azure Pipelines steps (GH-11493)
Steve Dower
webhook-mailer at python.org
Tue Jan 22 15:31:35 EST 2019
https://github.com/python/cpython/commit/128efcade63480b5860a6d045a41ba4abf5eea2f
commit: 128efcade63480b5860a6d045a41ba4abf5eea2f
branch: 3.7
author: Steve Dower <steve.dower at microsoft.com>
committer: GitHub <noreply at github.com>
date: 2019-01-22T12:31:30-08:00
summary:
bpo-35683: Improve Azure Pipelines steps (GH-11493)
files:
A .azure-pipelines/posix-deps-apt.sh
A Misc/NEWS.d/next/Build/2019-01-10-11-37-18.bpo-35683.pf5Oos.rst
D .azure-pipelines/docker-steps.yml
D .azure-pipelines/posix-deps.sh
D .azure-pipelines/windows-appx-test.yml
M .azure-pipelines/ci.yml
M .azure-pipelines/posix-steps.yml
M .azure-pipelines/pr.yml
M .azure-pipelines/windows-layout-steps.yml
M .azure-pipelines/windows-steps.yml
M Lib/idlelib/idle_test/test_help_about.py
M Lib/test/libregrtest/main.py
M Lib/test/test_symbol.py
M PC/layout/main.py
diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
index 78075bcfc147..15a83dd0370e 100644
--- a/.azure-pipelines/ci.yml
+++ b/.azure-pipelines/ci.yml
@@ -2,6 +2,11 @@ variables:
manylinux: false
coverage: false
+resources:
+ containers:
+ - container: manylinux1
+ image: pyca/cryptography-manylinux1:x86_64
+
jobs:
- job: Prebuild
displayName: Pre-build checks
@@ -54,10 +59,12 @@ jobs:
variables:
testRunTitle: '$(build.sourceBranchName)-linux'
testRunPlatform: linux
- openssl_version: 1.1.0g
+ openssl_version: 1.1.0j
steps:
- template: ./posix-steps.yml
+ parameters:
+ dependencies: apt
- job: ManyLinux1_CI_Tests
@@ -75,13 +82,20 @@ jobs:
pool:
vmImage: ubuntu-16.04
+ container: manylinux1
+
variables:
testRunTitle: '$(build.sourceBranchName)-manylinux1'
testRunPlatform: manylinux1
- imageName: 'dockcross/manylinux-x64'
+ openssl_version: ''
steps:
- - template: ./docker-steps.yml
+ - template: ./posix-steps.yml
+ parameters:
+ dependencies: yum
+ sudo_dependencies: ''
+ xvfb: false
+ patchcheck: false
- job: Ubuntu_Coverage_CI_Tests
@@ -102,11 +116,12 @@ jobs:
variables:
testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
testRunPlatform: linux-coverage
- openssl_version: 1.1.0g
+ openssl_version: 1.1.0j
steps:
- template: ./posix-steps.yml
parameters:
+ dependencies: apt
coverage: true
@@ -144,3 +159,4 @@ jobs:
- template: ./windows-layout-steps.yml
parameters:
kind: appx
+ fulltest: true
diff --git a/.azure-pipelines/docker-steps.yml b/.azure-pipelines/docker-steps.yml
deleted file mode 100644
index ba4dfd72dd8b..000000000000
--- a/.azure-pipelines/docker-steps.yml
+++ /dev/null
@@ -1,76 +0,0 @@
-steps:
-- checkout: self
- clean: true
- fetchDepth: 5
-
-- ${{ if ne(parameters.targetBranch, '') }}:
- - script: |
- git fetch -q origin ${{ parameters.targetbranch }}
- if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)'
- then
- echo "Only docs were updated, stopping build process."
- echo "##vso[task.setvariable variable=DocOnly]true"
- exit
- fi
- displayName: Detect doc-only changes
-
-- task: docker at 0
- displayName: 'Configure CPython (debug)'
- inputs:
- action: 'Run an image'
- imageName: $(imageName)
- volumes: |
- $(build.sourcesDirectory):/src
- $(build.binariesDirectory):/build
- workDir: '/src'
- containerCommand: './configure --with-pydebug'
- detached: false
- condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
-
-- task: docker at 0
- displayName: 'Build CPython'
- inputs:
- action: 'Run an image'
- imageName: $(imageName)
- volumes: |
- $(build.sourcesDirectory):/src
- $(build.binariesDirectory):/build
- workDir: '/src'
- containerCommand: 'make -s -j4'
- detached: false
- condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
-
-- task: docker at 0
- displayName: 'Display build info'
- inputs:
- action: 'Run an image'
- imageName: $(imageName)
- volumes: |
- $(build.sourcesDirectory):/src
- $(build.binariesDirectory):/build
- workDir: '/src'
- containerCommand: 'make pythoninfo'
- detached: false
- condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
-
-- task: docker at 0
- displayName: 'Tests'
- inputs:
- action: 'Run an image'
- imageName: $(imageName)
- volumes: |
- $(build.sourcesDirectory):/src
- $(build.binariesDirectory):/build
- workDir: '/src'
- containerCommand: 'make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=/build/test-results.xml"'
- detached: false
- condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
-
-- task: PublishTestResults at 2
- displayName: 'Publish Test Results'
- inputs:
- testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
- mergeTestResults: true
- testRunTitle: $(testRunTitle)
- platform: $(testRunPlatform)
- condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true'))
diff --git a/.azure-pipelines/posix-deps.sh b/.azure-pipelines/posix-deps-apt.sh
similarity index 92%
rename from .azure-pipelines/posix-deps.sh
rename to .azure-pipelines/posix-deps-apt.sh
index a57210756601..4f489903ab56 100755
--- a/.azure-pipelines/posix-deps.sh
+++ b/.azure-pipelines/posix-deps-apt.sh
@@ -1,6 +1,6 @@
-sudo apt-get update
+apt-get update
-sudo apt-get -yq install \
+apt-get -yq install \
build-essential \
zlib1g-dev \
libbz2-dev \
diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml
index 9fec9be8014f..a4160e5a1bf5 100644
--- a/.azure-pipelines/posix-steps.yml
+++ b/.azure-pipelines/posix-steps.yml
@@ -1,12 +1,16 @@
parameters:
coverage: false
+ sudo_dependencies: sudo
+ dependencies: apt
+ patchcheck: true
+ xvfb: true
steps:
- checkout: self
clean: true
fetchDepth: 5
-- script: ./.azure-pipelines/posix-deps.sh $(openssl_version)
+- script: ${{ parameters.sudo_dependencies }} ./.azure-pipelines/posix-deps-${{ parameters.dependencies }}.sh $(openssl_version)
displayName: 'Install dependencies'
- script: ./configure --with-pydebug
@@ -23,7 +27,7 @@ steps:
displayName: 'Display build info'
- script: |
- xvfb-run ./venv/bin/python -m coverage run --pylib -m test \
+ $COMMAND -m coverage run --pylib -m test \
--fail-env-changed \
-uall,-cpu \
--junit-xml=$(build.binariesDirectory)/test-results.xml" \
@@ -32,6 +36,11 @@ steps:
-x test_multiprocessing_spawn \
-x test_concurrent_futures
displayName: 'Tests with coverage'
+ env:
+ ${{ if eq(parameters.xvfb, 'true') }}:
+ COMMAND: xvfb-run ./venv/bin/python
+ ${{ if ne(parameters.xvfb, 'true') }}:
+ COMMAND: ./venv/bin/python
- script: ./venv/bin/python -m coverage xml
displayName: 'Generate coverage.xml'
@@ -44,13 +53,18 @@ steps:
- script: make pythoninfo
displayName: 'Display build info'
- - script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
+ - script: $COMMAND buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
displayName: 'Tests'
-
-
-- script: ./python Tools/scripts/patchcheck.py --travis true
- displayName: 'Run patchcheck.py'
- condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
+ env:
+ ${{ if eq(parameters.xvfb, 'true') }}:
+ COMMAND: xvfb-run make
+ ${{ if ne(parameters.xvfb, 'true') }}:
+ COMMAND: make
+
+- ${{ if eq(parameters.patchcheck, 'true') }}:
+ - script: ./python Tools/scripts/patchcheck.py --travis true
+ displayName: 'Run patchcheck.py'
+ condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
- task: PublishTestResults at 2
diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml
index 2d7fba9cf328..0bd7921bcbef 100644
--- a/.azure-pipelines/pr.yml
+++ b/.azure-pipelines/pr.yml
@@ -1,3 +1,12 @@
+variables:
+ manylinux: false
+ coverage: false
+
+resources:
+ containers:
+ - container: manylinux1
+ image: pyca/cryptography-manylinux1:x86_64
+
jobs:
- job: Prebuild
displayName: Pre-build checks
@@ -50,12 +59,70 @@ jobs:
variables:
testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
testRunPlatform: linux
- openssl_version: 1.1.0g
+ openssl_version: 1.1.0j
steps:
- template: ./posix-steps.yml
parameters:
- targetBranch: $(System.PullRequest.TargetBranch)
+ dependencies: apt
+
+
+- job: ManyLinux1_PR_Tests
+ displayName: ManyLinux1 PR Tests
+ dependsOn: Prebuild
+ condition: |
+ and(
+ and(
+ succeeded(),
+ eq(variables['manylinux'], 'true')
+ ),
+ eq(dependencies.Prebuild.outputs['tests.run'], 'true')
+ )
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ container: manylinux1
+
+ variables:
+ testRunTitle: '$(system.pullRequest.TargetBranch)-manylinux1'
+ testRunPlatform: manylinux1
+ openssl_version: ''
+
+ steps:
+ - template: ./posix-steps.yml
+ parameters:
+ dependencies: yum
+ sudo_dependencies: ''
+ xvfb: false
+ patchcheck: false
+
+
+- job: Ubuntu_Coverage_PR_Tests
+ displayName: Ubuntu PR Tests (coverage)
+ dependsOn: Prebuild
+ condition: |
+ and(
+ and(
+ succeeded(),
+ eq(variables['coverage'], 'true')
+ ),
+ eq(dependencies.Prebuild.outputs['tests.run'], 'true')
+ )
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ variables:
+ testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
+ testRunPlatform: linux-coverage
+ openssl_version: 1.1.0j
+
+ steps:
+ - template: ./posix-steps.yml
+ parameters:
+ dependencies: apt
+ coverage: true
- job: Windows_PR_Tests
diff --git a/.azure-pipelines/windows-appx-test.yml b/.azure-pipelines/windows-appx-test.yml
deleted file mode 100644
index cad752b0a1e7..000000000000
--- a/.azure-pipelines/windows-appx-test.yml
+++ /dev/null
@@ -1,67 +0,0 @@
-jobs:
-- job: Prebuild
- displayName: Pre-build checks
-
- pool:
- vmImage: ubuntu-16.04
-
- steps:
- - template: ./prebuild-checks.yml
-
-
-- job: Windows_Appx_Tests
- displayName: Windows Appx Tests
- dependsOn: Prebuild
- condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
-
- pool:
- vmImage: vs2017-win2016
-
- strategy:
- matrix:
- win64:
- arch: amd64
- buildOpt: '-p x64'
- testRunTitle: '$(Build.SourceBranchName)-win64-appx'
- testRunPlatform: win64
- maxParallel: 2
-
- steps:
- - checkout: self
- clean: true
- fetchDepth: 5
-
- - powershell: |
- # Relocate build outputs outside of source directory to make cleaning faster
- Write-Host '##vso[task.setvariable variable=Py_IntDir]$(Build.BinariesDirectory)\obj'
- # UNDONE: Do not build to a different directory because of broken tests
- Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.SourcesDirectory)\PCbuild'
- Write-Host '##vso[task.setvariable variable=EXTERNALS_DIR]$(Build.BinariesDirectory)\externals'
- displayName: Update build locations
-
- - script: PCbuild\build.bat -e $(buildOpt)
- displayName: 'Build CPython'
- env:
- IncludeUwp: true
-
- - script: python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Py_IntDir)\layout-tmp-$(arch)" --copy "$(Py_IntDir)\layout-$(arch)" --precompile --preset-appx --include-tests
- displayName: 'Create APPX layout'
-
- - script: .\python.exe -m test.pythoninfo
- workingDirectory: $(Py_IntDir)\layout-$(arch)
- displayName: 'Display build info'
-
- - script: .\python.exe -m test -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml" --tempdir "$(Py_IntDir)\tmp-$(arch)"
- workingDirectory: $(Py_IntDir)\layout-$(arch)
- displayName: 'Tests'
- env:
- PREFIX: $(Py_IntDir)\layout-$(arch)
-
- - task: PublishTestResults at 2
- displayName: 'Publish Test Results'
- inputs:
- testResultsFiles: '$(Build.BinariesDirectory)\test-results.xml'
- mergeTestResults: true
- testRunTitle: $(testRunTitle)
- platform: $(testRunPlatform)
- condition: succeededOrFailed()
diff --git a/.azure-pipelines/windows-layout-steps.yml b/.azure-pipelines/windows-layout-steps.yml
index 62e5259375f5..e15729fac344 100644
--- a/.azure-pipelines/windows-layout-steps.yml
+++ b/.azure-pipelines/windows-layout-steps.yml
@@ -1,11 +1,28 @@
parameters:
kind: nuget
extraOpts: --precompile
+ fulltest: false
steps:
-- script: .\python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Py_IntDir)\layout-tmp-${{ parameters['kind'] }}-$(arch)" --copy "$(Py_OutDir)\layout-${{ parameters['kind'] }}-$(arch)" ${{ parameters['extraOpts'] }} --preset-${{ parameters['kind'] }} --include-tests
- displayName: Create ${{ parameters['kind'] }} layout
+- script: .\python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Build.BinariesDirectory)\layout-tmp-${{ parameters.kind }}-$(arch)" --copy "$(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch)" ${{ parameters.extraOpts }} --preset-${{ parameters.kind }} --include-tests
+ displayName: Create ${{ parameters.kind }} layout
- script: .\python.exe -m test.pythoninfo
- workingDirectory: $(Py_OutDir)\layout-${{ parameters['kind'] }}-$(arch)
- displayName: Show layout info (${{ parameters['kind'] }})
+ workingDirectory: $(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch)
+ displayName: Show layout info (${{ parameters.kind }})
+
+- ${{ if eq(parameters.fulltest, 'true') }}:
+ - script: .\python.exe -m test -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results-${{ parameters.kind }}.xml" --tempdir "$(Build.BinariesDirectory)\tmp-${{ parameters.kind }}-$(arch)"
+ workingDirectory: $(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch)
+ displayName: ${{ parameters.kind }} Tests
+ env:
+ PREFIX: $(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch)
+
+ - task: PublishTestResults at 2
+ displayName: Publish ${{ parameters.kind }} Test Results
+ inputs:
+ testResultsFiles: $(Build.BinariesDirectory)\test-results-${{ parameters.kind }}.xml
+ mergeTestResults: true
+ testRunTitle: ${{ parameters.kind }}-$(testRunTitle)
+ platform: $(testRunPlatform)
+ condition: succeededOrFailed()
diff --git a/.azure-pipelines/windows-steps.yml b/.azure-pipelines/windows-steps.yml
index 3651ae03bc1d..794a23a5d77e 100644
--- a/.azure-pipelines/windows-steps.yml
+++ b/.azure-pipelines/windows-steps.yml
@@ -1,6 +1,6 @@
steps:
- checkout: self
- clean: true
+ clean: false
fetchDepth: 5
- powershell: |
@@ -8,6 +8,7 @@ steps:
Write-Host '##vso[task.setvariable variable=Py_IntDir]$(Build.BinariesDirectory)\obj'
# UNDONE: Do not build to a different directory because of broken tests
Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.SourcesDirectory)\PCbuild'
+ #Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.BinariesDirectory)\bin'
Write-Host '##vso[task.setvariable variable=EXTERNALS_DIR]$(Build.BinariesDirectory)\externals'
displayName: Update build locations
diff --git a/Lib/idlelib/idle_test/test_help_about.py b/Lib/idlelib/idle_test/test_help_about.py
index 5839b5d045d8..7c148d23a135 100644
--- a/Lib/idlelib/idle_test/test_help_about.py
+++ b/Lib/idlelib/idle_test/test_help_about.py
@@ -61,6 +61,8 @@ def test_printer_buttons(self):
button.invoke()
get = dialog._current_textview.viewframe.textframe.text.get
lines = printer._Printer__lines
+ if len(lines) < 2:
+ self.fail(name + ' full text was not found')
self.assertEqual(lines[0], get('1.0', '1.end'))
self.assertEqual(lines[1], get('2.0', '2.end'))
dialog._current_textview.destroy()
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 8d44caf2999a..32ac44029bc3 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -1,5 +1,6 @@
import datetime
import faulthandler
+import json
import locale
import os
import platform
@@ -565,6 +566,9 @@ def main(self, tests=None, **kwargs):
if self.ns.tempdir:
TEMPDIR = self.ns.tempdir
+ elif self.ns.worker_args:
+ ns_dict, _ = json.loads(self.ns.worker_args)
+ TEMPDIR = ns_dict.get("tempdir") or TEMPDIR
os.makedirs(TEMPDIR, exist_ok=True)
diff --git a/Lib/test/test_symbol.py b/Lib/test/test_symbol.py
index c1306f54327f..32564f3df214 100644
--- a/Lib/test/test_symbol.py
+++ b/Lib/test/test_symbol.py
@@ -2,6 +2,7 @@
from test import support
import os
import sys
+import sysconfig
import subprocess
@@ -35,8 +36,8 @@ def compare_files(self, file1, file2):
lines2 = fp.readlines()
self.assertEqual(lines1, lines2)
- @unittest.skipIf(not os.path.exists(GRAMMAR_FILE),
- 'test only works from source build directory')
+ @unittest.skipUnless(sysconfig.is_python_build(),
+ 'test only works from source build directory')
def test_real_grammar_and_symbol_file(self):
output = support.TESTFN
self.addCleanup(support.unlink, output)
diff --git a/Misc/NEWS.d/next/Build/2019-01-10-11-37-18.bpo-35683.pf5Oos.rst b/Misc/NEWS.d/next/Build/2019-01-10-11-37-18.bpo-35683.pf5Oos.rst
new file mode 100644
index 000000000000..f39610169370
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-01-10-11-37-18.bpo-35683.pf5Oos.rst
@@ -0,0 +1 @@
+Improved Azure Pipelines build steps and now verifying layouts correctly
diff --git a/PC/layout/main.py b/PC/layout/main.py
index d372fe50df32..910085c01bb2 100644
--- a/PC/layout/main.py
+++ b/PC/layout/main.py
@@ -156,6 +156,8 @@ def in_build(f, dest="", new_name=None):
for dest, src in rglob(ns.build, "vcruntime*.dll"):
yield dest, src
+ yield "LICENSE.txt", ns.source / "LICENSE"
+
for dest, src in rglob(ns.build, ("*.pyd", "*.dll")):
if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS:
continue
More information about the Python-checkins
mailing list