[pypy-commit] pypy detect_cpu_count: (catalin_m) Renamed support.detect_number_of_processors to support.detect_number_of_usable_processors to (somewhat) avoid confusion. Put back the usable cpu counting logic (cpu_count / 2 for >= 4 cpus with a minimum of 3, cpu_count otherwise). Now applies to BSD and MACOS as well. Moved the logic to support.usable_processors_from_total_processors(total) and added tests for it. Changed tests to reflect changes in the support module
catalin_m
pypy.commits at gmail.com
Thu Apr 14 09:01:14 EDT 2016
Author: Catalin Gabriel Manciu <catalin.gabriel.manciu at intel.com>
Branch: detect_cpu_count
Changeset: r83670:2f0bd0017db0
Date: 2016-04-14 15:52 +0300
http://bitbucket.org/pypy/pypy/changeset/2f0bd0017db0/
Log: (catalin_m) Renamed support.detect_number_of_processors to
support.detect_number_of_usable_processors to (somewhat) avoid
confusion. Put back the usable cpu counting logic (cpu_count / 2 for
>= 4 cpus with a minimum of 3, cpu_count otherwise). Now applies to
BSD and MACOS as well. Moved the logic to
support.usable_processors_from_total_processors(total) and added
tests for it. Changed tests to reflect changes in the support module
diff --git a/rpython/config/support.py b/rpython/config/support.py
--- a/rpython/config/support.py
+++ b/rpython/config/support.py
@@ -4,32 +4,40 @@
import re, sys, os, subprocess
-def detect_number_of_processors_fallback(filename_or_file):
- if sys.platform == 'darwin':
- return sysctl_get_cpu_count('/usr/sbin/sysctl')
- elif sys.platform.startswith('freebsd'):
- return sysctl_get_cpu_count('/sbin/sysctl')
- elif not sys.platform.startswith('linux'):
- return 1 # try to use cpu_count on other platforms or fallback to 1
- try:
- if isinstance(filename_or_file, str):
- f = open(filename_or_file, "r")
- else:
- f = filename_or_file
- return max([int(re.split('processor.*?(\d+)', line)[1])
- for line in f.readlines()
- if line.startswith('processor')]) + 1 # returning the actual number of available CPUs
- except:
- return 1 # we really don't want to explode here, at worst we have 1
+def usable_processors_from_total_processors(total):
+ if total >= 4:
+ return max(total // 2, 3)
+ return total
-def detect_number_of_processors(filename_or_file='/proc/cpuinfo'):
+def detect_number_of_usable_processors(filename_or_file='/proc/cpuinfo'):
if os.environ.get('MAKEFLAGS'):
return 1 # don't override MAKEFLAGS. This will call 'make' without any '-j' option
+
+ def fallback(filename_or_file):
+ if sys.platform == 'darwin':
+ return sysctl_get_cpu_count('/usr/sbin/sysctl')
+ elif sys.platform.startswith('freebsd'):
+ return sysctl_get_cpu_count('/sbin/sysctl')
+ elif not sys.platform.startswith('linux'):
+ return 1 # try to use cpu_count on other platforms or fallback to 1
+ try:
+ if isinstance(filename_or_file, str):
+ f = open(filename_or_file, "r")
+ else:
+ f = filename_or_file
+ return max([int(re.split('processor.*?(\d+)', line)[1])
+ for line in f.readlines()
+ if line.startswith('processor')]) + 1 # returning the actual number of available CPUs
+ except:
+ return 1 # we really don't want to explode here, at worst we have 1
+
try:
import multiprocessing
- return multiprocessing.cpu_count()
+ count = multiprocessing.cpu_count()
except:
- return detect_number_of_processors_fallback(filename_or_file)
+ count = fallback(filename_or_file)
+ return usable_processors_from_total_processors(count)
+
def sysctl_get_cpu_count(cmd, name='hw.ncpu'):
try:
diff --git a/rpython/config/test/test_support.py b/rpython/config/test/test_support.py
--- a/rpython/config/test/test_support.py
+++ b/rpython/config/test/test_support.py
@@ -46,9 +46,18 @@
assert varname == 'MAKEFLAGS'
return self._value
+def test_usable_cpus_from_total_cpus():
+ # test usable_processors_from_total_processors function
+ assert support.usable_processors_from_total_processors(1) == 1
+ assert support.usable_processors_from_total_processors(3) == 3
+ assert support.usable_processors_from_total_processors(4) == 3
+ assert support.usable_processors_from_total_processors(8) == 4
+ assert support.usable_processors_from_total_processors(16) == 8
+
def test_cpuinfo_linux():
if not sys.platform.startswith('linux'):
py.test.skip("linux only")
+
saved = os.environ
# old_cpu_count will be multiprocessing.cpu_count if multiprocessing module is available or None if the import fails
try:
@@ -56,14 +65,15 @@
old_cpu_count = multiprocessing.cpu_count
except:
old_cpu_count = None
+
if old_cpu_count != None: # if multiprocessing module is available
# test common behavior
- assert support.detect_number_of_processors() == multiprocessing.cpu_count()
+ assert support.detect_number_of_usable_processors() == support.usable_processors_from_total_processors(multiprocessing.cpu_count())
# test common behaviour when MAKEFLAGS is set
os.environ = FakeEnviron('-j2')
- assert support.detect_number_of_processors() == 1
+ assert support.detect_number_of_usable_processors() == 1
# create an override for cpu_count that throws an exception in order to test the fallback behavior of
- # support.detect_number_of_processors()
+ # support.detect_number_of_usable_processors()
def fail_cpu_count():
raise Exception("Failure")
multiprocessing.cpu_count = fail_cpu_count
@@ -71,10 +81,10 @@
# test fallback behavior (multiprocessing.cpu_count() throwing an exception or multiprocessing module
# not available)
os.environ = FakeEnviron(None)
- assert support.detect_number_of_processors(StringIO(cpuinfo)) == 11
- assert support.detect_number_of_processors('random crap that does not exist') == 1
+ assert support.detect_number_of_usable_processors(StringIO(cpuinfo)) == support.usable_processors_from_total_processors(11)
+ assert support.detect_number_of_usable_processors('random crap that does not exist') == 1
os.environ = FakeEnviron('-j2')
- assert support.detect_number_of_processors(StringIO(cpuinfo)) == 1
+ assert support.detect_number_of_usable_processors(StringIO(cpuinfo)) == 1
finally:
os.environ = saved
if old_cpu_count != None:
@@ -94,9 +104,9 @@
try:
support.sysctl_get_cpu_count = count
os.environ = FakeEnviron(None)
- assert support.detect_number_of_processors() == 42
+ assert support.detect_number_of_usable_processors() == support.usable_processors_from_total_processors(42)
os.environ = FakeEnviron('-j2')
- assert support.detect_number_of_processors() == 1
+ assert support.detect_number_of_usable_processors() == 1
finally:
os.environ = saved
support.sysctl_get_cpu_count = saved_func
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -3,7 +3,7 @@
from rpython.config.config import OptionDescription, BoolOption, IntOption, ArbitraryOption, FloatOption
from rpython.config.config import ChoiceOption, StrOption, Config, ConflictConfigError
from rpython.config.config import ConfigError
-from rpython.config.support import detect_number_of_processors
+from rpython.config.support import detect_number_of_usable_processors
from rpython.translator.platform import platform as compiler
@@ -172,7 +172,7 @@
negation=False),
IntOption("make_jobs", "Specify -j argument to make for compilation"
" (C backend only)",
- cmdline="--make-jobs", default=detect_number_of_processors()),
+ cmdline="--make-jobs", default=detect_number_of_usable_processors()),
# Flags of the TranslationContext:
BoolOption("list_comprehension_operations",
More information about the pypy-commit
mailing list