From linkmauve at linkmauve.fr Tue Nov 3 01:37:50 2015 From: linkmauve at linkmauve.fr (Emmanuel Gil Peyrot) Date: Tue, 3 Nov 2015 07:37:50 +0100 Subject: [Cython] [PATCH] Add a pass transforming Python-style properties in cdef class into Cython-style properties. Message-ID: <20151103063750.GA8939@yuyuko> This makes properties work at all, in cdef classes, and gives them almost the same features as the ?property something:? blocks. The only missing feature being the ability to assignate a docstring to the property itself, not only to the getter, setter and deleter. Fixes T264. --- Cython/Compiler/ParseTreeTransforms.py | 72 ++++++++++++++++++++++++ Cython/Compiler/Pipeline.py | 3 +- tests/run/cdef_class_property_decorator_T264.pyx | 51 +++++++++++++++++ 3 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 tests/run/cdef_class_property_decorator_T264.pyx diff --git a/Cython/Compiler/ParseTreeTransforms.py b/Cython/Compiler/ParseTreeTransforms.py index b34f44b..67507ce 100644 --- a/Cython/Compiler/ParseTreeTransforms.py +++ b/Cython/Compiler/ParseTreeTransforms.py @@ -1270,6 +1270,78 @@ class WithTransform(CythonTransform, SkipDeclarations): return node +class PropertyTransform(ScopeTrackingTransform): + """Originally, this was the only place where decorators were + transformed into the corresponding calling code. Now, this is + done directly in DefNode and PyClassDefNode to avoid reassignments + to the function/class name - except for cdef class methods. For + those, the reassignment is required as methods are originally + defined in the PyMethodDef struct. + + The IndirectionNode allows DefNode to override the decorator + """ + + properties = {} + + def visit_DefNode(self, func_node): + func_node = self.visit_FuncDefNode(func_node) + if self.scope_type != 'cclass' or not func_node.decorators: + return func_node + return self.handle_decorators(func_node, func_node.decorators, + func_node.name) + + def handle_decorators(self, node, decorators, name): + properties = self.properties.setdefault(self.scope_node, {}) + for decorator in decorators[::-1]: + decorator_node = decorator.decorator + if (isinstance(decorator_node, ExprNodes.NameNode) and + decorator_node.name == 'property'): + node.name = '__get__' + node.decorators.remove(decorator) + stat_list = [node] + if name in properties: + property = properties[name] + property.body.stats = stat_list + return [] + else: + property = Nodes.PropertyNode(node.pos, name=name) + property.doc = EncodedString(u'') + property.body = Nodes.StatListNode(node.pos, stats=stat_list) + properties[name] = property + return [property] + elif (isinstance(decorator_node, ExprNodes.AttributeNode) and + decorator_node.attribute == 'setter' and + decorator_node.obj.name in properties): + assert decorator_node.obj.name == name + node.name = '__set__' + node.decorators.remove(decorator) + property = properties[name] + stats = property.body.stats + for i, stat in enumerate(stats): + if stat.name == '__set__': + stats[i] = node + break + else: + stats.append(node) + return [] + elif (isinstance(decorator_node, ExprNodes.AttributeNode) and + decorator_node.attribute == 'deleter' and + decorator_node.obj.name in properties): + assert decorator_node.obj.name == name + node.name = '__del__' + node.decorators.remove(decorator) + property = properties[name] + stats = property.body.stats + for i, stat in enumerate(stats): + if stat.name == '__del__': + stats[i] = node + break + else: + stats.append(node) + return [] + return node + + class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): """Originally, this was the only place where decorators were transformed into the corresponding calling code. Now, this is diff --git a/Cython/Compiler/Pipeline.py b/Cython/Compiler/Pipeline.py index 833e22c..0506aff 100644 --- a/Cython/Compiler/Pipeline.py +++ b/Cython/Compiler/Pipeline.py @@ -171,7 +171,7 @@ def create_pipeline(context, mode, exclude_classes=()): from .ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor, DecoratorTransform from .ParseTreeTransforms import InterpretCompilerDirectives, TransformBuiltinMethods from .ParseTreeTransforms import ExpandInplaceOperators, ParallelRangeTransform - from .ParseTreeTransforms import CalculateQualifiedNamesTransform + from .ParseTreeTransforms import CalculateQualifiedNamesTransform, PropertyTransform from .TypeInference import MarkParallelAssignments, MarkOverflowingArithmetic from .ParseTreeTransforms import AdjustDefByDirectives, AlignFunctionDefinitions from .ParseTreeTransforms import RemoveUnreachableCode, GilCheck @@ -216,6 +216,7 @@ def create_pipeline(context, mode, exclude_classes=()): RemoveUnreachableCode(context), ConstantFolding(), FlattenInListTransform(), + PropertyTransform(context), DecoratorTransform(context), ForwardDeclareTypes(context), AnalyseDeclarationsTransform(context), diff --git a/tests/run/cdef_class_property_decorator_T264.pyx b/tests/run/cdef_class_property_decorator_T264.pyx new file mode 100644 index 0000000..94b862c --- /dev/null +++ b/tests/run/cdef_class_property_decorator_T264.pyx @@ -0,0 +1,51 @@ +# mode: run +# ticket: 264 +# tag: property, decorator + +cdef class Prop(object): + """ + >>> p = Prop() + >>> p.prop + GETTING 'None' + >>> p.prop = 1 + SETTING '1' (previously: 'None') + >>> p.prop + GETTING '1' + 1 + >>> p.prop = 2 + SETTING '2' (previously: '1') + >>> p.prop + GETTING '2' + 2 + >>> del p.prop + DELETING '2' + >>> p.prop + GETTING 'None' + """ + cdef _value + def __init__(self): + self._value = None + + @property + def prop(self): + print("FAIL") + return 0 + + @prop.deleter + def prop(self): + print("FAIL") + + @property + def prop(self): + print("GETTING '%s'" % self._value) + return self._value + + @prop.setter + def prop(self, value): + print("SETTING '%s' (previously: '%s')" % (value, self._value)) + self._value = value + + @prop.deleter + def prop(self): + print("DELETING '%s'" % self._value) + self._value = None -- 2.6.2 From thouis at gmail.com Tue Nov 3 09:34:30 2015 From: thouis at gmail.com (Thouis (Ray) Jones) Date: Tue, 03 Nov 2015 14:34:30 +0000 Subject: [Cython] BUG: Invalid code generated from prange() Message-ID: Cythoning and then trying to compile the attached test causes gcc-4.9 to error out with: physics.c:1840:165: error: invalid entry to OpenMP structured block if (unlikely(!__pyx_v_XY.memview)) { __Pyx_RaiseUnboundMemoryviewSliceNogil("XY"); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L5_error;} } clang version 3.5 (with openmp) gives the less helpful "use of undeclared label '__pyx_L5_error'". Ray Jones -------------- next part -------------- An HTML attachment was scrubbed... URL: -------------- next part -------------- A non-text attachment was scrubbed... Name: physics.pyx Type: application/octet-stream Size: 710 bytes Desc: not available URL: From mr.pastafarianist at gmail.com Tue Nov 3 12:19:08 2015 From: mr.pastafarianist at gmail.com (Pastafarianist) Date: Tue, 3 Nov 2015 20:19:08 +0300 Subject: [Cython] AttributeError: 'LetNode' object has no attribute 'generate_rhs_evaluation_code' Message-ID: <5638EC8C.9060608@gmail.com> I'm running Python 3.4.3 and Cython 0.23.4 under Gentoo Linux. I think I've found a bug in Cython. Steps to reproduce: ## test.py ## import pyximport pyximport.install() import mymodule print(mymodule.get(1, 2)) ## mymodule.pyx ## cdef int[2][3] arr = ( (0, 1, 2), (3, 4, 5) ) cpdef int get(int i, int j): return arr[i][j] #### `python test.py` fails with the following traceback: Traceback (most recent call last): File "/usr/lib64/python3.4/site-packages/pyximport/pyximport.py", line 209, in load_module inplace=build_inplace, language_level=language_level) File "/usr/lib64/python3.4/site-packages/pyximport/pyximport.py", line 186, in build_module reload_support=pyxargs.reload_support) File "/usr/lib64/python3.4/site-packages/pyximport/pyxbuild.py", line 100, in pyx_to_dll dist.run_commands() File "/usr/lib64/python3.4/distutils/dist.py", line 955, in run_commands self.run_command(cmd) File "/usr/lib64/python3.4/distutils/dist.py", line 974, in run_command cmd_obj.run() File "/usr/lib64/python3.4/site-packages/Cython/Distutils/build_ext.py", line 164, in run _build_ext.build_ext.run(self) File "/usr/lib64/python3.4/distutils/command/build_ext.py", line 339, in run self.build_extensions() File "/usr/lib64/python3.4/site-packages/Cython/Distutils/build_ext.py", line 171, in build_extensions ext.sources = self.cython_sources(ext.sources, ext) File "/usr/lib64/python3.4/site-packages/Cython/Distutils/build_ext.py", line 320, in cython_sources full_module_name=module_name) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/Main.py", line 677, in compile return compile_single(source, options, full_module_name) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/Main.py", line 630, in compile_single return run_pipeline(source, options, full_module_name) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/Main.py", line 487, in run_pipeline err, enddata = Pipeline.run_pipeline(pipeline, source) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/Pipeline.py", line 328, in run_pipeline data = phase(data) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/Pipeline.py", line 53, in generate_pyx_code_stage module_node.process_implementation(options, result) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/ModuleNode.py", line 118, in process_implementation self.generate_c_code(env, options, result) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/ModuleNode.py", line 348, in generate_c_code self.generate_module_init_func(modules[:-1], env, globalstate['init_module']) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/ModuleNode.py", line 2173, in generate_module_init_func self.body.generate_execution_code(code) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/Nodes.py", line 442, in generate_execution_code stat.generate_execution_code(code) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/UtilNodes.py", line 334, in generate_execution_code self.body.generate_execution_code(code) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/Nodes.py", line 5223, in generate_execution_code stat.generate_rhs_evaluation_code(code) AttributeError: 'LetNode' object has no attribute 'generate_rhs_evaluation_code' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/pastafarianist/ownCloud/dev/sandbox/cython/test.py", line 4, in import mymodule File "/usr/lib64/python3.4/site-packages/pyximport/pyximport.py", line 435, in load_module language_level=self.language_level) File "/usr/lib64/python3.4/site-packages/pyximport/pyximport.py", line 225, in load_module raise exc.with_traceback(tb) File "/usr/lib64/python3.4/site-packages/pyximport/pyximport.py", line 209, in load_module inplace=build_inplace, language_level=language_level) File "/usr/lib64/python3.4/site-packages/pyximport/pyximport.py", line 186, in build_module reload_support=pyxargs.reload_support) File "/usr/lib64/python3.4/site-packages/pyximport/pyxbuild.py", line 100, in pyx_to_dll dist.run_commands() File "/usr/lib64/python3.4/distutils/dist.py", line 955, in run_commands self.run_command(cmd) File "/usr/lib64/python3.4/distutils/dist.py", line 974, in run_command cmd_obj.run() File "/usr/lib64/python3.4/site-packages/Cython/Distutils/build_ext.py", line 164, in run _build_ext.build_ext.run(self) File "/usr/lib64/python3.4/distutils/command/build_ext.py", line 339, in run self.build_extensions() File "/usr/lib64/python3.4/site-packages/Cython/Distutils/build_ext.py", line 171, in build_extensions ext.sources = self.cython_sources(ext.sources, ext) File "/usr/lib64/python3.4/site-packages/Cython/Distutils/build_ext.py", line 320, in cython_sources full_module_name=module_name) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/Main.py", line 677, in compile return compile_single(source, options, full_module_name) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/Main.py", line 630, in compile_single return run_pipeline(source, options, full_module_name) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/Main.py", line 487, in run_pipeline err, enddata = Pipeline.run_pipeline(pipeline, source) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/Pipeline.py", line 328, in run_pipeline data = phase(data) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/Pipeline.py", line 53, in generate_pyx_code_stage module_node.process_implementation(options, result) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/ModuleNode.py", line 118, in process_implementation self.generate_c_code(env, options, result) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/ModuleNode.py", line 348, in generate_c_code self.generate_module_init_func(modules[:-1], env, globalstate['init_module']) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/ModuleNode.py", line 2173, in generate_module_init_func self.body.generate_execution_code(code) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/Nodes.py", line 442, in generate_execution_code stat.generate_execution_code(code) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/UtilNodes.py", line 334, in generate_execution_code self.body.generate_execution_code(code) File "/usr/lib64/python3.4/site-packages/Cython/Compiler/Nodes.py", line 5223, in generate_execution_code stat.generate_rhs_evaluation_code(code) ImportError: Building module mymodule failed: ["AttributeError: 'LetNode' object has no attribute 'generate_rhs_evaluation_code'\n"] From carlosjosepita at gmail.com Fri Nov 6 14:12:24 2015 From: carlosjosepita at gmail.com (Carlos Pita) Date: Fri, 6 Nov 2015 16:12:24 -0300 Subject: [Cython] [Bug] Generator already executing Message-ID: Say you define: def f(*ts): ts = (t for t in ts) list(ts) def g(*ts): list(t for t in ts) def h(*ts): list((t for t in ts)) Then f(1,2,3) will fail with "Generator already executing" while g(1,2,3) and h(1,2, 3) won't. All versions should work the same. Cheers -- Carlos -------------- next part -------------- An HTML attachment was scrubbed... URL: From stefan_ml at behnel.de Sat Nov 7 03:14:24 2015 From: stefan_ml at behnel.de (Stefan Behnel) Date: Sat, 7 Nov 2015 09:14:24 +0100 Subject: [Cython] [Bug] Generator already executing In-Reply-To: References: Message-ID: <563DB2E0.9080807@behnel.de> Carlos Pita schrieb am 06.11.2015 um 20:12: > Say you define: > > def f(*ts): > ts = (t for t in ts) > list(ts) > > def g(*ts): > list(t for t in ts) > > def h(*ts): > list((t for t in ts)) > > Then f(1,2,3) will fail with "Generator already executing" while g(1,2,3) > and h(1,2, 3) won't. All versions should work the same. It's a known bug. http://trac.cython.org/ticket/600 Stefan From stefan_ml at behnel.de Sat Nov 7 05:05:50 2015 From: stefan_ml at behnel.de (Stefan Behnel) Date: Sat, 7 Nov 2015 11:05:50 +0100 Subject: [Cython] [PATCH] Add a pass transforming Python-style properties in cdef class into Cython-style properties. In-Reply-To: <20151103063750.GA8939@yuyuko> References: <20151103063750.GA8939@yuyuko> Message-ID: <563DCCFE.5080902@behnel.de> Emmanuel Gil Peyrot schrieb am 03.11.2015 um 07:37: > This makes properties work at all, in cdef classes, and gives them > almost the same features as the ?property something:? blocks. Superseded by https://github.com/cython/cython/pull/462 Stefan From kasal at ucw.cz Fri Nov 6 10:18:50 2015 From: kasal at ucw.cz (Stepan Kasal) Date: Fri, 6 Nov 2015 16:18:50 +0100 Subject: [Cython] bug in doc Tutorials/Working with NumPy Message-ID: <20151106151850.GB16635@ucw.cz> Hello, I was learning Cython from this tutorial when I noticed a glitch: Near the end, after boundscheck False, some variables are changed to unsigned int. But this has unexpected consequences: variables t_from and such are not determined correctly, as one of the arguments is negative, but gets cast to unsigned for comaprison. (Yes, this is a weird feature of C.) Actually, the use of unsigned does not seem to have any impact on performance, so I suggest to scratch that part of tutorial. Just explain that with boundscheck off, negaitive indices no longer work. (But there might perhaps be one small improvent in cython itself: with boundscheck off, the index might be cast to unsigned before it is used: this way we probably crash on negative values, instead of silently accessing the negative offset of the array data.) Thank you for nice piece of software, Stepan Kasal From yi.codeplayer at gmail.com Mon Nov 9 19:01:33 2015 From: yi.codeplayer at gmail.com (yi huang) Date: Tue, 10 Nov 2015 08:01:33 +0800 Subject: [Cython] [Bug] Don't compile lambda with tuple argument. Message-ID: ? $ cat /tmp/test.pyx f = lambda (name,v):True ? $ cython /tmp/test.pyx Error compiling Cython file: ------------------------------------------------------------ ... f = lambda (name,v):True ^ ------------------------------------------------------------ /tmp/test.pyx:1:19: Empty declarator Tested with both 0.21 and 0.23 From yi.codeplayer at gmail.com Mon Nov 9 19:15:06 2015 From: yi.codeplayer at gmail.com (yi huang) Date: Tue, 10 Nov 2015 08:15:06 +0800 Subject: [Cython] [Bug] Don't compile lambda with tuple argument. In-Reply-To: References: Message-ID: Even function with tuple match parameter don't work either. ``` def test((name, v)): return True ``` ``` Error compiling Cython file: ------------------------------------------------------------ ... def test((name, v)): ^ ------------------------------------------------------------ /private/tmp/test.pyx:3:18: Empty declarator ``` On Tue, Nov 10, 2015 at 8:01 AM, yi huang wrote: > ? $ cat /tmp/test.pyx > f = lambda (name,v):True > > ? $ cython /tmp/test.pyx > > Error compiling Cython file: > ------------------------------------------------------------ > ... > f = lambda (name,v):True > ^ > ------------------------------------------------------------ > > /tmp/test.pyx:1:19: Empty declarator > > > Tested with both 0.21 and 0.23 -- http://yi-programmer.com/ From robertwb at gmail.com Tue Nov 10 00:25:52 2015 From: robertwb at gmail.com (Robert Bradshaw) Date: Mon, 9 Nov 2015 21:25:52 -0800 Subject: [Cython] [Bug] Don't compile lambda with tuple argument. In-Reply-To: References: Message-ID: On Mon, Nov 9, 2015 at 4:15 PM, yi huang wrote: > Even function with tuple match parameter don't work either. > > ``` > def test((name, v)): > return True > ``` > > ``` > Error compiling Cython file: > ------------------------------------------------------------ > ... > def test((name, v)): > ^ > ------------------------------------------------------------ > > /private/tmp/test.pyx:3:18: Empty declarator > ``` > > On Tue, Nov 10, 2015 at 8:01 AM, yi huang wrote: >> ? $ cat /tmp/test.pyx >> f = lambda (name,v):True >> >> ? $ cython /tmp/test.pyx >> >> Error compiling Cython file: >> ------------------------------------------------------------ >> ... >> f = lambda (name,v):True >> ^ >> ------------------------------------------------------------ >> >> /tmp/test.pyx:1:19: Empty declarator >> >> >> Tested with both 0.21 and 0.23 Yes, this went away in Python 3 as well: https://www.python.org/dev/peps/pep-3113/ . From robertwb at gmail.com Tue Nov 10 00:44:15 2015 From: robertwb at gmail.com (Robert Bradshaw) Date: Mon, 9 Nov 2015 21:44:15 -0800 Subject: [Cython] bug in doc Tutorials/Working with NumPy In-Reply-To: <20151106151850.GB16635@ucw.cz> References: <20151106151850.GB16635@ucw.cz> Message-ID: On Fri, Nov 6, 2015 at 7:18 AM, Stepan Kasal wrote: > Hello, > > I was learning Cython from this tutorial when I noticed a glitch: > > Near the end, after boundscheck False, some variables are changed to > unsigned int. But this has unexpected consequences: variables t_from and such > are not determined correctly, as one of the arguments is negative, but gets > cast to unsigned for comaprison. (Yes, this is a weird feature of C.) > > Actually, the use of unsigned does not seem to have any impact on performance, > so I suggest to scratch that part of tutorial. Just explain that with boundscheck > off, negaitive indices no longer work. > > (But there might perhaps be one small improvent in cython itself: with boundscheck > off, the index might be cast to unsigned before it is used: this way we probably > crash on negative values, instead of silently accessing the negative offset of > the array data.) There's two directives: wraparound and boundscheck. The wraparound directive can be used to disable the check for negative values. I heartily agree that using unsigned types is the source of much surprising pain, we shouldn't be recommending it: https://github.com/cython/cython/commit/06f75b803d91805f3068bbad2b9241efbd5cdc44 From kasal at ucw.cz Tue Nov 10 08:53:59 2015 From: kasal at ucw.cz (Stepan Kasal) Date: Tue, 10 Nov 2015 14:53:59 +0100 Subject: [Cython] bug in doc Tutorials/Working with NumPy Message-ID: <20151110135359.GA3009@ucw.cz> Hello, On Mon, Nov 09, 2015 at 09:44:15PM -0800, Robert Bradshaw wrote: > I heartily agree that using unsigned types is the source of much > surprising pain, we shouldn't be recommending it: great, thanks a lot for updating the dev doc instantly. > There's two directives: wraparound and boundscheck. The wraparound > directive can be used to disable the check for negative values. And thanks for catching this; I misunderstood this. Stepan From h.goebel at crazy-compilers.com Tue Nov 24 15:50:27 2015 From: h.goebel at crazy-compilers.com (Hartmut Goebel) Date: Tue, 24 Nov 2015 21:50:27 +0100 Subject: [Cython] Store list of imported modules in .so file Message-ID: <5654CD93.6050500@crazy-compilers.com> Hi, I'm a core-developer for PyInstaller. We would like to enhance cython support in PyInstaller and resolve dependencies of cython-compiled modules automatically. This would ease freezing cython modules with PyInstaller a lot and thus help users including cythonized-modules into their shipped application. For us, the most elegant way to get the dependencies would be to have the list of imported modules in the .so-file. E.g. some simple string marked by some cookie which is easily grepped out of the binary. Is there any chance to get suche a list into the .so/.dll-file? If you like, you can comment directly on our related ticket: https://github.com/pyinstaller/pyinstaller/issues/1674 -- Regards Hartmut Goebel | Hartmut Goebel | h.goebel at crazy-compilers.com | | www.crazy-compilers.com | compilers which you thought are impossible | From robertwb at gmail.com Thu Nov 26 01:15:21 2015 From: robertwb at gmail.com (Robert Bradshaw) Date: Wed, 25 Nov 2015 22:15:21 -0800 Subject: [Cython] Store list of imported modules in .so file In-Reply-To: <5654CD93.6050500@crazy-compilers.com> References: <5654CD93.6050500@crazy-compilers.com> Message-ID: On Tue, Nov 24, 2015 at 12:50 PM, Hartmut Goebel wrote: > Hi, > > I'm a core-developer for PyInstaller. We would like to enhance cython > support in PyInstaller and resolve dependencies of cython-compiled > modules automatically. This would ease freezing cython modules with > PyInstaller a lot and thus help users including cythonized-modules into > their shipped application. > > For us, the most elegant way to get the dependencies would be to have > the list of imported modules in the .so-file. E.g. some simple string > marked by some cookie which is easily grepped out of the binary. > > Is there any chance to get suche a list into the .so/.dll-file? It's not clear how to embed such information deterministically into the so/dll file (which are produced by the C compilers), but if you had/could create the generated C files we do put some metadata at the top of those when compiled with cythonize, e.g. /* BEGIN: Cython Metadata { [JSON data about cimports, distutils options, etc] } END: Cython Metadata */ We could consider putting imports there as well. > If you like, you can comment directly on our related ticket: > https://github.com/pyinstaller/pyinstaller/issues/1674 > > -- > Regards > Hartmut Goebel > > | Hartmut Goebel | h.goebel at crazy-compilers.com | > | www.crazy-compilers.com | compilers which you thought are impossible | > > > _______________________________________________ > cython-devel mailing list > cython-devel at python.org > https://mail.python.org/mailman/listinfo/cython-devel From nykevin.norris at gmail.com Fri Nov 27 12:13:42 2015 From: nykevin.norris at gmail.com (Kevin Norris) Date: Fri, 27 Nov 2015 12:13:42 -0500 Subject: [Cython] Store list of imported modules in .so file Message-ID: On Tue, Nov 24, 2015 at 12:50 PM, Hartmut Goebel wrote: > Hi, > > I'm a core-developer for PyInstaller. We would like to enhance cython > support in PyInstaller and resolve dependencies of cython-compiled > modules automatically. This would ease freezing cython modules with > PyInstaller a lot and thus help users including cythonized-modules into > their shipped application. > > For us, the most elegant way to get the dependencies would be to have > the list of imported modules in the .so-file. E.g. some simple string > marked by some cookie which is easily grepped out of the binary. > > Is there any chance to get suche a list into the .so/.dll-file? Import statements are "just code." They can appear inside functions and more generally can occur at runtime. In the absolute worst case, you could have something like (Python 3.x): importlib.import_module(input('What should I import today?')) This is not specific to Cython, but I imagine it's a lot harder to pull out via static analysis when you don't have Python-like source (see also: pkgutil.get_data(), any use of __file__, etc.). OTOH, if you're dealing with importlib et al., you're already in a bad place for static analysis since people can do things like: import importlib as foo; bar = foo.import_module; baz = bar('qux') (which is roughly equivalent to import qux as baz, but good luck spotting it automatically) Anyway, I think it might help to clarify what *exactly* you mean by "dependency." If you mean "any module that might possibly get imported by module X," you're going to have a hard time with this. Setuptools takes a fully manual approach to this problem, requiring developers to explicitly list all dependencies. That may not work for your needs, but it does seem to be more flexible since it lets you specify PEP 440 version constraints. From robertwb at gmail.com Fri Nov 27 13:00:43 2015 From: robertwb at gmail.com (Robert Bradshaw) Date: Fri, 27 Nov 2015 10:00:43 -0800 Subject: [Cython] Store list of imported modules in .so file In-Reply-To: References: Message-ID: On Fri, Nov 27, 2015 at 9:13 AM, Kevin Norris wrote: > On Tue, Nov 24, 2015 at 12:50 PM, Hartmut Goebel > wrote: >> Hi, >> >> I'm a core-developer for PyInstaller. We would like to enhance cython >> support in PyInstaller and resolve dependencies of cython-compiled >> modules automatically. This would ease freezing cython modules with >> PyInstaller a lot and thus help users including cythonized-modules into >> their shipped application. >> >> For us, the most elegant way to get the dependencies would be to have >> the list of imported modules in the .so-file. E.g. some simple string >> marked by some cookie which is easily grepped out of the binary. >> >> Is there any chance to get suche a list into the .so/.dll-file? > > Import statements are "just code." They can appear inside functions > and more generally can occur at runtime. In the absolute worst case, > you could have something like (Python 3.x): > > importlib.import_module(input('What should I import today?')) > > This is not specific to Cython, but I imagine it's a lot harder to > pull out via static analysis when you don't have Python-like source > (see also: pkgutil.get_data(), any use of __file__, etc.). OTOH, if > you're dealing with importlib et al., you're already in a bad place > for static analysis since people can do things like: > > import importlib as foo; bar = foo.import_module; baz = bar('qux') > > (which is roughly equivalent to import qux as baz, but good luck > spotting it automatically) > > Anyway, I think it might help to clarify what *exactly* you mean by > "dependency." If you mean "any module that might possibly get > imported by module X," you're going to have a hard time with this. > Setuptools takes a fully manual approach to this problem, requiring > developers to explicitly list all dependencies. That may not work for > your needs, but it does seem to be more flexible since it lets you > specify PEP 440 version constraints. Yes, you can do all sorts of crazy stuff. On the other hand, setuptools explicitness might be overkill for the 98% of projects that whose dependencies could be easily inferred, and at least give you a starting point. - Robert From h.goebel at crazy-compilers.com Sat Nov 28 05:35:06 2015 From: h.goebel at crazy-compilers.com (Hartmut Goebel) Date: Sat, 28 Nov 2015 11:35:06 +0100 Subject: [Cython] Store list of imported modules in .so file In-Reply-To: References: Message-ID: <5659835A.4000106@crazy-compilers.com> Am 27.11.2015 um 18:13 schrieb Kevin Norris: > Anyway, I think it might help to clarify what *exactly* you mean by > "dependency." If you mean "any module that might possibly get > imported by module X," you're going to have a hard time with this. We know about the problems you wrote -- this is PyInstallers job and we know where he limits are. So we would be happy to get a list od fully qualified module names for each "import" cython finds (and as I assume generated code for). PyInstaller too can only analyse the import-statements, which is enough for 90% of the cases. (We do a bit more, e.g. finding libraries used via ctypes.) Setuptools is not an option, since this are meant to list al* requirements of some package, while PyInstaller only includes the *used* modules - which might be quite a difference :-) -- Regards Hartmut Goebel | Hartmut Goebel | h.goebel at crazy-compilers.com | | www.crazy-compilers.com | compilers which you thought are impossible | -------------- next part -------------- An HTML attachment was scrubbed... URL: From linkmauve at linkmauve.fr Mon Nov 30 20:12:32 2015 From: linkmauve at linkmauve.fr (Emmanuel Gil Peyrot) Date: Tue, 1 Dec 2015 02:12:32 +0100 Subject: [Cython] Implementing tp_richcmp using __eq__ etc. Message-ID: <20151201011232.GB25936@yuyuko> Hello, I?m trying to implement the compilation of __eq__, __ne__, etc. in extension types, to have them available in pure mode (see ticket #130). I currently have an implementation which basically pastes the body.stats from those six functions in a __richcmp__ one using a template, but I find that very inelegant, as it removes the information about the original location of this code which is very bad for debugging, forces all six (or less) functions to use the same argument names (or rename them), and probably some other issues. So my idea was to convert each of those special function into a CFuncDefNode from a DefNode, in order to remove the Python wrapper, then to implement the new tp_richcmp slot as an util function directly in C. If this plan is sound, I?d like a few more informations, for now mostly about the DefNode to CFuncDefNode conversion, which I?ve been unable to do programmatically, with some later passes complaining about missing base_type or declarator properties. The next question is about determining in the C code whether a specific method exists, and otherwise returning NotImplemented instead. Thanks for listening, I?m not sure I?ve been very understandable, I?m still quite confused. ^^' -- Emmanuel Gil Peyrot -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 181 bytes Desc: not available URL: