Theano-1.0.0/00007500163562€ÿÿÿþ0000000000013203336031014265 5ustar boccoset00000000000000Theano-1.0.0/DESCRIPTION.txt00006000163562€ÿÿÿþ0000000216213203335527016620 0ustar boccoset00000000000000Theano is a Python library that allows you to define, optimize, and efficiently evaluate mathematical expressions involving multi-dimensional arrays. It is built on top of NumPy_. Theano features: * **tight integration with NumPy:** a similar interface to NumPy's. numpy.ndarrays are also used internally in Theano-compiled functions. * **transparent use of a GPU:** perform data-intensive computations up to 140x faster than on a CPU (support for float32 only). * **efficient symbolic differentiation:** Theano can compute derivatives for functions of one or many inputs. * **speed and stability optimizations:** avoid nasty bugs when computing expressions such as log(1 + exp(x)) for large values of x. * **dynamic C code generation:** evaluate expressions faster. * **extensive unit-testing and self-verification:** includes tools for detecting and diagnosing bugs and/or potential problems. Theano has been powering large-scale computationally intensive scientific research since 2007, but it is also approachable enough to be used in the classroom (IFT6266 at the University of Montreal). .. _NumPy: http://numpy.scipy.org/ Theano-1.0.0/bin/00007500163562€ÿÿÿþ0000000000013203336007015040 5ustar boccoset00000000000000Theano-1.0.0/bin/theano-cache00007000163562€ÿÿÿþ0000000007613203335527017311 0ustar boccoset00000000000000#!/usr/bin/env python import theano_cache theano_cache.main() Theano-1.0.0/bin/theano-nose00007000163562€ÿÿÿþ0000000007413203335527017210 0ustar boccoset00000000000000#!/usr/bin/env python import theano_nose theano_nose.main() Theano-1.0.0/bin/__init__.py00006000163562€ÿÿÿþ0000000000013203335527017142 0ustar boccoset00000000000000Theano-1.0.0/bin/theano_cache.py00006000163562€ÿÿÿþ0000000771313203335527020026 0ustar boccoset00000000000000#!/usr/bin/env python from __future__ import print_function import logging import os import sys if sys.platform == 'win32': config_for_theano_cache_script = 'cxx=,device=cpu' theano_flags = os.environ['THEANO_FLAGS'] if 'THEANO_FLAGS' in os.environ else '' if theano_flags: theano_flags += ',' theano_flags += config_for_theano_cache_script os.environ['THEANO_FLAGS'] = theano_flags import theano from theano import config import theano.gof.compiledir from theano.gof.cc import get_module_cache _logger = logging.getLogger('theano.bin.theano-cache') def print_help(exit_status): if exit_status: print('command "%s" not recognized' % (' '.join(sys.argv))) print('Type "theano-cache" to print the cache location') print('Type "theano-cache help" to print this help') print('Type "theano-cache clear" to erase the cache') print('Type "theano-cache list" to print the cache content') print('Type "theano-cache unlock" to unlock the cache directory') print('Type "theano-cache cleanup" to delete keys in the old ' 'format/code version') print('Type "theano-cache purge" to force deletion of the cache directory') print('Type "theano-cache basecompiledir" ' 'to print the parent of the cache directory') print('Type "theano-cache basecompiledir list" ' 'to print the content of the base compile dir') print('Type "theano-cache basecompiledir purge" ' 'to remove everything in the base compile dir, ' 'that is, erase ALL cache directories') sys.exit(exit_status) def main(): if len(sys.argv) == 1: print(config.compiledir) elif len(sys.argv) == 2: if sys.argv[1] == 'help': print_help(exit_status=0) if sys.argv[1] == 'clear': # We skip the refresh on module cache creation because the refresh will # be done when calling clear afterwards. cache = get_module_cache(init_args=dict(do_refresh=False)) cache.clear(unversioned_min_age=-1, clear_base_files=True, delete_if_problem=True) # Print a warning if some cached modules were not removed, so that the # user knows he should manually delete them, or call # theano-cache purge, # to properly clear the cache. items = [item for item in sorted(os.listdir(cache.dirname)) if item.startswith('tmp')] if items: _logger.warning( 'There remain elements in the cache dir that you may ' 'need to erase manually. The cache dir is:\n %s\n' 'You can also call "theano-cache purge" to ' 'remove everything from that directory.' % config.compiledir) _logger.debug('Remaining elements (%s): %s' % (len(items), ', '.join(items))) elif sys.argv[1] == 'list': theano.gof.compiledir.print_compiledir_content() elif sys.argv[1] == 'cleanup': theano.gof.compiledir.cleanup() cache = get_module_cache(init_args=dict(do_refresh=False)) cache.clear_old() elif sys.argv[1] == 'unlock': theano.gof.compilelock.force_unlock() print('Lock successfully removed!') elif sys.argv[1] == 'purge': theano.gof.compiledir.compiledir_purge() elif sys.argv[1] == 'basecompiledir': # Simply print the base_compiledir print(theano.config.base_compiledir) else: print_help(exit_status=1) elif len(sys.argv) == 3 and sys.argv[1] == 'basecompiledir': if sys.argv[2] == 'list': theano.gof.compiledir.basecompiledir_ls() elif sys.argv[2] == 'purge': theano.gof.compiledir.basecompiledir_purge() else: print_help(exit_status=1) else: print_help(exit_status=1) if __name__ == '__main__': main() Theano-1.0.0/bin/theano_nose.py00006000163562€ÿÿÿþ0000002536313203335527017730 0ustar boccoset00000000000000#!/usr/bin/env python """ This script should behave the same as the `nosetests` command. The reason for its existence is that on some systems, it may not be obvious to find where nosetests is installed in order to run it in a different process. It is also used to load the KnownFailure plugin, in order to hide KnownFailureTests error messages. Use --without-knownfailure to disable that plugin. `run_tests_in_batch.py` will in turn call back this script in another process. """ from __future__ import print_function __authors__ = "Olivier Delalleau, Pascal Lamblin, Eric Larsen" __contact__ = "delallea@iro" import logging _logger = logging.getLogger('theano.bin.theano-nose') import os import nose import textwrap import sys from nose.plugins import Plugin def main_function(): # Handle the --theano arguments if "--theano" in sys.argv: i = sys.argv.index("--theano") import theano sys.argv[i] = theano.__path__[0] # Many Theano tests suppose device=cpu, so we need to raise an # error if device==gpu. # I don't know how to do this check only if we use theano-nose on # Theano tests. So I make an try..except in case the script get # reused elsewhere. # We should not import theano before call nose.main() # As this cause import problem with nosetests. # Should we find a way to don't modify sys.path? if not os.path.exists('theano/__init__.py'): try: from theano import config if config.device != "cpu": raise ValueError("Theano tests must be run with device=cpu." " This will also run GPU tests when possible.\n" " If you want GPU-related tests to run on a" " specific GPU device, and not the default one," " you should use the init_gpu_device theano flag.") except ImportError: pass # Handle --batch[=n] arguments batch_args = [arg for arg in sys.argv if arg.startswith('--batch')] for arg in batch_args: sys.argv.remove(arg) batch_size = None if len(batch_args): if len(batch_args) > 1: _logger.warn( 'Multiple --batch arguments detected, using the last one ' 'and ignoring the first ones.') batch_arg = batch_args[-1] elems = batch_arg.split('=', 1) if len(elems) == 2: batch_size = int(elems[1]) # Handle the --debug-batch argument. display_batch_output = False if '--debug-batch' in sys.argv: if not batch_args: raise AssertionError( 'You can only use the --debug-batch argument with the ' '--batch[=n] option') while '--debug-batch' in sys.argv: sys.argv.remove('--debug-batch') sys.argv += ['--verbose', '--nocapture', '--detailed-errors'] display_batch_output = True # Handle --time_prof arguments time_prof_args = [arg for arg in sys.argv if arg=='--time-profile'] for arg in time_prof_args: sys.argv.remove(arg) # Time-profiling and batch modes if time_prof_args or batch_args: from theano.tests import run_tests_in_batch return run_tests_in_batch.main( theano_nose=os.path.realpath(__file__), batch_size=batch_size, time_profile=bool(time_prof_args), display_batch_output=display_batch_output) # Non-batch mode. addplugins = [] # We include KnownFailure plugin by default, unless # it is disabled by the "--without-knownfailure" arg. if '--without-knownfailure' not in sys.argv: try: from numpy.testing.noseclasses import KnownFailure addplugins.append(KnownFailure()) except ImportError: _logger.warn( 'KnownFailure plugin from NumPy could not be imported. ' 'Use --without-knownfailure to disable this warning.') else: sys.argv.remove('--without-knownfailure') # When 'theano-nose' is called-back under the time-profile option, an # instance of the custom Nosetests plugin class 'DisabDocString' (see # below) is loaded. The latter ensures that the test name will not be # replaced in display by the first line of the documentation string. if '--disabdocstring' in sys.argv: addplugins.append(DisabDocString()) try: if addplugins: ret = nose.main(addplugins=addplugins) else: ret = nose.main() return ret except TypeError as e: if "got an unexpected keyword argument 'addplugins'" in e.message: # This means nose is too old and does not support plugins _logger.warn( 'KnownFailure plugin from NumPy can\'t' ' be used as nosetests is too old. ' 'Use --without-knownfailure to disable this warning.') nose.main() else: raise def help(): help_msg = """ This script behaves mostly the same as the `nosetests` command. The main difference is that it loads automatically the KnownFailure plugin, in order to hide KnownFailureTests error messages. It also supports executing tests by batches. Local options: --help, -h: Displays this help. --batch[=n]: If specified without option '--time-profile', do not run all the tests in one run, but split the execution in batches of `n` tests each. Default n is 100. --time-profile: Each test will be run and timed separately and the results will be deposited in the files 'timeprof_sort', 'timeprof_nosort' and 'timeprof_rawlog' in the current directory. If the '--batch[=n]' option is also specified, notification of the progresses will be made to standard output after every group of n tests. Otherwise, notification will occur after every group of 100 tests. The files 'timeprof_sort' and 'timeprof_nosort' both contain one record for each test and comprise the following fields: - test running-time - nosetests sequential test number - test name - name of class to which test belongs (if any), otherwise full information is contained in test name - test outcome ('OK', 'SKIPPED TEST', 'FAILED TEST' or 'FAILED PARSING') In 'timeprof_sort', test records are sorted according to running-time whereas in 'timeprof_nosort' records are reported according to sequential number. The former classification is the main information source for time-profiling. Since tests belonging to same or close classes and files have close sequential, the latter may be used to identify duration patterns among the tests numbers. A full log is also saved as 'timeprof_rawlog'. --without-knownfailure: Do not load the KnownFailure plugin. --theano: This parameter is replaced with the path to the theano library. As theano-nose is a wrapper to nosetests, it expects a path to the tests to run. If you do not know where theano is installed, use this option to have it inserted automatically. --debug-batch: Use this parameter to run nosetests with options '--verbose', '--nocapture' and '--detailed-errors' and show the output of nosetests during batch execution. This can be useful to debug situations where re-running only the failed tests after batch execution is not working properly. This option can only be used in conjunction with the '--batch=[n]' argument. The other options will be passed to nosetests, see ``nosetests -h``. """ print(textwrap.dedent(help_msg)) def main(): if '--help' in sys.argv or '-h' in sys.argv: help() else: result = main_function() sys.exit(result) class DisabDocString(Plugin): """ When activated, a custom Nosetests plugin created through this class will preclude automatic replacement in display of the name of the test by the first line in its documentation string. Sources: http://nose.readthedocs.org/en/latest/developing.html http://nose.readthedocs.org/en/latest/further_reading.html http://www.siafoo.net/article/54 https://github.com/nose-devs/nose/issues/294 http://python-nose.googlecode.com/svn/trunk/nose/plugins/base.py Nat Williams: https://github.com/Merino/nose-description-fixer-plugin/commit/ df94596f29c04fea8001713dd9b04bf3720aebf4 """ enabled = False # plugin disabled by default score = 2000 # high score ensures priority over other plugins def __init__(self): # 'super.__init__(self):' would have achieved exactly the same if self.name is None: self.name = self.__class__.__name__.lower() if self.enableOpt is None: self.enableOpt = ("enable_plugin_%s" % self.name.replace('-', '_')) def options(self, parser, env): env_opt = 'NOSE_WITH_%s' % self.name.upper() # latter expression to be used if plugin called from the command line parser.add_option("--%s" % self.name, # will be called with Nosetests 'main' or 'run' # function's' argument '--disabdocstring' action="store_true", dest=self.enableOpt, # the latter entails that the boolean self.enableOpt # is set to 'True' when plugin is called through a # function's argument default=env.get(env_opt), # entails that plugin will be enabled when command # line trigger 'env_opt' will be activated help="Enable plugin %s: %s [%s]" % (self.__class__.__name__, self.help(), env_opt)) def configure(self, options, conf): self.conf = conf # plugin will be enabled when called through argument self.enabled = getattr(options, self.enableOpt) def describeTest(self, test): # 'describeTest' is also called when the test result in Nosetests calls # 'test.shortDescription()' and can thus be used to alter the display. return False if __name__ == '__main__': main() Theano-1.0.0/README.rst00006000163562€ÿÿÿþ0000000356413203335527015772 0ustar boccoset00000000000000============================================================================================================ `MILA will stop developing Theano `_. ============================================================================================================ To install the package, see this page: http://deeplearning.net/software/theano/install.html For the documentation, see the project website: http://deeplearning.net/software/theano/ Related Projects: https://github.com/Theano/Theano/wiki/Related-projects It is recommended that you look at the documentation on the website, as it will be more current than the documentation included with the package. In order to build the documentation yourself, you will need sphinx. Issue the following command: :: python ./doc/scripts/docgen.py Documentation is built into ``html/`` The PDF of the documentation can be found at ``html/theano.pdf`` ================ DIRECTORY LAYOUT ================ ``Theano`` (current directory) is the distribution directory. * ``Theano/theano`` contains the package * ``Theano/theano`` has several submodules: * ``gof`` + ``compile`` are the core * ``scalar`` depends upon core * ``tensor`` depends upon ``scalar`` * ``sparse`` depends upon ``tensor`` * ``sandbox`` can depend on everything else * ``Theano/examples`` are copies of the example found on the wiki * ``Theano/benchmark`` and ``Theano/examples`` are in the distribution, but not in the Python package * ``Theano/bin`` contains executable scripts that are copied to the bin folder when the Python package is installed * Tests are distributed and are part of the package, i.e. fall in the appropriate submodules * ``Theano/doc`` contains files and scripts used to generate the documentation * ``Theano/html`` is where the documentation will be generated Theano-1.0.0/HISTORY.txt00006000163562€ÿÿÿþ0000041375113203335527016210 0ustar boccoset00000000000000 .. _HISTORY: ================= Old Release Notes ================= Theano 1.0.0rc1 (30th of October, 2017) ======================================= This release contains new features, improvements and bug fixes to prepare the upcoming release. We recommend that every developer updates to this version. Highlights: - Make sure MKL uses GNU OpenMP - **NB**: Matrix dot product (``gemm``) with ``mkl`` from conda could return wrong results in some cases. We have reported the problem upstream and we have a work around that raises an error with information about how to fix it. - Optimized ``SUM(x^2)``, ``SUM(ABS(X))`` and ``MAX(ABS(X))`` operations with cuDNN reductions - Added Python scripts to help test cuDNN convolutions - Fixed invalid casts and index overflows in ``theano.tensor.signal.pool`` A total of 71 people contributed to this release since 0.9.0, see list below. Commiters since 0.9.0: - Frederic Bastien - Steven Bocco - João Victor Tozatti Risso - Arnaud Bergeron - Mohammed Affan - amrithasuresh - Pascal Lamblin - Reyhane Askari - Alexander Matyasko - Shawn Tan - Simon Lefrancois - Adam Becker - Vikram - Gijs van Tulder - Faruk Ahmed - Thomas George - erakra - Andrei Costinescu - Boris Fomitchev - Zhouhan LIN - Aleksandar Botev - jhelie - xiaoqie - Tegan Maharaj - Matt Graham - Cesar Laurent - Gabe Schwartz - Juan Camilo Gamboa Higuera - Tim Cooijmans - Anirudh Goyal - Saizheng Zhang - Yikang Shen - vipulraheja - Florian Bordes - Sina Honari - Chiheb Trabelsi - Shubh Vachher - Daren Eiri - Joseph Paul Cohen - Laurent Dinh - Mohamed Ishmael Diwan Belghazi - Jeff Donahue - Ramana Subramanyam - Bogdan Budescu - Dzmitry Bahdanau - Ghislain Antony Vaillant - Jan Schlüter - Nan Jiang - Xavier Bouthillier - fo40225 - mrTsjolder - wyjw - Aarni Koskela - Adam Geitgey - Adrian Keet - Adrian Seyboldt - Anmol Sahoo - Chong Wu - Holger Kohr - Jayanth Koushik - Lilian Besson - Lv Tao - Michael Manukyan - Murugesh Marvel - NALEPA - Rebecca N. Palmer - Zotov Yuriy - dareneiri - lrast - morrme - naitonium Theano 0.10.0beta4 (16th of October, 2017) ========================================== This release contains new features, improvements and bug fixes to prepare the upcoming release candidate. We recommend that every developer updates to this version. Highlights: - Announcing that `MILA will stop developing Theano `_ - Bug fixes, crash fixes, warning improvements and documentation updates A total of 70 people contributed to this release since 0.9.0, see list below. Interface changes: - Generalized ``AllocDiag`` for any non-scalar input Convolution updates: - Implemented fractional bilinear upsampling cuDNN (GPU): - Disallowed ``float16`` precision for convolution gradients - Fixed memory alignment detection - Added profiling in C debug mode (with theano flag ``cmodule.debug=True``) New features: - Implemented truncated normal distribution with box-muller transform - Added ``L_op()`` overriding option for ``OpFromGraph`` - Added NumPy C-API based fallback implementation for ``[sd]gemv_`` and ``[sd]dot_`` Other more detailed changes: - Improved stack trace follow-up for GPU optimizations - Fixed gradient error for elemwise ``minimum`` and ``maximum`` when compared values are the same - Fixed gradient for ``ARange`` - Removed ``ViewOp`` subclass during optimization Commiters since 0.9.0: - Frederic Bastien - João Victor Tozatti Risso - Arnaud Bergeron - Steven Bocco - Mohammed Affan - amrithasuresh - Pascal Lamblin - Reyhane Askari - Alexander Matyasko - Shawn Tan - Simon Lefrancois - Adam Becker - Vikram - Gijs van Tulder - Faruk Ahmed - Thomas George - erakra - Andrei Costinescu - Boris Fomitchev - Zhouhan LIN - Aleksandar Botev - jhelie - xiaoqie - Tegan Maharaj - Matt Graham - Cesar Laurent - Gabe Schwartz - Juan Camilo Gamboa Higuera - Tim Cooijmans - Anirudh Goyal - Saizheng Zhang - Yikang Shen - vipulraheja - Florian Bordes - Sina Honari - Chiheb Trabelsi - Shubh Vachher - Daren Eiri - Joseph Paul Cohen - Laurent Dinh - Mohamed Ishmael Diwan Belghazi - Jeff Donahue - Ramana Subramanyam - Bogdan Budescu - Dzmitry Bahdanau - Ghislain Antony Vaillant - Jan Schlüter - Nan Jiang - Xavier Bouthillier - fo40225 - mrTsjolder - wyjw - Aarni Koskela - Adam Geitgey - Adrian Keet - Adrian Seyboldt - Anmol Sahoo - Chong Wu - Holger Kohr - Jayanth Koushik - Lilian Besson - Lv Tao - Michael Manukyan - Murugesh Marvel - NALEPA - Zotov Yuriy - dareneiri - lrast - morrme - naitonium Theano 0.10.0beta3 (20th of September, 2017) ============================================ This release contains new features, improvements and bug fixes to prepare the upcoming release candidate. We recommend that every developer updates to this version. Highlights: - conda packages now available and updated in our own conda channel ``mila-udem``. To install it: ``conda install -c mila-udem -c mila-udem/label/pre theano pygpu`` - Improved elemwise operations - Speed-up elemwise ops based on SciPy - Fixed memory leak related to elemwise ops on GPU - Fixed pygpu detection - Bug fixes, crash fixes, warning improvements and documentation updates A total of 69 people contributed to this release since 0.9.0, see list below. Interface changes: - Removed op ``ExtractDiag`` from ``theano.tensor.nlinalg``, now only in ``theano.tensor.basic`` Convolution updates: - Added dilated causal convolutions for 2D New features: - Implemented ``topk`` and ``argtopk`` on CPU and GPU - Added ``unravel_index`` and ``ravel_multi_index`` functions on CPU - Implemented ``max()`` and ``min()`` functions for booleans and unsigned integers types Others: - Added ``R_op()`` for ``ZeroGrad`` - Added description for rnnblock Commiters since 0.9.0: - Frederic Bastien - João Victor Tozatti Risso - Arnaud Bergeron - Steven Bocco - Mohammed Affan - amrithasuresh - Pascal Lamblin - Reyhane Askari - Alexander Matyasko - Simon Lefrancois - Adam Becker - Shawn Tan - Vikram - Gijs van Tulder - Thomas George - Andrei Costinescu - Faruk Ahmed - Boris Fomitchev - Zhouhan LIN - Aleksandar Botev - jhelie - xiaoqie - Tegan Maharaj - Matt Graham - Cesar Laurent - Gabe Schwartz - Juan Camilo Gamboa Higuera - Tim Cooijmans - Anirudh Goyal - Saizheng Zhang - Yikang Shen - vipulraheja - Florian Bordes - Sina Honari - erakra - Chiheb Trabelsi - Shubh Vachher - Daren Eiri - Joseph Paul Cohen - Laurent Dinh - Mohamed Ishmael Diwan Belghazi - Jeff Donahue - Ramana Subramanyam - Bogdan Budescu - Dzmitry Bahdanau - Ghislain Antony Vaillant - Jan Schlüter - Nan Jiang - Xavier Bouthillier - fo40225 - wyjw - Aarni Koskela - Adam Geitgey - Adrian Keet - Adrian Seyboldt - Anmol Sahoo - Chong Wu - Holger Kohr - Jayanth Koushik - Lilian Besson - Lv Tao - Michael Manukyan - Murugesh Marvel - NALEPA - Zotov Yuriy - dareneiri - lrast - morrme - naitonium Theano 0.10.0beta2 (7th of September, 2017) =========================================== This release contains new features, improvements and bug fixes to prepare the upcoming release candidate. We recommend that every developer updates to this version. Highlights: - Support NumPy ``1.13`` - Support pygpu ``0.7`` - Added conda recipe - Optional faster optimization step with new destroy handler - Added documentation for RNNBlock - Bug fixes, crash fixes, warning improvements and documentation updates A total of 67 people contributed to this release since 0.9.0, see list below. Interface changes: - Added new parameter ``target`` for MRG functions Convolution updates: - Added unshared convolutions - Added 3D separable convolutions - Added 3D grouped convolutions - Removed old ``conv3d`` interface - Deprecated old ``conv2d`` interface - Updated ``conv`` documentation GPU: - Added a meta-optimizer to select the fastest GPU implementations for convolutions - cuDNN: - Official support for ``v6.*`` and ``v7.*``, support for ``v5.*`` will be removed in next release - Added spatial transformation operation based on cuDNN - Updated and improved caching system for runtime-chosen cuDNN convolution algorithms - Support cuDNN v7 tensor core operations for convolutions with runtime timed algorithms - Restricted cuDNN reductions to contiguous inputs - Automatic addition of cuDNN DLL path to ``PATH`` environment variable on Windows New features: - Added ``tensor6()`` and ``tensor7()`` in ``theano.tensor`` module - Added boolean indexing for sub-tensors - Added covariance matrix function ``theano.tensor.cov`` - Added new Theano flag ``pickle_test_value`` to help disable pickling test values Others: - Kept stack trace for optimizations in new GPU backend Other more detailed changes: - Moved all C code files into separate folder ``c_code`` in every Theano module - Improvements for Jenkins tests Commiters since 0.9.0: - Frederic Bastien - João Victor Tozatti Risso - Arnaud Bergeron - Steven Bocco - Mohammed Affan - amrithasuresh - Pascal Lamblin - Reyhane Askari - Alexander Matyasko - Simon Lefrancois - Shawn Tan - Gijs van Tulder - Thomas George - Vikram - Andrei Costinescu - Faruk Ahmed - Boris Fomitchev - Zhouhan LIN - Aleksandar Botev - jhelie - xiaoqie - Tegan Maharaj - Matt Graham - Cesar Laurent - Gabe Schwartz - Juan Camilo Gamboa Higuera - Tim Cooijmans - Anirudh Goyal - Saizheng Zhang - vipulraheja - Florian Bordes - Sina Honari - Yikang Shen - erakra - Chiheb Trabelsi - Shubh Vachher - Daren Eiri - Joseph Paul Cohen - Laurent Dinh - Mohamed Ishmael Diwan Belghazi - Jeff Donahue - Ramana Subramanyam - Bogdan Budescu - Dzmitry Bahdanau - Ghislain Antony Vaillant - Jan Schlüter - Xavier Bouthillier - fo40225 - Aarni Koskela - Adam Becker - Adam Geitgey - Adrian Keet - Adrian Seyboldt - Anmol Sahoo - Chong Wu - Holger Kohr - Jayanth Koushik - Lilian Besson - Lv Tao - Michael Manukyan - Murugesh Marvel - NALEPA - Zotov Yuriy - dareneiri - lrast - morrme - wyjw Theano 0.10.0beta1 (9th of August, 2017) ======================================== This release contains a lot of bug fixes, improvements and new features to prepare the upcoming release candidate. We recommend that every developer updates to this version. Highlights: - Moved Python 3.* minimum supported version from 3.3 to 3.4 - Replaced deprecated package ``nose-parameterized`` with up-to-date package ``parameterized`` for Theano requirements - Theano now internally uses ``sha256`` instead of ``md5`` to work on systems that forbide ``md5`` for security reason - Removed old GPU backend ``theano.sandbox.cuda``. New backend ``theano.gpuarray`` is now the official GPU backend - Support more debuggers for ``PdbBreakpoint`` - Scan improvements - Speed up Theano scan compilation and gradient computation - Added meaningful message when missing inputs to scan - Speed up graph toposort algorithm - Faster C compilation by massively using a new interface for op params - Faster optimization step - Documentation updated and more complete - Many bug fixes, crash fixes and warning improvements A total of 65 people contributed to this release since 0.9.0, see list below. Interface changes: - Merged duplicated diagonal functions into two ops: ``ExtractDiag`` (extract a diagonal to a vector), and ``AllocDiag`` (set a vector as a diagonal of an empty array) - Renamed ``MultinomialWOReplacementFromUniform`` to ``ChoiceFromUniform`` - Removed or deprecated Theano flags: - ``cublas.lib`` - ``cuda.enabled`` - ``enable_initial_driver_test`` - ``gpuarray.sync`` - ``home`` - ``lib.cnmem`` - ``nvcc.*`` flags - ``pycuda.init`` - Changed ``grad()`` method to ``L_op()`` in ops that need the outputs to compute gradient Convolution updates: - Extended Theano flag ``dnn.enabled`` with new option ``no_check`` to help speed up cuDNN importation - Implemented separable convolutions - Implemented grouped convolutions GPU: - Prevent GPU initialization when not required - Added disk caching option for kernels - Added method ``my_theano_function.sync_shared()`` to help synchronize GPU Theano functions - Added useful stats for GPU in profile mode - Added Cholesky op based on ``cusolver`` backend - Added GPU ops based on `magma library `_: SVD, matrix inverse, QR, cholesky and eigh - Added ``GpuCublasTriangularSolve`` - Added atomic addition and exchange for ``long long`` values in ``GpuAdvancedIncSubtensor1_dev20`` - Support log gamma function for all non-complex types - Support GPU SoftMax in both OpenCL and CUDA - Support offset parameter ``k`` for ``GpuEye`` - ``CrossentropyCategorical1Hot`` and its gradient are now lifted to GPU - Better cuDNN support - Official support for ``v5.*`` and ``v6.*`` - Better support and loading on Windows and Mac - Support cuDNN v6 dilated convolutions - Support cuDNN v6 reductions - Added new Theano flags ``cuda.include_path``, ``dnn.base_path`` and ``dnn.bin_path`` to help configure Theano when CUDA and cuDNN can not be found automatically. - Updated ``float16`` support - Added documentation for GPU float16 ops - Support ``float16`` for ``GpuGemmBatch`` - Started to use ``float32`` precision for computations that don't support ``float16`` on GPU New features: - Added a wrapper for `Baidu's CTC `_ cost and gradient functions - Added scalar and elemwise CPU ops for modified Bessel function of order 0 and 1 from ``scipy.special``. - Added Scaled Exponential Linear Unit (SELU) activation - Added sigmoid_binary_crossentropy function - Added tri-gamma function - Added modes ``half`` and ``full`` for ``Images2Neibs`` ops - Implemented gradient for ``AbstractBatchNormTrainGrad`` - Implemented gradient for matrix pseudoinverse op - Added new prop `replace` for ``ChoiceFromUniform`` op - Added new prop ``on_error`` for CPU ``Cholesky`` op - Added new Theano flag ``deterministic`` to help control how Theano optimize certain ops that have deterministic versions. Currently used for subtensor Ops only. - Added new Theano flag ``cycle_detection`` to speed-up optimization step by reducing time spending in inplace optimizations - Added new Theano flag ``check_stack_trace`` to help check the stack trace during optimization process - Added new Theano flag ``cmodule.debug`` to allow a debug mode for Theano C code. Currently used for cuDNN convolutions only. Others: - Added deprecation warning for the softmax and logsoftmax vector case - Added a warning to announce that C++ compiler will become mandatory in next Theano release ``0.11`` Other more detailed changes: - Removed useless warning when profile is manually disabled - Added tests for abstract conv - Added options for `disconnected_outputs` to Rop - Removed ``theano/compat/six.py`` - Removed ``COp.get_op_params()`` - Support of list of strings for ``Op.c_support_code()``, to help not duplicate support codes - Macro names provided for array properties are now standardized in both CPU and GPU C codes - Started to move C code files into separate folder ``c_code`` in every Theano module - Many improvements for Travis CI tests (with better splitting for faster testing) - Many improvements for Jenkins CI tests: daily testings on Mac and Windows in addition to Linux Commiters since 0.9.0: - Frederic Bastien - Arnaud Bergeron - amrithasuresh - João Victor Tozatti Risso - Steven Bocco - Pascal Lamblin - Mohammed Affan - Reyhane Askari - Alexander Matyasko - Simon Lefrancois - Shawn Tan - Thomas George - Faruk Ahmed - Zhouhan LIN - Aleksandar Botev - jhelie - xiaoqie - Tegan Maharaj - Matt Graham - Cesar Laurent - Gabe Schwartz - Juan Camilo Gamboa Higuera - AndroidCloud - Saizheng Zhang - vipulraheja - Florian Bordes - Sina Honari - Vikram - erakra - Chiheb Trabelsi - Shubh Vachher - Daren Eiri - Gijs van Tulder - Laurent Dinh - Mohamed Ishmael Diwan Belghazi - mila - Jeff Donahue - Ramana Subramanyam - Bogdan Budescu - Ghislain Antony Vaillant - Jan Schlüter - Xavier Bouthillier - fo40225 - Aarni Koskela - Adam Becker - Adam Geitgey - Adrian Keet - Adrian Seyboldt - Andrei Costinescu - Anmol Sahoo - Chong Wu - Holger Kohr - Jayanth Koushik - Jenkins - Lilian Besson - Lv Tao - Michael Manukyan - Murugesh Marvel - NALEPA - Ubuntu - Zotov Yuriy - dareneiri - lrast - morrme - yikang Theano 0.9.0 (20th of March, 2017) ================================== This is a final release of Theano, version ``0.9.0``, with a lot of new features, interface changes, improvements and bug fixes. We recommend that everybody update to this version. Highlights (since 0.8.0): - Better Python 3.5 support - Better numpy 1.12 support - Conda packages for Mac, Linux and Windows - Support newer Mac and Windows versions - More Windows integration: - Theano scripts (``theano-cache`` and ``theano-nose``) now works on Windows - Better support for Windows end-lines into C codes - Support for space in paths on Windows - Scan improvements: - More scan optimizations, with faster compilation and gradient computation - Support for checkpoint in scan (trade off between speed and memory usage, useful for long sequences) - Fixed broadcast checking in scan - Graphs improvements: - More numerical stability by default for some graphs - Better handling of corner cases for theano functions and graph optimizations - More graph optimizations with faster compilation and execution - smaller and more readable graph - New GPU back-end: - Removed warp-synchronous programming to get good results with newer CUDA drivers - More pooling support on GPU when cuDNN isn't available - Full support of ignore_border option for pooling - Inplace storage for shared variables - float16 storage - Using PCI bus ID of graphic cards for a better mapping between theano device number and nvidia-smi number - Fixed offset error in ``GpuIncSubtensor`` - Less C code compilation - Added support for bool dtype - Updated and more complete documentation - Bug fixes related to merge optimizer and shape inference - Lot of other bug fixes, crashes fixes and warning improvements A total of 123 people contributed to this release since 0.8.0, see list below. Interface changes: - Merged ``CumsumOp/CumprodOp`` into ``CumOp`` - In MRG module: - Replaced method ``multinomial_wo_replacement()`` with new method ``choice()`` - Random generator now tries to infer the broadcast pattern of its output - New pooling interface - Pooling parameters can change at run time - Moved ``softsign`` out of sandbox to ``theano.tensor.nnet.softsign`` - Using floatX dtype when converting empty list/tuple - ``Roll`` make the shift be modulo the size of the axis we roll on - ``round()`` default to the same as NumPy: half_to_even Convolution updates: - Support of full and half modes for 2D and 3D convolutions including in ``conv3d2d`` - Allowed pooling of empty batch - Implement ``conv2d_transpose`` convenience function - Multi-cores convolution and pooling on CPU - New abstract 3d convolution interface similar to the 2d convolution interface - Dilated convolution GPU: - cuDNN: support versoin 5.1 and wrap batch normalization (2d and 3d) and RNN functions - Multiple-GPU, synchrone update (via platoon, use NCCL) - Gemv(matrix-vector product) speed up for special shape - cublas gemv workaround when we reduce on an axis with a dimensions size of 0 - Warn user that some cuDNN algorithms may produce unexpected results in certain environments for convolution backward filter operations - ``GPUMultinomialFromUniform`` op now supports multiple dtypes - Support for ``MaxAndArgMax`` for some axis combination - Support for solve (using cusolver), erfinv and erfcinv - Implemented ``GpuAdvancedSubtensor`` New features: - ``OpFromGraph`` now allows gradient overriding for every input - Added Abstract Ops for batch normalization that use cuDNN when available and pure Theano CPU/GPU alternatives otherwise - Added gradient of solve, tensorinv (CPU), tensorsolve (CPU), searchsorted (CPU), DownsampleFactorMaxGradGrad (CPU) - Added Multinomial Without Replacement - Allowed partial evaluation of compiled function - More Rop support - Indexing support ellipsis: ``a[..., 3]```, ``a[1,...,3]`` - Added ``theano.tensor.{tensor5,dtensor5, ...}`` - compiledir_format support device - Added New Theano flag ``conv.assert_shape`` to check user-provided shapes at runtime (for debugging) - Added new Theano flag ``cmodule.age_thresh_use`` - Added new Theano flag ``cuda.enabled`` - Added new Theano flag ``nvcc.cudafe`` to enable faster compilation and import with old CUDA back-end - Added new Theano flag ``print_global_stats`` to print some global statistics (time spent) at the end - Added new Theano flag ``profiling.ignore_first_call``, useful to profile the new gpu back-end - remove ProfileMode (use Theano flag ``profile=True`` instead) Others: - Split op now has C code for CPU and GPU - ``theano-cache list`` now includes compilation times - Speed up argmax only on GPU (without also needing the max) - More stack trace in error messages - Speed up cholesky grad - ``log(sum(exp(...)))`` now get stability optimized Other more detailed changes: - Added Jenkins (gpu tests run on pull requests in addition to daily buildbot) - Removed old benchmark directory and other old files not used anymore - Use of 64-bit indexing in sparse ops to allow matrix with more then 2\ :sup:`31`\ -1 elements - Allowed more then one output to be an destructive inplace - More support of negative axis - Added the keepdims parameter to the norm function - Make scan gradient more deterministic Commiters since 0.8.0: - Frederic Bastien - Arnaud Bergeron - Pascal Lamblin - Steven Bocco - Ramana Subramanyam - Simon Lefrancois - Gijs van Tulder - Benjamin Scellier - khaotik - Chiheb Trabelsi - Chinnadhurai Sankar - Cesar Laurent - Reyhane Askari - Mohammad Pezeshki - Alexander Matyasko - Alexandre de Brebisson - Mathieu Germain - Nan Rosemary Ke - Pierre Luc Carrier - Olivier Mastropietro - Thomas George - Saizheng Zhang - Iulian Vlad Serban - Francesco Visin - Caglar - Faruk Ahmed - Harm de Vries - Samira Shabanian - Vincent Dumoulin - Nicolas Ballas - Jakub Sygnowski - Jan Schlüter - Samira Ebrahimi Kahou - Mikhail Korobov - Fei Wang - Kv Manohar - Jesse Livezey - Kelvin Xu - Matt Graham - Ruslana Makovetsky - Sina Honari - Bryn Keller - Ciyong Chen - Vitaliy Kurlin - Zhouhan LIN - Gokula Krishnan - Kumar Krishna Agrawal - Ozan ÇaÄŸlayan - Vincent Michalski - affanv14 - Amjad Almahairi - Ray Donnelly - Tim Cooijmans - happygds - mockingjamie - Christos Tsirigotis - Florian Bordes - Ilya Kulikov - RadhikaG - Taesup (TS) Kim - Ying Zhang - Anton Chechetka - Karthik Karanth - Kirill Bobyrev - Rebecca N. Palmer - Yang Zhang - Yaroslav Ganin - Jonas Degrave - Liwei Cai - Lucas Beyer - Michael Harradon - Morgan Stuart - Tim Gasper - Xavier Bouthillier - p - texot - Andrés Gottlieb - Ben Poole - Bhavishya Pohani - Carl Thomé - David Bau - Dimitar Dimitrov - Evelyn Mitchell - Fei Zhan - Fuchai - Fábio Perez - Gennadiy Tupitsin - Gilles Louppe - Greg Ciccarelli - He - Huan Zhang - Kaixhin - Kevin Keraudren - Maltimore - Marc-Alexandre Cote - Marco - Marius F. Killinger - Martin Drawitsch - Maxim Kochurov - Micah Bojrab - Neil - Nizar Assaf - Rithesh Kumar - Rizky Luthfianto - Robin Millette - Roman Ring - Sander Dieleman - Sebastin Santy - Shawn Tan - Wazeer Zulfikar - Wojciech GÅ‚ogowski - Yann N. Dauphin - gw0 [http://gw.tnode.com/] - hexahedria - hsintone - jakirkham - joncrall - root - superantichrist - tillahoffmann - valtron - wazeerzulfikar - you-n-g Theano 0.9.0rc4 (13th of March, 2017) ===================================== This release extends the 0.9.0rc3 and announces the upcoming final release 0.9. Highlights (since 0.9.0rc3): - Documentation updates - DebugMode fixes, cache cleanup fixes and other small fixes - New GPU back-end: - Fixed offset error in GpuIncSubtensor - Fixed indexing error in GpuAdvancedSubtensor for more than 2 dimensions A total of 5 people contributed to this release since 0.9.0rc3 and 123 since 0.8.0, see the lists below. Committers since 0.9.0rc3: - Frederic Bastien - Pascal Lamblin - Arnaud Bergeron - Cesar Laurent - Martin Drawitsch Theano 0.9.0rc3 (6th of March, 2017) ==================================== This release extends the 0.9.0rc2 and announces the upcoming final release 0.9. Highlights (since 0.9.0rc2): - Graph clean up and faster compilation - New Theano flag conv.assert_shape to check user-provided shapes at runtime (for debugging) - Fix overflow in pooling - Warn if taking softmax over broadcastable dimension - Removed old files not used anymore - Test fixes and crash fixes - New GPU back-end: - Removed warp-synchronous programming, to get good results with newer CUDA drivers A total of 5 people contributed to this release since 0.9.0rc2 and 122 since 0.8.0, see the lists below. Committers since 0.9.0rc2: - Frederic Bastien - Arnaud Bergeron - Pascal Lamblin - Florian Bordes - Jan Schlüter Theano 0.9.0rc2 (27th of February, 2017) ======================================== This release extends the 0.9.0rc1 and announces the upcoming final release 0.9. Highlights (since 0.9.0rc1): - Fixed dnn conv grad issues - Allowed pooling of empty batch - Use of 64-bit indexing in sparse ops to allow matrix with more then 2\ :sup:`31`\ -1 elements. - Removed old benchmark directory - Crash fixes, bug fixes, warnings improvements, and documentation update A total of 9 people contributed to this release since 0.9.0rc1 and 121 since 0.8.0, see the lists below. Committers since 0.9.0rc1: - Frederic Bastien - Pascal Lamblin - Steven Bocco - Simon Lefrancois - Lucas Beyer - Michael Harradon - Rebecca N. Palmer - David Bau - Micah Bojrab Theano 0.9.0rc1 (20th of February, 2017) ======================================== This release extends the 0.9.0beta1 and announces the upcoming final release 0.9. Highlights (since 0.9.0beta1): - Better integration of Theano+libgpuarray packages into conda distribution - Better handling of Windows end-lines into C codes - Better compatibility with NumPy 1.12 - Faster scan optimizations - Fixed broadcast checking in scan - Bug fixes related to merge optimizer and shape inference - many other bug fixes and improvements - Updated documentation - New GPU back-end: - Value of a shared variable is now set inplace A total of 26 people contributed to this release since 0.9.0beta1 and 117 since 0.8.0, see the list at the bottom. Interface changes: - In MRG, replaced method `multinomial_wo_replacement()` with new method `choice()` Convolution updates: - Implement conv2d_transpose convenience function GPU: - GPUMultinomialFromUniform op now supports multiple dtypes New features: - OpFromGraph now allows gradient overriding for every input - Added Abstract Ops for batch normalization that use cuDNN when available and pure Theano CPU/GPU alternatives otherwise - Added new Theano flag cuda.enabled - Added new Theano flag print_global_stats to print some global statistics (time spent) at the end Others: - Split op now has C code for CPU and GPU - "theano-cache list" now includes compilation times Committers since 0.9.0beta1: - Frederic Bastien - Benjamin Scellier - khaotik - Steven Bocco - Arnaud Bergeron - Pascal Lamblin - Gijs van Tulder - Reyhane Askari - Chinnadhurai Sankar - Vincent Dumoulin - Alexander Matyasko - Cesar Laurent - Nicolas Ballas - affanv14 - Faruk Ahmed - Anton Chechetka - Alexandre de Brebisson - Amjad Almahairi - Dimitar Dimitrov - Fuchai - Jan Schlüter - Jonas Degrave - Mathieu Germain - Rebecca N. Palmer - Simon Lefrancois - valtron Theano 0.9.0beta1 (24th of January, 2017) ========================================= This release contains a lot of bug fixes and improvements + new features, to prepare the upcoming release candidate. Highlights: - Many computation and compilation speed up - More numerical stability by default for some graph - Jenkins (gpu tests run on PR in addition to daily buildbot) - Better handling of corner cases for theano functions and graph optimizations - More graph optimization (faster execution and smaller graph, so more readable) - Less c code compilation - Better Python 3.5 support - Better numpy 1.12 support - Support newer Mac and Windows version - Conda packages for Mac, Linux and Windows - Theano scripts now works on Windows - scan with checkpoint (trade off between speed and memory usage, useful for long sequences) - Added a bool dtype - New GPU back-end: - float16 storage - better mapping between theano device number and nvidia-smi number, using the PCI bus ID of graphic cards - More pooling support on GPU when cuDNN isn't there - ignore_border=False is now implemented for pooling A total of 111 people contributed to this release since 0.8.0, see the list at the bottom. Interface changes: - New pooling interface - Pooling parameters can change at run time - When converting empty list/tuple, now we use floatX dtype - The MRG random generator now try to infer the broadcast pattern of its output - Move softsign out of sandbox to theano.tensor.nnet.softsign - Roll make the shift be modulo the size of the axis we roll on - Merge CumsumOp/CumprodOp into CumOp - round() default to the same as NumPy: half_to_even Convolution updates: - Multi-cores convolution and pooling on CPU - New abstract 3d convolution interface similar to the 2d convolution interface - Dilated convolution GPU: - cuDNN: support versoin 5.1 and wrap batch normalization (2d and 3d) and RNN functions - Multiple-GPU, synchrone update (via platoon, use NCCL) - GpuAdvancedSubtensor in new back-end - Gemv(matrix-vector product) speed up for special shape - Support for MaxAndArgMax for some axis combination - Support for solve (using cusolver), erfinv and erfcinv - cublas gemv workaround when we reduce on an axis with a dimensions size of 0 - Warn user that some cuDNN algorithms may produce unexpected results in certain environments for convolution backward filter operations New features: - Add gradient of solve, tensorinv (CPU), tensorsolve (CPU) searchsorted (CPU) - Add Multinomial Without Replacement - conv3d2d support full and half mode (REMOVE?) - Add DownsampleFactorMaxGradGrad.grad - Allow partial evaluation of compiled function - More Rop support - Indexing support ellipsis: a[..., 3], a[1,...,3] - Added theano.tensor.{tensor5,dtensor5, ...} - compiledir_format support device - Added new Theano flag cmodule.age_thresh_use Others: - Speed up argmax only on gpu (without also needing the max) - A few unfrequent bugfix - More stack trace in error message - Speed up cholesky grad - log(sum(exp(...))) now get stability optimized Other more detailed changes: - Allow more then one output to be an destructive inplace - Add flag profiling.ignore_first_call, useful to profile the new gpu back-end - Doc/error message fixes/updates - More support of negative axis - Added the keepdims parameter to the norm function - Crash fixes - Make scan gradient more deterministic - Add support for space in path on Windows - remove ProfileMode (use Theano flag profile=True instead) Committers since 0.8.0: - Frederic Bastien - Arnaud Bergeron - Pascal Lamblin - Ramana Subramanyam - Simon Lefrancois - Steven Bocco - Gijs van Tulder - Cesar Laurent - Chiheb Trabelsi - Chinnadhurai Sankar - Mohammad Pezeshki - Reyhane Askari - Alexander Matyasko - Alexandre de Brebisson - Nan Rosemary Ke - Pierre Luc Carrier - Mathieu Germain - Olivier Mastropietro - khaotik - Saizheng Zhang - Thomas George - Iulian Vlad Serban - Benjamin Scellier - Francesco Visin - Caglar - Harm de Vries - Samira Shabanian - Jakub Sygnowski - Samira Ebrahimi Kahou - Mikhail Korobov - Faruk Ahmed - Fei Wang - Jan Schlüter - Kv Manohar - Jesse Livezey - Kelvin Xu - Matt Graham - Ruslana Makovetsky - Sina Honari - Bryn Keller - Ciyong Chen - Nicolas Ballas - Vitaliy Kurlin - Zhouhan LIN - Gokula Krishnan - Kumar Krishna Agrawal - Ozan ÇaÄŸlayan - Vincent Michalski - Ray Donnelly - Tim Cooijmans - Vincent Dumoulin - happygds - mockingjamie - Amjad Almahairi - Christos Tsirigotis - Ilya Kulikov - RadhikaG - Taesup (TS) Kim - Ying Zhang - Karthik Karanth - Kirill Bobyrev - Yang Zhang - Yaroslav Ganin - Liwei Cai - Morgan Stuart - Tim Gasper - Xavier Bouthillier - p - texot - Andrés Gottlieb - Ben Poole - Bhavishya Pohani - Carl Thomé - Evelyn Mitchell - Fei Zhan - Fábio Perez - Gennadiy Tupitsin - Gilles Louppe - Greg Ciccarelli - He - Huan Zhang - Jonas Degrave - Kaixhin - Kevin Keraudren - Maltimore - Marc-Alexandre Cote - Marco - Marius F. Killinger - Maxim Kochurov - Neil - Nizar Assaf - Rithesh Kumar - Rizky Luthfianto - Robin Millette - Roman Ring - Sander Dieleman - Sebastin Santy - Shawn Tan - Wazeer Zulfikar - Wojciech GÅ‚ogowski - Yann N. Dauphin - gw0 [http://gw.tnode.com/] - hexahedria - hsintone - jakirkham - joncrall - root - superantichrist - tillahoffmann - wazeerzulfikar - you-n-g Theano 0.8.2 (21th of April, 2016) ================================== This is a point release with only the support for cudnn v5 convolution and minor fixes. Highlights: - cuDNN v5 convolution support (cuDNN v3 isn't supported anymore) - A few crash fixes Theano 0.8.1 (29th of March, 2016) ================================== This is a point release without any new feature. It fixes compilation issues on MacOS X with the command line tools for XCode 7.3, which was released shortly after Theano 0.8.0. Theano 0.8 (21th of March, 2016) ================================ We recommend that everybody update to this version. Highlights: - Python 2 and 3 support with the same code base - Faster optimization - Integration of cuDNN for better GPU performance - Many Scan improvements (execution speed up, ...) - optimizer=fast_compile moves computation to the GPU. - Better convolution on CPU and GPU. (CorrMM, cudnn, 3d conv, more parameter) - Interactive visualization of graphs with d3viz - cnmem (better memory management on GPU) - BreakpointOp - Multi-GPU for data parallism via Platoon (https://github.com/mila-udem/platoon/) - More pooling parameter supported - Bilinear interpolation of images - New GPU back-end: * Float16 new back-end (need cuda 7.5) * Multi dtypes * Multi-GPU support in the same process A total of 141 people contributed to this release, see the list at the bottom. Installation: - Better BLAS detection - Fixes for more recent software and OS versions - Support Anaconda on Windows Bug fixes: - GpuJoin now supports negative axis - Fix GpuCumsum for negative axis Interface Deprecation (a warning is printed): - Deprecate Param class, use In instead Interface Changes: - Rename DownsampleFactorMax to Pool. - tensor.stack now uses the same interface as numpy.stack - optimizer=fast_compile moves computation to the GPU - Raise the user stack trace more frequently. - Change dev version numbering to follow the PEP 440 New Interface (reuses existing functionality): - theano.tensor.nnet.relu - theano.tensor.nnet.elu - BatchNormalization. - MaxAndArgmax support axis=None - Add theano.tensor.compress (equivalent of numpy.compress) - theano.tensor.signal.downsamples.max_pool_2d_same_size - COp - __props__ New features - tensor.unique - map_variables - erfcx - mgrid, ogrid - allclose - BreakpointOp - Make bincount work on GPU - SolveOp on GPU - Optional optimization remove_all_assert - AllocEmpty - LogSoftmax, for stability optimization when the crossentropy optimization does not apply. - theano.tensor.repeat works on GPU - BatchedDot on the GPU and faster on the CPU. - Faster batched_tensordot and make it work on GPU. - SoftmaxGrad grad - 3d conv via CorrMM on the GPU - CPU Max Pool support of padding and strides!=windows size - theano.function() now accepts a dict for the outputs. When doing this, the function will return a dict. Helpful to keep track of which output is what. - Warn for unknown or misspelled theano config variables - theano.tensor.tile update (accept symbolic reps, work on GPU) - scan how have a strict flag. If set to True, this make scan building faster and could make execution faster. - theano.tensor.signal.conv2d(2d,2d) output 2d answer - More convolution parameter supported - Bilinear interpolation of images Speed-ups: - Faster SetSubtensor on the GPU. - Support more reduction pattern on the GPU. - More graph optimization - Faster graph optimization - GpuCrossentropySoftmaxArgmax1HotWithBias Crash/no return fixes: - Fix crash in the assert op grad - Fix curand crash on Mac - Multiple Fix scan crashes - Finish to update all Op.grad() implementation to the new interface Others: - Support ARM processor. - Better tests - Code clean up. - Doc updates - doctest and sphinx test in travis - More tests tagged as slow - Better same_shape implementation - More op with c code to lower overhead - Custom pickler for SharedVariable theano.misc.pkl_utils.{dump,load} - function_dump to help us reproduce user error during compilation - assert_no_cpu_op - pep8, flake8 - Better error messages - On non-default modes, reduce the number of allocation when allow_gc=False - Better lock Committers for this dev version only: - Frederic Bastien - Arnaud Bergeron - Pierre Luc Carrier - Iban Harlouchet - Pascal Lamblin - Chienli Ma - Tim Cooijmans - Nicolas Ballas - Amjad Almahairi - David Warde-Farley - Christof Angermueller - Ziye Fan - Caglar - Sina Honari - Roy Xue - hantek - Mohammad Pezeshki - Melanie Ducoffe - Alexandre de Brebisson - Harm de Vries - Samira Shabanian - Alex Lamb - Ramana.S - Francesco Visin - Saizheng Zhang - Ying Zhang - Jan Schlüter - Xavier Bouthillier - Bart van Merrienboer - Cesar Laurent - Iulian Vlad Serban - Li Yao - Sigurd Spieckermann - Dmitrii Serdiuk - Kelvin Xu - Sebastien Jean - Thomas Mesnard - Seon-Wook Park - Vincent Michalski - Dustin Webb - Mikhail Korobov - Orhan Firat - Olivier Mastropietro - Daniel Renshaw - Julien Rebetez - Peng Liu - Sean Lee - TimSalimans - Andre Holzner - Gijs van Tulder - Guillaume Alain - Julien Demouth - Markus Beissinger - Mehdi Mirza - Moslem Kazemi - Saxenauts - Søren Kaae Sønderby - sentient07 - Anatoly Belikov - Diogo Moitinho de Almeida - Jakub Sygnowski - Kashif Rasul - Laurent Dinh - Rémy Léone - Taesup (TS) Kim - gw0 [http://gw.tnode.com/] - mronian - vesis84 - Benni - Chiheb Trabelsi - JesseLivezey - Marius Killinger - Matt Graham - Matthew Willson - Piotr Frankowski - Stefan Krastanov - vdumoulin - Adithya Ganesh - Anish Shah - Balázs Hidasi - Colin Raffel - Cory Lorenz - Doug - Jesse Livezey - John Salvatier - John Zedlewski - Jonathan Ho - Kaixhin - Liang-Chi Hsieh - Lucas Beyer - Luke Metz - Marc-Alexandre Cote - Martin Arjovsky - Matthias Kümmerer - Sirisha Rambhatla - briancheung - cai-lw - ivdorelian - jan-matthis - jojolalpin - joncrall - peterjsadowski - scottsievert - Étienne Simon - A. Flaxman - AlOa - Albert Zeyer - Andrea - Andy Jiang - Balázs - Ben Poole - Brian Cheung - Christophe Van Gysel - Claude Coulombe - Clay McLeod - Dario Garcia - Jakob Lombacher - Joao Felipe Santos - John Arevalo - Jonas Degrave - Martin Thoma - Mathieu Germain - Matthew Koichi Grimes - Michael Eickenberg - Michael Opitz - Paul Hollensen - Prayag Verma - Saatvik Shah - Sergei Lebedev - Vik Kamath - Wei Ouyang - Wojciech GÅ‚ogowski - Yi-Lin Juang - Yurii Shevchuk - Zach Dwiel - dan - eulerreich - jotterbach - rolf - theaverageguy - wuaalb Theano 0.7 (26th of March, 2015) ================================ We recommand to everyone to upgrade to this version. Highlights: * Integration of cuDNN for 2D convolutions and pooling on supported GPUs * Too many optimizations and new features to count * Various fixes and improvements to scan * Better support for GPU on Windows * On Mac OS X, clang is used by default * Many crash fixes * Some bug fixes as well Theano 0.6 (December 3th, 2013) =================================== We recommend that everybody update to this version. Highlights (since 0.6rc5): * Last release with support for Python 2.4 and 2.5. * We will try to release more frequently. * Fix crash/installation problems. * Use less memory for conv3d2d. 0.6rc4 skipped for a technical reason. Highlights (since 0.6rc3): * Python 3.3 compatibility with buildbot test for it. * Full advanced indexing support. * Better Windows 64 bit support. * New profiler. * Better error messages that help debugging. * Better support for newer NumPy versions (remove useless warning/crash). * Faster optimization/compilation for big graph. * Move in Theano the Conv3d2d implementation. * Better SymPy/Theano bridge: Make an Theano op from SymPy expression and use SymPy c code generator. * Bug fixes. Change from 0.6rc5: * Fix crash when specifing march in cxxflags Theano flag. (Frederic B., reported by FiReTiTi) * code cleanup (Jorg Bornschein) * Fix Canopy installation on windows when it was installed for all users: Raingo * Fix Theano tests due to a scipy change. (Frederic B.) * Work around bug introduced in scipy dev 0.14. (Frederic B.) * Fix Theano tests following bugfix in SciPy. (Frederic B., reported by Ziyuan Lin) * Add Theano flag cublas.lib (Misha Denil) * Make conv3d2d work more inplace (so less memory usage) (Frederic B., repoted by Jean-Philippe Ouellet) Committers since 0.5: Frederic Bastien Pascal Lamblin Ian Goodfellow Olivier Delalleau Razvan Pascanu abalkin Arnaud Bergeron Nicolas Bouchard + Jeremiah Lowin + Matthew Rocklin Eric Larsen + James Bergstra David Warde-Farley John Salvatier + Vivek Kulkarni + Yann N. Dauphin Ludwig Schmidt-Hackenberg + Gabe Schwartz + Rami Al-Rfou' + Guillaume Desjardins Caglar + Sigurd Spieckermann + Steven Pigeon + Bogdan Budescu + Jey Kottalam + Mehdi Mirza + Alexander Belopolsky + Ethan Buchman + Jason Yosinski Nicolas Pinto + Sina Honari + Ben McCann + Graham Taylor Hani Almousli Ilya Dyachenko + Jan Schlüter + Jorg Bornschein + Micky Latowicki + Yaroslav Halchenko + Eric Hunsberger + Amir Elaguizy + Hannes Schulz + Huy Nguyen + Ilan Schnell + Li Yao Misha Denil + Robert Kern + Sebastian Berg + Vincent Dumoulin + Wei Li + XterNalz + A total of 51 people contributed to this release. People with a "+" by their names contributed a patch for the first time. Theano 0.6rc5 (November 25th, 2013) =================================== We recommend that everybody update to this version. We plan to release 0.6 in one week if there is no problem introduced with this release candidate. Theano 0.6rc4 was skipped due to a problem with pypi Highlights: * Python 3.3 compatibility with buildbot test for it. * Full advanced indexing support. * Better Windows 64 bit support. * New profiler. * Better error messages that help debugging. * Better support for newer NumPy versions (remove useless warning/crash). * Faster optimization/compilation for big graph. * Move in Theano the Conv3d2d implementation. * Better SymPy/Theano bridge: Make an Theano op from SymPy expression and use SymPy c code generator. * Bug fixes. Committers for this rc5 only: Frederic Bastien Pascal Lamblin Arnaud Bergeron abalkin Olivier Delalleau John Salvatier Razvan Pascanu Jeremiah Lowin Ludwig Schmidt-Hackenberg + Vivek Kulkarni Matthew Rocklin Gabe Schwartz James Bergstra Sigurd Spieckermann + Bogdan Budescu + Mehdi Mirza + Nicolas Bouchard Ethan Buchman + Guillaume Desjardins Ian Goodfellow Jason Yosinski Sina Honari + Ben McCann + David Warde-Farley Ilya Dyachenko + Jan Schluter + Micky Latowicki + Yaroslav Halchenko + Alexander Belopolsky Hannes Schulz + Huy Nguyen + Robert Kern + Sebastian Berg + Vincent Dumoulin + Wei Li + XterNalz + A total of 36 people contributed to this release. People with a "+" by their names contributed a patch for the first time. Installation: * Canopy support (direct link to MKL): * On Linux and Mac OSX (Frederic B., Robert Kern) * On Windows (Edward Shi, Frederic B.) * Anaconda instructions (Pascal L., Frederic B.) * Doc Ubuntu 13.04 (Frederic B.) * Better support of newer NumPy version(remove useless warning/crash) (Frederic B., Huy Nguyen) Bug fixes: * Scan: if a scan node was cloned (by theano.clone) with different inputs, and if both the initial and the cloned nodes are used in the function being compiled, the value of the outputs of one would be replaced with the outputs of the other one. (Pascal L.) * Sparse: Disable the optimization that introduce the CSMGradC op as it doesn't work correctly with unsorted indices. (Frederic B.) * Mac: Fix wrong result of GpuDownsampleFactorMaxGrad on Mac OSX. (Pascal L.) * Mac: Auto-Detect and work around a bug in BLAS on MacOS X (Pascal L.) * Mac: Work around bug in MacOS X. If 2 compiled modules had the same name, the OS or Python was not always the right one even when we used the right handle to it. (Pascal L.) Use this hash in the Python module, and in %(nodename)s, so that different helper functions in the support code for different Ops will always have different names. * Sparse grad: Fix ConstructSparseFromList.infer_shape (Pascal L., reported by Rami Al-Rfou') * (introduced in the development version after 0.6rc3 release) (Frederic B.) Reduction that upcasts the input on no axis (ex: call theano.sum() on a scalar when the original dtype isn't float64 or [u]int64). It produced bad results as we did not upcasted the inputs in the code, we just copy them. * Fix some cases of theano.clone() when we get a replacement of x that is a function of x. (Razvan P., reported by Akio Takano) * Fix grad of Alloc when we unbroadcast the value and it isn't a scalar. (Frederic B., reported Ian G.) * In some cases (I think most cases), there was an exception raised in the theano.tensor.grad() method. But in theory, there could be bad shapes produced in the unbroadcasted dimensions. Interface Deprecation (a warning is printed): * The mode ProfileMode is now deprecated, use the Theano flag profile=True to replace it. * New theano.sparse_grad() interface to get the sparse grad of a_tensor[an_int_vector]. (Frederic B.) This can speed up the sparse computations when a small fraction of a_tensor is taken. Deprecate the old interface for this. (Frederic B.) Interface Changes: * Interface change subtensor and take are not in tensor.basic anymore. They were available from tensor.* and are still available from there. (Frederic B., Matthew Rocklin) * This lowers the basic.py size to 191k, so under 200k for github search. * Add -m32 or -m64 in the module cache key and add the python bitwidth in the compiledir path. (Pascal L.) * mrg.normal now has the parameter size mandatory. It was crashing with the default value of None. (Olivier D.) * Remove the deprecated passing of multiple modes to theano function. (Frederic B.) * Change FunctionGraph Features interface of the {on_prune(),on_import()} call back to take a reason. (Frederic B.) * FunctionGraph now clone the input graph by default. (Frederic B.) * Added a parameter to optionally not do this cloning. * This was needed to speed up compilation New Interface (reuses existing functionality): * Add hostname as a var in compiledir_format (Frederic B.) * Add a new Theano flag: compute_test_value_opt. It takes the same values as compute_test_value. It enables compute_test_value during Theano optimization. Only useful to debug Theano optimization. Also small changes to some optimization to work correctly in that setup. (Frederic B.) * Add the value pdb to the Theano flag: compute_test_value and compute_test_value_opt. (Frederic B.) * Add the Theano flag: optimizer_verbose. Default False. When True, we print all the optimization being applied.(Frederic B.) * Add Op.c_init_code() to allow running the code when the c cmodule is imported (Pascal L.) * Allow theano.tensor.ones(3) to support scalar and not just list of scalar as numpy.ones (Jeremiah Lowin) * Make the memory profiler print the FLOPS used for the ops that know how to compute it. (Frederic B.) New Features: * Make tensor.{constant,as_tensor_variable} work with memmap. (Christian Hudon, Frederic Bastien) * compilation work on ARM processor (Raspberry Pi, Vincent Dumoulin) * Add numpy.random.choice wrapper to our random number generator (Sigurd Spieckermann) * Better SymPy/Theano bridge: Make an Theano op from SymPy expression and use SymPy c code generator (Matthew Rocklin) * Move in Theano the Conv3d2d implementation (James Bergstra, Frederic B., Pascal L.) * First version of the new GPU back-end available (Arnaud Bergeron, Frederic B.) * Not all Ops have been converted to this new back-end. To use, use Theano flag device=cudaN or device=openclN, where N is a integer. * Python 3.3 compatible (abalkin, Gabe Schwartz, Frederic B., Pascal L.) * A new profiler (Frederic B.) The new profiler now can profile the memory with the Theano flag profile_memory=True. The ProfileMode now can't profile memory anymore and prints a message about it. Now we raise an error if we try to profile when the gpu is enabled if we didn't set correctly the env variable to force the driver to sync the kernel launch. Otherwise the profile information are useless. The new profiler supports the enabling/disabling of the garbage collection. * Adds tensor.tri, tensor.triu, and tensor.tril functions that wrap Numpy equivalents (Jeremiah Lowin) * Adds tensor.nonzero, tensor.flatnonzero functions that wrap Numpy equivalents (Jeremiah Lowin) * Adds tensor.nonzero_values to get around lack of advanced indexing for nonzero elements (Jeremiah Lowin) * Make {inc,set}_subtensor work on output of take. (Pascal L.) * When device=cpu and force_device=True, force that we disable the gpu. (Frederic B.) * Better Windows 64 bit support for indexing/reshaping (Pascal L.) * Full advanced indexing support (John Salvatier, seberg) * Add theano.tensor.stacklist(). Recursivly stack lists of tensors to maintain similar structure (Matthew R.) * Add Theano flag value: on_opt_error=pdb (Olivier D.) * GpuSoftmax[WithBias] for bigger row. (Frederic B.) * Make Erfinv work on the GPU (Guillaume Desjardin, Pascal L.) * Add "theano-cache basecompiledir purge" (Pascal L.) This purges all the compiledirs that are in the base compiledir. * A_tensor_variable.zeros_like() now supports the dtype parameter (Pascal L.) * More stable reduce operations by default (Pascal L.) Add an accumulator dtype to CAReduceDtype (acc_dtype) by default, acc_dtype is float64 for float32 inputs, then cast to specified output dtype (float32 for float32 inputs) * Test default blas flag before using it (Pascal L.) This makes it work correctly by default if no blas library is installed. * Add cuda.unuse() to help tests that need to enable/disable the GPU (Frederic B.) * Add theano.tensor.nnet.ultra_fast_sigmoid and the opt (disabled by default) local_ultra_fast_sigmoid. (Frederic B.) * Add theano.tensor.nnet.hard_sigmoid and the opt (disabled by default) local_hard_sigmoid. (Frederic B.) * Add class theano.compat.python2x.Counter() (Mehdi Mirza) * Allow a_cuda_ndarray += another_cuda_ndarray for 6d tensor. (Frederic B.) * Make the op ExtractDiag work on the GPU. (Frederic B.) * New op theano.tensor.chi2sf (Ethan Buchman) * Lift Flatten/Reshape toward input on unary elemwise. (Frederic B.) This makes the "log(1-sigmoid) -> softplus" stability optimization being applied with a flatten/reshape in the middle. * Make MonitorMode use the default optimizers config and allow it to change used optimizers (Frederic B.) * Add support for ScalarOp.c_support_code in GpuElemwise. (Frederic B.) * Also make the Psi function run on GPU. (Frederic B.) * Make tensor.outer(x,y) work when ndim != 1 as numpy.outer. * Kron op: Speed up/generalize/GPU friendly. (Frederic B.) (It is not an op anymore, but reuses current op) * Add gpu max for pattern (0, 1) and added all gpu max pattern for gpu min. (Frederic B.) * Add GpuEye (Frederic B.) * Make GpuCrossentropySoftmaxArgmax1HotWithBias and GpuCrossentropySoftmax1HotWithBiasDx work for bigger inputs (Frederic B., reported by Ryan Price) * Finish and move out of sandbox theano.sparse.basic.true_dot (Nicolas Bouchard, Frederic B.) And document all sparse dot variants. * Implement the mode ignore_borders for GpuImages2Neibs (Frederic B.) * Make many reduction functions accept a numpy scalar as axis (Jeremiah Lowin) * Allow numpy.asarray(cuda_ndarray, dtype=...) (Frederic B.) * theano-cache cleanup now remove cached module old version of code. (Frederic B.) Speed-ups: * Optimizer speed up. (Frederic B.) * Fix warning on newer llvm version on Mac. (Pascal L., reported by Jeremiah Lowin and Chris Fonnesbeck) * Allow pickling of more Ops to allow reusing the compiled code (Pascal L., Frederic B.) * Optimize more cases of dot22 and scalar when we can't make a gemm (Pascal L., Frederic B.) * Speed up GpuJoin with c code (Ludwig Schmidt-Hackenberg, Frederic B.) * Faster GpuAdvancedIncSubtensor1 on Fermi GPU (and up) on matrix. (Vivek Kulkarni) * Faster GPUAdvancedIncSubtensor1 in some cases on all GPU (Vivek Kulkarni) * Implemented c_code for AdvancedSubtensor1 (abalkin) * Add the equivalent of -march=native to g++ command line. (Frederic B., Pascal L.) * Speed up compilation with Scan (Jan Schluter) * Merge more Scan nodes together (Pascal L., Yao Li). * Add MakeVector.c_code (Frederic B.) * Add Shape.c_code (Frederic B.) * Optimize Elemwise when all the inputs are fortran (Frederic B.) We now generate a fortran output and use vectorisable code. * Add ScalarOp.c_code_contiguous interface and do a default version. (Frederic B.) This could optimize elemwise by helping the compiler generate SIMD instruction. * Use ScalarOp.c_code_contiguous with amdlibm. (Frederic B.) This speeds up exp, pow, sin, cos, log, log2, log10 and sigmoid when the input is contiguous in memory. * A fix that removes a local_setsubtensor_of_allocs optimization warning and enables it in that case. (Frederic B., reported by John Salvatier) * Make inv_as_solve optimization work (Matthew Rocklin) Crash/no return fixes: * Fix scan crash in the grad of grad of a scan with special structure (including scan in a scan) (Razvan P., Bitton Tenessi) * Fix various crashes when calling scan() with inputs specified in unusual ways. (Pascal L.) * Fix shape crash inserted by Scan optimization. The gradient of some recursive scan was making the PushOutSeqScan optimization insert crash during the execution of a Theano function. (Frederic B., reported by Hugo Larochelle) * Fix command not returning with recent mingw64 on Windows (Pascal L., reported by many people) * Fix infinite loop related to Scan on the GPU. (Pascal L.) * Fix infinite loop when the compiledir is full. (Frederic B.) * Fix a shape cycle crash in the optimizer (Pascal L., Frederic B., reported by Cho KyungHyun) * Fix MRG normal() now allow it to generate scalars. (Pascal L.) * Fix some GPU compilation issue on Mac (John Yani, Frederic B.) * Fix crash when building symbolic random variables with a mix of symbolic and numeric scalar in the "size" parameter. (Pascal L., Reported by Wu Zhen Zhou) * Make some Op.grad() implementions not return None (Pascal L.) * Crash fix in the grad of elemwise about an DisconnectedType (Pascal L, reported by Thomas Wiecki) * Fix local_gpu_multinomial optimization handling of broadcast information. (Frederic B., reported by Caglar) * Fix crash with change introduced in NumPy 1.7.1 (Pascal L., reported by Thomas Wiecki) * Compilation failure with complex (Pascal L., reported by autumncat) * Gpu reduction on all dimensions of a 4d tensor. (Frederic B., reported by Arjun Jain) * Fix crash for a combination of grad of dot and dimshuffle when only one of the inputs for a corresponding dimensions was knowing that it was broadcastable. (Frederic B., reported by Micky Latowicki) * AdvancedSubtensor1: allow broadcasted index vector. (Frederic B., reported by Jeremiah Lowin) * Fix compute_test_value for ifelse (Olivier D., reported by Bitton Tenessi) * Fix import error with some versions of NumPy (Olivier D.) * Fix Scan grad exception (Razvan P., reported by Nicolas BL) * Fix compute_test_value for a non_sequence when calling the gradient of Scan (Pascal L., reported by Bitton Tenessi). * Crash fix in Scan following interface change in 0.6rc2 (Razvan P.) * Crash fix on Scan (Razvan P.) * Crash fix on Scan (Pascal L., reported by Sina Honari and Sigurd) * Fix crash in Scan gradient related to compute_test_value (Frederic B., reported by Bitton Tenessi) * Fix a scan optimization warning/error depending of Theano flags (Frederic B.) * Fixed crash for unimplemented elemwise gradient (Olivier D., reported by Michael McNeil Forbes) * Fix crash in the elemwise python code for some big shape with power of 2. (Sina Honari, Pascal L.) * Fix compile and import errors on Windows including for the GPU. (Bogdan Budescu) * Fix GPU compilation on Windows (XterNalz) * Fix local_abs_merge optimization crash (Pascal L., reported by Jeremiah Lowin) * Fix import theano crash when g++ isn't there (Olivier D.) * Fix crash related to rebuild of Theano graph (Pascal L., reported by Divine Eguzouwa) * Fix crash during compilation (David Ward-Farley) * Crash fix in the grad of GPU op in corner case (Pascal L.) * Crash fix on MacOS X (Robert Kern) * theano.misc.gnumpy_utils.garray_to_cudandarray() set strides correctly for dimensions of 1. (Frederic B., reported by Justin Bayer) * Fix crash during optimization with consecutive sums and some combination of axis (Frederic B., reported by Caglar Gulcehre) * Fix crash with keepdims and negative axis (Frederic B., reported by David W.-F.) * Fix crash of theano.[sparse.]dot(x,y) when x or y is a vector. (Frederic B., reported by Zsolt Bitvai) * Fix opt crash/disabled with ifelse on the gpu (Frederic B, reported by Ryan Price) * Fix crash in optimization involving dot22, (Pascal L., reported by @micklat) * Prevent shape optimizations from introducing cycles in the graph (Frederic Bastien, Pascal Lamblin, reported by Kyunghyun Cho) Others: * Update/Fixes/Typo/pep8 documentation and/or tutorial (Olivier D., David W.-F., Frederic B., Yaroslav Halchenko, Micky Latowicki, Ben McCann, Jason Yosinski, reported by Arnaud Bergeron) * Doc how to make a sparse Op. (Frederic B.) * Doc compatibility guide (abalkin) * Fix problem in remove_constants_and_unused_inputs_scan. (useless warning and maybe slow down) (Pascal L.) * Fix rop dot.(Razvan P., reported by Jeremiah Lowin) * Raise better error related to pydot bug. (Frederic B., reported by Jason Yosinski and Ludwig Schmidt-Hackenberg) * Fix to Theano tutorial examples. (reported by Ilya Dyachenko) * Fix SharedVar.value property to make it raise an exception (Frederic B., reported by Drew Duncan) * Fix verification with compute_test_value in grad() (Frederic B.) * Theano flags are now evaluated lazily, only if requested (Frederic B.) * Fix test when g++ is not avail (Frederic B.) * Add manual instructions for OpenBLAS on Ubuntu by (Jianri Li ) * Better/more error messages (Frederic B., Pascal L., Ian Goodfellow) * Fix Error reporting with GpuConv (Frederic B., reported by Heng Luo and Nicolas Pinto) * Now travis-ci tests with scipy the parts that need it (Frederic B.) * Export some functions that work on CudaNdarray for windows (Frederic B.) * If the user specifies a -arch=sm_* value in the Theano flags for the gpu, don't add one (Frederic B., Pascal L.) * If a C thunk returns an error, check if a python exception is set. Otherwise, set a default one (Pascal L.) * Crash fix introduced in the development version (Wei LI) * Added BLAS benchmark result (Frederic B., Ben McCann) * Fix code comment (Hannes Schulz) * More stable tests (Frederic B.) * Add utt.asset_allclose(a, b) to have better error message. (Frederic B.) * Better error message with compute_test_value (Frederic, reported by John Salvatier) * Stochastic order behavior fix (Frederic B.) * Simpler initial graph for subtensor infer shape (Olivier D.) The optimization was doing the optimization, but this allows better reading of the graph before optimization. * Better detection of non-aligned ndarray (Frederic B.) * Update MRG multinomial gradient to the new interface (Mehdi Mirza) * Implement Image2Neibs.perform() to help debug (Frederic B.) * Remove some Theano flags from the compilation key (Frederic B.) * Make theano-nose work on executable '\*.py' files. (Alistair Muldal) * Make theano-nose work with older nose version (Frederic B.) * Add extra debug info in verify_grad() (Frederic B.) Theano 0.6rc3 (February 14th, 2013) =================================== Highlights: * Windows related fixes. * Speed-ups. * Crash fixes. * A few small interface changes. * GPU memory leak fix. * A few corner cases fixes without incidence. * More Theano determinism * tensor.{dot,tensordot} more complete/faster/GPU friendly. * tensor.tensordot now support Rop/Lop * tensor.dot support n-dimensional inputs as NumPy * To support more NumPy syntax: * Add theano.tensor.take() * Add a_tensor_variable.{sort,dot,std,argmin,argmax,argsort,clip,conj,conjugate,repeat,round,trace,real,imag,take} Commiters for this rc3 only: Frederic Bastien Ian Goodfellow Pascal Lamblin Jeremiah Lowin abalkin Olivier Delalleau Razvan Pascanu Rami Al-Rfou' Vivek Kulkarni Guillaume Desjardins David Warde-Farley Eric Hunsberger Amir Elaguizy James Bergstra Bug fix: * Fix memory leak on the GPU in some corner cases with the Theano flags `allow_gc=False`. (Frederic B., reported by Jonas Gehring) * Fix copy of random state between graph. (Guillaume D.) http://deeplearning.net/software/theano/tutorial/examples.html#copying-random-state-between-theano-graphs * Fix wrong dtype in sandbox.linalg.ExtractDiag with shape of 0. (Frederic B., reported by abalkin) * Correctly support array with more then 2*10e32 element in AdvancedSubtensor1. (Abalkin) * Fix wrong broadcast dimensions of output of Repeat op. (Abalkin) We where using the inputs broadcasting pattern in some cases when we shouldn't. * Fix theano.sandbox.linalg.eigh grad that didn't always returned the right dtype. (Frederic B., Olivier D.) New Features: * More Theano determinism (Ian G., Olivier D., Pascal L.) * Add and use a new class OrderedSet. * theano.grad is now deterministic. * Warn when the user uses a (non ordered) dictionary and this causes non-determinism in Theano. * The Updates class was non-deterministic; replaced it with the OrderedUpdates class. * tensor.tensordot now support Rop/Lop (Jeremiah Lowin) This remove the class TensorDot and TensorDotGrad. It is the Dot/Elemwise ops that are used. * tensor.dot support n-dimensional inputs as NumPy (Jeremiah Lowin) Work on the GPU too. * The Theano flag `nvcc.flags` now accept `-ftz=true`, `--prec-div=false` and `--prec=sqrt=false` as value. (Frederic B.) To enable all of them, use the Theano flag `nvcc.flags=--use_fast_math`. * New op theano.sparse.ConstructSparseFromList (Rami Al-Rfou' Vivek Kulkarni) * Make Theano work with Anaconda on Windows. (Pascal L.) * Add tensor_var.diagonal and theano.tensor.{diag,diagonal}. (abalkin) * AdvencedSubtensor1 can now have a sparse gradient. (Rami Al-Rfou', Vivek Kulkarni) * Implemented GpuContiguous.grad. (Ian G.) Interface Deprecation (a warning is printed): * theano.misc.strutil.renderString -> render_string (Ian G.) * Print a warning when using dictionary and this makes Theano non-deterministic. Interface Change: * Raise an error when theano.shared called with a theano variable. (Frederic B.) * Don't print warning for bug before Theano 0.5 by default. (Frederic B.) * Theano functions now always have a field name, default to None. (Frederic B.) * Theano function fct.fgraph have a copy of the Theano function name field. (Ian G.) This is needed to allow the fgraph to know it. * In the grad method, if it were asked to raise an error if there is no path between the variables, we didn't always returned an error. (Ian G.) We returned the mathematical right answer 0 in those cases. * get_constant_value() renamed get_scalar_constant_value() and raise a new exception tensor.basic.NotScalarConstantError. (Ian G.) * theano.function raises an error when trying to replace inputs with the 'given' parameter. (Olivier D.) This was doing nothing, the error message explains what the user probably wants to do. New Interface (reuse existing functionality): * tensor_var.sort() as a shortcut for theano.tensor.sort. (Jeremiah Lowin) We where already doing this for argsort. * Add theano.tensor.take() and a_tensor_var.take() to support NumPy syntax. (abalkin) * Add a_tensor_variable.{dot,std,argmin,argmax,argsort,clip,conj,conjugate,repeat,round,trace,real,imag}. (abalkin) New debug feature: * DebugMode print more info when there is an error. (Frederic B.) * Better profiling of test time with `theano-nose --time-profile`. (Frederic B.) * Detection of infinite loop with global optimizer. (Pascal L.) * DebugMode.check_preallocated_output now also work on Theano function output. (Pascal L.) * DebugMode will now complain when the strides of CudaNdarray of dimensions of 1 are not 0. (Frederic B.) Speed-ups: * c_code for SpecifyShape op. (Frederic B.) * cross-entropy optimization now work when specify_shape is used. (Pascal L.) * The Scan optimization ScanSaveMem and PushOutDot1 applied more frequently. (Razvan P, reported Abalkin) A skipped optimization warning was printed. * dot(vector, vector) now faster with some BLAS implementation. (Eric Hunsberger) OpenBLAS and possibly others didn't call {s,d}dot internally when we called {s,d}gemv. MKL was doing this. * Compilation speed up: Take the compiledir lock only for op that generate c_code. (Frederic B) * More scan optimization (Razvan P.) * Opt to make RNN fast in Theano. * Optimize some case of dot, by moving them outside of Scan. * Move some sequences outside of scan too. * Merge more scan inputs, mostly byproduct of other Scan optimizations. * c_code for theano.sparse.AddSD. (Rami Al-Rfou', Vivek Kulkarni) Crash Fixes: * Fix crash about dimshuffle. (abalkin) * Fix crash at compilation. (Olivier D.) * Fix openmp detection. (Pascal L.) Resulted in a crash with EPD on Windows. * Fix for new BLAS interface in SciPy. (Olivier D.) Fix crash with some development version of SciPy. * GpuSum work with bigger shape when summing on the first dim on 3d tensor. (Frederic B., reported Chris Currivan) * Windows compilation crash fix. (Frederic B.) * Make CrossentropySoftmax1HotWithBiasDx and CrossentropySoftmaxArgmax1HotWithBias support uint* dtype. (Frederic B., reported by Mark Fenner) * Fix GpuSoftmax and GpuSoftmaxWithBias crash on GTX285. (Frederic B.) * Fix crash due to a race condition when importing theano. (Ian G.) * Fix crash from path problem with `theano-nose --batch`. (Abalkin) * Fix crash with tensor.roll(Var, iscalar). (Frederic B., reported by Jeremiah Lowin) * Fix compilation crash with llvm on Mac. (Abalkin) * Fix the grad of Scan that told wrongly that there is no connection between cost and parameters. (Razvan P.) * The infer shape mechanism now force that broadcasted dimensions have a shape know to be equivalent to one during compilation. Sometimes, we where not able knowing this before run time and resulted in crash. (Frederic B.) * Fix compilation problems on GPU on Windows. (Frederic B.) * Fix copy on the GPU with big shape for 4d tensor (Pascal L.) * GpuSubtensor didn't set the stride to 0 for dimensions of 1. This could lead to check failing later that caused a crash. (Frederic B., reported by vmichals) Theoretical bugfix (bug that won't happen with current Theano code, but if you messed with the internal, could have affected you): * GpuContiguous, GpuAlloc, GpuDownSampleGrad, Conv2d now check the preallocated outputs strides before using it. (Pascal L.) * GpuDownSample, GpuDownSampleGrad didn't work correctly with negative strides in their output due to problem with nvcc (Pascal L, reported by abalkin?) Others: * Fix race condition when determining if g++ is available. (Abalkin) * Documentation improvements. (Many people including David W-F, abalkin, Amir Elaguizy, Olivier D., Frederic B.) * The current GPU back-end have a new function CudaNdarray_prep_output(CudaNdarray ** arr, int nd, const int * dims) (Ian G) Theano 0.6rc2 (November 21th, 2012) =================================== Highlights: * Fix for a few regressions introduced in 0.6rc1. * A few new features. * Speed-ups. * Scan fixes. * Crash fixes. * A few small interface changes. Commiters for this rc2 only: Razvan Pascanu Pascal Lamblin Frederic Bastien Ian Goodfellow Jeremiah Lowin Caglar Gulcehre Jey Kottalam Matthew Rocklin abalkin Regressions in 0.6rc1 fixed: * Fixed the scan gradient dtype issue. In 0.6rc1, some upcast were inserted. (Razvan P.) * Now grad() will do as before 0.6rc1 for float, i.e. the grad dtype will be the same as the inputs inside the graph. If you ask for the direct grad, it will return the computed dtype. (Pascal L.) Wrong results fixes: * Scan fix in some case didn't returned the good results. (Razvan P., reported by Jeremiah L.) This happened if you had a state with only neg tap and the output of the state was a function of some sequence. If you had multiple states, there was no problem. * Fixed bug in Scan with multiple outputs, where one output would sometimes overwrite another one. (Razvan P.) * Clip.grad treated the gradient with respect to the clipping boundary as always 0. (Ian G.) Interface changes: * We do not support anymore unaligned ndarray in Python code. (Frederic B.) We did not support it in C code and supporting it in Python code made the detection harder. * Now we only officially support SciPy 0.7.2 and NumPy 1.5.0 (Frederic B.) We weren't and aren't testing with older versions. * The theano.sparse.SparseType is available even when SciPy is not (Frederic B.) * Fixed issue where members of consider_constant grad parameter were treated differently from Constant variables. (Ian G.) * Removed the parameter g_cost from theano.grad(). (Ian G.) Use the new more powerful parameter known_grads instead. NumPy interface support: * theano.tensor.where is an alias for theano.tensor.switch to support NumPy semantic. (Ian G.) * TensorVariable objects now have dot, argmin, argmax, clip, conj, repeat, trace, std, round, ravel and argsort functions and the real and imag properties as numpy.ndarray objects. The functionality was already available in Theano. (abalkin) Speed-ups: * A C version of the SoftMax op (Razvan P.) There was C code for the softmax with bias code. * Faster GpuIncSubtensor (Ian G.) * Faster copy on the GPU for 4d tensor. (Ian G.) * The fix of flatten infer_shape re-enables an optimization (Pascal L.) * The bug was introduced in 0.6rc1. * Enable inc_subtensor on the GPU when updating it with a float64 dtype. (Ian G.) It was causing an optimization warning. * Make DeepCopy reuse preallocated memory. (Frederic B.) * Move the convolution to the GPU when the image shape and logical image shape differ. (Frederic Bastien) * C code for the View Op (Razvan P., Pascal L.) New Features: * Added a monitoring mode "MonitorMode" as a debugging tool. (Olivier D.) * Allow integer axes when keepdims==True (Jeremiah Lowin) * Added erfinv and erfcinv op. (Jey Kottalam) * Added tensor.batched_dot(). (Caglar Gulcehre) It uses scan behind the scenes, but makes doing this easier. * theano.get_constant_value(x) (Frederic B.) This tries to have x as a constant int. This does some constant folding to try to convert x into an int. Used by some optimizations. * Add theano.tensor.io.{MPIRecv,MPIRecvWait,MPISend,MPISendWait} (Matthew Rocklin) Theano does not automatically use them. It is up to you to use them and split your computations. * Added theano.sandbox.linalg.eig (abalkin) * Started some support for Python3 (abalkin) setup.py supports python3 now. It calls 2to3 during the setup. Python3 is not fully supported as we didn't update the C code. Crash Fixes: * Fix a crash related to scan.grad due to the new mechanism. (Ian G.) * Fix an optimization warning. Now it gets optimized. (Frederic B.) * Fix crash introduced in 0.6rc1 in theano.grad (Ian G.) * Fix crash introduced in 0.6rc1 in the grad of scan (Razvan P.) * Fix crash introduced in 0.6rc1 in the grad of clip (Ian G.) Also implement the gradient on the min/max bound. * Fix crash in the grad of tensor.switch for int (Ian G.) * Fix crash when mixing shared variable on the GPU and sparse dot. (Pascal L.) * Fix crash as sometimes sparse.dot would return a different dtype number that is equivalent but not the one expected. (Pascal L., reported by Rami Al-Rfou) * Better error msg (Ian G.) * Move all sparse random functions back to sandbox as they don't have a state inside Theano. (Pascal L.) They were moved outside the sandbox in 0.6rc1 * LoadFromDisk now is allowed to only support some memmap mode. (Pascal L.) Otherwise, this was causing errors, segmentation faults or wrong results. * Fix import problem on PiCloud (Jeremiah Lowin) * You need to use the c|py linker with the default environment. Otherwise, you need to create your own environment. * Fix a crash during optimization when we take a subtensor of a constant with a non constant index. (Ian G.) * Better handling and error message of gradients on integer. (Ian G.) * Fixed a crash where Scan assumed all TypeErrors raised by the grad function were due to undefined gradients (Ian G.) Other: * Doc typo fixes, Doc updates, Better error messages: Olivier D., David W.F., Frederic B., James B., Matthew Rocklin, Ian G., abalkin. Theano 0.6rc1 (October 1st, 2012) ================================= Highlights: * Bug fixes, crash fixes, CPU and GPU speed up. * theano_var.eval({other_var: val[,...]} to simplify the usage of Theano (Ian G.) * New default linker `cvm`. This is the execution engine that tells ops to run in certain orders. It is now implemented in C and enables lazy evaluation of ifelse op. * Faster theano.function compilation. (Pascal L., Ian G.) * Big sparse submodule update and documentation of it. (Nicolas Bouchard) * Use GPU asynchronous functionality (Frederic B.) * Better Windows support. Known bugs: * A few crash cases that will be fixed by the final release. Bug fixes: * Outputs of Scan nodes could contain corrupted values: some parts of the output would be repeated a second time, instead of the correct values. It happened randomly, and quite infrequently, but the bug has been present (both in Python and Cython) since April 2011. (Pascal L.) * In Sparse sandbox, fix the grad of theano.sparse.sandbox.sp.row_scale. It did not return the right number of elements. (Frederic B.) * set_subtensor(x[int vector], new_value) when moved to the GPU was transformed into inc_subtensor on the GPU. Now we have a correct (but slow) GPU implementation. Note 1: set_subtensor(x[slice[,...]], new_value) was working correctly in all cases as well as all inc_subtensor. Note 2: If your code was affected by the incorrect behavior, we now print a warning by default (Frederic B.) * Fixed an issue whereby config values were used as default arguments, with those defaults then stuck at old values if the config variables were changed during program execution. (David W-F) * Fixed many subtle bugs involving mutable default arguments which may have led to unexpected behavior, such as objects sharing instance variables they were not supposed to share. (David W-F) * Correctly record the GPU device number used when we let the driver select it. (Frederic B.) * Min, max with NaN in inputs did not return the right output. (Pascal L.) * The grad of TensorDot, was returning the wrong shape for some combination of axes. We now raise NotImplementedError in those cases. (Frederic B.) * conv2d with subsample >2 returned wrong values. (Pascal L.) * Fixed when mode==valid, disabled when mode==full * theano.sparse.CSMGrad op (generated by the grad of CSM) didn't handle unsorted input correctly and gradient that is sparser than the input. In that case, a bad result was returned. But this could happen only when a sparse input of a Theano function was not sorted. This happens for example with sparse advanced indexing from scipy. The conclusion is most of time Nan in the graph. (Yann Dauphin) * theano.sparse._dot(CSC matrix, dense) optimized version UsmmCSCDense didn't handle correctly not contiguous inputs/outputs. (Pascal L.) * Fix a corner case CVM updates case. (Pascal L.) This happened if the update to a shared variable is itself after optimization. The CVM was not used by default. * Fix the view_map of sparse.Transpose and sparse.sandbow.sp.RowScale. (Frederic B.) This probably didn't cause problem as there is only the UsmmCscDense op (used call to Usmm with CSC matrix) that could interfere with them. Deprecation: * Deprecated the Module class (Ian G.) This was a predecessor of SharedVariable with a less pythonic philosophy. Interface changes: * Now the base version requirements are numpy >= 1.5.0 and the optional scipy >= 0.7.2. * In Theano 0.5, we removed the deprecated sharedvar.value property. Now we raise an error if you access it. (Frederic B.) * theano.function does not accept duplicate inputs, so function([x, x], ...) does not work anymore. (Pascal L.) * theano.function now raises an error if some of the provided inputs are not part of the computational graph needed to compute the output, for instance, function([x, y], [y]). You can use the kwarg ``on_unused_input={'raise', 'warn', 'ignore'}`` to control this. (Pascal L.) * New Theano flag "on_unused_input" that defines the default value of the previous point. (Frederic B.) * tensor.alloc() now raises an error during graph build time when we try to create less dimensions than the number of dimensions the provided value have. In the past, the error was at run time. (Frederic B.) * Remove theano.Value and related stuff (Ian G.) This was a test of what ended up as SharedVariable. * Renamed Env to FunctionGraph, and object attribute "env" to "fgraph" (Ian G.) Deprecation warning printed when you try to access the "env" attribute. * Renamed the FunctionGraph.nodes attribute to FunctionNodes.apply_nodes (Ian G.) * Warn when we don't handle correctly the parameter in Theano flags `nvcc.flags` (Frederic B.) * Do not reorder the user flags passed to the compiler. They get set after other flags. (Frederic B.) * Make setuptools optional (Ilan Schnell) * We warn when a user tries to use an old GPU with which Theano is untested. This could cause crash and will also be very slow. (Frederic B.) * Make theano.grad able to differentiate between not implemented, undefined and disconnected grad. Op.grad function should return theano.gradient.{grad_not_implemented,grad_undefined} or something of DisconectedType (Ian G.) * Make theano.grad expect to always receive a float or undefined gradient and enforce that op with integer output values always return 0. (Ian G.) New memory output contract (was mentioned in the release notes of Theano 0.5): * Now the output memory received can be preallocated by other stuff. In the past it was always the previous output an Apply node allocated. So this means that the shape and strides can be different from previous calls and there can be links to this memory at other places. This means it could receive preallocated output that is not c_contiguous. But we don't do that now. (Pascal L.) * New Theano flags to test this DebugMode.check_preallocated_output (Pascal L.) * Updated a few ops to respect this contract (Pascal L.) New Features: * GPU scan now works (does not crash) when there is a mixture of float32 and other dtypes. * theano_var.eval({other_var:val[,...]} to simplify the usage of Theano (Ian G.) * debugprint new param ids=["CHAR", "id", "int", ""] This makes the identifier printed to be a unique char, the Python id, a unique int, or not have it printed. We changed the default to be "CHAR" as this is more readable. (Frederic B.) * debugprint new param stop_on_name=[False, True]. If True, we don't print anything below an intermediate variable that has a name. Defaults to False. (Frederic B.) * debugprint does not print anymore the "|" symbol in a column after the last input. (Frederic B.) * If you use Enthought Python Distribution (EPD) now we use its blas implementation by default. (Frederic B., Graham Taylor, Simon McGregor) * MRG random now raises an error with a clear message when the passed shape contains dimensions with bad value like 0. (Frederic B. reported by Ian G.) * "CudaNdarray[*] = ndarray" works in more cases (Frederic B.) * "CudaNdarray[*] += ndarray" works in more cases (Frederic B.) * We add dimensions to CudaNdarray to automatically broadcast more frequently. (Frederic B.) * New theano flag cmodule.warn_no_version. Default False. If True, will print a warning when compiling one or more Op with C code that can't be cached because there is no c_code_cache_version() function associated to at least one of those Ops. (Frederic B.) * CPU alloc now always generate C code (Pascal L.) * New Theano flag cmodule.warn_no_version=False. When True, warn when an op with C code is not versioned (which forces to recompile it everytimes). (Frederic B.) * C code reuses preallocated outputs (only done by Scan) (Pascal L.) * Garbage collection of intermediate results during Theano function calls for Ops with C code (Pascal L.) * Theano flag compiledir_format now supports the parameter "numpy_version" and "g++". (Frederic B.) * Theano GPU variables, shared variables and constants now support <, <=, > and >= similar to those not on the GPU. * AdvancedIncSubtensor now supports the set_instead_of_inc parameter. (Eric L.) * Added Advanced Indexing support to inc_subtensor and set_subtensor. (Eric L.) * theano.tensor.{any,all,std,var,mean,prod,sum,argmin,argmax,min,max,max_and_argman} have a new parameter keepdims (Eric L.) This allows to broadcast it correctly against the input data to normalize it. * The Updates objects now check that the keys are SharedVariable when we pass them in the __init__ function. (Pascal L.) * Set a Theano Variable name on transposed op when the input has one (Frederic B). * The cvm linker now supports garbage collection (enabled by default). (James B. Arnaud B., Pascal L.) * The cvm linker is now the default linker. This makes the "loop" around the execution of apply node in C. So this lowers the overhead. * theano_variable[numpy.newaxis] is now supported (James B.) * Enable ifelse on the GPU. (Frederic B.) * Correctly support numpy.memmap everywhere (Pascal L.) We add partial support for them before. Just use the normal tensor operation on them and it should work. But be careful not to exhaust your computer memory! (we always generate normal ndarray) * Add an optimization that stabilizes log(softmax(x)). (Ian G.) * Re-enable the Images2Neibs grad. It was not broken, the problem was how we tested it. (Frederic B.) * If `theano_fn.trust_input` is set to False, do not check if the inputs are good when calling the theano function. (Frederic B.) * Add theano.tensor.blas,gem{m,v} as shortcut. * theano.grad(..., add_names=True). False for the old behavior. Otherwise it tries to name the grad variables. (Ian G.) * theano-nose (Pascal L.) A wrapper around nosetests that adds needed extensions. * --profile-time option, to print time spent in each test (Eric L.) * --batch option, to allow to run tests in batch to lower memory requirement. * m = mean(log(1 - sigm(x))) x - scalar * theano.grad(m, x) There is a stabilization optimization for this. Now it is applied more frequently. (Pascal L.) New Op/functions: * Added element-wise operation theano.tensor.{GammaLn,Psi} (John Salvatier, Nicolas Bouchard) * Added element-wise operation theano.tensor.{arcsin,arctan,arccosh,arcsinh,arctanh,exp2,arctan2} (Nicolas Bouchard) * Added element-wise operation theano.tensor.{gamma,conj,complex_from_polar,expm1,deg2rad,rad2deg,trunc,gamma} (Nicolas Bouchard) * Added theano.tensor.argsort that wraps numpy.argsort (Hani Almousli). * Added theano.tensor.diff that wraps numpy.diff (Nicolas B.) * Added theano.tensor.bincount that wraps numpy.bincount (Nicolas B., Pascal L, Frederic B.) * Added theano.tensor.squeeze (Nicolas B.) This removes broadcasted dimensions from the variable. Theano-esque version of numpy.squeeze. * Added theano.tensor.repeat that wraps numpy.repeat (Nicolas B. + PL) * Added theano.tensor.bartlett that wraps numpy.bartlett (Eric L.) * Added theano.tensor.fill_diagonal that wraps numpy.fill_diagonal (Eric L., Frederic B.) * Added tensor.square that is an alias for tensor.sqr as NumPy (Ian G.) * Added theano.tensor.load(path, dtype, broadcastable, mmap_mode=None) op that allows to load a .npy file in a theano graph (Matthew Rocklin) * theano.sandbox.linalg.kron.py:Kron op. (Eric L.) Kronecker product Speed up: * CPU convolutions are now parallelized (Frederic B.) By default use all cores/hyper-threads. To control it, use the `OMP_NUM_THREADS=N` environment variable where N is the number of parallel threads to use. By default it is equal to the number of CPU cores/hyper threads that you have. There is a new Theano flag `openmp` to allow/disallow openmp op. If your BLAS library is parallelized, this flag won't affect it, but the env variable will. * Remove a corner case causing duplicated dot22/gemm in the graph. (Frederic B., Ian G.) * Enable fusion of elemwise that have the same clients multiple times. (Frederic B.) * New optimization: Remove reduction over broadcastable dimensions (James B., Frederic B.) * Faster theano.function compilation. (Pascal L., Ian G.) * Remove GPU transfer around specify_shape op. (Frederic B.) * Implemented/tested MANY op.infer_shape method (Eric Larsen) This allows Theano to make better shape inferance. * Implement Solve.infer_shape (Matthew Rocklin) * Scan memory optimizations now work more frequently. (Razvan P.) There was a warning printed by the subtensor optimization in those cases. * Faster rng_mrg Python code. (mostly used for tests) (Frederic B.) Speed up GPU: * Convolution on the GPU now checks the generation of the card to make it faster in some cases (especially medium/big ouput image) (Frederic B.) * We had hardcoded 512 as the maximum number of threads per block. Newer cards support up to 1024 threads per block. * Faster GpuAdvancedSubtensor1, GpuSubtensor, GpuAlloc (Frederic B.) * We now pass the GPU architecture to nvcc when compiling (Frederic B.) * Now we use the GPU function async feature by default. (Frederic B.) Set the environment variable `CUDA_LAUNCH_BLOCKING` to `1` to disable this for profiling or debugging. * Faster creation of CudaNdarray objects (Frederic B.) * Now some Max reductions are implemented on the GPU. (Ian G.) Sparse Sandbox graduate (moved from theano.sparse.sandbox.sp): * sparse.remove0 (Frederic B., Nicolas B.) * sparse.sp_sum(a, axis=None) (Nicolas B.) * bugfix: the not structured grad was returning a structured grad. * sparse.{col_scale,row_scale,ensure_sorted_indices,clean} (Nicolas B.) * sparse.{diag,square_diagonal} (Nicolas B.) Sparse: * Support for uint* dtype. * Implement theano.sparse.mul(sparse1, sparse2) when both inputs don't have the same sparsity pattern. (Frederic B.) * New Ops: sparse.{expm1,deg2rad,rad2deg,trunc} (Nicolas B.) * New Ops: sparse.{sqrt,sqr,log1p,floor,ceil,sgn,round_half_to_even} (Nicolas B.) * New Ops: sparse.{arctanh,tanh,arcsinh,sinh,arctan,arcsin,tan,sin} (Nicolas B.) * New functions: structured_{add,exp,log,pow,minimum,maximum,sigmoid} (Yann D., Nicolas B.) * Optimized op: StructuredAddSV, StrucutedAddSVCSR (inserted automatically) * New Op: sparse.mul_s_v multiplication of sparse matrix by broadcasted vector (Yann D.) * New Op: sparse.Cast() (Yann D., Nicolas B.) * Add sparse_variable.astype() and theano.sparse.cast() and theano.sparse.{b,w,i,l,f,d,c,z}cast() as their tensor equivalent (Nicolas B.) * Op class: SamplingDot (Yann D., Nicolas B.) * Optimized version: SamplingDotCsr, StructuredDotCSC * Optimizations to insert the optimized version: local_sampling_dot_csr, local_structured_add_s_v * New Ops: sparse.{Multinomial,Poisson,Binomial} (Yann D., NB) * Implement the CSMProperties grad method (Yann Dauphin) * Move optimizations to theano/sparse/opt.py (Nicolas B.) New flags: * `profile=True` flag now prints the sum of all printed profiles. (Frederic B.) * It works with the linkers vm/cvm (default). * Also print compile time, optimizer time and linker time. * Also print a summary by op class. * new flag "profile_optimizer" (Frederic B.) when profile=True, will also print the time spent in each optimizer. Useful to find optimization bottleneck. * new flag "cmodule.remove_gxx_opt" (Frederic B.) If True, will remove -O* parameter passed to g++. This is useful to debug in gdb module compiled by Theano. The parameter -g is passed by default to g++. * new flag cmodule.compilation_warning if True, will print compilation warning. * new flag `allow_gc` (Frederic B.) When False, do not garbage collect intermediate results when they are not needed. This uses more memory, but allocates memory less frequently so faster. * new flag `vm.lazy` (Frederic B.) Useful only for the vm linkers. When lazy is None, auto detect if lazy evaluation is needed and use the apropriate version. If lazy is True/False, force the version used between Loop/LoopGC and Stack. * new flag `cxx`. This is the C++ compiler to use. If empty do not compile C code. (Frederic B.) * New flag `print_active_device` that defaults to True. (Matthew R.) Documentation: * Added in the tutorial documentation on how to extend Theano. This explains how to make a Theano Op from a Python function. http://deeplearning.net/software/theano/tutorial/extending_theano.html (Frederic B.) * New installation instructions for Windows using EPD (Pascal L.) * New installation on Windows by using a Linux VM from ContinuumIO (Frederic B.) * Revisions of Theano tutorial and addition of exercises to it. (Eric L.) * New tutorial on Sparse variable. (Nicolas B., Sebastien Lemieux, Frederic Bastien http://www.deeplearning.net/software/theano/tutorial/sparse.html * Installation documentation for CentOS6 (Frederic B.) * Installation documentation for Ubuntu (with GPU) (Frederic B., Matthias Zoehrer) * Doc typo fixes, Doc updates, Better error messages: Olivier D., David W.F., Frederic B., James B., Matthew Rocklin, Ian G. * Python Memory Management tutorial (Steven Pigeon, Olivier D.) Proposal: * Math framework for complex gradients (Pascal L.) Internal changes: * Define new exceptions MissingInputError and UnusedInputError, and use them in theano.function, instead of TypeError and ValueError. (Pascal L.) * Better handling of bitwidth and max values of integers and pointers across platforms (Pascal L.) * Made a few Ops with C code versioned to reduce compilation time. (Frederic B, Pascal L.) * Better deletion of files in the compiledir (Frederic B.) * Safer import on sort op (Nicolas Pinto) * hash_from_dict for elemwise op (Fredric B.) * Renamed BadCLinkerOutput into BadThunkOutput. (PL) * tensor.utils.shape_of_variables (Matthew R.) * Add the numpy abi version and g++/nvcc version in the key of compiled code. (Frederic B.) * env.replace_all_validate_remove (Frederic B.) This allows global optimizer to ensure it removed some nodes from the graph. This is a generic way to catch errors that would otherwise duplicate computation. * It was used for GEMM and Scan optimization (Frederic B., Razvan P.) * Fix how exception are raised in GPU code (James B.) * Made code respect pep8: OD, Fred, Pascal L., Nicolas Bouchard, Eric Larsen and others. * TensorType and CudaNdarrayType now have a value_zeros method that call CudaNdarray.zeros or numpy.zeros with the right dtype. (Pascal L., Olivier D.) This allows to have the same code work with both types. * Renamed FunctionGraph.extend function to FunctionGraph.attach_feature. (Ian G.) * New exception MissingGXX when we try to compile but there is no cxx compiler. (Frederic B.) * New fct theano.gof.utils.give_variables_names(...) that gives unique names to variables. (Matthew R.) * Use most of the time the new NumPy C-API for later NumPy release. (Frederic B.) * New theano.gof.sched.sort_apply_nodes() that will allow other execution ordering. (Matthew R.) * New attribute sort_schedule_fn, a way to specify a scheduler to use. (Matthew R.) Crash Fix: * Fix import conflict name (usaar33, Frederic B.) * This makes Theano work with PiCloud. * Do not try to use the BLAS library when blas.ldflags is manually set to an empty string (Frederic B., Pascal L.) * When importing theano on a computer without GPU with the Theano flags 'device' or 'init_gpu_device' set to gpu* (Frederic B., reported by Luo Heng) * Optimization printed a useless error when scipy was not available. (Frederic B.) * GPU conv crash/slowdown on newer hardware (James B.) * Better error handling in GPU conv (Frederic B.) * GPU optimization that moves element-wise Ops to the GPU. Crash happened in a particular execution order of this optimization and the element-wise fusion optimization when upcasting some inputs to float32 (to compute them on the GPU). (Frederic B., reported by Sander Dieleman) * GpuReshape in some particular case when the input is not contiguous (Frederic B., reported by Sander Dieleman) * GpuSoftmaxWithBias with shape (0, N) with N > 1. (Frederic B., reported by Razvan P.) * Fix crash under 64-bit Windows, when taking subtensors of the form a[n:] (Pascal L., reported by Simon McGregor) * Fixed issue with the MaxAndArgmax Op not properly preserving broadcastable dimensions, which could typically result in optimization crashes (Olivier D.) * Fixed crash when concatenating some arrays with specific broadcasting patterns (Olivier D.) * Work around a known issue with nvcc 4.1 on MacOS X. (Graham Taylor) * In advanced indexing, if some inputs are constant, no need to call constant(...) on their value any more. (Pascal L., reported by John Salvatier) * Fix crash on GPU when the GpuSubtensor didn't put the right stride when the result tensor had a dimension with size of 1. (Pascal L, reported Graham T.) * Fix scan crash that made it not run on the GPU in one case. (Guillaume D.) * If you grad again a random state, don't crash (Razvan P.) * GpuDownsampleFactorMax and its grad with inputs dimensions 0 and 1 bigger then 65535. (Frederic B. reported by Gabe Schwartz) * Potential crash due to parallel compilation when importing theano.sandbox.cuda (Olivier D.) * Crash fix on python 2.4 with slicing. (Pascal L.) * grad of argmin and argmax (Razvan P.) * Don't compute the Rop for shared variables with updates (mostly random). We don't use them and they caused crash. (Razvan P.) * MaxArgmax.grad() when one of the gradient it receives is None. (Razvan P, reported by Mark Fenner) * Fix crash of GpuSum when some dimensions shape was 0. (Frederic B.) Tests: * Use less memory (Olivier D.) (fix crash on 32-bit computers) * Fix test with Theano flag "blas.ldflags=". (Frederic B., Pascal L.) * Fix crash with advanced subtensor and numpy constant. * Fix random tests crash due to random value. (Pascal L.) * Always introduce Alloc node when calling alloc and let the optimizer remove them if needed. This allows DebugMode to catch some shape error. (Pascal L.) * DebugMode now checks the view_map for all types of Theano variables. It was doing only variables of tensor type. (Frederic B.) Others: * Remove python warning for some python version. (Gabe Schwartz) * Remove useless fill op in fast_compile mode to make the graph more readable. (Fredric B.) * Remove GpuOuter as it is a subset of the new GpuGer (Frederic B.) * Now we use http://travis-ci.org/ to run all CPU tests (without SciPy) with the default mode on all Pull Requests. This should make the trunk more stable. (Fredric B.) * Our nightly buildbot now checks on python 2.4 (Frederic B.) This should make the trunk work on it more frequently. Other thanks: * blaxill reported an error introduced into the trunk. New stuff that will probably be reworked/removed before the release: * Better PyCUDA sharing of the GPU context.(fix crash at exit) (Frederic B.) TODO: there is still a crash at exit! Theano 0.5 (23 February 2012) ============================= Highlights: * Moved to github: http://github.com/Theano/Theano/ * Old trac tickets moved to assembla tickets: http://www.assembla.com/spaces/theano/tickets * Theano vision: http://deeplearning.net/software/theano/introduction.html#theano-vision (Many people) * Theano with GPU works in some cases on Windows now. Still experimental. (Sebastian Urban) * Faster dot() call: New/Better direct call to cpu and gpu ger, gemv, gemm and dot(vector, vector). (James, Frédéric, Pascal) * C implementation of Alloc. (James, Pascal) * theano.grad() now also works with sparse variables. (Arnaud) * Macro to implement the Jacobian/Hessian with theano.tensor.{jacobian,hessian} (Razvan) * See the Interface changes. Interface Behavior Changes: * The current default value of the parameter axis of theano.{max,min,argmax,argmin,max_and_argmax} is now the same as numpy: None. i.e. operate on all dimensions of the tensor. (Frédéric Bastien, Olivier Delalleau) (was deprecated and generated a warning since Theano 0.3 released Nov. 23rd, 2010) * The current output dtype of sum with input dtype [u]int* is now always [u]int64. You can specify the output dtype with a new dtype parameter to sum. The output dtype is the one used for the summation. There is no warning in previous Theano versions about this. The consequence is that the sum is done in a dtype with more precision than before. So the sum could be slower, but will be more resistant to overflow. This new behavior is the same as numpy. (Olivier, Pascal) * When using a GPU, detect faulty nvidia drivers. This was detected when running Theano tests. Now this is always tested. Faulty drivers result in wrong results for reduce operations. (Frederic B.) Interface Features Removed (most were deprecated): * The string modes FAST_RUN_NOGC and STABILIZE are not accepted. They were accepted only by theano.function(). Use Mode(linker='c|py_nogc') or Mode(optimizer='stabilize') instead. * tensor.grad(cost, wrt) now always returns an object of the "same type" as wrt (list/tuple/TensorVariable). (Ian Goodfellow, Olivier) * A few tag.shape and Join.vec_length left have been removed. (Frederic) * The .value attribute of shared variables is removed, use shared.set_value() or shared.get_value() instead. (Frederic) * Theano config option "home" is not used anymore as it was redundant with "base_compiledir". If you use it, Theano will now raise an error. (Olivier D.) * scan interface changes: (Razvan Pascanu) * The use of `return_steps` for specifying how many entries of the output to return has been removed. Instead, apply a subtensor to the output returned by scan to select a certain slice. * The inner function (that scan receives) should return its outputs and updates following this order: [outputs], [updates], [condition]. One can skip any of the three if not used, but the order has to stay unchanged. Interface bug fix: * Rop in some case should have returned a list of one Theano variable, but returned the variable itself. (Razvan) New deprecation (will be removed in Theano 0.6, warning generated if you use them): * tensor.shared() renamed to tensor._shared(). You probably want to call theano.shared() instead! (Olivier D.) Bug fixes (incorrect results): * On CPU, if the convolution had received explicit shape information, they were not checked at runtime. This caused wrong result if the input shape was not the one expected. (Frederic, reported by Sander Dieleman) * Theoretical bug: in some case we could have GPUSum return bad value. We were not able to reproduce this problem * patterns affected ({0,1}*nb dim, 0 no reduction on this dim, 1 reduction on this dim): 01, 011, 0111, 010, 10, 001, 0011, 0101 (Frederic) * div by zero in verify_grad. This hid a bug in the grad of Images2Neibs. (James) * theano.sandbox.neighbors.Images2Neibs grad was returning a wrong value. The grad is now disabled and returns an error. (Frederic) * An expression of the form "1 / (exp(x) +- constant)" was systematically matched to "1 / (exp(x) + 1)" and turned into a sigmoid regardless of the value of the constant. A warning will be issued if your code was affected by this bug. (Olivier, reported by Sander Dieleman) * When indexing into a subtensor of negative stride (for instance, x[a:b:-1][c]), an optimization replacing it with a direct indexing (x[d]) used an incorrect formula, leading to incorrect results. (Pascal, reported by Razvan) * The tile() function is now stricter in what it accepts to allow for better error-checking/avoiding nonsensical situations. The gradient has been disabled for the time being as it only implemented (incorrectly) one special case. The `reps` argument must be a constant (not a tensor variable), and must have the same length as the number of dimensions in the `x` argument; this is now checked. (David) Scan fixes: * computing grad of a function of grad of scan (reported by Justin Bayer, fix by Razvan) before: most of the time crash, but could be wrong value with bad number of dimensions (so a visible bug) now: do the right thing. * gradient with respect to outputs using multiple taps (reported by Timothy, fix by Razvan) before: it used to return wrong values now: do the right thing. Note: The reported case of this bug was happening in conjunction with the save optimization of scan that give run time errors. So if you didn't manually disable the same memory optimization (number in the list4), you are fine if you didn't manually request multiple taps. * Rop of gradient of scan (reported by Timothy and Justin Bayer, fix by Razvan) before: compilation error when computing R-op now: do the right thing. * save memory optimization of scan (reported by Timothy and Nicolas BL, fix by Razvan) before: for certain corner cases used to result in a runtime shape error now: do the right thing. * Scan grad when the input of scan has sequences of different lengths. (Razvan, reported by Michael Forbes) * Scan.infer_shape now works correctly when working with a condition for the number of loops. In the past, it returned n_steps as the length, which is not always true. (Razvan) * Scan.infer_shape crash fix. (Razvan) New features: * AdvancedIncSubtensor grad defined and tested (Justin Bayer) * Adding 1D advanced indexing support to inc_subtensor and set_subtensor (James Bergstra) * tensor.{zeros,ones}_like now supports the dtype param as numpy (Frederic) * Added configuration flag "exception_verbosity" to control the verbosity of exceptions (Ian) * theano-cache list: list the content of the theano cache (Frederic) * theano-cache unlock: remove the Theano cache lock (Olivier) * tensor.ceil_int_div to compute ceil(a / float(b)) (Frederic) * MaxAndArgMax.grad now works with any axis (The op supports only 1 axis) (Frederic) * used by tensor.{max,min,max_and_argmax} * tensor.{all,any} (Razvan) * tensor.roll as numpy: (Matthew Rocklin, David Warde-Farley) * Theano with GPU works in some cases on Windows now. Still experimental. (Sebastian Urban) * IfElse now allows to have a list/tuple as the result of the if/else branches. * They must have the same length and corresponding type (Razvan) * Argmax output dtype is now int64 instead of int32. (Olivier) * Added the element-wise operation arccos. (Ian) * Added sparse dot with dense grad output. (Yann Dauphin) * Optimized to Usmm and UsmmCscDense in some case (Yann) * Note: theano.dot and theano.sparse.structured_dot() always had a gradient with the same sparsity pattern as the inputs. The new theano.sparse.dot() has a dense gradient for all inputs. * GpuAdvancedSubtensor1 supports broadcasted dimensions. (Frederic) * TensorVariable.zeros_like() and SparseVariable.zeros_like() * theano.sandbox.cuda.cuda_ndarray.cuda_ndarray.device_properties() (Frederic) * theano.sandbox.cuda.cuda_ndarray.cuda_ndarray.mem_info() return free and total gpu memory (Frederic) * Theano flags compiledir_format. Keep the same default as before: compiledir_%(platform)s-%(processor)s-%(python_version)s. (Josh Bleecher Snyder) * We also support the "theano_version" substitution. * IntDiv C code (faster and allows this elemwise to be fused with other elemwise) (Pascal) * Internal filter_variable mechanism in Type. (Pascal, Ian) * Ifelse works on sparse. * It makes use of gpu shared variable more transparent with theano.function updates and givens parameter. * Added a_tensor.transpose(axes) axes is optional (James) * theano.tensor.transpose(a_tensor, kwargs) We were ignoring kwargs, now it is used as the axes. * a_CudaNdarray_object[*] = int, now works (Frederic) * tensor_variable.size (as numpy) computes the product of the shape elements. (Olivier) * sparse_variable.size (as scipy) computes the number of stored values. (Olivier) * sparse_variable[N, N] now works (Li Yao, Frederic) * sparse_variable[M:N, O:P] now works (Li Yao, Frederic, Pascal) M, N, O, and P can be Python int or scalar tensor variables, None, or omitted (sparse_variable[:, :M] or sparse_variable[:M, N:] work). * tensor.tensordot can now be moved to GPU (Sander Dieleman, Pascal, based on code from Tijmen Tieleman's gnumpy, http://www.cs.toronto.edu/~tijmen/gnumpy.html) * Many infer_shape implemented on sparse matrices op. (David W.F.) * Added theano.sparse.verify_grad_sparse to easily allow testing grad of sparse op. It supports testing the full and structured gradients. * The keys in our cache now store the hash of constants and not the constant values themselves. This is significantly more efficient for big constant arrays. (Frederic B.) * 'theano-cache list' lists key files bigger than 1M (Frederic B.) * 'theano-cache list' prints an histogram of the number of keys per compiled module (Frederic B.) * 'theano-cache list' prints the number of compiled modules per op class (Frederic B.) * The Theano flag "nvcc.fastmath" is now also used for the cuda_ndarray.cu file. * Add the header_dirs to the hard part of the compilation key. This is currently used only by cuda, but if we use libraries that are only headers, this can be useful. (Frederic B.) * The Theano flag "nvcc.flags" is now included in the hard part of the key. This means that now we recompile all modules for each value of "nvcc.flags". A change in "nvcc.flags" used to be ignored for modules that were already compiled. (Frederic B.) * Alloc, GpuAlloc are not always pre-computed (constant_folding optimization) at compile time if all their inputs are constant. (Frederic B., Pascal L., reported by Sander Dieleman) * New Op tensor.sort(), wrapping numpy.sort (Hani Almousli) New optimizations: * AdvancedSubtensor1 reuses preallocated memory if available (scan, c|py_nogc linker) (Frederic) * dot22, dot22scalar work with complex. (Frederic) * Generate Gemv/Gemm more often. (James) * Remove scan when all computations can be moved outside the loop. (Razvan) * scan optimization done earlier. This allows other optimizations to be applied. (Frederic, Guillaume, Razvan) * exp(x) * sigmoid(-x) is now correctly optimized to the more stable form sigmoid(x). (Olivier) * Added Subtensor(Rebroadcast(x)) => Rebroadcast(Subtensor(x)) optimization. (Guillaume) * Made the optimization process faster. (James) * Allow fusion of elemwise when the scalar op needs support code. (James) * Better opt that lifts transpose around dot. (James) Crashes fixed: * T.mean crash at graph building time. (Ian) * "Interactive debugger" crash fix. (Ian, Frederic) * Do not call gemm with strides 0, some blas refuse it. (Pascal Lamblin) * Optimization crash with gemm and complex. (Frederic) * GPU crash with elemwise. (Frederic, some reported by Chris Currivan) * Compilation crash with amdlibm and the GPU. (Frederic) * IfElse crash. (Frederic) * Execution crash fix in AdvancedSubtensor1 on 32 bit computers. (Pascal) * GPU compilation crash on MacOS X. (Olivier) * Support for OSX Enthought Python Distribution 7.x. (Graham Taylor, Olivier) * When the subtensor inputs had 0 dimensions and the outputs 0 dimensions. (Frederic) * Crash when the step to subtensor was not 1 in conjunction with some optimization. (Frederic, reported by Olivier Chapelle) * Runtime crash related to an optimization with subtensor of alloc (reported by Razvan, fixed by Frederic) * Fix dot22scalar cast of integer scalars (Justin Bayer, Frédéric, Olivier) * Fix runtime crash in gemm, dot22. FB * Fix on 32 bit computer: make sure all shapes are int64. (Olivier) * Fix to deque on python 2.4 (Olivier) * Fix crash when not using C code (or using DebugMode) (not used by default) with numpy 1.6*. Numpy has a bug in the reduction code that made it crash. (Pascal) * Crashes of blas functions (Gemv on CPU; Ger, Gemv and Gemm on GPU) when matrices had non-unit stride in both dimensions (CPU and GPU), or when matrices had negative strides (GPU only). In those cases, we are now making copies. (Pascal) * More cases supported in AdvancedIncSubtensor1. (Olivier D.) * Fix crash when a broadcasted constant was used as input of an elemwise Op and needed to be upcasted to match the op's output. (Reported by John Salvatier, fixed by Pascal L.) * Fixed a memory leak with shared variable (we kept a pointer to the original value) (Ian G.) Known bugs: * CAReduce with nan in inputs don't return the good output (`Ticket `_). * This is used in tensor.{max,mean,prod,sum} and in the grad of PermuteRowElements. Sandbox: * cvm interface more consistent with current linker. (James) * Now all tests pass with the linker=cvm flags. * vm linker has a callback parameter. (James) * review/finish/doc: diag/extract_diag. (Arnaud Bergeron, Frederic, Olivier) * review/finish/doc: AllocDiag/diag. (Arnaud, Frederic, Guillaume) * review/finish/doc: MatrixInverse, matrix_inverse. (Razvan) * review/finish/doc: matrix_dot. (Razvan) * review/finish/doc: det (determinent) op. (Philippe Hamel) * review/finish/doc: Cholesky determinent op. (David) * review/finish/doc: ensure_sorted_indices. (Li Yao) * review/finish/doc: spectral_radius_boud. (Xavier Glorot) * review/finish/doc: sparse sum. (Valentin Bisson) * review/finish/doc: Remove0 (Valentin) * review/finish/doc: SquareDiagonal (Eric) Sandbox New features (not enabled by default): * CURAND_RandomStreams for uniform and normal (not picklable, GPU only) (James) * New sandbox.linalg.ops.pinv(pseudo-inverse) op (Razvan) Documentation: * Many updates. (Many people) * Updates to install doc on MacOS. (Olivier) * Updates to install doc on Windows. (David, Olivier) * Doc on the Rop function (Ian) * Added how to use scan to loop with a condition as the number of iteration. (Razvan) * Added how to wrap in Theano an existing python function (in numpy, scipy, ...). (Frederic) * Refactored GPU installation of Theano. (Olivier) Others: * Better error messages in many places. (Many people) * PEP8 fixes. (Many people) * Add a warning about numpy bug when using advanced indexing on a tensor with more than 2**32 elements (the resulting array is not correctly filled and ends with zeros). (Pascal, reported by David WF) * Added Scalar.ndim=0 and ScalarSharedVariable.ndim=0 (simplify code) (Razvan) * New min_informative_str() function to print graph. (Ian) * Fix catching of exception. (Sometimes we used to catch interrupts) (Frederic, David, Ian, Olivier) * Better support for utf string. (David) * Fix pydotprint with a function compiled with a ProfileMode (Frederic) * Was broken with change to the profiler. * Warning when people have old cache entries. (Olivier) * More tests for join on the GPU and CPU. (Frederic) * Do not request to load the GPU module by default in scan module. (Razvan) * Fixed some import problems. (Frederic and others) * Filtering update. (James) * On Windows, the default compiledir changed to be local to the computer/user and not transferred with roaming profile. (Sebastian Urban) * New theano flag "on_shape_error". Defaults to "warn" (same as previous behavior): it prints a warning when an error occurs when inferring the shape of some apply node. The other accepted value is "raise" to raise an error when this happens. (Frederic) * The buidbot now raises optimization/shape errors instead of just printing a warning. (Frederic) * better pycuda tests (Frederic) * check_blas.py now accepts the shape and the number of iterations as parameter (Frederic) * Fix opt warning when the opt ShapeOpt is disabled (enabled by default) (Frederic) * More internal verification on what each op.infer_shape return. (Frederic, James) * Improved docstring and basic tests for the Tile Op (David). Reviewers (alphabetical order): * David, Frederic, Ian, James, Olivier, Razvan Theano 0.4.1 (12 August 2011) ============================= New features: * `R_op `_ macro like theano.tensor.grad * Not all tests are done yet (TODO) * Added alias theano.tensor.bitwise_{and,or,xor,not}. They are the numpy names. * Updates returned by Scan (you need to pass them to the theano.function) are now a new Updates class. That allow more check and easier work with them. The Updates class is a subclass of dict * Scan can now work in a "do while" loop style. * We scan until a condition is met. * There is a minimum of 1 iteration(can't do "while do" style loop) * The "Interactive Debugger" (compute_test_value theano flags) * Now should work with all ops (even the one with only C code) * In the past some errors were caught and re-raised as unrelated errors (ShapeMismatch replaced with NotImplemented). We don't do that anymore. * The new Op.make_thunk function(introduced in 0.4.0) is now used by constant_folding and DebugMode * Added A_TENSOR_VARIABLE.astype() as a way to cast. NumPy allows this syntax. * New BLAS GER implementation. * Insert GEMV more frequently. * Added new ifelse(scalar condition, rval_if_true, rval_if_false) Op. * This is a subset of the elemwise switch (tensor condition, rval_if_true, rval_if_false). * With the new feature in the sandbox, only one of rval_if_true or rval_if_false will be evaluated. Optimizations: * Subtensor has C code * {Inc,Set}Subtensor has C code * ScalarFromTensor has C code * dot(zeros,x) and dot(x,zeros) * IncSubtensor(x, zeros, idx) -> x * SetSubtensor(x, x[idx], idx) -> x (when x is a constant) * subtensor(alloc,...) -> alloc * Many new scan optimization * Lower scan execution overhead with a Cython implementation * Removed scan double compilation (by using the new Op.make_thunk mechanism) * Certain computations from the inner graph are now Pushed out into the outer graph. This means they are not re-comptued at every step of scan. * Different scan ops get merged now into a single op (if possible), reducing the overhead and sharing computations between the two instances GPU: * PyCUDA/CUDAMat/Gnumpy/Theano bridge and `documentation `_. * New function to easily convert pycuda GPUArray object to and from CudaNdarray object * Fixed a bug if you crated a view of a manually created CudaNdarray that are view of GPUArray. * Removed a warning when nvcc is not available and the user did not requested it. * renamed config option cuda.nvccflags -> nvcc.flags * Allow GpuSoftmax and GpuSoftmaxWithBias to work with bigger input. Bugs fixed: * In one case an AdvancedSubtensor1 could be converted to a GpuAdvancedIncSubtensor1 insted of GpuAdvancedSubtensor1. It probably didn't happen due to the order of optimizations, but that order is not guaranteed to be the same on all computers. * Derivative of set_subtensor was wrong. * Derivative of Alloc was wrong. Crash fixed: * On an unusual Python 2.4.4 on Windows * When using a C cache copied from another location * On Windows 32 bits when setting a complex64 to 0. * Compilation crash with CUDA 4 * When wanting to copy the compilation cache from a computer to another * This can be useful for using Theano on a computer without a compiler. * GPU: * Compilation crash fixed under Ubuntu 11.04 * Compilation crash fixed with CUDA 4.0 Know bug: * CAReduce with nan in inputs don't return the good output (`Ticket `_). * This is used in tensor.{max,mean,prod,sum} and in the grad of PermuteRowElements. * This is not a new bug, just a bug discovered since the last release that we didn't had time to fix. Deprecation (will be removed in Theano 0.5, warning generated if you use them): * The string mode (accepted only by theano.function()) FAST_RUN_NOGC. Use Mode(linker='c|py_nogc') instead. * The string mode (accepted only by theano.function()) STABILIZE. Use Mode(optimizer='stabilize') instead. * scan interface change: * The use of `return_steps` for specifying how many entries of the output scan has been deprecated * The same thing can be done by applying a subtensor on the output return by scan to select a certain slice * The inner function (that scan receives) should return its outputs and updates following this order: [outputs], [updates], [condition]. One can skip any of the three if not used, but the order has to stay unchanged. * tensor.grad(cost, wrt) will return an object of the "same type" as wrt (list/tuple/TensorVariable). * Currently tensor.grad return a type list when the wrt is a list/tuple of more than 1 element. Decrecated in 0.4.0(Reminder, warning generated if you use them): * Dividing integers with / is deprecated: use // for integer division, or cast one of the integers to a float type if you want a float result (you may also change this behavior with config.int_division). * tag.shape attribute deprecated (#633) * CudaNdarray_new_null is deprecated in favour of CudaNdarray_New Sandbox: * MRG random generator now implements the same casting behavior as the regular random generator. Sandbox New features(not enabled by default): * New Linkers (theano flags linker={vm,cvm}) * The new linker allows lazy evaluation of the new ifelse op, meaning we compute only the true or false branch depending of the condition. This can speed up some types of computation. * Uses a new profiling system (that currently tracks less stuff) * The cvm is implemented in C, so it lowers Theano's overhead. * The vm is implemented in python. So it can help debugging in some cases. * In the future, the default will be the cvm. * Some new not yet well tested sparse ops: theano.sparse.sandbox.{SpSum, Diag, SquareDiagonal, ColScaleCSC, RowScaleCSC, Remove0, EnsureSortedIndices, ConvolutionIndices} Documentation: * How to compute the `Jacobian, Hessian, Jacobian times a vector, Hessian times a vector `_. * Slide for a 3 hours class with exercises that was done at the HPCS2011 Conference in Montreal. Others: * Logger name renamed to be consistent. * Logger function simplified and made more consistent. * Fixed transformation of error by other not related error with the compute_test_value Theano flag. * Compilation cache enhancements. * Made compatible with NumPy 1.6 and SciPy 0.9 * Fix tests when there was new dtype in NumPy that is not supported by Theano. * Fixed some tests when SciPy is not available. * Don't compile anything when Theano is imported. Compile support code when we compile the first C code. * Python 2.4 fix: * Fix the file theano/misc/check_blas.py * For python 2.4.4 on Windows, replaced float("inf") with numpy.inf. * Removes useless inputs to a scan node * Beautification mostly, making the graph more visible. Such inputs would appear as a consequence of other optimizations Core: * there is a new mechanism that lets an Op permit that one of its inputs to be aliased to another destroyed input. This will generally result in incorrect calculation, so it should be used with care! The right way to use it is when the caller can guarantee that even if these two inputs look aliased, they actually will never overlap. This mechanism can be used, for example, by a new alternative approach to implementing Scan. If an op has an attribute called "destroyhandler_tolerate_aliased" then this is what's going on. IncSubtensor is thus far the only Op to use this mechanism.Mechanism Theano 0.4.0 (2011-06-13) ========================= Change in output memory storage for Ops: If you implemented custom Ops, with either C or Python implementation, this will concern you. The contract for memory storage of Ops has been changed. In particular, it is no longer guaranteed that output memory buffers are either empty, or allocated by a previous execution of the same Op. Right now, here is the situation: * For Python implementation (perform), what is inside output_storage may have been allocated from outside the perform() function, for instance by another node (e.g., Scan) or the Mode. If that was the case, the memory can be assumed to be C-contiguous (for the moment). * For C implementations (c_code), nothing has changed yet. In a future version, the content of the output storage, both for Python and C versions, will either be NULL, or have the following guarantees: * It will be a Python object of the appropriate Type (for a Tensor variable, a numpy.ndarray, for a GPU variable, a CudaNdarray, for instance) * It will have the correct number of dimensions, and correct dtype However, its shape and memory layout (strides) will not be guaranteed. When that change is made, the config flag DebugMode.check_preallocated_output will help you find implementations that are not up-to-date. Deprecation: * tag.shape attribute deprecated (#633) * CudaNdarray_new_null is deprecated in favour of CudaNdarray_New * Dividing integers with / is deprecated: use // for integer division, or cast one of the integers to a float type if you want a float result (you may also change this behavior with config.int_division). * Removed (already deprecated) sandbox/compile module * Removed (already deprecated) incsubtensor and setsubtensor functions, inc_subtensor and set_subtensor are to be used instead. Bugs fixed: * In CudaNdarray.__{iadd,idiv}__, when it is not implemented, return the error. * THEANO_FLAGS='optimizer=None' now works as expected * Fixed memory leak in error handling on GPU-to-host copy * Fix relating specifically to Python 2.7 on Mac OS X * infer_shape can now handle Python longs * Trying to compute x % y with one or more arguments being complex now raises an error. * The output of random samples computed with uniform(..., dtype=...) is guaranteed to be of the specified dtype instead of potentially being of a higher-precision dtype. * The perform() method of DownsampleFactorMax did not give the right result when reusing output storage. This happen only if you use the Theano flags 'linker=c|py_nogc' or manually specify the mode to be 'c|py_nogc'. Crash fixed: * Work around a bug in gcc 4.3.0 that make the compilation of 2d convolution crash. * Some optimizations crashed when the "ShapeOpt" optimization was disabled. Optimization: * Optimize all subtensor followed by subtensor. GPU: * Move to the gpu fused elemwise that have other dtype then float32 in them (except float64) if the input and output are float32. * This allow to move elemwise comparisons to the GPU if we cast it to float32 after that. * Implemented CudaNdarray.ndim to have the same interface in ndarray. * Fixed slowdown caused by multiple chained views on CudaNdarray objects * CudaNdarray_alloc_contiguous changed so as to never try to free memory on a view: new "base" property * Safer decref behaviour in CudaNdarray in case of failed allocations * New GPU implementation of tensor.basic.outer * Multinomial random variates now available on GPU New features: * ProfileMode * profile the scan overhead * simple hook system to add profiler * reordered the output to be in the order of more general to more specific * DebugMode now checks Ops with different patterns of preallocated memory, configured by config.DebugMode.check_preallocated_output. * var[vector of index] now work, (grad work recursively, the direct grad work inplace, gpu work) * limitation: work only of the outer most dimensions. * New way to test the graph as we build it. Allow to easily find the source of shape mismatch error: `http://deeplearning.net/software/theano/tutorial/debug_faq.html#interactive-debugger`__ * cuda.root inferred if nvcc is on the path, otherwise defaults to /usr/local/cuda * Better graph printing for graphs involving a scan subgraph * Casting behavior can be controlled through config.cast_policy, new (experimental) mode. * Smarter C module cache, avoiding erroneous usage of the wrong C implementation when some options change, and avoiding recompiling the same module multiple times in some situations. * The "theano-cache clear" command now clears the cache more thoroughly. * More extensive linear algebra ops (CPU only) that wrap scipy.linalg now available in the sandbox. * CUDA devices 4 - 16 should now be available if present. * infer_shape support for the View op, better infer_shape support in Scan * infer_shape supported in all case of subtensor * tensor.grad now gives an error by default when computing the gradient wrt a node that is disconnected from the cost (not in the graph, or no continuous path from that op to the cost). * New tensor.isnan and isinf functions. Documentation: * Better commenting of cuda_ndarray.cu * Fixes in the scan documentation: add missing declarations/print statements * Better error message on failed __getitem__ * Updated documentation on profile mode * Better documentation of testing on Windows * Better documentation of the 'run_individual_tests' script Unit tests: * More strict float comparaison by default * Reuse test for subtensor of tensor for gpu tensor(more gpu test) * Tests that check for aliased function inputs and assure appropriate copying (#374) * Better test of copies in CudaNdarray * New tests relating to the new base pointer requirements * Better scripts to run tests individually or in batches * Some tests are now run whenever cuda is available and not just when it has been enabled before * Tests display less pointless warnings. Other: * Correctly put the broadcast flag to True in the output var of a Reshape op when we receive an int 1 in the new shape. * pydotprint: high contrast mode is now the default, option to print more compact node names. * pydotprint: How trunk label that are too long. * More compact printing (ignore leading "Composite" in op names) Theano 0.3.1 (2011-02-21) ========================= Deprecation: * The theano shared variable attribute `value` is deprecated, use `get_value()` or `set_value()`! See http://deeplearning.net/software/theano/tutorial/aliasing.html Bugs fixed: * The random number generator in theano/sandbox/rng_mrg.py did not always return the same sequence of number on the CPU and GPU. * In some cases, there was a (possibly large) fraction of non-random garbage in the returned sequence. * In python mode (not the default mode) when input of elemwise operation was an empty ndarray, we were not returning an empty ndarray. * Scan cached the number of steps. This caused no problem because each time you called scan the number of steps would got refreshed. The problem was when you called ScanGrad which would use the cached number of steps without refreshing it. To be affected by this bug, one would have to compile two graph, one that would contain a Scan and the other the corresponding GradScan, and call the first function to cache the number of steps, and then call the second function with a different number of steps. * In GpuConv, errors in conv_patch_stack_reduce when the entire kernel doesn't fit into shared memory. The error was not found before as the impact was less then the relative tolerance of 1e-3. Now the relative tolerance is 1e-5. Crash fixed: * Add a feature to not have an exception that makes Theano crash when taking the gradient on DimShuffle in some particular case. * Compilation crash for GpuElemwise with tensor with high number of dimensions (~6 or more). * Disabled C code generator that make gcc crash on complex type. * Crash in optimization when an Op has no input. * Output shape is now computed correctly for matrix-vector multiplication on GPU. * In Scan, when using numbers as inputs, not symbolic variables. * In GradScan, when there is only 1 inputs in the Scan. * In GpuSum, bug in calculation of n_blocks for the 10 pattern. (Sum on the row of a matrix) * Some segfault at exit with GPU code. Optimization: * New SpecifyShape op that allow to pass more shape info in the graph. * Speed up gemv by a work around scipy gemv slowness when the matrix is in C order (the default). * Remove join of only 1 element. * During optimization, consider one more case in get_constant_value. GPU: * cuda_shared.value = X now works inplace! * cuda_shared_var.set_value(new_ndarray) will overwrite the old value inplace in the most common case. * Allow to create a CudaNdarraySharedVariable from a CudaNdarray. * New init_gpu_device theano flags. * Fuse GpuElemwise more often (in the case where there are so many inputs that fusing them all would bust the 256 bytes limit of parameter to gpu function). * CPU join of only 1 element that was not moved to the GPU. New features: * tensor.reshape now makes dimensions of length 1 broadcastable. * tensor.prod now implements the gradient. * DebugMode now warns if an Op declared itself as returning a view of the input but did not do so. * This behaviour is a problem, because it can block other Ops from being inplace on the same inputs. This could lower the reuse of memory. * Sparse.structured_dot now works when both matrices are sparse * Sparse type is now supported by the shape op, and the ShapeFeature optimizer works correctly with them. * New 3D convolution ops, with CPU and GPU implementations. * New colors in pydotprint. Documentation: * Documented lib.amdlibm and (new) init_gpu_device config variables. * A new page (was done for 0.3 but an error was hiding it on the web page) on the memory aliasing contract of Theano. * Revision to the Windows installation instructions. * The cuda documentation is now generated on the web server. * Better documentation of .theanorc and its sections. Unit tests: * Stop usage of deprecated functions or syntax in the unit tests. * Better testing of GPU convolution nets. * Make more tests able to use different random seeds. * Tests of sparse now use default mode, not a hard-coded one. * Remove some tests of unimplemented features. Other: * The name of compiledir now includes the Python version to make it easier for people with many Python versions * Added theano.tensor.std as a shortcut to sqrt(var(input=input, axis=axis)). * Whitespace, tabulation and indentation clean-up in the code. * Better detection of memory sharing between variables. Theano 0.3 (2010-11-23) ======================= This is the first major release of Theano since 0.1. Version 0.2 development started internally but it was never advertised as a release. There have been so many changes since 0.1 that we have lost track of many of them. Below is a *partial* list of changes since 0.1. * GPU code using NVIDIA's CUDA framework is now generated for many Ops. * Some interface changes since 0.1: * A new "shared variable" system to allow reusing memory space between Theano functions. * A new memory contract has been formally written for Theano, for people who want to minimize memory copies. * The old module system has been deprecated. * By default, inputs to a Theano function will not be silently downcasted (e.g. from float64 to float32). * An error is now raised when using the result of logical operation on Theano variable in an 'if' (i.e. an implicit call to __nonzeros__). * An error is now raised when we receive a non-aligned ndarray as input to a function (this is not supported). * An error is raised when the list of dimensions passed to dimshuffle() contains duplicates or is otherwise not sensible. * Call NumPy BLAS bindings for gemv operations in addition to the already supported gemm. * If gcc is unavailable at import time, Theano now falls back to a Python-based emulation mode after raising a warning. * An error is now raised when tensor.grad is called on a non-scalar Theano variable (in the past we would implicitly do a sum on the tensor to make it a scalar). * Added support for "erf" and "erfc" functions. * The current default value of the parameter axis of theano.{max,min,argmax,argmin,max_and_argmax} is deprecated. We now use the default NumPy behavior of operating on the entire tensor. * Theano is now available from PyPI and installable through "easy_install" or "pip". Theano 0.1 ========== *Release date: 2009-04-02* What works ---------- - building symbolic expression. - arranging symbolic expressions into Modules so that multiple functions can work on the same data. - symbolic gradient descent. - graph optimization. - compilation to C for many kinds of expression. - a debugging mode that checks that your expression results are correct, using a variety of sanity checks. What's missing? --------------- - An algorithm library. We're missing a library of examples and standard component implementations. Some examples will find their way into the Theano repo, but standard algorithms will go into the 'pylearn' project (toolbox style). Now that we have a stable foundation, we can reach a consensus on style for algorithms. Theano-1.0.0/LICENSE.txt00007770163562€ÿÿÿþ0000000000013203335527020513 2doc/LICENSE.txtustar boccoset00000000000000Theano-1.0.0/requirement-rtd.txt00006000163562€ÿÿÿþ0000000014013203335531020151 0ustar boccoset00000000000000sphinx>=1.3.0 pygments nose>=1.3.0 numpy gnumpy pydot pydot2 Cython parameterized scipy==0.13 Theano-1.0.0/NEWS.txt00006000163562€ÿÿÿþ0000002313313203335527015612 0ustar boccoset00000000000000============= Release Notes ============= Theano 1.0.0 (15th of November, 2017) ===================================== This is a final release of Theano, version ``1.0.0``, with a lot of new features, interface changes, improvements and bug fixes. We recommend that everybody update to this version. Highlights (since 0.9.0): - Announcing that `MILA will stop developing Theano `_ - conda packages now available and updated in our own conda channel ``mila-udem`` To install it: ``conda install -c mila-udem theano pygpu`` - Support NumPy ``1.13`` - Support pygpu ``0.7`` - Moved Python ``3.*`` minimum supported version from ``3.3`` to ``3.4`` - Added conda recipe - Replaced deprecated package ``nose-parameterized`` with up-to-date package ``parameterized`` for Theano requirements - Theano now internally uses ``sha256`` instead of ``md5`` to work on systems that forbid ``md5`` for security reason - Removed old GPU backend ``theano.sandbox.cuda``. New backend ``theano.gpuarray`` is now the official GPU backend - Make sure MKL uses GNU OpenMP - **NB**: Matrix dot product (``gemm``) with ``mkl`` from conda could return wrong results in some cases. We have reported the problem upstream and we have a work around that raises an error with information about how to fix it. - Improved elemwise operations - Speed-up elemwise ops based on SciPy - Fixed memory leaks related to elemwise ops on GPU - Scan improvements - Speed up Theano scan compilation and gradient computation - Added meaningful message when missing inputs to scan - Speed up graph toposort algorithm - Faster C compilation by massively using a new interface for op params - Faster optimization step, with new optional destroy handler - Documentation updated and more complete - Added documentation for RNNBlock - Updated ``conv`` documentation - Support more debuggers for ``PdbBreakpoint`` - Many bug fixes, crash fixes and warning improvements A total of 71 people contributed to this release since 0.9.0, see list below. Interface changes: - Merged duplicated diagonal functions into two ops: ``ExtractDiag`` (extract a diagonal to a vector), and ``AllocDiag`` (set a vector as a diagonal of an empty array) - Removed op ``ExtractDiag`` from ``theano.tensor.nlinalg``, now only in ``theano.tensor.basic`` - Generalized ``AllocDiag`` for any non-scalar input - Added new parameter ``target`` for MRG functions - Renamed ``MultinomialWOReplacementFromUniform`` to ``ChoiceFromUniform`` - Changed ``grad()`` method to ``L_op()`` in ops that need the outputs to compute gradient - Removed or deprecated Theano flags: - ``cublas.lib`` - ``cuda.enabled`` - ``enable_initial_driver_test`` - ``gpuarray.sync`` - ``home`` - ``lib.cnmem`` - ``nvcc.*`` flags - ``pycuda.init`` Convolution updates: - Implemented separable convolutions for 2D and 3D - Implemented grouped convolutions for 2D and 3D - Added dilated causal convolutions for 2D - Added unshared convolutions - Implemented fractional bilinear upsampling - Removed old ``conv3d`` interface - Deprecated old ``conv2d`` interface GPU: - Added a meta-optimizer to select the fastest GPU implementations for convolutions - Prevent GPU initialization when not required - Added disk caching option for kernels - Added method ``my_theano_function.sync_shared()`` to help synchronize GPU Theano functions - Added useful stats for GPU in profile mode - Added Cholesky op based on ``cusolver`` backend - Added GPU ops based on `magma library `_: SVD, matrix inverse, QR, cholesky and eigh - Added ``GpuCublasTriangularSolve`` - Added atomic addition and exchange for ``long long`` values in ``GpuAdvancedIncSubtensor1_dev20`` - Support log gamma function for all non-complex types - Support GPU SoftMax in both OpenCL and CUDA - Support offset parameter ``k`` for ``GpuEye`` - ``CrossentropyCategorical1Hot`` and its gradient are now lifted to GPU - cuDNN: - Official support for ``v6.*`` and ``v7.*`` - Added spatial transformation operation based on cuDNN - Updated and improved caching system for runtime-chosen cuDNN convolution algorithms - Support cuDNN v7 tensor core operations for convolutions with runtime timed algorithms - Better support and loading on Windows and Mac - Support cuDNN v6 dilated convolutions - Support cuDNN v6 reductions for contiguous inputs - Optimized ``SUM(x^2)``, ``SUM(ABS(X))`` and ``MAX(ABS(X))`` operations with cuDNN reductions - Added new Theano flags ``cuda.include_path``, ``dnn.base_path`` and ``dnn.bin_path`` to help configure Theano when CUDA and cuDNN can not be found automatically - Extended Theano flag ``dnn.enabled`` with new option ``no_check`` to help speed up cuDNN importation - Disallowed ``float16`` precision for convolution gradients - Fixed memory alignment detection - Added profiling in C debug mode (with theano flag ``cmodule.debug=True``) - Added Python scripts to help test cuDNN convolutions - Automatic addition of cuDNN DLL path to ``PATH`` environment variable on Windows - Updated ``float16`` support - Added documentation for GPU float16 ops - Support ``float16`` for ``GpuGemmBatch`` - Started to use ``float32`` precision for computations that don't support ``float16`` on GPU New features: - Implemented truncated normal distribution with box-muller transform - Added ``L_op()`` overriding option for ``OpFromGraph`` - Added NumPy C-API based fallback implementation for ``[sd]gemv_`` and ``[sd]dot_`` - Implemented ``topk`` and ``argtopk`` on CPU and GPU - Implemented ``max()`` and ``min()`` functions for booleans and unsigned integers types - Added ``tensor6()`` and ``tensor7()`` in ``theano.tensor`` module - Added boolean indexing for sub-tensors - Added covariance matrix function ``theano.tensor.cov`` - Added a wrapper for `Baidu's CTC `_ cost and gradient functions - Added scalar and elemwise CPU ops for modified Bessel function of order 0 and 1 from ``scipy.special`` - Added Scaled Exponential Linear Unit (SELU) activation - Added sigmoid_binary_crossentropy function - Added tri-gamma function - Added ``unravel_index`` and ``ravel_multi_index`` functions on CPU - Added modes ``half`` and ``full`` for ``Images2Neibs`` ops - Implemented gradient for ``AbstractBatchNormTrainGrad`` - Implemented gradient for matrix pseudoinverse op - Added new prop `replace` for ``ChoiceFromUniform`` op - Added new prop ``on_error`` for CPU ``Cholesky`` op - Added new Theano flag ``deterministic`` to help control how Theano optimize certain ops that have deterministic versions. Currently used for subtensor Ops only. - Added new Theano flag ``cycle_detection`` to speed-up optimization step by reducing time spending in inplace optimizations - Added new Theano flag ``check_stack_trace`` to help check the stack trace during optimization process - Added new Theano flag ``cmodule.debug`` to allow a debug mode for Theano C code. Currently used for cuDNN convolutions only. - Added new Theano flag ``pickle_test_value`` to help disable pickling test values Others: - Kept stack trace for optimizations in new GPU backend - Added deprecation warning for the softmax and logsoftmax vector case - Added a warning to announce that C++ compiler will become mandatory in next Theano release ``0.11`` - Added ``R_op()`` for ``ZeroGrad`` - Added description for rnnblock Other more detailed changes: - Fixed invalid casts and index overflows in ``theano.tensor.signal.pool`` - Fixed gradient error for elemwise ``minimum`` and ``maximum`` when compared values are the same - Fixed gradient for ``ARange`` - Removed ``ViewOp`` subclass during optimization - Removed useless warning when profile is manually disabled - Added tests for abstract conv - Added options for `disconnected_outputs` to Rop - Removed ``theano/compat/six.py`` - Removed ``COp.get_op_params()`` - Support of list of strings for ``Op.c_support_code()``, to help not duplicate support codes - Macro names provided for array properties are now standardized in both CPU and GPU C codes - Moved all C code files into separate folder ``c_code`` in every Theano module - Many improvements for Travis CI tests (with better splitting for faster testing) - Many improvements for Jenkins CI tests: daily testings on Mac and Windows in addition to Linux Commiters since 0.9.0: - Frederic Bastien - Steven Bocco - João Victor Tozatti Risso - Arnaud Bergeron - Mohammed Affan - amrithasuresh - Pascal Lamblin - Reyhane Askari - Alexander Matyasko - Shawn Tan - Simon Lefrancois - Adam Becker - Vikram - Gijs van Tulder - Faruk Ahmed - Thomas George - erakra - Andrei Costinescu - Boris Fomitchev - Zhouhan LIN - Aleksandar Botev - jhelie - xiaoqie - Tegan Maharaj - Matt Graham - Cesar Laurent - Gabe Schwartz - Juan Camilo Gamboa Higuera - Tim Cooijmans - Anirudh Goyal - Saizheng Zhang - Yikang Shen - vipulraheja - Florian Bordes - Sina Honari - Chiheb Trabelsi - Shubh Vachher - Daren Eiri - Joseph Paul Cohen - Laurent Dinh - Mohamed Ishmael Diwan Belghazi - Jeff Donahue - Ramana Subramanyam - Bogdan Budescu - Dzmitry Bahdanau - Ghislain Antony Vaillant - Jan Schlüter - Nan Jiang - Xavier Bouthillier - fo40225 - mrTsjolder - wyjw - Aarni Koskela - Adam Geitgey - Adrian Keet - Adrian Seyboldt - Anmol Sahoo - Chong Wu - Holger Kohr - Jayanth Koushik - Lilian Besson - Lv Tao - Michael Manukyan - Murugesh Marvel - NALEPA - Rebecca N. Palmer - Zotov Yuriy - dareneiri - lrast - morrme - naitonium Theano-1.0.0/setup.cfg00006000163562€ÿÿÿþ0000000045313203336031016105 0ustar boccoset00000000000000[nosetest] match = ^test nocapture = 1 [flake8] ignore = E501,E123,E133,FI12,FI14,FI15,FI50,FI51,FI53 [versioneer] vcs = git style = pep440 versionfile_source = theano/_version.py versionfile_build = theano/_version.py tag_prefix = rel- [egg_info] tag_build = tag_date = 0 tag_svn_revision = 0 Theano-1.0.0/MANIFEST.in00006000163562€ÿÿÿþ0000000046713203335527016040 0ustar boccoset00000000000000global-include *.txt global-include *.c global-include *.cu global-include *.cuh global-include *.cpp global-include *.h global-include *.sh global-include *.pkl recursive-include doc include bin/theano-cache include bin/theano-nose prune .jenkins prune .travis include versioneer.py include theano/_version.py Theano-1.0.0/EMAIL.txt00006000163562€ÿÿÿþ0000000703313203335527015666 0ustar boccoset00000000000000=========================== Announcing Theano 0.5 =========================== ## You can select and adapt one of the following templates. ## Basic text for major version release: This is a release for a major version, with lots of new features, bug fixes, and some interface changes (deprecated or potentially misleading features were removed). Upgrading to Theano 0.5 is recommended for everyone, but you should first make sure that your code does not raise deprecation warnings with Theano 0.4.1. Otherwise, in one case the results can change. In other cases, the warnings are turned into errors (see below for details). For those using the bleeding edge version in the git repository, we encourage you to update to the `0.5` tag. ## Basic text for major version release candidate: This is a release candidate for a major version, with lots of new features, bug fixes, and some interface changes (deprecated or potentially misleading features were removed). The upgrade is recommended for developers who want to help test and report bugs, or want to use new features now. If you have updated to 0.5rc1, you are highly encouraged to update to 0.5rc2. For those using the bleeding edge version in the git repository, we encourage you to update to the `0.5rc2` tag. ## Basic text for minor version release: TODO ## Basic text for minor version release candidate: TODO What's New ---------- [Include the content of NEWS.txt here] Download and Install -------------------- You can download Theano from http://pypi.python.org/pypi/Theano Installation instructions are available at http://deeplearning.net/software/theano/install.html Description ----------- Theano is a Python library that allows you to define, optimize, and efficiently evaluate mathematical expressions involving multi-dimensional arrays. It is built on top of NumPy. Theano features: * tight integration with NumPy: a similar interface to NumPy's. numpy.ndarrays are also used internally in Theano-compiled functions. * transparent use of a GPU: perform data-intensive computations much faster than on a CPU. * efficient symbolic differentiation: Theano can compute derivatives for functions of one or many inputs. * speed and stability optimizations: avoid nasty bugs when computing expressions such as log(1+ exp(x)) for large values of x. * dynamic C code generation: evaluate expressions faster. * extensive unit-testing and self-verification: includes tools for detecting and diagnosing bugs and/or potential problems. Theano has been powering large-scale computationally intensive scientific research since 2007, but it is also approachable enough to be used in the classroom (IFT6266 at the University of Montreal). Resources --------- About Theano: http://deeplearning.net/software/theano/ Theano-related projects: http://github.com/Theano/Theano/wiki/Related-projects About NumPy: http://numpy.scipy.org/ About SciPy: http://www.scipy.org/ Machine Learning Tutorial with Theano on Deep Architectures: http://deeplearning.net/tutorial/ Acknowledgments --------------- I would like to thank all contributors of Theano. For this particular release, many people have helped, notably (in alphabetical order): [Generate the list of commiters: git shortlog -s ...| cut -c8-] I would also like to thank users who submitted bug reports, notably: [TODO] Also, thank you to all NumPy and Scipy developers as Theano builds on their strengths. All questions/comments are always welcome on the Theano mailing-lists ( http://deeplearning.net/software/theano/#community ) Theano-1.0.0/theano/00007500163562€ÿÿÿþ0000000000013203336031015543 5ustar boccoset00000000000000Theano-1.0.0/theano/scalar/00007500163562€ÿÿÿþ0000000000013203336026017014 5ustar boccoset00000000000000Theano-1.0.0/theano/scalar/tests/00007500163562€ÿÿÿþ0000000000013203336026020156 5ustar boccoset00000000000000Theano-1.0.0/theano/scalar/tests/__init__.py00006000163562€ÿÿÿþ0000000000013203335532022253 0ustar boccoset00000000000000Theano-1.0.0/theano/scalar/tests/test_div_future.py00006000163562€ÿÿÿþ0000000062113203335532023740 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import theano import theano.tensor as T import unittest class test_FutureDiv(unittest.TestCase): def test_divide_floats(self): a = T.dscalar('a') b = T.dscalar('b') c = theano.function([a, b], b / a) d = theano.function([a, b], b // a) assert c(6, 3) == 0.5 assert d(6, 3) == 0.0 Theano-1.0.0/theano/scalar/tests/test_div_no_future.py00006000163562€ÿÿÿþ0000000062013203335532024433 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import theano import theano.tensor as T import unittest class test_FutureDiv(unittest.TestCase): def test_divide_floats(self): a = T.dscalar('a') b = T.dscalar('b') c = theano.function([a, b], b / a) d = theano.function([a, b], b // a) assert c(6, 3) == 0.5 assert d(6, 3) == 0.0 Theano-1.0.0/theano/scalar/tests/test_basic_sympy.py00006000163562€ÿÿÿþ0000000203213203335532024104 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import theano from theano.scalar.basic_sympy import SymPyCCode from theano.scalar.basic import floats from nose.plugins.skip import SkipTest try: import sympy xs = sympy.Symbol('x') ys = sympy.Symbol('y') except ImportError: raise SkipTest('optional package sympy disabled') xt, yt = floats('xy') def test_SymPyCCode(): if not theano.config.cxx: raise SkipTest("Need cxx for this test") op = SymPyCCode([xs, ys], xs + ys) e = op(xt, yt) g = theano.gof.FunctionGraph([xt, yt], [e]) fn = theano.gof.CLinker().accept(g).make_function() assert fn(1.0, 2.0) == 3.0 def test_grad(): op = SymPyCCode([xs], xs**2) zt = op(xt) ztprime = theano.grad(zt, xt) assert ztprime.owner.op.expr == 2 * xs def test_multivar_grad(): op = SymPyCCode([xs, ys], xs ** 2 + ys ** 3) zt = op(xt, yt) dzdx, dzdy = theano.grad(zt, [xt, yt]) assert dzdx.owner.op.expr == 2 * xs assert dzdy.owner.op.expr == 3 * ys ** 2 Theano-1.0.0/theano/scalar/tests/test_basic.py00006000163562€ÿÿÿþ0000004230113203335532022646 0ustar boccoset00000000000000""" These routines are not well-tested. They are also old. OB says that it is not important to test them well because Scalar Ops are rarely used by themselves, instead they are the basis for Tensor Ops (which should be checked thoroughly). Moreover, Scalar will be changed to use numpy's scalar routines. If you do want to rewrite these tests, bear in mind: * You don't need to use Composite. * FunctionGraph and DualLinker are old, use compile.function instead. """ from __future__ import absolute_import, print_function, division import unittest import numpy as np import theano from theano.gof import FunctionGraph from theano import gof from theano.tests import unittest_tools as utt from theano.scalar.basic import (floats, float16, float32, float64, ints, int8, int32, complex64, uint8, ComplexError, IntDiv, TrueDiv, Composite, add, div_proxy, and_, eq, neq, invert, mul, Scalar, InRange, cast, constant, switch) from theano.scalar.basic import ( true_div, inv, log, log2, log10, log1p, exp, exp2, expm1, sqrt, deg2rad, rad2deg, cos, arccos, sin, arcsin, tan, arctan, arctan2, cosh, arccosh, sinh, arcsinh, tanh, arctanh) def inputs(): return floats('xyz') class test_ScalarOps(unittest.TestCase): def test_straightforward(self): x, y, z = inputs() e = mul(add(x, y), div_proxy(x, y)) g = FunctionGraph([x, y], [e]) fn = gof.DualLinker().accept(g).make_function() assert fn(1.0, 2.0) == 1.5 # This test is moved to theano.tensor.tests.test_basic.py:test_mod # We move it their as under ubuntu the c_extract call of theano.scalar # call PyInt_check and it fail under some os. If work in other case. # As we use theano.scalar normally, but we use theano.tensor.scalar # that is not important. Also this make the theano fct fail at call time # so this is not a silent bug. # --> This is why it is purposedly named 'tes_mod' instead of 'test_mod'. def tes_mod(self): # We add this test as not all language and C implementation give the same # sign to the result. This check that the c_code of `Mod` is implemented # as Python. That is what we want. x, y = ints('xy') fn = gof.DualLinker().accept(FunctionGraph([x, y], [x % y])).make_function() for a, b in ((0, 1), (1, 1), (0, -1), (1, -1), (-1, -1), (1, 2), (-1, 2), (1, -2), (-1, -2), (5, 3), (-5, 3), (5, -3), (-5, -3) ): self.assertTrue(fn(a, b) == a % b, (a,)) def has_f16(comp): if any(v.type == float16 for v in comp.fgraph.variables): return True return False class test_composite(unittest.TestCase): def test_composite_clone_float32(self): w = int8() x = float16() y = float32() cz = Composite([x, y], [tanh(x + cast(y, 'float16'))]) c = Composite([w, x, y], [cz(x, y) - cz(x, y)**2 + cast(x, 'int16') + cast(x, 'float32') + cast(w, 'float16') - constant(np.float16(1.0))]) assert has_f16(c) nc = c.clone_float32() assert not has_f16(nc) v = uint8() w = float16() x = float16() y = float16() z = float16() c = Composite([v, w, x, y, z], [switch(v, mul(w, x, y), z)]) assert has_f16(c) nc = c.clone_float32() assert not has_f16(nc) def test_straightforward(self): x, y, z = inputs() e = mul(add(x, y), div_proxy(x, y)) C = Composite([x, y], [e]) c = C.make_node(x, y) # print c.c_code(['x', 'y'], ['z'], dict(id = 0)) g = FunctionGraph([x, y], [c.out]) fn = gof.DualLinker().accept(g).make_function() assert fn(1.0, 2.0) == 1.5 def test_flatten(self): # Test that we flatten multiple Composite. x, y, z = inputs() C = Composite([x, y], [x + y]) CC = Composite([x, y], [C(x * y, y)]) assert not isinstance(CC.outputs[0].owner.op, Composite) # Test with multiple outputs CC = Composite([x, y, z], [C(x * y, y), C(x * z, y)]) # We don't flatten that case. assert isinstance(CC.outputs[0].owner.op, Composite) def test_with_constants(self): x, y, z = inputs() e = mul(add(70.0, y), div_proxy(x, y)) C = Composite([x, y], [e]) c = C.make_node(x, y) assert "70.0" in c.op.c_code(c, 'dummy', ['x', 'y'], ['z'], dict(id=0)) # print c.c_code(['x', 'y'], ['z'], dict(id = 0)) g = FunctionGraph([x, y], [c.out]) fn = gof.DualLinker().accept(g).make_function() assert fn(1.0, 2.0) == 36.0 def test_many_outputs(self): x, y, z = inputs() e0 = x + y + z e1 = x + y * z e2 = x / y C = Composite([x, y, z], [e0, e1, e2]) c = C.make_node(x, y, z) # print c.c_code(['x', 'y', 'z'], ['out0', 'out1', 'out2'], dict(id = 0)) g = FunctionGraph([x, y, z], c.outputs) fn = gof.DualLinker().accept(g).make_function() assert fn(1.0, 2.0, 3.0) == [6.0, 7.0, 0.5] def test_composite_printing(self): x, y, z = floats('xyz') e0 = x + y + z e1 = x + y * z e2 = x / y e3 = x // 5 e4 = -x e5 = x - y e6 = x ** y + (-z) e7 = x % 3 C = Composite([x, y, z], [e0, e1, e2, e3, e4, e5, e6, e7]) c = C.make_node(x, y, z) g = FunctionGraph([x, y, z], c.outputs) gof.DualLinker().accept(g).make_function() assert str(g) == ('[*1 -> Composite{((i0 + i1) + i2),' ' (i0 + (i1 * i2)), (i0 / i1), ' '(i0 // Constant{5}), ' '(-i0), (i0 - i1), ((i0 ** i1) + (-i2)),' ' (i0 % Constant{3})}(x, y, z), ' '*1::1, *1::2, *1::3, *1::4, *1::5, *1::6, *1::7]') def test_make_node_continue_graph(self): # This is a test for a bug (now fixed) that disabled the # local_gpu_elemwise_0 optimization and printed an # optimization warning on the terminal. # We test that Composite.make_node accept as inputs Variable # some that represent existing computation. si0 = theano.scalar.int8() si1 = theano.scalar.int8() si2 = theano.scalar.float32() sout = (si0 * si1) / si2 sop = theano.scalar.Composite([si0, si1, si2], [sout]) si0 = theano.scalar.int8() si1 = theano.scalar.int8() si2 = theano.scalar.float32() si3 = theano.scalar.float32() sop.make_node(si0 * si3, si1, si2) class test_logical(unittest.TestCase): def test_gt(self): x, y, z = inputs() fn = gof.DualLinker().accept(FunctionGraph([x, y], [x > y])).make_function() for a, b in ((3., 9), (3, 0.9), (3, 3)): self.assertTrue(fn(a, b) == (a > b)) def test_lt(self): x, y, z = inputs() fn = gof.DualLinker().accept(FunctionGraph([x, y], [x < y])).make_function() for a, b in ((3., 9), (3, 0.9), (3, 3)): self.assertTrue(fn(a, b) == (a < b)) def test_le(self): x, y, z = inputs() fn = gof.DualLinker().accept(FunctionGraph([x, y], [x <= y])).make_function() for a, b in ((3., 9), (3, 0.9), (3, 3)): self.assertTrue(fn(a, b) == (a <= b)) def test_ge(self): x, y, z = inputs() fn = gof.DualLinker().accept(FunctionGraph([x, y], [x >= y])).make_function() for a, b in ((3., 9), (3, 0.9), (3, 3)): self.assertTrue(fn(a, b) == (a >= b)) def test_eq(self): x, y, z = inputs() fn = gof.DualLinker().accept(FunctionGraph([x, y], [eq(x, y)])).make_function() for a, b in ((3., 9), (3, 0.9), (3, 3)): self.assertTrue(fn(a, b) == (a == b)) def test_neq(self): x, y, z = inputs() fn = gof.DualLinker().accept(FunctionGraph([x, y], [neq(x, y)])).make_function() for a, b in ((3., 9), (3, 0.9), (3, 3)): self.assertTrue(fn(a, b) == (a != b)) def test_or(self): x, y, z = ints('xyz') fn = gof.DualLinker().accept(FunctionGraph([x, y], [x | y])).make_function() for a, b in ((0, 1), (0, 0), (1, 0), (1, 1)): self.assertTrue(fn(a, b) == (a | b), (a, b)) def test_xor(self): x, y, z = ints('xyz') fn = gof.DualLinker().accept(FunctionGraph([x, y], [x ^ y])).make_function() for a, b in ((0, 1), (0, 0), (1, 0), (1, 1)): self.assertTrue(fn(a, b) == (a ^ b), (a, b)) def test_and(self): x, y, z = ints('xyz') fn = gof.DualLinker().accept(FunctionGraph([x, y], [and_(x, y)])).make_function() for a, b in ((0, 1), (0, 0), (1, 0), (1, 1)): self.assertTrue(fn(a, b) == (a & b), (a, b)) x, y, z = ints('xyz') fn = gof.DualLinker().accept(FunctionGraph([x, y], [x & y])).make_function() for a, b in ((0, 1), (0, 0), (1, 0), (1, 1)): self.assertTrue(fn(a, b) == (a & b), (a, b)) def test_not(self): x, y, z = ints('xyz') fn = gof.DualLinker().accept(FunctionGraph([x, y], [invert(x)])).make_function() for a, b in ((0, 1), (0, 0), (1, 0), (1, 1)): self.assertTrue(fn(a, b) == ~a, (a,)) x, y, z = ints('xyz') fn = gof.DualLinker().accept(FunctionGraph([x, y], [~x])).make_function() for a, b in ((0, 1), (0, 0), (1, 0), (1, 1)): self.assertTrue(fn(a, b) == ~a, (a,)) # This class does not inherit from unittest.TestCase, because it would # interfere with the "yield" mechanism that automatically generates test, see # http://stackoverflow.com/questions/6689537/nose-test-generators-inside-class # Therefore, it needs to be named "test_..." or "Test_...", so nose can pick # it up by name, otherwise the tests would not be executed. class test_upgrade_to_float(object): # Test for Ops whose output has to be floating point, even when all # inputs are ints. # In particular, when the inputs are int8, the output should be # at least float32, not float16. unary_ops_vals = [ (inv, list(range(-127, 0)) + list(range(1, 127))), (sqrt, list(range(0, 128))), (log, list(range(1, 128))), (log2, list(range(1, 128))), (log10, list(range(1, 128))), (log1p, list(range(0, 128))), (exp, list(range(-127, 89))), (exp2, list(range(-127, 89))), (expm1, list(range(-127, 89))), (deg2rad, list(range(-127, 128))), (rad2deg, list(range(-127, 128))), (cos, list(range(-127, 128))), (arccos, list(range(-1, 2))), (cosh, list(range(-89, 90))), (arccosh, list(range(1, 128))), (sin, list(range(-127, 128))), (arcsin, list(range(-1, 2))), (sinh, list(range(-89, 90))), (arcsinh, list(range(-127, 128))), (tan, list(range(-3, 4))), (arctan, list(range(-127, 128))), (tanh, list(range(-127, 128))), (arctanh, [0])] binary_ops_vals = [ (arctan2, list(range(-127, 128)), list(range(-127, 128)))] @staticmethod def _test_unary(unary_op, x_range): xi = int8('xi') xf = float32('xf') ei = unary_op(xi) fi = theano.function([xi], ei) ef = unary_op(xf) ff = theano.function([xf], ef) for x_val in x_range: outi = fi(x_val) outf = ff(x_val) assert outi.dtype == outf.dtype, 'incorrect dtype' assert np.allclose(outi, outf), 'insufficient precision' @staticmethod def _test_binary(binary_op, x_range, y_range): xi = int8('xi') yi = int8('yi') xf = float32('xf') yf = float32('yf') ei = binary_op(xi, yi) fi = theano.function([xi, yi], ei) ef = binary_op(xf, yf) ff = theano.function([xf, yf], ef) for x_val in x_range: for y_val in y_range: outi = fi(x_val, y_val) outf = ff(x_val, y_val) assert outi.dtype == outf.dtype, 'incorrect dtype' assert np.allclose(outi, outf), 'insufficient precision' def test_true_div(self): # true_div's upcast policy is not exactly "upgrade_to_float", # so the test is a little bit different x_range = list(range(-127, 128)) y_range = list(range(-127, 0)) + list(range(1, 127)) xi = int8('xi') yi = int8('yi') xf = Scalar(theano.config.floatX)('xf') yf = Scalar(theano.config.floatX)('yf') ei = true_div(xi, yi) fi = theano.function([xi, yi], ei) ef = true_div(xf, yf) ff = theano.function([xf, yf], ef) for x_val in x_range: for y_val in y_range: outi = fi(x_val, y_val) outf = ff(x_val, y_val) assert outi.dtype == outf.dtype, 'incorrect dtype' assert np.allclose(outi, outf), 'insufficient precision' def test_unary(self): # Automatically define all individual unary tests for unary_op, x_range in self.unary_ops_vals: test_name = 'test_%s' % unary_op.name def test(): self._test_unary(unary_op, x_range) test.description = test_name yield test def test_binary(self): # Automatically define all individual binary tests for binary_op, x_range, y_range in self.binary_ops_vals: test_name = 'test_%s' % binary_op.name def test(): self._test_binary(binary_op, x_range, y_range) test.description = test_name yield test class test_complex_mod(unittest.TestCase): # Make sure % fails on complex numbers. def test_fail(self): x = complex64() y = int32() try: x % y assert False except ComplexError: pass class test_div(unittest.TestCase): def test_0(self): a = int8() b = int32() c = complex64() d = float64() f = float32() assert isinstance((a // b).owner.op, IntDiv) assert isinstance((b // a).owner.op, IntDiv) assert isinstance((b / d).owner.op, TrueDiv) assert isinstance((b / f).owner.op, TrueDiv) assert isinstance((f / a).owner.op, TrueDiv) assert isinstance((d / b).owner.op, TrueDiv) assert isinstance((d / f).owner.op, TrueDiv) assert isinstance((f / c).owner.op, TrueDiv) assert isinstance((a / c).owner.op, TrueDiv) def test_grad_gt(): x = float32(name='x') y = float32(name='y') z = x > y g = theano.gradient.grad(z, y) assert g.eval({y: 1.}) == 0. def test_grad_switch(): # This is a code snippet from the mailing list # It caused an assert to be raised due to the # switch op's grad method not handling integer # inputs correctly x = theano.tensor.matrix() c = theano.tensor.matrix() s = theano.tensor.switch(c, x, 0) l = s.sum() theano.gradient.grad(l, x) def test_grad_identity(): # Check that the grad method of Identity correctly handles int dytpes x = theano.tensor.imatrix('x') # tensor_copy is Elemwise{Identity} y = theano.tensor.tensor_copy(x) l = y.sum(dtype=theano.config.floatX) theano.gradient.grad(l, x) def test_grad_inrange(): for bound_definition in [(True, True), (False, False)]: # Instantiate op, and then take the gradient op = InRange(*bound_definition) x = theano.tensor.fscalar('x') low = theano.tensor.fscalar('low') high = theano.tensor.fscalar('high') out = op(x, low, high) gx, glow, ghigh = theano.tensor.grad(out, [x, low, high]) # We look if the gradient are equal to zero # if x is lower than the lower bound, # equal to the lower bound, between lower and higher bound, # equal to the higher bound and higher than the higher # bound. # Mathematically we should have an infinite gradient when # x is equal to the lower or higher bound but in that case # Theano defines the gradient to be zero for stability. f = theano.function([x, low, high], [gx, glow, ghigh]) utt.assert_allclose(f(0, 1, 5), [0, 0, 0]) utt.assert_allclose(f(1, 1, 5), [0, 0, 0]) utt.assert_allclose(f(2, 1, 5), [0, 0, 0]) utt.assert_allclose(f(5, 1, 5), [0, 0, 0]) utt.assert_allclose(f(7, 1, 5), [0, 0, 0]) def test_grad_abs(): a = theano.tensor.fscalar("a") b = theano.tensor.nnet.relu(a) c = theano.grad(b, a) f = theano.function([a], c, mode=theano.Mode(optimizer=None)) # Currently Theano return 0.5, but it isn't sure it won't change # in the futur. ret = f(0.) assert ret == 0.5, ret # Testing of Composite is done in tensor/tests/test_opt.py # in test_fusion, TestCompositeCodegen def test_constant(): c = constant(2, name='a') assert c.name == 'a' assert c.dtype == 'int8' c = constant(2, dtype='float32') assert c.name is None assert c.dtype == 'float32' if __name__ == '__main__': unittest.main() Theano-1.0.0/theano/scalar/sharedvar.py00006000163562€ÿÿÿþ0000000413713203335532021350 0ustar boccoset00000000000000""" A shared variable container for true scalars - for internal use. Why does this file exist? ------------------------- Scalars are used to index subtensors. Subtensor indexing is the heart of what looks like the new scan mechanism. This little file made it possible to catch up to the Python interpreter in benchmarking tests. We don't want to encourage people to use scalars (rather than 0-d tensors), but the reason is just to keep the docs simple, not because scalars are bad. If we just don't register this shared variable constructor to handle any values by default when calling theano.shared(value) then users must really go out of their way (as scan does) to create a shared variable of this kind. """ from __future__ import absolute_import, print_function, division import numpy as np from six import integer_types from theano.compile import SharedVariable from .basic import Scalar, _scalar_py_operators __authors__ = "James Bergstra" __copyright__ = "(c) 2010, Universite de Montreal" __license__ = "3-clause BSD License" __contact__ = "theano-dev " __docformat__ = "restructuredtext en" class ScalarSharedVariable(_scalar_py_operators, SharedVariable): pass # this is not installed in the default shared variable registry so that # scalars are typically 0-d tensors. # still, in case you need a shared variable scalar, you can get one # by calling this function directly. def shared(value, name=None, strict=False, allow_downcast=None): """ SharedVariable constructor for scalar values. Default: int64 or float64. Notes ----- We implement this using 0-d tensors for now. """ if not isinstance(value, (np.number, float, integer_types, complex)): raise TypeError() try: dtype = value.dtype except AttributeError: dtype = np.asarray(value).dtype dtype = str(dtype) value = getattr(np, dtype)(value) scalar_type = Scalar(dtype=dtype) rval = ScalarSharedVariable( type=scalar_type, value=value, name=name, strict=strict, allow_downcast=allow_downcast) return rval Theano-1.0.0/theano/scalar/basic.py00006000163562€ÿÿÿþ0000042207313203335532020455 0ustar boccoset00000000000000""" .. warning:: This directory is for the internal of Theano. You are strongly advised not to use it, except if you know what you are doing! If you want to use a scalar variable in a Theano graph, you probably want to use theano.tensor.[c,z,f,d,b,w,i,l,]scalar! """ from __future__ import absolute_import, print_function, division from itertools import chain import math import warnings from copy import copy from textwrap import dedent import numpy as np import six from six.moves import xrange import theano from theano.compat import imap, izip from theano import gof, printing from theano.gof import (Op, utils, Variable, Constant, Type, Apply, FunctionGraph) from functools import partial from theano import config from theano.gradient import DisconnectedType from theano.gradient import grad_undefined from theano.printing import pprint import collections builtin_bool = bool builtin_complex = complex builtin_int = int builtin_float = float class ComplexError(NotImplementedError): """ Raised if complex numbers are used in an unsupported operation. """ pass class IntegerDivisionError(Exception): """ Raised if someone tries to divide integers with '/' instead of '//'. """ pass def upcast(dtype, *dtypes): # This tries to keep data in floatX or lower precision, unless we # explicitely request a higher precision datatype. keep_float32 = [(config.cast_policy == 'numpy+floatX' and config.floatX == 'float32')] keep_float16 = [(config.cast_policy == 'numpy+floatX' and config.floatX == 'float16')] def make_array(dt): if dt == 'float64': # There is an explicit float64 dtype: we cannot keep float32. keep_float32[0] = False keep_float16[0] = False if dt == 'float32': keep_float16[0] = False return np.zeros((), dtype=dt) z = make_array(dtype) for dt in dtypes: z = z + make_array(dt=dt) rval = str(z.dtype) if rval == 'float64': if keep_float16[0]: return 'float16' if keep_float32[0]: return 'float32' elif rval == 'float32': if keep_float16[0]: return 'float16' return rval def as_common_dtype(*vars): """ For for theano.scalar.Scalar and TensorVariable. """ dtype = upcast(*[v.dtype for v in vars]) return (v.astype(dtype) for v in vars) def get_scalar_type(dtype): """ Return a Scalar(dtype) object. This caches objects to save allocation and run time. """ if dtype not in get_scalar_type.cache: get_scalar_type.cache[dtype] = Scalar(dtype=dtype) return get_scalar_type.cache[dtype] get_scalar_type.cache = {} def as_scalar(x, name=None): from ..tensor import TensorType, scalar_from_tensor if isinstance(x, gof.Apply): if len(x.outputs) != 1: raise ValueError("It is ambiguous which output of a multi-output" " Op has to be fetched.", x) else: x = x.outputs[0] if isinstance(x, Variable): if isinstance(x.type, Scalar): return x elif isinstance(x.type, TensorType) and x.ndim == 0: return scalar_from_tensor(x) else: raise TypeError("Variable type field must be a Scalar.", x, x.type) try: return constant(x) except TypeError: raise TypeError("Cannot convert %s to Scalar" % x, type(x)) class NumpyAutocaster(object): """ This class is used to cast python ints and floats to numpy arrays. The behavior when called on scalar `x` depends on `config.cast_policy`: - 'numpy' will simply use the same type as found by `numpy.asarray(x)`. - 'numpy+floatX' will do the same, except it will use float32 instead of float64 if `x` is a Python float and `config.floatX` is set to 'float32' (note that if `x` is a numpy scalar whose data type is float64, it is not modified since we assume the user is purposedly using float64). - 'custom' lets one define a tuple of data types such that: - if `x` is already a numpy scalar and its data type is in this tuple, then it is returned unchanged; - otherwise, the first data type in this tuple that can represent `x` without loss of precision will be used, unless `x` is a float and 'float32' is in the tuple (in which case `x` is cast as a float32); - if no data type can represent `x` without loss of precision, then the last data type in the tuple will be used. Parameters ---------- dtypes: tuple of strings The ordered list of preferred data types (only used when `config.cast_policy` is set to 'custom', see the `NumpyAutocaster` help for details). """ def __init__(self, dtypes): self.dtypes = tuple(dtypes) def __call__(self, x): # Make sure we only deal with scalars. assert (isinstance(x, six.integer_types) or isinstance(x, builtin_float) or (isinstance(x, np.ndarray) and x.ndim == 0)) if config.cast_policy == 'numpy': return np.asarray(x) elif config.cast_policy == 'numpy+floatX': rval = np.asarray(x) if ((not hasattr(x, 'dtype') and rval.dtype in ('float64', 'float32') and rval.dtype != config.floatX)): rval = theano._asarray(rval, dtype=config.floatX) return rval # The following is the original code, corresponding to the 'custom' # option for `config.cast_policy`. assert config.cast_policy == 'custom' try: # Pass through numpy scalars, since they are already typed on # purpose typically. if str(x.dtype) in self.dtypes: # No need to cast `x` into a new dtype. Note that we still # need to convert it into an array, because it may not be # one already (e.g. if x == numpy.float64(1.1)). return np.asarray(x) except AttributeError: # Means `x` has no 'dtype' attribute. pass # unsafe downcast of float64 variables when config.floatX == 'float32' # recall: float is numpy.float if ((isinstance(x, float) and config.floatX in self.dtypes and config.floatX != 'float64')): return theano._asarray(x, dtype=config.floatX) # Don't autocast to float16 unless config.floatX is float16 try_dtypes = [d for d in self.dtypes if config.floatX == 'float16' or d != 'float16'] for dtype in try_dtypes: x_ = theano._asarray(x, dtype=dtype) if np.all(x == x_): break # returns either an exact x_==x, or the last cast x_ return x_ autocast_int = NumpyAutocaster(('int8', 'int16', 'int32', 'int64')) # autocast_float dtypes might be manipulated in tensor.* autocast_float = NumpyAutocaster(('float16', 'float32', 'float64')) class autocast_float_as(object): """ Temporarily adjust autocasting behavior. This class makes it possible to temporarily and locally adjust autocasting behavior when `config.cast_policy` is set to 'custom'. If `config.cast_policy` is not 'custom', an exception is raised. This class might be convenient in some code, but it definitely helps to test the autocasting mechanism. Examples -------- >>> with autocast_float_as('float32'): ... assert (fvector() + 1.1).dtype == 'float32' # temporary downcasting >>> assert (fvector() + 1.1).dtype == 'float64' # back to default behaviour """ def __init__(self, *dtypes): self.dtypes = dtypes assert config.cast_policy == 'custom' def __enter__(self): assert config.cast_policy == 'custom' self.old_dtypes = autocast_float.dtypes autocast_float.dtypes = self.dtypes def __exit__(self, *args): assert config.cast_policy == 'custom' autocast_float.dtypes = self.old_dtypes def convert(x, dtype=None): """ Convert the input to a properly typed numpy value according to the current casting policy. Work with scalars and tensors. """ if dtype is not None: # in this case, the semantics are that the caller is forcing the dtype x_ = theano._asarray(x, dtype=dtype) else: # In this case, this function should infer the dtype according to the # autocasting rules. See autocasting above. x_ = None if isinstance(x, six.integer_types): try: x_ = autocast_int(x) except OverflowError: # This is to imitate numpy behavior which tries to fit # bigger numbers into a uint64. x_ = theano._asarray(x, dtype='uint64') elif isinstance(x, builtin_float): x_ = autocast_float(x) elif isinstance(x, np.ndarray): x_ = x else: # Here x is probably a list or a tuple. If it contains a # long, we will behave like the current NumPy version: it # will work if the long fits in int64 or uint64. x_ = np.asarray(x) if x_.size == 0 and not hasattr(x, 'dtype'): x_ = np.asarray(x, dtype=config.floatX) assert type(x_) in [np.ndarray, np.memmap] return x_ def constant(x, name=None, dtype=None): x = convert(x, dtype=dtype) assert x.ndim == 0 return ScalarConstant(get_scalar_type(str(x.dtype)), x, name=name) class Scalar(Type): """ Internal class, should not be used by clients. Primarily used by tensor.elemwise and tensor.reduce. Analogous to TensorType, but for zero-dimensional objects. Maps directly to C primitives. TODO: refactor to be named ScalarType for consistency with TensorType. """ __props__ = ('dtype',) ndim = 0 def __init__(self, dtype): if dtype == 'floatX': dtype = config.floatX self.dtype = dtype self.dtype_specs() # error checking @staticmethod def may_share_memory(a, b): # This class represent basic c type, represented in python # with numpy.scalar. They are read only. So from python, they # can never share memory. return False def filter(self, data, strict=False, allow_downcast=None): py_type = self.dtype_specs()[0] if strict and not isinstance(data, py_type): raise TypeError("%s expected a %s, got %s of type %s" % ( self, py_type, data, type(data)), data) try: converted_data = py_type(data) if (allow_downcast or (allow_downcast is None and type(data) is float and self.dtype == theano.config.floatX) or data == converted_data): return py_type(data) else: raise TypeError('Value cannot accurately be converted to dtype' ' (%s) and allow_downcast is not True' % self.dtype) except Exception as e: raise TypeError("Could not convert %s (value=%s) to %s" % ( type(data), data, self.dtype), e) def values_eq_approx(self, a, b, tolerance=1e-4): # The addition have risk of overflow especially with [u]int8 if self.dtype == 'bool': return a == b diff = a - b if diff == 0: return True return abs(diff) <= (abs(a) * tolerance) + (abs(b) * tolerance) def c_element_type(self): return self.dtype_specs()[1] def c_headers(self, c_compiler): l = [''] # These includes are needed by Scalar and TensorType, # we declare them here and they will be re-used by TensorType l.append('') l.append('') if config.lib.amdlibm and c_compiler.supports_amdlibm: l += [''] return l def c_libraries(self, c_compiler): l = [] if config.lib.amdlibm and c_compiler.supports_amdlibm: l += ['amdlibm'] return l def c_compile_args(self, c_compiler): if config.lib.amdlibm and c_compiler.supports_amdlibm: return ['-DREPLACE_WITH_AMDLIBM'] else: return [] def dtype_specs(self): try: # To help debug dtype/typenum problem, here is code to get # the list of numpy typenum. This list change between 32 # and 64 bit platform and probably also also between # Windows and Linux. # NOTE: equivalent type on a platform can have different typenum. # This is the source of all dtype/typenum problem found up to # now, as Theano always expect the exact typenum that # correspond to our supported dtype. """ for dtype in ['bool', 'int8', 'uint8', 'short', 'ushort', 'intc', 'uintc', 'longlong', 'ulonglong', 'single', 'double', 'longdouble', 'csingle', 'cdouble', 'clongdouble', 'float32', 'float64', 'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', 'complex64', 'complex128', 'float', 'double', 'int', 'uint']: print(dtype, np.zeros(1, dtype=dtype).dtype.num) """ return { # dtype: (py_type, c_type, cls_name) 'float16': (np.float16, 'npy_float16', 'Float16'), 'float32': (np.float32, 'npy_float32', 'Float32'), 'float64': (np.float64, 'npy_float64', 'Float64'), 'complex128': (np.complex128, 'theano_complex128', 'Complex128'), 'complex64': (np.complex64, 'theano_complex64', 'Complex64'), 'bool': (np.bool_, 'npy_bool', 'Bool'), 'uint8': (np.uint8, 'npy_uint8', 'UInt8'), 'int8': (np.int8, 'npy_int8', 'Int8'), 'uint16': (np.uint16, 'npy_uint16', 'UInt16'), 'int16': (np.int16, 'npy_int16', 'Int16'), 'uint32': (np.uint32, 'npy_uint32', 'UInt32'), 'int32': (np.int32, 'npy_int32', 'Int32'), 'uint64': (np.uint64, 'npy_uint64', 'UInt64'), 'int64': (np.int64, 'npy_int64', 'Int64') }[self.dtype] except KeyError: raise TypeError("Unsupported dtype for %s: %s" % ( self.__class__.__name__, self.dtype)) def upcast(self, *others): return upcast(*[x.dtype for x in [self] + list(others)]) def make_variable(self, name=None): return ScalarVariable(self, name=name) def __str__(self): return str(self.dtype) def __repr__(self): return "Scalar(%s)" % self.dtype def c_literal(self, data): if 'complex' in self.dtype: raise NotImplementedError("No literal for complex values.") if self.dtype == 'bool': return '1' if data else '0' return str(data) def c_declare(self, name, sub, check_input=True): if(check_input): pre = """ typedef %(dtype)s dtype_%(name)s; """ % dict(name=name, dtype=self.dtype_specs()[1]) else: pre = "" return pre + """ %(dtype)s %(name)s; """ % dict(name=name, dtype=self.dtype_specs()[1]) def c_init(self, name, sub): return """ %(name)s = 0; """ % locals() def c_extract(self, name, sub, check_input=True): if self.dtype == 'float16': # This doesn't work at the numpy level raise NotImplementedError('float16') specs = self.dtype_specs() if(check_input): pre = """ if (!PyObject_TypeCheck(py_%(name)s, &%(pyarr_type)s)) { PyErr_Format(PyExc_ValueError, "Scalar check failed (%(dtype)s)"); %(fail)s } """ % dict(sub, name=name, dtype=specs[1], pyarr_type='Py%sArrType_Type' % specs[2]) else: pre = "" return pre + """ PyArray_ScalarAsCtype(py_%(name)s, &%(name)s); """ % dict(sub, name=name) def c_sync(self, name, sub): specs = self.dtype_specs() return """ Py_XDECREF(py_%(name)s); py_%(name)s = PyArrayScalar_New(%(cls)s); if (!py_%(name)s) { Py_XINCREF(Py_None); py_%(name)s = Py_None; PyErr_Format(PyExc_MemoryError, "Instantiation of new Python scalar failed (%(dtype)s)"); %(fail)s } PyArrayScalar_ASSIGN(py_%(name)s, %(cls)s, %(name)s); """ % dict(sub, name=name, dtype=specs[1], cls=specs[2]) def c_cleanup(self, name, sub): return "" def c_support_code(self): if self.dtype.startswith('complex'): cplx_types = ['theano_complex64', 'theano_complex128'] real_types = ['npy_int8', 'npy_int16', 'npy_int32', 'npy_int64', 'npy_float32', 'npy_float64'] # If the 'int' C type is not exactly the same as an existing # 'npy_intX', some C code may not compile, e.g. when assigning # the value 0 (cast to 'int' in C) to a theano_complex64. if (np.dtype('intc').num not in [np.dtype(d[4:]).num for d in real_types]): # In that case we add the 'int' type to the real types. real_types.append('int') template = """ struct theano_complex%(nbits)s : public npy_complex%(nbits)s { typedef theano_complex%(nbits)s complex_type; typedef npy_float%(half_nbits)s scalar_type; complex_type operator +(const complex_type &y) const { complex_type ret; ret.real = this->real + y.real; ret.imag = this->imag + y.imag; return ret; } complex_type operator -() const { complex_type ret; ret.real = -this->real; ret.imag = -this->imag; return ret; } bool operator ==(const complex_type &y) const { return (this->real == y.real) && (this->imag == y.imag); } bool operator ==(const scalar_type &y) const { return (this->real == y) && (this->imag == 0); } complex_type operator -(const complex_type &y) const { complex_type ret; ret.real = this->real - y.real; ret.imag = this->imag - y.imag; return ret; } complex_type operator *(const complex_type &y) const { complex_type ret; ret.real = this->real * y.real - this->imag * y.imag; ret.imag = this->real * y.imag + this->imag * y.real; return ret; } complex_type operator /(const complex_type &y) const { complex_type ret; scalar_type y_norm_square = y.real * y.real + y.imag * y.imag; ret.real = (this->real * y.real + this->imag * y.imag) / y_norm_square; ret.imag = (this->imag * y.real - this->real * y.imag) / y_norm_square; return ret; } template complex_type& operator =(const T& y); theano_complex%(nbits)s() {} template theano_complex%(nbits)s(const T& y) { *this = y; } template theano_complex%(nbits)s(const TR& r, const TI& i) { this->real=r; this->imag=i; } }; """ def operator_eq_real(mytype, othertype): return ''' template <> %(mytype)s & %(mytype)s::operator=<%(othertype)s>(const %(othertype)s & y) { this->real=y; this->imag=0; return *this; } ''' % dict(mytype=mytype, othertype=othertype) def operator_eq_cplx(mytype, othertype): return ''' template <> %(mytype)s & %(mytype)s::operator=<%(othertype)s>(const %(othertype)s & y) { this->real=y.real; this->imag=y.imag; return *this; } ''' % dict(mytype=mytype, othertype=othertype) operator_eq = (''.join(operator_eq_real(ctype, rtype) for ctype in cplx_types for rtype in real_types) + ''.join(operator_eq_cplx(ctype1, ctype2) for ctype1 in cplx_types for ctype2 in cplx_types)) # We are not using C++ generic templating here, because this would # generate two different functions for adding a complex64 and a # complex128, one returning a complex64, the other a complex128, # and the compiler complains it is ambiguous. # Instead, we generate code for known and safe types only. def operator_plus_real(mytype, othertype): return ''' const %(mytype)s operator+(const %(mytype)s &x, const %(othertype)s &y) { return %(mytype)s(x.real+y, x.imag); } const %(mytype)s operator+(const %(othertype)s &y, const %(mytype)s &x) { return %(mytype)s(x.real+y, x.imag); } ''' % dict(mytype=mytype, othertype=othertype) operator_plus = ''.join(operator_plus_real(ctype, rtype) for ctype in cplx_types for rtype in real_types) def operator_minus_real(mytype, othertype): return ''' const %(mytype)s operator-(const %(mytype)s &x, const %(othertype)s &y) { return %(mytype)s(x.real-y, x.imag); } const %(mytype)s operator-(const %(othertype)s &y, const %(mytype)s &x) { return %(mytype)s(y-x.real, -x.imag); } ''' % dict(mytype=mytype, othertype=othertype) operator_minus = ''.join(operator_minus_real(ctype, rtype) for ctype in cplx_types for rtype in real_types) def operator_mul_real(mytype, othertype): return ''' const %(mytype)s operator*(const %(mytype)s &x, const %(othertype)s &y) { return %(mytype)s(x.real*y, x.imag*y); } const %(mytype)s operator*(const %(othertype)s &y, const %(mytype)s &x) { return %(mytype)s(x.real*y, x.imag*y); } ''' % dict(mytype=mytype, othertype=othertype) operator_mul = ''.join(operator_mul_real(ctype, rtype) for ctype in cplx_types for rtype in real_types) return (template % dict(nbits=64, half_nbits=32) + template % dict(nbits=128, half_nbits=64) + operator_eq + operator_plus + operator_minus + operator_mul) else: return "" def c_init_code(self): return ["import_array();"] def c_code_cache_version(self): return (13, np.__version__) def get_shape_info(self, obj): return obj.itemsize def get_size(self, shape_info): return shape_info # Register C code for ViewOp on Scalars. theano.compile.register_view_op_c_code( Scalar, """ %(oname)s = %(iname)s; """, 1) bool = get_scalar_type('bool') int8 = get_scalar_type('int8') int16 = get_scalar_type('int16') int32 = get_scalar_type('int32') int64 = get_scalar_type('int64') uint8 = get_scalar_type('uint8') uint16 = get_scalar_type('uint16') uint32 = get_scalar_type('uint32') uint64 = get_scalar_type('uint64') float16 = get_scalar_type('float16') float32 = get_scalar_type('float32') float64 = get_scalar_type('float64') complex64 = get_scalar_type('complex64') complex128 = get_scalar_type('complex128') int_types = int8, int16, int32, int64 uint_types = uint8, uint16, uint32, uint64 float_types = float16, float32, float64 complex_types = complex64, complex128 integer_types = int_types + uint_types discrete_types = (bool,) + integer_types continuous_types = float_types + complex_types all_types = discrete_types + continuous_types class _scalar_py_operators: # So that we can simplify checking code when we have a mixture of Scalar # variables and Tensor variables ndim = 0 dtype = property(lambda self: self.type.dtype) """The dtype of this scalar.""" # UNARY def __abs__(self): return abs_(self) def __neg__(self): return neg(self) # CASTS # def __int__(self): return AsInt(self).out # def __float__(self): return AsDouble(self).out # def __complex__(self): return AsComplex(self).out # BITWISE def __invert__(self): return invert(self) def __and__(self, other): return and_(self, other) def __or__(self, other): return or_(self, other) def __xor__(self, other): return xor(self, other) def __rand__(self, other): return and_(other, self) def __ror__(self, other): return or_(other, self) def __rxor__(self, other): return xor(other, self) # COMPARISONS def __lt__(self, other): return lt(self, other) def __le__(self, other): return le(self, other) def __gt__(self, other): return gt(self, other) def __ge__(self, other): return ge(self, other) # ARITHMETIC - NORMAL def __add__(self, other): return add(self, other) def __sub__(self, other): return sub(self, other) def __mul__(self, other): return mul(self, other) def __truediv__(self, other): return div_proxy(self, other) def __div__(self, other): return div_proxy(self, other) def __floordiv__(self, other): return int_div(self, other) def __mod__(self, other): return mod_check(self, other) def __pow__(self, other): return pow(self, other) # ARITHMETIC - RIGHT-OPERAND def __radd__(self, other): return add(other, self) def __rsub__(self, other): return sub(other, self) def __rmul__(self, other): return mul(other, self) def __rdiv__(self, other): return div_proxy(other, self) def __rmod__(self, other): return mod(other, self) def __rpow__(self, other): return pow(other, self) def zeros_like(self, dtype=None): # The second is needed for Elemwise ops to work right if dtype is None: dtype = str(self.type.dtype) return second(self, ScalarConstant(get_scalar_type(dtype), 0)) def ones_like(self, dtype=None): # The second is needed for Elemwise ops to work right if dtype is None: dtype = str(self.type.dtype) return second(self, ScalarConstant(get_scalar_type(dtype), 1)) def astype(self, dtype): return cast(self, dtype) class ScalarVariable(_scalar_py_operators, Variable): pass class ScalarConstant(_scalar_py_operators, Constant): pass # Register ScalarConstant as the type of Constant corresponding to Scalar Scalar.Constant = ScalarConstant # Easy constructors def _multi(*fns): def f2(f, names): if len(names) == 1: return f(names) else: return [f(name) for name in names] if len(fns) == 1: return partial(f2, fns[0]) else: return [partial(f2, f) for f in fns] ints = _multi(int64) floats = _multi(float64) complexs = _multi(complex128) complexs64 = _multi(complex64) complexs128 = _multi(complex128) def upcast_out(*types): dtype = Scalar.upcast(*types) return get_scalar_type(dtype), def upcast_out_nobool(*types): type = upcast_out(*types) if type[0] == bool: raise TypeError("bool output not supported") return type def upcast_out_min8(*types): type = upcast_out(*types) if type[0] == bool: return int8, return type def upgrade_to_float(*types): """ Upgrade any int types to float32 or float64 to avoid losing precision. """ conv = {bool: float32, int8: float32, int16: float32, int32: float64, int64: float64, uint8: float32, uint16: float32, uint32: float64, uint64: float64} return get_scalar_type(Scalar.upcast(*[conv.get(type, type) for type in types])), def upgrade_to_float64(*types): """ Upgrade any int and float32 to float64 to do as SciPy. """ return get_scalar_type('float64'), def same_out(type): return type, def same_out_nobool(type): if type == bool: raise TypeError("bool input not supported") return type, def same_out_min8(type): if type == bool: return int8, return type, def upcast_out_no_complex(*types): if any([type in complex_types for type in types]): raise TypeError('complex type are not supported') return get_scalar_type(dtype=Scalar.upcast(*types)), def same_out_float_only(type): if type not in float_types: raise TypeError('only float type are supported') return type, class transfer_type(gof.utils.object2): __props__ = ('transfer',) def __init__(self, *transfer): assert all(type(x) in [int, str] or x is None for x in transfer) self.transfer = transfer def __str__(self): return 'transfer_type{%s}' % self.transfer def __call__(self, *types): upcast = upcast_out(*types) retval = [] for i in self.transfer: if i is None: retval += [upcast] elif isinstance(i, str): retval += [i] else: retval += [types[i]] return retval # return [upcast if i is None else types[i] for i in self.transfer] class specific_out(gof.utils.object2): __props__ = ('spec',) def __init__(self, *spec): self.spec = spec def __call__(self, *types): return self.spec def int_out(*types): return int64, def float_out(*types): return float64, def upgrade_to_float_no_complex(*types): """ Don't accept complex, otherwise call upgrade_to_float(). """ for type in types: if type in complex_types: raise TypeError('complex argument not supported') return upgrade_to_float(*types) def same_out_nocomplex(type): if type in complex_types: raise TypeError('complex argument not supported') return type, def int_out_nocomplex(*types): for type in types: if type in complex_types: raise TypeError('complex argument not supported') return int64, def float_out_nocomplex(*types): for type in types: if type in complex_types: raise TypeError('complex argument not supported') return float64, class unary_out_lookup(gof.utils.object2): """ Get a output_types_preference object by passing a dictionary: unary_out_lookup({int8:int32, float32:complex128}) The result is an op that maps in8 to int32 and float32 to complex128 and other input types lead to a TypeError. """ def __init__(self, type_table): self.tbl = type_table def __call__(self, *types): if len(types) == 1: types = types[0] try: rval = self.tbl[types] except Exception: raise TypeError(types) if isinstance(types, (list, tuple)): return rval else: return [rval] def __eq__(self, other): return type(self) == type(other) and self.tbl == other.tbl def __hash__(self): return hash(type(self)) # ignore hash of table def real_out(type): if type == complex64: return float32, if type == complex128: return float64, return type, class ScalarOp(Op): nin = -1 nout = 1 def __init__(self, output_types_preference=None, name=None): self.name = name if output_types_preference is not None: if not isinstance(output_types_preference, collections.Callable): raise TypeError( "Expected a callable for the 'output_types_preference' argument to %s. (got: %s)" % (self.__class__, output_types_preference)) self.output_types_preference = output_types_preference def make_node(self, *inputs): if self.nin >= 0: if len(inputs) != self.nin: raise TypeError("Wrong number of inputs for %s.make_node (got %i(%s), expected %i)" % (self, len(inputs), str(inputs), self.nin)) inputs = [as_scalar(input) for input in inputs] outputs = [t() for t in self.output_types([input.type for input in inputs])] if len(outputs) != self.nout: raise TypeError("Not the right number of outputs produced for %s(%s). Expected %s, got %s." % (self, ", ".join(str(input) for input in inputs), self.nout, len(outputs))) return Apply(self, inputs, outputs) def output_types(self, types): if hasattr(self, 'output_types_preference'): variables = self.output_types_preference(*types) if not isinstance(variables, (list, tuple)) or any(not isinstance(x, Type) for x in variables): raise TypeError( "output_types_preference should return a list or a tuple of types", self.output_types_preference, variables) if len(variables) != self.nout: raise TypeError("Not the right number of outputs types produced for %s(%s) by %s. Expected %s, got %s." % (self, ", ".join(str(type) for type in variables), self.output_types_preference, self.nout, len(variables))) return variables else: raise NotImplementedError( "Cannot calculate the output types for %s" % self) def perform(self, node, inputs, output_storage): if self.nout == 1: output_storage[0][0] = self.impl(*inputs) else: variables = utils.from_return_values(self.impl(*inputs)) assert len(variables) == len(output_storage) for storage, variable in zip(output_storage, variables): storage[0] = variable def impl(self, *inputs): raise utils.MethodNotDefined("impl", type(self), self.__class__.__name__) def grad(self, inputs, output_gradients): raise utils.MethodNotDefined("grad", type(self), self.__class__.__name__) def L_op(self, inputs, outputs, output_gradients): return self.grad(inputs, output_gradients) def __eq__(self, other): test = (type(self) == type(other) and getattr(self, 'output_types_preference', None) == getattr(other, 'output_types_preference', None)) return test def __hash__(self): return hash((type(self), getattr(self, 'output_types_preference', 0))) def __str__(self): if hasattr(self, 'name') and self.name: return self.name else: param = [(k, v) for k, v in self.__dict__.items() if k not in ["name", "_op_use_c_code", "bool", "output_types_preference"]] if param: return "%s{%s}" % (self.__class__.__name__, ", ".join("%s=%s" % (k, v) for k, v in param)) else: return self.__class__.__name__ def c_code_cache_version(self): return (4,) def c_code_contiguous(self, node, name, inp, out, sub): """ This function is called by Elemwise when all inputs and outputs are c_contiguous. This allows to use the SIMD version of this op. The inputs are the same as c_code except that: - inp and out must be the names of the variables associated to the ndarrays in the C code - node must be the elemwise node (this is needed to know the inputs/outputs types) """ raise theano.gof.utils.MethodNotDefined() def supports_c_code(self, inputs, outputs): """Returns True if the current op has functioning C code for the given Elemwise inputs, outputs. """ try: tmp_s_input = [] # To keep the same aliasing between inputs mapping = dict() for ii in inputs: if ii in mapping: tmp_s_input.append(mapping[ii]) else: tmp = get_scalar_type(ii.dtype).make_variable() tmp_s_input.append(tmp) mapping[ii] = tmp_s_input[-1] with theano.change_flags(compute_test_value='ignore'): s_op = self(*tmp_s_input, return_list=True) # if the scalar_op don't have a c implementation, # we skip its fusion to allow the fusion of the # other ops. self.c_code(s_op[0].owner, "test_presence_of_c_code", ["x" for x in inputs], ["z" for z in outputs], {"fail": "%(fail)s"}) except (theano.gof.utils.MethodNotDefined, NotImplementedError): return False return True class UnaryScalarOp(ScalarOp): nin = 1 amd_float32 = None amd_float64 = None def c_code_contiguous(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if (not theano.config.lib.amdlibm or # We compare the dtype AND the broadcast flag # as this function do not broadcast node.inputs[0].type != node.outputs[0].type): raise theano.gof.utils.MethodNotDefined() dtype = node.inputs[0].type.dtype_specs()[1] fct_call = self.c_code_contiguous_raw(dtype, 'n', 'x', 'z') return """ { npy_intp n = PyArray_SIZE(%(z)s); %(dtype)s * x = (%(dtype)s*) PyArray_DATA(%(x)s); %(dtype)s * z = (%(dtype)s*) PyArray_DATA(%(z)s); %(fct_call)s; } """ % locals() def c_code_contiguous_raw(self, dtype, n, i, o): if not config.lib.amdlibm: raise theano.gof.utils.MethodNotDefined() if dtype.startswith('npy_'): dtype = dtype[4:] if dtype == 'float32' and self.amd_float32 is not None: dtype = 'float' fct = self.amd_float32 elif dtype == 'float64' and self.amd_float64 is not None: dtype = 'double' fct = self.amd_float64 else: raise theano.gof.utils.MethodNotDefined() return "%(fct)s(%(n)s, %(i)s, %(o)s)" % locals() class BinaryScalarOp(ScalarOp): # One may define in subclasses the following fields: # - `identity`: for an associative operation, identity corresponds to # the neutral element. For instance, it will be 0 for addition, 1 for # multiplication, True for "and", False for "or". # - `commutative`: whether op(a, b) == op(b, a) # - `associative`: whether op(op(a, b), c) == op(a, op(b, c)) nin = 2 ############### # Comparisons ############### class LogicalComparison(BinaryScalarOp): def __init__(self, *args, **kwargs): BinaryScalarOp.__init__(self, *args, **kwargs) # This is for compat with old pickles. self.bool = True def __eq__(self, other): return (BinaryScalarOp.__eq__(self, other) and getattr(self, 'bool', False) == getattr(other, 'bool', False)) def __hash__(self): # bool should always be True return BinaryScalarOp.__hash__(self) def output_types(self, *input_dtypes): return [bool] if getattr(self, 'bool', False) else [int8] def L_op(self, inputs, outputs, output_gradients): x, y = inputs assert outputs[0].type == bool return [x.zeros_like().astype(theano.config.floatX), y.zeros_like().astype(theano.config.floatX)] def c_code_cache_version(self): super_version = super(LogicalComparison, self).c_code_cache_version() return super_version + (0,) class FixedLogicalComparison(UnaryScalarOp): """ Comparison to a fixed value. """ def __init__(self, *args, **kwargs): UnaryScalarOp.__init__(self, *args, **kwargs) # This is for compat with old pickles self.bool = True def __eq__(self, other): return (UnaryScalarOp.__eq__(self, other) and getattr(self, 'bool', False) == getattr(other, 'bool', False)) def __hash__(self): # bool should always be True return UnaryScalarOp.__hash__(self) def output_types(self, *input_dtypes): return [bool] if getattr(self, 'bool', False) else [int8] def L_op(self, inputs, outputs, output_gradients): x, = inputs assert outputs[0].type == bool return [x.zeros_like().astype(theano.config.floatX)] def c_code_cache_version(self): super_version = super(FixedLogicalComparison, self).c_code_cache_version() return super_version + (0,) class LT(LogicalComparison): identity = False commutative = False associative = False nfunc_spec = ('less', 2, 1) def impl(self, x, y): # built-in < don't support complex return np.less(x, y) def c_code(self, node, name, inputs, outputs, sub): (x, y) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError() return "%(z)s = (%(x)s < %(y)s);" % locals() lt = LT() class GT(LogicalComparison): identity = False commutative = False associative = False nfunc_spec = ('greater', 2, 1) def impl(self, x, y): # built-in > don't support complex return np.greater(x, y) def c_code(self, node, name, inputs, outputs, sub): (x, y) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError() return "%(z)s = (%(x)s > %(y)s);" % locals() gt = GT() class LE(LogicalComparison): identity = False commutative = False associative = False nfunc_spec = ('less_equal', 2, 1) def impl(self, x, y): # built-in <= don't support complex return np.less_equal(x, y) def c_code(self, node, name, inputs, outputs, sub): (x, y) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError() return "%(z)s = (%(x)s <= %(y)s);" % locals() le = LE() class GE(LogicalComparison): identity = False commutative = False associative = False nfunc_spec = ('greater_equal', 2, 1) def impl(self, x, y): # built-in >= don't support complex return np.greater_equal(x, y) def c_code(self, node, name, inputs, outputs, sub): (x, y) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError() return "%(z)s = (%(x)s >= %(y)s);" % locals() ge = GE() class EQ(LogicalComparison): identity = False commutative = True associative = False nfunc_spec = ('equal', 2, 1) def impl(self, x, y): return x == y def c_code(self, node, name, inputs, outputs, sub): (x, y) = inputs (z,) = outputs return "%(z)s = (%(x)s == %(y)s);" % locals() eq = EQ() class NEQ(LogicalComparison): identity = False commutative = True associative = False nfunc_spec = ('not_equal', 2, 1) def impl(self, x, y): return x != y def c_code(self, node, name, inputs, outputs, sub): (x, y) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError() return "%(z)s = (%(x)s != %(y)s);" % locals() neq = NEQ() class IsNan(FixedLogicalComparison): nfunc_spec = ('isnan', 1, 1) def impl(self, x): return np.isnan(x) def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError() # Discrete type can never be nan if node.inputs[0].type in discrete_types: return "%(z)s = false;" % locals() # Windows tries to be different and sometimes return -1, but we want # to be consistent with numpy (which returns True), hence the "abs". return "%(z)s = abs(isnan(%(x)s));" % locals() def c_code_cache_version(self): scalarop_version = super(IsNan, self).c_code_cache_version() return tuple(scalarop_version) + (3,) isnan = IsNan() class IsInf(FixedLogicalComparison): nfunc_spec = ('isinf', 1, 1) def impl(self, x): return np.isinf(x) def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError() # Discrete type can never be inf if node.inputs[0].type in discrete_types: return "%(z)s = false;" % locals() # Note that the C isinf returns -1 for -Inf and +1 for +Inf, while # numpy simply returns True: we mimic numpy's behavior here, thus # the absolute value. return "%(z)s = abs(isinf(%(x)s));" % locals() def c_code_cache_version(self): scalarop_version = super(IsInf, self).c_code_cache_version() return tuple(scalarop_version) + (3,) isinf = IsInf() class InRange(LogicalComparison): nin = 3 def __init__(self, openlow, openhi): self.openlow = openlow self.openhi = openhi def impl(self, x, low, hi): if self.openlow and x <= low: return False elif not self.openlow and x < low: return False if self.openhi and x >= hi: return False elif not self.openhi and x > hi: return False return True def c_code(self, node, name, inputs, outputs, sub): (x, low, hi) = inputs (z,) = outputs cmp1 = '>' if self.openlow else '>=' cmp2 = '<' if self.openhi else '<=' return ("%(z)s = %(x)s %(cmp1)s %(low)s &&" " %(x)s %(cmp2)s %(hi)s;" % locals()) def get_grad(self, elem): if elem.type in complex_types: msg = ("No gradient implemented for complex numbers in " "class scalar.basic.InRange") raise NotImplementedError(msg) elif elem.type in discrete_types: return elem.zeros_like().astype(theano.config.floatX) else: return elem.zeros_like() def L_op(self, inputs, outputs, gout): (x, low, hi) = inputs (gz,) = gout grads = [] for elem in [x, low, hi]: grads.append(self.get_grad(elem)) return grads inopenrange = InRange(True, True) inclosedrange = InRange(False, False) class Switch(ScalarOp): nin = 3 nfunc_spec = ('where', 3, 1) def impl(self, cond, ift, iff): return ift if cond else iff def c_code(self, node, name, inputs, outputs, sub): (cond, ift, iff) = inputs (z,) = outputs return "%(z)s = %(cond)s ? %(ift)s : %(iff)s;" % locals() def L_op(self, inputs, outputs, gout): (cond, ift, iff) = inputs (gz,) = gout first_part = switch(cond, gz, 0.) second_part = switch(cond, 0., gz) if (outputs[0].type.dtype in discrete_types): first_part = 0. second_part = 0. # cond does affect the elements of the output so it is connected. # For the sake of making the gradient convenient we assume that # condition + epsilon always triggers the same branch as condition condition_grad = cond.zeros_like().astype(theano.config.floatX) return (condition_grad, first_part, second_part) def output_types(self, types): (cond_t, ift_t, iff_t) = types return upcast_out(ift_t, iff_t) switch = Switch() #################### # BIT-WISE OPERATORS #################### class UnaryBitOp(UnaryScalarOp): def output_types(self, *input_types): for i in input_types[0]: if i not in discrete_types: raise TypeError('input to a BitOp must have type (u)int8, ' '(u)int16, (u)int32 or (u)int64 or bool not %s' % i) return upcast_out(*input_types[0]) def grad(self, inputs, output_gradients): return [inputs[0].zeros_like().astype(theano.config.floatX)] class BinaryBitOp(BinaryScalarOp): def output_types(self, *input_types): t0, t1 = input_types[0] if t0 == bool and t1 == bool: return [bool] for i in input_types[0]: if i not in integer_types: raise TypeError('input to a BitOp must have type (u)int8, ' '(u)int16, (u)int32 or (u)int64 or ' 'be all bools not %s' % i) return upcast_out(*input_types[0]) def grad(self, inputs, output_gradients): a, b = inputs return [a.zeros_like().astype(theano.config.floatX), b.zeros_like().astype(theano.config.floatX)] class OR(BinaryBitOp): identity = 0 commutative = True associative = True nfunc_spec = ('bitwise_or', 2, 1) def impl(self, x, y): return x | y def c_code(self, node, name, inputs, outputs, sub): (x, y) = inputs (z,) = outputs return "%(z)s = (%(x)s | %(y)s);" % locals() or_ = OR() class XOR(BinaryBitOp): identity = 0 commutative = True associative = True nfunc_spec = ('bitwise_xor', 2, 1) def impl(self, x, y): return x ^ y def c_code(self, node, name, inputs, outputs, sub): (x, y) = inputs (z,) = outputs return "%(z)s = (%(x)s ^ %(y)s);" % locals() xor = XOR() class AND(BinaryBitOp): identity = -1 commutative = True associative = True nfunc_spec = ('bitwise_and', 2, 1) def impl(self, x, y): return x & y def c_code(self, node, name, inputs, outputs, sub): (x, y) = inputs (z,) = outputs return "%(z)s = (%(x)s & %(y)s);" % locals() def c_code_cache_version(self): super_version = super(AND, self).c_code_cache_version() return super_version + (3,) and_ = AND() class Invert(UnaryBitOp): nfunc_spec = ('invert', 1, 1) def impl(self, x): return ~x def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.outputs[0].type == bool: return "%(z)s = (!%(x)s);" % locals() return "%(z)s = (~%(x)s);" % locals() invert = Invert() ############## # Arithmetic ############## class Maximum(BinaryScalarOp): commutative = True associative = True nfunc_spec = ('maximum', 2, 1) def impl(self, *inputs): # The built-in max function don't support complex type return np.maximum(*inputs) def c_code(self, node, name, inputs, outputs, sub): (x, y) = inputs (z,) = outputs if any([i.type in complex_types for i in node.inputs]): raise NotImplementedError() # Test for both y>x and x>=y to detect NaN return ('%(z)s = ((%(y)s)>(%(x)s)? (%(y)s): ' '((%(x)s)>=(%(y)s)? (%(x)s): nan("")));' % locals()) def L_op(self, inputs, outputs, gout): (x, y) = inputs (gz,) = gout if gz.type in complex_types: # max is currently defined for complex_types, # but the gradient for complex is not. raise NotImplementedError() if outputs[0].type in discrete_types: return [x.zeros_like().astype(theano.config.floatX), y.zeros_like().astype(theano.config.floatX)] # This form handle the case when both value are the same. # In that case, gx will be gz, gy will be 0. e = eq(outputs[0], x) gx = e * gz gy = (constant(1, dtype=gz.dtype) - e) * gz return (gx, gy) maximum = Maximum(upcast_out, name='maximum') class Minimum(BinaryScalarOp): commutative = True associative = True nfunc_spec = ('minimum', 2, 1) def impl(self, *inputs): # The built-in min function don't support complex type return np.minimum(*inputs) def c_code(self, node, name, inputs, outputs, sub): (x, y) = inputs (z,) = outputs if any([i.type in complex_types for i in node.inputs]): raise NotImplementedError() return ('%(z)s = ((%(y)s)<(%(x)s)? (%(y)s): ' '((%(x)s)<=(%(y)s)? (%(x)s): nan("")));' % locals()) def L_op(self, inputs, outputs, gout): (x, y) = inputs (gz,) = gout if gz.type in complex_types: # min is currently defined for complex_types, # but the gradient for complex is not. raise NotImplementedError() if outputs[0].type in discrete_types: return [x.zeros_like().astype(theano.config.floatX), y.zeros_like().astype(theano.config.floatX)] # This form handle the case when both value are the same. # In that case, gx will be gz, gy will be 0. e = eq(outputs[0], x) gx = e * gz gy = (constant(1, dtype=gz.dtype) - e) * gz return (gx, gy) minimum = Minimum(upcast_out, name='minimum') class Add(ScalarOp): identity = 0 commutative = True associative = True nfunc_spec = ('add', 2, 1) def impl(self, *inputs): return sum(inputs) def c_code(self, node, name, inputs, outputs, sub): (z,) = outputs op = " + " if node.outputs[0].type == bool: op = " || " if not inputs: return z + " = 0;" else: return z + " = " + op.join(inputs) + ";" def L_op(self, inputs, outputs, gout): (gz,) = gout if gz.type in complex_types: raise NotImplementedError() if (outputs[0].type in discrete_types): assert gz is not None retval = [] for ii, inp in enumerate(inputs): if hasattr(inp, 'zeros_like'): retval.append( inp.zeros_like().astype(theano.config.floatX)) else: retval.append(grad_undefined(self, ii, inp)) else: retval = [] for i in inputs: retval += [gz] return retval add = Add(upcast_out, name='add') class Mul(ScalarOp): identity = 1 commutative = True associative = True nfunc_spec = ('multiply', 2, 1) def impl(self, *inputs): return np.product(inputs) def c_code(self, node, name, inputs, outputs, sub): (z,) = outputs op = " * " if node.outputs[0].type == bool: op = " && " if not inputs: return z + " = 1;" else: return z + " = " + op.join(inputs) + ";" def grad(self, inputs, gout): (gz,) = gout retval = [] # The following 3 lines verify that gz is complex when the # output is complex. The rest of this function make this supposition. output_type = self.output_types([i.type for i in inputs])[0] if output_type in complex_types: if gz.type not in complex_types: raise TypeError( 'Mul with output_type ' + str(output_type) + ' expected gz type to be complex, got gz with type ' + str(gz.type)) if output_type in discrete_types: return [ipt.zeros_like().astype(theano.config.floatX) for ipt in inputs] for input in inputs: if gz.type in complex_types: # zr+zi = (xr + xi)(yr + yi) # zr+zi = (xr*yr - xi*yi) + (xr yi + xi yr ) otherprod = mul(*(utils.difference(inputs, [input]))) yr = real(otherprod) yi = imag(otherprod) if input.type in complex_types: retval += [complex(yr * real(gz) + yi * imag(gz), yr * imag(gz) - yi * real(gz))] else: retval += [yr * real(gz) + yi * imag(gz)] else: retval += [mul(*([gz] + utils.difference(inputs, [input])))] return retval mul = Mul(upcast_out, name='mul') class Sub(BinaryScalarOp): nfunc_spec = ('subtract', 2, 1) def impl(self, x, y): return x - y def c_code(self, node, name, inputs, outputs, sub): (x, y) = inputs (z,) = outputs return "%(z)s = %(x)s - %(y)s;" % locals() def L_op(self, inputs, outputs, gout): (x, y) = inputs (gz,) = gout if gz.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: return [x.zeros_like().astype(theano.config.floatX), y.zeros_like().astype(theano.config.floatX)] first_part = gz second_part = -gz return first_part, second_part sub = Sub(upcast_out_nobool, name='sub') def int_or_true_div(x_discrete, y_discrete): """ Return 'int' or 'true' depending on the type of division used for x / y. Parameters ---------- x_discrete : bool True if `x` is discrete ([unsigned] integer). y_discrete : bool True if `y` is discrete ([unsigned] integer). Returns ------- str 'int' if `x / y` should be an integer division, or `true` if it should be a true division. Raises ------ IntegerDivisionError If both `x_discrete` and `y_discrete` are True and `config.int_division` is set to 'raise'. Notes ----- This function is used by both scalar/basic.py and tensor/basic.py. """ if (x_discrete and y_discrete): if config.int_division == 'raise': raise IntegerDivisionError( "With `config.int_division` set to 'raise', dividing two " "integer types with '/' is forbidden to avoid confusion " "between integer and floating point divisions. Please " "use // for integer division, or if you want a float result " "either cast one of the arguments to a float or directly call " "`x.__truediv__(y)`.") elif config.int_division == 'int': warnings.warn( "Division of two integer types with x / y is deprecated, " "please use x // y for an integer division.", DeprecationWarning, stacklevel=4) return int_div elif config.int_division == 'floatX': return true_div else: raise NotImplementedError(config.int_division) else: return true_div def div_proxy(x, y): """ Proxy for either true_div or int_div, depending on types of x, y. """ f = int_or_true_div(as_scalar(x).type in discrete_types, as_scalar(y).type in discrete_types) return f(x, y) class TrueDiv(BinaryScalarOp): nfunc_spec = ('true_divide', 2, 1) def output_types(self, types): if all(t in discrete_types for t in types): return [get_scalar_type(config.floatX)] else: return super(TrueDiv, self).output_types(types) def impl(self, x, y): x = np.asarray(x) y = np.asarray(y) if all(a.dtype in discrete_types for a in (x, y)): return np.sctypeDict[config.floatX](float(x) / y) else: return x / y def c_code(self, node, name, inputs, outputs, sub): # we generate good c code only when both are complex! (x, y) = inputs (z,) = outputs if sum([node.inputs[0].type in complex_types, node.inputs[1].type in complex_types]) == 1: raise NotImplementedError('type not supported', type) if (node.inputs[0].type in discrete_types and node.inputs[1].type in discrete_types): return "%(z)s = ((double)%(x)s) / %(y)s;" % locals() return "%(z)s = %(x)s / %(y)s;" % locals() def grad(self, inputs, gout): (x, y) = inputs (gz,) = gout if x.type in complex_types: raise NotImplementedError() # If the output of this op is discrete, then it # it is locally flat everywhere, so the gradient # through it is 0. # This is different from it not being connected # to the output; x/y is still a function of x # and y; it's just a step function. if all(a.dtype in discrete_types for a in (x, y)): return [x.zeros_like(), y.zeros_like()] first_part = gz / y if y.type in complex_types: raise NotImplementedError() second_part = -(gz * x) / (y * y) return first_part, second_part true_div = TrueDiv(upcast_out, name='true_div') class IntDiv(BinaryScalarOp): nfunc_spec = ('floor_divide', 2, 1) complex_error = ComplexError( "Theano does not support integer division (//) on " "complex numbers, since numpy deprecated it.") def impl(self, x, y): return x // y def c_support_code(self): # We use a macro as python use % as a special string character, # and the output of c_code may be run through another level # of string formatting. return "#define THEANO_MACRO_MOD(x,y) (x % y)" def c_code(self, node, name, inputs, outputs, sub): (x, y) = inputs (z,) = outputs fail = sub['fail'] t = node.inputs[0].type.upcast(*[i.type for i in node.inputs[1:]]) if t in imap(str, discrete_types): x_div_y_pp = '(%(x)s / %(y)s)' % locals() x_div_y_mp = '((-%(x)s) / %(y)s)' % locals() x_mod_y_mp = 'THEANO_MACRO_MOD((-%(x)s), %(y)s)' % locals() x_div_y_pm = '(%(x)s / (-%(y)s))' % locals() x_mod_y_pm = 'THEANO_MACRO_MOD(%(x)s, (-%(y)s))' % locals() x_div_y_mm = '((-%(x)s) / (-%(y)s))' % locals() # If we are in a gpuarray kernel, %(fail)s exits the kernel, # and we do not have any error report, and we cannot set # Python error messages either, so for now we just call the # cuda function, which return a binary pattern of all 1s. div_zero = dedent(''' #ifdef KERNEL %(z)s = %(x_div_y_pp)s; #else PyErr_SetString(PyExc_ZeroDivisionError, "integer division by zero"); %(fail)s #endif ''') % locals() elif t in imap(str, float_types): # We need to call different functions of math.h # depending on the type if t == 'float32': floor = 'floorf' fmod = 'fmodf' elif t == 'float64': floor = 'floor' fmod = 'fmod' else: raise NotImplementedError('type not supported', t) x_div_y_pp = '%(floor)s(%(x)s / %(y)s)' % locals() x_div_y_mp = '%(floor)s((-%(x)s) / %(y)s)' % locals() x_mod_y_mp = '%(fmod)s((-%(x)s), %(y)s)' % locals() x_div_y_pm = '%(floor)s(%(x)s / (-%(y)s))' % locals() x_mod_y_pm = '%(fmod)s(%(x)s, (-%(y)s))' % locals() x_div_y_mm = '%(floor)s((-%(x)s) / (-%(y)s))' % locals() div_zero = '%(z)s = %(x_div_y_pp)s;' % locals() elif t in complex_types: raise self.complex_error else: raise NotImplementedError('type not supported', t) return dedent(""" if (%(y)s == 0) { %(div_zero)s; } else if (%(y)s < 0) { if (%(x)s < 0) { %(z)s = %(x_div_y_mm)s; } else { %(z)s = - %(x_div_y_pm)s - ((%(x_mod_y_pm)s == 0) ? 0 : 1); } } else { if (%(x)s < 0) { %(z)s = - %(x_div_y_mp)s - ((%(x_mod_y_mp)s == 0) ? 0 : 1); } else { %(z)s = %(x_div_y_pp)s; } } """) % locals() def c_code_cache_version(self): return (6,) def grad(self, inputs, g_output): return [inp.zeros_like(dtype=theano.config.floatX) for inp in inputs] int_div = IntDiv(upcast_out, name='int_div') floor_div = int_div def mod_check(x, y): if (as_scalar(x).type in complex_types or as_scalar(y).type in complex_types): # Currently forbidden. raise Mod.complex_error else: return mod(x, y) class Mod(BinaryScalarOp): nfunc_spec = ('mod', 2, 1) complex_error = ComplexError( "Theano does not support the mod operator (%) on " "complex numbers, since numpy deprecated it.") def impl(self, x, y): if isinstance(x, np.complex) or isinstance(y, np.complex): raise self.complex_error return x % y def c_code_cache_version(self): return (9,) def c_support_code(self): # We use a macro as python use % as a special string character, # and the output of c_code may be run through another level # of string formatting. return "#define THEANO_MACRO_MOD(x, y) (x % y)" def c_code(self, node, name, inputs, outputs, sub): """ We want the result to have the same sign as Python, not the other implementation of mod. """ (x, y) = inputs (z,) = outputs fail = sub['fail'] t = node.inputs[0].type.upcast(*[i.type for i in node.inputs[1:]]) if (str(t) in imap(str, discrete_types) or t in ['uint8', 'int8', 'uint16', 'int16'] or t in ['uint32', 'int32', 'uint64', 'int64'] or t in discrete_types): # The above or's should not be needed anymore. However, for now we # keep them out of safety, and verify they are useless with an # assert. assert str(t) in imap(str, discrete_types) x_mod_y = "THEANO_MACRO_MOD(%(x)s, %(y)s)" % locals() x_mod_ymm = "THEANO_MACRO_MOD(-%(x)s, -%(y)s)" % locals() x_mod_ypm = "THEANO_MACRO_MOD(%(x)s, -%(y)s)" % locals() x_mod_ymp = "THEANO_MACRO_MOD(-%(x)s, %(y)s)" % locals() # If we are in a gpuarray kernel, %(fail)s exits the kernel, # and we do not have any error report, and we cannot set # Python error messages either, so for now we just call the # cuda function, returning a binary pattern depending on dtype mod_zero = dedent(''' #ifdef KERNEL %(z)s = %(x_mod_y)s; #else PyErr_SetString(PyExc_ZeroDivisionError, "integer modulo by zero"); %(fail)s #endif ''') % locals() elif (str(t) in imap(str, float_types) or t in ['float32', 'float64'] or t in float_types): # The above or's should not be needed anymore. However, for now we # keep them out of safety, and verify they are useless with an # assert. assert str(t) in imap(str, float_types) x_mod_y = "fmod(%(x)s, %(y)s)" % locals() x_mod_ymm = "fmod(-%(x)s, -%(y)s)" % locals() x_mod_ypm = "fmod(%(x)s, -%(y)s)" % locals() x_mod_ymp = "fmod(-%(x)s, %(y)s)" % locals() mod_zero = "%(z)s = %(x_mod_y)s;" % locals() elif str(t) in imap(str, complex_types): raise self.complex_error else: raise NotImplementedError('type not supported', t) return dedent(""" if (%(y)s == 0) { %(mod_zero)s; } else if (%(y)s < 0){ if (%(x)s < 0){ %(z)s = -(%(x_mod_ymm)s); } else { %(z)s = (%(x_mod_ypm)s) + (%(x_mod_ypm)s != 0 ? %(y)s : 0); } } else { if (%(x)s < 0){ %(z)s = - %(x_mod_ymp)s + (%(x_mod_ymp)s != 0 ? %(y)s : 0); } else { %(z)s = %(x_mod_y)s; } } """) % locals() def L_op(self, inputs, outputs, gout): (x, y) = inputs (gz,) = gout if outputs[0].type.dtype in discrete_types: # The gradient does not flow in if the output is discrete return [x.zeros_like(dtype=theano.config.floatX), y.zeros_like(dtype=theano.config.floatX)] return [gz, -(x // y) * gz] mod = Mod(upcast_out, name='mod') class Pow(BinaryScalarOp): nfunc_spec = ('power', 2, 1) def impl(self, x, y): return x ** y def c_code(self, node, name, inputs, outputs, sub): (x, y) = inputs (z,) = outputs if (node.inputs[0].type in complex_types or node.inputs[1].type in complex_types): raise NotImplementedError('type not supported', type) return "%(z)s = pow(%(x)s, %(y)s);" % locals() def L_op(self, inputs, outputs, gout): (x, y) = inputs (gz,) = gout if gz.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: return [x.zeros_like().astype(theano.config.floatX), y.zeros_like().astype(theano.config.floatX)] first_part = gz * y * x ** (y - 1) second_part = gz * log(x) * x ** y second_part = switch(eq(x, 0), 0, second_part) return (first_part, second_part) def c_code_contiguous(self, node, name, inputs, outputs, sub): (x, y) = inputs (z,) = outputs if not theano.config.lib.amdlibm: raise theano.gof.utils.MethodNotDefined() # We compare the dtype AND the broadcast flag # as this function do not broadcast if (node.inputs[0].type == node.outputs[0].type and node.inputs[1].type == node.outputs[0].type and # amdlibm 3.0 do not have a float64 version of this SIMD function node.inputs[0].dtype == 'float32' and node.inputs[1].dtype == 'float32'): dtype = 'float' fct = "amd_vrsa_powf" return """ npy_intp n = PyArray_SIZE(%(z)s); %(dtype)s * x = (%(dtype)s*) PyArray_DATA(%(x)s); %(dtype)s * y = (%(dtype)s*) PyArray_DATA(%(y)s); %(dtype)s * z = (%(dtype)s*) PyArray_DATA(%(z)s); %(fct)s(n, x, y, z); """ % locals() # We compare the dtype and check we broadcast a scalar elif (node.inputs[0].type == node.outputs[0].type and node.inputs[1].dtype == node.outputs[0].dtype and all(node.inputs[1].broadcastable) and # amdlibm 3.0 do not have a float64 version of this SIMD function node.inputs[0].dtype == 'float32' and node.inputs[1].dtype == 'float32'): dtype = 'float' fct = "amd_vrsa_powxf" return """ npy_intp n = PyArray_SIZE(%(z)s); %(dtype)s * x = (%(dtype)s*) PyArray_DATA(%(x)s); %(dtype)s * y = (%(dtype)s*) PyArray_DATA(%(y)s); %(dtype)s * z = (%(dtype)s*) PyArray_DATA(%(z)s); %(fct)s(n, x, *y, z); """ % locals() raise theano.gof.utils.MethodNotDefined() pow = Pow(upcast_out_min8, name='pow') class Clip(ScalarOp): nin = 3 # The numpy.clip don't work correctly when the min is bigger then the max, # So we do not use nfunc_spec = ('clip', 3, 1) def impl(self, x, min, max): if x < min: return min elif x > max: return max else: return x def c_code(self, node, name, inputs, outputs, sub): (x, min, max) = inputs (z,) = outputs return "%(z)s = %(x)s < %(min)s ? %(min)s : %(x)s > %(max)s ? %(max)s : %(x)s;" % locals() def L_op(self, inputs, outputs, gout): (x, mn, mx) = inputs (gz,) = gout assert gz.type not in complex_types gx = ((x >= mn) & (x <= mx)) * gz gmn = (x < mn) * gz gmx = (x > mx) * gz def handle_int(v): if outputs[0].type in int_types: return v.zeros_like().astype(config.floatX) return v return list(map(handle_int, [gx, gmn, gmx])) # Don't allow complex even if numpy do # As there is no mathematical reason for this function on complex clip = Clip(upcast_out_no_complex, name='clip') class Second(BinaryScalarOp): def impl(self, x, y): return y def c_code(self, node, name, inputs, outputs, sub): (x, y) = inputs (z,) = outputs return "%(z)s = %(y)s;" % locals() def connection_pattern(self, node): # x is never connected because its elements are never used # y is connected because its elements are copied over return [[False], [True]] def grad(self, inputs, gout): (x, y) = inputs (gz,) = gout if y.type in continuous_types: # x is disconnected because the elements of x are not used return DisconnectedType()(), gz else: # when y is discrete, we assume the function can be extended # to deal with real-valued inputs by rounding them to the # nearest integer. f(x+eps) thus equals f(x) so the gradient # is zero, not disconnected or undefined return DisconnectedType()(), y.zeros_like() second = Second(transfer_type(1), name='second') class Identity(UnaryScalarOp): def impl(self, input): return input def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs return "%(z)s = %(x)s;" % locals() def grad(self, inputs, gout): (x,) = inputs (gz,) = gout if x.type in continuous_types: return gz, else: return x.zeros_like(dtype=theano.config.floatX), identity = Identity(same_out, name='identity') # CASTING OPERATIONS class Cast(UnaryScalarOp): def __init__(self, o_type, name=None): if not isinstance(o_type, Scalar): raise TypeError(o_type) super(Cast, self).__init__(specific_out(o_type), name=name) self.o_type = o_type self.ctor = getattr(np, o_type.dtype) def __str__(self): return '%s{%s}' % (self.__class__.__name__, self.o_type.dtype) def clone_float32(self): if self.o_type == float16: return convert_to_float32 return self def make_new_inplace(self, output_types_preference=None, name=None): """ This op.__init__ fct don't have the same parameter as other scalar op. This breaks the insert_inplace_optimizer optimization. This function is a fix to patch this, by ignoring the output_types_preference passed by the optimization, and replacing it by the current output type. This should only be triggered when both input and output have the same dtype anyway. """ return self.__class__(self.o_type, name) def impl(self, input): return self.ctor(input) def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.outputs[0].type == bool: return "%s = (%s) ? 1 : 0;" % (z, x) return "%s = (%s)%s;" % (z, node.outputs[0].type.dtype_specs()[1], x) def grad(self, inputs, gout): (x,) = inputs (gz,) = gout if self.o_type in continuous_types: return [gz] else: return [x.zeros_like().astype(theano.config.floatX)] def c_code_cache_version(self): s = super(Cast, self).c_code_cache_version() if s: return (4,) + s else: return s convert_to_bool = Cast(bool, name='convert_to_bool') convert_to_int8 = Cast(int8, name='convert_to_int8') convert_to_int16 = Cast(int16, name='convert_to_int16') convert_to_int32 = Cast(int32, name='convert_to_int32') convert_to_int64 = Cast(int64, name='convert_to_int64') convert_to_uint8 = Cast(uint8, name='convert_to_uint8') convert_to_uint16 = Cast(uint16, name='convert_to_uint16') convert_to_uint32 = Cast(uint32, name='convert_to_uint32') convert_to_uint64 = Cast(uint64, name='convert_to_uint64') convert_to_float16 = Cast(float16, name='convert_to_float16') convert_to_float32 = Cast(float32, name='convert_to_float32') convert_to_float64 = Cast(float64, name='convert_to_float64') convert_to_complex64 = Cast(complex64, name='convert_to_complex64') convert_to_complex128 = Cast(complex128, name='convert_to_complex128') _cast_mapping = { 'bool': convert_to_bool, 'int8': convert_to_int8, 'int16': convert_to_int16, 'int32': convert_to_int32, 'int64': convert_to_int64, 'uint8': convert_to_uint8, 'uint16': convert_to_uint16, 'uint32': convert_to_uint32, 'uint64': convert_to_uint64, 'float16': convert_to_float16, 'float32': convert_to_float32, 'float64': convert_to_float64, 'complex64': convert_to_complex64, 'complex128': convert_to_complex128} def cast(x, dtype): """ Symbolically cast `x` to a Scalar of given `dtype`. """ if dtype == 'floatX': dtype = config.floatX _x = as_scalar(x) if _x.type.dtype == dtype: return _x if _x.type.dtype.startswith('complex') and not dtype.startswith('complex'): raise TypeError('Casting from complex to real is ambiguous: consider' ' real(), imag(), angle() or abs()') return _cast_mapping[dtype](_x) class Abs(UnaryScalarOp): nfunc_spec = ('abs', 1, 1) def make_node(self, x): inputs = [as_scalar(input) for input in [x]] if inputs[0].type == complex64: outputs = [float32()] elif inputs[0].type == complex128: outputs = [float64()] else: outputs = [t() for t in self.output_types( [input.type for input in inputs])] return Apply(self, inputs, outputs) def impl(self, x): return np.abs(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if (outputs[0].type in discrete_types): if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] if x.type in float_types: return gz * sgn(x), return gz * x / abs(x), # formula works for complex and real def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs type = node.inputs[0].type if type in int_types: return "%(z)s = abs(%(x)s);" % locals() if type in float_types: return "%(z)s = fabs(%(x)s);" % locals() if type in complex_types: return "%(z)s = sqrt(%(x)s.real*%(x)s.real + %(x)s.imag*%(x)s.imag);" % locals() if node.outputs[0].type == bool: return "%(z)s = (%(x)s) ? 1 : 0;" % locals() if type in uint_types: # uint are always already absolute value. return "%(z)s = %(x)s;" % locals() raise NotImplementedError('type not supported', type) abs_ = Abs(same_out) class Sgn(UnaryScalarOp): nfunc_spec = ('sign', 1, 1) @staticmethod def output_types_preference(x): if x == bool: raise TypeError(x) return same_out_nocomplex(x) def impl(self, x): # casting to output type is handled by filter return np.sign(x) def grad(self, inputs, gout): (x,) = inputs (gz,) = gout rval = x.zeros_like() if rval.type.dtype in discrete_types: rval = rval.astype(theano.config.floatX) return [rval] def c_code(self, node, name, inputs, outputs, sub): # casting is done by compiler # TODO: use copysign (x,) = inputs (z,) = outputs type = node.inputs[0].type if type in float_types: return '%(z)s = (%(x)s > 0) ? 1. : ((%(x)s < 0) ? -1. : (isnan(%(x)s) ? NAN : 0.));' % locals() if type in int_types: return "%(z)s = (%(x)s >= 0) ? (%(x)s == 0) ? 0 : 1 : -1;" % locals() raise ComplexError('complex has no sgn') def c_code_cache_version(self): s = super(Sgn, self).c_code_cache_version() if s: return (4,) + s else: # if parent is unversioned, we are too return s sgn = Sgn(name='sgn') class Ceil(UnaryScalarOp): nfunc_spec = ('ceil', 1, 1) def impl(self, x): return np.ceil(x) def grad(self, inputs, gout): (x,) = inputs (gz,) = gout rval = x.zeros_like() if rval.type.dtype in discrete_types: rval = rval.astype(theano.config.floatX) return [rval] def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = ceil((%(cast)s)%(x)s);" % locals() ceil = Ceil(upgrade_to_float_no_complex, name='ceil') class Floor(UnaryScalarOp): nfunc_spec = ('floor', 1, 1) def impl(self, x): return np.floor(x) def grad(self, inputs, gout): (x,) = inputs (gz,) = gout rval = x.zeros_like() if rval.type.dtype in discrete_types: rval = rval.astype(theano.config.floatX) return [rval] def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = floor((%(cast)s)%(x)s);" % locals() floor = Floor(upgrade_to_float_no_complex, name='floor') class Trunc(UnaryScalarOp): nfunc_spec = ('trunc', 1, 1) def impl(self, x): return np.trunc(x) def grad(self, inputs, gout): (x,) = inputs (gz,) = gout return [x.zeros_like().astype(theano.config.floatX)] def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs return "%(z)s = %(x)s >= 0? floor(%(x)s): -floor(-%(x)s);" % locals() trunc = Trunc(upgrade_to_float_no_complex, name='trunc') class RoundHalfToEven(UnaryScalarOp): """ This function implement the same rounding than numpy: Round half to even. c/c++ round fct IS DIFFERENT! See http://en.wikipedia.org/wiki/Rounding for more details. """ nfunc_spec = ('around', 1, 1) def impl(self, x): return np.round(x) def grad(self, inputs, gout): (x,) = inputs (gz,) = gout rval = x.zeros_like() if rval.type.dtype in discrete_types: rval = rval.astype(theano.config.floatX) return [rval] def c_code_cache_version(self): return (1,) def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs typ = node.outputs[0].type.dtype if typ not in ['float32', 'float64']: raise NotImplementedError("The output should be float32 or float64") if typ == 'float32': ctype = 'float' floor_function = 'floorf' else: ctype = 'double' floor_function = 'floor' return """ /* Code inspired from NumPy npy_rint implementation. */ { %(ctype)s y, r; y = %(floor_function)s(%(x)s); r = %(x)s - y; if(r > 0.5) { y += 1; } else if(r == 0.5) { r = y - 2.0*%(floor_function)s(0.5*y); /* If y is even, then r == 0 If y is odd, then r == 1 So we can just add r to y, so that y will be incremented only if he's odd. */ y += (int)r; } %(z)s = y; } """ % locals() round_half_to_even = RoundHalfToEven(same_out_float_only) def round_half_away_from_zero_(a): if a > 0: return np.floor(a + 0.5) else: return np.ceil(a - 0.5) round_half_away_from_zero_vec64 = np.vectorize( round_half_away_from_zero_, doc='round_half_away_from_zero_vec64') round_half_away_from_zero_vec32 = np.vectorize( round_half_away_from_zero_, doc='round_half_away_from_zero_vec32', otypes=['float32']) def round_half_away_from_zero_vec(a): if getattr(a, 'dtype', None) == np.float32: return round_half_away_from_zero_vec32(a) return round_half_away_from_zero_vec64(a) class RoundHalfAwayFromZero(UnaryScalarOp): """ Implement the same rounding algo as c round() fct. numpy.round fct IS DIFFERENT! See http://en.wikipedia.org/wiki/Rounding for more details. """ def impl(self, x): return round_half_away_from_zero_vec(x) def grad(self, inputs, gout): (x,) = inputs (gz,) = gout rval = x.zeros_like() if rval.type.dtype in discrete_types: rval = rval.astype(theano.config.floatX) return [rval] def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.outputs[0].type.dtype in ['float32', 'float64']: return "%(z)s = round(%(x)s);" % locals() else: raise NotImplementedError("The output should be float32 or float64") round_half_away_from_zero = RoundHalfAwayFromZero(same_out_float_only) class Neg(UnaryScalarOp): # We can use numpy.negative here, because even if it gives unexpected # results on Boolean arrays, it will be passed other dtypes as Theano # does not have a Boolean type for tensors. nfunc_spec = ('negative', 1, 1) def impl(self, x): return -x def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return -gz, def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs return "%(z)s = -%(x)s;" % locals() neg = Neg(same_out_nobool, name='neg') pprint.assign(add, printing.OperatorPrinter('+', -2, 'either')) pprint.assign(mul, printing.OperatorPrinter('*', -1, 'either')) pprint.assign(sub, printing.OperatorPrinter('-', -2, 'left')) pprint.assign(neg, printing.OperatorPrinter('-', 0, 'either')) pprint.assign(true_div, printing.OperatorPrinter('/', -1, 'left')) pprint.assign(int_div, printing.OperatorPrinter('//', -1, 'left')) pprint.assign(pow, printing.OperatorPrinter('**', 1, 'right')) pprint.assign(mod, printing.OperatorPrinter('%', -1, 'left')) class Inv(UnaryScalarOp): """ Multiplicative inverse. Also called reciprocal. """ def impl(self, x): return np.float32(1.0) / x def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return -gz / (x * x), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError() return "%(z)s = 1.0 / %(x)s;" % locals() inv = Inv(upgrade_to_float, name='inv') class Log(UnaryScalarOp): """ log base e. """ nfunc_spec = ('log', 1, 1) amd_float32 = "amd_vrsa_logf" amd_float64 = "amd_vrda_log" def impl(self, x): # If x is an int8 or uint8, numpy.log will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.log(x, sig='f') return np.log(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz / x, def c_code(self, node, name, inputs, outputs, sub): # todo: the version using log2 seems to be very slightly faster # on some machines for some reason, check if it's worth switching # return "%(z)s = log2(%(x)s) * 0.69314718055994529;" % locals() (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = log((%(cast)s)%(x)s);" % locals() log = Log(upgrade_to_float, name='log') class Log2(UnaryScalarOp): """ log base 2. """ nfunc_spec = ('log2', 1, 1) amd_float32 = "amd_vrsa_log2f" amd_float64 = "amd_vrda_log2" def impl(self, x): # If x is an int8 or uint8, numpy.log2 will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.log2(x, sig='f') return np.log2(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz / (x * np.asarray(math.log(2.0)).astype(x.dtype)), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = log2((%(cast)s)%(x)s);" % locals() log2 = Log2(upgrade_to_float, name='log2') class Log10(UnaryScalarOp): """ log base 10. """ nfunc_spec = ('log10', 1, 1) amd_float32 = "amd_vrsa_log10f" amd_float64 = "amd_vrda_log10" def impl(self, x): # If x is an int8 or uint8, numpy.log10 will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.log10(x, sig='f') return np.log10(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz / (x * np.log(10.0)), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = log10((%(cast)s)%(x)s);" % locals() log10 = Log10(upgrade_to_float, name='log10') class Log1p(UnaryScalarOp): """ log(1+x). """ nfunc_spec = ('log1p', 1, 1) def impl(self, x): # If x is an int8 or uint8, numpy.log1p will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.log1p(x, sig='f') return np.log1p(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if gz.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return [gz / (1 + x)] def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = log1p((%(cast)s)%(x)s);" % locals() log1p = Log1p(upgrade_to_float, name='log1p') class Exp(UnaryScalarOp): nfunc_spec = ('exp', 1, 1) amd_float32 = "amd_vrsa_expf" amd_float64 = "amd_vrda_exp" def impl(self, x): # If x is an int8 or uint8, numpy.exp will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.exp(x, sig='f') return np.exp(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz * exp(x), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = exp((%(cast)s)%(x)s);" % locals() exp = Exp(upgrade_to_float, name='exp') class Exp2(UnaryScalarOp): nfunc_spec = ('exp2', 1, 1) def impl(self, x): # If x is an int8 or uint8, numpy.exp2 will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.exp2(x, sig='f') return np.exp2(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz * exp2(x) * log(np.cast[x.type](2)), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = exp2((%(cast)s)%(x)s);" % locals() exp2 = Exp2(upgrade_to_float, name='exp2') class Expm1(UnaryScalarOp): nfunc_spec = ('expm1', 1, 1) def impl(self, x): # If x is an int8 or uint8, numpy.expm1 will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.expm1(x, sig='f') return np.expm1(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz * exp(x), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = expm1((%(cast)s)%(x)s);" % locals() def c_code_cache_version(self): return (5,) expm1 = Expm1(upgrade_to_float, name='expm1') class Sqr(UnaryScalarOp): nfunc_spec = ('square', 1, 1) def impl(self, x): return x * x def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if gz.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz * x * 2, def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs return "%(z)s = %(x)s * %(x)s;" % locals() sqr = Sqr(same_out, name='sqr') class Sqrt(UnaryScalarOp): nfunc_spec = ('sqrt', 1, 1) def impl(self, x): # If x is an int8 or uint8, numpy.sqrt will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.sqrt(x, sig='f') return np.sqrt(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if gz.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return (gz * 0.5) / sqrt(x), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = sqrt((%(cast)s)%(x)s);" % locals() sqrt = Sqrt(upgrade_to_float, name='sqrt') class Deg2Rad(UnaryScalarOp): nfunc_spec = ('deg2rad', 1, 1) def impl(self, x): # If x is an int8 or uint8, numpy.deg2rad will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.deg2rad(x, sig='f') return np.deg2rad(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if gz.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz * np.asarray(np.pi / 180, gz.type), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) return "%(z)s = %(x)s * (M_PI / 180.0);" % locals() deg2rad = Deg2Rad(upgrade_to_float, name='deg2rad') class Rad2Deg(UnaryScalarOp): nfunc_spec = ('rad2deg', 1, 1) def impl(self, x): # If x is an int8 or uint8, numpy.rad2deg will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.rad2deg(x, sig='f') return np.rad2deg(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if gz.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz * np.asarray(180. / np.pi, gz.type), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) return "%(z)s = %(x)s * (180.0 / M_PI);" % locals() rad2deg = Rad2Deg(upgrade_to_float, name='rad2deg') class Cos(UnaryScalarOp): nfunc_spec = ('cos', 1, 1) amd_float32 = "amd_vrsa_cosf" amd_float64 = "amd_vrda_cos" def impl(self, x): # If x is an int8 or uint8, numpy.cos will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.cos(x, sig='f') return np.cos(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if gz.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return -gz * sin(x), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = cos((%(cast)s)%(x)s);" % locals() cos = Cos(upgrade_to_float, name='cos') class ArcCos(UnaryScalarOp): nfunc_spec = ('arccos', 1, 1) def impl(self, x): # If x is an int8 or uint8, numpy.arccos will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.arccos(x, sig='f') return np.arccos(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if gz.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return - gz / sqrt(np.cast[x.type](1) - sqr(x)), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = acos((%(cast)s)%(x)s);" % locals() arccos = ArcCos(upgrade_to_float, name='arccos') class Sin(UnaryScalarOp): nfunc_spec = ('sin', 1, 1) amd_float32 = "amd_vrsa_sinf" amd_float64 = "amd_vrda_sin" def impl(self, x): # If x is an int8 or uint8, numpy.sin will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.sin(x, sig='f') return np.sin(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz * cos(x), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = sin((%(cast)s)%(x)s);" % locals() sin = Sin(upgrade_to_float, name='sin') class ArcSin(UnaryScalarOp): nfunc_spec = ('arcsin', 1, 1) def impl(self, x): # If x is an int8 or uint8, numpy.arcsin will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.arcsin(x, sig='f') return np.arcsin(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if gz.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz / sqrt(np.cast[x.type](1) - sqr(x)), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = asin((%(cast)s)%(x)s);" % locals() arcsin = ArcSin(upgrade_to_float, name='arcsin') class Tan(UnaryScalarOp): nfunc_spec = ('tan', 1, 1) def impl(self, x): # If x is an int8 or uint8, numpy.tan will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.tan(x, sig='f') return np.tan(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz / sqr(cos(x)), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = tan((%(cast)s)%(x)s);" % locals() tan = Tan(upgrade_to_float, name='tan') class ArcTan(UnaryScalarOp): nfunc_spec = ('arctan', 1, 1) def impl(self, x): # If x is an int8 or uint8, numpy.arctan will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.arctan(x, sig='f') return np.arctan(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if gz.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz / (np.cast[x.type](1) + sqr(x)), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = atan((%(cast)s)%(x)s);" % locals() arctan = ArcTan(upgrade_to_float, name='arctan') class ArcTan2(BinaryScalarOp): nfunc_spec = ('arctan2', 2, 1) def impl(self, y, x): # If x and y are int8 or uint8, numpy.arctan2 will compute the result # in half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): y_dtype = str(getattr(x, 'dtype', '')) if y_dtype in ('int8', 'uint8'): return np.arctan2(y, x, sig='f') return np.arctan2(y, x) def L_op(self, inputs, outputs, gout): (y, x) = inputs (gz,) = gout if gz.type in complex_types: raise NotImplementedError() else: if outputs[0].type in discrete_types: if x.type in discrete_types: gx = x.zeros_like(dtype=theano.config.floatX) else: gx = x.zeros_like() if y.type in discrete_types: gy = y.zeros_like(dtype=theano.config.floatX) else: gy = y.zeros_like() return [gx, gy] # If the output is float, the gradient should flow, # even if the inputs are ints return [gz * x / (sqr(x) + sqr(y)), gz * neg(y) / (sqr(x) + sqr(y))] def c_code(self, node, name, inputs, outputs, sub): (y, x) = inputs (z,) = outputs if (node.inputs[0].type in complex_types or node.inputs[1].type in complex_types): raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = atan2((%(cast)s)%(y)s, (%(cast)s)%(x)s);" % locals() arctan2 = ArcTan2(upgrade_to_float, name='arctan2') class Cosh(UnaryScalarOp): """ cosh(x) = (exp(x) + exp(-x)) / 2. """ nfunc_spec = ('cosh', 1, 1) def impl(self, x): # If x is an int8 or uint8, numpy.cosh will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.cosh(x, sig='f') return np.cosh(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz * sinh(x), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = cosh((%(cast)s)%(x)s);" % locals() cosh = Cosh(upgrade_to_float, name='cosh') class ArcCosh(UnaryScalarOp): nfunc_spec = ('arccosh', 1, 1) def impl(self, x): # If x is an int8 or uint8, numpy.arccosh will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.arccosh(x, sig='f') return np.arccosh(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz / sqrt(sqr(x) - np.cast[x.type](1)), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = acosh((%(cast)s)%(x)s);" % locals() arccosh = ArcCosh(upgrade_to_float, name='arccosh') class Sinh(UnaryScalarOp): """ sinh(x) = (exp(x) - exp(-x)) / 2. """ nfunc_spec = ('sinh', 1, 1) def impl(self, x): # If x is an int8 or uint8, numpy.sinh will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.sinh(x, sig='f') return np.sinh(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz * cosh(x), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = sinh((%(cast)s)%(x)s);" % locals() sinh = Sinh(upgrade_to_float, name='sinh') class ArcSinh(UnaryScalarOp): nfunc_spec = ('arcsinh', 1, 1) def impl(self, x): # If x is an int8 or uint8, numpy.arcsinh will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.arcsinh(x, sig='f') return np.arcsinh(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz / sqrt(sqr(x) + np.cast[x.type](1)), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = asinh((%(cast)s)%(x)s);" % locals() arcsinh = ArcSinh(upgrade_to_float, name='arcsinh') class Tanh(UnaryScalarOp): """ tanh(x) = sinh(x) / cosh(x) = (exp(2*x) - 1) / (exp(2*x) + 1). """ nfunc_spec = ('tanh', 1, 1) def impl(self, x): # If x is an int8 or uint8, numpy.tanh will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.tanh(x, sig='f') return np.tanh(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz * (1 - sqr(tanh(x))), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = tanh((%(cast)s)%(x)s);" % locals() tanh = Tanh(upgrade_to_float, name='tanh') class ArcTanh(UnaryScalarOp): nfunc_spec = ('arctanh', 1, 1) def impl(self, x): # If x is an int8 or uint8, numpy.arctanh will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.arctanh(x, sig='f') return np.arctanh(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz / (np.cast[x.type](1) - sqr(x)), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = atanh((%(cast)s)%(x)s);" % locals() arctanh = ArcTanh(upgrade_to_float, name='arctanh') class Real(UnaryScalarOp): """ Extract the real coordinate of a complex number. """ # numpy.real(float32) return a view on the inputs. # nfunc_spec = ('real', 1, 1) def impl(self, x): return np.real(x) def grad(self, inputs, gout): (x,) = inputs (gz,) = gout return [complex(gz, 0)] real = Real(real_out, name='real') class Imag(UnaryScalarOp): nfunc_spec = ('imag', 1, 1) def impl(self, x): return np.imag(x) def grad(self, inputs, gout): (x,) = inputs (gz,) = gout if x.type in complex_types: return [complex(0, gz)] elif x.type in float_types: return [second(x, 0)] else: return [x.zeros_like(dtype=theano.config.floatX)] imag = Imag(real_out, name='imag') class Angle(UnaryScalarOp): nfunc_spec = ('angle', 1, 1) def impl(self, x): return np.angle(x) def grad(self, inputs, gout): # y = x.imag # r = sqrt(y**2 + x.real**2) # g = y/r # if x == 0 and y == 0: # theta = 0 # elif x >= 0: # theta = numpy.arcsin(g) # else: # theta = -numpy.arcsin(g)+numpy.pi (c,) = inputs (gtheta,) = gout x = real(c) y = imag(c) r = abs(c) gr = -gtheta * y / (r ** 2 * sqrt(1 - (y / r) ** 2)) gx = gr * x / r gy = gr * y / r if c in complex_types: return [cast(complex(gx, gy), x.type.dtype)] elif c in float_types: return [cast(second(x, 0), x.type.dtype)] else: return [c.zeros_like(dtype=theano.config.floatX)] angle = Angle(specific_out(float64), name='angle') class Complex(BinaryScalarOp): @staticmethod def output_types_preference(x, y): if x in complex_types: raise TypeError(x) if y in complex_types: raise TypeError(y) up = Scalar.upcast(x, y) if up in ('float64', 'int64', 'uint64', 'int32', 'uint32'): return [complex128] else: return [complex64] def impl(self, x, y): return np.complex(x, y) def grad(self, inputs, gout): (x, y) = inputs (gz,) = gout return [cast(real(gz), x.type.dtype), cast(imag(gz), y.type.dtype)] complex = Complex(name='complex') class Conj(UnaryScalarOp): nfunc_spec = ('conj', 1, 1) def impl(self, x): return np.conj(x) def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in complex_types: # For non complex, th raise NotImplementedError('type have no c code', node.inputs[0].type) return "%(z)s = %(x)s;" % locals() conj = Conj(same_out_min8, name='conj') class ComplexFromPolar(BinaryScalarOp): @staticmethod def output_types_preference(x, y): return Complex.output_types_preference(x, y) def impl(self, r, theta): if r < 0: raise ValueError('polar radius must be non-negative', r) x = r * np.cos(theta) y = r * np.sin(theta) if x.dtype == 'float32': return np.complex64(np.complex(x, y)) else: return np.complex128(np.complex(x, y)) def grad(self, inputs, gout): (r, theta) = inputs (gz,) = gout gr = gz * complex_from_polar(1, theta) gtheta = gz * complex_from_polar(r, -theta) return [gr, gtheta] complex_from_polar = ComplexFromPolar(name='complex_from_polar') class Composite(ScalarOp): """ Composite is an Op that takes a graph of scalar operations and produces c code for the whole graph. Its purpose is to implement loop fusion. Composite depends on all the Ops in its graph having C code. """ init_param = ('inputs', 'outputs') def __str__(self): if self.name is None: self.init_name() return self.name def make_new_inplace(self, output_types_preference=None, name=None): """ This op.__init__ fct don't have the same parameter as other scalar op. This break the insert_inplace_optimizer optimization. This fct allow fix patch this. """ d = dict([(k, getattr(self, k)) for k in self.init_param]) out = self.__class__(**d) if name: out.name = name else: name = out.name super(Composite, out).__init__(output_types_preference, name) return out def init_c_code(self): """ Assemble the C code for this Composite Op. The result is assigned to `self._c_code`. """ # It was already called if hasattr(self, '_c_code'): return subd = dict(chain( ((e, "%%(i%i)s" % i) for i, e in enumerate(self.fgraph.inputs)), ((e, "%%(o%i)s" % i) for i, e in enumerate(self.fgraph.outputs)))) for var in self.fgraph.variables: if var.owner is None: if var not in self.fgraph.inputs: # This is an orphan if isinstance(var, Constant): subd[var] = var.type.c_literal(var.data) else: raise ValueError( "All orphans in the fgraph to Composite must" " be Constant instances.") elif (any(i.dtype == 'float16' for i in var.owner.inputs) or any(o.dtype == 'float16' for o in var.owner.outputs)): # flag for elemwise ops to check. self.inner_float16 = True _c_code = "{\n" self.nodenames = ["%(nodename)s_" + ('subnode%i' % j) for j, n in enumerate(self.fgraph.toposort())] i = 0 for j, node in enumerate(self.fgraph.toposort()): for output in node.outputs: if output not in subd: i += 1 name = "V%%(id)s_tmp%i" % i subd[output] = name _c_code += "%s %s;\n" % ( output.type.dtype_specs()[1], name) s = node.op.c_code( node, self.nodenames[j], [subd[input] for input in node.inputs], [subd[output] for output in node.outputs], dict(fail="%(fail)s", id="%%(id)s_%i" % j)) _c_code += s _c_code += "\n" _c_code += "}\n" self._c_code = _c_code def init_py_impls(self): """ Return a list of functions that compute each output of self. """ # In the case where the graph is a dag, but not a tree like: # add(*1 -> mul(x, y), *1) # We have an efficent way to build the executable (we build # and traverse each node only once). # But we don't have an efficient execution. We will execute # like a tree, so nodes that have more then 1 client will be # executed as many times as there number of clients. In the # example aboce, it will calculate *1 twice. Doing otherwise # imply making a complicated execution engine. # We need the fast creation of the executor as we always do it # even if we will use the c code. The Python implementation is # already slow, so it is not as much important to have a fast # execution there. memo = {} def compose_impl(r): if r in memo: return memo[r] if r in self.fgraph.inputs: idx = self.fgraph.inputs.index(r) def f(inputs): return inputs[idx] memo[r] = f return f elif r.owner is None: # in fgraph.orphans: def f(inputs): return r.data memo[r] = f return f node = r.owner producers = [compose_impl(input) for input in node.inputs] def f(inputs): return node.op.impl(*[p(inputs) for p in producers]) memo[r] = f return f self._impls = [compose_impl(r) for r in self.fgraph.outputs] def init_name(self): """ Return a readable string representation of self.fgraph. """ rval = self.name if rval is None: for i, r in enumerate(self.fgraph.inputs): r.name = 'i%i' % i for i, r in enumerate(self.fgraph.outputs): r.name = 'o%i' % i io = set(self.fgraph.inputs + self.fgraph.outputs) for i, r in enumerate(self.fgraph.variables): if r not in io and len(r.clients) > 1: r.name = 't%i' % i rval = "Composite{%s}" % ', '.join([pprint(output) for output in self.fgraph.outputs]) self.name = rval def init_fgraph(self): # The clone done by FunctionGraph is needed as we don't want # the fgraph to be set to the variable as we need to pickle # them for the cache of c module to work. fgraph = FunctionGraph(self.inputs, self.outputs) gof.MergeOptimizer().optimize(fgraph) for node in fgraph.apply_nodes: if not isinstance(node.op, ScalarOp): raise ValueError("The fgraph to Composite must be exclusively" " composed of ScalarOp instances.") self.fgraph = fgraph def __init__(self, inputs, outputs): # We need to clone the graph as sometimes its nodes already # contain a reference to an fgraph. As we want the Composite # to be pickable, we can't have reference to fgraph. # Also, if there is Composite in the inner graph, we want to # remove them. In that case, we do a more complicated clone # that will flatten Composite. We don't need to do this # recusively, as the way the fusion optimizer work, we have # only 1 new Composite each time at the output. for i in inputs: assert i not in outputs # This isn't supported, use identity if len(outputs) > 1 or not any([isinstance(var.owner.op, Composite) for var in outputs]): # No inner Composite inputs, outputs = gof.graph.clone(inputs, outputs) else: # Inner Composite that we need to flatten assert len(outputs) == 1 # 1. Create a new graph from inputs up to the # Composite res = theano.compile.rebuild_collect_shared( inputs=inputs, outputs=outputs[0].owner.inputs, copy_inputs_over=False) # Clone also the inputs # 2. We continue this partial clone with the graph in # the inner Composite res2 = theano.compile.rebuild_collect_shared( inputs=outputs[0].owner.op.inputs, outputs=outputs[0].owner.op.outputs, replace=dict(izip(outputs[0].owner.op.inputs, res[1])) ) assert len(res2[1]) == len(outputs) assert len(res[0]) == len(inputs) assert res[0] != inputs inputs, outputs = res[0], res2[1] # Next assert comment just for speed # assert not any([isinstance(node.op, Composite) for node in # theano.gof.graph.ops(inputs, outputs)]) self.inputs = copy(inputs) self.outputs = copy(outputs) self.inputs_type = tuple([input.type for input in inputs]) self.outputs_type = tuple([output.type for output in outputs]) self.nin = len(inputs) self.nout = len(outputs) self.init_fgraph() # self.fgraph # Postpone the creation in case it isn't needed. # self.init_name() # self.name self.name = None self.prepare_node_called = set() def prepare_node(self, node, storage_map, compute_map, impl): if impl == 'py': self.init_py_impls() # self._impls if impl not in self.prepare_node_called: for n in theano.gof.graph.list_of_nodes(self.inputs, self.outputs): n.op.prepare_node(n, None, None, impl) self.prepare_node_called.add(impl) def clone_float32(self): # This will not modify the fgraph or the nodes new_ins, new_outs = composite_f32.apply(self.fgraph) return Composite(new_ins, new_outs) def output_types(self, input_types): if tuple(input_types) != self.inputs_type: raise TypeError("Wrong types for Composite. Expected %s, got %s." % (self.inputs_type, tuple(input_types))) return self.outputs_type def make_node(self, *inputs): if (tuple([i.type for i in self.inputs]) == tuple([i.type for i in inputs])): return super(Composite, self).make_node(*inputs) else: # Make a new op with the right input type. assert len(inputs) == self.nin res = theano.compile.rebuild_collect_shared( self.outputs, replace=dict(izip(self.inputs, inputs)), rebuild_strict=False) # After rebuild_collect_shared, the Variable in inputs # are not necessarily in the graph represented by res. # res[2][0] is a dict that map from the original variable to the # cloned variable. cloned_inputs = [res[2][0][i] for i in inputs] node = Composite(cloned_inputs, res[1]).make_node(*inputs) return node def perform(self, node, inputs, output_storage): for storage, impl in zip(output_storage, self._impls): storage[0] = impl(inputs) def impl(self, *inputs): output_storage = [[None] for i in xrange(self.nout)] self.perform(None, inputs, output_storage) ret = utils.to_return_values([storage[0] for storage in output_storage]) if self.nout > 1: ret = tuple(ret) return ret def grad(self, inputs, output_grads): raise NotImplementedError("grad is not implemented for Composite") def c_code(self, node, nodename, inames, onames, sub): self.init_c_code() d = dict(chain(izip(("i%i" % i for i in xrange(len(inames))), inames), izip(("o%i" % i for i in xrange(len(onames))), onames)), **sub) d['nodename'] = nodename if 'id' not in sub: # The use of a dummy id is safe as the code is in a separate block. # It won't generate conflicting variable name. d['id'] = '_DUMMY_ID_' return self._c_code % d def c_code_cache_version(self): rval = [3] for x in self.fgraph.toposort(): xv = x.op.c_code_cache_version() if xv: rval.append(xv) else: return () return tuple(rval) def c_support_code(self): rval = [] for subnode in self.fgraph.toposort(): try: rval.append(subnode.op.c_support_code().strip()) except gof.utils.MethodNotDefined: pass # remove duplicate code blocks return "\n".join(sorted(set(rval))) def c_support_code_apply(self, node, name): self.init_c_code() rval = [] for subnode, subnodename in zip(self.fgraph.toposort(), self.nodenames): try: subnode_support_code = subnode.op.c_support_code_apply( subnode, subnodename % dict(nodename=name)) if subnode_support_code: rval.append(subnode_support_code) except gof.utils.MethodNotDefined: pass # there should be no need to remove duplicate code blocks because # each block should have been specialized for the given nodename. # Any block that isn't specialized should be returned via # c_support_code instead of c_support_code_apply. return "\n".join(rval) def __eq__(self, other): if self is other: return True if (type(self) != type(other) or self.nin != other.nin or self.nout != other.nout): return False # see __hash__ for comment on why there is no mention of fgraph # or module cache key here. self.init_c_code() # self._c_code and self.nodenames other.init_c_code() return (self._c_code == other._c_code) def __hash__(self): self.init_c_code() # self._c_code and self.nodenames rval = hash((type(self), self.nin, self.nout, self._c_code)) # Note that in general, the configparser settings at the time # of code generation (__init__) affect the semantics of this Op. # This function assumes that all relevant info about the configparser # is embodied in _c_code. So the _c_code, rather than self.fgraph, # is the signature of the semantics of this Op. # _c_code is preserved through unpickling, so the Op will not change # semantics when it is reloaded with different configparser # settings. return rval def __getstate__(self): rval = dict(self.__dict__) rval.pop('_impls', None) rval.pop('prepare_node_called', None) del rval['fgraph'] return rval def __setstate__(self, d): self.__dict__.update(d) # We must call init to set fgraph and _impls again, as otherwise # self.perform will not work. self.prepare_node_called = set() self.init_fgraph() self.init_py_impls() class Compositef32(object): # This is a dict of scalar op classes that need special handling special = {} def apply(self, fgraph): mapping = {} topo = fgraph.toposort() for i in fgraph.inputs: if i.dtype == 'float16': mapping[i] = get_scalar_type('float32')() if hasattr(i.tag, 'test_value'): mapping[i].tag.test_value = i.tag.test_value else: mapping[i] = i for node in topo: # Patch up for constants for i in node.inputs: if i not in mapping: assert type(i) is ScalarConstant if i.type == float16: ni = ScalarConstant(float32, i.data) else: ni = i mapping[i] = ni if type(node.op) in self.special: self.special[type(node.op)](node, mapping) continue new_node = node.clone_with_new_inputs( [mapping[inp] for inp in node.inputs], strict=False) # make sure we don't produce any float16. assert not any(o.dtype == 'float16' for o in new_node.outputs) for o, no in zip(node.outputs, new_node.outputs): mapping[o] = no new_ins = [mapping[inp] for inp in fgraph.inputs] new_outs = [mapping[out] for out in fgraph.outputs] return new_ins, new_outs composite_f32 = Compositef32() def handle_cast(node, mapping): inp = mapping[node.inputs[0]] out = node.outputs[0] node_ok = False if node.op.o_type == float16: if node.inputs[0].type == float32: # cast f32 -> f16, remove mapping[out] = inp return else: # cast to f16, convert to f32 new_out = cast(inp, 'float32') # change the node for the following if node = new_out.owner mapping[out] = new_out node_ok = True if node.inputs[0].type == float16: if node.op.o_type == inp.type: # cast f16 to new input type, remove mapping[out] = inp return if not node_ok: new_node = node.clone_with_new_inputs([inp], strict=False) mapping[out] = new_node.outputs[0] Compositef32.special[Cast] = handle_cast def handle_composite(node, mapping): new_op = node.op.clone_float32() new_outs = new_op(*[mapping[i] for i in node.inputs], return_list=True) assert len(new_outs) == len(node.outputs) for o, no in zip(node.outputs, new_outs): mapping[o] = no Compositef32.special[Composite] = handle_composite Theano-1.0.0/theano/scalar/basic_sympy.py00006000163562€ÿÿÿþ0000000631013203335532021706 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import itertools as it from theano.scalar.basic import Apply, ScalarOp, as_scalar, float64, float32, int64 from theano.gof.utils import remove imported_sympy = False try: from sympy.utilities.codegen import get_default_datatype, codegen imported_sympy = True except ImportError: pass names = ("sympy_func_%d" % i for i in it.count(0)) def include_line(line): return '#include' in line def sympy_dtype(expr): return get_default_datatype(expr).cname def theano_dtype(expr): return {'double': float64, 'float': float32, 'int': int64}[sympy_dtype(expr)] class SymPyCCode(ScalarOp): """ An Operator that wraps SymPy's C code generation. Examples -------- >>> from sympy.abc import x, y # SymPy Variables >>> from theano.scalar.basic_sympy import SymPyCCode >>> op = SymPyCCode([x, y], x + y) >>> from theano.scalar.basic import floats >>> xt, yt = floats('xy') # Theano variables >>> zt = op(xt, yt) >>> import theano >>> f = theano.function([xt, yt], zt) >>> f(1.0, 2.0) 3.0 """ def __init__(self, inputs, expr, name=None): self.name = name or next(names) self.inputs = inputs self.expr = expr def _sympy_c_code(self): [(c_name, c_code), (h_name, c_header)] = codegen( (self.name, self.expr), 'C', 'project_name', header=False, argument_sequence=self.inputs) return c_code def c_support_code(self): c_code = self._sympy_c_code() return '\n'.join(remove(include_line, c_code.split('\n'))) def c_headers(self): c_code = self._sympy_c_code() return [line.replace("#include", "").strip() for line in c_code.split('\n') if include_line(line) and 'project_name' not in line] def c_code(self, node, name, input_names, output_names, sub): y, = output_names xs = ', '.join(input_names) f = self.name return "%(y)s = %(f)s(%(xs)s);" % locals() def output_types_preference(self, *inputs): return [theano_dtype(self.expr)] def make_node(self, *inputs): # TODO: assert input types are correct use get_default_datatype if len(inputs) != len(self.inputs): raise TypeError("Wrong number of inputs for %s.make_node (got %i(%s), expected %i)" % (self, len(inputs), str(inputs), self.nin)) inputs = [as_scalar(input) for input in inputs] outputs = [t() for t in self.output_types([input.type for input in inputs])] return Apply(self, inputs, outputs) def perform(self, node, inputs, output_storage): raise NotImplementedError() def grad(self, inputs, output_grads): return [SymPyCCode(self.inputs, self.expr.diff(inp), name=self.name + "_grad_%d" % i)(*inputs) for i, inp in enumerate(self.inputs)] def _info(self): return type(self), self.name, tuple(self.inputs), self.expr def __eq__(self, other): return type(self) == type(other) and self._info() == other._info() def __hash__(self): return hash(self._info()) Theano-1.0.0/theano/scalar/__init__.py00006000163562€ÿÿÿþ0000000016313203335532021123 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division from .basic import * from .basic_scipy import * Theano-1.0.0/theano/scalar/basic_scipy.py00006000163562€ÿÿÿþ0000004450013203335532021657 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division # Definitions of theano.scalar ops that have their python implementation taken # from SciPy. As SciPy is not always available, we treat them separately. import numpy as np import theano from theano.gradient import grad_not_implemented from theano.scalar.basic import (UnaryScalarOp, BinaryScalarOp, exp, upgrade_to_float, upgrade_to_float64, float_types) from theano.scalar.basic import (upgrade_to_float_no_complex, complex_types, discrete_types, upcast) imported_scipy_special = False try: import scipy.special import scipy.stats imported_scipy_special = True # Importing scipy.special may raise ValueError. # See http://projects.scipy.org/scipy/ticket/1739 except (ImportError, ValueError): pass class Erf(UnaryScalarOp): nfunc_spec = ('scipy.special.erf', 1, 1) def impl(self, x): if imported_scipy_special: return scipy.special.erf(x) else: super(Erf, self).impl(x) def L_op(self, inputs, outputs, grads): x, = inputs gz, = grads if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] cst = np.asarray(2. / np.sqrt(np.pi), dtype=upcast(x.type.dtype, gz.type.dtype)) return gz * cst * exp(-x * x), def c_code(self, node, name, inp, out, sub): x, = inp z, = out if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = erf((%(cast)s)%(x)s);" % locals() erf = Erf(upgrade_to_float, name='erf') class Erfc(UnaryScalarOp): nfunc_spec = ('scipy.special.erfc', 1, 1) def impl(self, x): if imported_scipy_special: return scipy.special.erfc(x) else: super(Erfc, self).impl(x) def L_op(self, inputs, outputs, grads): x, = inputs gz, = grads if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] cst = np.asarray(2. / np.sqrt(np.pi), dtype=upcast(x.type.dtype, gz.type.dtype)) return - gz * cst * exp(-x * x), def c_code(self, node, name, inp, out, sub): x, = inp z, = out if node.inputs[0].type in complex_types: raise NotImplementedError('type not supported', type) cast = node.outputs[0].type.dtype_specs()[1] return "%(z)s = erfc((%(cast)s)%(x)s);" % locals() # scipy.special.erfc don't support complex. Why? erfc = Erfc(upgrade_to_float_no_complex, name='erfc') class Erfcx(UnaryScalarOp): """ Implements the scaled complementary error function exp(x**2)*erfc(x) in a numerically stable way for large x. This is useful for calculating things like log(erfc(x)) = log(erfcx(x)) - x ** 2 without causing underflow. Should only be used if x is known to be large and positive, as using erfcx(x) for large negative x may instead introduce overflow problems. Notes ----- This op can still be executed on GPU, despite not having c_code. When running on GPU an optimization will replace it with a gpu version. """ nfunc_spec = ('scipy.special.erfcx', 1, 1) def impl(self, x): if imported_scipy_special: return scipy.special.erfcx(x) else: super(Erfcx, self).impl(x) def L_op(self, inputs, outputs, grads): x, = inputs gz, = grads if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] cst = np.asarray(2. / np.sqrt(np.pi), dtype=upcast(x.type.dtype, gz.type.dtype)) return gz * (-cst + (2. * x) * erfcx(x)), erfcx = Erfcx(upgrade_to_float_no_complex, name='erfcx') class Erfinv(UnaryScalarOp): """ Implements the inverse error function. Notes ----- This op can still be executed on GPU, despite not having c_code. When running on GPU, an optimization will replace it with a GPU version. (TODO) Find a C implementation of erfinv for CPU. """ nfunc_spec = ('scipy.special.erfinv', 1, 1) def impl(self, x): if imported_scipy_special: return scipy.special.erfinv(x) else: super(Erfinv, self).impl(x) def L_op(self, inputs, outputs, grads): x, = inputs gz, = grads if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] cst = np.asarray(np.sqrt(np.pi) / 2., dtype=upcast(x.type.dtype, gz.type.dtype)) return gz * cst * exp(erfinv(x) ** 2), # TODO: erfinv() is not provided by the C standard library # def c_code(self, node, name, inp, out, sub): # x, = inp # z, = out # if node.inputs[0].type in complex_types: # raise NotImplementedError('type not supported', type) # return "%(z)s = erfinv(%(x)s);" % locals() erfinv = Erfinv(upgrade_to_float_no_complex, name='erfinv') class Erfcinv(UnaryScalarOp): nfunc_spec = ('scipy.special.erfcinv', 1, 1) def impl(self, x): if imported_scipy_special: return scipy.special.erfcinv(x) else: super(Erfcinv, self).impl(x) def L_op(self, inputs, outputs, grads): x, = inputs gz, = grads if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] cst = np.asarray(np.sqrt(np.pi) / 2., dtype=upcast(x.type.dtype, gz.type.dtype)) return - gz * cst * exp(erfcinv(x) ** 2), # TODO: erfcinv() is not provided by the C standard library # def c_code(self, node, name, inp, out, sub): # x, = inp # z, = out # if node.inputs[0].type in complex_types: # raise NotImplementedError('type not supported', type) # return "%(z)s = erfcinv(%(x)s);" % locals() erfcinv = Erfcinv(upgrade_to_float_no_complex, name='erfcinv') class Gamma(UnaryScalarOp): nfunc_spec = ('scipy.special.gamma', 1, 1) @staticmethod def st_impl(x): return scipy.special.gamma(x) def impl(self, x): if imported_scipy_special: return Gamma.st_impl(x) else: super(Gamma, self).impl(x) def L_op(self, inputs, outputs, gout): (x,) = inputs (gz,) = gout if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return gz * gamma(x) * psi(x), def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs if node.inputs[0].type in float_types: return """%(z)s = tgamma(%(x)s);""" % locals() raise NotImplementedError('only floating point is implemented') gamma = Gamma(upgrade_to_float, name='gamma') class GammaLn(UnaryScalarOp): """ Log gamma function. """ nfunc_spec = ('scipy.special.gammaln', 1, 1) @staticmethod def st_impl(x): return scipy.special.gammaln(x) def impl(self, x): if imported_scipy_special: return GammaLn.st_impl(x) else: super(GammaLn, self).impl(x) def L_op(self, inputs, outputs, grads): x, = inputs gz, = grads if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return [gz * psi(x)] def c_code(self, node, name, inp, out, sub): x, = inp z, = out # no c code for complex # [u]int* will be casted to float64 before computation if node.inputs[0].type in complex_types: raise NotImplementedError( 'gammaln complex c code is not implemented') # For some reason, on the GPU, uint64 inputs don't get casted # automatically to float64. This make the compilation crash dtype = "" cast = node.outputs[0].type.dtype_specs()[1] return """%(z)s = lgamma((%(cast)s)%(x)s);""" % locals() gammaln = GammaLn(upgrade_to_float, name='gammaln') class Psi(UnaryScalarOp): """ Derivative of log gamma function. """ nfunc_spec = ('scipy.special.psi', 1, 1) @staticmethod def st_impl(x): return scipy.special.psi(x) def impl(self, x): if imported_scipy_special: return Psi.st_impl(x) else: super(Psi, self).impl(x) def L_op(self, inputs, outputs, grads): x, = inputs gz, = grads if x.type in complex_types: raise NotImplementedError() if outputs[0].type in discrete_types: if x.type in discrete_types: return [x.zeros_like(dtype=theano.config.floatX)] else: return [x.zeros_like()] return [gz * tri_gamma(x)] def c_support_code(self): return ( """ // For GPU support #ifdef WITHIN_KERNEL #define DEVICE WITHIN_KERNEL #else #define DEVICE #endif #ifndef ga_double #define ga_double double #endif #ifndef _PSIFUNCDEFINED #define _PSIFUNCDEFINED DEVICE double _psi(ga_double x) { /*taken from Bernardo, J. M. (1976). Algorithm AS 103: Psi (Digamma) Function. Applied Statistics. 25 (3), 315-317. http://www.uv.es/~bernardo/1976AppStatist.pdf */ ga_double y, R, psi_ = 0; ga_double S = 1.0e-5; ga_double C = 8.5; ga_double S3 = 8.333333333e-2; ga_double S4 = 8.333333333e-3; ga_double S5 = 3.968253968e-3; ga_double D1 = -0.5772156649; y = x; if (y <= 0.0) return psi_; if (y <= S) return D1 - 1.0/y; while (y < C) { psi_ = psi_ - 1.0 / y; y = y + 1; } R = 1.0 / y; psi_ = psi_ + log(y) - .5 * R ; R= R*R; psi_ = psi_ - R * (S3 - R * (S4 - R * S5)); return psi_; } #endif """) def c_code(self, node, name, inp, out, sub): x, = inp z, = out if node.inputs[0].type in float_types: return """%(z)s = _psi(%(x)s);""" % locals() raise NotImplementedError('only floating point is implemented') psi = Psi(upgrade_to_float, name='psi') class TriGamma(UnaryScalarOp): """ Second derivative of log gamma function. """ @staticmethod def st_impl(x): return scipy.special.polygamma(1, x) def impl(self, x): if imported_scipy_special: return TriGamma.st_impl(x) else: super(TriGamma, self).impl(x) def grad(self, inputs, outputs_gradients): raise NotImplementedError() def c_support_code(self): # The implementation has been copied from # http://people.sc.fsu.edu/~jburkardt/cpp_src/asa121/asa121.html return ( """ // For GPU support #ifdef WITHIN_KERNEL #define DEVICE WITHIN_KERNEL #else #define DEVICE #endif #ifndef ga_double #define ga_double double #endif #ifndef _TRIGAMMAFUNCDEFINED #define _TRIGAMMAFUNCDEFINED DEVICE double _tri_gamma(ga_double x) { double a = 0.0001; double b = 5.0; double b2 = 0.1666666667; double b4 = -0.03333333333; double b6 = 0.02380952381; double b8 = -0.03333333333; double value; double y; double z; if (x <= 0) { return 0.0; } if ( x <= a ) { value = 1.0 / x / x; return value; } value = 0.0; z = x; while ( z < b ) { value += 1.0 / z / z; z += 1.0; } y = 1.0 / z / z; value += 0.5 * y + (1.0 + y * (b2 + y * (b4 + y * (b6 + y * b8 )))) / z; return value; } #endif """) def c_code(self, node, name, inp, out, sub): x, = inp z, = out if node.inputs[0].type in float_types: return """%(z)s = _tri_gamma(%(x)s);""" % locals() raise NotImplementedError('only floating point is implemented') tri_gamma = TriGamma(upgrade_to_float, name='tri_gamma') class Chi2SF(BinaryScalarOp): """ Compute (1 - chi2_cdf(x)) ie. chi2 pvalue (chi2 'survival function'). C code is provided in the Theano_lgpl repository. This make it faster. https://github.com/Theano/Theano_lgpl.git """ nfunc_spec = ('scipy.stats.chi2.sf', 2, 1) @staticmethod def st_impl(x, k): return scipy.stats.chi2.sf(x, k) def impl(self, x, k): if imported_scipy_special: return Chi2SF.st_impl(x, k) else: super(Chi2SF, self).impl(x, k) chi2sf = Chi2SF(upgrade_to_float64, name='chi2sf') class Jv(BinaryScalarOp): """ Bessel function of the first kind of order v (real). """ nfunc_spec = ('scipy.special.jv', 2, 1) @staticmethod def st_impl(v, x): return scipy.special.jv(v, x) def impl(self, v, x): if imported_scipy_special: return self.st_impl(v, x) else: super(Jv, self).impl(v, x) def grad(self, inputs, grads): v, x = inputs gz, = grads return [grad_not_implemented(self, 0, v), gz * (jv(v - 1, x) - jv(v + 1, x)) / 2.] jv = Jv(upgrade_to_float, name='jv') class J1(UnaryScalarOp): """ Bessel function of the first kind of order 1. """ nfunc_spec = ('scipy.special.j1', 1, 1) @staticmethod def st_impl(x): return scipy.special.j1(x) def impl(self, x): if imported_scipy_special: return self.st_impl(x) else: super(J1, self).impl(x) def grad(self, inputs, grads): x, = inputs gz, = grads return [gz * (j0(x) - jv(2, x)) / 2.] def c_code(self, node, name, inp, out, sub): x, = inp z, = out if node.inputs[0].type in float_types: return """%(z)s = j1(%(x)s);""" % locals() raise NotImplementedError('only floating point is implemented') j1 = J1(upgrade_to_float, name='j1') class J0(UnaryScalarOp): """ Bessel function of the first kind of order 0. """ nfunc_spec = ('scipy.special.j0', 1, 1) @staticmethod def st_impl(x): return scipy.special.j0(x) def impl(self, x): if imported_scipy_special: return self.st_impl(x) else: super(J0, self).impl(x) def grad(self, inp, grads): x, = inp gz, = grads return [gz * -1 * j1(x)] def c_code(self, node, name, inp, out, sub): x, = inp z, = out if node.inputs[0].type in float_types: return """%(z)s = j0(%(x)s);""" % locals() raise NotImplementedError('only floating point is implemented') j0 = J0(upgrade_to_float, name='j0') class Iv(BinaryScalarOp): """ Modified Bessel function of the first kind of order v (real). """ nfunc_spec = ('scipy.special.iv', 2, 1) @staticmethod def st_impl(v, x): return scipy.special.iv(v, x) def impl(self, v, x): if imported_scipy_special: return self.st_impl(v, x) else: super(Iv, self).impl(v, x) def grad(self, inputs, grads): v, x = inputs gz, = grads return [grad_not_implemented(self, 0, v), gz * (iv(v - 1, x) + iv(v + 1, x)) / 2.] iv = Iv(upgrade_to_float, name='iv') class I1(UnaryScalarOp): """ Modified Bessel function of the first kind of order 1. """ nfunc_spec = ('scipy.special.i1', 1, 1) @staticmethod def st_impl(x): return scipy.special.i1(x) def impl(self, x): if imported_scipy_special: return self.st_impl(x) else: super(I1, self).impl(x) def grad(self, inputs, grads): x, = inputs gz, = grads return [gz * (i0(x) + iv(2, x)) / 2.] i1 = I1(upgrade_to_float, name='i1') class I0(UnaryScalarOp): """ Modified Bessel function of the first kind of order 0. """ nfunc_spec = ('scipy.special.i0', 1, 1) @staticmethod def st_impl(x): return scipy.special.i0(x) def impl(self, x): if imported_scipy_special: return self.st_impl(x) else: super(I0, self).impl(x) def grad(self, inp, grads): x, = inp gz, = grads return [gz * i1(x)] i0 = I0(upgrade_to_float, name='i0') Theano-1.0.0/theano/tensor/00007500163562€ÿÿÿþ0000000000013203336026017061 5ustar boccoset00000000000000Theano-1.0.0/theano/tensor/sharedvar.py00006000163562€ÿÿÿþ0000000727713203335532021425 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import traceback import numpy as np from six import integer_types import theano.tensor.basic from theano.tensor.basic import TensorType, _tensor_py_operators from theano.compile import shared_constructor, SharedVariable def load_shared_variable(val): """ This function is only here to keep some pickles loading after a failed fix done in August 2011. It can be removed after sufficient time has passed. """ return tensor_constructor(val) # _tensor_py_operators is first to have its version of __{gt,ge,lt,le}__ class TensorSharedVariable(_tensor_py_operators, SharedVariable): pass @shared_constructor def tensor_constructor(value, name=None, strict=False, allow_downcast=None, borrow=False, broadcastable=None, target='cpu'): """ SharedVariable Constructor for TensorType. Notes ----- Regarding the inference of the broadcastable pattern... The default is to assume that the value might be resized in any dimension, so the default broadcastable is ``(False,)*len(value.shape)``. The optional `broadcastable` argument will override this default. """ if target != 'cpu': raise TypeError('not for cpu') if not isinstance(value, np.ndarray): raise TypeError() # if no broadcastable is given, then the default is to assume that # the value might be resized in any dimension in the future. # if broadcastable is None: broadcastable = (False,) * len(value.shape) type = TensorType(value.dtype, broadcastable=broadcastable) return TensorSharedVariable(type=type, value=np.array(value, copy=(not borrow)), name=name, strict=strict, allow_downcast=allow_downcast) # TensorSharedVariable brings in the tensor operators, is not ideal, but works # as long as we dont do purely scalar-scalar operations # _tensor_py_operators is first to have its version of __{gt,ge,lt,le}__ # # N.B. THERE IS ANOTHER CLASS CALLED ScalarSharedVariable in the # theano.scalar.sharedvar file. It is not registered as a shared_constructor, # this one is. class ScalarSharedVariable(_tensor_py_operators, SharedVariable): pass @shared_constructor def scalar_constructor(value, name=None, strict=False, allow_downcast=None, borrow=False, target='cpu'): """ SharedVariable constructor for scalar values. Default: int64 or float64. Notes ----- We implement this using 0-d tensors for now. We ignore the borrow parameter as we convert ``value`` to an ndarray (this is a new object). This respects the semantic of borrow, as it is a hint to Theano that we can reuse it. """ if target != 'cpu': raise TypeError('not for cpu') if not isinstance(value, (np.number, float, integer_types, complex)): raise TypeError() try: dtype = value.dtype except Exception: dtype = np.asarray(value).dtype dtype = str(dtype) value = theano._asarray(value, dtype=dtype) tensor_type = TensorType(dtype=str(value.dtype), broadcastable=[]) try: # Do not pass the dtype to asarray because we want this to fail if # strict is True and the types do not match. rval = ScalarSharedVariable(type=tensor_type, value=np.array(value, copy=True), name=name, strict=strict, allow_downcast=allow_downcast) return rval except Exception: traceback.print_exc() raise Theano-1.0.0/theano/tensor/extra_ops.py00006000163562€ÿÿÿþ0000012646213203335532021450 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import numpy as np from six.moves import xrange import theano from theano.tensor import basic from theano.tensor import nlinalg # noqa from theano import gof, scalar from theano.gof import Generic, ParamsType, EnumList from theano import gradient from theano.gradient import DisconnectedType, disconnected_type from theano.scalar import int32 as int_t tensor = basic class CpuContiguous(theano.Op): """ Check to see if the input is c-contiguous, if it is, do nothing, else return a contiguous array. """ __props__ = () view_map = {0: [0]} check_input = False def make_node(self, x): x_ = theano.tensor.as_tensor_variable(x) return theano.Apply(self, [x_], [x_.type()]) def perform(self, node, inputs, output_storage): x, = inputs y = output_storage[0] # if the ouput is contiguous do nothing, else copy # the input if not x.flags['C_CONTIGUOUS']: x = x.copy() assert x.flags['C_CONTIGUOUS'] y[0] = x def grad(self, inputs, dout): return [theano.tensor.as_tensor_variable(dout[0])] def c_code(self, node, name, inames, onames, sub): x, = inames y, = onames code = """ if (!PyArray_CHKFLAGS(%(x)s, NPY_ARRAY_C_CONTIGUOUS)){ // check to see if output is contiguous first if (%(y)s != NULL && PyArray_CompareLists(PyArray_DIMS(%(y)s), PyArray_DIMS(%(x)s), PyArray_NDIM(%(x)s)) && PyArray_CHKFLAGS(%(y)s, NPY_ARRAY_C_CONTIGUOUS)){ PyArray_CopyInto(%(y)s, %(x)s); } else{ Py_XDECREF(%(y)s); %(y)s = PyArray_GETCONTIGUOUS(%(x)s); } } else{ Py_XINCREF(%(x)s); Py_XDECREF(%(y)s); %(y)s = %(x)s; } """ % locals() return code def c_code_cache_version(self): return (1,) cpu_contiguous = CpuContiguous() class SearchsortedOp(theano.Op): """Wrapper of numpy.searchsorted. For full documentation, see :func:`searchsorted`. See Also -------- searchsorted : numpy-like function to use the SearchsortedOp """ params_type = Generic() __props__ = ("side", ) check_input = False def __init__(self, side='left'): if side == 'left' or side == 'right': self.side = side else: raise ValueError('\'%(side)s\' is an invalid value for keyword \'side\'' % locals()) def get_params(self, node): return self.side def make_node(self, x, v, sorter=None): x = basic.as_tensor(x, ndim=1) v = basic.as_tensor(v) out_type = v.type.clone(dtype='int64') if sorter is None: return theano.Apply(self, [x, v], [out_type()]) else: sorter = basic.as_tensor(sorter, ndim=1) if (theano.configdefaults.python_int_bitwidth() == 32 and sorter.dtype == 'int64'): raise TypeError( "numpy.searchsorted with Python 32bit do not support a" " sorter of int64.") if sorter.type not in basic.int_vector_types: raise TypeError('sorter must be an integer vector', sorter.type) return theano.Apply(self, [x, v, sorter], [out_type()]) def infer_shape(self, node, shapes): return [shapes[1]] def perform(self, node, inputs, output_storage, params): x = inputs[0] v = inputs[1] if len(node.inputs) == 3: sorter = inputs[2] else: sorter = None z = output_storage[0] z[0] = np.searchsorted(x, v, side=params, sorter=sorter).astype( node.outputs[0].dtype) def c_support_code_struct(self, node, name): return """ int right_%(name)s; """ % locals() def c_init_code_struct(self, node, name, sub): side = sub['params'] fail = sub['fail'] return """ PyObject* tmp_%(name)s = PyUnicode_FromString("right"); if (tmp_%(name)s == NULL) %(fail)s; right_%(name)s = PyUnicode_Compare(%(side)s, tmp_%(name)s); Py_DECREF(tmp_%(name)s); """ % locals() def c_code(self, node, name, inames, onames, sub): sorter = None if len(node.inputs) == 3: x, v, sorter = inames else: x, v = inames if not sorter: sorter = "NULL" z, = onames fail = sub['fail'] return """ Py_XDECREF(%(z)s); %(z)s = (PyArrayObject*) PyArray_SearchSorted(%(x)s, (PyObject*) %(v)s, right_%(name)s ? NPY_SEARCHLEFT : NPY_SEARCHRIGHT, (PyObject*) %(sorter)s); if (!%(z)s) %(fail)s; if (PyArray_TYPE(%(z)s) != NPY_INT64){ PyObject * tmp = PyArray_Cast(%(z)s, NPY_INT64); Py_XDECREF(%(z)s); %(z)s = (PyArrayObject*) tmp; } """ % locals() def c_code_cache_version(self): return (2,) def grad(self, inputs, output_gradients): num_ins = len(inputs) if num_ins == 3: x, v, sorter = inputs else: x, v = inputs x_grad = gradient._float_zeros_like(x) v_grad = gradient._float_zeros_like(v) if num_ins == 3: return [x_grad, v_grad, disconnected_type()] else: return [x_grad, v_grad] def searchsorted(x, v, side='left', sorter=None): """Find indices where elements should be inserted to maintain order. Wrapping of numpy.searchsorted. Find the indices into a sorted array `x` such that, if the corresponding elements in `v` were inserted before the indices, the order of `x` would be preserved. Parameters ---------- x: 1-D tensor (array-like) Input array. If `sorter` is None, then it must be sorted in ascending order, otherwise `sorter` must be an array of indices which sorts it. v: tensor (array-like) Contains the values to be inserted into `x`. side: {'left', 'right'}, optional. If 'left' (default), the index of the first suitable location found is given. If 'right', return the last such index. If there is no suitable index, return either 0 or N (where N is the length of `x`). sorter: 1-D tensor of integers (array-like), optional Contains indices that sort array `x` into ascending order. They are typically the result of argsort. Returns ------- indices : tensor of integers (int64) Array of insertion points with the same shape as `v`. See Also -------- `numpy.searchsorted `_ Notes ----- * Binary search is used to find the required insertion points. * This Op is working **only on CPU** currently. Examples -------- >>> from theano import tensor >>> x = tensor.dvector() >>> idx = x.searchsorted(3) >>> idx.eval({x: [1,2,3,4,5]}) array(2) >>> tensor.extra_ops.searchsorted([1,2,3,4,5], 3).eval() array(2) >>> tensor.extra_ops.searchsorted([1,2,3,4,5], 3, side='right').eval() array(3) >>> tensor.extra_ops.searchsorted([1,2,3,4,5], [-10, 10, 2, 3]).eval() array([0, 5, 1, 2]) .. versionadded:: 0.9 """ return SearchsortedOp(side=side)(x, v, sorter) class CumOp(theano.Op): # See function cumsum/cumprod for docstring __props__ = ("axis", "mode") check_input = False params_type = ParamsType(c_axis=int_t, mode=EnumList(('MODE_ADD', 'add'), ('MODE_MUL', 'mul'))) def __init__(self, axis=None, mode='add'): if mode not in ('add', 'mul'): raise ValueError('%s: Unknown mode "%s"' % (type(self).__name__, mode)) self.axis = axis self.mode = mode c_axis = property(lambda self: np.MAXDIMS if self.axis is None else self.axis) def make_node(self, x): x = basic.as_tensor_variable(x) out_type = x.type() if self.axis is None: out_type = theano.tensor.vector(dtype=x.dtype) # Flatten elif self.axis >= x.ndim or self.axis < -x.ndim: raise ValueError('axis(={0}) out of bounds'.format(self.axis)) return theano.Apply(self, [x], [out_type]) def perform(self, node, inputs, output_storage, params): x = inputs[0] z = output_storage[0] z[0] = {'add': np.cumsum, 'mul': np.cumprod}[self.mode](x, axis=self.axis) def grad(self, inputs, output_gradients): x, = inputs gi, = output_gradients if self.axis is None: if self.mode == 'add': return [cumsum(gi[::-1])[::-1].reshape(x.shape)] elif self.mode == 'mul': fx = cumprod(x, axis=self.axis) return [cumsum( (fx * gi)[::-1])[::-1].reshape(x.shape) / x] else: raise NotImplementedError( '%s: unknown gradient for mode "%s"' % (type(self).__name__, self.mode)) reverse_slicing = [slice(None, None, None)] * gi.ndim reverse_slicing[self.axis] = slice(None, None, -1) reverse_slicing = tuple(reverse_slicing) # We need to reverse the gradients along ``self.axis``, # compute cumsum, then reverse again if self.mode == 'add': return [cumsum(gi[reverse_slicing], self.axis)[reverse_slicing]] elif self.mode == 'mul': fx = cumprod(x, axis=self.axis) return [cumsum( (fx * gi)[reverse_slicing], self.axis)[reverse_slicing] / x] else: raise NotImplementedError( '%s: unknown gradient for mode "%s"' % (type(self).__name__, self.mode)) def infer_shape(self, node, shapes): if self.axis is None: return [(tensor.prod(shapes[0]),)] # Flatten return shapes def c_code(self, node, name, inames, onames, sub): x, = inames z, = onames axis = self.axis fail = sub['fail'] params = sub['params'] code = """ int axis = %(params)s->c_axis; if (axis == 0 && PyArray_NDIM(%(x)s) == 1) axis = NPY_MAXDIMS; npy_intp shape[1] = { PyArray_SIZE(%(x)s) }; if(axis == NPY_MAXDIMS && !(%(z)s && PyArray_DIMS(%(z)s)[0] == shape[0])) { Py_XDECREF(%(z)s); %(z)s = (PyArrayObject*) PyArray_SimpleNew(1, shape, PyArray_TYPE((PyArrayObject*) py_%(x)s)); } else if(axis != NPY_MAXDIMS && !(%(z)s && PyArray_CompareLists(PyArray_DIMS(%(z)s), PyArray_DIMS(%(x)s), PyArray_NDIM(%(x)s)))) { Py_XDECREF(%(z)s); %(z)s = (PyArrayObject*) PyArray_SimpleNew(PyArray_NDIM(%(x)s), PyArray_DIMS(%(x)s), PyArray_TYPE(%(x)s)); } if (!%(z)s) %(fail)s; { PyObject * t = NULL; if(%(params)s->mode == MODE_ADD) t = PyArray_CumSum( %(x)s, axis, PyArray_TYPE(%(x)s), %(z)s); else if(%(params)s->mode == MODE_MUL) t = PyArray_CumProd( %(x)s, axis, PyArray_TYPE(%(x)s), %(z)s); if (!t){ %(fail)s; } // Because PyArray_CumSum/CumProd returns a newly created reference on t. Py_XDECREF(t); } """ % locals() return code def c_code_cache_version(self): return (8,) def __str__(self): return "%s{%s, %s}" % (self.__class__.__name__, self.axis, self.mode) def cumsum(x, axis=None): """Return the cumulative sum of the elements along a given axis. Wraping of numpy.cumsum. Parameters ---------- x Input tensor variable. axis The axis along which the cumulative sum is computed. The default (None) is to compute the cumsum over the flattened array. .. versionadded:: 0.7 """ return CumOp(axis=axis, mode='add')(x) def cumprod(x, axis=None): """Return the cumulative product of the elements along a given axis. Wraping of numpy.cumprod. Parameters ---------- x Input tensor variable. axis The axis along which the cumulative product is computed. The default (None) is to compute the cumprod over the flattened array. .. versionadded:: 0.7 """ return CumOp(axis=axis, mode='mul')(x) # CumsumOp and CumprodOp are for compatibility with old version, # just in case unpickling a theano function with old Ops. class CumsumOp(theano.Op): __props__ = ("axis",) def __new__(typ, *args, **kwargs): obj = object.__new__(CumOp, *args, **kwargs) obj.mode = 'add' return obj class CumprodOp(theano.Op): __props__ = ("axis",) def __new__(typ, *args, **kwargs): obj = object.__new__(CumOp, *args, **kwargs) obj.mode = 'mul' return obj class DiffOp(theano.Op): # See function diff for docstring __props__ = ("n", "axis") def __init__(self, n=1, axis=-1): self.n = n self.axis = axis # numpy return a view in that case. # TODO, make an optimization that remove this op in this case. if n == 0: self.view_map = {0: [0]} def make_node(self, x): x = basic.as_tensor_variable(x) return theano.Apply(self, [x], [x.type()]) def perform(self, node, inputs, output_storage): x = inputs[0] z = output_storage[0] z[0] = np.diff(x, n=self.n, axis=self.axis) def grad(self, inputs, outputs_gradients): inputs = inputs[0] if inputs.ndim != 1: raise NotImplementedError("Grad is not implemented for inputs with" "number of dimension other than 1.") z = outputs_gradients[0] def _grad_helper(z): pre = basic.concatenate([[0.], z]) app = basic.concatenate([z, [0.]]) return pre - app for k in range(self.n): z = _grad_helper(z) return [z] def infer_shape(self, node, ins_shapes): i0_shapes = ins_shapes[0] out_shape = list(i0_shapes) out_shape[self.axis] = out_shape[self.axis] - self.n return [out_shape] def diff(x, n=1, axis=-1): """Calculate the n-th order discrete difference along given axis. The first order difference is given by out[i] = a[i + 1] - a[i] along the given axis, higher order differences are calculated by using diff recursively. Wraping of numpy.diff. Parameters ---------- x Input tensor variable. n The number of times values are differenced, default is 1. axis The axis along which the difference is taken, default is the last axis. .. versionadded:: 0.6 """ return DiffOp(n=n, axis=axis)(x) def bincount(x, weights=None, minlength=None, assert_nonneg=False): """Count number of occurrences of each value in array of ints. The number of bins (of size 1) is one larger than the largest value in x. If minlength is specified, there will be at least this number of bins in the output array (though it will be longer if necessary, depending on the contents of x). Each bin gives the number of occurrences of its index value in x. If weights is specified the input array is weighted by it, i.e. if a value n is found at position i, out[n] += weight[i] instead of out[n] += 1. Parameters ---------- x : 1 dimension, nonnegative ints weights : array of the same shape as x with corresponding weights. Optional. minlength : A minimum number of bins for the output array. Optional. assert_nonneg : A flag that inserts an assert_op to check if every input x is nonnegative. Optional. .. versionadded:: 0.6 """ if x.ndim != 1: raise TypeError("Inputs must be of dimension 1.") if assert_nonneg: from theano.tensor.opt import Assert assert_op = Assert('Input to bincount has negative values!') x = assert_op(x, theano.tensor.all(x >= 0)) max_value = theano.tensor.cast(x.max() + 1, 'int64') if minlength is not None: max_value = theano.tensor.maximum(max_value, minlength) # Note: we do not use inc_subtensor(out[x], ...) in the following lines, # since out[x] raises an exception if the indices (x) are int8. if weights is None: out = theano.tensor.zeros([max_value], dtype=x.dtype) out = theano.tensor.advanced_inc_subtensor1(out, 1, x) else: out = theano.tensor.zeros([max_value], dtype=weights.dtype) out = theano.tensor.advanced_inc_subtensor1(out, weights, x) return out def squeeze(x): """ Remove broadcastable dimensions from the shape of an array. It returns the input array, but with the broadcastable dimensions removed. This is always `x` itself or a view into `x`. .. versionadded:: 0.6 Parameters ---------- x Input data, tensor variable. Returns ------- object `x` without its broadcastable dimensions. """ view = x.dimshuffle([i for i in range(x.ndim) if not x.broadcastable[i]]) return view def compress(condition, x, axis=None): """ Return selected slices of an array along given axis. It returns the input tensor, but with selected slices along a given axis retained. If no axis is provided, the tensor is flattened. Corresponds to numpy.compress .. versionadded:: 0.7 Parameters ---------- x Input data, tensor variable. condition 1 dimensional array of non-zero and zero values corresponding to indices of slices along a selected axis. Returns ------- object `x` with selected slices. """ indices = theano.tensor.basic.flatnonzero(condition) return x.take(indices, axis=axis) class RepeatOp(theano.Op): # See the repeat function for docstring __props__ = ("axis",) def __init__(self, axis=None): self.axis = axis def make_node(self, x, repeats): x = basic.as_tensor_variable(x) repeats = basic.as_tensor_variable(repeats) if repeats.dtype not in tensor.integer_dtypes: raise TypeError("repeats.dtype must be an integer.") # Some dtypes are not supported by numpy's implementation of repeat. # Until another one is available, we should fail at graph construction # time, not wait for execution. ptr_bitwidth = theano.configdefaults.local_bitwidth() if ptr_bitwidth == 64: numpy_unsupported_dtypes = ('uint64',) if ptr_bitwidth == 32: numpy_unsupported_dtypes = ('uint32', 'int64', 'uint64') if repeats.dtype in numpy_unsupported_dtypes: raise TypeError( ("dtypes %s are not supported by numpy.repeat " "for the 'repeats' parameter, " % str(numpy_unsupported_dtypes)), repeats.dtype) if self.axis is None: broadcastable = [False] else: try: const_reps = basic.get_scalar_constant_value(repeats) except basic.NotScalarConstantError: const_reps = None if const_reps == 1: broadcastable = x.broadcastable else: broadcastable = list(x.broadcastable) broadcastable[self.axis] = False out_type = theano.tensor.TensorType(x.dtype, broadcastable) return theano.Apply(self, [x, repeats], [out_type()]) def perform(self, node, inputs, output_storage): x = inputs[0] repeats = inputs[1] z = output_storage[0] z[0] = np.repeat(x, repeats=repeats, axis=self.axis) def connection_pattern(self, node): return [[True], [False]] def grad(self, inputs, gout): (x, repeats) = inputs (gz,) = gout if repeats.ndim == 0: if self.axis is None: axis = x.ndim else: if self.axis >= 0: axis = self.axis + 1 else: axis = self.axis + x.ndim + 1 shape = [x.shape[k] for k in range(x.ndim)] shape.insert(axis, repeats) return [gz.reshape(shape, x.ndim + 1).sum(axis=axis), DisconnectedType()()] elif repeats.ndim == 1: # For this implementation, we would need to specify the length # of repeats in order to split gz in the right way to sum # the good part. raise NotImplementedError() else: raise ValueError() def infer_shape(self, node, ins_shapes): i0_shapes = ins_shapes[0] repeats = node.inputs[1] out_shape = list(i0_shapes) # uint64 shape are not supported. dtype = None if repeats.dtype in ['uint8', 'uint16', 'uint32']: dtype = 'int64' if self.axis is None: if repeats.ndim == 0: if len(i0_shapes) == 0: out_shape = [repeats] else: res = 1 for d in i0_shapes: res = res * d out_shape = (res * repeats, ) else: out_shape = [theano.tensor.sum(repeats, dtype=dtype)] else: if repeats.ndim == 0: out_shape[self.axis] = out_shape[self.axis] * repeats else: out_shape[self.axis] = theano.tensor.sum(repeats, dtype=dtype) return [out_shape] def repeat(x, repeats, axis=None): """Repeat elements of an array. It returns an array which has the same shape as `x`, except along the given axis. The axis is used to speficy along which axis to repeat values. By default, use the flattened input array, and return a flat output array. The number of repetitions for each element is `repeat`. `repeats` is broadcasted to fit the length of the given `axis`. Parameters ---------- x Input data, tensor variable. repeats int, scalar or tensor variable axis : int, optional See Also -------- tensor.tile .. versionadded:: 0.6 """ repeats = tensor.as_tensor_variable(repeats) if repeats.ndim > 1: raise ValueError('The dimension of repeats should not exceed 1.') if repeats.ndim == 1 and not repeats.broadcastable[0]: return RepeatOp(axis=axis)(x, repeats) else: if repeats.ndim == 1: repeats = repeats[0] if x.dtype == 'uint64': raise TypeError("theano.tensor.repeat don't support dtype uint64") if axis is None: axis = 0 x = x.flatten() else: if axis >= x.ndim: raise ValueError('Axis should not exceed x.ndim-1.') if axis < 0: axis = x.ndim + axis shape = [x.shape[i] for i in xrange(x.ndim)] # shape_ is the shape of the intermediate tensor which has # an additional dimension comparing to x. We use alloc to # allocate space for this intermediate tensor to replicate x # along that additional dimension. shape_ = shape[:] shape_.insert(axis + 1, repeats) # shape is now the shape of output, where shape[axis] becomes # shape[axis]*repeats. shape[axis] = shape[axis] * repeats # dims_ is the dimension of that intermediate tensor. dims_ = list(np.arange(x.ndim)) dims_.insert(axis + 1, 'x') # After the original tensor is duplicated along the additional # dimension, we reshape it to the expected output shape, and # return the output z. z = tensor.alloc(x.dimshuffle(*dims_), *shape_).reshape(shape) return z class Bartlett(gof.Op): # See function bartlett for docstring __props__ = () def make_node(self, M): M = tensor.as_tensor_variable(M) if M.ndim != 0: raise TypeError('%s only works on scalar input' % self.__class__.__name__) elif M.dtype not in theano.tensor.integer_dtypes: # dtype is a theano attribute here raise TypeError('%s only works on integer input' % self.__class__.__name__) return gof.Apply(self, [M], [tensor.dvector()]) def perform(self, node, inputs, out_): M = inputs[0] out, = out_ out[0] = np.bartlett(M) def infer_shape(self, node, in_shapes): temp = node.inputs[0] M = tensor.switch(tensor.lt(temp, 0), tensor.cast(0, temp.dtype), temp) return [[M]] def grad(self, inputs, output_grads): return [None for i in inputs] bartlett_ = Bartlett() # I create a function only to have the doc show well. def bartlett(M): """ An instance of this class returns the Bartlett spectral window in the time-domain. The Bartlett window is very similar to a triangular window, except that the end points are at zero. It is often used in signal processing for tapering a signal, without generating too much ripple in the frequency domain. .. versionadded:: 0.6 Parameters ---------- M : integer scalar Number of points in the output window. If zero or less, an empty vector is returned. Returns ------- vector of doubles The triangular window, with the maximum value normalized to one (the value one appears only if the number of samples is odd), with the first and last samples equal to zero. """ return bartlett_(M) class FillDiagonal(gof.Op): # See function fill_diagonal for docstring __props__ = () def infer_shape(self, node, in_shapes): return [in_shapes[0]] def make_node(self, a, val): a = tensor.as_tensor_variable(a) val = tensor.as_tensor_variable(val) if a.ndim < 2: raise TypeError('%s: first parameter must have at least' ' two dimensions' % self.__class__.__name__) elif val.ndim != 0: raise TypeError('%s: second parameter must be a scalar' % self.__class__.__name__) val = tensor.cast(val, dtype=scalar.upcast(a.dtype, val.dtype)) if val.dtype != a.dtype: raise TypeError('%s: type of second parameter must be the same as' ' the first\'s' % self.__class__.__name__) return gof.Apply(self, [a, val], [a.type()]) def perform(self, node, inputs, output_storage): a = inputs[0].copy() val = inputs[1] if a.ndim == 2: # numpy.fill_diagonal up to date(including 1.6.2) have a # bug for tall matrix. # For 2-d arrays, we accept rectangular ones. step = a.shape[1] + 1 end = a.shape[1] * a.shape[1] # Write the value out into the diagonal. a.flat[:end:step] = val else: np.fill_diagonal(a, val) output_storage[0][0] = a def grad(self, inp, cost_grad): """ Notes ----- The gradient is currently implemented for matrices only. """ a, val = inp grad = cost_grad[0] if (a.dtype.startswith('complex')): return [None, None] elif a.ndim > 2: raise NotImplementedError('%s: gradient is currently implemented' ' for matrices only' % self.__class__.__name__) wr_a = fill_diagonal(grad, 0) # valid for any number of dimensions # diag is only valid for matrices wr_val = theano.tensor.nlinalg.diag(grad).sum() return [wr_a, wr_val] fill_diagonal_ = FillDiagonal() # I create a function only to have the doc show well. def fill_diagonal(a, val): """ Returns a copy of an array with all elements of the main diagonal set to a specified scalar value. .. versionadded:: 0.6 Parameters ---------- a Rectangular array of at least two dimensions. val Scalar value to fill the diagonal whose type must be compatible with that of array 'a' (i.e. 'val' cannot be viewed as an upcast of 'a'). Returns ------- array An array identical to 'a' except that its main diagonal is filled with scalar 'val'. (For an array 'a' with a.ndim >= 2, the main diagonal is the list of locations a[i, i, ..., i] (i.e. with indices all identical).) Support rectangular matrix and tensor with more than 2 dimensions if the later have all dimensions are equals. """ return fill_diagonal_(a, val) class FillDiagonalOffset(gof.Op): # See function fill_diagonal_offset for docstring __props__ = () def infer_shape(self, node, in_shapes): return [in_shapes[0]] def make_node(self, a, val, offset): a = tensor.as_tensor_variable(a) val = tensor.as_tensor_variable(val) offset = tensor.as_tensor_variable(offset) if a.ndim != 2: raise TypeError('%s: first parameter must have exactly' ' two dimensions' % self.__class__.__name__) elif val.ndim != 0: raise TypeError('%s: second parameter must be a scalar' % self.__class__.__name__) elif offset.ndim != 0: raise TypeError('%s: third parameter must be a scalar' % self.__class__.__name__) val = tensor.cast(val, dtype=scalar.upcast(a.dtype, val.dtype)) if val.dtype != a.dtype: raise TypeError('%s: type of second parameter must be the same' ' as the first\'s' % self.__class__.__name__) elif offset.dtype not in theano.tensor.integer_dtypes: raise TypeError('%s: type of third parameter must be as integer' ' use theano.tensor.cast( input, \'int32/int64\')' % self.__class__.__name__) return gof.Apply(self, [a, val, offset], [a.type()]) def perform(self, node, inputs, output_storage): a = inputs[0].copy() val = inputs[1] offset = inputs[2] height, width = a.shape """ Notes ----- The fill_diagonal only support rectangular matrix. The output of tall matrix is "wrapped", which is an option in numpy 1.9.0 but was regarded as a bug in numpy 1.6.2. Here I implement the fill_diagonal_offset with unwrapped output, so fill_diagonal_offset supports tall matrix.(This make a little difference between the output of fill_diagonal and fill_diagonal_offset only in the case of tall matrix) """ if offset >= 0: start = offset num_of_step = min(min(width, height), width - offset) else: start = - offset * a.shape[1] num_of_step = min(min(width, height), height + offset) step = a.shape[1] + 1 end = start + step * num_of_step # Write the value out into the diagonal. a.flat[start:end:step] = val output_storage[0][0] = a def grad(self, inp, cost_grad): """ Notes ----- The gradient is currently implemented for matrices only. """ a, val, offset = inp grad = cost_grad[0] height, width = grad.shape if (a.dtype.startswith('complex')): return [None, None] # only valid for matrices wr_a = fill_diagonal_offset(grad, 0, offset) offset_abs = basic.abs_(offset) pos_offset_flag = basic.ge(offset, 0) neg_offset_flag = basic.lt(offset, 0) min_wh = basic.minimum(width, height) start = offset * pos_offset_flag + offset_abs * width * neg_offset_flag num_of_step = basic.minimum(min_wh, width * pos_offset_flag + height * neg_offset_flag - offset_abs) step = a.shape[1] + 1 end = start + step * num_of_step # input of slice should be integer start = basic.cast(start, 'int32') step = basic.cast(step, 'int32') end = basic.cast(end, 'int32') wr_val = grad.flatten()[start:end:step].sum() wr_offset = theano.gradient.grad_undefined( self, 2, offset, "offset is not defined for non-integer offset so" " fill_diagonal_offset(a,val,offset+eps) is undefined") return [wr_a, wr_val, wr_offset] fill_diagonal_offset_ = FillDiagonalOffset() def fill_diagonal_offset(a, val, offset): """ Returns a copy of an array with all elements of the main diagonal set to a specified scalar value. Parameters ---------- a Rectangular array of two dimensions. val Scalar value to fill the diagonal whose type must be compatible with that of array 'a' (i.e. 'val' cannot be viewed as an upcast of 'a'). offset Scalar value Offset of the diagonal from the main diagonal. Can be positive or negative integer. Returns ------- array An array identical to 'a' except that its offset diagonal is filled with scalar 'val'. The output is unwrapped. """ return fill_diagonal_offset_(a, val, offset) def to_one_hot(y, nb_class, dtype=None): """ Return a matrix where each row correspond to the one hot encoding of each element in y. Parameters ---------- y A vector of integer value between 0 and nb_class - 1. nb_class : int The number of class in y. dtype : data-type The dtype of the returned matrix. Default floatX. Returns ------- object A matrix of shape (y.shape[0], nb_class), where each row ``i`` is the one hot encoding of the corresponding ``y[i]`` value. """ ret = theano.tensor.zeros((y.shape[0], nb_class), dtype=dtype) ret = theano.tensor.set_subtensor(ret[theano.tensor.arange(y.shape[0]), y], 1) return ret class Unique(theano.Op): """ Wraps numpy.unique. This op is not implemented on the GPU. Examples -------- >>> import numpy as np >>> import theano >>> x = theano.tensor.vector() >>> f = theano.function([x], Unique(True, True, False)(x)) >>> f([1, 2., 3, 4, 3, 2, 1.]) [array([ 1., 2., 3., 4.]), array([0, 1, 2, 3]), array([0, 1, 2, 3, 2, 1, 0])] >>> y = theano.tensor.matrix() >>> g = theano.function([y], Unique(True, True, False)(y)) >>> g([[1, 1, 1.0], (2, 3, 3.0)]) [array([ 1., 2., 3.]), array([0, 3, 4]), array([0, 0, 0, 1, 2, 2])] """ __props__ = ("return_index", "return_inverse", "return_counts") def __init__(self, return_index=False, return_inverse=False, return_counts=False): self.return_index = return_index self.return_inverse = return_inverse self.return_counts = return_counts numpy_ver = [int(n) for n in np.__version__.split('.')[:2]] if self.return_counts and bool(numpy_ver < [1, 9]): raise RuntimeError( "Numpy version = " + np.__version__ + ". Option 'return_counts=True' works starting" " from version 1.9.0.") def make_node(self, x): x = basic.as_tensor_variable(x) outputs = [basic.TensorType(broadcastable=[False], dtype=x.dtype)()] typ = basic.TensorType(broadcastable=[False], dtype='int64') if self.return_index: outputs.append(typ()) if self.return_inverse: outputs.append(typ()) if self.return_counts: outputs.append(typ()) return theano.Apply(self, [x], outputs) def perform(self, node, inputs, output_storage): x = inputs[0] z = output_storage param = {} if self.return_index: param['return_index'] = True if self.return_inverse: param['return_inverse'] = True if self.return_counts: param['return_counts'] = True outs = np.unique(x, **param) if ((not self.return_inverse) and (not self.return_index) and (not self.return_counts)): z[0][0] = outs else: for i in range(len(outs)): z[i][0] = outs[i] def infer_shape(self, node, i0_shapes): ret = node.fgraph.shape_feature.default_infer_shape(node, i0_shapes) if self.return_inverse: shape = (basic.prod(i0_shapes[0]), ) if self.return_index: ret[2] = shape return ret ret[1] = shape return ret return ret class UnravelIndex(gof.Op): __props__ = ('ndim', 'order') def __init__(self, ndim, order='C'): assert order in ('C', 'F') if not isinstance(ndim, int) or ndim < 1: raise ValueError('ndim must be an integer greater than 0') self.ndim = int(ndim) self.order = order def make_node(self, indices, dims): indices = basic.as_tensor_variable(indices) dims = basic.as_tensor_variable(dims) if indices.dtype not in basic.int_dtypes: raise TypeError("'%s' object cannot be interpreted as an index" % str(indices.dtype)) if dims.dtype not in basic.int_dtypes: raise TypeError("'%s' object cannot be interpreted as an index" % str(dims.dtype)) if dims.ndim != 1: raise TypeError("dims must be a 1D array") return gof.Apply( self, [indices, dims], [basic.TensorType(dtype='int64', broadcastable=(False,) * indices.ndim)() for i in xrange(self.ndim)]) def infer_shape(self, node, input_shapes): return [input_shapes[0]] * len(node.outputs) def perform(self, node, inp, out): indices, dims = inp res = np.unravel_index(indices, dims) assert len(res) == len(out) for i in xrange(len(out)): ret = theano._asarray(res[i], node.outputs[0].dtype) if ret.base is not None: # NumPy will return a view when it can. # But we don't want that. ret = ret.copy() out[i][0] = ret def unravel_index(indices, dims, order='C', ndim=None): """ Converts a flat index or array of flat indices into a tuple of coordinate arrays. This method is similar to the NumPy version, except for the additional ``ndim`` parameter. This parameter is required if the length of ``dims`` cannot be determined automatically. Parameters ---------- indices : Theano or NumPy array An integer array whose elements are indices into the flattened version of an array of dimensions ``dims``. dims : tuple of ints The shape of the array to use for unraveling ``indices``. order : {'C', 'F'}, optional Determines whether the indices should be viewed as indexing in row-major (C-style) or column-major (Fortran-style) order. ndim : int, optional Specifies the number of dimensions, i.e., the length of ``dims``. This is required if the dimensions cannot be determined automatically from ``dims`` itself. Returns ------- unraveled_coords : tuple of ndarray Each array in the tuple has the same shape as the ``indices`` array. See Also -------- ravel_multi_index """ if ndim is None: try: ndim = basic.get_vector_length(dims) except ValueError: raise ValueError( "The length of the provided dimension list (%s) cannot " "be automatically determined, so Theano is not able " "to know what the number of dimensions of the unraveled " "index will be. You can provide the 'ndim' keyword " "argument to 'unravel_index' to avoid this problem." % str(dims)) res = UnravelIndex(ndim=ndim, order=order)(indices, dims) if ndim == 1: return (res,) else: return tuple(res) class RavelMultiIndex(gof.Op): __props__ = ('mode', 'order') def __init__(self, mode='raise', order='C'): assert mode in ('raise', 'wrap', 'clip') assert order in ('C', 'F') self.mode = mode self.order = order def make_node(self, *inp): multi_index = [basic.as_tensor_variable(i) for i in inp[:-1]] dims = basic.as_tensor_variable(inp[-1]) for i in multi_index: if i.dtype not in basic.int_dtypes: raise TypeError("'%s' object cannot be interpreted as an index" % str(i.dtype)) if dims.dtype not in basic.int_dtypes: raise TypeError("'%s' object cannot be interpreted as an index" % str(dims.dtype)) if dims.ndim != 1: raise TypeError("dims must be a 1D array") return gof.Apply( self, multi_index + [dims], [basic.TensorType(dtype='int64', broadcastable=(False,) * multi_index[0].ndim)()]) def infer_shape(self, node, input_shapes): return [input_shapes[0]] def perform(self, node, inp, out): multi_index, dims = inp[:-1], inp[-1] res = np.ravel_multi_index(multi_index, dims, mode=self.mode, order=self.order) out[0][0] = theano._asarray(res, node.outputs[0].dtype) def ravel_multi_index(multi_index, dims, mode='raise', order='C'): """ Converts a tuple of index arrays into an array of flat indices, applying boundary modes to the multi-index. Parameters ---------- multi_index : tuple of Theano or NumPy arrays A tuple of integer arrays, one array for each dimension. dims : tuple of ints The shape of array into which the indices from ``multi_index`` apply. mode : {'raise', 'wrap', 'clip'}, optional Specifies how out-of-bounds indices are handled. Can specify either one mode or a tuple of modes, one mode per index. * 'raise' -- raise an error (default) * 'wrap' -- wrap around * 'clip' -- clip to the range In 'clip' mode, a negative index which would normally wrap will clip to 0 instead. order : {'C', 'F'}, optional Determines whether the multi-index should be viewed as indexing in row-major (C-style) or column-major (Fortran-style) order. Returns ------- raveled_indices : Theano array An array of indices into the flattened version of an array of dimensions ``dims``. See Also -------- unravel_index """ if not isinstance(multi_index, (tuple, list)): raise TypeError('multi_index must be a tuple or a list.') args = tuple(multi_index) + (dims,) return RavelMultiIndex(mode=mode, order=order)(*args) Theano-1.0.0/theano/tensor/elemwise_cgen.py00006000163562€ÿÿÿþ0000004762413203335532022254 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division from six.moves import xrange import theano def make_declare(loop_orders, dtypes, sub): """ Produce code to declare all necessary variables. """ decl = "" for i, (loop_order, dtype) in enumerate(zip(loop_orders, dtypes)): var = sub['lv%i' % i] # input name corresponding to ith loop variable # we declare an iteration variable # and an integer for the number of dimensions decl += """ %(dtype)s* %(var)s_iter; """ % locals() for j, value in enumerate(loop_order): if value != 'x': # If the dimension is not broadcasted, we declare # the number of elements in that dimension, # the stride in that dimension, # and the jump from an iteration to the next decl += """ npy_intp %(var)s_n%(value)i; ssize_t %(var)s_stride%(value)i; int %(var)s_jump%(value)i_%(j)i; """ % locals() else: # if the dimension is broadcasted, we only need # the jump (arbitrary length and stride = 0) decl += """ int %(var)s_jump%(value)s_%(j)i; """ % locals() return decl def make_checks(loop_orders, dtypes, sub): init = "" for i, (loop_order, dtype) in enumerate(zip(loop_orders, dtypes)): var = "%%(lv%i)s" % i # List of dimensions of var that are not broadcasted nonx = [x for x in loop_order if x != 'x'] if nonx: # If there are dimensions that are not broadcasted # this is a check that the number of dimensions of the # tensor is as expected. min_nd = max(nonx) + 1 init += """ if (PyArray_NDIM(%(var)s) < %(min_nd)s) { PyErr_SetString(PyExc_ValueError, "Not enough dimensions on input."); %%(fail)s } """ % locals() # In loop j, adjust represents the difference of values of the # data pointer between the beginning and the end of the # execution of loop j+1 (the loop inside the current one). It # is equal to the stride in loop j+1 times the length of loop # j+1, or 0 for the inner-most loop. adjust = "0" # We go from the inner loop to the outer loop for j, index in reversed(list(enumerate(loop_order))): if index != 'x': # Initialize the variables associated to the jth loop # jump = stride - adjust jump = "(%s) - (%s)" % ("%(var)s_stride%(index)s" % locals(), adjust) init += """ %(var)s_n%(index)s = PyArray_DIMS(%(var)s)[%(index)s]; %(var)s_stride%(index)s = PyArray_STRIDES(%(var)s)[%(index)s] / sizeof(%(dtype)s); %(var)s_jump%(index)s_%(j)s = %(jump)s; """ % locals() adjust = "%(var)s_n%(index)s*%(var)s_stride%(index)s" % locals() else: jump = "-(%s)" % adjust init += """ %(var)s_jump%(index)s_%(j)s = %(jump)s; """ % locals() adjust = "0" check = "" # This loop builds multiple if conditions to verify that the # dimensions of the inputs match, and the first one that is true # raises an informative error message for matches in zip(*loop_orders): to_compare = [(j, x) for j, x in enumerate(matches) if x != "x"] # elements of to_compare are pairs ( input_variable_idx, input_variable_dim_idx ) if len(to_compare) < 2: continue j0, x0 = to_compare[0] for (j, x) in to_compare[1:]: check += """ if (%%(lv%(j0)s)s_n%(x0)s != %%(lv%(j)s)s_n%(x)s) { PyErr_Format(PyExc_ValueError, "Input dimension mis-match. (input[%%%%i].shape[%%%%i] = %%%%lld, input[%%%%i].shape[%%%%i] = %%%%lld)", %(j0)s, %(x0)s, (long long int) %%(lv%(j0)s)s_n%(x0)s, %(j)s, %(x)s, (long long int) %%(lv%(j)s)s_n%(x)s ); %%(fail)s } """ % locals() return init % sub + check % sub def make_alloc(loop_orders, dtype, sub, fortran='0'): """Generate C code to allocate outputs. Parameters ---------- fortran : str A string included in the generated code. If it evaluate to non-zero, an ndarray in fortran order will be created, otherwise it will be c order. """ type = dtype.upper() if type.startswith('THEANO_COMPLEX'): type = type.replace('THEANO_COMPLEX', 'NPY_COMPLEX') nd = len(loop_orders[0]) init_dims = "" # For each dimension, the tensors are either all broadcasted, in # which case the output will also be broadcastable (dimension = # 1), or one or more are not broadcasted, in which case the number # of elements of the output in that dimension will be equal to the # number of elements of any of them. for i, candidates in enumerate(zip(*loop_orders)): for j, candidate in enumerate(candidates): if candidate != 'x': var = sub['lv%i' % j] init_dims += "dims[%(i)s] = %(var)s_n%(candidate)s;\n" % locals() break else: init_dims += "dims[%(i)s] = 1;\n" % locals() # TODO: it would be interesting to allocate the output in such a # way that its contiguous dimensions match one of the input's # contiguous dimensions, or the dimension with the smallest # stride. Right now, it is allocated to be C_CONTIGUOUS. return """ { npy_intp dims[%(nd)s]; //npy_intp* dims = (npy_intp*)malloc(%(nd)s * sizeof(npy_intp)); %(init_dims)s if (!%(olv)s) { %(olv)s = (PyArrayObject*)PyArray_EMPTY(%(nd)s, dims, %(type)s, %(fortran)s); } else { PyArray_Dims new_dims; new_dims.len = %(nd)s; new_dims.ptr = dims; PyObject* success = PyArray_Resize(%(olv)s, &new_dims, 0, NPY_CORDER); if (!success) { // If we can't resize the ndarray we have we can allocate a new one. PyErr_Clear(); Py_XDECREF(%(olv)s); %(olv)s = (PyArrayObject*)PyArray_EMPTY(%(nd)s, dims, %(type)s, 0); } else { Py_DECREF(success); } } if (!%(olv)s) { %(fail)s } } """ % dict(locals(), **sub) def make_loop(loop_orders, dtypes, loop_tasks, sub, openmp=None): """ Make a nested loop over several arrays and associate specific code to each level of nesting. Parameters ---------- loop_orders : list of N tuples of length M Each value of each tuple can be either the index of a dimension to loop over or the letter 'x' which means there is no looping to be done over that variable at that point (in other words we broadcast over that dimension). If an entry is an integer, it will become an alias of the entry of that rank. loop_tasks : list of M+1 pieces of code The ith loop_task is a pair of strings, the first string is code to be executed before the ith loop starts, the second one contains code to be executed just before going to the next element of the ith dimension. The last element if loop_tasks is a single string, containing code to be executed at the very end. sub : dictionary Maps 'lv#' to a suitable variable name. The 'lvi' variable corresponds to the ith element of loop_orders. """ def loop_over(preloop, code, indices, i): iterv = 'ITER_%i' % i update = "" suitable_n = "1" for j, index in enumerate(indices): var = sub['lv%i' % j] dtype = dtypes[j] update += "%(dtype)s &%(var)s_i = * ( %(var)s_iter + %(iterv)s * %(var)s_jump%(index)s_%(i)s );\n" % locals() if index != 'x': suitable_n = "%(var)s_n%(index)s" % locals() if openmp: openmp_elemwise_minsize = theano.config.openmp_elemwise_minsize forloop = """#pragma omp parallel for if( %(suitable_n)s >=%(openmp_elemwise_minsize)s)\n""" % locals() else: forloop = "" forloop += """for (int %(iterv)s = 0; %(iterv)s<%(suitable_n)s; %(iterv)s++)""" % locals() return""" %(preloop)s %(forloop)s { %(update)s %(code)s } """ % locals() preloops = {} for i, (loop_order, dtype) in enumerate(zip(loop_orders, dtypes)): for j, index in enumerate(loop_order): if index != 'x': preloops.setdefault(j, "") preloops[j] += ("%%(lv%(i)s)s_iter = (%(dtype)s*)(PyArray_DATA(%%(lv%(i)s)s));\n" % locals()) % sub break else: # all broadcastable preloops.setdefault(0, "") preloops[0] += ("%%(lv%(i)s)s_iter = (%(dtype)s*)(PyArray_DATA(%%(lv%(i)s)s));\n" % locals()) % sub s = "" for i, (pre_task, task), indices in reversed(list(zip(xrange(len(loop_tasks) - 1), loop_tasks, list(zip(*loop_orders))))): s = loop_over(preloops.get(i, "") + pre_task, s + task, indices, i) s += loop_tasks[-1] return "{%s}" % s def make_reordered_loop(init_loop_orders, olv_index, dtypes, inner_task, sub, openmp=None): """A bit like make_loop, but when only the inner-most loop executes code. All the loops will be reordered so that the loops over the output tensor are executed with memory access as contiguous as possible. For instance, if the output tensor is c_contiguous, the inner-most loop will be on its rows; if it's f_contiguous, it will be on its columns. The output tensor's index among the loop variables is indicated by olv_index. """ # Number of variables nvars = len(init_loop_orders) # Number of loops (dimensionality of the variables) nnested = len(init_loop_orders[0]) # This is the var from which we'll get the loop order ovar = sub['lv%i' % olv_index] # The loops are ordered by (decreasing) absolute values of ovar's strides. # The first element of each pair is the absolute value of the stride # The second element correspond to the index in the initial loop order order_loops = """ std::vector< std::pair > %(ovar)s_loops(%(nnested)i); std::vector< std::pair >::iterator %(ovar)s_loops_it = %(ovar)s_loops.begin(); """ % locals() # Fill the loop vector with the appropriate pairs for i, index in enumerate(init_loop_orders[olv_index]): if index != 'x': order_loops += """ %(ovar)s_loops_it->first = abs(PyArray_STRIDES(%(ovar)s)[%(index)i]); """ % locals() else: # Stride is 0 when dimension is broadcastable order_loops += """ %(ovar)s_loops_it->first = 0; """ % locals() order_loops += """ %(ovar)s_loops_it->second = %(i)i; ++%(ovar)s_loops_it; """ % locals() # We sort in decreasing order so that the outermost loop (loop 0) # has the largest stride, and the innermost loop (nnested - 1) has # the smallest stride. order_loops += """ // rbegin and rend are reversed iterators, so this sorts in decreasing order std::sort(%(ovar)s_loops.rbegin(), %(ovar)s_loops.rend()); """ % locals() # Get the (sorted) total number of iterations of each loop # Get totals in the initial order # For each dimension, the tensors are either all broadcasted, in # which case there is only one iteration of the loop, or one or # more are not broadcasted, in which case the number of elements # of any of them will be equal to the number of iterations we have # to do. totals = [] for i, candidates in enumerate(zip(*init_loop_orders)): for j, candidate in enumerate(candidates): if candidate != 'x': var = sub['lv%i' % j] total = "%(var)s_n%(candidate)s" % locals() break else: total = '1' totals.append(total) declare_totals = """ int init_totals[%(nnested)s] = {%(totals)s}; """ % dict(nnested=nnested, totals=', '.join(totals)) # Sort totals to match the new order that was computed by sorting # the loop vector. One integer variable per loop is declared. declare_totals += """ %(ovar)s_loops_it = %(ovar)s_loops.begin(); """ % locals() for i in xrange(nnested): declare_totals += """ int TOTAL_%(i)i = init_totals[%(ovar)s_loops_it->second]; ++%(ovar)s_loops_it; """ % locals() # Get sorted strides # Get strides in the initial order def get_loop_strides(loop_order, i): """ Returns a list containing a C expression representing the stride for each dimension of the ith variable, in the specified loop_order. """ var = sub["lv%i" % i] r = [] for index in loop_order: # Note: the stride variable is not declared for broadcasted variables if index != 'x': r.append("%(var)s_stride%(index)s" % locals()) else: r.append('0') return r # We declare the initial strides as a 2D array, nvars x nnested declare_strides = """ int init_strides[%(nvars)i][%(nnested)i] = { %(strides)s };""" % dict(nvars=nvars, nnested=nnested, strides=', \n'.join(', '.join(get_loop_strides(lo, i)) for i, lo in enumerate(init_loop_orders) if len(lo) > 0)) # Declare (sorted) stride and for each variable # we iterate from innermost loop to outermost loop declare_strides += """ std::vector< std::pair >::reverse_iterator %(ovar)s_loops_rit; """ % locals() for i in xrange(nvars): var = sub["lv%i" % i] declare_strides += """ %(ovar)s_loops_rit = %(ovar)s_loops.rbegin();""" % locals() for j in reversed(xrange(nnested)): declare_strides += """ int %(var)s_stride_l%(j)i = init_strides[%(i)i][%(ovar)s_loops_rit->second]; ++%(ovar)s_loops_rit; """ % locals() declare_iter = "" for i, dtype in enumerate(dtypes): var = sub["lv%i" % i] declare_iter += "%(var)s_iter = (%(dtype)s*)(PyArray_DATA(%(var)s));\n" % locals() pointer_update = '' for j, dtype in enumerate(dtypes): var = sub["lv%i" % j] pointer_update += "%(dtype)s &%(var)s_i = * ( %(var)s_iter" % locals() tot_jump = '' for i in reversed(range(nnested)): iterv = 'ITER_%i' % i pointer_update += "+%(var)s_stride_l%(i)i*%(iterv)s" % locals() pointer_update += ");\n" loop = inner_task for i in reversed(range(nnested)): iterv = 'ITER_%i' % i total = 'TOTAL_%i' % i update = '' forloop = '' # The pointers are defined only in the most inner loop if i == nnested - 1: update = pointer_update if i == 0: if openmp: openmp_elemwise_minsize = theano.config.openmp_elemwise_minsize forloop += """#pragma omp parallel for if( %(total)s >=%(openmp_elemwise_minsize)s)\n""" % locals() forloop += "for(int %(iterv)s = 0; %(iterv)s<%(total)s; %(iterv)s++)" % locals() loop = """ %(forloop)s { // begin loop %(i)i %(update)s %(loop)s } // end loop %(i)i """ % locals() return '\n'.join(['{', order_loops, declare_totals, declare_strides, declare_iter, loop, '}\n']) # print make_declare(((0, 1, 2, 3), ('x', 1, 0, 3), ('x', 'x', 'x', 0)), # ('double', 'int', 'float'), # dict(lv0='x', lv1='y', lv2='z', fail="FAIL;")) # print make_checks(((0, 1, 2, 3), ('x', 1, 0, 3), ('x', 'x', 'x', 0)), # ('double', 'int', 'float'), # dict(lv0='x', lv1='y', lv2='z', fail="FAIL;")) # print make_alloc(((0, 1, 2, 3), ('x', 1, 0, 3), ('x', 'x', 'x', 0)), # 'double', # dict(olv='out', lv0='x', lv1='y', lv2='z', fail="FAIL;")) # print make_loop(((0, 1, 2, 3), ('x', 1, 0, 3), ('x', 'x', 'x', 0)), # ('double', 'int', 'float'), # (("C00;", "C%01;"), ("C10;", "C11;"), ("C20;", "C21;"), ("C30;", "C31;"),"C4;"), # dict(lv0='x', lv1='y', lv2='z', fail="FAIL;")) # print make_loop(((0, 1, 2, 3), (3, 'x', 0, 'x'), (0, 'x', 'x', 'x')), # ('double', 'int', 'float'), # (("C00;", "C01;"), ("C10;", "C11;"), ("C20;", "C21;"), ("C30;", "C31;"),"C4;"), # dict(lv0='x', lv1='y', lv2='z', fail="FAIL;")) ################## # DimShuffle # ################## ################# # Broadcast # ################# ################ # CAReduce # ################ def make_loop_careduce(loop_orders, dtypes, loop_tasks, sub): """ Make a nested loop over several arrays and associate specific code to each level of nesting. Parameters ---------- loop_orders : list of N tuples of length M Each value of each tuple can be either the index of a dimension to loop over or the letter 'x' which means there is no looping to be done over that variable at that point (in other words we broadcast over that dimension). If an entry is an integer, it will become an alias of the entry of that rank. loop_tasks : list of M+1 pieces of code The ith loop_task is a pair of strings, the first string is code to be executed before the ith loop starts, the second one contains code to be executed just before going to the next element of the ith dimension. The last element if loop_tasks is a single string, containing code to be executed at the very end. sub: dictionary Maps 'lv#' to a suitable variable name. The 'lvi' variable corresponds to the ith element of loop_orders. """ def loop_over(preloop, code, indices, i): iterv = 'ITER_%i' % i update = "" suitable_n = "1" for j, index in enumerate(indices): var = sub['lv%i' % j] update += "%(var)s_iter += %(var)s_jump%(index)s_%(i)s;\n" % locals() if index != 'x': suitable_n = "%(var)s_n%(index)s" % locals() return """ %(preloop)s for (int %(iterv)s = %(suitable_n)s; %(iterv)s; %(iterv)s--) { %(code)s %(update)s } """ % locals() preloops = {} for i, (loop_order, dtype) in enumerate(zip(loop_orders, dtypes)): for j, index in enumerate(loop_order): if index != 'x': preloops.setdefault(j, "") preloops[j] += ("%%(lv%(i)s)s_iter = (%(dtype)s*)(PyArray_DATA(%%(lv%(i)s)s));\n" % locals()) % sub break else: # all broadcastable preloops.setdefault(0, "") preloops[0] += ("%%(lv%(i)s)s_iter = (%(dtype)s*)(PyArray_DATA(%%(lv%(i)s)s));\n" % locals()) % sub if len(loop_tasks) == 1: s = preloops.get(0, "") else: s = "" for i, (pre_task, task), indices in reversed(list(zip(xrange(len(loop_tasks) - 1), loop_tasks, list(zip(*loop_orders))))): s = loop_over(preloops.get(i, "") + pre_task, s + task, indices, i) s += loop_tasks[-1] return "{%s}" % s Theano-1.0.0/theano/tensor/shared_randomstreams.py00006000163562€ÿÿÿþ0000001151313203335532023637 0ustar boccoset00000000000000""" Define RandomStreams, providing random number variables for Theano graphs. """ from __future__ import absolute_import, print_function, division import copy import numpy as np from theano.compile.sharedvalue import (SharedVariable, shared_constructor, shared) from theano.tensor import raw_random __docformat__ = "restructuredtext en" class RandomStateSharedVariable(SharedVariable): pass @shared_constructor def randomstate_constructor(value, name=None, strict=False, allow_downcast=None, borrow=False): """ SharedVariable Constructor for RandomState. """ if not isinstance(value, np.random.RandomState): raise TypeError if not borrow: value = copy.deepcopy(value) return RandomStateSharedVariable( type=raw_random.random_state_type, value=value, name=name, strict=strict, allow_downcast=allow_downcast) class RandomStreams(raw_random.RandomStreamsBase): """ Module component with similar interface to numpy.random (numpy.random.RandomState) Parameters ---------- seed: None or int A default seed to initialize the RandomState instances after build. See `RandomStreamsInstance.__init__` for more details. """ def updates(self): return list(self.state_updates) def __init__(self, seed=None): super(RandomStreams, self).__init__() # A list of pairs of the form (input_r, output_r). This will be # over-ridden by the module instance to contain stream generators. self.state_updates = [] # Instance variable should take None or integer value. Used to seed the # random number generator that provides seeds for member streams. self.default_instance_seed = seed # numpy.RandomState instance that gen() uses to seed new streams. self.gen_seedgen = np.random.RandomState(seed) def seed(self, seed=None): """ Re-initialize each random stream. Parameters ---------- seed : None or integer in range 0 to 2**30 Each random stream will be assigned a unique state that depends deterministically on this value. Returns ------- None """ if seed is None: seed = self.default_instance_seed seedgen = np.random.RandomState(seed) for old_r, new_r in self.state_updates: old_r_seed = seedgen.randint(2 ** 30) old_r.set_value(np.random.RandomState(int(old_r_seed)), borrow=True) def __getitem__(self, item): """ Retrieve the numpy RandomState instance associated with a particular stream. Parameters ---------- item A variable of type RandomStateType, associated with this RandomStream. Returns ------- numpy RandomState (or None, before initialize) Notes ----- This is kept for compatibility with `tensor.randomstreams.RandomStreams`. The simpler syntax ``item.rng.get_value()`` is also valid. """ return item.get_value(borrow=True) def __setitem__(self, item, val): """ Set the numpy RandomState instance associated with a particular stream. Parameters ---------- item A variable of type RandomStateType, associated with this RandomStream. val : numpy RandomState The new value. Returns ------- None Notes ----- This is kept for compatibility with `tensor.randomstreams.RandomStreams`. The simpler syntax ``item.rng.set_value(val)`` is also valid. """ item.set_value(val, borrow=True) def gen(self, op, *args, **kwargs): """ Create a new random stream in this container. Parameters ---------- op A RandomFunction instance to args Interpreted by `op`. kwargs Interpreted by `op`. Returns ------- Tensor Variable The symbolic random draw part of op()'s return value. This function stores the updated RandomStateType Variable for use at `build` time. """ seed = int(self.gen_seedgen.randint(2 ** 30)) random_state_variable = shared(np.random.RandomState(seed)) # Add a reference to distinguish from other shared variables random_state_variable.tag.is_rng = True new_r, out = op(random_state_variable, *args, **kwargs) out.rng = random_state_variable out.update = (random_state_variable, new_r) self.state_updates.append(out.update) random_state_variable.default_update = new_r return out Theano-1.0.0/theano/tensor/nnet/00007500163562€ÿÿÿþ0000000000013203336027020026 5ustar boccoset00000000000000Theano-1.0.0/theano/tensor/nnet/tests/00007500163562€ÿÿÿþ0000000000013203336027021170 5ustar boccoset00000000000000Theano-1.0.0/theano/tensor/nnet/tests/test_bn.py00006000163562€ÿÿÿþ0000005672613203335532023215 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import theano import theano.tensor as T from theano.tests import unittest_tools as utt import numpy as np from collections import OrderedDict from theano.tensor.nnet import bn def test_BNComposite(): try: orig = theano.config.compute_test_value theano.config.compute_test_value = 'raise' def bn_ref(x, G, B, M, V): n = (x - M) / V return n * G + B np.random.seed(1234) X = 1 + np.random.random([10, 20]).astype('float32') B = 1 + np.random.random([20]).astype('float32') G = 1 + np.random.random([20]).astype('float32') M = 1 + np.random.random([20]).astype('float32') V = 1 + np.random.random([20]).astype('float32') x = theano.tensor.matrix('x') b = theano.tensor.vector('b') g = theano.tensor.vector('g') m = theano.tensor.vector('m') v = theano.tensor.vector('v') x.tag.test_value = np.random.rand(2, 2).astype(theano.config.floatX) b.tag.test_value = np.random.rand(2).astype(theano.config.floatX) g.tag.test_value = np.random.rand(2).astype(theano.config.floatX) m.tag.test_value = np.random.rand(2).astype(theano.config.floatX) v.tag.test_value = np.random.rand(2).astype(theano.config.floatX) bn_ref_op = bn_ref(x, g, b, m, v) f_ref = theano.function([x, b, g, m, v], [bn_ref_op]) res_ref = f_ref(X, G, B, M, V) for mode in ['low_mem', 'high_mem']: bn_op = bn.batch_normalization(x, g, b, m, v, mode=mode) f = theano.function([x, b, g, m, v], [bn_op]) res = f(X, G, B, M, V) utt.assert_allclose(res_ref, res) finally: theano.config.compute_test_value = orig def test_batch_normalization(): def bn_ref(x, G, B, M, V): n = (x - M) / V return n * G + B np.random.seed(1234) X = 1 + np.random.random([10, 20]).astype('float32') B = 1 + np.random.random([20]).astype('float32') G = 1 + np.random.random([20]).astype('float32') M = 1 + np.random.random([20]).astype('float32') V = 1 + np.random.random([20]).astype('float32') x = theano.tensor.matrix('x') b = theano.tensor.vector('b') g = theano.tensor.vector('g') m = theano.tensor.vector('m') v = theano.tensor.vector('v') bn_ref_op = bn_ref(x, g, b, m, v) f_ref = theano.function([x, b, g, m, v], [bn_ref_op]) res_ref = f_ref(X, G, B, M, V) for mode in ['low_mem', 'high_mem']: bn_op = bn.batch_normalization(x, g, b, m, v, mode=mode) f = theano.function([x, b, g, m, v], [bn_op]) res = f(X, G, B, M, V) utt.assert_allclose(res_ref, res) def bn_f(inputs, gamma, beta, mean, std): return bn.batch_normalization(inputs, gamma, beta, mean, std, mode=mode) utt.verify_grad(bn_f, [X, G, B, M, V]) bn_ref_op = bn_ref(x, g, b, x.mean(axis=0, keepdims=True), x.std(axis=0, keepdims=True)) f_ref = theano.function([x, b, g], [bn_ref_op]) res_ref = f_ref(X, G, B) for mode in ['low_mem', 'high_mem']: bn_op = bn.batch_normalization(x, g, b, x.mean(axis=0, keepdims=True), x.std(axis=0, keepdims=True), mode=mode) f = theano.function([x, b, g], [bn_op]) res = f(X, G, B) utt.assert_allclose(res_ref, res) def bn_f(inputs, gamma, beta, mean, std): return bn.batch_normalization(inputs, gamma, beta, mean, std, mode=mode) utt.verify_grad(bn_f, [X, G, B, X.mean(axis=0)[np.newaxis], X.std(axis=0)[np.newaxis]]) def test_bn_feature_maps(): def bn_ref(x, G, B, M, V): n = (x - M) / V return n * G + B np.random.seed(1234) X = 1 + np.random.random([2, 3, 4, 4]).astype('float32') B = 1 + np.random.random([3]).astype('float32') G = 1 + np.random.random([3]).astype('float32') M = 1 + np.random.random([3]).astype('float32') V = 1 + np.random.random([3]).astype('float32') x = theano.tensor.tensor4('x') b = theano.tensor.vector('b') g = theano.tensor.vector('g') m = theano.tensor.vector('m') v = theano.tensor.vector('v') bn_ref_op = bn_ref(x, g.dimshuffle('x', 0, 'x', 'x'), b.dimshuffle('x', 0, 'x', 'x'), m.dimshuffle('x', 0, 'x', 'x'), v.dimshuffle('x', 0, 'x', 'x')) f_ref = theano.function([x, b, g, m, v], [bn_ref_op]) res_ref = f_ref(X, G, B, M, V) for mode in ['low_mem', 'high_mem']: bn_op = bn.batch_normalization(x, g.dimshuffle('x', 0, 'x', 'x'), b.dimshuffle('x', 0, 'x', 'x'), m.dimshuffle('x', 0, 'x', 'x'), v.dimshuffle('x', 0, 'x', 'x'), mode=mode) f = theano.function([x, b, g, m, v], [bn_op]) res = f(X, G, B, M, V) utt.assert_allclose(res_ref, res) def conv_bn(inputs, gamma, beta, mean, std): return bn.batch_normalization(inputs, gamma.dimshuffle('x', 0, 'x', 'x'), beta.dimshuffle('x', 0, 'x', 'x'), mean.dimshuffle('x', 0, 'x', 'x'), std.dimshuffle('x', 0, 'x', 'x'), mode=mode) utt.verify_grad(conv_bn, [X, G, B, M, V]) def test_batch_normalization_train(): utt.seed_rng() for axes in ('per-activation', 'spatial', (1, 2, 3, 4)): for vartype in (T.tensor5, T.tensor3, T.vector): x, scale, bias, running_mean, running_var = (vartype(n) for n in ('x', 'scale', 'bias', 'running_mean', 'running_var')) ndim = x.ndim eps = 5e-3 # some non-standard value to test if it's used running_average_factor = 0.3 # remove non-existing axes if isinstance(axes, tuple): axes = tuple(i for i in axes if i < ndim) if len(axes) == 0: continue # forward pass out, x_mean, x_invstd, out_running_mean, out_running_var = \ bn.batch_normalization_train( x, scale, bias, axes, eps, running_average_factor, running_mean, running_var) # reference forward pass if axes == 'per-activation': axes2 = (0,) elif axes == 'spatial': axes2 = (0,) + tuple(range(2, ndim)) else: axes2 = axes x_mean2 = x.mean(axis=axes2, keepdims=True) x_var2 = x.var(axis=axes2, keepdims=True) x_invstd2 = T.inv(T.sqrt(x_var2 + eps)) scale2 = T.addbroadcast(scale, *axes2) bias2 = T.addbroadcast(bias, *axes2) out2 = (x - x_mean2) * (scale2 * x_invstd2) + bias2 m = T.cast(T.prod(x.shape) / T.prod(scale.shape), theano.config.floatX) out_running_mean2 = running_mean * (1 - running_average_factor) + \ x_mean2 * running_average_factor out_running_var2 = running_var * (1 - running_average_factor) + \ (m / (m - 1)) * x_var2 * running_average_factor # backward pass dy = vartype('dy') grads = T.grad(None, wrt=[x, scale, bias], known_grads={out: dy}) # reference backward pass grads2 = T.grad(None, wrt=[x, scale, bias], known_grads={out2: dy}) # second-order backward pass dx = vartype('dinputs') dscale = vartype('dscale') dbias = vartype('dbias') grad_grads = T.grad(None, wrt=[x, dy, scale], known_grads=OrderedDict( {grads[0]: dx, grads[1]: dscale, grads[2]: dbias}), consider_constant=[x, dy, scale, bias, x_mean, x_invstd, running_mean, running_var], return_disconnected='zero') # reference second-order backward pass grad_grads2 = T.grad(None, wrt=[x, dy, scale], known_grads=OrderedDict( {grads2[0]: dx, grads2[1]: dscale, grads2[2]: dbias}), consider_constant=[x, dy, scale, bias, x_mean2, x_var2, running_mean, running_var], return_disconnected='zero') # compile f = theano.function([x, scale, bias, running_mean, running_var, dy, dx, dscale, dbias], [out, x_mean, x_invstd, out_running_mean, out_running_var, out2, x_mean2, x_invstd2, out_running_mean2, out_running_var2] + grads + grads2 + grad_grads + grad_grads2) # check if the abstract Ops have been replaced assert not any([isinstance(n.op, (bn.AbstractBatchNormTrain, bn.AbstractBatchNormInference, bn.AbstractBatchNormTrainGrad)) for n in f.maker.fgraph.toposort()]) # run for data_shape in ((5, 10, 30, 40, 10), (4, 3, 1, 1, 1), (2, 3, 5, 5, 5)): data_shape = data_shape[:ndim] param_shape = tuple(1 if d in axes2 else s for d, s in enumerate(data_shape)) X = 4 + 3 * np.random.randn(*data_shape).astype(theano.config.floatX) Dy = -1 + 2 * np.random.randn(*data_shape).astype(theano.config.floatX) Scale = np.random.randn(*param_shape).astype(theano.config.floatX) Bias = np.random.randn(*param_shape).astype(theano.config.floatX) Running_mean = np.random.randn(*param_shape).astype(theano.config.floatX) Running_var = np.random.randn(*param_shape).astype(theano.config.floatX) Dx = 4 + 3 * np.random.randn(*data_shape).astype(theano.config.floatX) Dscale = -1 + 2 * np.random.randn(*param_shape).astype(theano.config.floatX) Dbias = np.random.randn(*param_shape).astype(theano.config.floatX) outputs = f(X, Scale, Bias, Running_mean, Running_var, Dy, Dx, Dscale, Dbias) # compare outputs utt.assert_allclose(outputs[0], outputs[0 + 5]) # out utt.assert_allclose(outputs[1], outputs[1 + 5]) # mean utt.assert_allclose(outputs[2], outputs[2 + 5]) # invstd utt.assert_allclose(outputs[3], outputs[3 + 5]) # running_mean utt.assert_allclose(np.nan_to_num(outputs[4]), np.nan_to_num(outputs[4 + 5])) # running_var # compare gradients utt.assert_allclose(outputs[10], outputs[10 + 3], atol=1e-4) # dx utt.assert_allclose(outputs[11], outputs[11 + 3], rtol=2e-4, atol=1e-4) # dscale utt.assert_allclose(outputs[12], outputs[12 + 3]) # dbias # compare second-order gradients utt.assert_allclose(outputs[16], outputs[16 + 3], atol=1e-4) # ddx utt.assert_allclose(outputs[17], outputs[17 + 3]) # ddy utt.assert_allclose(outputs[18], outputs[18 + 3], rtol=3e-4, atol=1e-4) # ddscale def test_batch_normalization_train_grad_grad(): utt.seed_rng() for axes in ('per-activation', 'spatial', (1, 2, 3, 4)): for vartype in (T.tensor5, T.tensor4, T.tensor3, T.matrix, T.vector): # run these experiments with float64 for sufficient numerical stability x, dy, scale, x_mean, x_invstd = (vartype(n, dtype='float64') for n in ('x', 'dy', 'scale', 'x_mean', 'x_invstd')) ndim = x.ndim # reference forward pass if axes == 'per-activation': axes = (0,) elif axes == 'spatial': axes = (0,) + tuple(range(2, ndim)) else: # remove non-existing axes axes = tuple(i for i in axes if i < ndim) if len(axes) == 0: continue def bn_grad_wrt_inputs_f(x, dy, scale, x_mean, x_invstd): g_inputs, g_scale, g_bias = bn.AbstractBatchNormTrainGrad(axes)(x, dy, scale, x_mean, x_invstd) return g_inputs def bn_grad_wrt_scale_f(x, dy, scale, x_mean, x_invstd): g_inputs, g_scale, g_bias = bn.AbstractBatchNormTrainGrad(axes)(x, dy, scale, x_mean, x_invstd) return g_scale def bn_grad_wrt_bias_f(x, dy, scale, x_mean, x_invstd): g_inputs, g_scale, g_bias = bn.AbstractBatchNormTrainGrad(axes)(x, dy, scale, x_mean, x_invstd) return g_bias # run for data_shape in ((4, 3, 3, 3, 3), (4, 3, 1, 1, 1), (2, 3, 5, 3, 2)): data_shape = data_shape[:ndim] param_shape = tuple(1 if d in axes else s for d, s in enumerate(data_shape)) # force float64 for sufficient numerical stability x_val = 4 + 3 * np.random.randn(*data_shape).astype('float64') dy_val = -1 + 2 * np.random.randn(*data_shape).astype('float64') scale_val = np.random.randn(*param_shape).astype('float64') x_mean_val = np.random.randn(*param_shape).astype('float64') x_invstd_val = np.random.randn(*param_shape).astype('float64') utt.verify_grad(bn_grad_wrt_inputs_f, [x_val, dy_val, scale_val, x_mean_val, x_invstd_val], abs_tol=5e-4, rel_tol=5e-4) utt.verify_grad(bn_grad_wrt_scale_f, [x_val, dy_val, scale_val, x_mean_val, x_invstd_val]) utt.verify_grad(bn_grad_wrt_bias_f, [x_val, dy_val, scale_val, x_mean_val, x_invstd_val]) def test_batch_normalization_train_without_running_averages(): # compile and run batch_normalization_train without running averages utt.seed_rng() x, scale, bias, dy = T.tensor4('x'), T.tensor4('scale'), T.tensor4('bias'), T.tensor4('dy') data_shape = (5, 10, 30, 25) param_shape = (1, 10, 30, 25) # forward pass out, x_mean, x_invstd = bn.batch_normalization_train(x, scale, bias, 'per-activation') # backward pass grads = T.grad(None, wrt=[x, scale, bias], known_grads={out: dy}) # compile f = theano.function([x, scale, bias, dy], [out, x_mean, x_invstd] + grads) # check if the abstract Ops have been replaced assert not any([isinstance(n.op, (bn.AbstractBatchNormTrain, bn.AbstractBatchNormInference, bn.AbstractBatchNormTrainGrad)) for n in f.maker.fgraph.toposort()]) # run X = 4 + 3 * np.random.randn(*data_shape).astype(theano.config.floatX) Dy = -1 + 2 * np.random.randn(*data_shape).astype(theano.config.floatX) Scale = np.random.randn(*param_shape).astype(theano.config.floatX) Bias = np.random.randn(*param_shape).astype(theano.config.floatX) f(X, Scale, Bias, Dy) def test_batch_normalization_train_broadcast(): for axes in ('per-activation', 'spatial', (1, 2, 3, 4)): for vartype in (T.tensor5, T.tensor4, T.tensor3, T.matrix, T.vector): x = vartype('x') ndim = x.ndim eps = 5e-3 # some non-standard value to test if it's used running_average_factor = 0.3 # remove non-existing axes if isinstance(axes, tuple): axes = tuple(i for i in axes if i < ndim) if len(axes) == 0: continue # convert axes to explicit list if axes == 'per-activation': axes2 = (0,) elif axes == 'spatial': axes2 = (0,) + tuple(range(2, ndim)) else: axes2 = axes # compute axes for parameter tensors non_bc_axes = tuple(i for i in range(ndim) if i not in axes2) params_dimshuffle = ['x'] * ndim for i, axis in enumerate(non_bc_axes): params_dimshuffle[axis] = i # construct non-broadcasted parameter variables param_type = T.TensorType(x.dtype, (False,) * len(non_bc_axes)) scale, bias, running_mean, running_var = (param_type(n) for n in ('scale', 'bias', 'running_mean', 'running_var')) # broadcast parameter variables scale_bc = scale.dimshuffle(params_dimshuffle) bias_bc = bias.dimshuffle(params_dimshuffle) running_mean_bc = running_mean.dimshuffle(params_dimshuffle) running_var_bc = running_var.dimshuffle(params_dimshuffle) # batch_normalization_train with original, non-broadcasted variables train_non_bc = \ bn.batch_normalization_train( x, scale, bias, axes, eps, running_average_factor, running_mean, running_var) # batch_normalization_train with broadcasted variables train_bc = \ bn.batch_normalization_train( x, scale_bc, bias_bc, axes, eps, running_average_factor, running_mean_bc, running_var_bc) train_bc = tuple([train_bc[0]] + # out [r.dimshuffle(non_bc_axes) for r in train_bc[1:]]) # batch_normalization_test with original, non-broadcasted variables test_non_bc = \ bn.batch_normalization_test( x, scale, bias, running_mean, running_var, axes, eps) # batch_normalization_test with broadcasted variables test_bc = \ bn.batch_normalization_test( x, scale_bc, bias_bc, running_mean_bc, running_var_bc, axes, eps) # subtract the results of the non-broadcasted and broadcasted calls results_non_bc = train_non_bc + (test_non_bc,) results_bc = train_bc + (test_bc,) results = [abs(r - r_bc) for (r, r_bc) in zip(results_non_bc, results_bc)] # compile to compute all differences f = theano.function([x, scale, bias, running_mean, running_var], T.sum(sum(results))) # the paired ops are exactly the same, so the optimizer should have # collapsed the sum of differences to a constant zero nodes = f.maker.fgraph.toposort() if theano.config.mode != "FAST_COMPILE": assert len(nodes) == 1 assert isinstance(nodes[0].op, theano.compile.DeepCopyOp) inputs = [np.asarray(np.random.rand(*((4,) * n)), x.dtype) for n in [x.ndim, scale.ndim, bias.ndim, running_mean.ndim, running_var.ndim]] assert 0.0 == f(*inputs) def test_batch_normalization_test(): for axes in ('per-activation', 'spatial', (1, 2, 3, 4)): for vartype in (T.tensor5, T.tensor3, T.vector): x, scale, bias, mean, var = (vartype(n) for n in ('x', 'scale', 'bias', 'mean', 'var')) ndim = x.ndim eps = 5e-3 # some non-standard value to test if it's used # remove non-existing axes if isinstance(axes, tuple): axes = tuple(i for i in axes if i < ndim) if len(axes) == 0: continue # forward pass out = bn.batch_normalization_test(x, scale, bias, mean, var, axes, eps) # reference forward pass if axes == 'per-activation': axes2 = (0,) elif axes == 'spatial': axes2 = (0,) + tuple(range(2, ndim)) else: axes2 = axes scale2, bias2, mean2, var2 = (T.addbroadcast(t, *axes2) for t in (scale, bias, mean, var)) out2 = (x - mean2) * (scale2 / T.sqrt(var2 + eps)) + bias2 # backward pass dy = vartype('dy') grads = T.grad(None, wrt=[x, scale, bias, mean, var], known_grads={out: dy}) # reference backward pass grads2 = T.grad(None, wrt=[x, scale, bias, mean, var], known_grads={out2: dy}) # compile f = theano.function([x, scale, bias, mean, var, dy], [out, out2] + grads + grads2) # check if the abstract Ops have been replaced assert not any([isinstance(n.op, (bn.AbstractBatchNormTrain, bn.AbstractBatchNormInference, bn.AbstractBatchNormTrainGrad)) for n in f.maker.fgraph.toposort()]) # run for data_shape in ((10, 20, 30, 40, 10), (4, 3, 1, 1, 1), (1, 1, 5, 5, 5)): data_shape = data_shape[:ndim] param_shape = tuple(1 if d in axes2 else s for d, s in enumerate(data_shape)) X = 4 + 3 * np.random.randn(*data_shape).astype(theano.config.floatX) Dy = -1 + 2 * np.random.randn(*data_shape).astype(theano.config.floatX) Scale = np.random.randn(*param_shape).astype(theano.config.floatX) Bias = np.random.randn(*param_shape).astype(theano.config.floatX) Mean = np.random.randn(*param_shape).astype(theano.config.floatX) Var = np.random.rand(*param_shape).astype(theano.config.floatX) outputs = f(X, Scale, Bias, Mean, Var, Dy) # compare outputs utt.assert_allclose(outputs[0], outputs[1]) # out # compare gradients utt.assert_allclose(outputs[2], outputs[2 + 5], atol=4e-5) # dx utt.assert_allclose(outputs[3], outputs[3 + 5], atol=4e-5) # dscale utt.assert_allclose(outputs[4], outputs[4 + 5]) # dbias utt.assert_allclose(outputs[5], outputs[5 + 5]) # dmean utt.assert_allclose(outputs[6], outputs[6 + 5], rtol=2e-3, atol=4e-5) # dvar def test_batch_normalization_broadcastable(): # check if the broadcastable pattern is preserved by the optimizations x, dy, scale, bias, mean, var = (T.scalar(n).dimshuffle(['x'] * 5) for n in ('x', 'dy', 'scale', 'bias', 'mean', 'var')) # forward pass out_train, x_mean, x_invstd = bn.batch_normalization_train(x, scale, bias, 'spatial') out_test = bn.batch_normalization_test(x, scale, bias, mean, var, 'spatial') # backward pass grads_train = T.grad(None, wrt=[x, scale, bias], known_grads={out_train: dy}) grads_test = T.grad(None, wrt=[x, scale, bias], known_grads={out_test: dy}) # compile f = theano.function([x, scale, bias, mean, var, dy], [out_train, x_mean, x_invstd, out_test] + grads_train + grads_test) assert not any([isinstance(n.op, (bn.AbstractBatchNormTrain, bn.AbstractBatchNormInference, bn.AbstractBatchNormTrainGrad)) for n in f.maker.fgraph.toposort()]) Theano-1.0.0/theano/tensor/nnet/tests/test_corr3d.py00006000163562€ÿÿÿþ0000005066713203335532024010 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division from nose.plugins.skip import SkipTest from nose.plugins.attrib import attr from nose.tools import assert_equals from parameterized import parameterized import numpy as np from six import integer_types import theano import theano.tensor as T from theano.tests import unittest_tools as utt from theano.tensor.nnet import corr3d, conv from theano.tensor.nnet.tests.test_abstract_conv import Grouped_conv3d_noOptim class TestCorr3D(utt.InferShapeTester): if theano.config.mode == "FAST_COMPILE": mode = theano.compile.get_mode("FAST_RUN") else: mode = None dtype = theano.config.floatX def setUp(self): super(TestCorr3D, self).setUp() self.input = T.tensor5('input', dtype=self.dtype) self.input.name = 'default_V' self.filters = T.tensor5('filters', dtype=self.dtype) self.filters.name = 'default_filters' if not conv.imported_scipy_signal and theano.config.cxx == "": raise SkipTest("Corr3dMM tests need SciPy or a c++ compiler") # This tests can run even when theano.config.blas.ldflags is empty. def validate(self, image_shape, filter_shape, border_mode='valid', subsample=(1, 1, 1), input=None, filters=None, verify_grad=True, non_contiguous=False, filter_dilation=(1, 1, 1)): """ :param image_shape: The constant shape info passed to corr3dMM. :param filter_shape: The constant shape info passed to corr3dMM. """ if not theano.config.cxx: raise SkipTest("Need cxx for this test") N_image_shape = [T.get_scalar_constant_value(T.as_tensor_variable(x)) for x in image_shape] N_filter_shape = [T.get_scalar_constant_value(T.as_tensor_variable(x)) for x in filter_shape] if input is None: input = self.input if filters is None: filters = self.filters # THEANO IMPLEMENTATION # we create a symbolic function so that verify_grad can work def sym_Corr3dMM(input, filters): # define theano graph and function input.name = 'input' filters.name = 'filters' rval = corr3d.Corr3dMM(border_mode, subsample, filter_dilation)(input, filters) rval.name = 'corr_output' return rval output = sym_Corr3dMM(input, filters) output.name = 'Corr3dMM()(%s,%s)' % (input.name, filters.name) theano_corr = theano.function([input, filters], output, mode=self.mode) # initialize input and compute result image_data = np.random.random(N_image_shape).astype(self.dtype) filter_data = np.random.random(N_filter_shape).astype(self.dtype) image_data /= 10 filter_data /= 10 if non_contiguous: image_data = np.transpose(image_data, axes=(0, 1, 4, 3, 2)) image_data = image_data.copy() image_data = np.transpose(image_data, axes=(0, 1, 4, 3, 2)) filter_data = np.transpose(filter_data, axes=(0, 1, 4, 3, 2)) filter_data = filter_data.copy() filter_data = np.transpose(filter_data, axes=(0, 1, 4, 3, 2)) assert not image_data.flags['CONTIGUOUS'] assert not filter_data.flags['CONTIGUOUS'] theano_output = theano_corr(image_data, filter_data) # REFERENCE IMPLEMENTATION # Testing correlation, not convolution. Reverse filters. filter_data_corr = np.array(filter_data[:, :, ::-1, ::-1, ::-1], copy=True, order='C') orig_image_data = image_data img_shape3d = np.array(N_image_shape[-3:]) fil_shape3d = np.array(N_filter_shape[-3:]) dil_shape3d = np.array(filter_dilation) dil_fil_shape3d = (fil_shape3d - 1) * dil_shape3d + 1 subsample3d = np.array(subsample) if border_mode == 'full': padHWD = (dil_fil_shape3d - 1) elif border_mode == 'valid': padHWD = np.array([0, 0, 0]) elif border_mode == 'half': padHWD = np.floor(dil_fil_shape3d / 2).astype('int32') elif isinstance(border_mode, tuple): padHWD = np.array(border_mode) elif isinstance(border_mode, integer_types): padHWD = np.array([border_mode, border_mode, border_mode]) else: raise NotImplementedError('Unsupported border_mode {}'.format(border_mode)) out_shape3d = np.floor((img_shape3d + 2 * (padHWD) - dil_fil_shape3d) / subsample3d) + 1 # avoid numpy deprecation out_shape3d = out_shape3d.astype('int32') out_shape = (N_image_shape[0], N_filter_shape[0]) + tuple(out_shape3d) ref_output = np.zeros(out_shape) # loop over output feature maps ref_output.fill(0) image_data2 = np.zeros((N_image_shape[0], N_image_shape[1], N_image_shape[2] + 2 * padHWD[0], N_image_shape[3] + 2 * padHWD[1], N_image_shape[4] + 2 * padHWD[2])) image_data2[:, :, padHWD[0]:padHWD[0] + N_image_shape[2], padHWD[1]:padHWD[1] + N_image_shape[3], padHWD[2]:padHWD[2] + N_image_shape[4]] = image_data image_data = image_data2 N_image_shape = image_data.shape for bb in range(N_image_shape[0]): for nn in range(N_filter_shape[0]): for im0 in range(N_image_shape[1]): filter3d = filter_data_corr[nn, im0, :, :, :] image3d = image_data[bb, im0, :, :, :] for row in range(ref_output.shape[2]): irow = row * subsample[0] # image row for col in range(ref_output.shape[3]): icol = col * subsample[1] # image col for slc in range(ref_output.shape[4]): islc = slc * subsample[2] # image slice ref_output[bb, nn, row, col, slc] += (image3d[ irow:irow + dil_fil_shape3d[0]:filter_dilation[0], icol:icol + dil_fil_shape3d[1]:filter_dilation[1], islc:islc + dil_fil_shape3d[2]:filter_dilation[2] ] * filter3d[::-1, ::-1, ::-1] ).sum() utt.assert_allclose(theano_output, ref_output) # TEST GRADIENT if verify_grad: utt.verify_grad(sym_Corr3dMM, [orig_image_data, filter_data], mode=self.mode) @attr('slow') def test_basic(self): # Tests that basic correlations work for odd and even # dimensions of image and filter shapes, as well as rectangular # images and filters. border_modes = ['valid', 'full', 'half', (1, 1, 1), (2, 1, 1), (1, 2, 1), (1, 1, 2), (3, 3, 3), 1] img_shapes = [(2, 2, 3, 3, 3), (3, 2, 8, 8, 8), (3, 2, 7, 5, 5), (3, 2, 7, 5, 5), (1, 2, 8, 8, 8), (1, 2, 7, 5, 5)] fil_shapes = [(2, 2, 2, 2, 2), (1, 2, 5, 5, 5), (2, 2, 2, 3, 2), (2, 2, 3, 2, 2), (1, 2, 5, 5, 5), (1, 2, 2, 3, 3)] for border_mode in border_modes: for img, fil in zip(img_shapes, fil_shapes): self.validate(img, fil, border_mode, verify_grad=False) # Very slow on with 'full' or 'half' self.validate((1, 2, 53, 29, 11), (13, 2, 12, 1, 1), 'valid', verify_grad=False) def test_img_kernel_same_shape(self): self.validate((3, 2, 3, 3, 3), (1, 2, 3, 3, 3), 'full') self.validate((3, 2, 3, 3, 3), (1, 2, 3, 3, 3), 'valid') self.validate((3, 2, 3, 3, 3), (1, 2, 3, 3, 3), 'half') self.validate((3, 2, 3, 3, 3), (1, 2, 3, 3, 3), (1, 1, 1)) self.validate((3, 2, 3, 3, 3), (1, 2, 3, 3, 3), 1) @attr('slow') def test_subsample(self): # Tests correlation where subsampling != (1,1,1) self.validate((3, 2, 7, 5, 5), (2, 2, 2, 3, 3), 'valid', subsample=(2, 2, 2)) self.validate((3, 2, 7, 5, 5), (2, 2, 2, 3, 3), 'valid', subsample=(2, 1, 1)) self.validate((1, 1, 6, 6, 6), (1, 1, 3, 3, 3), 'valid', subsample=(3, 3, 3)) self.validate((3, 2, 7, 5, 5), (2, 2, 2, 3, 3), 'full', subsample=(2, 2, 2)) self.validate((3, 2, 7, 5, 5), (2, 2, 2, 3, 3), 'full', subsample=(2, 1, 1)) self.validate((1, 1, 6, 6, 6), (1, 1, 3, 3, 3), 'full', subsample=(3, 3, 3)) self.validate((3, 2, 7, 5, 5), (2, 2, 2, 3, 3), 'half', subsample=(2, 2, 2)) self.validate((3, 2, 7, 5, 5), (2, 2, 2, 3, 3), 'half', subsample=(2, 1, 1)) self.validate((1, 1, 6, 6, 6), (1, 1, 3, 3, 3), 'half', subsample=(3, 3, 3)) self.validate((3, 2, 7, 5, 5), (2, 2, 2, 3, 3), (1, 1, 1), subsample=(2, 2, 2)) self.validate((3, 2, 7, 5, 5), (2, 2, 2, 3, 3), (2, 1, 1), subsample=(2, 1, 1)) self.validate((1, 1, 6, 6, 6), (1, 1, 3, 3, 3), (1, 2, 2), subsample=(3, 3, 3)) self.validate((1, 1, 6, 6, 6), (1, 1, 3, 3, 3), 1, subsample=(3, 3, 3)) # Tests correlation where filter dilation != (1,1,1) @parameterized.expand([ ((3, 2, 7, 5, 5), (2, 2, 2, 3, 3), 'valid', (2, 2, 2)), ((3, 2, 14, 10, 10), (2, 2, 2, 3, 3), 'valid', (3, 1, 1)), ((1, 1, 14, 14, 14), (1, 1, 3, 3, 3), 'valid', (2, 3, 3)), ((3, 2, 7, 5, 5), (2, 2, 2, 3, 3), 'full', (2, 2, 2)), ((3, 2, 7, 5, 5), (2, 2, 2, 3, 3), 'full', (3, 1, 1)), ((1, 1, 6, 6, 6), (1, 1, 3, 3, 3), 'full', (2, 3, 3)), ((3, 2, 7, 5, 5), (2, 2, 2, 3, 3), 'half', (2, 2, 2)), ((3, 2, 7, 5, 5), (2, 2, 2, 3, 3), 'half', (3, 1, 1)), ((1, 1, 6, 6, 6), (1, 1, 3, 3, 3), 'half', (2, 3, 3)), ((3, 2, 7, 5, 5), (2, 2, 2, 3, 3), (1, 1, 1), (2, 2, 2)), ((3, 2, 7, 5, 5), (2, 2, 2, 3, 3), (2, 1, 1), (2, 1, 1)), ((1, 1, 6, 6, 6), (1, 1, 3, 3, 3), (1, 2, 1), (1, 2, 1)), ((1, 1, 6, 6, 6), (1, 1, 3, 3, 3), (1, 1, 2), (1, 1, 2))]) def test_filter_dilation(self, image_shape, filter_shape, border_mode, filter_dilation): self.validate(image_shape, filter_shape, border_mode, filter_dilation=filter_dilation) def test_filter_dilation_subsample(self): self.validate((1, 1, 6, 6, 6), (1, 1, 3, 3, 3), 1, subsample=(3, 3, 3), filter_dilation=(2, 2, 2)) @parameterized.expand([('valid',), ('full',), ('half',), ((1, 1, 1),), ((2, 1, 1),), ((1, 2, 1),), ((1, 1, 2),), ((3, 3, 3),), (1,)]) # @attr('slow') def test_shape_Constant_tensor(self, border_mode): # Tests correlation where the {image,filter}_shape is a Constant tensor as_t = T.as_tensor_variable self.validate((as_t(3), as_t(2), as_t(7), as_t(5), as_t(5)), (5, 2, 2, 3, 3), border_mode) self.validate(as_t([3, 2, 7, 5, 5]), (5, 2, 2, 3, 3), border_mode) self.validate(as_t((3, 2, 7, 5, 5)), (5, 2, 2, 3, 3), border_mode) self.validate((3, 2, 7, 5, 5), (as_t(5), as_t(2), as_t(2), as_t(3), as_t(3)), 'valid') self.validate((3, 2, 7, 5, 5), as_t([5, 2, 2, 3, 3]), border_mode) self.validate(as_t([3, 2, 7, 5, 5]), as_t([5, 2, 2, 3, 3]), border_mode) def test_invalid_filter_shape(self): # Tests scenario where filter_shape[1] != input_shape[1] self.assertRaises(ValueError, self.validate, (3, 2, 8, 8, 8), (4, 3, 5, 5, 8), 'valid') def test_full_mode(self): # Tests basic correlation in full mode and case where filter # is larger than the input image. self.validate((3, 1, 4, 4, 4), (2, 1, 5, 5, 5), 'full') def f(): self.validate((3, 2, 5, 5, 5), (4, 2, 8, 8, 8), 'valid') self.assertRaises(Exception, f) def test_wrong_input(self): # Make sure errors are raised when image and kernel are not 5D tensors self.assertRaises(Exception, self.validate, (3, 2, 8, 8, 8), (4, 2, 5, 5, 5), 'valid', input=T.dmatrix()) self.assertRaises(Exception, self.validate, (3, 2, 8, 8, 8), (4, 2, 5, 5, 5), 'valid', filters=T.dvector()) self.assertRaises(Exception, self.validate, (3, 2, 8, 8, 8), (4, 2, 5, 5, 5), 'valid', input=T.dtensor3()) self.assertRaises(Exception, self.validate, (3, 2, 8, 8, 8), (4, 2, 5, 5, 5), 'valid', input=T.dtensor4()) def test_dtype_upcast(self): # Checks dtype upcast for Corr3dMM methods. if not theano.config.cxx: raise SkipTest("Need cxx for this test") def rand(shape, dtype='float64'): r = np.asarray(np.random.rand(*shape), dtype=dtype) return r * 2 - 1 ops = [corr3d.Corr3dMM, corr3d.Corr3dMM_gradWeights, corr3d.Corr3dMM_gradInputs] a_shapes = [[4, 5, 6, 3, 3], [1, 5, 6, 3, 3], [1, 5, 6, 3, 3]] b_shapes = [[7, 5, 3, 2, 2], [1, 5, 3, 1, 1], [7, 1, 3, 1, 1]] dtypes = ['float32', 'float64'] for op, a_shape, b_shape in zip(ops, a_shapes, b_shapes): for a_dtype in dtypes: for b_dtype in dtypes: c_dtype = theano.scalar.upcast(a_dtype, b_dtype) a_tens = T.tensor5(dtype=a_dtype) b_tens = T.tensor5(dtype=b_dtype) a_tens_val = rand(a_shape, dtype=a_dtype) b_tens_val = rand(b_shape, dtype=b_dtype) c_tens = op()(a_tens, b_tens) f = theano.function([a_tens, b_tens], c_tens, mode=self.mode) assert_equals(f(a_tens_val, b_tens_val).dtype, c_dtype) @attr('slow') def test_infer_shape_forward(self): if theano.config.mode == "FAST_COMPILE": raise SkipTest("Corr3dMM don't work in FAST_COMPILE") if not theano.config.cxx: raise SkipTest("Need cxx for this test") def rand(*shape): r = np.asarray(np.random.rand(*shape), dtype='float64') return r * 2 - 1 corr3dMM = corr3d.Corr3dMM adtens = T.dtensor5() bdtens = T.dtensor5() aivec_vals = [[4, 5, 6, 3, 3], [6, 2, 8, 3, 3], [3, 6, 7, 5, 5], [3, 6, 7, 5, 5], [5, 2, 4, 3, 3]] bivec_vals = [[7, 5, 3, 2, 2], [4, 2, 5, 3, 3], [5, 6, 3, 2, 2], [5, 6, 2, 3, 3], [6, 2, 4, 3, 3]] modes = ['valid', 'full', 'half', (1, 1, 1), (2, 1, 1), (1, 2, 1), (1, 1, 2), 1] subsamples = [(1, 1, 1), (2, 1, 1), (1, 2, 1), (1, 1, 2)] for aivec_val, bivec_val in zip(aivec_vals, bivec_vals): adtens_val = rand(*aivec_val) bdtens_val = rand(*bivec_val) for mode in modes: for subsample in subsamples: # Corr3dMM cdtens = corr3dMM(border_mode=mode, subsample=subsample)(adtens, bdtens) self._compile_and_check([adtens, bdtens], [cdtens], [adtens_val, bdtens_val], corr3dMM, warn=False) @attr('slow') def test_infer_shape_gradW(self): if theano.config.mode == "FAST_COMPILE": raise SkipTest("Corr3dMM don't work in FAST_COMPILE") if not theano.config.cxx: raise SkipTest("Need cxx for this test") def rand(*shape): r = np.asarray(np.random.rand(*shape), dtype='float64') return r * 2 - 1 corr3dMM = corr3d.Corr3dMM gradW = corr3d.Corr3dMM_gradWeights adtens = T.dtensor5() bdtens = T.dtensor5() aivec_vals = [[1, 5, 6, 3, 3], [8, 2, 7, 3, 3], [1, 6, 9, 4, 4], [9, 6, 8, 5, 5], [9, 1, 6, 8, 8]] bivec_vals = [[7, 5, 3, 1, 1], [4, 2, 5, 3, 3], [12, 6, 3, 2, 2], [5, 6, 1, 3, 3], [11, 1, 3, 3, 3]] modes = ['valid', 'full', 'half', (1, 1, 1), (2, 1, 1), (1, 2, 1), (1, 1, 2), 1] subsamples = [(1, 1, 1), (2, 1, 1), (1, 2, 1), (1, 1, 2)] for aivec_val, bivec_val in zip(aivec_vals, bivec_vals): adtens_val = rand(*aivec_val) bdtens_val = rand(*bivec_val) for mode in modes: for subsample in subsamples: # Corr3dMM cdtens = corr3dMM(border_mode=mode, subsample=subsample)(adtens, bdtens) f = theano.function([adtens, bdtens], cdtens) cdtens_val = f(adtens_val, bdtens_val) # Corr3dMM_gradWeights shape = (theano.shared(bivec_val[2]), theano.shared(bivec_val[3]), theano.shared(bivec_val[4])) bdtens_g = gradW(border_mode=mode, subsample=subsample)(adtens, cdtens, shape=shape) self._compile_and_check([adtens, cdtens], [bdtens_g], [adtens_val, cdtens_val], gradW, warn=False) @attr('slow') def test_infer_shape_gradI(self): if theano.config.mode == "FAST_COMPILE": raise SkipTest("Corr3dMM don't work in FAST_COMPILE") if not theano.config.cxx: raise SkipTest("Need cxx for this test") def rand(*shape): r = np.asarray(np.random.rand(*shape), dtype='float64') return r * 2 - 1 corr3dMM = corr3d.Corr3dMM gradI = corr3d.Corr3dMM_gradInputs adtens = T.dtensor5() bdtens = T.dtensor5() aivec_vals = [[1, 5, 6, 3, 3], [8, 2, 7, 3, 3], [1, 6, 9, 4, 4], [9, 6, 8, 5, 5], [9, 1, 6, 8, 8]] bivec_vals = [[7, 5, 3, 1, 1], [4, 2, 5, 3, 3], [12, 6, 3, 2, 2], [5, 6, 1, 3, 3], [7, 1, 3, 4, 4]] modes = ['valid', 'full', 'half', (1, 1, 1), (2, 1, 1), (1, 2, 1), (1, 1, 2), 1] subsamples = [(1, 1, 1), (2, 1, 1), (1, 2, 1), (1, 1, 2)] for aivec_val, bivec_val in zip(aivec_vals, bivec_vals): adtens_val = rand(*aivec_val) bdtens_val = rand(*bivec_val) for mode in modes: for subsample in subsamples: # Corr3dMM cdtens = corr3dMM(border_mode=mode, subsample=subsample)(adtens, bdtens) f = theano.function([adtens, bdtens], cdtens) cdtens_val = f(adtens_val, bdtens_val) # Corr3dMM_gradInputs shape = (theano.shared(aivec_val[2]), theano.shared(aivec_val[3]), theano.shared(aivec_val[4])) adtens_g = gradI(border_mode=mode, subsample=subsample)(bdtens, cdtens, shape=shape) self._compile_and_check([bdtens, cdtens], [adtens_g], [bdtens_val, cdtens_val], gradI, warn=False) def test_non_contiguous(self): self.validate((2, 2, 3, 3, 3), (2, 2, 2, 2, 2), 'valid', non_contiguous=True) self.validate((3, 2, 8, 8, 8), (2, 2, 5, 5, 5), 'valid', non_contiguous=True) self.validate((3, 2, 7, 5, 5), (3, 2, 2, 3, 3), 'valid', non_contiguous=True) self.validate((3, 2, 7, 5, 5), (3, 2, 3, 2, 2), 'valid', non_contiguous=True) self.validate((3, 1, 8, 8, 8), (2, 1, 5, 5, 5), 'full', non_contiguous=True) self.validate((3, 1, 8, 8, 8), (2, 1, 5, 5, 5), 'half', non_contiguous=True) self.validate((3, 1, 8, 8, 8), (2, 1, 5, 5, 5), (1, 1, 1), non_contiguous=True) self.validate((3, 1, 7, 5, 5), (2, 1, 2, 3, 3), (1, 1, 2), non_contiguous=True) self.validate((3, 1, 7, 5, 5), (2, 1, 2, 3, 3), (1, 2, 1), non_contiguous=True) self.validate((3, 1, 7, 5, 5), (2, 1, 2, 3, 3), (2, 1, 1), non_contiguous=True) class TestGroupCorr3d(Grouped_conv3d_noOptim): mode = theano.compile.get_mode("FAST_RUN") conv_op = corr3d.Corr3dMM conv_gradw_op = corr3d.Corr3dMM_gradWeights conv_gradi_op = corr3d.Corr3dMM_gradInputs flip_filter = True is_dnn = False if __name__ == '__main__': t = TestCorr3D('setUp') t.setUp() t.test_infer_shape() Theano-1.0.0/theano/tensor/nnet/tests/__init__.py00006000163562€ÿÿÿþ0000000010113203335532023266 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division Theano-1.0.0/theano/tensor/nnet/tests/test_neighbours.py00006000163562€ÿÿÿþ0000005772013203335532024756 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import numpy as np import unittest import theano from theano import shared, function import theano.tensor as T from theano.tensor.nnet.neighbours import images2neibs, neibs2images, Images2Neibs from theano.tests import unittest_tools from theano import change_flags mode_without_gpu = theano.compile.mode.get_default_mode().excluding('gpu') class T_Images2Neibs(unittest_tools.InferShapeTester): mode = mode_without_gpu op = Images2Neibs dtypes = ['int64', 'float32', 'float64'] def test_neibs(self): for shape, pshape in [((10, 7, 18, 18), (2, 2)), ((10, 7, 6, 18), (3, 2)), ((5, 7, 66, 66), (33, 33)), ((5, 7, 68, 66), (34, 33))]: for border in ['valid', 'ignore_borders']: for dtype in self.dtypes: images = shared( np.arange(np.prod(shape), dtype=dtype).reshape(shape)) neib_shape = T.as_tensor_variable(pshape) f = function([], images2neibs(images, neib_shape, mode=border), mode=self.mode) # print images.get_value(borrow=True) neibs = f() # print neibs g = function([], neibs2images(neibs, neib_shape, images.shape), mode=self.mode) assert any([isinstance(node.op, self.op) for node in f.maker.fgraph.toposort()]) # print g() assert np.allclose(images.get_value(borrow=True), g()) def test_neibs_manual(self): shape = (2, 3, 4, 4) for dtype in self.dtypes: images = shared( np.arange(np.prod(shape), dtype=dtype).reshape(shape)) neib_shape = T.as_tensor_variable((2, 2)) for border in ['valid', 'ignore_borders']: f = function([], images2neibs(images, neib_shape, mode=border), mode=self.mode) assert any([isinstance(node.op, self.op) for node in f.maker.fgraph.toposort()]) # print images.get_value(borrow=True) neibs = f() # print neibs assert np.allclose(neibs, [ [0, 1, 4, 5], [2, 3, 6, 7], [8, 9, 12, 13], [10, 11, 14, 15], [16, 17, 20, 21], [18, 19, 22, 23], [24, 25, 28, 29], [26, 27, 30, 31], [32, 33, 36, 37], [34, 35, 38, 39], [40, 41, 44, 45], [42, 43, 46, 47], [48, 49, 52, 53], [50, 51, 54, 55], [56, 57, 60, 61], [58, 59, 62, 63], [64, 65, 68, 69], [66, 67, 70, 71], [72, 73, 76, 77], [74, 75, 78, 79], [80, 81, 84, 85], [82, 83, 86, 87], [88, 89, 92, 93], [90, 91, 94, 95]]) g = function([], neibs2images(neibs, neib_shape, images.shape), mode=self.mode) assert np.allclose(images.get_value(borrow=True), g()) def test_neibs_manual_step(self): shape = (2, 3, 5, 5) for dtype in self.dtypes: images = shared(np.asarray(np.arange(np.prod( shape)).reshape(shape), dtype=dtype)) neib_shape = T.as_tensor_variable((3, 3)) neib_step = T.as_tensor_variable((2, 2)) for border in ['valid', 'ignore_borders']: f = function([], images2neibs(images, neib_shape, neib_step, mode=border), mode=self.mode) neibs = f() assert self.op in [type(node.op) for node in f.maker.fgraph.toposort()] assert np.allclose(neibs, [ [0, 1, 2, 5, 6, 7, 10, 11, 12], [2, 3, 4, 7, 8, 9, 12, 13, 14], [10, 11, 12, 15, 16, 17, 20, 21, 22], [12, 13, 14, 17, 18, 19, 22, 23, 24], [25, 26, 27, 30, 31, 32, 35, 36, 37], [27, 28, 29, 32, 33, 34, 37, 38, 39], [35, 36, 37, 40, 41, 42, 45, 46, 47], [37, 38, 39, 42, 43, 44, 47, 48, 49], [50, 51, 52, 55, 56, 57, 60, 61, 62], [52, 53, 54, 57, 58, 59, 62, 63, 64], [60, 61, 62, 65, 66, 67, 70, 71, 72], [62, 63, 64, 67, 68, 69, 72, 73, 74], [75, 76, 77, 80, 81, 82, 85, 86, 87], [77, 78, 79, 82, 83, 84, 87, 88, 89], [85, 86, 87, 90, 91, 92, 95, 96, 97], [87, 88, 89, 92, 93, 94, 97, 98, 99], [100, 101, 102, 105, 106, 107, 110, 111, 112], [102, 103, 104, 107, 108, 109, 112, 113, 114], [110, 111, 112, 115, 116, 117, 120, 121, 122], [112, 113, 114, 117, 118, 119, 122, 123, 124], [125, 126, 127, 130, 131, 132, 135, 136, 137], [127, 128, 129, 132, 133, 134, 137, 138, 139], [135, 136, 137, 140, 141, 142, 145, 146, 147], [137, 138, 139, 142, 143, 144, 147, 148, 149]]) # neibs2images do not seam to support step != neib_shape # g = function([], neibs2images(neibs, neib_shape, images.shape), # mode=self.mode) # print g() # assert numpy.allclose(images.get_value(borrow=True), g()) @change_flags(compute_test_value='off') def test_neibs_bad_shape(self): shape = (2, 3, 10, 10) for dtype in self.dtypes: images = shared(np.arange( np.prod(shape), dtype=dtype).reshape(shape)) for neib_shape in [(3, 2), (2, 3)]: neib_shape = T.as_tensor_variable(neib_shape) f = function([], images2neibs(images, neib_shape), mode=self.mode) self.assertRaises(TypeError, f) # Test that ignore border work in that case. f = function([], images2neibs(images, neib_shape, mode='ignore_borders'), mode=self.mode) assert self.op in [type(node.op) for node in f.maker.fgraph.toposort()] f() def test_neibs_wrap_centered_step_manual(self): expected1 = [[24, 20, 21, 4, 0, 1, 9, 5, 6], [21, 22, 23, 1, 2, 3, 6, 7, 8], [23, 24, 20, 3, 4, 0, 8, 9, 5], [9, 5, 6, 14, 10, 11, 19, 15, 16], [6, 7, 8, 11, 12, 13, 16, 17, 18], [8, 9, 5, 13, 14, 10, 18, 19, 15], [19, 15, 16, 24, 20, 21, 4, 0, 1], [16, 17, 18, 21, 22, 23, 1, 2, 3], [18, 19, 15, 23, 24, 20, 3, 4, 0]] expected2 = [[24, 20, 21, 4, 0, 1, 9, 5, 6], [22, 23, 24, 2, 3, 4, 7, 8, 9], [14, 10, 11, 19, 15, 16, 24, 20, 21], [12, 13, 14, 17, 18, 19, 22, 23, 24]] expected3 = [[19, 15, 16, 24, 20, 21, 4, 0, 1, 9, 5, 6, 14, 10, 11], [17, 18, 19, 22, 23, 24, 2, 3, 4, 7, 8, 9, 12, 13, 14], [9, 5, 6, 14, 10, 11, 19, 15, 16, 24, 20, 21, 4, 0, 1], [7, 8, 9, 12, 13, 14, 17, 18, 19, 22, 23, 24, 2, 3, 4]] expected4 = [[23, 24, 20, 21, 22, 3, 4, 0, 1, 2, 8, 9, 5, 6, 7], [21, 22, 23, 24, 20, 1, 2, 3, 4, 0, 6, 7, 8, 9, 5], [13, 14, 10, 11, 12, 18, 19, 15, 16, 17, 23, 24, 20, 21, 22], [11, 12, 13, 14, 10, 16, 17, 18, 19, 15, 21, 22, 23, 24, 20]] expected5 = [[24, 20, 21, 4, 0, 1, 9, 5, 6], [22, 23, 24, 2, 3, 4, 7, 8, 9], [9, 5, 6, 14, 10, 11, 19, 15, 16], [7, 8, 9, 12, 13, 14, 17, 18, 19], [19, 15, 16, 24, 20, 21, 4, 0, 1], [17, 18, 19, 22, 23, 24, 2, 3, 4]] expected6 = [[24, 20, 21, 4, 0, 1, 9, 5, 6], [21, 22, 23, 1, 2, 3, 6, 7, 8], [23, 24, 20, 3, 4, 0, 8, 9, 5], [14, 10, 11, 19, 15, 16, 24, 20, 21], [11, 12, 13, 16, 17, 18, 21, 22, 23], [13, 14, 10, 18, 19, 15, 23, 24, 20]] # TODO test discontinous image for shp_idx, (shape, neib_shape, neib_step, expected) in enumerate([ [(7, 8, 5, 5), (3, 3), (2, 2), expected1], [(7, 8, 5, 5), (3, 3), (3, 3), expected2], [(7, 8, 5, 5), (5, 3), (3, 3), expected3], [(7, 8, 5, 5), (3, 5), (3, 3), expected4], [(80, 90, 5, 5), (3, 3), (2, 3), expected5], [(1025, 9, 5, 5), (3, 3), (3, 2), expected6], [(1, 1, 5, 1035), (3, 3), (3, 3), None], [(1, 1, 1045, 5), (3, 3), (3, 3), None], ] ): for dtype in self.dtypes: images = shared(np.asarray(np.arange(np.prod( shape)).reshape(shape), dtype=dtype)) neib_shape = T.as_tensor_variable(neib_shape) neib_step = T.as_tensor_variable(neib_step) expected = np.asarray(expected) f = function([], images2neibs(images, neib_shape, neib_step, mode="wrap_centered"), mode=self.mode) neibs = f() if expected.size > 1: for i in range(shape[0] * shape[1]): assert np.allclose( neibs[i * expected.shape[0]:(i + 1) * expected.shape[0], :], expected + 25 * i), "wrap_centered" assert self.op in [type(node.op) for node in f.maker.fgraph.toposort()] # g = function([], neibs2images(neibs, neib_shape, images.shape), mode=self.mode) # TODO: why this is commented? # assert numpy.allclose(images.get_value(borrow=True), g()) def test_neibs_half_step_by_valid(self): neib_shapes = ((3, 3), (3, 5), (5, 3)) for shp_idx, (shape, neib_step) in enumerate([ [(7, 8, 5, 5), (1, 1)], [(7, 8, 5, 5), (2, 2)], [(7, 8, 5, 5), (4, 4)], [(7, 8, 5, 5), (1, 4)], [(7, 8, 5, 5), (4, 1)], [(80, 90, 5, 5), (1, 2)], [(1025, 9, 5, 5), (2, 1)], [(1, 1, 5, 1037), (2, 4)], [(1, 1, 1045, 5), (4, 2)]] ): for neib_shape in neib_shapes: for dtype in self.dtypes: x = theano.shared(np.random.randn(*shape).astype(dtype)) extra = (neib_shape[0] // 2, neib_shape[1] // 2) padded_shape = (x.shape[0], x.shape[1], x.shape[2] + 2 * extra[0], x.shape[3] + 2 * extra[1]) padded_x = T.zeros(padded_shape) padded_x = T.set_subtensor(padded_x[:, :, extra[0]:-extra[0], extra[1]:-extra[1]], x) x_using_valid = images2neibs(padded_x, neib_shape, neib_step, mode="valid") x_using_half = images2neibs(x, neib_shape, neib_step, mode="half") f_valid = theano.function([], x_using_valid, mode='FAST_RUN') f_half = theano.function([], x_using_half, mode=self.mode) unittest_tools.assert_allclose(f_valid(), f_half()) def test_neibs_full_step_by_valid(self): for shp_idx, (shape, neib_step, neib_shapes) in enumerate([ [(7, 8, 5, 5), (1, 1), ((3, 3), (3, 5), (5, 3))], [(7, 8, 5, 5), (2, 2), ((3, 3), (3, 5), (5, 3))], [(7, 8, 6, 6), (3, 3), ((2, 2), (2, 5), (5, 2))], [(7, 8, 6, 6), (1, 3), ((2, 2), (2, 5), (5, 2))], [(7, 8, 6, 6), (3, 1), ((2, 2), (2, 5), (5, 2))], [(80, 90, 5, 5), (1, 2), ((3, 3), (3, 5), (5, 3))], [(1025, 9, 5, 5), (2, 1), ((3, 3), (3, 5), (5, 3))], [(1, 1, 11, 1037), (2, 3), ((3, 3), (5, 3))], [(1, 1, 1043, 11), (3, 2), ((3, 3), (3, 5))]] ): for neib_shape in neib_shapes: for dtype in self.dtypes: x = theano.shared(np.random.randn(*shape).astype(dtype)) extra = (neib_shape[0] - 1, neib_shape[1] - 1) padded_shape = (x.shape[0], x.shape[1], x.shape[2] + 2 * extra[0], x.shape[3] + 2 * extra[1]) padded_x = T.zeros(padded_shape) padded_x = T.set_subtensor(padded_x[:, :, extra[0]:-extra[0], extra[1]:-extra[1]], x) x_using_valid = images2neibs(padded_x, neib_shape, neib_step, mode="valid") x_using_full = images2neibs(x, neib_shape, neib_step, mode="full") f_valid = theano.function([], x_using_valid, mode='FAST_RUN') f_full = theano.function([], x_using_full, mode=self.mode) unittest_tools.assert_allclose(f_valid(), f_full()) @change_flags(compute_test_value='off') def test_neibs_bad_shape_wrap_centered(self): shape = (2, 3, 10, 10) for dtype in self.dtypes: images = shared(np.arange( np.prod(shape), dtype=dtype ).reshape(shape)) for neib_shape in [(3, 2), (2, 3)]: neib_shape = T.as_tensor_variable(neib_shape) f = function([], images2neibs(images, neib_shape, mode="wrap_centered"), mode=self.mode) self.assertRaises(TypeError, f) for shape in [(2, 3, 2, 3), (2, 3, 3, 2)]: images = shared(np.arange(np.prod(shape)).reshape(shape)) neib_shape = T.as_tensor_variable((3, 3)) f = function([], images2neibs(images, neib_shape, mode="wrap_centered"), mode=self.mode) self.assertRaises(TypeError, f) # Test a valid shapes shape = (2, 3, 3, 3) images = shared(np.arange(np.prod(shape)).reshape(shape)) neib_shape = T.as_tensor_variable((3, 3)) f = function([], images2neibs(images, neib_shape, mode="wrap_centered"), mode=self.mode) f() def test_grad_wrap_centered(self): # It is not implemented for now. So test that we raise an error. shape = (2, 3, 6, 6) images_val = np.random.rand(*shape).astype('float32') def fn(images): return images2neibs(images, (3, 3), mode='wrap_centered') self.assertRaises(TypeError, unittest_tools.verify_grad, fn, [images_val], mode=self.mode) def test_grad_half(self): # It is not implemented for now. So test that we raise an error. shape = (2, 3, 6, 6) images_val = np.random.rand(*shape).astype('float32') def fn(images): return images2neibs(images, (3, 3), mode='half') self.assertRaises(TypeError, unittest_tools.verify_grad, fn, [images_val], mode=self.mode) def test_grad_full(self): # It is not implemented for now. So test that we raise an error. shape = (2, 3, 6, 6) images_val = np.random.rand(*shape).astype('float32') def fn(images): return images2neibs(images, (3, 3), mode='full') self.assertRaises(TypeError, unittest_tools.verify_grad, fn, [images_val], mode=self.mode) def test_grad_valid(self): shape = (2, 3, 6, 6) images_val = np.random.rand(*shape).astype('float32') def fn(images): return images2neibs(images, (2, 2)) unittest_tools.verify_grad(fn, [images_val], mode=self.mode, eps=0.1) def fn(images): return images2neibs(images, (3, 2), (1, 2)) unittest_tools.verify_grad(fn, [images_val], mode=self.mode, eps=0.1) def fn(images): return images2neibs(images, (1, 2), (5, 2)) unittest_tools.verify_grad(fn, [images_val], mode=self.mode, eps=0.1) def test_grad_ignore_border(self): shape = (2, 3, 5, 5) images_val = np.random.rand(*shape).astype('float32') def fn(images): return images2neibs(images, (2, 2), mode='ignore_borders') unittest_tools.verify_grad(fn, [images_val], mode=self.mode, eps=0.1) def test_neibs2images_grad(self): # say we had images of size (2, 3, 10, 10) # then we extracted 2x2 neighbors on this, we get (2 * 3 * 5 * 5, 4) neibs_val = np.random.rand(150, 4) def fn(neibs): return neibs2images(neibs, (2, 2), (2, 3, 10, 10)) unittest_tools.verify_grad(fn, [neibs_val], mode=self.mode, eps=0.1) def test_neibs_valid_with_inconsistent_borders(self): shape = (2, 3, 5, 5) images = T.dtensor4() images_val = np.arange(np.prod(shape), dtype='float32').reshape(shape) f = theano.function([images], T.sqr(images2neibs(images, (2, 2), mode='valid')), mode=self.mode) self.assertRaises(TypeError, f, images_val) def test_neibs_half_with_inconsistent_borders(self): shape = (2, 3, 5, 5) images = T.dtensor4() images_val = np.arange(np.prod(shape), dtype='float32').reshape(shape) f = theano.function([images], T.sqr(images2neibs(images, (2, 2), mode='half')), mode=self.mode) self.assertRaises(TypeError, f, images_val) def test_neibs_full_with_inconsistent_borders(self): shape = (2, 3, 5, 5) images = T.dtensor4() images_val = np.arange(np.prod(shape), dtype='float32').reshape(shape) f = theano.function([images], T.sqr(images2neibs(images, (2, 2), mode='full')), mode=self.mode) self.assertRaises(TypeError, f, images_val) def test_can_not_infer_nb_dim(self): # Was reported in gh-5613. Test that we do not crash # or that we crash in a few other case found while # investigating that case img = T.tensor4('img') patches = T.nnet.neighbours.images2neibs(img, [16, 16]) extractPatches = theano.function([img], patches, mode=self.mode) patsRecovery = T.matrix('patsRecovery') original_size = T.ivector('original_size') for mode in ['valid', 'ignore_borders']: out = neibs2images(patsRecovery, (16, 16), original_size, mode=mode) f = theano.function([patsRecovery, original_size], out, mode=self.mode) im_val = np.ones((1, 3, 320, 320), dtype=np.float32) neibs = extractPatches(im_val) f(neibs, im_val.shape) # Wrong number of dimensions self.assertRaises(ValueError, f, neibs, (1, 1, 3, 320, 320)) # End up with a step of 0 # This can lead to division by zero in DebugMode self.assertRaises((ValueError, ZeroDivisionError), f, neibs, (3, 320, 320, 1)) def speed_neibs(self): shape = (100, 40, 18, 18) images = shared(np.arange(np.prod(shape), dtype='float32').reshape(shape)) neib_shape = T.as_tensor_variable((3, 3)) f = function([], images2neibs(images, neib_shape), mode=self.mode) for i in range(1000): f() def speed_neibs_wrap_centered(self): shape = (100, 40, 18, 18) images = shared(np.arange(np.prod(shape), dtype='float32').reshape(shape)) neib_shape = T.as_tensor_variable((3, 3)) f = function([], images2neibs(images, neib_shape, mode="wrap_centered"), mode=self.mode) for i in range(1000): f() def speed_neibs_half(self): shape = (100, 40, 18, 18) images = shared(np.arange(np.prod(shape), dtype='float32').reshape(shape)) neib_shape = T.as_tensor_variable((3, 3)) f = function([], images2neibs(images, neib_shape, mode="half"), mode=self.mode) for i in range(1000): f() def speed_neibs_full(self): shape = (100, 40, 18, 18) images = shared(np.arange(np.prod(shape), dtype='float32').reshape(shape)) neib_shape = T.as_tensor_variable((3, 3)) f = function([], images2neibs(images, neib_shape, mode="full"), mode=self.mode) for i in range(1000): f() def test_infer_shape(self): shape = (100, 40, 6, 3) images = np.ones(shape).astype('float32') x = T.ftensor4() self._compile_and_check( [x], [images2neibs(x, neib_shape=(2, 1), mode='valid')], [images], Images2Neibs) self._compile_and_check( [x], [images2neibs(x, neib_shape=(2, 3), mode='valid')], [images], Images2Neibs) shape = (100, 40, 5, 4) images = np.ones(shape).astype('float32') x = T.ftensor4() self._compile_and_check( [x], [images2neibs( x, neib_shape=(2, 1), mode='ignore_borders')], [images], Images2Neibs) shape = (100, 40, 5, 3) images = np.ones(shape).astype('float32') x = T.ftensor4() self._compile_and_check( [x], [images2neibs( x, neib_shape=(2, 3), mode='ignore_borders')], [images], Images2Neibs) shape = (100, 40, 6, 7) images = np.ones(shape).astype('float32') x = T.ftensor4() self._compile_and_check( [x], [images2neibs( x, neib_shape=(2, 2), mode='ignore_borders')], [images], Images2Neibs) shape = (100, 40, 5, 10) images = np.ones(shape).astype('float32') x = T.ftensor4() self._compile_and_check( [x], [images2neibs( x, neib_shape=(3, 3), mode='wrap_centered')], [images], Images2Neibs) shape = (100, 40, 6, 4) images = np.ones(shape).astype('float32') x = T.ftensor4() self._compile_and_check( [x], [images2neibs(x, neib_shape=(2, 1), mode='half')], [images], Images2Neibs) self._compile_and_check( [x], [images2neibs(x, neib_shape=(2, 3), mode='half')], [images], Images2Neibs) shape = (100, 40, 6, 5) images = np.ones(shape).astype('float32') x = T.ftensor4() self._compile_and_check( [x], [images2neibs(x, neib_shape=(2, 1), mode='full')], [images], Images2Neibs) self._compile_and_check( [x], [images2neibs(x, neib_shape=(2, 3), mode='full')], [images], Images2Neibs) if __name__ == '__main__': unittest.main() Theano-1.0.0/theano/tensor/nnet/tests/test_ctc.py00006000163562€ÿÿÿþ0000001401513203335532023350 0ustar boccoset00000000000000from __future__ import (division, absolute_import, print_function) import unittest from nose.plugins.skip import SkipTest import numpy as np import theano import theano.tensor as T from theano.tests import unittest_tools as utt from theano.tensor.nnet.ctc import (ctc_available, ctc, ConnectionistTemporalClassification) def setup_torch_case(): # Test obtained from Torch tutorial at: # https://github.com/baidu-research/warp-ctc/blob/master/torch_binding/TUTORIAL.md # Layout, from slowest to fastest changing dimension, is (time, batchSize, inputLayerSize) activations = np.asarray([[[0, 0, 0, 0, 0], [1, 2, 3, 4, 5], [-5, -4, -3, -2, -1]], [[0, 0, 0, 0, 0], [6, 7, 8, 9, 10], [-10, -9, -8, -7, -6]], [[0, 0, 0, 0, 0], [11, 12, 13, 14, 15], [-15, -14, -13, -12, -11]]], dtype=np.float32) # Duration of each sequence activation_times = np.asarray([1, 3, 3], dtype=np.int32) # Labels for each sequence labels = np.asarray([[1, -1], [3, 3], [2, 3]], dtype=np.int32) expected_costs = np.asarray([1.609437943, 7.355742931, 4.938849926], dtype=np.float32) grads = [[[0.2, -0.8, 0.2, 0.2, 0.2], [0.01165623125, 0.03168492019, 0.08612854034, -0.7658783197, 0.636408627], [-0.02115798369, 0.03168492019, -0.8810571432, 0.2341216654, 0.636408627]], [[0, 0, 0, 0, 0], [-0.9883437753, 0.03168492019, 0.08612854034, 0.2341216654, 0.636408627], [-0.02115798369, 0.03168492019, -0.1891518533, -0.4577836394, 0.636408627]], [[0, 0, 0, 0, 0], [0.01165623125, 0.03168492019, 0.08612854034, -0.7658783197, 0.636408627], [-0.02115798369, 0.03168492019, 0.08612854034, -0.7330639958, 0.636408627]]] expected_gradients = np.asarray(grads, dtype=np.float32) return [activations, labels, activation_times, expected_costs, expected_gradients] def setup_ctc_case(): activations = np.asarray([[[0.1, 0.6, 0.1, 0.1, 0.1], [0.1, 0.1, 0.6, 0.1, 0.1]], [[0.6, 0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.5, 0.2, 0.1]]], dtype=np.float32) activation_times = np.asarray([2, 2], dtype=np.int32) labels = np.asarray([[1, 2], [1, 2]], dtype=np.int32) expected_costs = np.asarray([2.962858438, 3.053659201], dtype=np.float32) grads = [[[0.177031219, -0.7081246376, 0.177031219, 0.177031219, 0.177031219], [0.177031219, -0.8229685426, 0.291875124, 0.177031219, 0.177031219]], [[0.291875124, 0.177031219, -0.8229685426, 0.177031219, 0.177031219], [0.1786672771, 0.1786672771, -0.7334594727, 0.1974578798, 0.1786672771]]] expected_gradients = np.asarray(grads, dtype=np.float32) return [activations, labels, activation_times, expected_costs, expected_gradients] def setup_grad_case(): activations = np.asarray([[[0.1, 0.6, 0.1, 0.1, 0.1], [0.1, 0.1, 0.6, 0.1, 0.1]], [[0.6, 0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.5, 0.2, 0.1]]], dtype=np.float32) activation_times = np.asarray([2, 2], dtype=np.int32) labels = np.asarray([[1, 2], [1, 2]], dtype=np.int32) return [activations, labels, activation_times] class TestCTC(unittest.TestCase): """ Test Baidu CTC wrapper implementation. Expected values for costs and gradients are obtained through an external C implementation, that uses the library directly. """ def setUp(self): if theano.config.mode == "FAST_COMPILE" or theano.config.cxx == "": raise SkipTest("We need a c compiler") if not ctc_available(): self.skipTest('Optional library warp-ctc not available') def run_ctc(self, activations, labels, input_length, expected_costs, expected_grads): # Create symbolic variables t_activations = theano.shared(activations, name="activations") t_activation_times = theano.shared(input_length, name="activation_times") t_labels = theano.shared(labels, name="labels") t_cost = ctc(t_activations, t_labels, t_activation_times) # Symbolic gradient of CTC cost t_grad = T.grad(T.mean(t_cost), t_activations) # Compile symbolic functions train = theano.function([], [t_cost, t_grad]) cost, grad = train() utt.assert_allclose(expected_grads / cost.shape[0], grad) utt.assert_allclose(expected_costs, cost) self.check_grads_disabled(t_activations, t_labels, t_activation_times) def check_grads_disabled(self, activations, labels, input_length): """ Check if optimization to disable gradients is working """ ctc_cost = ctc(activations, labels, input_length) ctc_function = theano.function([], [ctc_cost]) for node in ctc_function.maker.fgraph.apply_nodes: if isinstance(node.op, ConnectionistTemporalClassification): assert (node.op.compute_grad is False) def test_torch_case(self): activations, labels, input_length, expected_costs, expected_grads = setup_torch_case() self.run_ctc(activations, labels, input_length, expected_costs, expected_grads) def test_ctc(self): activations, labels, input_length, expected_costs, expected_grads = setup_ctc_case() self.run_ctc(activations, labels, input_length, expected_costs, expected_grads) def test_verify_grad(self): def ctc_op_functor(labels, in_lengths): def wrapper(acts): # Create auxiliary symbolic variables t_activation_times = theano.shared(in_lengths, name="activation_times") t_labels = theano.shared(labels, name="labels") return ctc(acts, t_labels, t_activation_times) return wrapper activations, labels, activation_times = setup_grad_case() ctc_op = ctc_op_functor(labels, activation_times) utt.verify_grad(ctc_op, [activations]) Theano-1.0.0/theano/tensor/nnet/tests/test_sigm.py00006000163562€ÿÿÿþ0000005011213203335532023534 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import unittest import numpy as np from theano.compat import imap import theano.tensor.inplace from theano.tensor import basic as tensor from theano import tensor as T from theano import config from theano.gof.opt import check_stack_trace from theano.tests import unittest_tools as utt from theano.tensor.nnet import (sigmoid, sigmoid_inplace, softplus, ultra_fast_sigmoid, hard_sigmoid) from theano.tensor.nnet.sigm import ( compute_mul, is_1pexp, parse_mul_tree, perform_sigm_times_exp, register_local_1msigmoid, simplify_mul, ) from theano.tensor.tests.test_basic import (makeBroadcastTester, copymod, check_floatX, upcast_int8_nfunc, _good_broadcast_unary_normal_no_complex) class T_sigmoid(unittest.TestCase): def setUp(self): utt.seed_rng() def test_elemwise(self): utt.verify_grad(sigmoid, [np.random.rand(3, 4)]) SigmoidTester = makeBroadcastTester( op=sigmoid, expected=upcast_int8_nfunc(lambda inputs: check_floatX( inputs, 1 / (1 + np.exp(-inputs)))), good=copymod(_good_broadcast_unary_normal_no_complex, without=['uint16']), # The reason that 'uint16' is excluted is that # theano works well but numpy overflows resulting # in an assertion error. # grad=_grad_broadcast_unary_normal, name='SigmoidTester', ) UltraFastSigmoidTester = makeBroadcastTester( op=ultra_fast_sigmoid, expected=upcast_int8_nfunc(lambda inputs: check_floatX( inputs, 1 / (1 + np.exp(-inputs)))), good=copymod(_good_broadcast_unary_normal_no_complex, without=['uint16']), # numpy fucnting overflows with uint16. # grad=_grad_broadcast_unary_normal, name='UltraFastSigmoidTester', # This is an approx of the sigmoid. That is why we raise eps eps=5e-2) HardSigmoidTester = makeBroadcastTester( op=hard_sigmoid, expected=upcast_int8_nfunc(lambda inputs: check_floatX( inputs, 1 / (1 + np.exp(-inputs)))), good=copymod(_good_broadcast_unary_normal_no_complex, without=['uint16']), # numpy fucnting overflows with uint16. # grad=_grad_broadcast_unary_normal, name='HardSigmoidTester', # This is an approx of the sigmoid. That is why we raise eps eps=1e-1) SoftplusTester = makeBroadcastTester( op=softplus, expected=upcast_int8_nfunc(lambda inputs: check_floatX( inputs, np.log1p(np.exp(inputs)))), good=dict(copymod(_good_broadcast_unary_normal_no_complex, without=['uint8', 'uint16']), # numpy fucnting overflows with uint16. uint8=[np.arange(0, 89, dtype='uint8')], # the range is different in new added uint8. int8=[np.arange(-127, 89, dtype='int8')]), # grad=_grad_broadcast_unary_normal, name='SoftplusTester', ) class T_softplus(unittest.TestCase): def setUp(self): utt.seed_rng() def test_elemwise(self): utt.verify_grad(softplus, [np.random.rand(3, 4)]) class T_sigmoid_opts(unittest.TestCase): def get_mode(self, excluding=None): """ Return appropriate mode for the tests. :param excluding: List of optimizations to exclude. :return: The current default mode unless the `config.mode` option is set to 'FAST_COMPILE' (in which case it is replaced by the 'FAST_RUN' mode), without the optimizations specified in `excluding`. """ if excluding is None: excluding = [] m = theano.config.mode if m == 'FAST_COMPILE': mode = theano.compile.mode.get_mode('FAST_RUN') else: mode = theano.compile.mode.get_default_mode() if excluding: return mode.excluding(*excluding) else: return mode def test_exp_over_1_plus_exp(self): m = self.get_mode(excluding=['local_elemwise_fusion']) x = T.vector() data = np.random.rand(54).astype(config.floatX) backup = config.warn.identify_1pexp_bug config.warn.identify_1pexp_bug = False try: # tests exp_over_1_plus_exp f = theano.function([x], T.exp(x) / (1 + T.exp(x)), mode=m) assert [node.op for node in f.maker.fgraph.toposort()] == [sigmoid] f(data) f = theano.function([x], T.exp(x) / (2 + T.exp(x)), mode=m) assert [node.op for node in f.maker.fgraph.toposort()] != [sigmoid] f(data) f = theano.function([x], T.exp(x) / (1 - T.exp(x)), mode=m) assert [node.op for node in f.maker.fgraph.toposort()] != [sigmoid] f(data) f = theano.function([x], T.exp(x + 1) / (1 + T.exp(x)), mode=m) assert [node.op for node in f.maker.fgraph.toposort()] != [sigmoid] f(data) # tests inv_1_plus_exp f = theano.function([x], T.fill(x, 1.0) / (1 + T.exp(-x)), mode=m) # todo: solve issue #4589 first # assert check_stack_trace(f, ops_to_check=sigmoid) assert [node.op for node in f.maker.fgraph.toposort()] == [sigmoid] f(data) f = theano.function([x], T.fill(x, 1.0) / (2 + T.exp(-x)), mode=m) assert [node.op for node in f.maker.fgraph.toposort()] != [sigmoid] f(data) f = theano.function([x], T.fill(x, 1.0) / (1 - T.exp(-x)), mode=m) assert [node.op for node in f.maker.fgraph.toposort()] != [sigmoid] f(data) f = theano.function([x], T.fill(x, 1.1) / (1 + T.exp(-x)), mode=m) assert [node.op for node in f.maker.fgraph.toposort()] != [sigmoid] f(data) # tests inv_1_plus_exp with neg f = theano.function([x], T.fill(x, -1.0) / (1 + T.exp(-x)), mode=m) # todo: solve issue #4589 first # assert check_stack_trace( # f, ops_to_check=[sigmoid, theano.tensor.inplace.neg_inplace]) assert ([node.op for node in f.maker.fgraph.toposort()] == [sigmoid, theano.tensor.inplace.neg_inplace]) f(data) f = theano.function([x], T.fill(x, -1.0) / (1 - T.exp(-x)), mode=m) assert ([node.op for node in f.maker.fgraph.toposort()] != [sigmoid, theano.tensor.inplace.neg_inplace]) f(data) f = theano.function([x], T.fill(x, -1.0) / (2 + T.exp(-x)), mode=m) assert ([node.op for node in f.maker.fgraph.toposort()] != [sigmoid, theano.tensor.inplace.neg_inplace]) f(data) f = theano.function([x], T.fill(x, -1.1) / (1 + T.exp(-x)), mode=m) assert ([node.op for node in f.maker.fgraph.toposort()] != [sigmoid, theano.tensor.inplace.neg_inplace]) f(data) # tests double inv_1_plus_exp with neg # (-1)(exp(x)) / (1+exp(x))(1+exp(-x)) # = (-1)/(1+exp(-x)) * exp(x)/(1+exp(x)) # = - (sigm(x) * sigm(x)) f = theano.function([x], (T.fill(x, -1.0) * T.exp(x)) / ((1 + T.exp(x)) * (1 + T.exp(-x))), mode=m) # todo: solve issue #4589 first # assert check_stack_trace(f, ops_to_check=[sigmoid, T.mul]) assert ([node.op for node in f.maker.fgraph.toposort()] == [sigmoid, T.mul]) f(data) f = theano.function([x], (T.fill(x, -1.1) * T.exp(x)) / ((1 + T.exp(x)) * (1 + T.exp(-x))), mode=m) assert ([node.op for node in f.maker.fgraph.toposort()] != [sigmoid, T.mul, theano.tensor.inplace.neg_inplace]) f(data) f = theano.function([x], (T.fill(x, -1.0) * T.exp(x)) / ((2 + T.exp(x)) * (1 + T.exp(-x))), mode=m) assert ([node.op for node in f.maker.fgraph.toposort()] != [sigmoid, T.mul, theano.tensor.inplace.neg_inplace]) f(data) f = theano.function([x], (T.fill(x, -1.0) * T.exp(x)) / ((1 + T.exp(x)) * (2 + T.exp(-x))), mode=m) assert ([node.op for node in f.maker.fgraph.toposort()] != [sigmoid, T.mul, theano.tensor.inplace.neg_inplace]) f(data) f = theano.function([x], (T.fill(x, -1.0) * T.exp(x)) / ((1 + T.exp(x)) * (1 + T.exp(x))), mode=m) assert ([node.op for node in f.maker.fgraph.toposort()] != [sigmoid, T.mul, theano.tensor.inplace.neg_inplace]) f(data) f = theano.function([x], (T.fill(x, -1.0) * T.exp(x)) / ((1 + T.exp(x)) * (2 + T.exp(-x))), mode=m) assert ([node.op for node in f.maker.fgraph.toposort()] != [sigmoid, T.mul, theano.tensor.inplace.neg_inplace]) f(data) finally: # Restore config option. config.warn.identify_1pexp_bug = backup def test_1msigmoid(self): if not register_local_1msigmoid: return m = self.get_mode() x = T.fmatrix() # tests exp_over_1_plus_exp f = theano.function([x], 1 - T.exp(x) / (1 + T.exp(x)), mode=m) assert check_stack_trace(f, ops_to_check=[tensor.neg, sigmoid_inplace]) assert [node.op for node in f.maker.fgraph.toposort()] == [ tensor.neg, sigmoid_inplace] # tests inv_1_plus_exp f = theano.function([x], 1 - T.fill(x, 1.0) / (1 + T.exp(-x)), mode=m) assert check_stack_trace(f, ops_to_check=[tensor.neg, sigmoid_inplace]) assert ([node.op for node in f.maker.fgraph.toposort()] == [tensor.neg, sigmoid_inplace]) def test_local_sigm_times_exp(self): # Test the `local_sigm_times_exp` optimization. # exp(x) * sigm(-x) -> sigm(x) # exp(-x) * sigm(x) -> sigm(-x) def match(func, ops): # print [node.op.scalar_op for node in func.maker.fgraph.toposort()] assert [node.op for node in func.maker.fgraph.toposort()] == ops m = self.get_mode(excluding=['local_elemwise_fusion', 'inplace']) x, y = tensor.vectors('x', 'y') f = theano.function([x], sigmoid(-x) * tensor.exp(x), mode=m) match(f, [sigmoid]) assert check_stack_trace(f, ops_to_check=sigmoid) f = theano.function([x], sigmoid(x) * tensor.exp(-x), mode=m) match(f, [tensor.neg, sigmoid]) assert check_stack_trace(f, ops_to_check=sigmoid) f = theano.function([x], -(-(-(sigmoid(x)))) * tensor.exp(-x), mode=m) match(f, [tensor.neg, sigmoid, tensor.neg]) # assert check_stack_trace(f, ops_to_check=sigmoid) f = theano.function( [x, y], (sigmoid(x) * sigmoid(-y) * -tensor.exp(-x) * tensor.exp(x * y) * tensor.exp(y)), mode=m) topo = f.maker.fgraph.toposort() for op, nb in [(sigmoid, 2), (tensor.mul, 2), (tensor.neg, 1), (tensor.exp, 1)]: assert sum([n.op == op for n in topo]) == nb # assert check_stack_trace(f, ops_to_check=[sigmoid, tensor.mul, # tensor.exp]) def test_perform_sigm_times_exp(self): # Test the core function doing the `sigm_times_exp` optimization. # # It is easier to test different graph scenarios this way than by # compiling a theano function. x, y, z, t = tensor.vectors('x', 'y', 'z', 't') exp = tensor.exp def ok(expr1, expr2): trees = [parse_mul_tree(e) for e in (expr1, expr2)] perform_sigm_times_exp(trees[0]) trees[0] = simplify_mul(trees[0]) good = theano.gof.graph.is_same_graph( compute_mul(trees[0]), compute_mul(trees[1])) if not good: print(trees[0]) print(trees[1]) print('***') theano.printing.debugprint(compute_mul(trees[0])) print('***') theano.printing.debugprint(compute_mul(trees[1])) assert good ok(sigmoid(x) * exp(-x), sigmoid(-x)) ok(-x * sigmoid(x) * (y * (-1 * z) * exp(-x)), -x * sigmoid(-x) * (y * (-1 * z))) ok(-sigmoid(-x) * (exp(y) * (-exp(-z) * 3 * -exp(x)) * (y * 2 * (-sigmoid(-y) * (z + t) * exp(z)) * sigmoid(z))) * - sigmoid(x), sigmoid(x) * (-sigmoid(y) * (-sigmoid(-z) * 3) * (y * 2 * ((z + t) * exp(z)))) * (-sigmoid(x))) ok(exp(-x) * -exp(-x) * (-sigmoid(x) * -sigmoid(x)), -sigmoid(-x) * sigmoid(-x)) ok(-exp(x) * -sigmoid(-x) * -exp(-x), -sigmoid(-x)) def test_grad_log1msigm(self): # At some point, this returned nan, because (1 - sigm(x)) was # on both the numerator and the denominator of a fraction, # but the two nodes in question had not been merged. x = tensor.matrix('x') lr = tensor.scalar('lr') s = sigmoid(x) l = T.log(1 - s) c = l.mean() ux = x - lr * theano.grad(c, x) # Before the optimization, inf and NaN will be produced in the graph, # and DebugMode will complain. Everything is fine afterwards. mode = self.get_mode() if not isinstance(mode, theano.compile.DebugMode): f = theano.function([x, lr], ux, mode=mode) ux_v = f([[50]], 0.1) assert not np.isnan(ux_v) def test_local_ultra_fast_sigmoid(self): x = tensor.matrix('x') s = sigmoid(x) mode = self.get_mode('local_ultra_fast_sigmoid') f = theano.function([x], s, mode=mode) assert check_stack_trace(f, ops_to_check=sigmoid) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert topo[0].op == sigmoid mode = self.get_mode().including('local_ultra_fast_sigmoid') f = theano.function([x], s, mode=mode) assert check_stack_trace(f, ops_to_check=ultra_fast_sigmoid) topo = f.maker.fgraph.toposort() assert topo[0].op == ultra_fast_sigmoid assert len(topo) == 1 f([[-50, -10, -4, -1, 0, 1, 4, 10, 50]]) def test_local_hard_sigmoid(self): x = tensor.matrix('x') s = sigmoid(x) mode = self.get_mode('local_hard_sigmoid') f = theano.function([x], s, mode=mode) assert check_stack_trace(f, ops_to_check=sigmoid) topo = f.maker.fgraph.toposort() assert topo[0].op == sigmoid assert len(topo) == 1 mode = self.get_mode().including('local_hard_sigmoid') f = theano.function([x], s, mode=mode) topo = f.maker.fgraph.toposort() assert not any([n.op == sigmoid for n in topo]) f([[-50, -10, -4, -1, 0, 1, 4, 10, 50]]) mode2 = mode.excluding('fusion').excluding('inplace') f2 = theano.function([x], s, mode=mode2) self.assertTrue(check_stack_trace(f2, ops_to_check=theano.tensor.clip)) class T_softplus_opts(unittest.TestCase): def setUp(self): if theano.config.mode == 'FAST_COMPILE': m = theano.compile.mode.get_mode('FAST_RUN').excluding( 'local_elemwise_fusion') else: m = theano.compile.mode.get_default_mode().excluding( 'local_elemwise_fusion') self.m = m utt.seed_rng() def test_logsigm_to_softplus(self): x = T.vector() out = T.log(sigmoid(x)) f = theano.function([x], out, mode=self.m) # Fix ticket #4581 first # assert check_stack_trace( # f, ops_to_check=(theano.scalar.Neg, # theano.tensor.nnet.sigm.ScalarSoftplus)) topo = f.maker.fgraph.toposort() assert len(topo) == 3 assert isinstance(topo[0].op.scalar_op, theano.scalar.Neg) assert isinstance(topo[1].op.scalar_op, theano.tensor.nnet.sigm.ScalarSoftplus) assert isinstance(topo[2].op.scalar_op, theano.scalar.Neg) f(np.random.rand(54).astype(config.floatX)) def test_log1msigm_to_softplus(self): x = T.matrix() out = T.log(1 - sigmoid(x)) f = theano.function([x], out, mode=self.m) topo = f.maker.fgraph.toposort() assert len(topo) == 2 assert isinstance(topo[0].op.scalar_op, theano.tensor.nnet.sigm.ScalarSoftplus) assert isinstance(topo[1].op.scalar_op, theano.scalar.Neg) # assert check_stack_trace(f, ops_to_check='all') f(np.random.rand(54, 11).astype(config.floatX)) # Same test with a flatten out = T.log(1 - T.flatten(sigmoid(x))) f = theano.function([x], out, mode=self.m) # assert check_stack_trace(f, ops_to_check='all') topo = f.maker.fgraph.toposort() assert len(topo) == 3 assert tensor.is_flat(topo[0].outputs[0]) assert isinstance(topo[1].op.scalar_op, theano.tensor.nnet.sigm.ScalarSoftplus) assert isinstance(topo[2].op.scalar_op, theano.scalar.Neg) f(np.random.rand(54, 11).astype(config.floatX)) # Same test with a reshape out = T.log(1 - sigmoid(x).reshape([x.size])) f = theano.function([x], out, mode=self.m) topo = f.maker.fgraph.toposort() # assert len(topo) == 3 assert any(isinstance(node.op, T.Reshape) for node in topo) assert any(isinstance(getattr(node.op, 'scalar_op', None), theano.tensor.nnet.sigm.ScalarSoftplus) for node in topo) f(np.random.rand(54, 11).astype(config.floatX)) def test_log1pexp_to_softplus(self): m = theano.config.mode if m == 'FAST_COMPILE': m = 'FAST_RUN' x = T.vector() out = T.log(1 + T.exp(x)) f = theano.function([x], out, mode=self.m) # Fix ticket #4581 first # assert check_stack_trace(f, ops_to_check='all') topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op.scalar_op, theano.tensor.nnet.sigm.ScalarSoftplus) f(np.random.rand(54).astype(config.floatX)) class T_sigmoid_utils(unittest.TestCase): """ Test utility functions found in 'sigm.py'. """ def test_compute_mul(self): x, y, z = tensor.vectors('x', 'y', 'z') tree = (x * y) * -z mul_tree = parse_mul_tree(tree) assert parse_mul_tree(compute_mul(mul_tree)) == mul_tree assert theano.gof.graph.is_same_graph( compute_mul(parse_mul_tree(tree)), tree) def test_parse_mul_tree(self): x, y, z = tensor.vectors('x', 'y', 'z') assert parse_mul_tree(x * y) == [False, [[False, x], [False, y]]] assert parse_mul_tree(-(x * y)) == [True, [[False, x], [False, y]]] assert parse_mul_tree(-x * y) == [False, [[True, x], [False, y]]] assert parse_mul_tree(-x) == [True, x] assert parse_mul_tree((x * y) * -z) == [ False, [[False, [[False, x], [False, y]]], [True, z]]] def test_is_1pexp(self): backup = config.warn.identify_1pexp_bug config.warn.identify_1pexp_bug = False try: x = tensor.vector('x') exp = tensor.exp assert is_1pexp(1 + exp(x), False) == (False, x) assert is_1pexp(exp(x) + 1, False) == (False, x) for neg, exp_arg in imap(lambda x: is_1pexp(x, only_process_constants=False), [(1 + exp(-x)), (exp(-x) + 1)]): assert not neg and theano.gof.graph.is_same_graph(exp_arg, -x) assert is_1pexp(1 - exp(x), False) is None assert is_1pexp(2 + exp(x), False) is None assert is_1pexp(exp(x) + 2, False) is None assert is_1pexp(exp(x) - 1, False) is None assert is_1pexp(-1 + exp(x), False) is None assert is_1pexp(1 + 2 * exp(x), False) is None finally: config.warn.identify_1pexp_bug = backup Theano-1.0.0/theano/tensor/nnet/tests/test_nnet.py00006000163562€ÿÿÿþ0000020427613203335532023555 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import unittest import numpy as np from nose.plugins.skip import SkipTest from six.moves import xrange import theano from theano import config from theano import tensor as T from theano import tensor from theano import gof from theano.gof.opt import check_stack_trace from theano.tests import unittest_tools as utt from theano import printing from theano.tensor.nnet import (categorical_crossentropy, crossentropy_categorical_1hot, crossentropy_softmax_1hot, crossentropy_softmax_1hot_with_bias, crossentropy_softmax_1hot_with_bias_dx, crossentropy_softmax_argmax_1hot_with_bias, CrossentropySoftmax1HotWithBiasDx, CrossentropySoftmaxArgmax1HotWithBias, CrossentropyCategorical1Hot, CrossentropyCategorical1HotGrad, sigmoid, softplus, Softmax, softmax, softmax_op, softmax_graph, SoftmaxWithBias, softmax_with_bias, logsoftmax_op, softmax_grad, SoftmaxGrad, Prepend_scalar_constant_to_each_row, Prepend_scalar_to_each_row, relu, h_softmax, elu, selu, binary_crossentropy, sigmoid_binary_crossentropy, confusion_matrix) from theano.tensor import matrix, vector, lvector, scalar from theano.tensor.nnet.nnet import softsign from theano.tensor.tests.test_basic import (makeBroadcastTester, check_floatX, _good_broadcast_unary_normal_float_no_complex, upcast_int8_nfunc) class T_sigmoid(unittest.TestCase): def setUp(self): utt.seed_rng() def test_elemwise(self): utt.verify_grad(sigmoid, [np.random.rand(3, 4)]) class T_softplus(unittest.TestCase): def setUp(self): utt.seed_rng() def test_elemwise(self): utt.verify_grad(softplus, [np.random.rand(3, 4)]) class T_Softmax(utt.InferShapeTester): def test0(self): def f(a): return softmax_op(a)[:, 0] utt.verify_grad(f, [np.random.rand(3, 4)]) def test1(self): def f(a): return softmax_op(a)[:, 1] utt.verify_grad(f, [np.random.rand(3, 4)]) def test2(self): def f(a): return softmax_op(a)[:, 2] utt.verify_grad(f, [np.random.rand(3, 4)]) def test3(self): def f(a): return softmax_op(a)[:, 3] utt.verify_grad(f, [np.random.rand(3, 4)]) def test_infer_shape(self): admat = matrix() admat_val = np.random.rand(3, 4).astype(config.floatX) self._compile_and_check([admat], [Softmax()(admat)], [admat_val], Softmax) def test_vector(self): x = T.vector() f = theano.function([x], softmax_op(x)) xv = np.random.randn(6).astype(config.floatX) assert np.allclose(f(xv), np.exp(xv) / np.exp(xv).sum()) def test_vector_grad(self): def f(a): return softmax_op(a) utt.verify_grad(f, [np.random.rand(4)]) class T_SoftmaxWithBias(utt.InferShapeTester): def test0(self): def f(a, b): return softmax_with_bias(a, b)[:, 0] utt.verify_grad(f, [np.random.rand(3, 4), np.random.rand(4)]) def test1(self): def f(a, b): return softmax_with_bias(a, b)[:, 1] utt.verify_grad(f, [np.random.rand(3, 4), np.random.rand(4)]) def test2(self): def f(a, b): return softmax_with_bias(a, b)[:, 2] utt.verify_grad(f, [np.random.rand(3, 4), np.random.rand(4)]) def test3(self): def f(a, b): return softmax_with_bias(a, b)[:, 3] utt.verify_grad(f, [np.random.rand(3, 4), np.random.rand(4)]) def test_broadcast(self): # test that we don't raise an error during optimization for no good # reason as softmax_with_bias don't support correctly some/all # broadcasted inputs pattern initial_W = np.asarray([[0.1, 0.1, 0.1], [0.1, 0.1, 0.1], [0.1, 0.1, 0.1]], dtype=theano.config.floatX) W = theano.shared(value=initial_W, name='W') vbias = theano.shared(value=0.1, name='vbias') # 0.01 hid = T.vector('hid') f = theano.function([hid], T.nnet.softmax_op(T.dot(hid, W.T) + vbias)) ops = [node.op for node in f.maker.fgraph.toposort()] assert softmax_with_bias not in ops assert softmax_op in ops f([0, 1, 0]) # print f.maker.fgraph.toposort() def test_softmax_with_bias_trace(self): a = theano.shared( np.random.randn(3).astype(config.floatX)) b = theano.shared(np.float32(np.random.randn())) sm = T.nnet.softmax(a + b) f = theano.function([], sm) assert check_stack_trace(f, ops_to_check='last') def test_infer_shape(self): admat = matrix() advec = vector() admat_val = np.random.rand(3, 4).astype(config.floatX) advec_val = np.random.rand(4).astype(config.floatX) self._compile_and_check([admat, advec], [SoftmaxWithBias()(admat, advec)], [admat_val, advec_val], SoftmaxWithBias) class T_LogSoftmax(utt.InferShapeTester): def test0(self): def f(a): return logsoftmax_op(a)[:, 0] utt.verify_grad(f, [np.random.rand(3, 4)]) def test1(self): def f(a): return logsoftmax_op(a)[:, 1] utt.verify_grad(f, [np.random.rand(3, 4)]) def test2(self): def f(a): return logsoftmax_op(a)[:, 2] utt.verify_grad(f, [np.random.rand(3, 4)]) def test3(self): def f(a): return logsoftmax_op(a)[:, 3] utt.verify_grad(f, [np.random.rand(3, 4)]) def test_matrix(self): def f(a): return logsoftmax_op(a) utt.verify_grad(f, [np.random.rand(3, 4)]) def test_vector(self): x = T.vector() f = theano.function([x], logsoftmax_op(x)) xv = np.random.randn(6).astype(config.floatX) assert np.allclose(f(xv), np.log(np.exp(xv) / np.exp(xv).sum())) def test_vector_grad(self): def f(a): return logsoftmax_op(a) utt.verify_grad(f, [np.random.rand(4)]) def test_allclose(self): m = theano.config.mode m = theano.compile.get_mode(m) m.check_isfinite = False x, y = tensor.matrices('xy') # regular softmax and crossentropy sm = tensor.nnet.softmax(x) cm = tensor.nnet.categorical_crossentropy(sm, y) # numerically stable log-softmax with crossentropy logsm = tensor.nnet.logsoftmax(x) sm2 = tensor.exp(logsm) # just used to show equivalence with sm cm2 = -tensor.sum(y * logsm, axis=1) grad = tensor.grad(cm2.mean(), x) # create some inputs into a softmax that are large and labels a = np.exp(10 * np.random.rand(5, 10).astype(theano.config.floatX)) # create some one-hot coded labels b = np.eye(5, 10).astype(theano.config.floatX) # show equivalence of softmax and exponentiated numerically stable # log-softmax f1 = theano.function([x], [sm, sm2]) sm_, sm2_ = f1(a) utt.assert_allclose(sm_, sm2_) # now show that the two versions result in the same crossentropy cost # this indicates that the forward function does provide some numerical # stability f2 = theano.function([x, y], [cm, cm2], mode=m) cm_, cm2_ = f2(a, b) utt.assert_allclose(cm_, cm2_) # now, show that in the standard softmax case the gradients blow up # while in the log-softmax case they don't f3 = theano.function([x, y], [grad]) grad_ = f3(a, b) assert not np.any(np.isnan(grad_)) def test_isclose(self): def f(a): return logsoftmax_op(a) def test_local_softmax_optimization(self): # Test the Logsoftmax substitution # # Check that Log(Softmax(x)) is substituted with Logsoftmax(x). Note that # only the forward pass is checked (i.e., doesn't check the gradient) x, y = tensor.matrices('xy') sm = tensor.nnet.softmax(x) logsm = tensor.log(sm) f = theano.function([x], logsm) assert isinstance(f.maker.fgraph.outputs[0].owner.op, theano.tensor.nnet.nnet.LogSoftmax) assert check_stack_trace( f, ops_to_check=theano.tensor.nnet.nnet.LogSoftmax) def test_local_softmax_grad_optimization_and_big_input(self): # Test the Logsoftmax's grad substitution. # # Check that Log(Softmax(x))'s grad is substituted with Logsoftmax(x)'s # grad and that the new operation does not explode for big inputs. # Note that only the grad is checked. m = theano.config.mode m = theano.compile.get_mode(m) m.check_isfinite = False # some inputs that are large to make the gradient explode in the non # optimized case a = np.exp( 10 * np.random.rand(5, 10).astype(theano.config.floatX)) def myfunc(x): sm = tensor.nnet.softmax(x) logsm = tensor.log(sm) return logsm # We set step to 0.1 because for big values we need a big epsilon utt.verify_grad(myfunc, [a], eps=0.1, mode=m) sa = theano.shared(a) f = theano.function([], myfunc(sa)) self.assertTrue(check_stack_trace(f, ops_to_check='all')) def test_logsoftmax_grad_true_div_elemwise(self): # Checks that the gradient of an expression similar to a log(softmax) # but with a different elemwise operation than true_div is not # optimized. x = T.matrix('x') y = T.log(T.nnet.softmax(x)) g = T.grad(y.sum(), x) softmax_grad_node = g.owner assert softmax_grad_node.op == softmax_grad true_div_node = softmax_grad_node.inputs[0].owner assert true_div_node.op == tensor.true_div # We replace the elemwise true_div op by an elemwise add. new_g = softmax_grad(tensor.add(*true_div_node.inputs), softmax_grad_node.inputs[1]) fgraph = gof.FunctionGraph([x], [new_g]) theano.compile.mode.optdb.query( theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) assert softmax_grad in [n.op for n in fgraph.toposort()] class T_SoftmaxGrad(utt.InferShapeTester): def test_infer_shape(self): admat = matrix() bdmat = matrix() admat_val = np.random.rand(3, 4).astype(config.floatX) bdmat_val = np.random.rand(3, 4).astype(config.floatX) self._compile_and_check([admat, bdmat], [SoftmaxGrad()(admat, bdmat)], [admat_val, bdmat_val], SoftmaxGrad) class T_CrossentropySoftmax1Hot(unittest.TestCase): def setUp(self): utt.seed_rng() def test0(self): y_idx = [0, 1, 3] def f(a, b): return crossentropy_softmax_1hot_with_bias(a, b, y_idx)[0] utt.verify_grad(f, [np.random.rand(3, 4), np.random.rand(4)]) def test1(self): y_idx = [0, 1, 3] def f(a): return crossentropy_softmax_1hot(a, y_idx)[0] utt.verify_grad(f, [np.random.rand(3, 4)]) def test_vector(self): y_idx = [3] def f(a): return crossentropy_softmax_1hot(T.shape_padleft(a), y_idx)[0] utt.verify_grad(f, [np.random.rand(4)]) def test_vectors(self): y_idx = [3] def f(a, b): return crossentropy_softmax_1hot(T.shape_padleft(a) + b, y_idx)[0] utt.verify_grad(f, [np.random.rand(4), np.random.rand(4)]) class T_CrossentropySoftmax1HotWithBiasDx(utt.InferShapeTester): def test0(self): def ff(class_dtype): def f(sm): # Class indices y = np.random.randint(low=0, high=5, size=10).astype(class_dtype) return theano.tensor.nnet.crossentropy_softmax_1hot_with_bias_dx( np.random.rand(10), # Gradient w.r.t. NLL. sm, # Softmax output. y) return f # Build a random softmax output whose rows sum to 1. softmax_output = np.random.rand(10, 5) softmax_output /= softmax_output.sum(axis=1).reshape(10, 1) for dtype in ['uint8', 'int8', 'uint64', 'int64']: utt.verify_grad(ff(dtype), [softmax_output]) def test1(self): rng = np.random.RandomState(utt.fetch_seed()) softmax_output = rng.rand(10, 5) softmax_output /= softmax_output.sum(axis=1).reshape(10, 1) def f(dy): return (theano.tensor.nnet.crossentropy_softmax_1hot_with_bias_dx( dy, softmax_output, rng.randint(low=0, high=5, size=10))) utt.verify_grad(f, [rng.rand(10)]) def test_infer_shape(self): admat = matrix() advec = vector() alvec = lvector() rng = np.random.RandomState(utt.fetch_seed()) admat_val = rng.rand(10, 5).astype(config.floatX) admat_val /= admat_val.sum(axis=1).reshape(10, 1) advec_val = rng.rand(10).astype(config.floatX) alvec_val = rng.randint(low=0, high=5, size=10) self._compile_and_check( [advec, admat, alvec], [CrossentropySoftmax1HotWithBiasDx()(advec, admat, alvec)], [advec_val, admat_val, alvec_val], CrossentropySoftmax1HotWithBiasDx) def test_neg_idx(self): admat = matrix() advec = vector() alvec = lvector() rng = np.random.RandomState(utt.fetch_seed()) admat_val = rng.rand(10, 5).astype(config.floatX) admat_val /= admat_val.sum(axis=1).reshape(10, 1) advec_val = rng.rand(10).astype(config.floatX) alvec_val = rng.randint(low=0, high=5, size=10) alvec_val[1] = -1 out = CrossentropySoftmax1HotWithBiasDx()(advec, admat, alvec) f = theano.function([advec, admat, alvec], out) self.assertRaises(ValueError, f, advec_val, admat_val, alvec_val) class T_CrossentropySoftmaxArgmax1HotWithBias(utt.InferShapeTester): def setUp(self): super(T_CrossentropySoftmaxArgmax1HotWithBias, self).setUp() self.op = theano.tensor.nnet.crossentropy_softmax_argmax_1hot_with_bias def test0(self): n_classes = 5 n_samples = 3 # First test gradient when getting a gradient on the NLL output. def grad_on_nll_dtype(dtype): def grad_on_nll(x, b): y_idx = np.random.randint(low=0, high=n_classes, size=n_samples).astype(dtype) return self.op(x, b, y_idx=y_idx)[0] return grad_on_nll for dtype in ['uint8', 'int8', 'uint64', 'int64']: utt.verify_grad(grad_on_nll_dtype(dtype), [np.random.rand(n_samples, n_classes), np.random.rand(n_classes)]) # Then test gradient when getting a gradient on the softmax output. def grad_on_softmax(x, b): return self.op(x, b, y_idx=np.random.randint( low=0, high=n_classes, size=n_samples))[1] utt.verify_grad( grad_on_softmax, [np.random.rand(n_samples, n_classes), np.random.rand(n_classes)]) def test_infer_shape(self): admat = matrix() advec = vector() alvec = lvector() rng = np.random.RandomState(utt.fetch_seed()) admat_val = rng.rand(3, 5).astype(config.floatX) advec_val = rng.rand(5).astype(config.floatX) alvec_val = rng.randint(low=0, high=5, size=3) self._compile_and_check( [admat, advec, alvec], CrossentropySoftmaxArgmax1HotWithBias()(admat, advec, alvec), [admat_val, advec_val, alvec_val], CrossentropySoftmaxArgmax1HotWithBias) def test_neg_idx(self): admat = matrix() advec = vector() alvec = lvector() rng = np.random.RandomState(utt.fetch_seed()) admat_val = rng.rand(3, 5).astype(config.floatX) advec_val = rng.rand(5).astype(config.floatX) alvec_val = rng.randint(low=0, high=5, size=3) alvec_val[1] = -1 out = CrossentropySoftmaxArgmax1HotWithBias()(admat, advec, alvec) f = theano.function([admat, advec, alvec], out) self.assertRaises(ValueError, f, admat_val, advec_val, alvec_val) class T_prepend(utt.InferShapeTester): def test0(self): x = tensor.matrix('x') y = Prepend_scalar_constant_to_each_row(4.)(x) f = theano.function([x], y) m = np.random.rand(3, 5).astype(config.floatX) my = f(m) self.assertTrue(my.shape == (3, 6), my.shape) self.assertTrue(np.all(my[:, 0] == 4.0)) def test1(self): "basic functionality" x = tensor.matrix('x') y = Prepend_scalar_to_each_row()(5., x) f = theano.function([x], y) m = np.ones((3, 5), dtype="float32") my = f(m) self.assertTrue(my.shape == (3, 6)) self.assertTrue(np.all(my[:, 0] == 5.0)) def test_infer_shape(self): admat = matrix() adscal = scalar() rng = np.random.RandomState(utt.fetch_seed()) admat_val = rng.rand(3, 5).astype(config.floatX) adscal_val = np.asarray(rng.rand(), dtype=config.floatX).item() self._compile_and_check( [admat], [Prepend_scalar_constant_to_each_row(adscal_val)(admat)], [admat_val], Prepend_scalar_constant_to_each_row) self._compile_and_check( [adscal, admat], [Prepend_scalar_to_each_row()(adscal, admat)], [adscal_val, admat_val], Prepend_scalar_to_each_row) class T_CrossentropyCategorical1HotGrad(utt.InferShapeTester): def test_infer_shape(self): advec = vector() admat = matrix() alvec = lvector() rng = np.random.RandomState(utt.fetch_seed()) advec_val = rng.rand(3).astype(config.floatX) admat_val = rng.rand(3, 2).astype(config.floatX) alvec_val = [0, 1, 0] self._compile_and_check( [advec, admat, alvec], [CrossentropyCategorical1HotGrad()(advec, admat, alvec)], [advec_val, admat_val, alvec_val], CrossentropyCategorical1HotGrad) class T_CrossentropyCategorical1Hot(utt.InferShapeTester): def test_grad(self): x = tensor.matrix('x') one_of_n = tensor.lvector('one_of_n') op = crossentropy_categorical_1hot xe = op(x, one_of_n) f = theano.function([x, one_of_n], xe) x_val = np.asarray( [[.4, .6, .0], [.1, .8, .1]], dtype=config.floatX) xe_val = f(x_val, [0, 1]) assert np.allclose(xe_val, -np.log([.4, .8])) def oplike(x): return op(x, [0, 1]) tensor.verify_grad(oplike, [x_val], rng=np.random) def test_infer_shape(self): admat = matrix() alvec = lvector() rng = np.random.RandomState(utt.fetch_seed()) admat_val = rng.rand(3, 2).astype(config.floatX) alvec_val = [0, 1, 0] self._compile_and_check( [admat, alvec], [CrossentropyCategorical1Hot()(admat, alvec)], [admat_val, alvec_val], CrossentropyCategorical1Hot) def test_softmax_optimizations(self): x = tensor.matrix('x') one_of_n = tensor.lvector('one_of_n') op = crossentropy_categorical_1hot # xe = op(x, one_of_n) fgraph = gof.FunctionGraph( [x, one_of_n], [op(softmax_op(x), one_of_n)]) assert fgraph.outputs[0].owner.op == op theano.compile.mode.optdb.query( theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) assert (fgraph.outputs[0].owner.op == crossentropy_softmax_argmax_1hot_with_bias) def test_softmax_optimizations_vector(self): x = tensor.vector('x') one_of_n = tensor.lvector('one_of_n') op = crossentropy_categorical_1hot fgraph = gof.FunctionGraph( [x, one_of_n], [op(softmax_op(x), one_of_n)]) assert fgraph.outputs[0].owner.op == op theano.compile.mode.optdb.query( theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) assert (fgraph.outputs[0].owner.op == crossentropy_softmax_argmax_1hot_with_bias) def test_softmax_optimizations_w_bias(self): x = tensor.matrix('x') b = tensor.vector('b') one_of_n = tensor.lvector('one_of_n') op = crossentropy_categorical_1hot # xe = op(x, one_of_n) fgraph = gof.FunctionGraph( [x, b, one_of_n], [op(softmax_op(x + b), one_of_n)]) assert fgraph.outputs[0].owner.op == op # print 'BEFORE' # for node in fgraph.toposort(): # print node.op # print printing.pprint(node.outputs[0]) # print '----' theano.compile.mode.optdb.query( theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) # print 'AFTER' # for node in fgraph.toposort(): # print node.op # print printing.pprint(node.outputs[0]) # print '====' assert len(fgraph.toposort()) == 1 assert (fgraph.outputs[0].owner.op == crossentropy_softmax_argmax_1hot_with_bias) def test_softmax_optimizations_w_bias2(self): x = tensor.matrix('x') b = tensor.vector('b') c = tensor.vector('c') one_of_n = tensor.lvector('one_of_n') op = crossentropy_categorical_1hot fgraph = gof.FunctionGraph( [x, b, c, one_of_n], [op(softmax_op(T.add(x, b, c)), one_of_n)]) assert fgraph.outputs[0].owner.op == op # print 'BEFORE' # for node in fgraph.toposort(): # print node.op # print '----' theano.compile.mode.optdb.query( theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) # print 'AFTER' # for node in fgraph.toposort(): # print node.op # print '====' assert len(fgraph.toposort()) == 2 assert (fgraph.outputs[0].owner.op == crossentropy_softmax_argmax_1hot_with_bias) def test_softmax_optimizations_w_bias_vector(self): x = tensor.vector('x') b = tensor.vector('b') one_of_n = tensor.lvector('one_of_n') op = crossentropy_categorical_1hot fgraph = gof.FunctionGraph( [x, b, one_of_n], [op(softmax_op(x + b), one_of_n)]) assert fgraph.outputs[0].owner.op == op # print 'BEFORE' # for node in fgraph.toposort(): # print node.op # print printing.pprint(node.outputs[0]) # print '----' theano.compile.mode.optdb.query( theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) # print 'AFTER' # for node in fgraph.toposort(): # print node.op # print '====' assert len(fgraph.toposort()) == 2 assert (fgraph.outputs[0].owner.op == crossentropy_softmax_argmax_1hot_with_bias) def test_softmax_grad_optimizations(self): x = tensor.matrix('x') one_of_n = tensor.lvector('one_of_n') op = crossentropy_categorical_1hot xe = op(softmax_op(x), one_of_n) sum_xe = tensor.sum(xe) g_x = tensor.grad(sum_xe, x) fgraph = gof.FunctionGraph( [x, one_of_n], [g_x]) assert check_stack_trace( fgraph, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx, softmax_op]) # print 'BEFORE' # for node in fgraph.toposort(): # print node.op, node.inputs # print '----' theano.compile.mode.optdb.query( theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) # print 'AFTER' # for node in fgraph.toposort(): # print node.op, node.inputs has_cx1hot = False has_cx1hotdx = False has_softmax = False has_softmaxdx = False for node in fgraph.toposort(): if node.op == crossentropy_softmax_argmax_1hot_with_bias: has_cx1hot = True if node.op == crossentropy_softmax_1hot_with_bias_dx: has_cx1hotdx = True if node.op == softmax_op: has_softmax = True if node.op == softmax_grad: has_softmaxdx = True assert not has_cx1hot assert has_cx1hotdx assert has_softmax assert not has_softmaxdx def test_softmax_grad_optimizations_vector(self): x = tensor.vector('x') one_of_n = tensor.lvector('one_of_n') op = crossentropy_categorical_1hot xe = op(softmax_op(x), one_of_n) sum_xe = tensor.sum(xe) g_x = tensor.grad(sum_xe, x) fgraph = gof.FunctionGraph( [x, one_of_n], [g_x]) # print 'BEFORE' # for node in fgraph.toposort(): # print node.op, node.inputs # print '----' theano.compile.mode.optdb.query( theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) # print 'AFTER' # for node in fgraph.toposort(): # print node.op, node.inputs has_cx1hot = False has_cx1hotdx = False has_softmax = False has_softmaxdx = False for node in fgraph.toposort(): if node.op == crossentropy_softmax_argmax_1hot_with_bias: has_cx1hot = True if node.op == crossentropy_softmax_1hot_with_bias_dx: has_cx1hotdx = True if node.op == softmax_op: has_softmax = True if node.op == softmax_grad: has_softmaxdx = True assert not has_cx1hot assert has_cx1hotdx assert has_softmax assert not has_softmaxdx def test_get_rid_of_advanced_indexing_version_of_xent(self): verbose = 0 # TODO: add the optimization in FAST_COMPILE? # In the mean time, run it as 'FAST_RUN' instead mode = theano.compile.mode.get_default_mode() if mode == theano.compile.mode.get_mode('FAST_COMPILE'): mode = 'FAST_RUN' rng = np.random.RandomState(utt.fetch_seed()) x_val = rng.randn(3, 5).astype(config.floatX) b_val = rng.randn(5).astype(config.floatX) y_val = np.asarray([2, 4, 1]) x = T.matrix('x') b = T.vector('b') y = T.lvector('y') # Basic case expressions = [ T.sum( -T.log(softmax(x)[T.arange(y.shape[0]), y])), -T.sum(T.log(softmax(x)[T.arange(y.shape[0]), y])), -T.sum(T.log(softmax(x))[T.arange(y.shape[0]), y]), T.sum(-T.log(softmax(x))[T.arange(y.shape[0]), y])] for expr in expressions: # Verify the optimizer worked on the expressions f = theano.function([x, y], expr, mode=mode) # todo: only the first output of the op has a stack trace # assert check_stack_trace( # f, ops_to_check=crossentropy_softmax_argmax_1hot_with_bias) if verbose: theano.printing.debugprint(f) try: ops = [node.op for node in f.maker.fgraph.toposort()] assert len(ops) == 4 assert crossentropy_softmax_argmax_1hot_with_bias in ops assert not [1 for o in ops if isinstance(o, T.AdvancedSubtensor)] f(x_val, y_val) except Exception: theano.printing.debugprint(f) raise # Also verify the gradient wrt x g = theano.function([x, y], T.grad(expr, x), mode=mode) assert check_stack_trace( g, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx, softmax_op]) if verbose: theano.printing.debugprint(g) try: ops = [node.op for node in g.maker.fgraph.toposort()] assert len(ops) == 2 assert crossentropy_softmax_1hot_with_bias_dx in ops assert softmax_op in ops assert softmax_grad not in ops g(x_val, y_val) except Exception: theano.printing.debugprint(g) raise # Test that a biased softmax is optimized correctly bias_expressions = [ T.sum(-T.log(softmax(x + b)[T.arange(y.shape[0]), y])), -T.sum(T.log(softmax(b + x)[T.arange(y.shape[0]), y])), -T.sum(T.log(softmax(x + b))[T.arange(y.shape[0]), y]), T.sum(-T.log(softmax(b + x))[T.arange(y.shape[0]), y])] for expr in bias_expressions: f = theano.function([x, b, y], expr, mode=mode) # todo: only the first output of the op has a stack trace # assert check_stack_trace( # f, ops_to_check=crossentropy_softmax_argmax_1hot_with_bias) if verbose: theano.printing.debugprint(f) try: ops = [node.op for node in f.maker.fgraph.toposort()] assert len(ops) == 2 # [big_op, sum] assert crossentropy_softmax_argmax_1hot_with_bias in ops f(x_val, b_val, y_val) except Exception: theano.printing.debugprint(f) raise g = theano.function([x, b, y], T.grad(expr, x), mode=mode) assert check_stack_trace( g, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx, softmax_with_bias]) if verbose: theano.printing.debugprint(g) try: ops = [node.op for node in g.maker.fgraph.toposort()] assert len(ops) == 2 assert crossentropy_softmax_1hot_with_bias_dx in ops assert softmax_with_bias in ops assert softmax_grad not in ops g(x_val, b_val, y_val) except Exception: theano.printing.debugprint(g) raise # Test that using "mean" instead of sum works, too mean_expressions = [ T.mean(-T.log(softmax(x)[T.arange(y.shape[0]), y])), -T.mean(T.log(softmax(x)[T.arange(y.shape[0]), y])), -T.mean(T.log(softmax(x))[T.arange(y.shape[0]), y]), T.mean(-T.log(softmax(x))[T.arange(y.shape[0]), y])] for expr in mean_expressions: f = theano.function([x, y], expr, mode=mode) # todo: only the first output of the op has a stack trace # assert check_stack_trace( # f, ops_to_check=[crossentropy_softmax_argmax_1hot_with_bias]) if verbose: theano.printing.debugprint(f) try: ops = [node.op for node in f.maker.fgraph.toposort()] assert len(ops) == 6 assert crossentropy_softmax_argmax_1hot_with_bias in ops assert not [1 for o in ops if isinstance(o, T.AdvancedSubtensor)] f(x_val, y_val) except Exception: theano.printing.debugprint(f) raise g = theano.function([x, y], T.grad(expr, x), mode=mode) assert check_stack_trace( g, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx, softmax_op]) if verbose: theano.printing.debugprint(g) try: ops = [node.op for node in g.maker.fgraph.toposort()] assert len(ops) == 5 # there's an extra dimshuffle in there # but I can't think of a good rule to get rid of it assert crossentropy_softmax_1hot_with_bias_dx in ops assert softmax_op in ops assert softmax_grad not in ops g(x_val, y_val) except Exception: theano.printing.debugprint(g) raise mean_bias_expressions = [ T.mean(-T.log(softmax(x + b)[T.arange(y.shape[0]), y])), -T.mean(T.log(softmax(b + x)[T.arange(y.shape[0]), y])), -T.mean(T.log(softmax(x + b))[T.arange(y.shape[0]), y]), T.mean(-T.log(softmax(b + x))[T.arange(y.shape[0]), y])] for expr in mean_bias_expressions: f = theano.function([x, b, y], expr, mode=mode) # todo: only the first output of the op has a stack trace # assert check_stack_trace( # f, ops_to_check=crossentropy_softmax_argmax_1hot_with_bias) if verbose: theano.printing.debugprint(f) try: ops = [node.op for node in f.maker.fgraph.toposort()] assert len(ops) == 4 assert crossentropy_softmax_argmax_1hot_with_bias in ops assert not [1 for o in ops if isinstance(o, T.AdvancedSubtensor)] except Exception: theano.printing.debugprint(f) raise g = theano.function([x, b, y], T.grad(expr, x), mode=mode) assert check_stack_trace( g, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx, softmax_with_bias]) if verbose: theano.printing.debugprint(g) try: ops = [node.op for node in g.maker.fgraph.toposort()] assert len(ops) == 5 assert crossentropy_softmax_1hot_with_bias_dx in ops assert softmax_with_bias in ops assert softmax_grad not in ops g(x_val, b_val, y_val) except Exception: theano.printing.debugprint(g) raise def test_xent_thing_int32(self): verbose = 0 mode = theano.compile.mode.get_default_mode() if mode == theano.compile.mode.get_mode('FAST_COMPILE'): mode = 'FAST_RUN' rng = np.random.RandomState(utt.fetch_seed()) x_val = rng.randn(3, 5).astype(config.floatX) y_val = np.asarray([2, 4, 1], dtype='int64') x = T.matrix('x') y = T.lvector('y') yi = T.cast(y, 'int32') expressions = [ T.sum(-T.log(softmax(x)[T.arange(yi.shape[0]), yi])), -T.sum(T.log(softmax(x)[T.arange(yi.shape[0]), yi])), -T.sum(T.log(softmax(x))[T.arange(yi.shape[0]), yi]), T.sum(-T.log(softmax(x))[T.arange(yi.shape[0]), yi])] for expr in expressions: # Verify the optimizer worked on the expressions f = theano.function([x, y], expr, mode=mode) if verbose: theano.printing.debugprint(f) try: ops = [node.op for node in f.maker.fgraph.toposort()] assert len(ops) == 5 assert crossentropy_softmax_argmax_1hot_with_bias in ops assert not [1 for o in ops if isinstance(o, T.AdvancedSubtensor)] f(x_val, y_val) except Exception: theano.printing.debugprint(f) raise # Also verify the gradient wrt x g = theano.function([x, y], T.grad(expr, x), mode=mode) if verbose: theano.printing.debugprint(g) try: ops = [node.op for node in g.maker.fgraph.toposort()] assert len(ops) == 3 assert crossentropy_softmax_1hot_with_bias_dx in ops assert softmax_op in ops assert softmax_grad not in ops g(x_val, y_val) except Exception: theano.printing.debugprint(g) raise def test_optimize_xent_vector(self): verbose = 0 mode = theano.compile.mode.get_default_mode() if mode == theano.compile.mode.get_mode('FAST_COMPILE'): mode = 'FAST_RUN' rng = np.random.RandomState(utt.fetch_seed()) x_val = rng.randn(5).astype(config.floatX) y_val = np.asarray([2]) x = T.vector('x') y = T.lvector('y') # Test that a biased softmax is optimized correctly bias_expressions = [ T.sum(-T.log(softmax(x)[T.arange(y.shape[0]), y])), -T.sum(T.log(softmax(x)[T.arange(y.shape[0]), y]))] for expr in bias_expressions: f = theano.function([x, y], expr, mode=mode) if verbose: printing.debugprint(f) try: ops = [node.op for node in f.maker.fgraph.toposort()] assert len(ops) == 5 assert crossentropy_softmax_argmax_1hot_with_bias in ops assert not [1 for o in ops if isinstance(o, T.AdvancedSubtensor)] f(x_val, y_val) except Exception: theano.printing.debugprint(f) raise g = theano.function([x, y], T.grad(expr, x), mode=mode) if verbose: printing.debugprint(g) try: ops = [node.op for node in g.maker.fgraph.toposort()] assert len(ops) == 4 assert crossentropy_softmax_1hot_with_bias_dx in ops assert softmax_op in ops assert softmax_grad not in ops g(x_val, y_val) except Exception: theano.printing.debugprint(g) raise def test_optimize_xent_vector2(self): verbose = 0 mode = theano.compile.mode.get_default_mode() if mode == theano.compile.mode.get_mode('FAST_COMPILE'): mode = 'FAST_RUN' rng = np.random.RandomState(utt.fetch_seed()) x_val = rng.randn(5).astype(config.floatX) b_val = rng.randn(5).astype(config.floatX) y_val = np.asarray([2]) x = T.vector('x') b = T.vector('b') y = T.lvector('y') # Test that a biased softmax is optimized correctly bias_expressions = [ T.sum(-T.log(softmax(x + b)[T.arange(y.shape[0]), y])), -T.sum(T.log(softmax(b + x)[T.arange(y.shape[0]), y])), -T.sum(T.log(softmax(x + b))[T.arange(y.shape[0]), y]), T.sum(-T.log(softmax(b + x))[T.arange(y.shape[0]), y])] for expr in bias_expressions: f = theano.function([x, b, y], expr, mode=mode) if verbose: printing.debugprint(f) try: ops = [node.op for node in f.maker.fgraph.toposort()] # [big_op, sum, dim_shuffle] assert len(ops) == 3 assert crossentropy_softmax_argmax_1hot_with_bias in ops assert not [1 for o in ops if isinstance(o, T.AdvancedSubtensor)] f(x_val, b_val, y_val) except Exception: theano.printing.debugprint(f) raise backup = config.warn.sum_div_dimshuffle_bug config.warn.sum_div_dimshuffle_bug = False try: g = theano.function([x, b, y], T.grad(expr, x), mode=mode) finally: config.warn.sum_div_dimshuffle_bug = backup if verbose: printing.debugprint(g) try: ops = [node.op for node in g.maker.fgraph.toposort()] assert len(ops) <= 6 assert crossentropy_softmax_1hot_with_bias_dx in ops assert softmax_with_bias in ops assert softmax_grad not in ops g(x_val, b_val, y_val) except Exception: theano.printing.debugprint(g) raise def test_optimize_xent_vector3(self): # Same as test_optimize_xent_vector2, but y is the result of # a "flatten", and it used to make the constant-folding # of arange(y.shape[0]) happen before the xent optimization verbose = 0 mode = theano.compile.mode.get_default_mode() if mode == theano.compile.mode.get_mode('FAST_COMPILE'): mode = 'FAST_RUN' rng = np.random.RandomState(utt.fetch_seed()) x_val = rng.randn(5).astype(config.floatX) b_val = rng.randn(5).astype(config.floatX) y_val = np.asarray([2]) x = T.vector('x') b = T.vector('b') y_ = T.lvector('y_') y = y_.flatten() # Test that a biased softmax is optimized correctly bias_expressions = [ T.sum(-T.log(softmax(x + b)[T.arange(y.shape[0]), y])), -T.sum(T.log(softmax(b + x)[T.arange(y.shape[0]), y])), -T.sum(T.log(softmax(x + b))[T.arange(y.shape[0]), y]), T.sum(-T.log(softmax(b + x))[T.arange(y.shape[0]), y])] for expr in bias_expressions: f = theano.function([x, b, y_], expr, mode=mode) if verbose: printing.debugprint(f) try: ops = [node.op for node in f.maker.fgraph.toposort()] # [big_op, sum, dim_shuffle, flatten] assert len(ops) <= 4 assert crossentropy_softmax_argmax_1hot_with_bias in ops assert not [1 for o in ops if isinstance(o, T.AdvancedSubtensor)] f(x_val, b_val, y_val) except Exception: theano.printing.debugprint(f) raise backup = config.warn.sum_div_dimshuffle_bug config.warn.sum_div_dimshuffle_bug = False try: g = theano.function([x, b, y], T.grad(expr, x), mode=mode) finally: config.warn.sum_div_dimshuffle_bug = backup if verbose: printing.debugprint(g) try: ops = [node.op for node in g.maker.fgraph.toposort()] assert len(ops) <= 6 assert crossentropy_softmax_1hot_with_bias_dx in ops assert softmax_with_bias in ops assert softmax_grad not in ops g(x_val, b_val, y_val) except Exception: theano.printing.debugprint(g) raise def test_optimize_xent_vector4(self): # Same as test_optimize_xent_vector2, but y is the result of # a "specify_shape" that indicates its length is 1, so the # constant-folding of arange(y.shape[0]) happen before the xent # optimization verbose = 0 mode = theano.compile.mode.get_default_mode() if mode == theano.compile.mode.get_mode('FAST_COMPILE'): mode = 'FAST_RUN' rng = np.random.RandomState(utt.fetch_seed()) x_val = rng.randn(5).astype(config.floatX) b_val = rng.randn(5).astype(config.floatX) y_val = np.asarray([2]) x = T.vector('x') b = T.vector('b') y_ = T.lvector('y_') y = T.specify_shape(y_, (1,)) # Test that a biased softmax is optimized correctly bias_expressions = [ T.sum(-T.log(softmax(x + b)[T.arange(y.shape[0]), y])), -T.sum(T.log(softmax(b + x)[T.arange(y.shape[0]), y])), -T.sum(T.log(softmax(x + b))[T.arange(y.shape[0]), y]), T.sum(-T.log(softmax(b + x))[T.arange(y.shape[0]), y])] for expr in bias_expressions: f = theano.function([x, b, y_], expr, mode=mode) if verbose: printing.debugprint(f) try: ops = [node.op for node in f.maker.fgraph.toposort()] # [big_op, sum, dim_shuffle, specify_shape] assert len(ops) <= 4 assert crossentropy_softmax_argmax_1hot_with_bias in ops assert not [1 for o in ops if isinstance(o, T.AdvancedSubtensor)] f(x_val, b_val, y_val) except Exception: theano.printing.debugprint(f) raise backup = config.warn.sum_div_dimshuffle_bug config.warn.sum_div_dimshuffle_bug = False try: g = theano.function([x, b, y], T.grad(expr, x), mode=mode) finally: config.warn.sum_div_dimshuffle_bug = backup if verbose: printing.debugprint(g) try: ops = [node.op for node in g.maker.fgraph.toposort()] assert len(ops) <= 6 assert crossentropy_softmax_1hot_with_bias_dx in ops assert softmax_with_bias in ops assert softmax_grad not in ops g(x_val, b_val, y_val) except Exception: theano.printing.debugprint(g) raise def test_crossentropy_softmax_1hot_with_bias_dxcale_cost(self): # TODO: add the optimization in FAST_COMPILE? # In the mean time, run it as 'FAST_RUN' instead mode = theano.compile.mode.get_default_mode() if mode == theano.compile.mode.get_mode('FAST_COMPILE'): mode = 'FAST_RUN' rng = np.random.RandomState(utt.fetch_seed()) x_val = rng.randn(3, 5).astype(config.floatX) y_val = np.asarray([2, 4, 1]) x = T.matrix('x') y = T.lvector('y') a = T.scalar('a') def validate_fn_graph(func): # The graph of the function should not have softmax anymore has_cx1hot = False has_softmax = False for node in func.maker.fgraph.toposort(): if node.op == crossentropy_softmax_argmax_1hot_with_bias: has_cx1hot = True if node.op == softmax_op: has_softmax = True assert has_cx1hot assert not has_softmax def validate_grad_graph(func): # The graph of the gradient should not have softmaxgrad anymore has_cx1hotdx = False has_softmax = False has_softmaxdx = False for node in func.maker.fgraph.toposort(): if node.op == crossentropy_softmax_1hot_with_bias_dx: has_cx1hotdx = True if node.op == softmax_op: has_softmax = True if node.op == softmax_grad: has_softmaxdx = True assert has_cx1hotdx assert has_softmax assert not has_softmaxdx # Cases to test expressions = [ a * T.sum(-T.log(softmax(x)[T.arange(y.shape[0]), y])), -a * T.sum(T.log(softmax(x)[T.arange(y.shape[0]), y])), a * (-T.sum(T.log(softmax(x)[T.arange(y.shape[0]), y]))), a * T.sum(T.log(softmax(x)[T.arange(y.shape[0]), y])), a * T.sum(-T.log(softmax(x))[T.arange(y.shape[0]), y]), -a * T.sum(T.log(softmax(x))[T.arange(y.shape[0]), y]), a * (-T.sum(T.log(softmax(x))[T.arange(y.shape[0]), y])), a * T.sum(T.log(softmax(x))[T.arange(y.shape[0]), y]), a * T.mean(-T.log(softmax(x)[T.arange(y.shape[0]), y])), -a * T.mean(T.log(softmax(x)[T.arange(y.shape[0]), y])), a * (-T.mean(T.log(softmax(x)[T.arange(y.shape[0]), y]))), a * T.mean(T.log(softmax(x)[T.arange(y.shape[0]), y])), a * T.mean(-T.log(softmax(x))[T.arange(y.shape[0]), y]), -a * T.mean(T.log(softmax(x))[T.arange(y.shape[0]), y]), a * (-T.mean(T.log(softmax(x))[T.arange(y.shape[0]), y])), a * T.mean(T.log(softmax(x))[T.arange(y.shape[0]), y]), ] for expr in expressions: # Verify the optimizer worked on the expressions f = theano.function([x, y, a], expr, mode=mode) try: assert 5 <= len(f.maker.fgraph.toposort()) <= 10 validate_fn_graph(f) f(x_val, y_val, 0.1) except Exception: theano.printing.debugprint(f) raise # Verify the gradient wrt x g = theano.function([x, y, a], T.grad(expr, x), mode=mode) try: assert 3 <= len(g.maker.fgraph.toposort()) <= 6 validate_grad_graph(g) g(x_val, y_val, 0.1) except Exception: theano.printing.debugprint(g) raise # Verify the gradient when providing output gradient h = theano.function( [x, y, a], T.grad(expr, x, known_grads={expr: a * x.sum()}), mode=mode) try: assert 6 <= len(h.maker.fgraph.toposort()) <= 8 validate_grad_graph(h) h(x_val, y_val, 0.1) except Exception: theano.printing.debugprint(h) raise def test_argmax_pushdown(): x = tensor.matrix() for sm in [softmax_graph, softmax_op]: # test that the max_and_argmax is pushed down if the max is not used out = tensor.max_and_argmax( sm(tensor.exp(tensor.tanh(sigmoid(x)))), axis=-1)[1] fgraph = gof.FunctionGraph( [x], [out]) theano.compile.mode.optdb.query( theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) # print 'AFTER' # for node in fgraph.toposort(): # print node.op assert len(fgraph.toposort()) == 1 assert isinstance(fgraph.toposort()[0].op, tensor.basic.Argmax) assert check_stack_trace( fgraph, ops_to_check=tensor.basic.Argmax) x = tensor.matrix() # test that the max_and_argmax is not pushed down if the max is used out = tensor.max_and_argmax( sm(tensor.exp(tensor.tanh(sigmoid(x)))), axis=-1)[0] fgraph = gof.FunctionGraph( [x], [out]) assert hasattr(fgraph.outputs[0].tag, 'trace') backup = config.warn.argmax_pushdown_bug config.warn.argmax_pushdown_bug = False try: theano.compile.mode.optdb.query( theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) finally: config.warn.argmax_pushdown_bug = backup # print 'AFTER' # for node in fgraph.toposort(): # print node.op assert len(fgraph.toposort()) == 3 assert isinstance(fgraph.toposort()[0].op, tensor.Elemwise) assert isinstance(fgraph.toposort()[1].op, Softmax) assert isinstance(fgraph.toposort()[2].op, tensor.CAReduce) assert isinstance(fgraph.toposort()[2].op.scalar_op, theano.scalar.Maximum) def test_argmax_pushdown_bias(): x = tensor.matrix() b = tensor.vector() out = tensor.argmax(softmax_with_bias(x, b), axis=-1) fgraph = gof.FunctionGraph( [x, b], [out]) theano.compile.mode.optdb.query( theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) # print 'AFTER' # for node in fgraph.toposort(): # print node.op types_to_check = (tensor.DimShuffle, tensor.Elemwise, tensor.Argmax) assert len(fgraph.toposort()) == 3 for i, type in enumerate(types_to_check): assert isinstance(fgraph.toposort()[i].op, type) assert check_stack_trace(fgraph, ops_to_check=types_to_check) x = tensor.matrix() b = tensor.vector() out = tensor.max_and_argmax(softmax_with_bias(x, b), axis=-1)[0] fgraph = gof.FunctionGraph( [x, b], [out]) backup = config.warn.argmax_pushdown_bug config.warn.argmax_pushdown_bug = False try: theano.compile.mode.optdb.query( theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) finally: config.warn.argmax_pushdown_bug = backup # print 'AFTER' # for node in fgraph.toposort(): # print node.op assert len(fgraph.toposort()) == 2 assert isinstance(fgraph.toposort()[0].op, SoftmaxWithBias) assert isinstance(fgraph.toposort()[1].op, tensor.CAReduce) assert isinstance(fgraph.toposort()[1].op.scalar_op, theano.scalar.Maximum) assert check_stack_trace( fgraph, ops_to_check=(SoftmaxWithBias, tensor.CAReduce)) def test_asymptotic_32(): # This test makes sure that our functions behave sensibly when # huge values are present # TODO: consider adding the optimization of crossentropy into the current # mode for the purpose of running this test for dtype in 'float32', 'float64': if dtype == 'float32': x = tensor.fmatrix() x2 = tensor.fvector() else: x = tensor.dmatrix() x2 = tensor.dvector() y = tensor.lvector() c = categorical_crossentropy(softmax(x + x2), y) f = theano.function([x, y, x2], [c.sum(), tensor.grad(c.sum(), x)], mode='FAST_RUN') xval = np.zeros((5, 5), dtype=dtype).astype(dtype) x2val = np.zeros(5, dtype=xval.dtype).astype(dtype) for i in xrange(100): cval, gxval = f(xval, np.arange(5), x2val) xval -= 100.3 * gxval # print cval, gxval assert cval == 0 # no problem going to zero error # what about when x gets really big? xval = np.zeros((5, 5), dtype=dtype) x2val = np.zeros(5, dtype=xval.dtype) for i in xrange(100): cval, gxval = f(xval, np.arange(5), x2val) xval += 100000.3 * gxval # print cval, gxval assert cval > 61750000 assert gxval[0, 0] == -1.0 assert gxval[0, 1] == 0.25 class Test_softmax_opt: # Test that expressions of softmax in terms of exponentiated things # divided by row sums are replaced by softmax expressions. # # Softmax_grad isn't that interesting as an Op, but it has the signature # we look for when trying to insert CrossEntropySoftmax... grad. So, for # now, we add softmax_grad to graphs. In the future, we may modify the # CrossEntropySoftmax...grad to look for the more basic pattern. # def setUp(self): utt.seed_rng() self.rng = np.random.RandomState(utt.fetch_seed()) self.mode = theano.compile.mode.get_default_mode() self.mode = self.mode.including('canonicalize') def test_basic(self): c = T.matrix() p_y = T.exp(c) / T.exp(c).sum(axis=1).dimshuffle(0, 'x') # test that function contains softmax and no div. f = theano.function([c], p_y, mode=self.mode) assert check_stack_trace(f, ops_to_check=softmax_op) f_ops = [n.op for n in f.maker.fgraph.toposort()] # print '--- f =' # printing.debugprint(f) # print '===' assert len(f_ops) == 1 assert softmax_op in f_ops f(self.rng.rand(3, 4).astype(config.floatX)) def test_basic_keepdims(self): c = T.matrix() p_y = T.exp(c) / T.exp(c).sum(axis=1, keepdims=True) # test that function contains softmax and no div. f = theano.function([c], p_y, mode=self.mode) assert check_stack_trace(f, ops_to_check=softmax_op) f_ops = [n.op for n in f.maker.fgraph.toposort()] # print '--- f =' # printing.debugprint(f) # print '===' assert len(f_ops) == 1 assert softmax_op in f_ops f(self.rng.rand(3, 4).astype(config.floatX)) def test_grad(self): c = T.matrix() p_y = T.exp(c) / T.exp(c).sum(axis=1).dimshuffle(0, 'x') # test that function contains softmax and softmaxgrad w = T.matrix() backup = config.warn.sum_div_dimshuffle_bug config.warn.sum_div_dimshuffle_bug = False try: g = theano.function([c, w], T.grad((p_y * w).sum(), c)) finally: config.warn.sum_div_dimshuffle_bug = backup g_ops = [n.op for n in g.maker.fgraph.toposort()] # print '--- g =' # printing.debugprint(g) # print '===' raise SkipTest('Optimization not enabled for the moment') assert len(g_ops) == 2 assert softmax_op in g_ops assert softmax_grad in g_ops g(self.rng.rand(3, 4), self.rng.uniform(.5, 1, (3, 4))) def test_transpose_basic(self): # this should be a transposed softmax c = T.matrix() p_y = T.exp(c) / T.exp(c).sum(axis=0) # test that function contains softmax and no div. theano.function([c], p_y) # printing.debugprint(f) # test that function contains softmax and no div. backup = config.warn.sum_div_dimshuffle_bug config.warn.sum_div_dimshuffle_bug = False try: theano.function([c], T.grad(p_y.sum(), c)) finally: config.warn.sum_div_dimshuffle_bug = backup # printing.debugprint(g) raise SkipTest('Optimization not enabled for the moment') def test_1d_basic(self): # this should be a softmax, but of a one-row matrix c = T.vector() p_y = T.exp(c) / T.exp(c).sum() # test that function contains softmax and no div. theano.function([c], p_y) # printing.debugprint(f) # test that function contains softmax and no div. backup = config.warn.sum_div_dimshuffle_bug config.warn.sum_div_dimshuffle_bug = False try: theano.function([c], T.grad(p_y.sum(), c)) finally: config.warn.sum_div_dimshuffle_bug = backup # printing.debugprint(g) raise SkipTest('Optimization not enabled for the moment') # REPEAT 3 CASES in presence of log(softmax) with the advanced indexing # etc. def test_softmax_graph(): rng = np.random.RandomState(utt.fetch_seed()) x = theano.shared(rng.normal(size=(3, 4))) def f(inputs): y = softmax_graph(x) return theano.grad(None, x, known_grads={y: inputs}) utt.verify_grad(f, [rng.rand(3, 4)]) def test_grad_softmax_grad(): rng = np.random.RandomState(utt.fetch_seed()) x = theano.shared(rng.normal(size=(3, 4))) def f(inputs): y = softmax_op(x) return theano.grad(None, x, known_grads={y: inputs}) utt.verify_grad(f, [rng.rand(3, 4)]) def test_stabilize_log_softmax(): mode = theano.compile.mode.get_default_mode() mode = mode.including('local_log_softmax', 'specialize') x = matrix() y = softmax(x) z = theano.tensor.log(y) f = theano.function([x], z, mode=mode) assert check_stack_trace(f, ops_to_check='all') # check that the softmax has been optimized out for node in f.maker.fgraph.toposort(): assert not isinstance(node.op, y.owner.op.__class__) # call the function so debug mode can verify the optimized # version matches the unoptimized version rng = np.random.RandomState([2012, 8, 22]) f(np.cast[config.floatX](rng.randn(2, 3))) def test_relu(): x = matrix('x') seed = theano.tests.unittest_tools.fetch_seed() rng = np.random.RandomState(seed) X = rng.randn(20, 30).astype(config.floatX) # test the base case, without custom alpha value y = relu(x).eval({x: X}) assert np.allclose(y, np.maximum(X, 0)) # test for different constant alpha values (also outside of [0, 1]) for alpha in 0, 0.3, 1, 2, -0.3, -1, -2: y = relu(x, alpha).eval({x: X}) assert np.allclose(y, np.where(X > 0, X, alpha * X)) # test for variable alpha (scalar, vector and matrix) for alpha in scalar(), vector(), matrix(): # create value for alpha (correct ndim and broadcastable against X) A = np.array(rng.randn(*X.shape[::-1][:alpha.ndim][::-1]), dtype=config.floatX) y = relu(x, alpha).eval({x: X, alpha: A}) assert np.allclose(y, np.where(X > 0, X, A * X), rtol=3e-5) # test that for alpha of ndarray don't cause upcast. x = matrix('x', dtype='float32') rng = np.random.RandomState(seed) X = rng.randn(20, 30).astype('float32') alpha = np.asarray(.123, dtype='float32') y = relu(x, alpha).eval({x: X}) assert np.allclose(y, np.where(X > 0, X, alpha * X)) assert y.dtype == 'float32' def test_h_softmax(): # Tests the output dimensions of the h_softmax when a target is provided or # not. ############# # Config ############# input_size = 4 batch_size = 2 h_softmax_level1_size = 5 h_softmax_level2_size = 3 output_size = h_softmax_level1_size * h_softmax_level2_size ############# # Initialize shared variables ############# floatX = theano.config.floatX shared = theano.shared # First level of h_softmax W1 = np.asarray(np.random.normal( size=(input_size, h_softmax_level1_size)), dtype=floatX) W1 = shared(W1) b1 = shared(np.asarray(np.zeros((h_softmax_level1_size,)), dtype=floatX)) # Second level of h_softmax W2 = np.asarray(np.random.normal( size=(h_softmax_level1_size, input_size, h_softmax_level2_size)), dtype=floatX) W2 = shared(W2) b2 = shared( np.asarray(np.zeros((h_softmax_level1_size, h_softmax_level2_size)), dtype=floatX)) ############# # Build graph ############# x = tensor.matrix('x') y = tensor.ivector('y') # This only computes the output corresponding to the target y_hat_tg = h_softmax(x, batch_size, output_size, h_softmax_level1_size, h_softmax_level2_size, W1, b1, W2, b2, y) # This computes all the outputs y_hat_all = h_softmax(x, batch_size, output_size, h_softmax_level1_size, h_softmax_level2_size, W1, b1, W2, b2) ############# # Compile functions ############# fun_output_tg = theano.function([x, y], y_hat_tg) fun_output = theano.function([x], y_hat_all) ############# # Test ############# x_mat = np.random.normal(size=(batch_size, input_size)).astype(floatX) y_mat = np.random.randint(0, output_size, batch_size).astype('int32') tg_output = fun_output_tg(x_mat, y_mat) all_outputs = fun_output(x_mat) assert(tg_output.shape == (batch_size,)) assert(all_outputs.shape == (batch_size, output_size)) # Verifies that the outputs computed by fun_output_tg are the same as those # computed by fun_output. utt.assert_allclose( all_outputs[np.arange(0, batch_size), y_mat], tg_output) def test_elu(): x = matrix('x') seed = theano.tests.unittest_tools.fetch_seed() rng = np.random.RandomState(seed) X = rng.randn(20, 30).astype(config.floatX) # test the base case, without custom alpha value y = elu(x).eval({x: X}) utt.assert_allclose(y, np.where(X > 0, X, np.exp(X) - 1)) # test for different constant alpha values for alpha in 1.5, 2, -1, -1.5, -2: y = elu(x, alpha).eval({x: X}) utt.assert_allclose(y, np.where(X > 0, X, alpha * (np.exp(X) - 1))) def test_selu(): alpha = 1.6732632423543772848170429916717 scale = 1.0507009873554804934193349852946 x = matrix('x') seed = theano.tests.unittest_tools.fetch_seed() rng = np.random.RandomState(seed) X = rng.randn(20, 30).astype(config.floatX) y = selu(x).eval({x: X}) utt.assert_allclose(y, np.where(X > 0, scale * X, scale * alpha * (np.exp(X) - 1))) def test_binary_crossentropy_reshape(): # Reported as https://github.com/Theano/Theano/issues/4086 a = tensor.tensor4('a') for c in (binary_crossentropy(sigmoid(a.reshape((-1, 1))), 1).sum(), binary_crossentropy(sigmoid(a).reshape((-1, 1)), 1).sum()): ga = theano.grad(c, a) # This only works when "specialize" options are included mode = theano.compile.get_default_mode().including('fast_run') fga = theano.function([a], ga, mode=mode) utt.assert_allclose(fga(np.array([[[[30.]]]], dtype=config.floatX)), np.zeros((1, 1, 1, 1), dtype=config.floatX)) SoftsignTester = makeBroadcastTester( op=softsign, expected=upcast_int8_nfunc(lambda inputs: check_floatX( inputs, inputs / (1.0 + np.fabs(inputs)))), good=_good_broadcast_unary_normal_float_no_complex, name='SoftsignTester', ) class T_sigmoid_binary_crossentropy(unittest.TestCase): def setUp(self): utt.seed_rng() def _get_test_inputs(self, n=50): pred, target = np.random.randn(2, n).astype(config.floatX) # apply sigmoid to target, but not pred return [pred, 1 / (1 + np.exp(-target))] def test_matches_binary_crossentropy(self): # Test sigmoid_binary_crossentropy(p, t) == # binary_crossentropy(sigmoid(p), t). pred, target = inputs = tensor.vectors('pt') reference_val = binary_crossentropy(sigmoid(pred), target) f_reference = theano.function(inputs, reference_val) test_val = sigmoid_binary_crossentropy(pred, target) f_test = theano.function(inputs, test_val) test_inputs = self._get_test_inputs() utt.assert_allclose(f_reference(*test_inputs), f_test(*test_inputs)) def test_grad(self): utt.verify_grad(sigmoid_binary_crossentropy, self._get_test_inputs()) def test_confusion_matrix(): # Defining numpy implementation of confusion matrix def numpy_conf_mat(actual, pred): order = np.union1d(actual, pred) colA = np.matrix(actual).T colP = np.matrix(pred).T oneHotA = colA.__eq__(order).astype('int64') oneHotP = colP.__eq__(order).astype('int64') conf_mat = np.dot(oneHotA.T, oneHotP) conf_mat = np.asarray(conf_mat) return [conf_mat, order] x = tensor.vector() y = tensor.vector() f = theano.function([x, y], confusion_matrix(x, y)) list_inputs = [[[0, 1, 2, 1, 0], [0, 0, 2, 1, 2]], [[2, 0, 2, 2, 0, 1], [0, 0, 2, 2, 0, 2]]] for case in list_inputs: a = np.asarray(case[0]) b = np.asarray(case[1]) out_exp = numpy_conf_mat(a, b) outs = f(case[0], case[1]) for exp, out in zip(out_exp, outs): utt.assert_allclose(exp, out) Theano-1.0.0/theano/tensor/nnet/tests/test_opt.py00006000163562€ÿÿÿþ0000000333013203335532023377 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import theano from theano import tensor from theano.tests.unittest_tools import assertFailure_fast from theano.gof.opt import check_stack_trace from theano.tensor.nnet.blocksparse import ( sparse_block_dot, sparse_block_gemv_inplace, sparse_block_outer_inplace, sparse_block_gemv, sparse_block_outer) def test_blocksparse_inplace_gemv_opt(): b = tensor.fmatrix() W = tensor.ftensor4() h = tensor.ftensor3() iIdx = tensor.lmatrix() oIdx = tensor.lmatrix() o = sparse_block_dot(W, h, iIdx, b, oIdx) f = theano.function([W, h, iIdx, b, oIdx], o) if theano.config.mode == "FAST_COMPILE": assert not f.maker.fgraph.toposort()[-1].op.inplace assert check_stack_trace(f, ops_to_check=[sparse_block_gemv]) else: assert f.maker.fgraph.toposort()[-1].op.inplace assert check_stack_trace(f, ops_to_check=[sparse_block_gemv_inplace]) if theano.config.mode != 'FAST_COMPILE': test_blocksparse_inplace_gemv_opt = assertFailure_fast(test_blocksparse_inplace_gemv_opt) def test_blocksparse_inplace_outer_opt(): b = tensor.fmatrix() W = tensor.ftensor4() h = tensor.ftensor3() iIdx = tensor.lmatrix() oIdx = tensor.lmatrix() o = sparse_block_dot(W, h, iIdx, b, oIdx) f = theano.function([W, h, iIdx, b, oIdx], [o, tensor.grad(o.sum(), wrt=W)]) if theano.config.mode == "FAST_COMPILE": assert not f.maker.fgraph.toposort()[-1].op.inplace assert check_stack_trace(f, ops_to_check=sparse_block_outer) else: assert f.maker.fgraph.toposort()[-1].op.inplace assert check_stack_trace(f, ops_to_check=sparse_block_outer_inplace) Theano-1.0.0/theano/tensor/nnet/tests/test_blocksparse.py00006000163562€ÿÿÿþ0000002465713203335532025124 0ustar boccoset00000000000000""" Tests for block sparse dot """ from __future__ import absolute_import, print_function, division import numpy as np from numpy.random import randn import theano from theano import tensor import theano.tests.unittest_tools as utt from theano.tensor.nnet.blocksparse import ( sparse_block_dot, sparse_block_gemv, sparse_block_outer, SparseBlockGemv, SparseBlockOuter) class BlockSparse_Gemv_and_Outer(utt.InferShapeTester): def setUp(self): utt.seed_rng() mode = None if theano.config.mode == "FAST_COMPILE": mode = "FAST_RUN" self.mode = theano.compile.get_mode(mode).excluding( 'constant_folding' ) self.gemv_op = sparse_block_gemv self.outer_op = sparse_block_outer self.gemv_class = SparseBlockGemv self.outer_class = SparseBlockOuter @staticmethod def gemv_data(): nInputBlock = 8 nOutputBlock = 7 inputSize = 6 outputSize = 5 inputWindowSize = 4 outputWindowSize = 3 batchSize = 2 input = randn(batchSize, inputWindowSize, inputSize).astype('float32') permutation = np.random.permutation inputIndice = np.vstack(permutation(nInputBlock)[:inputWindowSize] for _ in range(batchSize)).astype('int32') outputIndice = np.vstack( permutation(nOutputBlock)[:outputWindowSize] for _ in range(batchSize)).astype('int32') weight = randn(nInputBlock, nOutputBlock, inputSize, outputSize).astype('float32') bias = randn(nOutputBlock, outputSize).astype('float32') return weight, input, inputIndice, bias, outputIndice @staticmethod def outer_data(): nInputBlock = 8 nOutputBlock = 7 xSize = 6 ySize = 5 xWindowSize = 4 yWindowSize = 3 batchSize = 2 o = randn(nInputBlock, nOutputBlock, xSize, ySize).astype('float32') x = randn(batchSize, xWindowSize, xSize).astype('float32') y = randn(batchSize, yWindowSize, ySize).astype('float32') randint = np.random.randint xIdx = np.vstack(randint(0, nInputBlock, size=xWindowSize) for _ in range(batchSize)).astype('int32') yIdx = np.vstack(randint(0, nOutputBlock, size=yWindowSize) for _ in range(batchSize)).astype('int32') return o, x, y, xIdx, yIdx @staticmethod def gemv_numpy(o, W, h, iIdx, oIdx): for b in range(o.shape[0]): for j in range(o.shape[1]): outputIdx = oIdx[b, j] for i in range(h.shape[1]): inputIdx = iIdx[b, i] w = W[inputIdx, outputIdx] o[b, j, :] += np.dot(h[b, i], w) return o @staticmethod def gemv_numpy2(o, W, h, iIdx, oIdx): """ Other implementation """ from numpy import ix_ for b in range(o.shape[0]): w = W[ix_(iIdx[b], oIdx[b])].swapaxes(1, 2) w = w.reshape((w.shape[0] * w.shape[1], w.shape[2] * w.shape[3])) o[b] += np.dot(h[b].ravel(), w).reshape(o.shape[1:]) return o @staticmethod def gemv_numpy3(o, W, h, iIdx, oIdx): """ Other implementation """ from numpy import ix_ for b in range(o.shape[0]): w = W[ix_(iIdx[b], oIdx[b])] # The next three lines do the same operation. The last one is the # fastest # o[b] += (h[b][:, None, :, None] * w).sum(axis=(0, 2)) # o[b] += np.tensordot(h[b], w, [(0,1),(0,2)]) o[b] += np.einsum('ik,ijkl', h[b], w) return o @staticmethod def outer_numpy(o, x, y, xIdx, yIdx): for b in range(x.shape[0]): for i in range(xIdx.shape[1]): for j in range(yIdx.shape[1]): o[xIdx[b, i], yIdx[b, j]] += np.outer(x[b, i, :], y[b, j, :]) return o def test_sparseblockdot(self): # Compares the numpy version of sparseblockgemv to sparse_block_dot. b = tensor.fmatrix() W = tensor.ftensor4() h = tensor.ftensor3() iIdx = tensor.imatrix() oIdx = tensor.imatrix() o = sparse_block_dot(W, h, iIdx, b, oIdx) f = theano.function([W, h, iIdx, b, oIdx], o, mode=self.mode) W_val, h_val, iIdx_val, b_val, oIdx_val = \ BlockSparse_Gemv_and_Outer.gemv_data() th_out = f(W_val, h_val, iIdx_val, b_val, oIdx_val) ref_out = BlockSparse_Gemv_and_Outer.gemv_numpy( b_val.take(oIdx_val, axis=0), W_val, h_val, iIdx_val, oIdx_val) utt.assert_allclose(ref_out, th_out) def test_sparseblockgemv(self): # Compares the numpy and theano versions of sparseblockgemv. b = tensor.fmatrix() W = tensor.ftensor4() h = tensor.ftensor3() iIdx = tensor.imatrix() oIdx = tensor.imatrix() o = self.gemv_op(b.take(oIdx, axis=0), W, h, iIdx, oIdx) f = theano.function([W, h, iIdx, b, oIdx], o, mode=self.mode) W_val, h_val, iIdx_val, b_val, oIdx_val = \ BlockSparse_Gemv_and_Outer.gemv_data() th_out = f(W_val, h_val, iIdx_val, b_val, oIdx_val) ref_out = BlockSparse_Gemv_and_Outer.gemv_numpy( b_val.take(oIdx_val, axis=0), W_val, h_val, iIdx_val, oIdx_val) utt.assert_allclose(ref_out, th_out) def test_sparseblockgemvF(self): # Test the fortan order for W (which can happen in the grad for some # graphs). b = tensor.fmatrix() W = tensor.ftensor4() h = tensor.ftensor3() iIdx = tensor.imatrix() oIdx = tensor.imatrix() o = self.gemv_op(b.take(oIdx, axis=0), tensor.DimShuffle((False, False, False, False), (0, 1, 3, 2)) (tensor.as_tensor_variable(W)), h, iIdx, oIdx) f = theano.function([W, h, iIdx, b, oIdx], o, mode=self.mode) W_val, h_val, iIdx_val, b_val, oIdx_val = \ BlockSparse_Gemv_and_Outer.gemv_data() th_out = f(np.swapaxes(W_val, 2, 3), h_val, iIdx_val, b_val, oIdx_val) ref_out = BlockSparse_Gemv_and_Outer.gemv_numpy( b_val.take(oIdx_val, axis=0), W_val, h_val, iIdx_val, oIdx_val) utt.assert_allclose(ref_out, th_out) def test_sparseblockgemv_grad(self): W_val, h_val, iIdx_val, b_val, oIdx_val = \ BlockSparse_Gemv_and_Outer.gemv_data() iIdx = theano.tensor.constant(iIdx_val) oIdx = theano.tensor.constant(oIdx_val) def metaop(b, h, W): return sparse_block_dot(W, h, iIdx, b, oIdx) def op(b, h, W): return self.gemv_op(b.take(oIdx, axis=0), W, h, iIdx, oIdx) eps = 3e-3 utt.verify_grad(metaop, [b_val, h_val, W_val], mode=self.mode, eps=eps) utt.verify_grad(op, [b_val, h_val, W_val], mode=self.mode, eps=eps) def test_sparseblockgemv_grad_1(self): # Test that we correctly handle cases where dimensions are 1. h_val = randn(1, 1, 1).astype('float32') iIdx_val = np.random.permutation(1)[:1][None, :] oIdx_val = np.random.permutation(1)[:1][None, :] W_val = randn(1, 1, 1, 1).astype('float32') b_val = randn(1, 1).astype('float32') iIdx = theano.tensor.constant(iIdx_val) oIdx = theano.tensor.constant(oIdx_val) def metaop(b, h, W): return sparse_block_dot(W, h, iIdx, b, oIdx) def op(b, h, W): return self.gemv_op(b.take(oIdx, axis=0), W, h, iIdx, oIdx) utt.verify_grad(metaop, [b_val, h_val, W_val], mode=self.mode) utt.verify_grad(op, [b_val, h_val, W_val], mode=self.mode) def test_sparseblockgemv_grad_shape(self): b = tensor.fmatrix() W = tensor.ftensor4() h = tensor.ftensor3() iIdx = tensor.imatrix() oIdx = tensor.imatrix() o = self.gemv_op(b.take(oIdx, axis=0), W, h, iIdx, oIdx) go = theano.grad(o.sum(), [b, W, h]) f = theano.function([W, h, iIdx, b, oIdx], go, mode=self.mode) W_val, h_val, iIdx_val, b_val, oIdx_val = \ BlockSparse_Gemv_and_Outer.gemv_data() # just make sure that it runs correcly and all the shapes are ok. b_g, W_g, h_g = f(W_val, h_val, iIdx_val, b_val, oIdx_val) assert b_g.shape == b_val.shape assert h_g.shape == h_val.shape assert W_g.shape == W_val.shape def test_sparseblockouter(self): o = tensor.ftensor4() x = tensor.ftensor3() y = tensor.ftensor3() xIdx = tensor.imatrix() yIdx = tensor.imatrix() out = self.outer_op(o, x, y, xIdx, yIdx) f = theano.function([o, x, y, xIdx, yIdx], out, on_unused_input="warn", mode=self.mode) o_val, x_val, y_val, xIdx_val, yIdx_val = \ BlockSparse_Gemv_and_Outer.outer_data() th_out = f(o_val, x_val, y_val, xIdx_val, yIdx_val) ref_out = BlockSparse_Gemv_and_Outer.outer_numpy( o_val, x_val, y_val, xIdx_val, yIdx_val) utt.assert_allclose(ref_out, th_out) def test_dot_infershape(self): b = tensor.fmatrix() W = tensor.ftensor4() h = tensor.ftensor3() iIdx = tensor.imatrix() oIdx = tensor.imatrix() self._compile_and_check([W, h, iIdx, b, oIdx], [sparse_block_dot(W, h, iIdx, b, oIdx)], self.gemv_data(), self.gemv_class) def test_gemv_infershape(self): b = tensor.fmatrix() W = tensor.ftensor4() h = tensor.ftensor3() iIdx = tensor.imatrix() oIdx = tensor.imatrix() self._compile_and_check( [W, h, iIdx, b, oIdx], [self.gemv_op(b.take(oIdx, axis=0), W, h, iIdx, oIdx)], self.gemv_data(), self.gemv_class) def test_outer_infershape(self): o = tensor.ftensor4() x = tensor.ftensor3() y = tensor.ftensor3() xIdx = tensor.imatrix() yIdx = tensor.imatrix() self._compile_and_check([o, x, y, xIdx, yIdx], [self.outer_op(o, x, y, xIdx, yIdx)], self.outer_data(), self.outer_class) Theano-1.0.0/theano/tensor/nnet/tests/test_conv3d2d.py00006000163562€ÿÿÿþ0000001570213203335532024225 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import time from nose.plugins.skip import SkipTest from parameterized import parameterized import numpy as np try: from scipy import ndimage except ImportError: ndimage = None from six.moves import xrange import theano from theano.gof.opt import check_stack_trace from theano.tensor.nnet.conv3d2d import conv3d, get_diagonal_subtensor_view, DiagonalSubtensor, IncDiagonalSubtensor import theano.tests.unittest_tools as utt def test_get_diagonal_subtensor_view(wrap=lambda a: a): x = np.arange(20).reshape(5, 4).astype('float32') x = wrap(x) xv01 = get_diagonal_subtensor_view(x, 0, 1) # test that it works in 2d assert np.all(np.asarray(xv01) == [[12, 9, 6, 3], [16, 13, 10, 7]]) x = np.arange(24).reshape(4, 3, 2) xv01 = get_diagonal_subtensor_view(x, 0, 1) xv02 = get_diagonal_subtensor_view(x, 0, 2) xv12 = get_diagonal_subtensor_view(x, 1, 2) # print 'x', x # print 'xv01', xv01 # print 'xv02', xv02 assert np.all(np.asarray(xv01) == [ [[12, 13], [8, 9], [4, 5]], [[18, 19], [14, 15], [10, 11]]]) assert np.all(np.asarray(xv02) == [ [[6, 1], [8, 3], [10, 5]], [[12, 7], [14, 9], [16, 11]], [[18, 13], [20, 15], [22, 17]], ]) # diagonal views of each leading matrix is the same # as the slices out of the diagonal view of the entire 3d tensor for xi, xvi in zip(x, xv12): assert np.all(xvi == get_diagonal_subtensor_view(xi, 0, 1)) def pyconv3d(signals, filters, border_mode='valid'): Ns, Ts, C, Hs, Ws = signals.shape Nf, Tf, C, Hf, Wf = filters.shape # if border_mode is not 'valid', the signals need zero-padding if border_mode == 'full': Tpad = Tf - 1 Hpad = Hf - 1 Wpad = Wf - 1 elif border_mode == 'half': Tpad = Tf // 2 Hpad = Hf // 2 Wpad = Wf // 2 else: Tpad = 0 Hpad = 0 Wpad = 0 if Tpad > 0 or Hpad > 0 or Wpad > 0: # zero-pad signals signals_padded = np.zeros((Ns, Ts + 2 * Tpad, C, Hs + 2 * Hpad, Ws + 2 * Wpad), 'float32') signals_padded[:, Tpad:(Ts + Tpad), :, Hpad:(Hs + Hpad), Wpad:(Ws + Wpad)] = signals Ns, Ts, C, Hs, Ws = signals_padded.shape signals = signals_padded Tf2 = Tf // 2 Hf2 = Hf // 2 Wf2 = Wf // 2 rval = np.zeros((Ns, Ts - Tf + 1, Nf, Hs - Hf + 1, Ws - Wf + 1)) for ns in xrange(Ns): for nf in xrange(Nf): for c in xrange(C): s_i = signals[ns, :, c, :, :] f_i = filters[nf, :, c, :, :] r_i = rval[ns, :, nf, :, :] o_i = ndimage.convolve(s_i, f_i, mode='constant', cval=1) o_i_sh0 = o_i.shape[0] # print s_i.shape, f_i.shape, r_i.shape, o_i.shape r_i += o_i[Tf2:o_i_sh0 - Tf2, Hf2:-Hf2, Wf2:-Wf2] return rval def check_diagonal_subtensor_view_traces(fn): assert check_stack_trace( fn, ops_to_check=(DiagonalSubtensor, IncDiagonalSubtensor)) @parameterized.expand(('valid', 'full', 'half'), utt.custom_name_func) def test_conv3d(border_mode): if ndimage is None or not theano.config.cxx: raise SkipTest("conv3d2d tests need SciPy and a c++ compiler") if theano.config.mode == 'FAST_COMPILE': mode = theano.compile.mode.get_mode('FAST_RUN') else: mode = theano.compile.mode.get_default_mode() shared = theano.tensor._shared Ns, Ts, C, Hs, Ws = 3, 10, 3, 32, 32 Nf, Tf, C, Hf, Wf = 32, 5, 3, 5, 5 signals = np.arange(Ns * Ts * C * Hs * Ws).reshape(Ns, Ts, C, Hs, Ws).astype('float32') filters = np.arange(Nf * Tf * C * Hf * Wf).reshape(Nf, Tf, C, Hf, Wf).astype('float32') t0 = time.time() pyres = pyconv3d(signals, filters, border_mode) print(time.time() - t0) s_signals = shared(signals) s_filters = shared(filters) s_output = shared(signals * 0) out = conv3d(s_signals, s_filters, signals_shape=signals.shape, filters_shape=filters.shape, border_mode=border_mode) newconv3d = theano.function([], [], updates={s_output: out}, mode=mode) check_diagonal_subtensor_view_traces(newconv3d) t0 = time.time() newconv3d() print(time.time() - t0) utt.assert_allclose(pyres, s_output.get_value(borrow=True)) gsignals, gfilters = theano.grad(out.sum(), [s_signals, s_filters]) gnewconv3d = theano.function([], [], updates=[(s_filters, gfilters), (s_signals, gsignals)], mode=mode, name='grad') check_diagonal_subtensor_view_traces(gnewconv3d) t0 = time.time() gnewconv3d() print('grad', time.time() - t0) Ns, Ts, C, Hs, Ws = 3, 3, 3, 5, 5 Nf, Tf, C, Hf, Wf = 4, 2, 3, 2, 2 signals = np.random.rand(Ns, Ts, C, Hs, Ws).astype('float32') filters = np.random.rand(Nf, Tf, C, Hf, Wf).astype('float32') utt.verify_grad(lambda s, f: conv3d(s, f, border_mode=border_mode), [signals, filters], eps=1e-1, mode=mode) # Additional Test that covers the case of patched implementation for filter with Tf=1 Ns, Ts, C, Hs, Ws = 3, 10, 3, 32, 32 Nf, Tf, C, Hf, Wf = 32, 1, 3, 5, 5 signals = np.arange(Ns * Ts * C * Hs * Ws).reshape(Ns, Ts, C, Hs, Ws).astype('float32') filters = np.arange(Nf * Tf * C * Hf * Wf).reshape(Nf, Tf, C, Hf, Wf).astype('float32') t0 = time.time() pyres = pyconv3d(signals, filters, border_mode) print(time.time() - t0) s_signals = shared(signals) s_filters = shared(filters) s_output = shared(signals * 0) out = conv3d(s_signals, s_filters, signals_shape=signals.shape, filters_shape=filters.shape, border_mode=border_mode) newconv3d = theano.function([], [], updates={s_output: out}, mode=mode) t0 = time.time() newconv3d() print(time.time() - t0) utt.assert_allclose(pyres, s_output.get_value(borrow=True)) gsignals, gfilters = theano.grad(out.sum(), [s_signals, s_filters]) gnewconv3d = theano.function([], [], updates=[(s_filters, gfilters), (s_signals, gsignals)], mode=mode, name='grad') t0 = time.time() gnewconv3d() print('grad', time.time() - t0) Ns, Ts, C, Hs, Ws = 3, 3, 3, 5, 5 Nf, Tf, C, Hf, Wf = 4, 1, 3, 2, 2 signals = np.random.rand(Ns, Ts, C, Hs, Ws).astype('float32') filters = np.random.rand(Nf, Tf, C, Hf, Wf).astype('float32') utt.verify_grad(lambda s, f: conv3d(s, f, border_mode=border_mode), [signals, filters], eps=1e-1, mode=mode) Theano-1.0.0/theano/tensor/nnet/tests/test_conv.py00006000163562€ÿÿÿþ0000006317613203335532023560 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import time from nose.plugins.skip import SkipTest import numpy as np import theano import theano.tensor as T from theano.tests import unittest_tools as utt from theano.tensor.nnet import conv from theano.tensor.basic import _allclose, NotScalarConstantError from theano.tests.unittest_tools import attr class TestConv2D(utt.InferShapeTester): # This class contains tests for the legacy 2d convolution, # but will also be inherited from for other implementations mode = None dtype = theano.config.floatX # This will be set to the appropriate function in the inherited classes. # The call to `staticmethod` is necessary to prevent Python from passing # `self` as the first argument. conv2d = staticmethod(conv.conv2d) def setUp(self): super(TestConv2D, self).setUp() self.input = T.tensor4('input', dtype=self.dtype) self.input.name = 'default_V' self.filters = T.tensor4('filters', dtype=self.dtype) self.filters.name = 'default_filters' if not conv.imported_scipy_signal and theano.config.cxx == "": raise SkipTest("conv2d tests need SciPy or a c++ compiler") def validate(self, image_shape, filter_shape, border_mode='valid', subsample=(1, 1), N_image_shape=None, N_filter_shape=None, input=None, filters=None, unroll_batch=None, unroll_kern=None, unroll_patch=None, verify_grad=True, should_raise=False): """ :param image_shape: The constant shape info passed to conv2d. :param filter_shape: The constant shape info passed to conv2d. :param N_image_shape: None(default to image_shape) or tuple of 4 elements with the shape of the input image :param N_filter_shape: None(default to filter_shape) or tuple of 4 elements with the shape of the input filter """ if N_image_shape is None: N_image_shape = [T.get_scalar_constant_value( T.as_tensor_variable(x)) for x in image_shape] if N_filter_shape is None: N_filter_shape = [T.get_scalar_constant_value( T.as_tensor_variable(x)) for x in filter_shape] if input is None: input = self.input if not filters: filters = self.filters # THEANO IMPLEMENTATION # we create a symbolic function so that verify_grad can work def sym_conv2d(input, filters): # define theano graph and function input.name = 'input' filters.name = 'filters' rval = conv.conv2d( input, filters, image_shape, filter_shape, border_mode, subsample, unroll_batch=unroll_batch, unroll_kern=unroll_kern, unroll_patch=unroll_patch) rval.name = 'conv_output' return rval output = sym_conv2d(input, filters) output.name = 'conv2d(%s,%s)' % (input.name, filters.name) theano_conv = theano.function([input, filters], output, mode=self.mode) # initialize input and compute result image_data = np.random.random(N_image_shape).astype(self.dtype) filter_data = np.random.random(N_filter_shape).astype(self.dtype) try: theano_output = theano_conv(image_data, filter_data) except ValueError: if not should_raise: raise return else: if should_raise: raise Exception( "ConvOp should have generated an error") # REFERENCE IMPLEMENTATION s = 1. orig_image_data = image_data if border_mode is not 'full': s = -1. out_shape2d = np.array(N_image_shape[-2:]) +\ s * np.array(N_filter_shape[-2:]) - s out_shape2d = np.ceil(out_shape2d / np.array(subsample)) # avoid numpy deprecation out_shape2d = out_shape2d.astype('int32') out_shape = (N_image_shape[0], N_filter_shape[0]) + tuple(out_shape2d) ref_output = np.zeros(out_shape) # loop over output feature maps ref_output.fill(0) if border_mode == 'full': image_data2 = np.zeros((N_image_shape[0], N_image_shape[1], N_image_shape[2] + 2 * N_filter_shape[2] - 2, N_image_shape[3] + 2 * N_filter_shape[3] - 2)) image_data2[ :, :, N_filter_shape[2] - 1:N_filter_shape[2] - 1 + N_image_shape[2], N_filter_shape[3] - 1:N_filter_shape[3] - 1 + N_image_shape[3]] = image_data image_data = image_data2 N_image_shape = image_data.shape for bb in range(N_image_shape[0]): for nn in range(N_filter_shape[0]): for im0 in range(N_image_shape[1]): filter2d = filter_data[nn, im0, :, :] image2d = image_data[bb, im0, :, :] for row in range(ref_output.shape[2]): irow = row * subsample[0] # image row for col in range(ref_output.shape[3]): icol = col * subsample[1] # image col ref_output[bb, nn, row, col] += (image2d[ irow:irow + N_filter_shape[2], icol:icol + N_filter_shape[3]] * filter2d[::-1, ::-1] ).sum() self.assertTrue(_allclose(theano_output, ref_output)) # TEST GRADIENT if verify_grad: utt.verify_grad(sym_conv2d, [orig_image_data, filter_data]) def test_basic1(self): # Tests that basic convolutions work for odd and even # dimensions of image and filter shapes, as well as rectangular # images and filters. self.validate((2, 2, 3, 3), (2, 2, 2, 2), 'valid', verify_grad=False) def test_basic(self): # Tests that basic convolutions work for odd and even # dimensions of image and filter shapes, as well as rectangular # images and filters. self.validate((3, 2, 8, 8), (4, 2, 5, 5), 'valid', verify_grad=False) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'valid') self.validate((3, 2, 7, 5), (5, 2, 3, 2), 'valid', verify_grad=False) self.validate((3, 2, 8, 8), (4, 2, 5, 5), 'full', verify_grad=False) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'full') # test filter same size as input def test_uint_image_shape_datatype(self): # Tests for uint datatype in image_shape. self.validate((2, 2, 3, np.uint8(3)), (3, 2, 3, 3), 'valid', verify_grad=False) self.validate((np.uint16(2), 2, 3, 3), (3, 2, 3, 3), 'valid', verify_grad=False) self.validate((2, np.uint32(2), 3, 3), (3, 2, 3, 3), 'valid', verify_grad=False) def test_uint_filter_shape_datatype(self): # Tests for uint datatype in filter_shape self.validate((3, 2, 3, 3), (2, 2, 3, np.uint8(3)), 'valid', verify_grad=False) self.validate((3, 2, 3, 3), (np.uint16(2), 2, 3, 3), 'valid', verify_grad=False) self.validate((3, 2, 3, 3), (2, np.uint32(2), 3, 3), 'valid', verify_grad=False) def test_img_kernel_same_shape(self): self.validate((3, 2, 3, 3), (4, 2, 3, 3), 'full') self.validate((3, 2, 3, 3), (4, 2, 3, 3), 'valid') def test_unroll_patch_true(self): # Test basic convs with True. self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'valid', unroll_patch=True) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'full', unroll_patch=True) self.validate( (3, 2, 3, 3), (4, 2, 3, 3), 'valid', unroll_patch=True, verify_grad=False) def test_unroll_patch_false(self): # Test basic convs with False. self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'valid', unroll_patch=False) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'full', unroll_patch=False) self.validate( (3, 2, 3, 3), (4, 2, 3, 3), 'valid', unroll_patch=False, verify_grad=False) def test_unroll_patch_true_fail(self): # Test basic convs with True. self.validate( (3, 2, 7, 5), (5, 2, 2, 3), 'valid', unroll_patch=True, N_image_shape=(1, 3, 3, 3), N_filter_shape=(6, 3, 2, 2), should_raise=True) self.validate( (3, 2, 7, 5), (5, 2, 2, 3), 'full', unroll_patch=True, N_image_shape=(1, 3, 3, 3), N_filter_shape=(6, 3, 2, 2), should_raise=True) self.validate( (3, 2, 3, 3), (4, 2, 3, 3), 'valid', unroll_patch=True, N_image_shape=(1, 3, 3, 3), N_filter_shape=(6, 3, 2, 2), should_raise=True) def test_unroll_special(self): # (unroll_kern, unroll_batch) in (0,1),(1,0) is special case. self.validate((6, 2, 3, 3), (3, 2, 2, 2), 'valid', unroll_batch=1) def test_unroll_batch(self): # Test mini-batch unrolling for various legal values. # mini-batch of size 6 is multiple of 2 and 3. Should work. self.validate( (6, 2, 3, 3), (3, 2, 2, 2), 'valid', unroll_batch=2, verify_grad=False) self.validate( (6, 2, 3, 3), (3, 2, 2, 2), 'valid', unroll_batch=3, verify_grad=False) def test_unroll_kern(self): # Test kernel unrolling for various legal values. # 6 filters is a multiple of 2 and 3. Should work. self.validate( (2, 3, 3, 3), (6, 3, 2, 2), 'valid', unroll_kern=2, verify_grad=False) self.validate( (2, 3, 3, 3), (6, 3, 2, 2), 'valid', unroll_kern=3, verify_grad=False) def test_unroll_batch_kern(self): # Test mini-batch unrolling with kernel unrolling for various # legal values. # mini-batch of size 6 is multiple of 2 and 3. Should work. self.validate( (6, 2, 3, 3), (3, 2, 2, 2), 'valid', unroll_batch=2, unroll_kern=3, verify_grad=False) self.validate( (6, 2, 3, 3), (3, 2, 2, 2), 'valid', unroll_batch=3, unroll_kern=3, verify_grad=False) # 6 filters is a multiple of 2 and 3. Should work. self.validate( (2, 3, 3, 3), (6, 3, 2, 2), 'valid', unroll_batch=2, unroll_kern=2, verify_grad=False) self.validate( (2, 3, 3, 3), (6, 3, 2, 2), 'valid', unroll_batch=2, unroll_kern=3, verify_grad=False) def test_unroll_batch_kern_fail(self): # Test mini-batch unrolling with kernel unrolling for various # legal values, but pass bad input. All those test must # generate errors # mini-batch of size 6 is multiple of 2 and 3. Should work. self.validate((6, 2, 3, 3), (3, 2, 2, 2), 'valid', unroll_batch=2, unroll_kern=3, N_image_shape=(7, 2, 3, 3), N_filter_shape=(3, 2, 2, 2), should_raise=True) self.validate((6, 2, 3, 3), (3, 2, 2, 2), 'valid', unroll_batch=3, unroll_kern=3, N_image_shape=(6, 2, 3, 3), N_filter_shape=(4, 2, 2, 2), should_raise=True) self.validate( (2, 3, 3, 3), (6, 3, 2, 2), 'valid', unroll_batch=2, unroll_kern=2, N_image_shape=(1, 3, 3, 3), N_filter_shape=(6, 3, 2, 2), should_raise=True) self.validate( (2, 3, 3, 3), (6, 3, 2, 2), 'valid', unroll_batch=2, unroll_kern=3, N_image_shape=(2, 3, 3, 3), N_filter_shape=(5, 3, 2, 2), should_raise=True) @attr('slow') def test_subsample(self): # Tests convolution where subsampling != (1,1) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'full', subsample=(2, 2)) # Fails as of 2012-07-11 self.assertRaises(NotImplementedError, self.validate, (1, 1, 6, 6), (1, 1, 3, 3), 'full', subsample=(3, 3)) # Fails as of 2017-08-10 self.assertRaises(NotImplementedError, self.validate, (3, 2, 7, 5), (5, 2, 2, 3), 'valid', subsample=(2, 2)) self.assertRaises(NotImplementedError, self.validate, (3, 2, 7, 5), (5, 2, 2, 3), 'valid', subsample=(2, 1)) self.assertRaises(NotImplementedError, self.validate, (1, 1, 6, 6), (1, 1, 3, 3), 'valid', subsample=(3, 3)) def test_shape_Constant_tensor(self): # Tests convolution where the {image,filter}_shape is a Constant tensor. as_t = T.as_tensor_variable self.validate( (as_t(3), as_t(2), as_t(7), as_t(5)), (5, 2, 2, 3), 'valid') self.validate(as_t([3, 2, 7, 5]), (5, 2, 2, 3), 'valid') self.validate(as_t((3, 2, 7, 5)), (5, 2, 2, 3), 'valid') self.validate( (3, 2, 7, 5), ( as_t(5), as_t(2), as_t(2), as_t(3)), 'valid') self.validate((3, 2, 7, 5), as_t([5, 2, 2, 3]), 'valid') self.validate((3, 2, 7, 5), as_t((5, 2, 2, 3)), 'valid') self.validate(as_t([3, 2, 7, 5]), as_t([5, 2, 2, 3]), 'full') def test_invalid_filter_shape(self): # Tests scenario where filter_shape[1] != input_shape[1] self.assertRaises(AssertionError, self.validate, (3, 2, 8, 8), (4, 3, 5, 5), 'valid') @attr('slow') def test_invalid_input_shape(self): # Tests that when the shape gived at build time is not the same as # run time we raise an error for unroll_batch in [None, 1, 3]: for unroll_kern in [None, 2, 4]: for unroll_patch in [None, True, False]: for mode in ['valid', 'full']: self.assertRaises(ValueError, self.validate, (3, 2, 8, 8), (4, 2, 5, 5), mode, N_image_shape=(2, 2, 8, 8), unroll_batch=unroll_batch, unroll_kern=unroll_kern, unroll_patch=unroll_patch) self.assertRaises(ValueError, self.validate, (3, 2, 8, 8), (4, 2, 5, 5), mode, N_image_shape=(3, 1, 8, 8), unroll_batch=unroll_batch, unroll_kern=unroll_kern, unroll_patch=unroll_patch) self.assertRaises(ValueError, self.validate, (3, 2, 8, 8), (4, 2, 5, 5), mode, N_image_shape=(3, 2, 7, 8), unroll_batch=unroll_batch, unroll_kern=unroll_kern, unroll_patch=unroll_patch) self.assertRaises(ValueError, self.validate, (3, 2, 8, 8), (4, 2, 5, 5), mode, N_image_shape=(3, 2, 8, 7), unroll_batch=unroll_batch, unroll_kern=unroll_kern, unroll_patch=unroll_patch) self.assertRaises(ValueError, self.validate, (3, 2, 8, 8), (4, 2, 5, 5), mode, N_filter_shape=(3, 2, 5, 5), unroll_batch=unroll_batch, unroll_kern=unroll_kern, unroll_patch=unroll_patch) self.assertRaises(ValueError, self.validate, (3, 2, 8, 8), (4, 2, 5, 5), mode, N_filter_shape=(4, 1, 5, 5), unroll_batch=unroll_batch, unroll_kern=unroll_kern, unroll_patch=unroll_patch) self.assertRaises(ValueError, self.validate, (3, 2, 8, 8), (4, 2, 5, 5), mode, N_filter_shape=(4, 2, 6, 5), unroll_batch=unroll_batch, unroll_kern=unroll_kern, unroll_patch=unroll_patch) self.assertRaises(ValueError, self.validate, (3, 2, 8, 8), (4, 2, 5, 5), mode, N_filter_shape=(4, 2, 5, 6), unroll_batch=unroll_batch, unroll_kern=unroll_kern, unroll_patch=unroll_patch) def test_missing_info(self): # Test convolutions for various pieces of missing info. self.validate(None, None, N_image_shape=(3, 2, 8, 8), N_filter_shape=(4, 2, 5, 5)) self.validate((3, 2, None, None), None, N_image_shape=(3, 2, 8, 8), N_filter_shape=(4, 2, 5, 5)) self.validate((None, 2, None, None), (None, 2, 5, 5), N_image_shape=(3, 2, 8, 8), N_filter_shape=(4, 2, 5, 5)) self.validate((3, 2, 8, 8), (4, 2, None, 5), N_image_shape=(3, 2, 8, 8), N_filter_shape=(4, 2, 5, 5)) self.validate((3, 2, 8, 8), (4, 2, 5, None), N_image_shape=(3, 2, 8, 8), N_filter_shape=(4, 2, 5, 5)) def test_wrong_info(self): # Test convolutions when we don't give a constant as shape information i = theano.scalar.basic.int32() self.assertRaises(NotScalarConstantError, self.validate, (3, 2, 8, i), (4, 2, 5, 5), N_image_shape=(3, 2, 8, 8), N_filter_shape=(4, 2, 5, 5)) self.assertRaises(NotScalarConstantError, self.validate, (3, 2, 8, 8), (4, 2, 5, i), N_image_shape=(3, 2, 8, 8), N_filter_shape=(4, 2, 5, 5)) def test_full_mode(self): # Tests basic convolution in full mode and case where filter # is larger than the input image. self.validate((3, 2, 5, 5), (4, 2, 8, 8), 'full') def f(): self.validate((3, 2, 5, 5), (4, 2, 8, 8), 'valid') self.assertRaises(Exception, f) def test_wrong_input(self): # Make sure errors are raised when image and kernel are not 4D tensors self.assertRaises(Exception, self.validate, (3, 2, 8, 8), (4, 2, 5, 5), 'valid', input=T.dmatrix()) self.assertRaises(Exception, self.validate, (3, 2, 8, 8), (4, 2, 5, 5), 'valid', filters=T.dvector()) self.assertRaises(Exception, self.validate, (3, 2, 8, 8), (4, 2, 5, 5), 'valid', input=T.dtensor3()) def test_gcc_crash(self): # gcc 4.3.0 20080428 (Red Hat 4.3.0-8) # # crashed in this following case. I changed the c code to don't hit # gcc bug. So it should not crash anymore self.validate((1, 10, 213, 129), (46, 10, 212, 1), 'valid', verify_grad=False) def speed(self): n_calls = 20000 print("n_calls", n_calls) for border_mode in ['valid', 'full']: print() print(border_mode) for openmp in [False, True]: print("OpenMP", openmp) image_shapes = [ (1, 5, 6, 6), (10, 5, 6, 6) # (10, 10, 16, 16), # (10, 10, 32, 32)] ] print("image_shape", image_shapes) for image_shape in image_shapes: filter_shapes = [(1, 5, 4, 4), (2, 5, 4, 4), (5, 5, 4, 4)] print("filter_shapes", filter_shapes) for filter_shape in filter_shapes: input = theano.shared(np.random.random(image_shape)) filters = theano.shared(np.random.random(filter_shape)) output = self.conv2d( input, filters, image_shape, filter_shape, border_mode, unroll_patch=True, openmp=openmp) mode = theano.Mode(linker=theano.gof.vm.VM_Linker( allow_gc=False, use_cloop=True)) theano_conv = theano.function([], output, mode=mode) t1 = time.time() theano_conv.fn(n_calls=n_calls) t2 = time.time() print(t2 - t1, end=' ') print() def test_infer_shape(self): # Note: infer_shape is incomplete and thus input and filter shapes # must be provided explicitly def rand(*shape): r = np.asarray(np.random.rand(*shape), dtype='float64') return r * 2 - 1 adtens = T.dtensor4() bdtens = T.dtensor4() aivec_val = [4, 5, 6, 3] bivec_val = [7, 5, 3, 2] adtens_val = rand(*aivec_val) bdtens_val = rand(*bivec_val) self._compile_and_check( [adtens, bdtens], [self.conv2d( adtens, bdtens, aivec_val, bivec_val, border_mode='valid')], [adtens_val, bdtens_val], conv.ConvOp, excluding=['conv_gemm']) self._compile_and_check( [adtens, bdtens], [self.conv2d( adtens, bdtens, aivec_val, bivec_val, border_mode='full')], [adtens_val, bdtens_val], conv.ConvOp, excluding=['conv_gemm']) aivec_val = [6, 2, 8, 3] bivec_val = [4, 2, 5, 3] adtens_val = rand(*aivec_val) bdtens_val = rand(*bivec_val) self._compile_and_check( [adtens, bdtens], [self.conv2d( adtens, bdtens, aivec_val, bivec_val, border_mode='valid')], [adtens_val, bdtens_val], conv.ConvOp, excluding=['conv_gemm']) self._compile_and_check( [adtens, bdtens], [self.conv2d( adtens, bdtens, aivec_val, bivec_val, border_mode='full')], [adtens_val, bdtens_val], conv.ConvOp, excluding=['conv_gemm']) aivec_val = [3, 6, 7, 5] bivec_val = [5, 6, 3, 2] adtens_val = rand(*aivec_val) bdtens_val = rand(*bivec_val) self._compile_and_check( [adtens, bdtens], [self.conv2d( adtens, bdtens, aivec_val, bivec_val, border_mode='valid')], [adtens_val, bdtens_val], conv.ConvOp, excluding=['conv_gemm']) self._compile_and_check( [adtens, bdtens], [self.conv2d( adtens, bdtens, aivec_val, bivec_val, border_mode='full')], [adtens_val, bdtens_val], conv.ConvOp, excluding=['conv_gemm']) aivec_val = [3, 6, 7, 5] bivec_val = [5, 6, 2, 3] adtens_val = rand(*aivec_val) bdtens_val = rand(*bivec_val) self._compile_and_check( [adtens, bdtens], [self.conv2d( adtens, bdtens, aivec_val, bivec_val, border_mode='valid')], [adtens_val, bdtens_val], conv.ConvOp, excluding=['conv_gemm']) self._compile_and_check( [adtens, bdtens], [self.conv2d( adtens, bdtens, aivec_val, bivec_val, border_mode='full')], [adtens_val, bdtens_val], conv.ConvOp, excluding=['conv_gemm']) aivec_val = [5, 2, 4, 3] bivec_val = [6, 2, 4, 3] adtens_val = rand(*aivec_val) bdtens_val = rand(*bivec_val) self._compile_and_check( [adtens, bdtens], [self.conv2d( adtens, bdtens, aivec_val, bivec_val, border_mode='valid')], [adtens_val, bdtens_val], conv.ConvOp, excluding=['conv_gemm']) self._compile_and_check( [adtens, bdtens], [self.conv2d( adtens, bdtens, aivec_val, bivec_val, border_mode='full')], [adtens_val, bdtens_val], conv.ConvOp, excluding=['conv_gemm']) # Test that broadcasting of gradients works correctly when using the # nnet.conv2d() interface. This was reported in #3763, and uses the example # code from that ticket. def test_broadcast_grad(): # rng = numpy.random.RandomState(utt.fetch_seed()) x1 = T.tensor4('x') # x1_data = rng.randn(1, 1, 300, 300) sigma = T.scalar('sigma') # sigma_data = 20 window_radius = 3 filter_1d = T.arange(-window_radius, window_radius + 1) filter_1d = filter_1d.astype(theano.config.floatX) filter_1d = T.exp(-0.5 * filter_1d**2 / sigma ** 2) filter_1d = filter_1d / filter_1d.sum() filter_W = filter_1d.dimshuffle(['x', 'x', 0, 'x']) y = theano.tensor.nnet.conv2d(x1, filter_W, border_mode='full', filter_shape=[1, 1, None, None]) theano.grad(y.sum(), sigma) if __name__ == '__main__': t = TestConv2D('setUp') t.setUp() t.test_infer_shape() Theano-1.0.0/theano/tensor/nnet/tests/speed_test_conv.py00006000163562€ÿÿÿþ0000003330313203335532024725 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import time import numpy as N from six.moves import xrange from theano import function, Mode import theano.tensor as T from theano.tensor.nnet.conv import ConvOp def flip(kern, kshp): "flip the kernel as scipy.convolv2d do it flipped." flip = N.zeros(kern.shape) if len(kern.shape) == 2: kern = kern.reshape(-1) it = reversed(kern) for i in xrange(kshp[0]): for j in xrange(kshp[1]): flip[i, j] = next(it) elif len(kern.shape) == 3: kern = kern.reshape(kern.shape[0], -1) for k in xrange(kern.shape[0]): it = reversed(kern[k, :]) for i in xrange(kshp[0]): for j in xrange(kshp[1]): flip[k, i, j] = next(it) elif len(kern.shape) == 4: kern = kern.reshape(kern.shape[0], kern.shape[1], -1) for k in xrange(kern.shape[0]): for m in xrange(kern.shape[1]): it = reversed(kern[k, m, :]) for i in xrange(kshp[0]): for j in xrange(kshp[1]): flip[k, m, i, j] = next(it) else: raise NotImplementedError() return flip global_rng = N.random.RandomState(3423489) dmatrix4 = T.TensorType('float64', (False, False, False, False)) def exec_multilayer_conv_nnet_old( conv_mode, ss, bsize, imshp, kshps, nkerns, unroll_batch=0, unroll_kern=0, img=T.dmatrix(), validate=True, conv_op_py=False, do_print=True, repeat=1, unroll_patch=False, unroll_patch_size=False, verbose=0): # build actual input images imgval = global_rng.rand(bsize, imshp[0], imshp[1], imshp[2]) a = T.dmatrix() kerns = [a for i in nkerns] inputs4 = dmatrix4() kerns4 = dmatrix4() # for each layer ntot = 0 tctot = 0 tpytot = 0 for kshp, kern, nkern, n_layer in zip(kshps, kerns, nkerns, xrange(len(nkerns))): if do_print: print('************* layer %i ***************' % n_layer) print(conv_mode, ss, n_layer, kshp, nkern) # actual values w = global_rng.random_sample(N.r_[nkern, imshp[0], kshp]) w_flip = flip(w, kshp).reshape(w.shape) # manual implementation # check first stage padimg = imgval if conv_mode == 'full': padimg_shp = N.array(imshp[1:]) + 2 * (N.array(kshp) - N.array([1, 1])) padimg = N.zeros(N.r_[bsize, imshp[0], padimg_shp]) padimg[ :, :, kshp[0] - 1:-kshp[0] + 1, kshp[1] - 1:-kshp[1] + 1] = imgval outshp = N.hstack((nkern, ConvOp.getOutputShape(imshp[1:], kshp, ss, conv_mode))) time1 = time.time() outval = N.zeros(N.r_[bsize, outshp]) if validate: # causes an atexit problem from scipy.signal.sigtools import _convolve2d from scipy.signal.signaltools import _valfrommode, _bvalfromboundary val = _valfrommode(conv_mode) bval = _bvalfromboundary('fill') for b in xrange(bsize): # loop over batches for n in xrange(nkern): # loop over filters for i in xrange(imshp[0]): # loop over input feature maps outval[b, n, ...] += _convolve2d( imgval[b, i, ...], w_flip[n, i, ...], 1, val, bval, 0)[0::ss[0], 0::ss[1]] ntot += time.time() - time1 # ConvOp if unroll_patch and not unroll_patch_size: conv_op = ConvOp(dx=ss[0], dy=ss[1], output_mode=conv_mode, unroll_patch=unroll_patch, verbose=verbose)(inputs4, kerns4) else: conv_op = ConvOp(imshp, kshp, nkern, bsize, ss[0], ss[1], conv_mode, unroll_batch=unroll_batch, unroll_kern=unroll_kern, unroll_patch=unroll_patch, verbose=verbose)(inputs4, kerns4) # l1shp = N.hstack((nkern, # ConvOp.getOutputShape(imshp[1:], kshp, ss, conv_mode))) propup2 = function([inputs4, kerns4], conv_op) propup3 = function([inputs4, kerns4], conv_op, mode=Mode(linker="py")) time1 = time.time() for i in xrange(repeat): hidval2_ = propup2(imgval, w_flip) hidval2 = hidval2_ # [:,:,0::ss[0],0::ss[1]] tctot += time.time() - time1 if conv_op_py: time1 = time.time() for i in xrange(repeat): hidval3_ = propup3(imgval, w_flip) hidval3 = hidval3_ # [:,:,0::ss[0],0::ss[1]] tpytot += time.time() - time1 assert (N.abs(hidval2 - hidval3) < 1e-5).all() else: tpytot += 0 if validate: temp = N.abs(outval - hidval2) assert (temp < 1e-5).all() if validate and conv_op_py: temp = N.abs(outval - hidval3) assert (temp < 1e-5).all() imshp = tuple(outshp) imgval = outval.reshape(bsize, outshp[0], outshp[1], outshp[2]) return tctot, tpytot, ntot def exec_multilayer_conv_nnet( conv_mode, ss, bsize, imshp, kshps, nkerns, unroll_batch=0, unroll_kern=0, img=T.dmatrix(), do_print=True, repeat=1, unroll_patch=False, unroll_patch_size=False, verbose=0): # build actual input images imgval = global_rng.rand(bsize, imshp[0], imshp[1], imshp[2]) a = T.dmatrix() kerns = [a for i in nkerns] inputs4 = dmatrix4() kerns4 = dmatrix4() # for each layer ntot = 0 tctot = 0 tpytot = 0 for kshp, kern, nkern, n_layer in zip(kshps, kerns, nkerns, xrange(len(nkerns))): if do_print: print('************* layer %i ***************' % n_layer) print(conv_mode, ss, n_layer, kshp, nkern) # actual values w = global_rng.random_sample(N.r_[nkern, imshp[0], kshp]) w_flip = flip(w, kshp).reshape(w.shape) outshp = N.hstack((nkern, ConvOp.getOutputShape(imshp[1:], kshp, ss, conv_mode))) time1 = time.time() # outval = N.zeros(N.r_[bsize, outshp]) # ConvOp if unroll_patch and not unroll_patch_size: conv_op = ConvOp(dx=ss[0], dy=ss[1], output_mode=conv_mode, unroll_patch=unroll_patch, verbose=verbose)(inputs4, kerns4) else: conv_op = ConvOp(imshp, kshp, nkern, bsize, ss[0], ss[1], conv_mode, unroll_batch=unroll_batch, unroll_kern=unroll_kern, unroll_patch=unroll_patch, verbose=verbose)(inputs4, kerns4) # l1shp = N.hstack((nkern, # ConvOp.getOutputShape(imshp[1:], kshp, ss, conv_mode))) propup2 = function([inputs4, kerns4], conv_op) time1 = time.time() for i in xrange(repeat): propup2(imgval, w_flip) tctot += time.time() - time1 imshp = tuple(outshp) # imgval = outval.reshape(bsize, outshp[0], outshp[1], outshp[2]) return tctot, tpytot, ntot def speed_multilayer_conv(): # calculate the speed up of different combination of unroll # put the paramter to the same you will try. # validate = False # we don't validate the result to have it much faster! repeat = 3 verbose = 1 unroll_batch = [1, 2, 3, 4, 5, 6, 10] # 15, 30, 60 always much slower unroll_kern = [1, 2, 3, 4, 5, 6, 10] # 15, 30, 60 always much slower # unroll_batch = [1,4,5] # unroll_kern = [1,4,5] # unroll_batch = [1,4] # unroll_kern = [1,4] # unroll_patch = [True, False] bsize = 60 # batch size imshp_start = (1, 48, 48) # un square shape to test more corner case. kshps = ([11, 12],) # un square shape to test more corner case. nkerns = [60] # per output pixel ssizes = [(1, 1), ] # (1,1)]#(2,2) bugged convmodes = ['valid', 'full'] # do_convolve2 = False a = T.dmatrix() kerns = [a for i in nkerns] assert len(kshps) == len(nkerns) == len(kerns) timing = N.zeros((len(unroll_batch), len(unroll_kern), 3, len(convmodes) * len(ssizes))) t_b_k = [] # calculate the timing with unrolling print('time unroll batch kern') best = [] worst = [] t_ = [] for unroll_b, n_b in zip(unroll_batch, xrange(len(unroll_batch))): for unroll_k, n_k in zip(unroll_kern, xrange(len(unroll_kern))): t_b_k.append(str(unroll_b) + "/" + str(unroll_k)) if not t_: tctot, tpytot, ntot = [], [], [] for conv_mode, n_mode in zip(convmodes, xrange(len(convmodes))): for ss, n_ss in zip(ssizes, xrange(len(ssizes))): # tctot_, tpytot_, ntot_ = exec_multilayer_conv_nnet_old(conv_mode, ss, bsize, imshp_start, kshps, nkerns, unroll_batch=unroll_b, unroll_kern=unroll_k, validate=validate, verbose=verbose,do_print=False) tctot_, tpytot_, ntot_ = exec_multilayer_conv_nnet(conv_mode, ss, bsize, imshp_start, kshps, nkerns, unroll_batch=unroll_b, unroll_kern=unroll_k, verbose=verbose, do_print=False, repeat=repeat) tctot += [tctot_] tpytot += [tpytot_] ntot += [ntot_] if unroll_b == 4 and unroll_k == 4: # print "unroll 4/4",tctot best = tctot if unroll_b == 1 and unroll_k == 1: # print "unroll 1/1",tctot worst = tctot timing[n_b, n_k] = [tctot, tpytot, ntot] # [sum(tctot), sum(tpytot), sum(ntot)] if not t_: t = timing[:, :, 0, :] # We select only the c timing. else: t = t_ t = N.asarray(t) # calculate the old timing print('time old version') tctot, tpytot, ntot = [], [], [] tctot_ = [] if not tctot_: for conv_mode, n_mode in zip(convmodes, xrange(len(convmodes))): for ss, n_ss in zip(ssizes, xrange(len(ssizes))): # tctot_, tpytot_, ntot_ = exec_multilayer_conv_nnet_old(conv_mode, ss, bsize, imshp_start, kshps, nkerns, unroll_batch=0, unroll_kern=0, validate=validate, verbose=verbose,do_print=False) tctot_, tpytot_, ntot_ = exec_multilayer_conv_nnet(conv_mode, ss, bsize, imshp_start, kshps, nkerns, unroll_batch=0, unroll_kern=0, verbose=verbose, do_print=False, repeat=repeat) tctot += [tctot_] tpytot += [tpytot_] ntot += [ntot_] else: tctot = N.asarray(tctot_) print("old code timing %.3fs" % sum(tctot), tctot) best = N.asarray(best) worst = N.asarray(worst) print("timing for unrolled version") print("unroll_batch/unroll_kern valid_mode full_mode") for n_b in xrange(len(unroll_batch)): for n_k in xrange(len(unroll_kern)): print((unroll_batch[n_b], unroll_kern[n_k]) + tuple(t[n_b, n_k]), ',') # t_detail = t t = t.sum(axis=2) print("max %.3fs" % t.max(), "max param(batch unloop size/kernel unloop size)", t_b_k[t.argmax()]) print("min %.3fs" % t.min(), "min param(batch unloop size/kernel unloop size)", t_b_k[t.argmin()]) print("speedup vs (1/1)%.3fx, vs old %.3fx" % (t.max() / t.min(), sum(tctot) / t.min())) print(worst / best, tctot / best) # calculate the timing of unroll_patch print('time unroll_patch') tctot_patch = [] tctot_patch_size = [] for conv_mode, n_mode in zip(convmodes, xrange(len(convmodes))): for ss, n_ss in zip(ssizes, xrange(len(ssizes))): # tctot_, tpytot_, ntot_ = exec_multilayer_conv_nnet_old(conv_mode, ss, bsize, imshp_start, kshps, nkerns, unroll_batch=0, unroll_kern=0, validate=validate,unroll_patch=True,verbose=verbose,do_print=False) tctot_, tpytot_, ntot_ = exec_multilayer_conv_nnet(conv_mode, ss, bsize, imshp_start, kshps, nkerns, unroll_batch=0, unroll_kern=0, unroll_patch=True, verbose=verbose, do_print=False, repeat=repeat) tctot_patch += [tctot_] # tctot_, tpytot_, ntot_ = exec_multilayer_conv_nnet_old(conv_mode, ss, bsize, imshp_start, kshps, nkerns, unroll_batch=0, unroll_kern=0, validate=validate,unroll_patch=True,verbose=verbose,do_print=False,unroll_patch_size=True) tctot_, tpytot_, ntot_ = exec_multilayer_conv_nnet(conv_mode, ss, bsize, imshp_start, kshps, nkerns, unroll_batch=0, unroll_kern=0, unroll_patch=True, verbose=verbose, do_print=False, unroll_patch_size=True, repeat=repeat) tctot_patch_size += [tctot_] t_patch = sum(tctot_patch) print("unroll_patch without shape time", tctot_patch) print("speedup vs (1/1)%.3fx, vs old %.3fx" % (t.max() / t_patch, sum(tctot) / t_patch)) print(best / tctot_patch, worst / tctot_patch) t_patch_size = sum(tctot_patch_size) print("unroll_patch with shape time", tctot_patch_size) print("speedup vs (1/1)%.3fx, vs old %.3fx" % (t.max() / t_patch_size, sum(tctot) / t_patch_size)) print(best / tctot_patch_size, worst / tctot_patch_size) return if __name__ == '__main__': speed_multilayer_conv() Theano-1.0.0/theano/tensor/nnet/tests/test_abstract_conv.py00006000163562€ÿÿÿþ0000031053313203335532025433 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import unittest import numpy as np from nose.plugins.skip import SkipTest from nose.tools import assert_raises, assert_true import theano from theano import tensor from theano import change_flags from theano.gof.opt import check_stack_trace from theano.tests import unittest_tools as utt from theano.tensor.nnet import (corr, corr3d, conv2d_transpose, abstract_conv as conv) from theano.tensor.nnet.abstract_conv import (get_conv_output_shape, get_conv_gradweights_shape, get_conv_gradinputs_shape, check_conv_gradinputs_shape, assert_conv_shape, assert_shape) from theano.tensor.nnet.abstract_conv import AbstractConv2d from theano.tensor.nnet.abstract_conv import AbstractConv2d_gradInputs from theano.tensor.nnet.abstract_conv import AbstractConv2d_gradWeights from theano.tensor.nnet.abstract_conv import bilinear_kernel_1D from theano.tensor.nnet.abstract_conv import bilinear_kernel_2D from theano.tensor.nnet.abstract_conv import bilinear_upsampling from theano.tensor.nnet.abstract_conv import separable_conv2d, separable_conv3d from theano.tensor.nnet.abstract_conv import causal_conv1d from theano.tensor.nnet.corr import (CorrMM, CorrMM_gradWeights, CorrMM_gradInputs) from theano.tensor.nnet.corr3d import (Corr3dMM, Corr3dMM_gradWeights, Corr3dMM_gradInputs) def conv2d_corr(inputs, filters, border_mode="valid", subsample=(1, 1), conv_mode='conv', filter_dilation=(1, 1)): if conv_mode == 'conv': filters = filters[:, :, ::-1, ::-1] return corr.CorrMM(border_mode, subsample, filter_dilation)(inputs, filters) def conv2d_corr_gw(inputs, topgrad, filters_shape, border_mode="valid", subsample=(1, 1), conv_mode='conv', filter_dilation=(1, 1)): rval = corr.CorrMM_gradWeights(border_mode, subsample, filter_dilation)(inputs, topgrad, filters_shape[2:]) if conv_mode == 'conv': rval = rval[:, :, ::-1, ::-1] return rval def conv2d_corr_gi(filters, topgrad, inputs_shape, border_mode="valid", subsample=(1, 1), conv_mode='conv', filter_dilation=(1, 1)): if conv_mode == 'conv': filters = filters[:, :, ::-1, ::-1] return corr.CorrMM_gradInputs(border_mode, subsample, filter_dilation)(filters, topgrad, inputs_shape[2:]) def conv3d_corr(inputs, filters, border_mode="valid", subsample=(1, 1, 1), conv_mode='conv', filter_dilation=(1, 1, 1)): if conv_mode == 'conv': filters = filters[:, :, ::-1, ::-1, ::-1] return corr3d.Corr3dMM(border_mode, subsample, filter_dilation)(inputs, filters) def conv3d_corr_gw(inputs, topgrad, filters_shape, border_mode="valid", subsample=(1, 1, 1), conv_mode='conv', filter_dilation=(1, 1, 1)): rval = corr3d.Corr3dMM_gradWeights(border_mode, subsample, filter_dilation)(inputs, topgrad, filters_shape[2:]) if conv_mode == 'conv': rval = rval[:, :, ::-1, ::-1, ::-1] return rval def conv3d_corr_gi(filters, topgrad, inputs_shape, border_mode="valid", subsample=(1, 1, 1), conv_mode='conv', filter_dilation=(1, 1, 1)): if conv_mode == 'conv': filters = filters[:, :, ::-1, ::-1, ::-1] return corr3d.Corr3dMM_gradInputs(border_mode, subsample, filter_dilation)(filters, topgrad, inputs_shape[2:]) class TestGetConvOutShape(unittest.TestCase): def test_basic(self): image_shape, kernel_shape = (3, 2, 12, 9), (4, 2, 5, 6) sub_sample = (1, 2) filter_dilation = (2, 1) test1_params = get_conv_output_shape( image_shape, kernel_shape, 'valid', sub_sample, filter_dilation) test2_params = get_conv_output_shape( image_shape, kernel_shape, 'half', sub_sample, filter_dilation) test3_params = get_conv_output_shape( image_shape, kernel_shape, 'full', sub_sample, filter_dilation) test4_params = get_conv_output_shape( image_shape, kernel_shape, (1, 2), sub_sample, filter_dilation) self.assertTrue(test1_params == (3, 4, 4, 2)) self.assertTrue(test2_params == (3, 4, 12, 5)) self.assertTrue(test3_params == (3, 4, 20, 7)) self.assertTrue(test4_params == (3, 4, 6, 4)) def test_basic_3d(self): image_shape, kernel_shape = (3, 2, 12, 9, 7), (4, 2, 5, 6, 4) sub_sample = (1, 2, 1) filter_dilation = (2, 1, 1) test1_params = get_conv_output_shape( image_shape, kernel_shape, 'valid', sub_sample, filter_dilation) test2_params = get_conv_output_shape( image_shape, kernel_shape, 'half', sub_sample, filter_dilation) test3_params = get_conv_output_shape( image_shape, kernel_shape, 'full', sub_sample, filter_dilation) test4_params = get_conv_output_shape( image_shape, kernel_shape, (1, 2, 3), sub_sample, filter_dilation) self.assertTrue(test1_params == (3, 4, 4, 2, 4)) self.assertTrue(test2_params == (3, 4, 12, 5, 8)) self.assertTrue(test3_params == (3, 4, 20, 7, 10)) self.assertTrue(test4_params == (3, 4, 6, 4, 10)) class TestConvGradInputsShape(unittest.TestCase): def test_check_shape(self): for i in range(1, 20): for k in range(1, 10): for b in ('valid', 'half', 'full', (0, 2)): for s in (1, 2, 3): for d in (1, 2, 3): image_shape = (59, 61, i, i) kernel_shape = (67, 61, k, k) # compute the output that these inputs and parameters would produce computed_shape = get_conv_output_shape( image_shape, kernel_shape, b, (s, s), (d, d)) # this should be accepted self.assertTrue(check_conv_gradinputs_shape( image_shape, kernel_shape, computed_shape, b, (s, s), (d, d))) # one or more None should also be accepted trial_shape = (None, None, computed_shape[2], None) self.assertTrue(check_conv_gradinputs_shape( image_shape, kernel_shape, trial_shape, b, (s, s), (d, d))) # the batch size and number of filters are important trial_shape = (1, 1, computed_shape[2], computed_shape[3]) self.assertFalse(check_conv_gradinputs_shape( image_shape, kernel_shape, trial_shape, b, (s, s), (d, d))) # outputs that are too large or too small should be rejected for o in (-3, -2, -1, 1, 2, 3): trial_shape = (computed_shape[0], computed_shape[1], computed_shape[2] + o, computed_shape[3] + o) self.assertFalse(check_conv_gradinputs_shape( image_shape, kernel_shape, trial_shape, b, (s, s), (d, d))) def test_get_shape(self): for i in range(1, 20): for k in range(1, 10): for b in ('valid', 'half', 'full', (0, 2)): for d in (1, 2, 3): image_shape = (59, 61, i, i) kernel_shape = (67, 61, k, k) # compute the output that these inputs and parameters would produce output_shape = get_conv_output_shape( image_shape, kernel_shape, b, (1, 1), (d, d)) # compute the image_shape given this output_shape computed_image_shape = get_conv_gradinputs_shape( kernel_shape, output_shape, b, (1, 1), (d, d)) self.assertEqual(computed_image_shape, image_shape) # if subsample > 1, the shape should be None computed_image_shape = get_conv_gradinputs_shape( kernel_shape, output_shape, b, (2, 3), (d, d)) image_shape_with_None = image_shape[:2] + (None, None) self.assertEqual(computed_image_shape, image_shape_with_None) # compute the kernel_shape given this output_shape computed_kernel_shape = get_conv_gradweights_shape( image_shape, output_shape, b, (1, 1), (d, d)) # if border_mode == 'half', the shape should be None if b == 'half': kernel_shape_with_None = kernel_shape[:2] + (None, None) self.assertEqual(computed_kernel_shape, kernel_shape_with_None) else: self.assertEqual(computed_kernel_shape, kernel_shape) # if subsample > 1, the shape should be None computed_kernel_shape = get_conv_gradweights_shape( kernel_shape, output_shape, b, (2, 3), (d, d)) kernel_shape_with_None = kernel_shape[:2] + (None, None) self.assertEqual(computed_kernel_shape, kernel_shape_with_None) class TestAssertConvShape(unittest.TestCase): def test_basic(self): shape = tuple(tensor.iscalar() for i in range(4)) f = theano.function(shape, assert_conv_shape(shape)) self.assertEqual([1, 2, 3, 4], f(1, 2, 3, 4)) self.assertEqual([0, 0, 1, 1], f(0, 0, 1, 1)) assert_raises(AssertionError, f, 3, 3, 3, 0) assert_raises(AssertionError, f, 3, 3, 0, 3) assert_raises(AssertionError, f, 3, 3, -1, 3) assert_raises(AssertionError, f, 3, -1, 3, 3) assert_raises(AssertionError, f, -1, 3, 3, 3) class TestAssertShape(unittest.TestCase): @change_flags([("conv.assert_shape", True)]) def test_basic(self): x = tensor.tensor4() s1 = tensor.iscalar() s2 = tensor.iscalar() expected_shape = [None, s1, s2, None] f = theano.function([x, s1, s2], assert_shape(x, expected_shape)) v = np.zeros((3, 5, 7, 11), dtype='float32') self.assertEqual(0, np.sum(f(v, 5, 7))) assert_raises(AssertionError, f, v, 5, 0) assert_raises(AssertionError, f, v, 5, 9) assert_raises(AssertionError, f, v, 0, 7) assert_raises(AssertionError, f, v, 7, 7) @change_flags([("conv.assert_shape", True)]) def test_shape_check_conv2d(self): input = tensor.tensor4() filters = tensor.tensor4() out = conv.conv2d(input, filters, input_shape=(3, 5, 7, 11), filter_shape=(7, 5, 3, 3)) f = theano.function([input, filters], out) # mismatched input_shape assert_raises(AssertionError, f, np.zeros((3, 5, 9, 11), dtype='float32'), np.zeros((7, 5, 3, 3), dtype='float32')) # mismatched filter_shape assert_raises(AssertionError, f, np.zeros((3, 5, 7, 11), dtype='float32'), np.zeros((7, 5, 2, 2), dtype='float32')) @change_flags([("conv.assert_shape", True)]) def test_shape_check_conv3d(self): if theano.config.cxx == "": raise SkipTest("test needs cxx") input = tensor.tensor5() filters = tensor.tensor5() out = conv.conv3d(input, filters, input_shape=(3, 5, 7, 11, 13), filter_shape=(7, 5, 3, 3, 3)) f = theano.function([input, filters], out) # mismatched input_shape assert_raises(AssertionError, f, np.zeros((3, 5, 9, 11, 13), dtype='float32'), np.zeros((7, 5, 3, 3, 3), dtype='float32')) # mismatched filter_shape assert_raises(AssertionError, f, np.zeros((3, 5, 7, 11, 13), dtype='float32'), np.zeros((7, 5, 2, 2, 2), dtype='float32')) @change_flags([("conv.assert_shape", True)]) def test_shape_check_conv2d_grad_wrt_inputs(self): output_grad = tensor.tensor4() filters = tensor.tensor4() out = conv.conv2d_grad_wrt_inputs(output_grad, filters, input_shape=(None, None, 7, 11), filter_shape=(7, 5, 3, 3)) f = theano.function([output_grad, filters], out) # mismatched filter_shape assert_raises(AssertionError, f, np.zeros((3, 6, 5, 9), dtype='float32'), np.zeros((7, 6, 3, 3), dtype='float32')) @change_flags([("conv.assert_shape", True)]) def test_shape_check_conv3d_grad_wrt_inputs(self): if theano.config.cxx == "": raise SkipTest("test needs cxx") output_grad = tensor.tensor5() filters = tensor.tensor5() out = conv.conv3d_grad_wrt_inputs(output_grad, filters, input_shape=(None, None, 7, 11, 13), filter_shape=(7, 5, 3, 3, 3)) f = theano.function([output_grad, filters], out) # mismatched filter_shape assert_raises(AssertionError, f, np.zeros((3, 6, 5, 9, 11), dtype='float32'), np.zeros((7, 6, 3, 3, 3), dtype='float32')) @change_flags([("conv.assert_shape", True)]) def test_shape_check_conv2d_grad_wrt_weights(self): input = tensor.tensor4() output_grad = tensor.tensor4() out = conv.conv2d_grad_wrt_weights(input, output_grad, filter_shape=(None, None, 3, 3), input_shape=(3, 5, 7, 11)) f = theano.function([input, output_grad], out) # mismatched filter_shape assert_raises(AssertionError, f, np.zeros((3, 6, 7, 11), dtype='float32'), np.zeros((3, 7, 5, 9), dtype='float32')) @change_flags([("conv.assert_shape", True)]) def test_shape_check_conv3d_grad_wrt_weights(self): if theano.config.cxx == "": raise SkipTest("test needs cxx") input = tensor.tensor5() output_grad = tensor.tensor5() out = conv.conv3d_grad_wrt_weights(input, output_grad, filter_shape=(None, None, 3, 3, 3), input_shape=(3, 5, 7, 11, 13)) f = theano.function([input, output_grad], out) # mismatched filter_shape assert_raises(AssertionError, f, np.zeros((3, 6, 7, 11, 13), dtype='float32'), np.zeros((3, 7, 5, 9, 11), dtype='float32')) class BaseTestConv(object): def get_output_shape(self, inputs_shape, filters_shape, subsample, border_mode, filter_dilation): dil_filters = tuple((s - 1) * d + 1 for s, d in zip(filters_shape[2:], filter_dilation)) if border_mode == "valid": border_mode = (0,) * (len(inputs_shape) - 2) if border_mode == "half": border_mode = tuple(d // 2 for d in dil_filters) if border_mode == "full": border_mode = tuple(d - 1 for d in dil_filters) batch_size = inputs_shape[0] num_filters = filters_shape[0] return ((batch_size, num_filters,) + tuple(None if i is None or k is None else ((i + 2 * pad - ((k - 1) * fd + 1)) // d + 1) for i, k, d, pad, fd in zip(inputs_shape[2:], filters_shape[2:], subsample, border_mode, filter_dilation))) def run_fwd(self, inputs_shape, filters_shape, conv_fn, conv_op, ref, subsample=None, verify_grad=True, mode=None, border_mode='valid', filter_flip=True, provide_shape=False, target_op=None, check_trace=False, filter_dilation=None): if subsample is None: subsample = (1,) * (len(inputs_shape) - 2) if filter_dilation is None: filter_dilation = (1,) * (len(inputs_shape) - 2) inputs_val = np.random.random(inputs_shape).astype('float32') filters_val = np.random.random(filters_shape).astype('float32') # scale down values to prevent rounding errors inputs_val /= 10 filters_val /= 10 inputs = self.shared(inputs_val) filters = self.shared(filters_val) if provide_shape: imshp = inputs_shape kshp = filters_shape else: imshp = None kshp = None if filter_flip: conv_mode = 'conv' else: conv_mode = 'cross' c_ref = ref(inputs, filters, border_mode=border_mode, subsample=subsample, conv_mode=conv_mode, filter_dilation=filter_dilation) c = conv_fn(inputs, filters, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, input_shape=imshp, filter_shape=kshp, filter_dilation=filter_dilation) f_ref = theano.function([], c_ref, mode='FAST_RUN') f = theano.function([], c, mode=mode) if target_op is not None: assert any([isinstance(n.op, target_op) for n in f.maker.fgraph.toposort()]) if check_trace: assert_true(check_stack_trace(f, ops_to_check=target_op)) res_ref = np.array(f_ref()) res = np.array(f()) utt.assert_allclose(res_ref, res) if verify_grad and inputs_val.size > 0 and filters_val.size > 0 and res.size > 0: utt.verify_grad(conv_op(border_mode=border_mode, imshp=imshp, kshp=kshp, subsample=subsample, filter_dilation=filter_dilation), [inputs_val, filters_val], mode=mode) def run_gradweight(self, inputs_shape, filters_shape, output_shape, gradWeights_fn, ref, subsample=None, filter_flip=True, verify_grad=True, mode=None, border_mode='valid', provide_shape=False, target_op=None, check_trace=False, filter_dilation=None): if subsample is None: subsample = (1,) * (len(inputs_shape) - 2) if filter_dilation is None: filter_dilation = (1,) * (len(inputs_shape) - 2) inputs_val = np.random.random(inputs_shape).astype('float32') output_val = np.random.random(output_shape).astype('float32') inputs = self.shared(inputs_val) output = self.shared(output_val) if provide_shape: imshp = inputs_shape kshp = filters_shape else: imshp = None kshp = None if filter_flip: conv_mode = 'conv' else: conv_mode = 'cross' c = gradWeights_fn(border_mode=border_mode, filter_flip=filter_flip, subsample=subsample, imshp=imshp, kshp=kshp, filter_dilation=filter_dilation) c = c(inputs, output, filters_shape[2:]) c_ref = ref(inputs, output, filters_shape, border_mode=border_mode, subsample=subsample, conv_mode=conv_mode, filter_dilation=filter_dilation) f = theano.function([], c, mode=mode) f_ref = theano.function([], c_ref, mode='FAST_RUN') if target_op is not None: assert any([isinstance(n.op, target_op) for n in f.maker.fgraph.toposort()]) if check_trace: assert_true(check_stack_trace(f, ops_to_check=target_op)) res_ref = np.array(f_ref()) res = np.array(f()) utt.assert_allclose(res_ref, res) def abstract_conv_gradweight(inputs_val, output_val): conv_op = gradWeights_fn(border_mode=border_mode, subsample=subsample, filter_dilation=filter_dilation) return conv_op(inputs_val, output_val, filters_shape[2:]) if verify_grad and inputs_val.size > 0 and output_val.size > 0 and res.size > 0: utt.verify_grad(abstract_conv_gradweight, [inputs_val, output_val], mode=mode, eps=1) def run_gradinput(self, inputs_shape, filters_shape, output_shape, gradInputs_fn, ref, subsample=None, filter_flip=True, verify_grad=True, mode=None, border_mode='valid', provide_shape=False, target_op=None, check_trace=False, filter_dilation=None): if subsample is None: subsample = (1,) * (len(inputs_shape) - 2) if filter_dilation is None: filter_dilation = (1,) * (len(inputs_shape) - 2) output_val = np.random.random(output_shape).astype('float32') filters_val = np.random.random(filters_shape).astype('float32') output = self.shared(output_val) filters = self.shared(filters_val) if provide_shape: imshp = inputs_shape kshp = filters_shape else: imshp = None kshp = None if filter_flip: conv_mode = 'conv' else: conv_mode = 'cross' c = gradInputs_fn(border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, imshp=imshp, kshp=kshp, filter_dilation=filter_dilation) c = c(filters, output, inputs_shape[2:]) f = theano.function([], c, mode=mode) # ref is set to None for the inconsistent-shape tests. # The reference function also raises an exception, which would # mask the exception generated by the target implementation. if ref is not None: c_ref = ref(filters, output, inputs_shape, border_mode=border_mode, subsample=subsample, conv_mode=conv_mode, filter_dilation=filter_dilation) f_ref = theano.function([], c_ref, mode='FAST_RUN') if target_op is not None: assert any([isinstance(n.op, target_op) for n in f.maker.fgraph.toposort()]) if check_trace: assert_true(check_stack_trace(f, ops_to_check=target_op)) res = np.array(f()) if ref is not None: res_ref = np.array(f_ref()) utt.assert_allclose(res_ref, res) def abstract_conv_gradinputs(filters_val, output_val): conv_op = gradInputs_fn(border_mode=border_mode, subsample=subsample, filter_dilation=filter_dilation) return conv_op(filters_val, output_val, inputs_shape[2:]) if verify_grad and filters_val.size > 0 and output_val.size > 0 and res.size > 0: utt.verify_grad(abstract_conv_gradinputs, [filters_val, output_val], mode=mode, eps=1) def test_all(self): if type(self) is BaseTestConv: raise SkipTest("base class") ds = self.default_subsamples db = self.default_border_mode dflip = self.default_filter_flip dprovide_shape = self.default_provide_shape for (i, f) in zip(self.inputs_shapes, self.filters_shapes): for provide_shape in self.provide_shape: yield (self.tcase, i, f, ds, db, dflip, provide_shape) if min(i) > 0 and min(f) > 0: for fd in self.filters_dilations: for s in self.subsamples: for b in self.border_modes: yield (self.tcase, i, f, s, b, dflip, dprovide_shape, fd) for flip in self.filter_flip: yield (self.tcase, i, f, ds, db, flip, dprovide_shape) class BaseTestConv2d(BaseTestConv): @classmethod def setup_class(cls): # This tests can run even when theano.config.blas.ldflags is empty. cls.inputs_shapes = [(8, 1, 6, 6), (8, 1, 8, 8), (2, 1, 7, 7), (6, 1, 10, 11), (2, 1, 6, 5), (1, 5, 9, 9), (0, 1, 6, 6), (1, 0, 6, 6), (1, 1, 6, 6)] cls.filters_shapes = [(5, 1, 2, 2), (4, 1, 3, 3), (2, 1, 3, 3), (1, 1, 2, 3), (4, 1, 1, 3), (4, 5, 3, 2), (1, 1, 2, 2), (1, 0, 2, 2), (0, 1, 2, 2)] cls.subsamples = [(1, 1), (2, 2), (2, 4)] cls.default_subsamples = (1, 1) cls.filters_dilations = [(1, 1), (1, 2), (2, 1)] cls.default_filters_dilations = (1, 1) cls.border_modes = ["valid", "half", "full", (0, 0), (1, 1), (5, 5), (5, 2)] cls.default_border_mode = (0, 0) cls.filter_flip = [True, False] cls.default_filter_flip = True cls.provide_shape = [True, False] cls.default_provide_shape = True cls.shared = staticmethod(theano.compile.shared) def test_gradinput_arbitrary_output_shapes(self): # this computes the grad wrt inputs for an output shape # that the forward convolution would not produce input_shape = (2, 1, 7, 7) filter_shape = (2, 1, 3, 3) for output_shape in [(2, 2, 8, 8), (2, 2, 9, 9), (2, 2, 12, 12)]: for border_mode in ["valid", "half", "full"]: computed_shape = get_conv_output_shape( input_shape, filter_shape, border_mode, self.default_subsamples, self.default_filters_dilations) # is this a valid combination? if tuple(computed_shape) == output_shape: yield (self.tcase_gi, input_shape, filter_shape, output_shape, self.default_subsamples, border_mode, True, True, self.default_filters_dilations, False) else: # expect an error yield (self.tcase_gi, input_shape, filter_shape, output_shape, self.default_subsamples, border_mode, True, True, self.default_filters_dilations, True) def test_gradinput_impossible_output_shapes(self): def run_for_output_offsets(image_shape, kernel_shape, s, border_mode, d): # outputs that are too large or too small should be rejected for o in (-3, -1, 1, 2): output_shape = (1, 1, computed_shape[2] + o, computed_shape[3] + o) # expect an error self.tcase_gi(image_shape, kernel_shape, output_shape, (s, s), border_mode, True, True, (d, d), True) for (i, k) in ((1, 1), (1, 2), (2, 1), (4, 2), (4, 3), (7, 3), (9, 5)): for border_mode in ('valid', 'half', 'full', (0, 2)): for (s, d) in ((1, 1), (1, 2), (2, 1), (2, 2), (3, 1), (1, 3)): image_shape = (1, 1, i, i) kernel_shape = (1, 1, k, k) # compute the output that these inputs and parameters would produce computed_shape = get_conv_output_shape( image_shape, kernel_shape, border_mode, (s, s), (d, d)) yield (run_for_output_offsets, image_shape, kernel_shape, s, border_mode, d) def run_fwd(self, inputs_shape, filters_shape, conv_fn=conv.conv2d, conv_op=conv.AbstractConv2d, ref=conv2d_corr, **kwargs): super(BaseTestConv2d, self).run_fwd( inputs_shape=inputs_shape, filters_shape=filters_shape, conv_fn=conv_fn, conv_op=conv_op, ref=ref, **kwargs) def run_gradweight(self, inputs_shape, filters_shape, output_shape, gradWeights_fn=conv.AbstractConv2d_gradWeights, ref=conv2d_corr_gw, **kwargs): super(BaseTestConv2d, self).run_gradweight( inputs_shape=inputs_shape, filters_shape=filters_shape, output_shape=output_shape, gradWeights_fn=gradWeights_fn, ref=ref, **kwargs) def run_gradinput(self, inputs_shape, filters_shape, output_shape, gradInputs_fn=conv.AbstractConv2d_gradInputs, ref=conv2d_corr_gi, **kwargs): super(BaseTestConv2d, self).run_gradinput( inputs_shape=inputs_shape, filters_shape=filters_shape, output_shape=output_shape, gradInputs_fn=gradInputs_fn, ref=ref, **kwargs) class TestCorrConv2d(BaseTestConv2d): @classmethod def setup_class(cls): # This tests can run even when theano.config.blas.ldflags is empty. BaseTestConv2d.setup_class() def tcase(self, i, f, s, b, flip, provide_shape, fd=(1, 1)): o = self.get_output_shape(i, f, s, b, fd) # This tests can run even when theano.config.blas.ldflags is empty. if (not theano.config.cxx or theano.config.mode == "FAST_COMPILE"): raise SkipTest("Need blas to test conv2d") self.run_fwd(inputs_shape=i, filters_shape=f, subsample=s, verify_grad=True, provide_shape=provide_shape, border_mode=b, filter_flip=flip, target_op=CorrMM, check_trace=True, filter_dilation=fd) self.run_gradweight(inputs_shape=i, filters_shape=f, output_shape=o, subsample=s, verify_grad=True, provide_shape=provide_shape, border_mode=b, filter_flip=flip, target_op=CorrMM_gradWeights, check_trace=True, filter_dilation=fd) self.run_gradinput(inputs_shape=i, filters_shape=f, output_shape=o, subsample=s, verify_grad=True, provide_shape=provide_shape, border_mode=b, filter_flip=flip, target_op=CorrMM_gradInputs, check_trace=True, filter_dilation=fd) def tcase_gi(self, i, f, o, s, b, flip, provide_shape, fd=(1, 1), expect_error=False): # This tests can run even when theano.config.blas.ldflags is empty. if (not theano.config.cxx or theano.config.mode == "FAST_COMPILE"): raise SkipTest("Need blas to test conv2d") if not expect_error: self.run_gradinput(inputs_shape=i, filters_shape=f, output_shape=o, subsample=s, verify_grad=True, provide_shape=provide_shape, border_mode=b, filter_flip=flip, target_op=CorrMM_gradInputs, check_trace=True, filter_dilation=fd) else: assert_raises(ValueError, self.run_gradinput, inputs_shape=i, filters_shape=f, output_shape=o, subsample=s, verify_grad=False, provide_shape=provide_shape, border_mode=b, filter_flip=flip, target_op=CorrMM_gradInputs, ref=None, check_trace=True, filter_dilation=fd) class TestAbstractConvNoOptim(BaseTestConv2d): @classmethod def setup_class(cls): # This tests can run even when theano.config.blas.ldflags is empty. BaseTestConv2d.setup_class() cls.inputs_shapes = [(8, 1, 6, 6)] cls.filters_shapes = [(5, 1, 2, 2)] cls.subsamples = [(1, 1), (2, 2)] cls.filters_dilations = [(1, 1), (1, 2), (2, 1)] cls.border_modes = ["valid", "half", "full"] cls.filter_flip = [True] cls.provide_shape = [False] if not theano.tensor.nnet.abstract_conv.imported_scipy_signal: raise SkipTest("SciPy needed") def tcase(self, i, f, s, b, flip, provide_shape, fd=(1, 1)): o = self.get_output_shape(i, f, s, b, fd) mode = theano.Mode(optimizer=None) if not theano.config.cxx: raise SkipTest("Need cxx to test conv2d") self.run_fwd(inputs_shape=i, filters_shape=f, subsample=s, verify_grad=True, provide_shape=provide_shape, border_mode=b, filter_flip=flip, target_op=None, check_trace=True, filter_dilation=fd, mode=mode) self.run_gradweight(inputs_shape=i, filters_shape=f, output_shape=o, subsample=s, verify_grad=True, provide_shape=provide_shape, border_mode=b, filter_flip=flip, target_op=None, check_trace=True, filter_dilation=fd, mode=mode) self.run_gradinput(inputs_shape=i, filters_shape=f, output_shape=o, subsample=s, verify_grad=True, provide_shape=provide_shape, border_mode=b, filter_flip=flip, target_op=None, check_trace=True, filter_dilation=fd, mode=mode) def tcase_gi(self, i, f, o, s, b, flip, provide_shape, fd=(1, 1), expect_error=False): if not theano.config.cxx: raise SkipTest("Need cxx to test conv2d") mode = theano.Mode(optimizer=None) if not expect_error: self.run_gradinput(inputs_shape=i, filters_shape=f, output_shape=o, subsample=s, verify_grad=True, provide_shape=provide_shape, border_mode=b, filter_flip=flip, target_op=None, check_trace=True, filter_dilation=fd, mode=mode) else: assert_raises(ValueError, self.run_gradinput, inputs_shape=i, filters_shape=f, output_shape=o, subsample=s, verify_grad=False, provide_shape=provide_shape, border_mode=b, filter_flip=flip, target_op=None, check_trace=True, filter_dilation=fd, ref=None, mode=mode) class BaseTestConv3d(BaseTestConv): @classmethod def setup_class(cls): # This tests can run even when theano.config.blas.ldflags is empty. cls.inputs_shapes = [(2, 1, 5, 5, 5), (1, 2, 7, 5, 6), (0, 1, 5, 5, 5), (1, 0, 5, 5, 5), (1, 1, 5, 5, 5)] cls.filters_shapes = [(2, 1, 2, 2, 2), (1, 2, 2, 1, 3), (1, 1, 2, 2, 2), (1, 0, 2, 2, 2), (0, 1, 2, 2, 2)] cls.subsamples = [(1, 1, 1), (2, 2, 2), (1, 2, 3)] cls.default_subsamples = (1, 1, 1) cls.filters_dilations = [(1, 1, 1), (1, 2, 1), (2, 1, 2)] cls.default_filters_dilations = (1, 1, 1) cls.border_modes = ["valid", "half", "full", (0, 0, 0), (2, 2, 3)] cls.default_border_mode = (0, 0, 0) cls.filter_flip = [True, False] cls.default_filter_flip = True cls.provide_shape = [True, False] cls.default_provide_shape = True cls.shared = staticmethod(theano.compile.shared) def test_gradinput_arbitrary_output_shapes(self): # this computes the grad wrt inputs for an output shape # that the forward convolution would not produce input_shape = (2, 1, 7, 7, 7) filter_shape = (1, 1, 3, 3, 3) for output_shape in [(2, 1, 8, 8, 8), (2, 1, 9, 9, 9), (2, 1, 12, 12, 12)]: for border_mode in ["valid", "half", "full"]: # compute the output that these inputs and parameters would produce computed_shape = get_conv_output_shape( input_shape, filter_shape, border_mode, self.default_subsamples, self.default_filters_dilations) # is this a valid combination? if tuple(computed_shape) == output_shape: yield (self.tcase_gi, input_shape, filter_shape, output_shape, self.default_subsamples, border_mode, True, True, self.default_filters_dilations, False) else: # expect an error yield (self.tcase_gi, input_shape, filter_shape, output_shape, self.default_subsamples, border_mode, True, True, self.default_filters_dilations, True) def test_gradinput_impossible_output_shapes(self): def run_for_output_offsets(image_shape, kernel_shape, s, border_mode, d): # outputs that are too large or too small should be rejected for o in (-3, -1, 1, 2): output_shape = (1, 1, computed_shape[2] + o, computed_shape[3] + o, computed_shape[4] + o) # expect an error self.tcase_gi(image_shape, kernel_shape, output_shape, (s, s), border_mode, True, True, (d, d), True) for (i, k) in ((1, 1), (1, 2), (2, 1), (4, 2), (4, 3), (7, 3), (9, 5)): for border_mode in ('valid', 'half', 'full', (0, 2, 1)): for (s, d) in ((1, 1), (1, 2), (2, 1), (2, 2), (3, 1), (1, 3)): image_shape = (1, 1, i, i, i) kernel_shape = (1, 1, k, k, k) # compute the output that these inputs and parameters would produce computed_shape = get_conv_output_shape( image_shape, kernel_shape, border_mode, (s, s, s), (d, d, d)) yield (run_for_output_offsets, image_shape, kernel_shape, s, border_mode, d) def run_fwd(self, inputs_shape, filters_shape, conv_fn=conv.conv3d, conv_op=conv.AbstractConv3d, ref=conv3d_corr, **kwargs): super(BaseTestConv3d, self).run_fwd( inputs_shape=inputs_shape, filters_shape=filters_shape, conv_fn=conv_fn, conv_op=conv_op, ref=ref, **kwargs) def run_gradweight(self, inputs_shape, filters_shape, output_shape, gradWeights_fn=conv.AbstractConv3d_gradWeights, ref=conv3d_corr_gw, **kwargs): super(BaseTestConv3d, self).run_gradweight( inputs_shape=inputs_shape, filters_shape=filters_shape, output_shape=output_shape, gradWeights_fn=gradWeights_fn, ref=ref, **kwargs) def run_gradinput(self, inputs_shape, filters_shape, output_shape, gradInputs_fn=conv.AbstractConv3d_gradInputs, ref=conv3d_corr_gi, **kwargs): super(BaseTestConv3d, self).run_gradinput( inputs_shape=inputs_shape, filters_shape=filters_shape, output_shape=output_shape, gradInputs_fn=gradInputs_fn, ref=ref, **kwargs) class TestCorrConv3d(BaseTestConv3d): @classmethod def setup_class(cls): # This tests can run even when theano.config.blas.ldflags is empty. BaseTestConv3d.setup_class() def tcase(self, i, f, s, b, flip, provide_shape, fd=(1, 1, 1)): o = self.get_output_shape(i, f, s, b, fd) # This test can run even when theano.config.blas.ldflags is empty. if (not theano.config.cxx or theano.config.mode == "FAST_COMPILE"): raise SkipTest("Need blas to test conv3d") self.run_fwd(inputs_shape=i, filters_shape=f, subsample=s, verify_grad=True, provide_shape=provide_shape, border_mode=b, filter_flip=flip, target_op=Corr3dMM, check_trace=True, filter_dilation=fd) self.run_gradweight(inputs_shape=i, filters_shape=f, output_shape=o, subsample=s, verify_grad=True, provide_shape=provide_shape, border_mode=b, filter_flip=flip, target_op=Corr3dMM_gradWeights, check_trace=True, filter_dilation=fd) self.run_gradinput(inputs_shape=i, filters_shape=f, output_shape=o, subsample=s, verify_grad=True, provide_shape=provide_shape, border_mode=b, filter_flip=flip, target_op=Corr3dMM_gradInputs, check_trace=True, filter_dilation=fd) def tcase_gi(self, i, f, o, s, b, flip, provide_shape, fd=(1, 1, 1), expect_error=False): # This test can run even when theano.config.blas.ldflags is empty. if (not theano.config.cxx or theano.config.mode == "FAST_COMPILE"): raise SkipTest("Need blas to test conv3d") if not expect_error: self.run_gradinput(inputs_shape=i, filters_shape=f, output_shape=o, subsample=s, verify_grad=True, provide_shape=provide_shape, border_mode=b, filter_flip=flip, target_op=Corr3dMM_gradInputs, check_trace=True, filter_dilation=fd) else: assert_raises(ValueError, self.run_gradinput, inputs_shape=i, filters_shape=f, output_shape=o, subsample=s, verify_grad=False, provide_shape=provide_shape, border_mode=b, filter_flip=flip, target_op=Corr3dMM_gradInputs, ref=None, check_trace=True, filter_dilation=fd) def test_constant_shapes(): # Check that the `imshp` and `kshp` parameters of the AbstractConv Ops # are rejected if not constant or None dummy_t4 = tensor.ftensor4() alloc_dummy_t4 = tensor.zeros((3, 5, 7, 11), dtype='float32') dummy_shape = tensor.lvector() dummy_one_shape = tensor.ones(4, dtype='int64') constant_vec_shape = tensor.constant([3, 5, 7, 11]) tuple_shape = (3, 5, 7, 11) list_shape = list(tuple_shape) constant_list_shape = [tensor.constant(i, dtype='int64') for i in tuple_shape] constant_tuple_shape = tuple(constant_list_shape) bad_shapes = ( dummy_shape, dummy_one_shape, dummy_t4.shape, alloc_dummy_t4.shape, constant_vec_shape, ) good_shapes = ( constant_list_shape, constant_tuple_shape, tuple_shape, list_shape ) ops_to_test = ( AbstractConv2d, AbstractConv2d_gradInputs, AbstractConv2d_gradWeights ) for op in ops_to_test: for shp in bad_shapes: assert_raises(ValueError, op, imshp=shp) assert_raises(ValueError, op, kshp=shp) for shp in good_shapes: op(imshp=shp) op(kshp=shp) class TestConvTypes(unittest.TestCase): def setUp(self): self.input = tensor.ftensor4() self.filters = tensor.ftensor4() self.topgrad = tensor.ftensor4() self.constant_tensor = np.zeros((3, 5, 7, 11), dtype='float32') def test_grad_types(self): # This function simply tests the behaviour of the AbstractConv # Ops, not their optimizations input = self.input filters = self.filters topgrad = self.topgrad out_shape = tensor.lvector() output = conv.conv2d(input, filters) grad_input, grad_filters = theano.grad(output.sum(), wrt=(input, filters)) assert grad_input.type == input.type, ( grad_input, grad_input.type, input, input.type) assert grad_filters.type == filters.type, ( grad_filters, grad_filters.type, filters, filters.type) grad_filters = conv.AbstractConv2d_gradWeights()( input, topgrad, out_shape) grad_input, grad_topgrad = theano.grad(grad_filters.sum(), wrt=(input, topgrad)) assert grad_input.type == input.type, ( grad_input, grad_input.type, input, input.type) assert grad_topgrad.type == topgrad.type, ( grad_topgrad, grad_topgrad.type, topgrad, topgrad.type) grad_input = conv.AbstractConv2d_gradInputs()( filters, topgrad, out_shape) grad_filters, grad_topgrad = theano.grad(grad_input.sum(), wrt=(filters, topgrad)) assert grad_filters.type == filters.type, ( grad_filters, grad_filters.type, filters, filters.type) assert grad_topgrad.type == topgrad.type, ( grad_topgrad, grad_topgrad.type, topgrad, topgrad.type) def test_constant_input(self): # Check the AbstractConv Ops for constant inputs input = self.input filters = self.filters topgrad = self.topgrad constant_tensor = self.constant_tensor out_shape = tensor.lvector() # Check the forward Op output = conv.conv2d(constant_tensor, filters) grad_filters = theano.grad(output.sum(), wrt=filters) assert grad_filters.type == filters.type, ( grad_filters, grad_filters.type, filters, filters.type) output = conv.conv2d(input, constant_tensor) grad_input = theano.grad(output.sum(), wrt=input) assert grad_input.type == input.type, ( grad_input, grad_input.type, input, input.type) # Check grad wrt weights grad_filters = conv.AbstractConv2d_gradWeights()( constant_tensor, topgrad, out_shape) grad_topgrad = theano.grad(grad_filters.sum(), wrt=topgrad) assert grad_topgrad.type == topgrad.type, ( grad_topgrad, grad_topgrad.type, topgrad, topgrad.type) grad_filters = conv.AbstractConv2d_gradWeights()( input, constant_tensor, out_shape) grad_input = theano.grad(grad_filters.sum(), wrt=input) assert grad_input.type == input.type, ( grad_input, grad_input.type, input, input.type) # Check grad wrt inputs grad_input = conv.AbstractConv2d_gradInputs()( constant_tensor, topgrad, out_shape) grad_topgrad = theano.grad(grad_input.sum(), wrt=topgrad) assert grad_topgrad.type == topgrad.type, ( grad_topgrad, grad_topgrad.type, topgrad, topgrad.type) grad_input = conv.AbstractConv2d_gradInputs()( filters, constant_tensor, out_shape) grad_filters = theano.grad(grad_input.sum(), wrt=filters) assert grad_filters.type == filters.type, ( grad_filters, grad_filters.type, filters, filters.type) class TestBilinearUpsampling(unittest.TestCase): # If theano.config.blas.ldflags is empty, Theano will use # a NumPy C implementation of [sd]gemm_. compile_mode = theano.compile.mode.get_default_mode() if theano.config.mode == "FAST_COMPILE": compile_mode = compile_mode.excluding("conv_gemm") compile_mode = compile_mode.excluding('AbstractConvCheck') elif not theano.config.cxx: compile_mode = compile_mode.excluding('AbstractConvCheck') def numerical_kernel_1D(self, ratio): """ Gets numerical 1D kernel for bilinear upsampling """ return np.array(list(range(1, ratio + 1)) + list(range(ratio - 1, 0, -1))) def numerical_kernel_2D(self, ratio): """ Gets numerical 2D kernel for bilinear upsampling """ return np.array([i * j for i in self.numerical_kernel_1D(ratio) for j in self.numerical_kernel_1D(ratio)]).\ reshape(2 * ratio - 1, 2 * ratio - 1) def test_bilinear_kernel_2D(self): # Test 2D kernels used in bilinear upsampling # # This method tests the correctness of the # 2D kernel values used in bilinear upsampling # for some upsampling ratios. for ratio in [2, 3, 4, 5, 6, 7, 8, 9]: # getting the un-normalized kernel kernel = bilinear_kernel_2D(ratio=ratio, normalize=False) f = theano.function([], kernel) kernel_2D = self.numerical_kernel_2D(ratio) utt.assert_allclose(kernel_2D, f()) # getting the normalized kernel kernel = bilinear_kernel_2D(ratio=ratio, normalize=True) f = theano.function([], kernel) kernel_2D = kernel_2D / float(ratio**2) utt.assert_allclose(kernel_2D, f()) def test_bilinear_kernel_1D(self): # Test 1D kernels used in bilinear upsampling # # This method tests the correctness of the # 1D kernel values used in bilinear upsampling # for some upsampling ratios. rat = tensor.iscalar() kernel_ten = bilinear_kernel_1D(ratio=rat, normalize=False) f_ten = theano.function([rat], kernel_ten) kernel_ten_norm = bilinear_kernel_1D(ratio=rat, normalize=True) f_ten_norm = theano.function([rat], kernel_ten_norm) for ratio in [2, 3, 4, 5, 6, 7, 8, 9]: # getting the un-normalized kernel kernel = bilinear_kernel_1D(ratio=ratio, normalize=False) f = theano.function([], kernel) kernel_1D = self.numerical_kernel_1D(ratio) utt.assert_allclose(kernel_1D, f()) utt.assert_allclose(kernel_1D, f_ten(ratio)) # getting the normalized kernel kernel = bilinear_kernel_1D(ratio=ratio, normalize=True) f = theano.function([], kernel) kernel_1D = kernel_1D / float(ratio) utt.assert_allclose(kernel_1D, f()) utt.assert_allclose(kernel_1D, f_ten_norm(ratio)) def numerical_upsampling_multiplier(self, ratio): """ Compute upsampling multiplier This method computes the multipliers of an array that will be upsampled using bilinear interpolation. Parameters ---------- ratio: int the ratio by which the array will be upsampled. Returns ------- 1D numpy array The multiplers that can be used in bilinear interpolation to upsample an array. int The size of the multipliers array """ kern = np.arange(ratio + 1) return kern, kern.shape[0] def get_upsampled_twobytwo_mat(self, two_by_two, ratio): """ Upsample 4D array with two rows and two columns This method gets a 4D numpy array with two rows and two columns and computes its upsampled array by using bilinear interpolation Parameters ---------- two_by_two: numpy 4D array The array that will be upsampled by bilinear interpolation. Array is of shape (batch size, num channels, 2, 2) ratio: int The ratio by which two_by_two's last two dimensions (row and col) will be upsampled. Returns ------- 4D numpy array The array upsampled by using bilinear interpolation. Array is of shape (batch size, num channels, 2*ratio, 2*ratio). """ kern, shp = self.numerical_upsampling_multiplier(ratio) up_1D = two_by_two[:, :, :, :1] * kern[::-1] + \ two_by_two[:, :, :, 1:] * kern up_2D = up_1D[:, :, :1, :] * kern[::-1][:, np.newaxis] + \ up_1D[:, :, 1:, :] * kern[:, np.newaxis] num_concat = (ratio - 1) // 2 for i in range(num_concat): up_2D = np.concatenate([up_2D[:, :, :1, :], up_2D], axis=2) up_2D = np.concatenate([up_2D, up_2D[:, :, -1:, :]], axis=2) up_2D = np.concatenate([up_2D[:, :, :, :1], up_2D], axis=3) up_2D = np.concatenate([up_2D, up_2D[:, :, :, -1:]], axis=3) if ratio % 2 == 0: up_2D = np.concatenate([up_2D, up_2D[:, :, -1:, :]], axis=2) up_2D = np.concatenate([up_2D, up_2D[:, :, :, -1:]], axis=3) return up_2D / float(ratio)**2 def test_bilinear_upsampling_1D(self): # Test bilinear upsampling using 1D kernels # # This method tests the bilinear_upsampling method # when using 1D kernels for some upsampling ratios. # upsampling for a ratio of two input_x = np.array([[[[1, 2], [3, 4]]]], dtype=theano.config.floatX) for ratio in [2, 3, 4, 5, 6, 7, 8, 9]: bilin_mat = bilinear_upsampling(input=input_x, ratio=ratio, batch_size=1, num_input_channels=1, use_1D_kernel=True) f = theano.function([], bilin_mat, mode=self.compile_mode) up_mat_2d = self.get_upsampled_twobytwo_mat(input_x, ratio) utt.assert_allclose(f(), up_mat_2d, rtol=1e-06) def test_bilinear_upsampling_reshaping(self): # Test bilinear upsampling without giving shape information # # This method tests the bilinear_upsampling method # without giving batch_size and num_input_channels # upsampling for a ratio of two input_x = np.array([[[[1, 2], [3, 4]]]], dtype=theano.config.floatX) for ratio in [2, 3]: for use_1D_kernel in [True, False]: bilin_mat = bilinear_upsampling(input=input_x, ratio=ratio, batch_size=None, num_input_channels=None, use_1D_kernel=use_1D_kernel) f = theano.function([], bilin_mat, mode=self.compile_mode) up_mat_2d = self.get_upsampled_twobytwo_mat(input_x, ratio) utt.assert_allclose(f(), up_mat_2d, rtol=1e-06) def test_compare_1D_and_2D_upsampling_values(self): # Compare 1D and 2D upsampling # # This method verifies the bilinear upsampling done by using # 1D and 2D kernels will generate the same result. # checking upsampling with ratio 5 input_x = np.random.rand(5, 4, 6, 7).astype(theano.config.floatX) mat_1D = bilinear_upsampling(input=input_x, ratio=5, batch_size=5, num_input_channels=4, use_1D_kernel=True) mat_2D = bilinear_upsampling(input=input_x, ratio=5, batch_size=5, num_input_channels=4, use_1D_kernel=False) f_1D = theano.function([], mat_1D, mode=self.compile_mode) f_2D = theano.function([], mat_2D, mode=self.compile_mode) utt.assert_allclose(f_1D(), f_2D(), rtol=1e-06) # checking upsampling with ratio 8 input_x = np.random.rand(12, 11, 10, 7).astype(theano.config.floatX) mat_1D = bilinear_upsampling(input=input_x, ratio=8, batch_size=12, num_input_channels=11, use_1D_kernel=True) mat_2D = bilinear_upsampling(input=input_x, ratio=8, batch_size=12, num_input_channels=11, use_1D_kernel=False) f_1D = theano.function([], mat_1D, mode=self.compile_mode) f_2D = theano.function([], mat_2D, mode=self.compile_mode) utt.assert_allclose(f_1D(), f_2D(), rtol=1e-06) def test_fractional_bilinear_upsampling(self): """Test bilinear upsampling with nonsimilar fractional row and col ratios """ input_x = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]], [[9, 10], [11, 12]]], ndmin=4).astype(theano.config.floatX) up_x = bilinear_upsampling(input=input_x, frac_ratio=((7, 4), (5, 3)), use_1D_kernel=False) num_up_x = np.array( [[[[1., 1.2, 1.8, 2.], [1.28571429, 1.48571429, 2.08571429, 2.28571429], [2.42857143, 2.62857143, 3.22857143, 3.42857143], [3., 3.2, 3.8, 4.]], [[5., 5.2, 5.8, 6.], [5.28571429, 5.48571429, 6.08571429, 6.28571429], [6.42857143, 6.62857143, 7.22857143, 7.42857143], [7., 7.2, 7.8, 8.]], [[9., 9.2, 9.8, 10.], [9.28571429, 9.48571429, 10.08571429, 10.28571429], [10.42857143, 10.62857143, 11.22857143, 11.42857143], [11., 11.2, 11.8, 12.]]]] ).astype(theano.config.floatX) f_up_x = theano.function([], up_x, mode=self.compile_mode) utt.assert_allclose(f_up_x(), num_up_x, rtol=1e-6) class TestConv2dTranspose(unittest.TestCase): mode = None def test_interface(self): # Test conv2d_transpose wrapper. # # This method tests that the order of the filter's # axes expected by the function produces the correct # output shape. if theano.config.cxx == "": raise SkipTest("test needs cxx") mode = self.mode if theano.config.mode == "FAST_COMPILE": mode = theano.compile.get_mode( mode).excluding("conv_gemm").excluding("AbstractConvCheck") output = theano.function( inputs=[], outputs=conv2d_transpose(input=tensor.ones((2, 2, 4, 4)), filters=tensor.ones((2, 1, 4, 4)), output_shape=(2, 1, 10, 10), input_dilation=(2, 2)), mode=mode)() expected_output = np.array( [[[[2, 2, 4, 4, 4, 4, 4, 4, 2, 2], [2, 2, 4, 4, 4, 4, 4, 4, 2, 2], [4, 4, 8, 8, 8, 8, 8, 8, 4, 4], [4, 4, 8, 8, 8, 8, 8, 8, 4, 4], [4, 4, 8, 8, 8, 8, 8, 8, 4, 4], [4, 4, 8, 8, 8, 8, 8, 8, 4, 4], [4, 4, 8, 8, 8, 8, 8, 8, 4, 4], [4, 4, 8, 8, 8, 8, 8, 8, 4, 4], [2, 2, 4, 4, 4, 4, 4, 4, 2, 2], [2, 2, 4, 4, 4, 4, 4, 4, 2, 2]]]] * 2) np.testing.assert_equal(output, expected_output) class TestConv2dGrads(unittest.TestCase): def setUp(self): if (not theano.config.cxx or theano.config.mode == "FAST_COMPILE"): raise SkipTest("Need blas to test conv2d") self.random_stream = np.random.RandomState(utt.fetch_seed()) self.inputs_shapes = [(8, 1, 12, 12), (1, 1, 5, 5), (1, 1, 5, 6), (1, 1, 6, 6)] self.filters_shapes = [(5, 1, 2, 2), (1, 1, 3, 3)] self.subsamples = [(1, 1), (2, 2)] self.border_modes = ["valid", "full"] self.filter_flip = [True, False] self.output_grad = theano.tensor.tensor4() self.output_grad_wrt = theano.tensor.tensor4() self.x = theano.tensor.tensor4('x', theano.config.floatX) # inputs self.w = theano.tensor.tensor4('w', theano.config.floatX) # filter weights def test_conv2d_grad_wrt_inputs(self): # Compares calculated abstract grads wrt inputs with the fwd grads # This method checks the outputs of conv2_grad_wrt_inputs against # the outputs of T.nnet.conv forward grads to make sure the # results are the same. for (in_shape, fltr_shape) in zip(self.inputs_shapes, self.filters_shapes): for bm in self.border_modes: for ss in self.subsamples: for ff in self.filter_flip: input_val = self.random_stream.random_sample(in_shape).astype(theano.config.floatX) filter_val = self.random_stream.random_sample(fltr_shape).astype(theano.config.floatX) out_grad_shape = theano.tensor.nnet.abstract_conv.get_conv_output_shape(image_shape=in_shape, kernel_shape=fltr_shape, border_mode=bm, subsample=ss) out_grad_val = self.random_stream.random_sample(out_grad_shape).astype(theano.config.floatX) conv_out = theano.tensor.nnet.conv2d(self.x, filters=self.w, border_mode=bm, subsample=ss, input_shape=in_shape, filter_shape=fltr_shape, filter_flip=ff ) conv_grad = theano.grad(conv_out.sum(), wrt=self.x, known_grads={conv_out: self.output_grad}) f_old = theano.function([self.x, self.w, self.output_grad], conv_grad) conv_wrt_i_out = theano.tensor.nnet.abstract_conv.conv2d_grad_wrt_inputs(output_grad=self.output_grad_wrt, filters=self.w, border_mode=bm, subsample=ss, input_shape=in_shape, filter_shape=fltr_shape, filter_flip=ff ) f_new = theano.function([self.w, self.output_grad_wrt], conv_wrt_i_out) # check that they're equal utt.assert_allclose(f_new(filter_val, out_grad_val), f_old(input_val, filter_val, out_grad_val)) def test_conv2d_grad_wrt_weights(self): # Compares calculated abstract grads wrt weights with the fwd grads # This method checks the outputs of conv2_grad_wrt_weights against # the outputs of T.nnet.conv forward grads to make sure the # results are the same. for (in_shape, fltr_shape) in zip(self.inputs_shapes, self.filters_shapes): for bm in self.border_modes: for ss in self.subsamples: for ff in self.filter_flip: input_val = self.random_stream.random_sample(in_shape).astype(theano.config.floatX) filter_val = self.random_stream.random_sample(fltr_shape).astype(theano.config.floatX) out_grad_shape = theano.tensor.nnet.abstract_conv.get_conv_output_shape(image_shape=in_shape, kernel_shape=fltr_shape, border_mode=bm, subsample=ss) out_grad_val = self.random_stream.random_sample(out_grad_shape).astype(theano.config.floatX) conv_out = theano.tensor.nnet.conv2d(self.x, filters=self.w, border_mode=bm, subsample=ss, input_shape=in_shape, filter_shape=fltr_shape, filter_flip=ff ) conv_grad = theano.grad(conv_out.sum(), wrt=self.w, known_grads={conv_out: self.output_grad}) f_old = theano.function([self.x, self.w, self.output_grad], conv_grad) conv_wrt_w_out = theano.tensor.nnet.abstract_conv.conv2d_grad_wrt_weights(self.x, output_grad=self.output_grad_wrt, border_mode=bm, subsample=ss, input_shape=in_shape, filter_shape=fltr_shape, filter_flip=ff ) f_new = theano.function([self.x, self.output_grad_wrt], conv_wrt_w_out) utt.assert_allclose(f_new(input_val, out_grad_val), f_old(input_val, filter_val, out_grad_val)) class Grouped_conv_noOptim(unittest.TestCase): conv = theano.tensor.nnet.abstract_conv.AbstractConv2d conv_gradw = theano.tensor.nnet.abstract_conv.AbstractConv2d_gradWeights conv_gradi = theano.tensor.nnet.abstract_conv.AbstractConv2d_gradInputs conv_op = theano.tensor.nnet.abstract_conv.AbstractConv2d conv_gradw_op = theano.tensor.nnet.abstract_conv.AbstractConv2d_gradWeights conv_gradi_op = theano.tensor.nnet.abstract_conv.AbstractConv2d_gradInputs mode = theano.Mode(optimizer=None) is_dnn = False def setUp(self): self.num_groups = [3, 2, 4, 4] self.border_mode = 'valid' self.subsample = (1, 1) self.img_shape = [(5, 6, 5, 5), (4, 4, 7, 5), (3, 8, 5, 3), (2, 4, 7, 7)] self.kern_shape = [(6, 2, 3, 3), (6, 2, 5, 3), (4, 2, 3, 3), (4, 1, 3, 5)] self.top_shape = [(5, 6, 3, 3), (4, 6, 3, 3), (3, 4, 3, 1), (2, 4, 5, 3)] self.filter_dilation = (1, 1) self.ref_mode = 'FAST_RUN' self.convdim = 2 self.corr_fwd = conv2d_corr self.corr_gradw = conv2d_corr_gw self.corr_gradi = conv2d_corr_gi if theano.config.cxx == "" or not theano.tensor.nnet.abstract_conv.imported_scipy_signal: raise SkipTest("CorrMM needs cxx and SciPy") def test_fwd(self): if self.convdim == 2: img_sym = theano.tensor.tensor4('img') kern_sym = theano.tensor.tensor4('kern') else: img_sym = theano.tensor.tensor5('img') kern_sym = theano.tensor.tensor5('kern') for imshp, kshp, groups in zip(self.img_shape, self.kern_shape, self.num_groups): img = np.random.random(imshp).astype(theano.config.floatX) kern = np.random.random(kshp).astype(theano.config.floatX) split_imgs = np.split(img, groups, axis=1) split_kern = np.split(kern, groups, axis=0) grouped_conv_op = self.conv(border_mode=self.border_mode, subsample=self.subsample, filter_dilation=self.filter_dilation, num_groups=groups) grouped_conv_output = grouped_conv_op(img_sym, kern_sym) grouped_func = theano.function([img_sym, kern_sym], grouped_conv_output, mode=self.mode) assert any([isinstance(node.op, self.conv_op) for node in grouped_func.maker.fgraph.toposort()]) grouped_output = grouped_func(img, kern) ref_conv_op = self.corr_fwd(img_sym, kern_sym, border_mode=self.border_mode, subsample=self.subsample, filter_dilation=self.filter_dilation) ref_func = theano.function([img_sym, kern_sym], ref_conv_op, mode=self.ref_mode) ref_concat_output = [ref_func(img_arr, kern_arr) for img_arr, kern_arr in zip(split_imgs, split_kern)] ref_concat_output = np.concatenate(ref_concat_output, axis=1) utt.assert_allclose(grouped_output, ref_concat_output) utt.verify_grad(grouped_conv_op, [img, kern], mode=self.mode, eps=1) def test_gradweights(self): if self.convdim == 2: img_sym = theano.tensor.tensor4('img') top_sym = theano.tensor.tensor4('kern') else: img_sym = theano.tensor.tensor5('img') top_sym = theano.tensor.tensor5('kern') for imshp, kshp, tshp, groups in zip(self.img_shape, self.kern_shape, self.top_shape, self.num_groups): img = np.random.random(imshp).astype(theano.config.floatX) top = np.random.random(tshp).astype(theano.config.floatX) split_imgs = np.split(img, groups, axis=1) split_top = np.split(top, groups, axis=1) grouped_convgrad_op = self.conv_gradw(border_mode=self.border_mode, subsample=self.subsample, filter_dilation=self.filter_dilation, num_groups=groups) grouped_conv_output = grouped_convgrad_op(img_sym, top_sym, tensor.as_tensor_variable( kshp[-self.convdim:])) grouped_func = theano.function([img_sym, top_sym], grouped_conv_output, mode=self.mode) assert any([isinstance(node.op, self.conv_gradw_op) for node in grouped_func.maker.fgraph.toposort()]) grouped_output = grouped_func(img, top) ref_conv_op = self.corr_gradw(img_sym, top_sym, kshp, border_mode=self.border_mode, subsample=self.subsample, filter_dilation=self.filter_dilation) ref_func = theano.function([img_sym, top_sym], ref_conv_op, mode=self.ref_mode) ref_concat_output = [ref_func(img_arr, top_arr) for img_arr, top_arr in zip(split_imgs, split_top)] ref_concat_output = np.concatenate(ref_concat_output, axis=0) utt.assert_allclose(grouped_output, ref_concat_output) def conv_gradweight(inputs_val, output_val): return grouped_convgrad_op(inputs_val, output_val, tensor.as_tensor_variable( kshp[-self.convdim:])) utt.verify_grad(conv_gradweight, [img, top], mode=self.mode, eps=1) def test_gradinputs(self): if self.convdim == 2: kern_sym = theano.tensor.tensor4('kern') top_sym = theano.tensor.tensor4('top') else: kern_sym = theano.tensor.tensor5('kern') top_sym = theano.tensor.tensor5('top') for imshp, kshp, tshp, groups in zip(self.img_shape, self.kern_shape, self.top_shape, self.num_groups): kern = np.random.random(kshp).astype(theano.config.floatX) top = np.random.random(tshp).astype(theano.config.floatX) split_kerns = np.split(kern, groups, axis=0) split_top = np.split(top, groups, axis=1) grouped_convgrad_op = self.conv_gradi(border_mode=self.border_mode, subsample=self.subsample, filter_dilation=self.filter_dilation, num_groups=groups) grouped_conv_output = grouped_convgrad_op(kern_sym, top_sym, tensor.as_tensor_variable( imshp[-self.convdim:])) grouped_func = theano.function([kern_sym, top_sym], grouped_conv_output, mode=self.mode) assert any([isinstance(node.op, self.conv_gradi_op) for node in grouped_func.maker.fgraph.toposort()]) grouped_output = grouped_func(kern, top) ref_conv_op = self.corr_gradi(kern_sym, top_sym, imshp, border_mode=self.border_mode, subsample=self.subsample, filter_dilation=self.filter_dilation) ref_func = theano.function([kern_sym, top_sym], ref_conv_op, mode=self.ref_mode) ref_concat_output = [ref_func(kern_arr, top_arr) for kern_arr, top_arr in zip(split_kerns, split_top)] ref_concat_output = np.concatenate(ref_concat_output, axis=1) utt.assert_allclose(grouped_output, ref_concat_output) def conv_gradinputs(filters_val, output_val): return grouped_convgrad_op(filters_val, output_val, tensor.as_tensor_variable( imshp[-self.convdim:])) utt.verify_grad(conv_gradinputs, [kern, top], mode=self.mode, eps=1) class Grouped_conv3d_noOptim(Grouped_conv_noOptim): conv = theano.tensor.nnet.abstract_conv.AbstractConv3d conv_gradw = theano.tensor.nnet.abstract_conv.AbstractConv3d_gradWeights conv_gradi = theano.tensor.nnet.abstract_conv.AbstractConv3d_gradInputs conv_op = theano.tensor.nnet.abstract_conv.AbstractConv3d conv_gradw_op = theano.tensor.nnet.abstract_conv.AbstractConv3d_gradWeights conv_gradi_op = theano.tensor.nnet.abstract_conv.AbstractConv3d_gradInputs mode = theano.Mode(optimizer=None) def setUp(self): self.num_groups = [3, 2, 4, 4] self.border_mode = 'valid' self.subsample = (1, 1, 1) self.img_shape = [(2, 6, 5, 5, 5), (1, 4, 7, 5, 7), (1, 8, 5, 3, 5), (2, 4, 7, 7, 7)] self.kern_shape = [(3, 2, 3, 3, 3), (6, 2, 5, 3, 5), (4, 2, 3, 3, 3), (4, 1, 3, 5, 3)] self.top_shape = [(2, 3, 3, 3, 3), (1, 6, 3, 3, 3), (1, 4, 3, 1, 3), (2, 4, 5, 3, 5)] self.filter_dilation = (1, 1, 1) self.ref_mode = 'FAST_RUN' self.convdim = 3 self.corr_fwd = conv3d_corr self.corr_gradw = conv3d_corr_gw self.corr_gradi = conv3d_corr_gi if theano.config.cxx == "" or not theano.tensor.nnet.abstract_conv.imported_scipy_signal: raise SkipTest("CorrMM needs cxx") class Separable_conv(unittest.TestCase): def setUp(self): self.x = np.array([[[[1, 2, 3, 4, 5], [3, 2, 1, 4, 5], [3, 3, 1, 3, 6], [5, 3, 2, 1, 1], [4, 7, 1, 2, 1]], [[3, 3, 1, 2, 6], [6, 5, 4, 3, 1], [3, 4, 5, 2, 3], [6, 4, 1, 3, 4], [2, 3, 4, 2, 5]]]]).astype(theano.config.floatX) self.depthwise_filter = np.array([[[[3, 2, 1], [5, 3, 2], [6, 4, 2]]], [[[5, 5, 2], [3, 7, 4], [3, 5, 4]]], [[[7, 4, 7], [5, 3, 3], [1, 3, 1]]], [[[4, 4, 4], [2, 4, 6], [0, 0, 7]]]]).astype(theano.config.floatX) self.pointwise_filter = np.array([[[[4]], [[1]], [[3]], [[5]]], [[[2]], [[1]], [[2]], [[8]]]]).astype(theano.config.floatX) self.precomp_output_valid = np.array([[[[1385, 1333, 1339], [1382, 1243, 1291], [1303, 1120, 1228]], [[1532, 1410, 1259], [1522, 1346, 1314], [1379, 1192, 1286]]]]).astype(theano.config.floatX) self.precomp_output_full = np.array([[[[140, 266, 343, 206, 59], [395, 697, 979, 585, 245], [429, 863, 1385, 919, 453], [243, 499, 864, 627, 371], [90, 183, 291, 254, 202]], [[149, 289, 359, 213, 58], [400, 750, 1076, 662, 266], [387, 854, 1532, 1091, 540], [174, 411, 971, 786, 518], [51, 110, 286, 299, 298]]]]).astype(theano.config.floatX) def test_interface2d(self): if theano.config.cxx == "": raise SkipTest("test needs cxx") x_sym = theano.tensor.tensor4('x') dfilter_sym = theano.tensor.tensor4('d') pfilter_sym = theano.tensor.tensor4('p') sep_op = separable_conv2d(x_sym, dfilter_sym, pfilter_sym, self.x.shape[1]) fun = theano.function([x_sym, dfilter_sym, pfilter_sym], sep_op, mode='FAST_RUN') # test for square matrix top = fun(self.x, self.depthwise_filter, self.pointwise_filter) utt.assert_allclose(top, self.precomp_output_valid) # test for non-square matrix top = fun(self.x[:, :, :3, :], self.depthwise_filter, self.pointwise_filter) utt.assert_allclose(top, self.precomp_output_valid[:, :, :1, :]) # test if it infers shape sep_op = separable_conv2d(x_sym, dfilter_sym, pfilter_sym, self.x.shape[1], input_shape=self.x.shape, depthwise_filter_shape=self.depthwise_filter.shape, pointwise_filter_shape=self.pointwise_filter.shape) fun = theano.function([x_sym, dfilter_sym, pfilter_sym], sep_op, mode='FAST_RUN') top = fun(self.x, self.depthwise_filter, self.pointwise_filter) utt.assert_allclose(top, self.precomp_output_valid) # test non-default subsample sep_op = separable_conv2d(x_sym, dfilter_sym, pfilter_sym, self.x.shape[1], subsample=(2, 2)) fun = theano.function([x_sym, dfilter_sym, pfilter_sym], sep_op, mode='FAST_RUN') top = fun(self.x, self.depthwise_filter, self.pointwise_filter) utt.assert_allclose(top, np.delete(np.delete(self.precomp_output_valid, 1, axis=3), 1, axis=2)) # test non-default border_mode sep_op = separable_conv2d(x_sym, dfilter_sym, pfilter_sym, self.x.shape[1], border_mode='full') fun = theano.function([x_sym, dfilter_sym, pfilter_sym], sep_op, mode='FAST_RUN') top = fun(self.x[:, :, :3, :3], self.depthwise_filter, self.pointwise_filter) utt.assert_allclose(top, self.precomp_output_full) def test_interface3d(self): if theano.config.cxx == "": raise SkipTest("test needs cxx") # Expand the filter along the depth x = np.tile(np.expand_dims(self.x, axis=2), (1, 1, 5, 1, 1)) depthwise_filter = np.tile(np.expand_dims(self.depthwise_filter, axis=2), (1, 1, 3, 1, 1)) pointwise_filter = np.expand_dims(self.pointwise_filter, axis=2) precomp_output = np.tile(np.expand_dims(self.precomp_output_valid, axis=2), (1, 1, 3, 1, 1)) * 3 x_sym = theano.tensor.tensor5('x') dfilter_sym = theano.tensor.tensor5('d') pfilter_sym = theano.tensor.tensor5('p') sep_op = separable_conv3d(x_sym, dfilter_sym, pfilter_sym, x.shape[1]) fun = theano.function([x_sym, dfilter_sym, pfilter_sym], sep_op, mode='FAST_RUN') # test for square matrix top = fun(x, depthwise_filter, pointwise_filter) utt.assert_allclose(top, precomp_output) # test for non-square matrix top = fun(x[:, :, :3, :, :3], depthwise_filter, pointwise_filter) utt.assert_allclose(top, precomp_output[:, :, :1, :, :1]) # test if it infers shape sep_op = separable_conv3d(x_sym, dfilter_sym, pfilter_sym, x.shape[1], input_shape=x.shape, depthwise_filter_shape=depthwise_filter.shape, pointwise_filter_shape=pointwise_filter.shape) fun = theano.function([x_sym, dfilter_sym, pfilter_sym], sep_op, mode='FAST_RUN') top = fun(x, depthwise_filter, pointwise_filter) utt.assert_allclose(top, precomp_output) # test non-default subsample sep_op = separable_conv3d(x_sym, dfilter_sym, pfilter_sym, x.shape[1], subsample=(2, 2, 2)) fun = theano.function([x_sym, dfilter_sym, pfilter_sym], sep_op, mode='FAST_RUN') top = fun(x, depthwise_filter, pointwise_filter) utt.assert_allclose(top, np.delete(np.delete( np.delete(precomp_output, 1, axis=4), 1, axis=3), 1, axis=2)) # test non-default border_mode precomp_output = np.tile(np.expand_dims(self.precomp_output_full, axis=2), (1, 1, 5, 1, 1)) * np.array([[[[[1]], [[2]], [[3]], [[2]], [[1]]]]]) sep_op = separable_conv3d(x_sym, dfilter_sym, pfilter_sym, x.shape[1], border_mode='full') fun = theano.function([x_sym, dfilter_sym, pfilter_sym], sep_op, mode='FAST_RUN') top = fun(x[:, :, :3, :3, :3], depthwise_filter, pointwise_filter) utt.assert_allclose(top, precomp_output) class TestUnsharedConv(unittest.TestCase): conv2d = theano.tensor.nnet.abstract_conv.AbstractConv2d conv2d_gradw = theano.tensor.nnet.abstract_conv.AbstractConv2d_gradWeights conv2d_gradi = theano.tensor.nnet.abstract_conv.AbstractConv2d_gradInputs conv2d_op = theano.tensor.nnet.abstract_conv.AbstractConv2d conv2d_gradw_op = theano.tensor.nnet.abstract_conv.AbstractConv2d_gradWeights conv2d_gradi_op = theano.tensor.nnet.abstract_conv.AbstractConv2d_gradInputs mode = theano.compile.mode.Mode(optimizer='None') def setUp(self): self.img_shape = [(2, 2, 4, 4), (3, 2, 4, 2), (3, 3, 5, 3), (3, 4, 4, 4)] self.kern_shape = [(2, 2, 2, 2, 3, 3), (2, 4, 2, 2, 4, 2), (3, 2, 1, 1, 3, 3), (4, 3, 3, 2, 4, 2)] self.topgrad_shape = [(2, 2, 2, 2), (3, 2, 4, 2), (3, 3, 2, 1), (3, 4, 3, 3)] self.border_mode = ['valid', 'full', 'valid', 'full'] self.subsample = [(1, 1), (2, 2), (2, 1), (3, 2)] self.filter_dilation = (1, 1) self.num_groups = [1, 1, 3, 2] # self.verify_flags = np.random.choice([True, False], 4, [0.5, 0.5]) # Above line can be used instead if speed is a concern self.verify_flags = [True] * 4 self.ref_mode = 'FAST_RUN' if theano.config.cxx == "" or not theano.tensor.nnet.abstract_conv.imported_scipy_signal: raise SkipTest("CorrMM needs cxx or SciPy") def test_fwd(self): tensor6 = theano.tensor.TensorType(theano.config.floatX, (False,) * 6) img_sym = theano.tensor.tensor4('img') kern_sym = tensor6('kern') ref_kern_sym = theano.tensor.tensor4('ref_kern') for imshp, kshp, mode, sub, groups, verify in zip(self.img_shape, self.kern_shape, self.border_mode, self.subsample, self.num_groups, self.verify_flags): img = np.random.random(imshp).astype(theano.config.floatX) kern = np.random.random(kshp).astype(theano.config.floatX) unshared_conv_op = self.conv2d(border_mode=mode, subsample=sub, filter_dilation=self.filter_dilation, num_groups=groups, unshared=True) unshared_out_sym = unshared_conv_op(img_sym, kern_sym) unshared_func = theano.function([img_sym, kern_sym], unshared_out_sym, mode=self.mode) assert any([isinstance(node.op, self.conv2d_op) for node in unshared_func.maker.fgraph.toposort()]) unshared_output = unshared_func(img, kern) single_kshp = kshp[:1] + kshp[3:] ref_conv_op = self.conv2d(border_mode=mode, subsample=sub, filter_dilation=self.filter_dilation, num_groups=groups, unshared=False) ref_out_sym = ref_conv_op(img_sym, ref_kern_sym) ref_func = theano.function([img_sym, ref_kern_sym], ref_out_sym, mode=self.mode) for i in range(0, kshp[1]): for j in range(0, kshp[2]): single_kern = kern[:, i, j, ...].reshape(single_kshp) ref_val = ref_func(img, single_kern) utt.assert_allclose(ref_val[:, :, i, j], unshared_output[:, :, i, j]) if verify: utt.verify_grad(unshared_conv_op, [img, kern], mode=self.mode, eps=1) def test_gradweight(self): img_sym = theano.tensor.tensor4('img') top_sym = theano.tensor.tensor4('top') for imshp, kshp, topshp, mode, sub, groups, verify in zip(self.img_shape, self.kern_shape, self.topgrad_shape, self.border_mode, self.subsample, self.num_groups, self.verify_flags): img = np.random.random(imshp).astype(theano.config.floatX) top = np.random.random(topshp).astype(theano.config.floatX) unshared_conv_op = self.conv2d_gradw(border_mode=mode, subsample=sub, filter_dilation=self.filter_dilation, num_groups=groups, unshared=True) unshared_out_sym = unshared_conv_op(img_sym, top_sym, tensor.as_tensor_variable(kshp[-2:])) unshared_func = theano.function([img_sym, top_sym], unshared_out_sym, mode=self.mode) assert any([isinstance(node.op, self.conv2d_gradw_op) for node in unshared_func.maker.fgraph.toposort()]) unshared_output = unshared_func(img, top) single_kshp = kshp[:1] + kshp[3:] ref_conv_op = self.conv2d_gradw(border_mode=mode, subsample=sub, filter_dilation=self.filter_dilation, num_groups=groups, unshared=False) ref_out_sym = ref_conv_op(img_sym, top_sym, tensor.as_tensor_variable(single_kshp[-2:])) ref_func = theano.function([img_sym, top_sym], ref_out_sym, mode=self.mode) for i in range(0, topshp[2]): for j in range(0, topshp[3]): top_single = np.zeros_like(top) top_single[:, :, i, j] = top[:, :, i, j] ref_output = ref_func(img, top_single) utt.assert_allclose(unshared_output[:, i, j, ...], ref_output) def conv_gradweight(inputs_val, output_val): return unshared_conv_op(inputs_val, output_val, tensor.as_tensor_variable(kshp[-2:])) if verify: utt.verify_grad(conv_gradweight, [img, top], mode=self.mode, eps=1) def test_gradinput(self): tensor6 = theano.tensor.TensorType(theano.config.floatX, (False,) * 6) kern_sym = tensor6('kern') top_sym = theano.tensor.tensor4('top') ref_kern_sym = theano.tensor.tensor4('ref_kern') for imshp, kshp, topshp, mode, sub, groups, verify in zip(self.img_shape, self.kern_shape, self.topgrad_shape, self.border_mode, self.subsample, self.num_groups, self.verify_flags): single_kshp = kshp[:1] + kshp[3:] kern = np.random.random(kshp).astype(theano.config.floatX) top = np.random.random(topshp).astype(theano.config.floatX) unshared_conv_op = self.conv2d_gradi(border_mode=mode, subsample=sub, filter_dilation=self.filter_dilation, num_groups=groups, unshared=True) unshared_out_sym = unshared_conv_op(kern_sym, top_sym, tensor.as_tensor_variable(imshp[-2:])) unshared_func = theano.function([kern_sym, top_sym], unshared_out_sym, mode=self.mode) assert any([isinstance(node.op, self.conv2d_gradi_op) for node in unshared_func.maker.fgraph.toposort()]) unshared_output = unshared_func(kern, top) ref_conv_op = self.conv2d_gradi(border_mode=mode, subsample=sub, filter_dilation=self.filter_dilation, num_groups=groups, unshared=False) ref_out_sym = ref_conv_op(ref_kern_sym, top_sym, tensor.as_tensor_variable(imshp[-2:])) ref_func = theano.function([ref_kern_sym, top_sym], ref_out_sym, mode=self.mode) ref_output = np.zeros(imshp) for i in range(0, topshp[2]): for j in range(0, topshp[3]): single_kern = kern[:, i, j, ...].reshape(single_kshp) top_single = np.zeros_like(top) top_single[:, :, i, j] = top[:, :, i, j] ref_output += ref_func(single_kern, top_single) utt.assert_allclose(ref_output, unshared_output) def conv_gradinputs(filters_val, output_val): return unshared_conv_op(filters_val, output_val, tensor.as_tensor_variable(imshp[-2:])) if verify: utt.verify_grad(conv_gradinputs, [kern, top], mode=self.mode, eps=1) class TestAsymmetricPadding(unittest.TestCase): conv2d = theano.tensor.nnet.abstract_conv.AbstractConv2d conv2d_gradw = theano.tensor.nnet.abstract_conv.AbstractConv2d_gradWeights conv2d_gradi = theano.tensor.nnet.abstract_conv.AbstractConv2d_gradInputs conv2d_op = theano.tensor.nnet.abstract_conv.AbstractConv2d conv2d_gradw_op = theano.tensor.nnet.abstract_conv.AbstractConv2d_gradWeights conv2d_gradi_op = theano.tensor.nnet.abstract_conv.AbstractConv2d_gradInputs mode = theano.compile.mode.Mode(optimizer='None') img_shape = [(2, 2, 4, 4), (3, 2, 4, 2), (3, 3, 5, 3)] kern_shape = [(4, 2, 2, 2), (2, 2, 4, 2), (2, 3, 3, 3)] topgrad_shape = [(2, 4, 6, 6), (3, 2, 3, 4), (3, 2, 6, 1)] border_mode = [((1, 2), (2, 1)), ((1, 1), (0, 3)), ((2, 1), (0, 0))] def test_fwd(self): if theano.config.cxx == "" or not theano.tensor.nnet.abstract_conv.imported_scipy_signal: raise SkipTest("SciPy and cxx needed") img_sym = theano.tensor.tensor4('img') kern_sym = theano.tensor.tensor4('kern') for imshp, kshp, pad in zip(self.img_shape, self.kern_shape, self.border_mode): img = np.random.random(imshp).astype(theano.config.floatX) kern = np.random.random(kshp).astype(theano.config.floatX) asymmetric_conv_op = self.conv2d(border_mode=pad, subsample=(1, 1), filter_dilation=(1, 1)) asymmetric_out_sym = asymmetric_conv_op(img_sym, kern_sym) asymmetric_func = theano.function([img_sym, kern_sym], asymmetric_out_sym, mode=self.mode) assert any([isinstance(node.op, self.conv2d_op) for node in asymmetric_func.maker.fgraph.toposort()]) asymmetric_output = asymmetric_func(img, kern) ref_conv_op = self.conv2d(border_mode="valid", subsample=(1, 1), filter_dilation=(1, 1)) ref_out_sym = ref_conv_op(img_sym, kern_sym) ref_func = theano.function([img_sym, kern_sym], ref_out_sym, mode=self.mode) exp_imshp = (imshp[0], imshp[1], imshp[2] + pad[0][0] + pad[0][1], imshp[3] + pad[1][0] + pad[1][1]) exp_img = np.zeros(exp_imshp, dtype=theano.config.floatX) exp_img[:, :, pad[0][0]:imshp[2] + pad[0][0], pad[1][0]:imshp[3] + pad[1][0]] = img ref_output = ref_func(exp_img, kern) utt.assert_allclose(asymmetric_output, ref_output) utt.verify_grad(asymmetric_conv_op, [img, kern], mode=self.mode, eps=1) def test_gradweight(self): if theano.config.cxx == "" or not theano.tensor.nnet.abstract_conv.imported_scipy_signal: raise SkipTest("SciPy and cxx needed") img_sym = theano.tensor.tensor4('img') top_sym = theano.tensor.tensor4('top') for imshp, kshp, topshp, pad in zip(self.img_shape, self.kern_shape, self.topgrad_shape, self.border_mode): img = np.random.random(imshp).astype(theano.config.floatX) top = np.random.random(topshp).astype(theano.config.floatX) asymmetric_conv_op = self.conv2d_gradw(border_mode=pad, subsample=(1, 1), filter_dilation=(1, 1)) asymmetric_out_sym = asymmetric_conv_op(img_sym, top_sym, kshp[-2:]) asymmetric_func = theano.function([img_sym, top_sym], asymmetric_out_sym, mode=self.mode) assert any([isinstance(node.op, self.conv2d_gradw_op) for node in asymmetric_func.maker.fgraph.toposort()]) asymmetric_output = asymmetric_func(img, top) ref_conv_op = self.conv2d_gradw(border_mode="valid", subsample=(1, 1), filter_dilation=(1, 1)) ref_out_sym = ref_conv_op(img_sym, top_sym, kshp[-2:]) ref_func = theano.function([img_sym, top_sym], ref_out_sym, mode=self.mode) exp_imshp = (imshp[0], imshp[1], imshp[2] + pad[0][0] + pad[0][1], imshp[3] + pad[1][0] + pad[1][1]) exp_img = np.zeros(exp_imshp, dtype=theano.config.floatX) exp_img[:, :, pad[0][0]:imshp[2] + pad[0][0], pad[1][0]:imshp[3] + pad[1][0]] = img ref_output = ref_func(exp_img, top) utt.assert_allclose(asymmetric_output, ref_output) def conv_gradweight(inputs_val, output_val): return asymmetric_conv_op(inputs_val, output_val, tensor.as_tensor_variable(kshp[-2:])) utt.verify_grad(conv_gradweight, [img, top], mode=self.mode, eps=1) def test_gradinput(self): if theano.config.cxx == "" or not theano.tensor.nnet.abstract_conv.imported_scipy_signal: raise SkipTest("test needs cxx and SciPy") kern_sym = theano.tensor.tensor4('kern') top_sym = theano.tensor.tensor4('top') for imshp, kshp, topshp, pad in zip(self.img_shape, self.kern_shape, self.topgrad_shape, self.border_mode): kern = np.random.random(kshp).astype(theano.config.floatX) top = np.random.random(topshp).astype(theano.config.floatX) asymmetric_conv_op = self.conv2d_gradi(border_mode=pad, subsample=(1, 1), filter_dilation=(1, 1)) asymmetric_out_sym = asymmetric_conv_op(kern_sym, top_sym, imshp[-2:]) asymmetric_func = theano.function([kern_sym, top_sym], asymmetric_out_sym, mode=self.mode) assert any([isinstance(node.op, self.conv2d_gradi_op) for node in asymmetric_func.maker.fgraph.toposort()]) asymmetric_output = asymmetric_func(kern, top) ref_conv_op = self.conv2d_gradi(border_mode="valid", subsample=(1, 1), filter_dilation=(1, 1)) exp_imshp = [imshp[2] + pad[0][0] + pad[0][1], imshp[3] + pad[1][0] + pad[1][1]] ref_out_sym = ref_conv_op(kern_sym, top_sym, exp_imshp) ref_func = theano.function([kern_sym, top_sym], ref_out_sym, mode=self.mode) ref_output = ref_func(kern, top) ref_output = ref_output[:, :, pad[0][0]:imshp[2] + pad[0][0], pad[1][0]:imshp[3] + pad[1][0]] utt.assert_allclose(asymmetric_output, ref_output) def conv_gradinputs(filters_val, output_val): return asymmetric_conv_op(filters_val, output_val, tensor.as_tensor_variable(imshp[-2:])) utt.verify_grad(conv_gradinputs, [kern, top], mode=self.mode, eps=1) class TestCausalConv(unittest.TestCase): mode = theano.compile.mode.Mode(optimizer='None') img = np.array([[[2, 4, 9, 5, 8], [0, 0, 4, 0, 5]], [[2, 5, 8, 5, 5], [1, 3, 0, 7, 9]], [[7, 0, 7, 1, 0], [0, 1, 4, 7, 2]]]).astype(theano.config.floatX) kern = np.array([[[5, 3, 1], [3, 1, 0]], [[6, 4, 9], [2, 2, 7]]]).astype(theano.config.floatX) dilation = 2 precomp_top = np.array([[[10, 20, 63, 37, 88], [12, 24, 70, 46, 120]], [[13, 34, 47, 64, 78], [14, 36, 58, 70, 105]], [[35, 3, 68, 27, 38], [42, 2, 78, 22, 103]]]).astype(theano.config.floatX) def test_interface(self): img_sym = theano.tensor.tensor3('img') kern_sym = theano.tensor.tensor3('kern') if theano.config.cxx == "" or not theano.tensor.nnet.abstract_conv.imported_scipy_signal: raise SkipTest("SciPy and cxx needed") sym_out = causal_conv1d(img_sym, kern_sym, self.kern.shape, filter_dilation=self.dilation) causal_func = theano.function([img_sym, kern_sym], sym_out, mode=self.mode) output = causal_func(self.img, self.kern) utt.assert_allclose(output, self.precomp_top) def causal_conv_fn(inputs_val, filters_val): return causal_conv1d(inputs_val, filters_val, self.kern.shape, filter_dilation=1) utt.verify_grad(causal_conv_fn, [self.img, self.kern], mode=self.mode, eps=1) Theano-1.0.0/theano/tensor/nnet/tests/test_corr.py00006000163562€ÿÿÿþ0000005202413203335532023546 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division from nose.plugins.skip import SkipTest from nose.plugins.attrib import attr from nose.tools import assert_equals import numpy as np from six import integer_types import theano import theano.tensor as T from theano.tests import unittest_tools as utt from theano.tensor.nnet import corr, conv from theano.tensor.nnet.tests.test_abstract_conv import Grouped_conv_noOptim, TestUnsharedConv from theano.tensor.nnet.tests.test_abstract_conv import TestAsymmetricPadding, TestCausalConv class TestCorr2D(utt.InferShapeTester): if theano.config.mode == "FAST_COMPILE": mode = theano.compile.get_mode("FAST_RUN") else: mode = None dtype = theano.config.floatX def setUp(self): super(TestCorr2D, self).setUp() self.input = T.tensor4('input', dtype=self.dtype) self.input.name = 'default_V' self.filters = T.tensor4('filters', dtype=self.dtype) self.filters.name = 'default_filters' if not conv.imported_scipy_signal and theano.config.cxx == "": raise SkipTest("CorrMM tests need SciPy or a c++ compiler") # This tests can run even when theano.config.blas.ldflags is empty. def validate(self, image_shape, filter_shape, border_mode='valid', subsample=(1, 1), input=None, filters=None, verify_grad=True, non_contiguous=False, filter_dilation=(1, 1)): """ :param image_shape: The constant shape info passed to corrMM. :param filter_shape: The constant shape info passed to corrMM. """ if not theano.config.cxx: raise SkipTest("Need cxx to test conv2d") N_image_shape = [T.get_scalar_constant_value(T.as_tensor_variable(x)) for x in image_shape] N_filter_shape = [T.get_scalar_constant_value(T.as_tensor_variable(x)) for x in filter_shape] if input is None: input = self.input if filters is None: filters = self.filters # THEANO IMPLEMENTATION # we create a symbolic function so that verify_grad can work def sym_CorrMM(input, filters): # define theano graph and function input.name = 'input' filters.name = 'filters' rval = corr.CorrMM(border_mode, subsample, filter_dilation)(input, filters) rval.name = 'corr_output' return rval output = sym_CorrMM(input, filters) output.name = 'CorrMM()(%s,%s)' % (input.name, filters.name) theano_corr = theano.function([input, filters], output, mode=self.mode) # initialize input and compute result image_data = np.random.random(N_image_shape).astype(self.dtype) filter_data = np.random.random(N_filter_shape).astype(self.dtype) if non_contiguous: image_data = np.transpose(image_data, axes=(0, 1, 3, 2)) image_data = image_data.copy() image_data = np.transpose(image_data, axes=(0, 1, 3, 2)) filter_data = np.transpose(filter_data, axes=(0, 1, 3, 2)) filter_data = filter_data.copy() filter_data = np.transpose(filter_data, axes=(0, 1, 3, 2)) assert not image_data.flags['CONTIGUOUS'] assert not filter_data.flags['CONTIGUOUS'] theano_output = theano_corr(image_data, filter_data) # REFERENCE IMPLEMENTATION # Testing correlation, not convolution. Reverse filters. filter_data_corr = np.array(filter_data[:, :, ::-1, ::-1], copy=True, order='C') orig_image_data = image_data img_shape2d = np.array(N_image_shape[-2:]) fil_shape2d = np.array(N_filter_shape[-2:]) dil_shape2d = np.array(filter_dilation) dil_fil_shape2d = (fil_shape2d - 1) * dil_shape2d + 1 subsample2d = np.array(subsample) if border_mode == 'full': padHW = (dil_fil_shape2d - 1) elif border_mode == 'valid': padHW = np.array([0, 0]) elif border_mode == 'half': padHW = np.floor(dil_fil_shape2d / 2).astype('int32') elif isinstance(border_mode, tuple): padHW = np.array(border_mode) elif isinstance(border_mode, integer_types): padHW = np.array([border_mode, border_mode]) else: raise NotImplementedError('Unsupported border_mode {}'.format(border_mode)) out_shape2d = np.floor((img_shape2d + 2 * (padHW) - dil_fil_shape2d) / subsample2d) + 1 # avoid numpy deprecation out_shape2d = out_shape2d.astype('int32') out_shape = (N_image_shape[0], N_filter_shape[0]) + tuple(out_shape2d) ref_output = np.zeros(out_shape) # loop over output feature maps ref_output.fill(0) image_data2 = np.zeros((N_image_shape[0], N_image_shape[1], N_image_shape[2] + 2 * padHW[0], N_image_shape[3] + 2 * padHW[1])) image_data2[:, :, padHW[0]:padHW[0] + N_image_shape[2], padHW[1]:padHW[1] + N_image_shape[3]] = image_data image_data = image_data2 N_image_shape = image_data.shape for bb in range(N_image_shape[0]): for nn in range(N_filter_shape[0]): for im0 in range(N_image_shape[1]): filter2d = filter_data_corr[nn, im0, :, :] image2d = image_data[bb, im0, :, :] for row in range(ref_output.shape[2]): irow = row * subsample[0] # image row for col in range(ref_output.shape[3]): icol = col * subsample[1] # image col ref_output[bb, nn, row, col] += (image2d[ irow:irow + dil_fil_shape2d[0]:filter_dilation[0], icol:icol + dil_fil_shape2d[1]:filter_dilation[1]] * filter2d[::-1, ::-1] ).sum() utt.assert_allclose(ref_output, theano_output) # TEST GRADIENT if verify_grad: utt.verify_grad(sym_CorrMM, [orig_image_data, filter_data], mode=self.mode) @attr('slow') def test_basic(self): # Tests that basic correlations work for odd and even # dimensions of image and filter shapes, as well as rectangular # images and filters. border_modes = ['valid', 'full', 'half', (1, 1), (2, 1), (1, 2), (3, 3), 1] img_shapes = [(2, 2, 3, 3), (3, 2, 8, 8), (3, 2, 7, 5), (3, 2, 7, 5), (3, 2, 8, 8), (3, 2, 7, 5)] fil_shapes = [(2, 2, 2, 2), (4, 2, 5, 5), (5, 2, 2, 3), (5, 2, 3, 2), (4, 2, 5, 5), (5, 2, 2, 3)] for border_mode in border_modes: for img, fil in zip(img_shapes, fil_shapes): self.validate(img, fil, border_mode, verify_grad=False) # Very slow on with 'full' or 'half' self.validate((1, 10, 213, 129), (46, 10, 212, 1), 'valid', verify_grad=False) def test_img_kernel_same_shape(self): self.validate((3, 2, 3, 3), (4, 2, 3, 3), 'full') self.validate((3, 2, 3, 3), (4, 2, 3, 3), 'valid') self.validate((3, 2, 3, 3), (4, 2, 3, 3), 'half') self.validate((3, 2, 3, 3), (4, 2, 3, 3), (1, 1)) self.validate((3, 2, 3, 3), (4, 2, 3, 3), 1) @attr('slow') def test_subsample(self): # Tests correlation where subsampling != (1,1) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'valid', subsample=(2, 2)) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'valid', subsample=(2, 1)) self.validate((1, 1, 6, 6), (1, 1, 3, 3), 'valid', subsample=(3, 3)) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'full', subsample=(2, 2)) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'full', subsample=(2, 1)) self.validate((1, 1, 6, 6), (1, 1, 3, 3), 'full', subsample=(3, 3)) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'half', subsample=(2, 2)) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'half', subsample=(2, 1)) self.validate((1, 1, 6, 6), (1, 1, 3, 3), 'half', subsample=(3, 3)) self.validate((3, 2, 7, 5), (5, 2, 2, 3), (1, 1), subsample=(2, 2)) self.validate((3, 2, 7, 5), (5, 2, 2, 3), (2, 1), subsample=(2, 1)) self.validate((1, 1, 6, 6), (1, 1, 3, 3), (1, 2), subsample=(3, 3)) self.validate((1, 1, 6, 6), (1, 1, 3, 3), 1, subsample=(3, 3)) def test_filter_dilation(self): # Tests correlation where filter dilation != (1,1) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'valid', filter_dilation=(2, 2)) self.validate((3, 2, 14, 10), (5, 2, 2, 3), 'valid', filter_dilation=(3, 1)) self.validate((1, 1, 14, 14), (1, 1, 3, 3), 'valid', filter_dilation=(2, 3)) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'full', filter_dilation=(2, 2)) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'full', filter_dilation=(3, 1)) self.validate((1, 1, 6, 6), (1, 1, 3, 3), 'full', filter_dilation=(2, 3)) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'half', filter_dilation=(2, 2)) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'half', filter_dilation=(3, 1)) self.validate((1, 1, 6, 6), (1, 1, 3, 3), 'half', filter_dilation=(2, 3)) self.validate((3, 2, 7, 5), (5, 2, 2, 3), (1, 1), filter_dilation=(2, 2)) self.validate((3, 2, 7, 5), (5, 2, 2, 3), (2, 1), filter_dilation=(2, 1)) self.validate((1, 1, 6, 6), (1, 1, 3, 3), (1, 2), filter_dilation=(1, 2)) self.validate((1, 1, 6, 6), (1, 1, 3, 3), 1, subsample=(3, 3), filter_dilation=(2, 2)) @attr('slow') def test_shape_Constant_tensor(self): # Tests correlation where the {image,filter}_shape is a Constant tensor. as_t = T.as_tensor_variable border_modes = ['valid', 'full', 'half', (1, 1), (2, 1), (1, 2), (3, 3), 1] for border_mode in border_modes: self.validate((as_t(3), as_t(2), as_t(7), as_t(5)), (5, 2, 2, 3), border_mode) self.validate(as_t([3, 2, 7, 5]), (5, 2, 2, 3), border_mode) self.validate(as_t((3, 2, 7, 5)), (5, 2, 2, 3), border_mode) self.validate((3, 2, 7, 5), (as_t(5), as_t(2), as_t(2), as_t(3)), 'valid') self.validate((3, 2, 7, 5), as_t([5, 2, 2, 3]), border_mode) self.validate(as_t([3, 2, 7, 5]), as_t([5, 2, 2, 3]), border_mode) def test_invalid_filter_shape(self): # Tests scenario where filter_shape[1] != input_shape[1] self.assertRaises(ValueError, self.validate, (3, 2, 8, 8), (4, 3, 5, 5), 'valid') def test_full_mode(self): # Tests basic correlation in full mode and case where filter # is larger than the input image. self.validate((3, 2, 5, 5), (4, 2, 8, 8), 'full') def f(): self.validate((3, 2, 5, 5), (4, 2, 8, 8), 'valid') self.assertRaises(Exception, f) def test_wrong_input(self): # Make sure errors are raised when image and kernel are not 4D tensors self.assertRaises(Exception, self.validate, (3, 2, 8, 8), (4, 2, 5, 5), 'valid', input=T.dmatrix()) self.assertRaises(Exception, self.validate, (3, 2, 8, 8), (4, 2, 5, 5), 'valid', filters=T.dvector()) self.assertRaises(Exception, self.validate, (3, 2, 8, 8), (4, 2, 5, 5), 'valid', input=T.dtensor3()) def test_dtype_upcast(self): # Checks dtype upcast for CorrMM methods. def rand(shape, dtype='float64'): r = np.asarray(np.random.rand(*shape), dtype=dtype) return r * 2 - 1 if not theano.config.cxx: raise SkipTest("Need cxx to test conv2d") ops = [corr.CorrMM, corr.CorrMM_gradWeights, corr.CorrMM_gradInputs] a_shapes = [[4, 5, 6, 3], [1, 5, 6, 3], [1, 5, 6, 3]] b_shapes = [[7, 5, 3, 2], [1, 5, 3, 1], [7, 1, 3, 1]] dtypes = ['float32', 'float64'] for op, a_shape, b_shape in zip(ops, a_shapes, b_shapes): for a_dtype in dtypes: for b_dtype in dtypes: c_dtype = theano.scalar.upcast(a_dtype, b_dtype) a_tens = T.tensor4(dtype=a_dtype) b_tens = T.tensor4(dtype=b_dtype) a_tens_val = rand(a_shape, dtype=a_dtype) b_tens_val = rand(b_shape, dtype=b_dtype) c_tens = op()(a_tens, b_tens) f = theano.function([a_tens, b_tens], c_tens, mode=self.mode) assert_equals(f(a_tens_val, b_tens_val).dtype, c_dtype) @attr('slow') def test_infer_shape_forward(self): if theano.config.mode == "FAST_COMPILE": raise SkipTest("CorrMM don't work in FAST_COMPILE") if not theano.config.cxx: raise SkipTest("Need cxx for this test") def rand(*shape): r = np.asarray(np.random.rand(*shape), dtype='float64') return r * 2 - 1 corrMM = corr.CorrMM adtens = T.dtensor4() bdtens = T.dtensor4() aivec_vals = [[4, 5, 6, 3], [6, 2, 8, 3], [3, 6, 7, 5], [3, 6, 7, 5], [5, 2, 4, 3]] bivec_vals = [[7, 5, 3, 2], [4, 2, 5, 3], [5, 6, 3, 2], [5, 6, 2, 3], [6, 2, 4, 3]] modes = ['valid', 'full', 'half', (1, 1), (2, 1), (1, 2), 1] subsamples = [(1, 1), (2, 1), (1, 2)] for aivec_val, bivec_val in zip(aivec_vals, bivec_vals): adtens_val = rand(*aivec_val) bdtens_val = rand(*bivec_val) for mode in modes: for subsample in subsamples: # CorrMM cdtens = corrMM(border_mode=mode, subsample=subsample)(adtens, bdtens) self._compile_and_check([adtens, bdtens], [cdtens], [adtens_val, bdtens_val], corrMM, warn=False) @attr('slow') def test_infer_shape_gradW(self): if theano.config.mode == "FAST_COMPILE": raise SkipTest("CorrMM don't work in FAST_COMPILE") if not theano.config.cxx: raise SkipTest("Need cxx for this test") def rand(*shape): r = np.asarray(np.random.rand(*shape), dtype='float64') return r * 2 - 1 corrMM = corr.CorrMM gradW = corr.CorrMM_gradWeights adtens = T.dtensor4() bdtens = T.dtensor4() aivec_vals = [[1, 5, 6, 3], [8, 2, 7, 3], [1, 6, 9, 4], [9, 6, 8, 5], [9, 1, 6, 8]] bivec_vals = [[7, 5, 3, 1], [4, 2, 5, 3], [12, 6, 3, 2], [5, 6, 1, 3], [11, 1, 3, 3]] modes = ['valid', 'full', 'half', (1, 1), (2, 1), (1, 2), 1] subsamples = [(1, 1), (2, 1), (1, 2)] for aivec_val, bivec_val in zip(aivec_vals, bivec_vals): adtens_val = rand(*aivec_val) bdtens_val = rand(*bivec_val) for mode in modes: for subsample in subsamples: # CorrMM cdtens = corrMM(border_mode=mode, subsample=subsample)(adtens, bdtens) f = theano.function([adtens, bdtens], cdtens) cdtens_val = f(adtens_val, bdtens_val) # CorrMM_gradWeights shape = (theano.shared(bivec_val[2]), theano.shared(bivec_val[3])) bdtens_g = gradW(border_mode=mode, subsample=subsample)(adtens, cdtens, shape=shape) self._compile_and_check([adtens, cdtens], [bdtens_g], [adtens_val, cdtens_val], gradW, warn=False) @attr('slow') def test_infer_shape_gradI(self): if theano.config.mode == "FAST_COMPILE": raise SkipTest("CorrMM don't work in FAST_COMPILE") if not theano.config.cxx: raise SkipTest("Need cxx for this test") def rand(*shape): r = np.asarray(np.random.rand(*shape), dtype='float64') return r * 2 - 1 corrMM = corr.CorrMM gradI = corr.CorrMM_gradInputs adtens = T.dtensor4() bdtens = T.dtensor4() aivec_vals = [[1, 5, 6, 3], [8, 2, 7, 3], [1, 6, 9, 4], [9, 6, 8, 5], [9, 1, 6, 8]] bivec_vals = [[7, 5, 3, 1], [4, 2, 5, 3], [12, 6, 3, 2], [5, 6, 1, 3], [7, 1, 3, 4]] modes = ['valid', 'full', 'half', (1, 1), (2, 1), (1, 2), 1] subsamples = [(1, 1), (2, 1), (1, 2)] for aivec_val, bivec_val in zip(aivec_vals, bivec_vals): adtens_val = rand(*aivec_val) bdtens_val = rand(*bivec_val) for mode in modes: for subsample in subsamples: # CorrMM cdtens = corrMM(border_mode=mode, subsample=subsample)(adtens, bdtens) f = theano.function([adtens, bdtens], cdtens) cdtens_val = f(adtens_val, bdtens_val) # CorrMM_gradInputs shape = (theano.shared(aivec_val[2]), theano.shared(aivec_val[3])) adtens_g = gradI(border_mode=mode, subsample=subsample)(bdtens, cdtens, shape=shape) self._compile_and_check([bdtens, cdtens], [adtens_g], [bdtens_val, cdtens_val], gradI, warn=False) def test_non_contiguous(self): self.validate((2, 2, 3, 3), (2, 2, 2, 2), 'valid', non_contiguous=True) self.validate((3, 2, 8, 8), (4, 2, 5, 5), 'valid', non_contiguous=True) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'valid', non_contiguous=True) self.validate((3, 2, 7, 5), (5, 2, 3, 2), 'valid', non_contiguous=True) self.validate((3, 2, 8, 8), (4, 2, 5, 5), 'full', non_contiguous=True) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'full', non_contiguous=True) self.validate((3, 2, 8, 8), (4, 2, 5, 5), 'half', non_contiguous=True) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 'half', non_contiguous=True) self.validate((3, 2, 8, 8), (4, 2, 5, 5), (1, 1), non_contiguous=True) self.validate((3, 2, 7, 5), (5, 2, 2, 3), (1, 2), non_contiguous=True) self.validate((3, 2, 7, 5), (5, 2, 2, 3), (2, 1), non_contiguous=True) self.validate((3, 2, 7, 5), (5, 2, 2, 3), 2, non_contiguous=True) class TestGroupCorr2d(Grouped_conv_noOptim): mode = theano.compile.get_mode("FAST_RUN").excluding('gpuarray') conv_op = corr.CorrMM conv_gradw_op = corr.CorrMM_gradWeights conv_gradi_op = corr.CorrMM_gradInputs def test_graph(self): # define common values first groups = 3 bottom = np.random.rand(3, 6, 5, 5).astype(theano.config.floatX) kern = np.random.rand(9, 2, 3, 3).astype(theano.config.floatX) bottom_sym = T.tensor4('bottom') kern_sym = T.tensor4('kern') # grouped convolution graph conv_group = self.conv(num_groups=groups)(bottom_sym, kern_sym) gconv_func = theano.function([bottom_sym, kern_sym], conv_group, mode=self.mode) # Graph for the normal hard way kern_offset = kern_sym.shape[0] // groups bottom_offset = bottom_sym.shape[1] // groups split_conv_output = [self.conv()(bottom_sym[:, i * bottom_offset:(i + 1) * bottom_offset, :, :], kern_sym[i * kern_offset:(i + 1) * kern_offset, :, :, :]) for i in range(groups)] concatenated_output = T.concatenate(split_conv_output, axis=1) conv_func = theano.function([bottom_sym, kern_sym], concatenated_output, mode=self.mode) # calculate outputs for each graph gconv_output = gconv_func(bottom, kern) conv_output = conv_func(bottom, kern) # compare values utt.assert_allclose(gconv_output, conv_output) class TestUnsharedCorr2d(TestUnsharedConv): if theano.config.mode == "FAST_COMPILE": mode = theano.compile.get_mode("FAST_RUN").excluding('gpuarray') else: mode = None conv2d_op = corr.CorrMM conv2d_gradw_op = corr.CorrMM_gradWeights conv2d_gradi_op = corr.CorrMM_gradInputs class TestAsymmetricCorr(TestAsymmetricPadding): if theano.config.mode == "FAST_COMPILE": mode = theano.compile.get_mode("FAST_RUN").excluding('gpuarray') else: mode = None conv2d_op = corr.CorrMM conv2d_gradw_op = corr.CorrMM_gradWeights conv2d_gradi_op = corr.CorrMM_gradInputs class TestCausalCorr(TestCausalConv): if theano.config.mode == "FAST_COMPILE": mode = theano.compile.get_mode("FAST_RUN").excluding('gpuarray') else: mode = None if __name__ == '__main__': t = TestCorr2D('setUp') t.setUp() t.test_infer_shape() Theano-1.0.0/theano/tensor/nnet/c_code/00007500163562€ÿÿÿþ0000000000013203336027021242 5ustar boccoset00000000000000Theano-1.0.0/theano/tensor/nnet/c_code/corr_gemm.c00006000163562€ÿÿÿþ0000006372713203335532023374 0ustar boccoset00000000000000// This uses a lot of code from Caffe (http://caffe.berkeleyvision.org/); // sources are clearly marked. Below we reproduce the original license of // the Caffe software. /* Copyright (c) 2014, The Regents of the University of California (Regents) All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ // (borrowed from Caffe: https://github.com/BVLC/caffe/blob/master/src/caffe/util/im2col.cpp) // Loops for fast unfold + copy void im2col(const %(float_type)s* data_im, const int channels, const int height, const int width, const int kernel_h, const int kernel_w, const int dilation_h, const int dilation_w, const int pad_hl, const int pad_hr, const int pad_wl, const int pad_wr, const int stride_h, const int stride_w, %(float_type)s* data_col) { // Implicit dilated kernel size int dil_kernel_h = (kernel_h - 1) * dilation_h + 1; int dil_kernel_w = (kernel_w - 1) * dilation_w + 1; int height_col = (height + pad_hl + pad_hr - dil_kernel_h) / stride_h + 1; int width_col = (width + pad_wl + pad_wr - dil_kernel_w) / stride_w + 1; int channels_col = channels * kernel_h * kernel_w; for (int c = 0; c < channels_col; ++c) { int w_offset = c %% kernel_w; int h_offset = (c / kernel_w) %% kernel_h; int c_im = c / kernel_h / kernel_w; for (int h = 0; h < height_col; ++h) { int h_pad = h * stride_h - pad_hl + h_offset * dilation_h; for (int w = 0; w < width_col; ++w) { int w_pad = w * stride_w - pad_wl + w_offset * dilation_w; if (h_pad >= 0 && h_pad < height && w_pad >= 0 && w_pad < width) data_col[(npy_intp)(c * height_col + h) * width_col + w] = data_im[(npy_intp)(c_im * height + h_pad) * width + w_pad]; else data_col[(npy_intp)(c * height_col + h) * width_col + w] = 0.; } } } } // Unlike the Caffe and Theano GPU verions, the data_im array is set to zero // before the col2im call rather than doing it here. So, the result is just // accumulated into data_im. void col2im(const %(float_type)s* data_col, const int channels, const int height, const int width, const int patch_h, const int patch_w, const int dilation_h, const int dilation_w, const int pad_hl, const int pad_hr, const int pad_wl, const int pad_wr, const int stride_h, const int stride_w, %(float_type)s* data_im) { // Implicit dilated patch int dil_patch_h = (patch_h - 1) * dilation_h + 1; int dil_patch_w = (patch_w - 1) * dilation_w + 1; int height_col = (height + pad_hl + pad_hr - dil_patch_h) / stride_h + 1; int width_col = (width + pad_wl + pad_wr - dil_patch_w) / stride_w + 1; int num_kernels = channels * height * width; int channels_col = channels * patch_h * patch_w; for (int c = 0; c < channels_col; ++c) { int w_offset = c %% patch_w; int h_offset = (c / patch_w) %% patch_h; int c_im = c / patch_h / patch_w; for (int h = 0; h < height_col; ++h) { int h_pad = h * stride_h - pad_hl + h_offset * dilation_h; for (int w = 0; w < width_col; ++w) { int w_pad = w * stride_w - pad_wl + w_offset * dilation_w; if (h_pad >= 0 && h_pad < height && w_pad >= 0 && w_pad < width) data_im[(npy_intp)(c_im * height + h_pad) * width + w_pad] += data_col[(npy_intp)(c * height_col + h) * width_col + w]; } } } } // Theano op code // GPU version authors: Arjun Jain, Frederic Bastien, Jan Schlueter // Reference code: https://github.com/BVLC/caffe/blob/master/src/caffe/layers/conv_layer.cu // and https://github.com/torch/cunn/blob/master/SpatialConvolutionMM.cu // CPU version author: Jesse Livezey // CPU version adapted from GPU version PyArrayObject* corrMM(PyArrayObject* bottom, PyArrayObject* weight, PyArrayObject* top, const int direction, const int dH = 1, const int dW = 1, const int dilH = 1, const int dilW = 1, const int padH_l = 0, const int padH_r = 0, const int padW_l = 0, const int padW_r = 0, const int numgroups = 1, const int unshared = 0) { if (PyArray_NDIM(bottom) != 4) { PyErr_SetString(PyExc_ValueError, "CorrMM requires bottom of 4D"); return NULL; } if (PyArray_TYPE(bottom) != %(float_typenum)s) { PyErr_SetString(PyExc_ValueError, "CorrMM received bottom with wrong type."); return NULL; } if (PyArray_NDIM(weight) != (unshared ? 6 : 4)) { PyErr_Format(PyExc_ValueError, "CorrMM requires weight of %%dD", unshared ? 6 : 4); return NULL; } if (PyArray_TYPE(weight) != %(float_typenum)s) { PyErr_SetString(PyExc_ValueError, "CorrMM received weight with wrong type."); return NULL; } if (PyArray_NDIM(top) != 4) { PyErr_SetString(PyExc_ValueError, "CorrMM requires top of 4D"); return NULL; } if (PyArray_TYPE(top) != %(float_typenum)s) { PyErr_SetString(PyExc_ValueError, "CorrMM received top with wrong type."); return NULL; } // Ensure data is contiguous bottom = PyArray_GETCONTIGUOUS(bottom); weight = PyArray_GETCONTIGUOUS(weight); top = PyArray_GETCONTIGUOUS(top); // Extract some shape information for later and check shape consistency // bottom: (batchSize, nChannels, bottomHeight, bottomWidth) const int batchSize = PyArray_DIMS(bottom)[0]; const int nChannels = PyArray_DIMS(bottom)[1]; const int bottomHeight = PyArray_DIMS(bottom)[2]; const int bottomWidth = PyArray_DIMS(bottom)[3]; // normal weights: (nFilters, nChannels, rows, columns) // unshared weights: (nFilters, topHeight, topWidth, nChannels, rows, columns) const int nFilters = PyArray_DIMS(weight)[0]; const int kH = PyArray_DIMS(weight)[unshared ? 4 : 2]; const int kW = PyArray_DIMS(weight)[unshared ? 5 : 3]; if (nChannels != PyArray_DIMS(weight)[unshared ? 3 : 1] * numgroups) { PyErr_SetString(PyExc_ValueError, "CorrMM images and kernel must have the same stack size\n"); return NULL; } if ((nFilters %% numgroups) != 0) { PyErr_SetString(PyExc_ValueError, "CorrMM the number of filters must be divisible by the number of groups\n"); return NULL; } // implicit dilated filter const int dil_kH = (kH - 1) * dilH + 1; const int dil_kW = (kW - 1) * dilW + 1; // top: (batchSize, nFilters, topHeight, topWidth) const int topHeightNoDH = (bottomHeight + padH_l + padH_r - dil_kH); const int topWidthNoDW = (bottomWidth + padW_l + padW_r - dil_kW); // the above values might be negative so we need to use Python-like // flooring integer division to be compatible with get_conv_output. // note: this macro implements Python's // for negative x only #define _CONV_FLOORDIV_X(x,y) ((x < 0) ? (- ((-x) / y) - (((-x) %% y) == 0 ? 0 : 1)) : (x / y)) const int topHeight = _CONV_FLOORDIV_X(topHeightNoDH, dH) + 1; const int topWidth = _CONV_FLOORDIV_X(topWidthNoDW, dW) + 1; #undef _CONV_FLOORDIV if (unshared) { if (topHeight != PyArray_DIMS(weight)[1] || topWidth != PyArray_DIMS(weight)[2]) { PyErr_Format(PyExc_ValueError, "CorrMM regions in kernel must match output regions:\n" " bottom shape: %%d %%d %%d %%d\n" " weight shape: %%d %%ld %%ld %%d %%d %%d" " (expected %%d %%d %%d %%d %%d %%d)\n" " top shape(calculated): %%d %%d %%d %%d\n", batchSize, nChannels, bottomHeight, bottomWidth, nFilters, PyArray_DIMS(weight)[1], PyArray_DIMS(weight)[2], nChannels / numgroups, kH, kW, nFilters, topHeight, topWidth, nChannels / numgroups, kH, kW, batchSize, nFilters, topHeight, topWidth); return NULL; } if (batchSize != PyArray_DIMS(top)[0] || nFilters != PyArray_DIMS(top)[1] || topHeight != PyArray_DIMS(top)[2] || topWidth != PyArray_DIMS(top)[3]) { PyErr_Format(PyExc_ValueError, "CorrMM shape inconsistency:\n" " bottom shape: %%d %%d %%d %%d\n" " weight shape: %%d %%d %%d %%d %%d %%d\n" " top shape: %%ld %%ld %%ld %%ld (expected %%d %%d %%d %%d)\n", batchSize, nChannels, bottomHeight, bottomWidth, nFilters, topHeight, topWidth, nChannels / numgroups, kH, kW, PyArray_DIMS(top)[0], PyArray_DIMS(top)[1], PyArray_DIMS(top)[2], PyArray_DIMS(top)[3], batchSize, nFilters, topHeight, topWidth); return NULL; } } else { if (batchSize != PyArray_DIMS(top)[0] || nFilters != PyArray_DIMS(top)[1] || topHeight != PyArray_DIMS(top)[2] || topWidth != PyArray_DIMS(top)[3]) { PyErr_Format(PyExc_ValueError, "CorrMM shape inconsistency:\n" " bottom shape: %%d %%d %%d %%d\n" " weight shape: %%d %%d %%d %%d\n" " top shape: %%ld %%ld %%ld %%ld (expected %%d %%d %%d %%d)\n", batchSize, nChannels, bottomHeight, bottomWidth, nFilters, nChannels / numgroups, kH, kW, PyArray_DIMS(top)[0], PyArray_DIMS(top)[1], PyArray_DIMS(top)[2], PyArray_DIMS(top)[3], batchSize, nFilters, topHeight, topWidth); return NULL; } } // Create temporary columns int max_threads = %(omp_get_max_threads)s; if (batchSize < max_threads) { max_threads = batchSize; } npy_intp col_dim[3]; col_dim[0] = (npy_intp)max_threads; col_dim[1] = (npy_intp)(nChannels * kW * kH); col_dim[2] = (npy_intp)(topHeight * topWidth); //Change to PyArray_ZEROS which is faster than PyArray_EMPTY. PyArrayObject* col = (PyArrayObject*)PyArray_ZEROS(3, col_dim, PyArray_TYPE(top), 0); if (NULL == col) { PyErr_Format(PyExc_RuntimeError, "CorrMM failed to allocate working memory of" " %%ld x %%ld x %%ld\n", col_dim[0], col_dim[1], col_dim[2]); return NULL; } // Define some useful variables const int batch_bottom_stride = PyArray_STRIDES(bottom)[0]/%(n_bytes)f; const int group_bottom_stride = (PyArray_STRIDES(bottom)[1] * nChannels / numgroups)/%(n_bytes)f; const int batch_top_stride = PyArray_STRIDES(top)[0]/%(n_bytes)f; const int group_top_stride = (PyArray_STRIDES(top)[1] * nFilters / numgroups)/%(n_bytes)f; const int K_ = col_dim[1] / numgroups; const int N_ = col_dim[2]; const int col_stride = (K_ * N_ * numgroups); const int group_col_stride = (K_ * N_); const int group_weight_stride = (PyArray_STRIDES(weight)[0] * nFilters / numgroups)/%(n_bytes)f; const int M_ = nFilters / numgroups; const int one_int = 1; const %(c_float_type)s one = 1.0; const %(c_float_type)s zero = 0.0; const int ldw = (K_ * N_); char NTrans = 'N'; char Trans = 'T'; PyArrayObject *output; if (batchSize == 0 || nChannels == 0 || nFilters == 0) { switch(direction) { case 0: output = top; break; case 1: output = weight; break; case 2: output = bottom; break; default: return NULL; } PyArray_FILLWBYTE(output, 0); } else if (direction == 0) { // forward pass output = top; // valid correlation: im2col, then gemm // Iterate over batch int blas_threads_saved = %(blas_get_num_threads)s; // Always forcing gemm to one thread when OpenMP is enalbed for best and stable performance. %(blas_set_num_threads)s(1); %(omp_flags)s for (int n = 0; n < batchSize; ++n) { int tid = %(omp_get_thread_num)s; // First, im2col im2col((%(float_type)s*)PyArray_DATA(bottom) + n * batch_bottom_stride, nChannels, bottomHeight,bottomWidth, kH, kW, dilH, dilW, padH_l, padH_r, padW_l, padW_r, dH, dW, (%(float_type)s*)PyArray_DATA(col)+ tid * col_stride); // Second, gemm if (unshared) { for (int g = 0; g < numgroups; ++g) { for (int reg = 0; reg < N_; ++reg) { %(gemv)s(&Trans, &K_, &M_, &one, (%(float_type)s*)PyArray_DATA(weight) + g * group_weight_stride + reg * K_, &ldw, (%(float_type)s*)PyArray_DATA(col) + tid * col_stride + g * group_col_stride + reg, &N_, &zero, (%(float_type)s*)PyArray_DATA(top) + n * batch_top_stride + g * group_top_stride + reg, &N_); } } } else { for ( int g = 0; g < numgroups; ++g){ // Second, gemm %(gemm)s(&NTrans, &NTrans, &N_, &M_, &K_, &one, (%(float_type)s*)PyArray_DATA(col) + tid * col_stride + g * group_col_stride, &N_, (%(float_type)s*)PyArray_DATA(weight) + g * group_weight_stride, &K_, &zero, (%(float_type)s*)PyArray_DATA(top) + n * batch_top_stride + g * group_top_stride, &N_); } } } // Restore to previous blas threads %(blas_set_num_threads)s(blas_threads_saved); /* // Original caffe code for comparison // Note that this code was translated from the Theano GPU code, // not the Caffe CPU code. // https://github.com/BVLC/caffe/blob/master/src/caffe/layers/conv_layer.cu // Note that this is for grouped convolution; we can ignore groups here, // but the group-related offsets help explain what M_, N_ and K_ are int weight_offset = M_ * K_; int col_offset = K_ * N_; int top_offset = M_ * N_; for (int n = 0; n < num_; ++n) { // First, im2col im2col_gpu(bottom_data + bottom[i]->offset(n), channels_, height_, width_, kernel_h_, kernel_w_, pad_h_, pad_w_, stride_h_, stride_w_, col_data); // Second, innerproduct with groups for (int g = 0; g < group_; ++g) { caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, M_, N_, K_, (Dtype)1., weight + weight_offset * g, col_data + col_offset * g, (Dtype)0., top_data + (*top)[i]->offset(n) + top_offset * g); == (see https://github.com/BVLC/caffe/blob/master/src/caffe/util/math_functions.cu#L16) cublasSgemm(CUBLAS_OP_N, CUBLAS_OP_N, N_, M_, K_, 1., col_data + col_offset * g, N_, weight + weight_offset * g, K_, 0., top_data + (*top)[i]->offset(n) + top_offset * g, N_); } } */ } else if (direction == 1) { // backprop wrt. weights output = weight; npy_intp weight_dim[2]; weight_dim[0] = (npy_intp)max_threads; if (unshared) weight_dim[1] = (npy_intp)(M_ * N_ * K_ * numgroups); else weight_dim[1] = (npy_intp)(M_ * K_ * numgroups); PyArrayObject* local_weight = (PyArrayObject*)PyArray_ZEROS(2, weight_dim, PyArray_TYPE(weight), 0); if (NULL == local_weight) { PyErr_Format(PyExc_RuntimeError, "CorrMM failed to allocate weight memory of %%ld x %%ld\n", weight_dim[0], weight_dim[1]); return NULL; } // valid convolution: im2col, then gemm // Iterate over batch int blas_threads_saved = %(blas_get_num_threads)s; // Always forcing gemm to one thread when OpenMP is enalbed for best and stable performance. %(blas_set_num_threads)s(1); // OMP for batch-level paralization %(omp_flags)s for (int n = 0; n < batchSize; ++n) { int tid = %(omp_get_thread_num)s; // First, im2col im2col((%(float_type)s*)PyArray_DATA(bottom) + n * batch_bottom_stride, nChannels, bottomHeight,bottomWidth, kH, kW, dilH, dilW, padH_l, padH_r, padW_l, padW_r, dH, dW, (%(float_type)s*)PyArray_DATA(col)+ tid * col_stride); // Second, gemm // Note that we accumulate into weight. We do so by setting beta = 0 // for the first iteration and beta = 1 for subsequent ones. (This // is faster than setting weight to all zeros before the loop.) if (unshared) { for (int g = 0; g < numgroups; ++g) { for (int reg = 0; reg < N_; ++reg) { %(gemm)s(&Trans, &NTrans, &K_, &M_, &one_int, &one, (%(float_type)s*)PyArray_DATA(col) + tid * col_stride + g * group_col_stride + reg, &N_, (%(float_type)s*)PyArray_DATA(top) + g * group_top_stride + n * batch_top_stride + reg, &N_, (n == 0) ? &zero : &one, (%(float_type)s*)PyArray_DATA(local_weight) + g * group_weight_stride + reg * K_ + tid * weight_dim[1], &ldw); } } } else { for(int g = 0; g < numgroups; ++g){ // Second, gemm // Note that we accumulate into weight. We do so by setting beta = 0 // for the first iteration and beta = 1 for subsequent ones. (This // is faster than setting weight to all zeros before the loop.) %(gemm)s(&Trans, &NTrans, &K_, &M_, &N_, &one, (%(float_type)s*)PyArray_DATA(col) + tid * col_stride + g * group_col_stride, &N_, (%(float_type)s*)PyArray_DATA(top) + g * group_top_stride + n * batch_top_stride, &N_, (n == 0) ? &zero : &one, (%(float_type)s*)PyArray_DATA(local_weight) + g * group_weight_stride + tid * weight_dim[1], &K_); } } } // Restore to previous blas threads %(blas_set_num_threads)s(blas_threads_saved); //aggregate weights memset((%(float_type)s*)PyArray_DATA(weight), 0, weight_dim[1]*sizeof(%(float_type)s)); /* * Put index "j" into outer loop to get the * correct result when openmp is used. */ %(omp_flags)s for(int j = 0; j < weight_dim[1]; ++j){ for(int i = 0; i < max_threads; ++i){ ((%(float_type)s*)PyArray_DATA(weight))[j] += *((%(float_type)s*)PyArray_DATA(local_weight) + i * weight_dim[1] + j); } } Py_DECREF(local_weight); /* // Original caffe code for comparison // Note that this code was translated from the Theano GPU code, // not the Caffe CPU code. // https://github.com/BVLC/caffe/blob/master/src/caffe/layers/conv_layer.cu // Note that this is for grouped convolution; we can ignore groups for (int n = 0; n < num_; ++n) { // Since we saved memory in the forward pass by not storing all col // data, we will need to recompute them. im2col_gpu(bottom_data + (*bottom)[i]->offset(n), channels_, height_, width_, kernel_h_, kernel_w_, pad_h_, pad_w_, stride_h_, stride_w_, col_data); // gradient w.r.t. weight. Note that we will accumulate diffs. for (int g = 0; g < group_; ++g) { caffe_gpu_gemm(CblasNoTrans, CblasTrans, M_, K_, N_, (Dtype)1., top_diff + top[i]->offset(n) + top_offset * g, col_data + col_offset * g, (Dtype)1., weight_diff + weight_offset * g); == (see https://github.com/BVLC/caffe/blob/master/src/caffe/util/math_functions.cu#L16) cublasSgemm(CUBLAS_OP_T, CUBLAS_OP_N, K_, M_, N_, 1.0, col_data + col_offset * g, N_, top_diff + top[i]->offset(n) + top_offset * g, N_, 1.0, weight_diff + weight_offset * g, K_); } } */ } else if (direction == 2) { // backprop wrt. inputs output = bottom; // bottom is set to zero here rather than inside of col2im PyArray_FILLWBYTE(bottom, 0); // full convolution: gemm, then col2im // Iterate over batch int blas_threads_saved = %(blas_get_num_threads)s; // Always forcing gemm to one thread when OpenMP is enalbed for best and stable performance. %(blas_set_num_threads)s(1); %(omp_flags)s for (int n = 0; n < batchSize; ++n) { int tid = %(omp_get_thread_num)s; if (unshared) { for (int g = 0; g < numgroups; ++g){ for (int reg = 0; reg < N_; ++reg){ %(gemm)s(&NTrans, &Trans, &one_int, &K_, &M_, &one, (%(float_type)s*)PyArray_DATA(top) + g * group_top_stride + n * batch_top_stride + reg, &N_, (%(float_type)s*)PyArray_DATA(weight) + g * group_weight_stride + reg * K_, &ldw, &zero, (%(float_type)s*)PyArray_DATA(col) + tid * col_stride + g * group_col_stride + reg, &N_); } } } else { for (int g = 0; g < numgroups; ++g) { %(gemm)s(&NTrans, &Trans, &N_, &K_, &M_, &one, (%(float_type)s*)PyArray_DATA(top) + g * group_top_stride + n * batch_top_stride, &N_, (%(float_type)s*)PyArray_DATA(weight) + g * group_weight_stride, &K_, &zero, (%(float_type)s*)PyArray_DATA(col) + tid * col_stride + g * group_col_stride, &N_); } } // col2im back to the data col2im((%(float_type)s*)PyArray_DATA(col) + tid * col_stride, nChannels, bottomHeight, bottomWidth, kH, kW, dilH, dilW, padH_l, padH_r, padW_l, padW_r, dH, dW, (%(float_type)s*)PyArray_DATA(bottom) + n * batch_bottom_stride); } // Restore to previous blas threads %(blas_set_num_threads)s(blas_threads_saved); /* // Original caffe code for comparison // Note that this code was translated from the Theano GPU code, // not the Caffe CPU code. // https://github.com/BVLC/caffe/blob/master/src/caffe/layers/conv_layer.cu for (int n = 0; n < num_; ++n) { // gradient w.r.t. bottom data, if necessary if (propagate_down[i]) { for (int g = 0; g < group_; ++g) { caffe_gpu_gemm(CblasTrans, CblasNoTrans, K_, N_, M_, (Dtype)1., weight + weight_offset * g, top_diff + top[i]->offset(n) + top_offset * g, (Dtype)0., col_diff + col_offset * g); == (see https://github.com/BVLC/caffe/blob/master/src/caffe/util/math_functions.cu#L16) cublasSgemm(CUBLAS_OP_N, CUBLAS_OP_T, N_, K_, M_, 1., top_diff + top[i]->offset(n) + top_offset * g, N_, weight + weight_offset * g, K_, 0., col_diff + col_offset * g, N_); } // col2im back to the data col2im_gpu(col_diff, channels_, height_, width_, kernel_h_, kernel_w_, pad_h_, pad_w_, stride_h_, stride_w_, bottom_diff + (*bottom)[i]->offset(n)); } } */ } // Free temporary columns Py_DECREF(col); // decref from contiguous check Py_DECREF(bottom); Py_DECREF(weight); Py_DECREF(top); // Note that we don't change the refcount of the output matrix here. Output // (re)allocation and refcounting is done in BaseCorrMM.c_code_helper(); // in here output is just aliased to one of bottom, weights, or top. return output; } Theano-1.0.0/theano/tensor/nnet/c_code/corr3d_gemm.c00006000163562€ÿÿÿþ0000004504613203335532023615 0ustar boccoset00000000000000// This uses a lot of code from Caffe (http://caffe.berkeleyvision.org/); // sources are clearly marked. Below we reproduce the original license of // the Caffe software. /* Copyright (c) 2014, The Regents of the University of California (Regents) All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ // (borrowed from Caffe: https://github.com/BVLC/caffe/blob/master/src/caffe/util/im2col.cpp) // Loops for fast unfold + copy void im3d2col(const %(float_type)s* data_im, const int channels, const int height, const int width, const int depth, const int kernel_h, const int kernel_w, const int kernel_d, const int dilation_h, const int dilation_w, const int dilation_d, const int pad_h, const int pad_w, const int pad_d, const int stride_h, const int stride_w, const int stride_d, %(float_type)s* data_col) { // Implicit dilated kernel size int dil_kernel_h = (kernel_h - 1) * dilation_h + 1; int dil_kernel_w = (kernel_w - 1) * dilation_w + 1; int dil_kernel_d = (kernel_d - 1) * dilation_d + 1; int height_col = (height + 2 * pad_h - dil_kernel_h) / stride_h + 1; int width_col = (width + 2 * pad_w - dil_kernel_w) / stride_w + 1; int depth_col = (depth + 2 * pad_d - dil_kernel_d) / stride_d + 1; int channels_col = channels * kernel_h * kernel_w * kernel_d; for (int c = 0; c < channels_col; ++c) { int d_offset = c %% kernel_d; int w_offset = (c / kernel_d) %% kernel_w; int h_offset = (c / kernel_w / kernel_d) %% kernel_h; int c_im = c / kernel_h / kernel_w / kernel_d; for (int h = 0; h < height_col; ++h) { int h_pad = h * stride_h - pad_h + h_offset * dilation_h; for (int w = 0; w < width_col; ++w) { int w_pad = w * stride_w - pad_w + w_offset * dilation_w; for (int d = 0; d < depth_col; ++d) { int d_pad = d * stride_d - pad_d + d_offset * dilation_d; if (h_pad >= 0 && h_pad < height && w_pad >= 0 && w_pad < width && d_pad >= 0 && d_pad < depth) data_col[(npy_intp)((c * height_col + h) * width_col + w) * depth_col + d] = data_im[(npy_intp)((c_im * height + h_pad) * width + w_pad) * depth + d_pad]; else data_col[(npy_intp)((c * height_col + h) * width_col + w) * depth_col + d] = 0.; } } } } } // Unlike the Caffe and Theano GPU verions, the data_im array is set to zero // before the col2im call rather than doing it here. So, the result is just // accumulated into data_im. void col2im3d(const %(float_type)s* data_col, const int channels, const int height, const int width, const int depth, const int patch_h, const int patch_w, const int patch_d, const int dilation_h, const int dilation_w, const int dilation_d, const int pad_h, const int pad_w, const int pad_d, const int stride_h, const int stride_w, const int stride_d, %(float_type)s* data_im) { // Implicit dilated patch int dil_patch_h = (patch_h - 1) * dilation_h + 1; int dil_patch_w = (patch_w - 1) * dilation_w + 1; int dil_patch_d = (patch_d - 1) * dilation_d + 1; int height_col = (height + 2 * pad_h - dil_patch_h) / stride_h + 1; int width_col = (width + 2 * pad_w - dil_patch_w) / stride_w + 1; int depth_col = (depth + 2 * pad_d - dil_patch_d) / stride_d + 1; int num_kernels = channels * height * width * depth; int channels_col = channels * patch_h * patch_w * patch_d; for (int c = 0; c < channels_col; ++c) { int d_offset = c %% patch_d; int w_offset = (c / patch_d) %% patch_w; int h_offset = (c / patch_w / patch_d) %% patch_h; int c_im = c / patch_h / patch_w / patch_d; for (int h = 0; h < height_col; ++h) { int h_pad = h * stride_h - pad_h + h_offset * dilation_h; for (int w = 0; w < width_col; ++w) { int w_pad = w * stride_w - pad_w + w_offset * dilation_w; for (int d = 0; d < depth_col; ++d) { int d_pad = d * stride_d - pad_d + d_offset * dilation_d; if (h_pad >= 0 && h_pad < height && w_pad >= 0 && w_pad < width && d_pad >= 0 && d_pad < depth) data_im[(npy_intp)((c_im * height + h_pad) * width + w_pad) * depth + d_pad] += data_col[(npy_intp)((c * height_col + h) * width_col + w) * depth_col + d]; } } } } } // Theano op code // GPU version authors: Arjun Jain, Frederic Bastien, Jan Schlueter // Reference code: https://github.com/BVLC/caffe/blob/master/src/caffe/layers/conv_layer.cu // and https://github.com/torch/cunn/blob/master/SpatialConvolutionMM.cu // CPU version author: Jesse Livezey // CPU version adapted from GPU version PyArrayObject* corr3dMM(PyArrayObject* bottom, PyArrayObject* weight, PyArrayObject* top, const int direction, const int dH = 1, const int dW = 1, const int dD = 1, const int dilH = 1, const int dilW = 1, const int dilD = 1, const int padH = 0, const int padW = 0, const int padD = 0, const int numgroups=1) { if (PyArray_NDIM(bottom) != 5) { PyErr_SetString(PyExc_ValueError, "Corr3dMM requires bottom of 5D"); return NULL; } if (PyArray_TYPE(bottom) != %(float_typenum)s) { PyErr_SetString(PyExc_ValueError, "Corr3dMM received bottom with wrong type."); return NULL; } if (PyArray_NDIM(weight) != 5) { PyErr_SetString(PyExc_ValueError, "Corr3dMM requires weight of 5D"); return NULL; } if (PyArray_TYPE(weight) != %(float_typenum)s) { PyErr_SetString(PyExc_ValueError, "Corr3dMM received weight with wrong type."); return NULL; } if (PyArray_NDIM(top) != 5) { PyErr_SetString(PyExc_ValueError, "Corr3dMM requires top of 5D"); return NULL; } if (PyArray_TYPE(top) != %(float_typenum)s) { PyErr_SetString(PyExc_ValueError, "Corr3dMM received top with wrong type."); return NULL; } // Ensure data is contiguous bottom = PyArray_GETCONTIGUOUS(bottom); weight = PyArray_GETCONTIGUOUS(weight); top = PyArray_GETCONTIGUOUS(top); // Extract some shape information for later and check shape consistency // bottom: (batchSize, nChannels, bottomHeight, bottomWidth, bottomDepth) const int batchSize = PyArray_DIMS(bottom)[0]; const int nChannels = PyArray_DIMS(bottom)[1]; const int bottomHeight = PyArray_DIMS(bottom)[2]; const int bottomWidth = PyArray_DIMS(bottom)[3]; const int bottomDepth = PyArray_DIMS(bottom)[4]; // weights: (nFilters, nChannels, rows, columns, slices) const int nFilters = PyArray_DIMS(weight)[0]; const int kH = PyArray_DIMS(weight)[2]; const int kW = PyArray_DIMS(weight)[3]; const int kD = PyArray_DIMS(weight)[4]; if (nChannels != PyArray_DIMS(weight)[1] * numgroups) { PyErr_SetString(PyExc_ValueError, "Corr3dMM images and kernel must have the same stack size\n"); return NULL; } if ((nFilters %% numgroups) != 0) { PyErr_SetString(PyExc_ValueError, "CorrMM the number of filters must be divisible by the number of groups\n"); return NULL; } // implicit dilated filter const int dil_kH = (kH - 1) * dilH + 1; const int dil_kW = (kW - 1) * dilW + 1; const int dil_kD = (kD - 1) * dilD + 1; // top: (batchSize, nFilters, topHeight, topWidth, topDepth) const int topHeightNoDH = (bottomHeight + 2*padH - dil_kH); const int topWidthNoDW = (bottomWidth + 2*padW - dil_kW); const int topDepthNoDD = (bottomDepth + 2*padD - dil_kD); // the above values might be negative so we need to use Python-like // flooring integer division to be compatible with get_conv_output. // note: this macro implements Python's // for negative x only #define _CONV_FLOORDIV_X(x,y) ((x < 0) ? (- ((-x) / y) - (((-x) %% y) == 0 ? 0 : 1)) : (x / y)) const int topHeight = _CONV_FLOORDIV_X(topHeightNoDH, dH) + 1; const int topWidth = _CONV_FLOORDIV_X(topWidthNoDW, dW) + 1; const int topDepth = _CONV_FLOORDIV_X(topDepthNoDD, dD) + 1; #undef _CONV_FLOORDIV if (batchSize != PyArray_DIMS(top)[0] || nFilters != PyArray_DIMS(top)[1] || topHeight != PyArray_DIMS(top)[2] || topWidth != PyArray_DIMS(top)[3] || topDepth != PyArray_DIMS(top)[4]) { PyErr_Format(PyExc_ValueError, "Corr3dMM shape inconsistency:\n" " bottom shape: %%d %%d %%d %%d %%d\n" " weight shape: %%d %%d %%d %%d %%d\n" " top shape: %%ld %%ld %%ld %%ld %%ld (expected %%d %%d %%d %%d %%d)\n", batchSize, nChannels, bottomHeight, bottomWidth, bottomDepth, nFilters, nChannels / numgroups, kH, kW, kD, PyArray_DIMS(top)[0], PyArray_DIMS(top)[1], PyArray_DIMS(top)[2], PyArray_DIMS(top)[3], PyArray_DIMS(top)[4], batchSize, nFilters, topHeight, topWidth, topDepth); return NULL; } // Create temporary columns int max_threads = %(omp_get_max_threads)s; if (batchSize < max_threads) { max_threads = batchSize; } npy_intp col_dim[3]; col_dim[0] = (npy_intp)max_threads; col_dim[1] = (npy_intp)(nChannels * kW * kH * kD); col_dim[2] = (npy_intp)(topHeight * topWidth * topDepth); //Change to PyArray_ZEROS which is faster than PyArray_EMPTY. PyArrayObject* col = (PyArrayObject*)PyArray_ZEROS(3, col_dim, PyArray_TYPE(top), 0); if (NULL == col) { PyErr_Format(PyExc_RuntimeError, "Corr3dMM failed to allocate working memory of" " %%ld x %%ld x %%ld\n", col_dim[0], col_dim[1], col_dim[2]); return NULL; } // Define some useful variables const int batch_bottom_stride = PyArray_STRIDES(bottom)[0]/%(n_bytes)f; const int group_bottom_stride = (PyArray_STRIDES(bottom)[1] * nChannels / numgroups)/%(n_bytes)f; const int batch_top_stride = PyArray_STRIDES(top)[0]/%(n_bytes)f; const int group_top_stride = (PyArray_STRIDES(top)[1] * nFilters / numgroups)/%(n_bytes)f; const int K_ = col_dim[1] / numgroups; const int N_ = col_dim[2]; const int col_stride = (K_ * N_ * numgroups); const int group_col_stride = (K_ * N_); const int group_weight_stride = (PyArray_STRIDES(weight)[0] * nFilters / numgroups)/%(n_bytes)f; const int M_ = nFilters / numgroups; const %(c_float_type)s one = 1.0; const %(c_float_type)s zero = 0.0; char NTrans = 'N'; char Trans = 'T'; PyArrayObject *output; if (batchSize == 0 || nChannels == 0 || nFilters == 0) { switch(direction) { case 0: output = top; break; case 1: output = weight; break; case 2: output = bottom; break; default: return NULL; } PyArray_FILLWBYTE(output, 0); } else if (direction == 0) { // forward pass output = top; // valid correlation: im3d2col, then gemm // Iterate over batch int blas_threads_saved = %(blas_get_num_threads)s; // Always forcing gemm to one thread when OpenMP is enalbed for best and stable performance. %(blas_set_num_threads)s(1); %(omp_flags)s for (int n = 0; n < batchSize; ++n) { int tid = %(omp_get_thread_num)s; // First, im3d2col im3d2col((%(float_type)s*)PyArray_DATA(bottom) + n * batch_bottom_stride, nChannels, bottomHeight, bottomWidth, bottomDepth, kH, kW, kD, dilH, dilW, dilD, padH, padW, padD, dH, dW, dD, (%(float_type)s*)PyArray_DATA(col)+ tid * col_stride); for ( int g = 0; g < numgroups; ++g){ // Second, gemm %(gemm)s(&NTrans, &NTrans, &N_, &M_, &K_, &one, (%(float_type)s*)PyArray_DATA(col)+ tid * col_stride + g * group_col_stride, &N_, (%(float_type)s*)PyArray_DATA(weight) + g * group_weight_stride, &K_, &zero, (%(float_type)s*)PyArray_DATA(top) + n * batch_top_stride + g * group_top_stride, &N_); } } // Restore to previous blas threads %(blas_set_num_threads)s(blas_threads_saved); } else if (direction == 1) { // backprop wrt. weights output = weight; npy_intp weight_dim[2]; weight_dim[0] = (npy_intp)max_threads; weight_dim[1] = (npy_intp)(M_ * K_ * numgroups); PyArrayObject* local_weight = (PyArrayObject*)PyArray_ZEROS(2, weight_dim, PyArray_TYPE(weight), 0); if (NULL == local_weight) { PyErr_Format(PyExc_RuntimeError, "Corr3dMM failed to allocate weight memory of %%ld x %%ld\n", weight_dim[0], weight_dim[1]); return NULL; } // valid convolution: im2col, then gemm // Iterate over batch int blas_threads_saved = %(blas_get_num_threads)s; // Always forcing gemm to one thread when OpenMP is enalbed for best and stable performance. %(blas_set_num_threads)s(1); // OMP for batch-level paralization %(omp_flags)s for (int n = 0; n < batchSize; ++n) { int tid = %(omp_get_thread_num)s; // First, im2col im3d2col((%(float_type)s*)PyArray_DATA(bottom) + n * batch_bottom_stride, nChannels, bottomHeight, bottomWidth, bottomDepth, kH, kW, kD, dilH, dilW, dilD, padH, padW, padD, dH, dW, dD, (%(float_type)s*)PyArray_DATA(col)+ tid * col_stride); for ( int g = 0; g < numgroups; ++g){ // Second, gemm // Note that we accumulate into weight. We do so by setting beta = 0 // for the first iteration and beta = 1 for subsequent ones. (This // is faster than setting weight to all zeros before the loop.) %(gemm)s(&Trans, &NTrans, &K_, &M_, &N_, &one, (%(float_type)s*)PyArray_DATA(col) + tid * col_stride + g * group_col_stride, &N_, (%(float_type)s*)PyArray_DATA(top) + n * batch_top_stride + g * group_top_stride, &N_, (n == 0) ? &zero : &one, (%(float_type)s*)PyArray_DATA(local_weight) + g * group_weight_stride + tid * weight_dim[1], &K_); } } // Restore to previous blas threads %(blas_set_num_threads)s(blas_threads_saved); //aggregate weights memset((%(float_type)s*)PyArray_DATA(weight), 0, M_ * K_*sizeof(%(float_type)s)); /* * Put index "j" into outer loop to get the * correct result when openmp is used. */ %(omp_flags)s for(int j = 0; j < weight_dim[1]; ++j){ for(int i = 0; i < max_threads; ++i){ ((%(float_type)s*)PyArray_DATA(weight))[j] += *((%(float_type)s*)PyArray_DATA(local_weight) + i * weight_dim[1] + j); } } Py_DECREF(local_weight); } else if (direction == 2) { // backprop wrt. inputs output = bottom; // bottom is set to zero here rather than inside of col2im PyArray_FILLWBYTE(bottom, 0); // full convolution: gemm, then col2im3d // Iterate over batch int blas_threads_saved = %(blas_get_num_threads)s; // Always forcing gemm to one thread when OpenMP is enalbed for best and stable performance. %(blas_set_num_threads)s(1); %(omp_flags)s for (int n = 0; n < batchSize; ++n) { int tid = %(omp_get_thread_num)s; for ( int g = 0; g < numgroups; ++g){ // gemm into columns %(gemm)s(&NTrans, &Trans, &N_, &K_, &M_, &one, (%(float_type)s*)PyArray_DATA(top) + n * batch_top_stride + g * group_top_stride, &N_, (%(float_type)s*)PyArray_DATA(weight) + g * group_weight_stride, &K_, &zero, (%(float_type)s*)PyArray_DATA(col) + tid * col_stride + g * group_col_stride, &N_); } // col2im back to the data col2im3d((%(float_type)s*)PyArray_DATA(col) + tid * col_stride, nChannels, bottomHeight, bottomWidth, bottomDepth, kH, kW, kD, dilH, dilW, dilD, padH, padW, padD, dH, dW, dD, (%(float_type)s*)PyArray_DATA(bottom) + n * batch_bottom_stride); } // Restore to previous blas threads %(blas_set_num_threads)s(blas_threads_saved); } // Free temporary columns Py_DECREF(col); // decref from contiguous check Py_DECREF(bottom); Py_DECREF(weight); Py_DECREF(top); // Note that we don't change the refcount of the output matrix here. Output // (re)allocation and refcounting is done in BaseCorr3dMM.c_code_helper(); // in here output is just aliased to one of bottom, weights, or top. return output; } Theano-1.0.0/theano/tensor/nnet/c_code/ctc_wrapper.c00006000163562€ÿÿÿþ0000002000313203335532023707 0ustar boccoset00000000000000#section support_code typedef struct ctc_context { struct ctcOptions options; void * workspace; int * input_lengths; int * flat_labels; int * label_lengths; } ctc_context_t; void ctc_context_init(ctc_context_t * context) { struct ctcOptions * options = &(context->options); memset(options, 0, sizeof(struct ctcOptions)); options->loc = CTC_CPU; #if defined(_OPENMP) options->num_threads = omp_get_num_threads(); #else options->num_threads = 1; #endif context->workspace = NULL; context->input_lengths = NULL; context->flat_labels = NULL; context->label_lengths = NULL; } void ctc_context_destroy(ctc_context_t * context) { free( context->workspace ); free( context->input_lengths ); free( context->flat_labels ); free( context->label_lengths ); } int ctc_check_result(ctcStatus_t retcode, const char * msg) { if( CTC_STATUS_SUCCESS != retcode ) { // Get error message from underlying library const char * ctc_msg = ctcGetStatusString( retcode ); PyErr_Format( PyExc_RuntimeError, "ConnectionistTemporalClassification: %s CTC error: %s", msg, ctc_msg ); return 1; } return 0; } void create_contiguous_input_lengths( PyArrayObject * input_lengths_arr, int ** input_lengths ) { npy_int num_elements = PyArray_DIMS( input_lengths_arr )[0]; *input_lengths = (int *) calloc( num_elements, sizeof(int) ); if ( NULL == (*input_lengths) ) return; for( npy_int elem_idx = 0; elem_idx < num_elements; ++elem_idx ) { (*input_lengths)[elem_idx] = *( (npy_int *) PyArray_GETPTR1( input_lengths_arr, elem_idx ) ); } } void create_flat_labels( PyArrayObject * label_matrix, int ** flat_labels, int ** label_lengths ) { npy_int rows = PyArray_DIMS( label_matrix )[0]; npy_int cols = PyArray_DIMS( label_matrix )[1]; *flat_labels = (int *) calloc( rows * cols, sizeof(int) ); if ( NULL == (*flat_labels) ) return; *label_lengths = (int *) calloc( rows, sizeof(int) ); if ( NULL == (*label_lengths) ) { free( *flat_labels ); *flat_labels = NULL; return; } npy_int label_index = 0; for( npy_int row_idx = 0; row_idx < rows; ++row_idx ) { npy_int label_length = 0; for( npy_int col_idx = 0; col_idx < cols; ++col_idx ) { npy_int label = *( (npy_int *) PyArray_GETPTR2( label_matrix, row_idx, col_idx ) ); if ( label >= 0 ) // negative values are assumed to be padding { (*flat_labels)[ label_index++ ] = label; ++label_length; } } (*label_lengths)[ row_idx ] = label_length; } } #section support_code_apply int APPLY_SPECIFIC(ctc_cost_cpu)(PyArrayObject * in_activations, PyArrayObject * in_labels, PyArrayObject * in_input_lengths, PyArrayObject ** out_costs, PyArrayObject ** out_gradients) { ctc_context_t ctc_object; ctc_context_t * context = &ctc_object; ctc_context_init( context ); if ( !PyArray_IS_C_CONTIGUOUS( in_activations ) ) { PyErr_SetString( PyExc_RuntimeError, "ConnectionistTemporalClassification: activations array must be C-contiguous." ); return 1; } npy_float32 * activations = (npy_float32 *) PyArray_DATA( in_activations ); create_contiguous_input_lengths( in_input_lengths, &(context->input_lengths) ); if ( NULL == context->input_lengths ) { // Destroy previous CTC context before returning exception ctc_context_destroy( context ); PyErr_Format( PyExc_MemoryError, "ConnectionistTemporalClassification: Could not allocate memory for input lengths" ); return 1; } // flatten labels to conform with library memory layout create_flat_labels( in_labels, &(context->flat_labels), &(context->label_lengths) ); if ( ( NULL == context->label_lengths ) || ( NULL == context->flat_labels ) ) { // Destroy previous CTC context before returning exception ctc_context_destroy( context ); PyErr_Format( PyExc_MemoryError, "ConnectionistTemporalClassification: Could not allocate memory for labels and their lengths" ); return 1; } npy_int minibatch_size = PyArray_DIMS( in_activations )[1]; npy_int alphabet_size = PyArray_DIMS( in_activations )[2]; npy_float32 * costs = NULL; npy_intp cost_size = minibatch_size; if ( (*out_costs) == NULL || // Symbolic variable has no memory backing PyArray_NDIM( *out_costs ) != 1 || // or, matrix has the wrong size PyArray_DIMS( *out_costs )[0] != cost_size ) { Py_XDECREF( *out_costs ); // Allocate new matrix *out_costs = (PyArrayObject *) PyArray_ZEROS( 1, &cost_size, NPY_FLOAT32, 0 ); if ( NULL == (*out_costs) ) { // Destroy previous CTC context before returning exception ctc_context_destroy( context ); PyErr_Format( PyExc_MemoryError, "ConnectionistTemporalClassification: Could not allocate memory for CTC costs" ); return 1; } } costs = (npy_float32 *) PyArray_DATA( *out_costs ); npy_float32 * gradients = NULL; if ( NULL != out_gradients ) // If gradient computation is not disabled { if ( NULL == (*out_gradients) || // Symbolic variable has no real backing PyArray_NDIM( *out_gradients ) != 3 || PyArray_DIMS( *out_gradients )[0] != PyArray_DIMS( in_activations )[0] || PyArray_DIMS( *out_gradients )[1] != PyArray_DIMS( in_activations )[1] || PyArray_DIMS( *out_gradients )[2] != PyArray_DIMS( in_activations )[2] ) { // Existing matrix is the wrong size. Make a new one. // Decrement ref counter to existing array Py_XDECREF( *out_gradients ); // Allocate new array *out_gradients = (PyArrayObject *) PyArray_ZEROS(3, PyArray_DIMS( in_activations ), NPY_FLOAT32, 0); if ( NULL == (*out_gradients) ) { // Destroy previous CTC context before returning exception ctc_context_destroy( context ); PyErr_Format( PyExc_MemoryError, "ConnectionistTemporalClassification: Could not allocate memory for CTC gradients!" ); return 1; } } gradients = (npy_float32 *) PyArray_DATA( *out_gradients ); } size_t cpu_workspace_size; int ctc_error; ctc_error = ctc_check_result( get_workspace_size( context->label_lengths, context->input_lengths, alphabet_size, minibatch_size, context->options, &cpu_workspace_size ), "Failed to obtain CTC workspace size." ); if ( ctc_error ) // Exception is set by ctc_check_result, return error here { // Destroy previous CTC context before returning exception ctc_context_destroy( context ); return 1; } context->workspace = malloc( cpu_workspace_size ); if ( NULL == context->workspace ) { // Destroy previous CTC context before returning exception ctc_context_destroy( context ); PyErr_Format( PyExc_MemoryError, "ConnectionistTemporalClassification: Failed to allocate memory for CTC workspace." ); return 1; } ctc_error = ctc_check_result( compute_ctc_loss( activations, gradients, context->flat_labels, context->label_lengths, context->input_lengths, alphabet_size, minibatch_size, costs, context->workspace, context->options ), "Failed to compute CTC loss function." ); if ( ctc_error ) // Exception is set by ctc_check_result, return error here { ctc_context_destroy( context ); return 1; } ctc_context_destroy( context ); return 0; } Theano-1.0.0/theano/tensor/nnet/nnet.py00006000163562€ÿÿÿþ0000026647313203335532021363 0ustar boccoset00000000000000""" Provides neural-network specific Ops. Notes ----- TODO: factor this out into a neural-network toolbox. We register all optimization with the gpu tag as we don't implement all the intermediate case on the GPU (in particular AdvancedSubtensor). So to make sure it run well on the gpu with fast_compile, we register them as needed for the GPU. This can be revisited later when all the intermediate part are on the GPU. """ from __future__ import absolute_import, print_function, division import logging import warnings import numpy as np from six.moves import xrange import theano from theano import gof from theano import scalar from theano.tensor import extra_ops, as_tensor_variable from theano.gof.opt import copy_stack_trace from theano.tensor import basic as tensor, subtensor, opt, elemwise from theano.tensor.type import (values_eq_approx_remove_inf, values_eq_approx_remove_nan) from theano.compile import optdb from theano.gof import Apply from theano.tensor.nnet.sigm import sigmoid, softplus from theano.gradient import DisconnectedType from theano.gradient import grad_not_implemented from theano.tensor.nnet.blocksparse import sparse_block_dot ############ # # TENSOR OPS # class SoftmaxWithBias(gof.Op): """ An L{Op} for the output of neural-net multiclass classifiers. Attributes ---------- x : a matrix of floats (32 or 64) b : a [row] vector of floats (32 or 64), length is number of cols in x This L{Op}'s output is softmax(x+b). softmax(x[i]) is the i'th distribution over len(x[i]) options. """ nin = 2 nout = 1 __props__ = () def make_node(self, x, b): x = tensor.as_tensor_variable(x) b = tensor.as_tensor_variable(b) if x.type.ndim != 2 \ or x.type.dtype not in tensor.float_dtypes: raise ValueError('x must be 2-d tensor of floats') if b.type.ndim != 1 \ or b.type.dtype not in tensor.float_dtypes: raise ValueError('b must be 1-d tensor of floats') sm = x.type() return Apply(self, [x, b], [sm]) def perform(self, node, input_storage, output_storage): x, b = input_storage if b.shape[0] != x.shape[1]: raise ValueError('b must have same number of columns as x') # sm = numpy.zeros_like(x) # for i in xrange(sm.shape[0]): # row = x[i] + b # sm[i] = numpy.exp(row - numpy.max(row)) # sm[i] *= 1.0 / numpy.sum(sm[i]) # output_storage[0][0] = sm if x.size == 0: # Numpy doesn't like the max of a zero-sized object. output_storage[0][0] = np.zeros(x.shape, dtype=x.dtype) return x_dtype = x.dtype # Perform computations in float32 otherwise the result is too imprecise if x.dtype == 'float16': x = x.astype('float32') x_plus_b = x + b[None, :] e_x = np.exp(x_plus_b - x_plus_b.max(axis=1)[:, None]) e_x *= 1.0 / e_x.sum(axis=1)[:, None] # default for copy is True and we don't need a copy if the # data type matches. output_storage[0][0] = e_x.astype(x_dtype, copy=False) def L_op(self, inp, outputs, grads): x, b = inp g_sm, = grads if isinstance(g_sm.type, DisconnectedType): return [DisconnectedType()(), DisconnectedType()()] dx = softmax_grad(g_sm, outputs[0]) db = tensor.sum(dx, axis=0) return dx, db def infer_shape(self, node, shape): return [shape[0]] def c_headers(self): return ['', ''] @staticmethod def c_code_template(dtype): # this implementation was lifted from # /u/bergstrj/cvs/bergstrj/src/feb07/nn.cxx # TODO: put this into a templated function, in the support code # TODO: declare the max of each row as an Op output # TODO: set error messages for failures in this code # TODO: use this to accept float32 and int32: # node.inputs[0].type.dtype_specs()[1] init_decl = """ npy_intp* Nx = PyArray_DIMS(%(x)s); npy_intp Sx = 0; npy_intp Sb = 0; npy_intp Ssm = 0; if (PyArray_NDIM(%(x)s) != 2) { PyErr_SetString(PyExc_ValueError, "not a 2d tensor"); %(fail)s; } if (PyArray_NDIM(%(b)s) != 1) { PyErr_SetString(PyExc_ValueError, "b not 1d tensor"); %(fail)s; } if ((PyArray_TYPE(%(x)s) != NPY_DOUBLE) && (PyArray_TYPE(%(x)s) != NPY_FLOAT)) { PyErr_SetString(PyExc_TypeError, "not a float"); %(fail)s; } if ((PyArray_TYPE(%(b)s) != NPY_DOUBLE) && (PyArray_TYPE(%(b)s) != NPY_FLOAT)) { PyErr_SetString(PyExc_TypeError, "b not float"); %(fail)s; } if ((PyArray_DIMS(%(x)s)[1] != PyArray_DIMS(%(b)s)[0])) { PyErr_Format(PyExc_ValueError, "number of columns in x (%%ld) does not match length of b (%%ld)", (long int)PyArray_DIMS(%(x)s)[1], (long int)PyArray_DIMS(%(b)s)[0]); %(fail)s; } if ((NULL == %(sm)s) || (PyArray_DIMS(%(sm)s)[0] != PyArray_DIMS(%(x)s)[0]) || (PyArray_DIMS(%(sm)s)[1] != PyArray_DIMS(%(x)s)[1])) { if (NULL != %(sm)s) Py_XDECREF(%(sm)s); %(sm)s = (PyArrayObject*)PyArray_SimpleNew(2, PyArray_DIMS(%(x)s), PyArray_TYPE(%(x)s)); if(!%(sm)s) { PyErr_SetString(PyExc_MemoryError, "failed to alloc sm output"); %(fail)s } } Sx = PyArray_STRIDES(%(x)s)[1]/sizeof(dtype_%(x)s); Sb = PyArray_STRIDES(%(b)s)[0]/sizeof(dtype_%(b)s); Ssm = PyArray_STRIDES(%(sm)s)[1]/sizeof(dtype_%(sm)s); """ begin_row_loop = """ for (size_t i = 0; i < Nx[0]; ++i) { size_t j; double sum = 0.0; const dtype_%(x)s* __restrict__ x_i = (dtype_%(x)s*)(PyArray_BYTES(%(x)s) + PyArray_STRIDES(%(x)s)[0] * i); const dtype_%(b)s* __restrict__ b_i = (dtype_%(b)s*)(PyArray_BYTES(%(b)s)); dtype_%(sm) s* __restrict__ sm_i = (dtype_%(sm)s*)(PyArray_BYTES(%(sm)s) + PyArray_STRIDES(%(sm)s)[0] * i); npy_intp Sx = PyArray_STRIDES(%(x)s)[1]/sizeof(dtype_%(x)s); npy_intp Sb = PyArray_STRIDES(%(b)s)[0]/sizeof(dtype_%(b)s); npy_intp Ssm = PyArray_STRIDES(%(sm)s)[1]/sizeof(dtype_%(sm)s); size_t row_max_j=0; dtype_%(sm)s row_max = x_i[0] + b_i[0]; //std::cout << "0 " << row_max << "\\n"; // Get the maximum value of the row for (j = 1; j < Nx[1]; ++j) { dtype_%(sm)s row_ij = x_i[j * Sx] + b_i[j * Sb]; //std::cout << "1 " << row_ij << "\\n"; row_max_j = (row_ij > row_max) ? j : row_max_j; row_max = (row_ij > row_max) ? row_ij : row_max; } """ inside_row_loop = """ for (j = 0; j < Nx[1]; ++j) { dtype_%(sm)s row_ij = x_i[j * Sx] + b_i[j * Sb]; //std::cout << "2 " << j << " " << row_ij << " " << row_max << "\\n"; dtype_%(sm)s sm_ij = exp(row_ij - row_max); //std::cout << "3 " << j << " " << sm_ij << "\\n"; sum += sm_ij; sm_i[j * Ssm] = sm_ij; } //cblas_dscal(x.N, 1.0 / sum, &mat_at(s,i,0), s.n); double sum_inv = 1.0 / sum; for (j = 0; j < Nx[1]; ++j) { sm_i[j * Ssm] *= sum_inv; } """ # Get the vectorized version of exp if it exist try: vec_exp = theano.scalar.exp.c_code_contiguous_raw(dtype, "Nx[1]", "sm_i", "sm_i") inside_row_loop_contig = """ for (j = 0; j < Nx[1]; ++j) { dtype_%%(sm)s row_ij = x_i[j * Sx] + b_i[j * Sb]; //std::cout << "2 " << j << " " << row_ij << " " << row_max << "\\n"; dtype_%%(sm)s sm_ij = row_ij - row_max; //std::cout << "3 " << j << " " << sm_ij << "\\n"; sm_i[j * Ssm] = sm_ij; } %(vec_exp)s; for (j = 0; j < Nx[1]; ++j) { sum += sm_i[j * Ssm]; } //cblas_dscal(x.N, 1.0 / sum, &mat_at(s,i,0), s.n); double sum_inv = 1.0 / sum; for (j = 0; j < Nx[1]; ++j) { sm_i[j * Ssm] *= sum_inv; } """ % locals() inside_row_loop = """ if(Ssm == 1){ %(inside_row_loop_contig)s }else{ %(inside_row_loop)s } """ % locals() except theano.gof.utils.MethodNotDefined: pass end_row_loop = """ } """ return (init_decl, begin_row_loop, inside_row_loop, end_row_loop) def c_code(self, node, name, inp, out, sub): x, b = inp sm, = out code_template = ''.join(self.c_code_template( node.inputs[0].type.dtype_specs()[1])) return code_template % dict(locals(), **sub) @staticmethod def c_code_cache_version(): return (8,) softmax_with_bias = SoftmaxWithBias() class SoftmaxGrad(gof.Op): """ Gradient wrt x of the Softmax Op. """ nin = 2 nout = 1 __props__ = () def make_node(self, dy, sm): dy = tensor.as_tensor_variable(dy) sm = tensor.as_tensor_variable(sm) if dy.type.ndim not in (1, 2) \ or dy.type.dtype not in tensor.float_dtypes: raise ValueError('dy must be 1-d or 2-d tensor of floats. Got ', dy.type) if dy.ndim == 1: dy = tensor.shape_padleft(dy, n_ones=1) if sm.ndim == 1: sm = tensor.shape_padleft(sm, n_ones=1) return Apply(self, [dy, sm], [sm.type()]) def perform(self, node, input_storage, output_storage): dy, sm = input_storage dx = np.zeros_like(sm) # dx[i,j] = - (\sum_k dy[i,k] sm[i,k]) sm[i,j] + dy[i,j] sm[i,j] for i in xrange(sm.shape[0]): dy_times_sm_i = dy[i] * sm[i] dx[i] = dy_times_sm_i - sum(dy_times_sm_i) * sm[i] output_storage[0][0] = dx def grad(self, inp, grads): dy, sm = inp g, = grads tmp = g + tensor.neg(tensor.sum(g * sm, axis=1).dimshuffle((0, 'x'))) g_dy = tmp * sm tmp2 = tensor.sum(dy * sm, axis=1).dimshuffle((0, 'x')) g_sm = tmp * dy - g * tmp2 return g_dy, g_sm def infer_shape(self, node, shape): return [shape[1]] def c_code_cache_version(self): return (3,) def c_code(self, node, name, inp, out, sub): dy, sm = inp dx, = out return ''' if ((PyArray_TYPE(%(dy)s) != NPY_DOUBLE) && (PyArray_TYPE(%(dy)s) != NPY_FLOAT)) { PyErr_SetString(PyExc_TypeError, "types should be float or float64"); %(fail)s; } if ((PyArray_TYPE(%(sm)s) != NPY_DOUBLE) && (PyArray_TYPE(%(sm)s) != NPY_FLOAT)) { PyErr_SetString(PyExc_TypeError, "types should be float or float64"); %(fail)s; } if ((PyArray_NDIM(%(dy)s) != 2) || (PyArray_NDIM(%(sm)s) != 2)) { PyErr_SetString(PyExc_ValueError, "rank error"); %(fail)s; } if (PyArray_DIMS(%(dy)s)[0] != PyArray_DIMS(%(sm)s)[0]) { PyErr_SetString(PyExc_ValueError, "dy.shape[0] != sm.shape[0]"); %(fail)s; } if ((NULL == %(dx)s) || (PyArray_DIMS(%(dx)s)[0] != PyArray_DIMS(%(sm)s)[0]) || (PyArray_DIMS(%(dx)s)[1] != PyArray_DIMS(%(sm)s)[1])) { Py_XDECREF(%(dx)s); %(dx)s = (PyArrayObject*) PyArray_SimpleNew(2, PyArray_DIMS(%(sm)s), PyArray_TYPE(%(sm)s)); if (!%(dx)s) { PyErr_SetString(PyExc_MemoryError, "failed to alloc dx output"); %(fail)s; } } for (size_t i = 0; i < PyArray_DIMS(%(dx)s)[0]; ++i) { const dtype_%(dy)s* __restrict__ dy_i = (dtype_%(dy)s*) (PyArray_BYTES(%(dy)s) + PyArray_STRIDES(%(dy)s)[0] * i); npy_intp Sdy = PyArray_STRIDES(%(dy)s)[1]/sizeof(dtype_%(dy)s); const dtype_%(sm)s* __restrict__ sm_i = (dtype_%(sm)s*) (PyArray_BYTES(%(sm)s) + PyArray_STRIDES(%(sm)s)[0] * i); npy_intp Ssm = PyArray_STRIDES(%(sm)s)[1]/sizeof(dtype_%(sm)s); dtype_%(dx) s* __restrict__ dx_i = (dtype_%(dx)s*) (PyArray_BYTES(%(dx)s) + PyArray_STRIDES(%(dx)s)[0] * i); npy_intp Sdx = PyArray_STRIDES(%(dx)s)[1]/sizeof(dtype_%(dx)s); double sum_dy_times_sm = 0.; for (size_t j = 0; j < PyArray_DIMS(%(dx)s)[1]; ++j) { dx_i[j * Sdx] = dy_i[j * Sdy] * sm_i[j * Ssm]; sum_dy_times_sm += dx_i[j * Sdx]; } for (size_t j = 0; j < PyArray_DIMS(%(dx)s)[1]; ++j) { dx_i[j * Sdx] -= sum_dy_times_sm * sm_i[j * Ssm]; } } ''' % dict(locals(), **sub) softmax_grad = SoftmaxGrad() class Softmax(gof.Op): """ Softmax activation function :math:`\\varphi(\\mathbf{x})_j = \\frac{e^{\mathbf{x}_j}}{\sum_{k=1}^K e^{\mathbf{x}_k}}` where :math:`K` is the total number of neurons in the layer. This activation function gets applied row-wise. """ nin = 1 nout = 1 __props__ = () def make_node(self, x): x = tensor.as_tensor_variable(x) if x.type.ndim not in (1, 2) \ or x.type.dtype not in tensor.float_dtypes: raise ValueError('x must be 1-d or 2-d tensor of floats. Got %s' % x.type) if x.ndim == 1: warnings.warn("DEPRECATION: If x is a vector, Softmax will not automatically pad x " "anymore in next releases. If you need it, please do it manually. The " "vector case is gonna be supported soon and the output will be a vector.", stacklevel=4) x = tensor.shape_padleft(x, n_ones=1) return Apply(self, [x], [x.type()]) def perform(self, node, input_storage, output_storage): x, = input_storage e_x = np.exp(x - x.max(axis=1)[:, None]) sm = e_x / e_x.sum(axis=1)[:, None] output_storage[0][0] = sm def L_op(self, inp, outputs, grads): x, = inp g_sm, = grads return [softmax_grad(g_sm, outputs[0])] def R_op(self, inputs, eval_points): # I think the Jacobian is symmetric so the R_op # is the same as the grad if None in eval_points: return [None] return self.L_op(inputs, [self(*inputs)], eval_points) def infer_shape(self, node, shape): return shape def c_headers(self): return ['', ''] @staticmethod def c_code_template(dtype): # this implementation was lifted from # /u/bergstrj/cvs/bergstrj/src/feb07/nn.cxx # TODO: put this into a templated function, in the support code # TODO: declare the max of each row as an Op output # TODO: set error messages for failures in this code # TODO: use this to accept float32 and int32: node.inputs[0].type.dtype_specs()[1] init_decl = """ npy_intp* Nx = PyArray_DIMS(%(x)s); npy_intp Sx1 = 0; npy_intp Ssm1 = 0; if (PyArray_NDIM(%(x)s) != 2) { PyErr_SetString(PyExc_ValueError, "not a 2d tensor"); %(fail)s; } if ((PyArray_TYPE(%(x)s) != NPY_DOUBLE) && (PyArray_TYPE(%(x)s) != NPY_FLOAT)) { PyErr_SetString(PyExc_TypeError, "not a float"); %(fail)s; } if ((NULL == %(sm)s) || (PyArray_DIMS(%(sm)s)[0] != PyArray_DIMS(%(x)s)[0]) || (PyArray_DIMS(%(sm)s)[1] != PyArray_DIMS(%(x)s)[1])) { Py_XDECREF(%(sm)s); %(sm)s = (PyArrayObject*)PyArray_SimpleNew(2, PyArray_DIMS(%(x)s), PyArray_TYPE(%(x)s)); if(!%(sm)s) { PyErr_SetString(PyExc_MemoryError, "failed to alloc sm output"); %(fail)s } } Sx1 = PyArray_STRIDES(%(x)s)[1]/sizeof(dtype_%(x)s); Ssm1 = PyArray_STRIDES(%(sm)s)[1]/sizeof(dtype_%(sm)s); """ begin_row_loop = """ for (size_t i = 0; i < Nx[0]; ++i) { size_t j; double sum = 0.0; const dtype_%(x)s* __restrict__ x_i = (dtype_%(x)s*)(PyArray_BYTES(%(x)s) + PyArray_STRIDES(%(x)s)[0] * i); dtype_%(sm) s* __restrict__ sm_i = (dtype_%(sm)s*)(PyArray_BYTES(%(sm)s) + PyArray_STRIDES(%(sm)s)[0] * i); dtype_%(sm)s row_max = x_i[0]; //std::cout << "0 " << row_max << "\\n"; // Get the maximum value of the row for (j = 1; j < Nx[1]; ++j) { dtype_%(sm)s row_ij = x_i[j * Sx1] ; //std::cout << "1 " << row_ij << "\\n"; row_max = (row_ij > row_max) ? row_ij : row_max; } """ inside_row_loop = """ for (j = 0; j < Nx[1]; ++j) { dtype_%(sm)s row_ij = x_i[j * Sx1] ; //std::cout << "2 " << j << " " << row_ij << " " << row_max << "\\n"; dtype_%(sm)s sm_ij = exp(row_ij - row_max); //std::cout << "3 " << j << " " << sm_ij << "\\n"; sum += sm_ij; sm_i[j * Ssm1] = sm_ij; } //cblas_dscal(x.N, 1.0 / sum, &mat_at(s,i,0), s.n); double sum_inv = 1.0 / sum; for (j = 0; j < Nx[1]; ++j) { sm_i[j * Ssm1] *= sum_inv; } """ # Get the vectorized version of exp if it exist try: vec_exp = theano.scalar.exp.c_code_contiguous_raw(dtype, "Nx[1]", "sm_i", "sm_i") inside_row_loop_contig = """ for (j = 0; j < Nx[1]; ++j) { sm_i[j * Ssm1] = x_i[j * Sx1] - row_max; } %(vec_exp)s; for (j = 0; j < Nx[1]; ++j) { sum += sm_i[j * Ssm1]; } //cblas_dscal(x.N, 1.0 / sum, &mat_at(s,i,0), s.n); double sum_inv = 1.0 / sum; for (j = 0; j < Nx[1]; ++j) { sm_i[j * Ssm1] *= sum_inv; } """ % locals() inside_row_loop = """ if(Ssm1 == 1){ %(inside_row_loop_contig)s }else{ %(inside_row_loop)s } """ % locals() except theano.gof.utils.MethodNotDefined: pass end_row_loop = """ } """ return (init_decl, begin_row_loop, inside_row_loop, end_row_loop) def c_code(self, node, name, inp, out, sub): x, = inp sm, = out code_template = ''.join(self.c_code_template( node.inputs[0].type.dtype_specs()[1])) return code_template % dict(locals(), **sub) @staticmethod def c_code_cache_version(): return (3,) softmax_op = Softmax() class LogSoftmax(gof.Op): """ LogSoftmax activation function :math:`\\varphi(\\mathbf{x})_j = \\e^{(\mathbf{x}_j - log{\sum_{k=1}^K e^{\mathbf{x}_k})}} where :math:`K` is the total number of neurons in the layer. This activation function gets applied row-wise. """ __props__ = () def make_node(self, x): x = tensor.as_tensor_variable(x) if x.type.ndim not in (1, 2) \ or x.type.dtype not in tensor.float_dtypes: raise ValueError('x must be 1-d or 2-d tensor of floats. Got %s' % x.type) if x.ndim == 1: warnings.warn("DEPRECATION: If x is a vector, LogSoftmax will not automatically pad x " "anymore in next releases. If you need it, please do it manually. The " "vector case is gonna be supported soon and the output will be a vector.", stacklevel=4) x = tensor.shape_padleft(x, n_ones=1) return Apply(self, [x], [x.type()]) def perform(self, node, input_storage, output_storage): x, = input_storage xdev = x - x.max(axis=1)[:, None] lsm = xdev - np.log(np.sum(np.exp(xdev), axis=1, keepdims=True)) output_storage[0][0] = lsm def grad(self, inp, grads): x, = inp sm = softmax_op(x) return [grads[0] - tensor.sum(grads[0], axis=1, keepdims=True) * sm] def R_op(self, inputs, eval_points): # I think the Jacobian is symmetric so the R_op # is the same as the grad if None in eval_points: return [None] return self.grad(inputs, eval_points) def infer_shape(self, node, shape): return shape def c_headers(self): return [''] @staticmethod def c_code_template(dtype): init_decl = """ npy_intp* Nx = PyArray_DIMS(%(x)s); npy_intp Sx1 = 0; npy_intp Ssm1 = 0; if (PyArray_NDIM(%(x)s) != 2) { PyErr_SetString(PyExc_ValueError, "not a 2d tensor"); %(fail)s; } if ((PyArray_TYPE(%(x)s) != NPY_DOUBLE) && (PyArray_TYPE(%(x)s) != NPY_FLOAT)) { PyErr_SetString(PyExc_TypeError, "not a float"); %(fail)s; } if ((NULL == %(sm)s) || (PyArray_DIMS(%(sm)s)[0] != PyArray_DIMS(%(x)s)[0]) || (PyArray_DIMS(%(sm)s)[1] != PyArray_DIMS(%(x)s)[1])) { Py_XDECREF(%(sm)s); %(sm)s = (PyArrayObject*)PyArray_SimpleNew( 2, PyArray_DIMS(%(x)s), PyArray_TYPE(%(x)s)); if(!%(sm)s) { PyErr_SetString(PyExc_MemoryError, "failed to alloc sm output"); %(fail)s } } Sx1 = PyArray_STRIDES(%(x)s)[1]/sizeof(dtype_%(x)s); Ssm1 = PyArray_STRIDES(%(sm)s)[1]/sizeof(dtype_%(sm)s); """ begin_row_loop = """ // minibatch loop for (size_t i = 0; i < Nx[0]; ++i) { size_t j; double sum = 0.0; const dtype_%(x)s* __restrict__ x_i = (dtype_%(x)s*)( PyArray_BYTES(%(x)s) + PyArray_STRIDES(%(x)s)[0] * i); dtype_%(sm)s* __restrict__ sm_i = (dtype_%(sm)s*)( PyArray_BYTES(%(sm)s) + PyArray_STRIDES(%(sm)s)[0] * i); dtype_%(sm)s row_max = x_i[0]; // Get the maximum value of the row for (j = 1; j < Nx[1]; ++j) { dtype_%(sm)s x_ij = x_i[j * Sx1] ; row_max = (x_ij > row_max) ? x_ij : row_max; } """ inside_row_loop = """ // Compute xdev and sum(exp(xdev), axis=1) double xdev_exp_row_sum = 0.0; for (j = 0; j < Nx[1]; j++) { // use sm_i to temporary store xdev sm_i[j * Ssm1] = (dtype_%(sm)s) (x_i[j * Sx1] - row_max); xdev_exp_row_sum += exp(sm_i[j * Ssm1]); } // Write sm = xdev - log(sum(exp(xdev), axis=1)) xdev_exp_row_sum = log(xdev_exp_row_sum); for (j = 0; j < Nx[1]; ++j) { sm_i[j * Ssm1] -= (dtype_%(sm)s) xdev_exp_row_sum; } """ end_row_loop = """ } """ return (init_decl, begin_row_loop, inside_row_loop, end_row_loop) def c_code(self, node, name, inp, out, sub): x, = inp sm, = out code_template = ''.join(self.c_code_template( node.inputs[0].type.dtype_specs()[1])) return code_template % dict(locals(), **sub) @staticmethod def c_code_cache_version(): return (0,) logsoftmax_op = LogSoftmax() # This is not registered in stabilize, as it cause some crossentropy # optimization to not be inserted. @opt.register_specialize('stabilize', 'fast_compile') @gof.local_optimizer([tensor.Elemwise]) def local_logsoftmax(node): """ Detect Log(Softmax(x)) and replace it with LogSoftmax(x) Note: only forward pass is affected """ if (isinstance(node.op, tensor.Elemwise) and isinstance(node.op.scalar_op, scalar.basic.Log) and len(node.inputs) == 1 and node.inputs[0].owner is not None and isinstance(node.inputs[0].owner.op, Softmax)): inVars = node.inputs[0].owner.inputs[0] new_op = LogSoftmax() ret = new_op(inVars) ret .tag.values_eq_approx = values_eq_approx_remove_inf copy_stack_trace([node.inputs[0], node.outputs[0]], ret) return [ret] # This is not registered in stabilize, as it cause some crossentropy # optimization to not be inserted. @opt.register_specialize('stabilize', 'fast_compile') @gof.local_optimizer([SoftmaxGrad]) def local_logsoftmax_grad(node): """ Detect Log(Softmax(x))'s grad and replace it with LogSoftmax(x)'s grad Note: only grad is affected """ if (isinstance(node.op, SoftmaxGrad) and len(node.inputs) == 2 and node.inputs[0].owner is not None and node.inputs[0].owner.op == tensor.true_div and len(node.inputs[0].owner.inputs) >= 2 and node.inputs[0].owner.inputs[1].owner is not None and node.inputs[0].owner.inputs[1].owner.op == softmax_op and node.inputs[1] == node.inputs[0].owner.inputs[1] and not ( # skip if it will be optimized by # local_advanced_indexing_crossentropy_onehot_grad node.inputs[0].owner.op == tensor.true_div and node.inputs[0].owner.inputs[0].owner is not None and isinstance(node.inputs[0].owner.inputs[0].owner.op, subtensor.AdvancedIncSubtensor))): # get parameters from unoptimized op sm = node.inputs[0].owner.inputs[1] # sm_input = node.inputs[1].owner.inputs[0] grads = node.inputs[0].owner.inputs[0] if grads.broadcastable[1] and not sm.broadcastable[1]: grads = tensor.alloc(grads, grads.shape[0], sm.shape[1]) ret = grads - tensor.sum(grads, axis=1, keepdims=True) * sm ret.tag.values_eq_approx = values_eq_approx_remove_nan copy_stack_trace(node.outputs[0], ret) return [ret] def softmax_graph(c): return tensor.exp(c) / tensor.exp(c).sum(axis=-1, keepdims=True) def softmax(c): c = as_tensor_variable(c) if c.broadcastable[-1]: warnings.warn("The softmax is applied on a dimension of shape 1, which does not have a semantic meaning.") return softmax_op(c) def logsoftmax(c): return logsoftmax_op(c) @opt.register_specialize('fast_compile_gpu') @gof.local_optimizer([softmax_op]) def local_softmax_with_bias(node): """ Try to turn softmax(sum_of_stuff) -> softmax_w_bias(matrix, bias). """ if node.op == softmax_op: x, = node.inputs if x.owner and x.owner.op == tensor.add: vectors = [] non_vectors = [] for x_in in x.owner.inputs: if list(x_in.type.broadcastable) == [True, False]: # print isinstance(x_in.owner.op, # tensor.DimShuffle) since specialization comes # relatively late in optimization, we don't want to # put in extra DimShuffles un-necessarily. if (x_in.owner and isinstance(x_in.owner.op, tensor.DimShuffle) and list(x_in.owner.inputs[0].type.broadcastable) == [False]): # cut out the DimShuffle that was broadcasting a vector vectors.append(x_in.owner.inputs[0]) else: # insert an extra DimShuffle to correct the old one vectors.append(tensor. DimShuffle((True, False), (1,))(x_in)) else: non_vectors.append(x_in) # If all the inputs were vectors or broadcasted vectors, # we broadcast one of them to be used as a matrix if len(non_vectors) == 0: assert len(vectors) > 0 # we should have at least 1 input... promoted_vector = vectors.pop() non_vectors.append(tensor.shape_padleft(promoted_vector)) assert non_vectors # not empty if vectors: # we're in business... if len(vectors) > 1: vector_sum = tensor.add(*vectors) copy_stack_trace(x_in, vector_sum) else: vector_sum = vectors[0] if len(non_vectors) > 1: non_vector_sum = tensor.add(*non_vectors) copy_stack_trace(x_in, non_vector_sum) else: non_vector_sum = non_vectors[0] try: sm_bias = softmax_with_bias(non_vector_sum, vector_sum) copy_stack_trace(node.outputs[0], sm_bias) except Exception: # if our arguments have the wrong types, then # forget about it return if sm_bias.type == node.outputs[0].type: # This condition is not always true. See the test # nnet/tests/test_nnet.py:T_SoftmaxWithBias.test_broadcast return [sm_bias] def softmax_simplifier(numerators, denominators): for numerator in list(numerators): # TODO: a single softmax'd vector?? if not numerator.type.dtype.startswith('float'): continue if numerator.ndim != 2: continue if numerator.owner and numerator.owner.op == tensor.exp: x = numerator.owner.inputs[0] else: continue matching_denom = None for denominator in denominators: if denominator.owner and isinstance(denominator.owner.op, tensor.DimShuffle): if denominator.owner.op.new_order == (0, 'x'): z = denominator.owner.inputs[0] # thing getting dimshuffled if z.owner and isinstance(z.owner.op, tensor.Sum): # print 'ASDF', denominator.owner.op.new_order # print z.owner.op.axis if z.owner.op.axis == (1,): # print "almost there.. softmax", x, z.owner.inputs[0] if z.owner.inputs[0] is numerator: matching_denom = denominator break if matching_denom: softmax = softmax_op(x) copy_stack_trace(numerator, softmax) numerators.remove(numerator) denominators.remove(matching_denom) numerators.append(softmax) return numerators, denominators opt.local_mul_canonizer.add_simplifier(softmax_simplifier, 'softmax_simplifier') class CrossentropySoftmaxArgmax1HotWithBias(gof.Op): """ A special compound L{Op} for the output of neural-net classifiers. Parameters ---------- x : a matrix of floats (32 or 64) b : a [row] vector of floats (32 or 64), length is number of cols in x y_idx : a [column] vector of int (32 or 64), length is number of rows in x Returns ------- object row-wise NLL, softmax(x+b), row-wise argmax of (x+b). @precondition: every entry in y_idx is a valid (non-negative) column index into x This L{Op} has three outputs: - KL(softmax(x+b), y) - softmax(x+b) - argmax(x+b) softmax(x[i]) is the i'th distribution over len(x[i]) options argmax(x) is the index of x's greatest element y_idx[i] is an integer index, encoding a 1-hot distribution. In practice, when we are trying to do classification, we have one row in x and y_idx per example, and y[i] is the index of the (correct) class of the i'th example. """ nin = 3 nout = 3 __props__ = () def __init__(self, **kwargs): gof.Op.__init__(self, **kwargs) def make_node(self, x, b, y_idx): x = tensor.as_tensor_variable(x) b = tensor.as_tensor_variable(b) y_idx = tensor.as_tensor_variable(y_idx) if x.type.ndim != 2 \ or x.type.dtype not in tensor.float_dtypes: raise ValueError('x must be 2-d tensor of floats', x.type) if b.type.ndim != 1 \ or x.type.dtype not in tensor.float_dtypes: raise ValueError('b must be 1-d tensor of floats', b.type) if y_idx.type.ndim != 1 \ or y_idx.type.dtype not in tensor.discrete_dtypes: raise ValueError('y_idx must be 1-d tensor of [u]ints', y_idx.type) # TODO: Is this correct? It used to be y, not y_idx nll = tensor.TensorType(x.type.dtype, y_idx.type.broadcastable).make_variable() # nll = TensorType(x.dtype, y.broadcastable) sm = x.type() am = y_idx.type() return Apply(self, [x, b, y_idx], [nll, sm, am]) def perform(self, node, input_storage, output_storage): """ The math, where x is an input vector, and t is a target index: softmax(x)[i] = exp(x[i]) / sum_j(exp(x[j])) nll(x,t) = -log(softmax(x)[t]) We compute this by subtracting off the max of x. This avoids numerical instability. m = max_j x[j] softmax(x)[i] = exp(x[i] -m) / sum_j(exp(x[j] - m)) nll = -log(exp(x[t] -m) / sum_j(exp(x[j] - m))) = -x[t] + m + log( sum_j(exp(x[j] - m))) """ x, b, y_idx = input_storage if b.shape[0] != x.shape[1]: raise ValueError('b must have same number of columns as x') if y_idx.shape[0] != x.shape[0]: raise ValueError('y_idx must have same number of rows as x') if any(y_idx < 0): raise ValueError("y_i value out of bounds") sm = np.zeros_like(x) # softmax nll = np.zeros(x.shape[0], dtype=node.outputs[0].type.dtype) # nll(y | softmax(x)) am = np.zeros_like(y_idx) for i in xrange(sm.shape[0]): # add the bias vector to the i'th row of x row = x[i] + b # get the maximum value of i'th row for numerically safe # softmax / nll am[i] = np.argmax(row) m = row[am[i]] # compute the unnormalized softmax, and normalization constant sm[i] = np.exp(row - m) sum_j = np.sum(sm[i]) # sum_j(exp(x[j] - m)) # normalized our softmax sm[i] *= 1.0 / sum_j # store the nll nll[i] = -row[y_idx[i]] + m + np.log(sum_j) output_storage[0][0] = nll output_storage[1][0] = sm output_storage[2][0] = am def infer_shape(self, node, shapes): x_shp, b_shp, idx_shp = shapes nll_shp = (x_shp[0],) sm_shp = x_shp am_shp = idx_shp return [nll_shp, sm_shp, am_shp] def connection_pattern(self, node): return [[True, True, True], # x [True, True, True], # b [False, False, True]] # y_idx def grad(self, inp, grads): x, b, y_idx = inp g_nll, g_sm, g_am = grads dx_terms = [] db_terms = [] d_idx_terms = [] if not isinstance(g_nll.type, DisconnectedType): nll, sm = crossentropy_softmax_1hot_with_bias(x, b, y_idx) dx = crossentropy_softmax_1hot_with_bias_dx(g_nll, sm, y_idx) db = tensor.sum(dx, axis=[0]) dx_terms.append(dx) db_terms.append(db) if not isinstance(g_sm.type, DisconnectedType): dx, db = softmax_with_bias.L_op((x, b), [softmax_with_bias(x, b)], (g_sm, )) dx_terms.append(dx) db_terms.append(db) if not isinstance(g_am.type, DisconnectedType): dx_terms.append(x.zeros_like()) db_terms.append(b.zeros_like()) d_idx_terms.append(y_idx.zeros_like()) def fancy_sum(terms): if len(terms) == 0: return DisconnectedType()() rval = terms[0] for term in terms[1:]: rval = rval + term return rval return [fancy_sum(terms) for terms in [dx_terms, db_terms, d_idx_terms]] def c_headers(self): return ['', ''] @staticmethod def c_code_template(dtype): # this implementation was lifted from # /u/bergstrj/cvs/bergstrj/src/feb07/nn.cxx # TODO: put this into a templated function, in the support code # TODO: declare the max of each row as an Op output # TODO: set error messages for failures in this code # TODO: use this to accept float32 and int32: node.inputs[0].type.dtype_specs()[1] (init_decl, begin_row_loop, inside_row_loop, end_row_loop) = \ SoftmaxWithBias.c_code_template(dtype) return (init_decl, """ if (PyArray_NDIM(%(y_idx)s) != 1) { PyErr_SetString(PyExc_ValueError, "y_idx not 1d tensor"); %(fail)s; } if (PyArray_DIMS(%(x)s)[0] != PyArray_DIMS(%(y_idx)s)[0]) { PyErr_Format(PyExc_ValueError, "number of rows in x (%%ld) does not match length of y (%%ld)", (long int)PyArray_DIMS(%(x)s)[0], (long int)PyArray_DIMS(%(y_idx)s)[0]); %(fail)s; } if ((NULL == %(nll)s) //initial condition || (PyArray_DIMS(%(nll)s)[0] != PyArray_DIMS(%(y_idx)s)[0])) { if (NULL != %(nll)s) Py_XDECREF(%(nll)s); %(nll)s = (PyArrayObject*)PyArray_SimpleNew(1, PyArray_DIMS(%(y_idx)s), PyArray_TYPE(%(x)s)); if(!%(nll)s) { PyErr_SetString(PyExc_MemoryError, "failed to alloc nll output"); %(fail)s; } } if ((NULL == %(am)s) || (PyArray_DIMS(%(am)s)[0] != PyArray_DIMS(%(y_idx)s)[0])) { Py_XDECREF(%(am)s); %(am)s = (PyArrayObject*) PyArray_SimpleNew(1, PyArray_DIMS(%(y_idx)s), PyArray_TYPE(%(y_idx)s)); if(!%(am)s) { PyErr_SetString(PyExc_MemoryError, "failed to alloc am output"); %(fail)s; } } """, begin_row_loop, """ const %(y_idx_type) s y_i = ((%(y_idx_type)s*)(PyArray_BYTES(%(y_idx)s) + PyArray_STRIDES(%(y_idx)s)[0] * i))[0]; dtype_%(nll) s* __restrict__ nll_i = (dtype_%(nll)s*)(PyArray_BYTES(%(nll)s) + PyArray_STRIDES(%(nll)s)[0] * i); %(am_type)s* __restrict__ am_i = (%(am_type)s*) (PyArray_BYTES(%(am)s) + PyArray_STRIDES(%(am)s)[0] * i); """, inside_row_loop, """ if ((y_i >= PyArray_DIMS(%(x)s)[1]) || (y_i < 0)) { PyErr_SetString(PyExc_ValueError, "y_i value out of bounds"); %(fail)s; } nll_i[0] = - x_i[y_i*Sx] - b_i[y_i*Sb] + row_max + log(sum); am_i[0] = row_max_j; """, end_row_loop) def c_code_cache_version(self): return (5,) + SoftmaxWithBias.c_code_cache_version() def c_code(self, node, name, inp, out, sub): x, b, y_idx = inp nll, sm, am = out y_idx_type = node.inputs[2].type.dtype_specs()[1] am_type = y_idx_type dtype = node.inputs[0].type.dtype_specs()[1] code_template = ''.join(self.c_code_template(dtype)) return code_template % dict(locals(), **sub) class CrossentropySoftmax1HotWithBiasDx(gof.Op): """ Gradient wrt x of the CrossentropySoftmaxArgmax1HotWithBias Op. """ nin = 3 nout = 1 __props__ = () def make_node(self, dy, sm, y_idx, **kwargs): dy = tensor.as_tensor_variable(dy) sm = tensor.as_tensor_variable(sm) y_idx = tensor.as_tensor_variable(y_idx) if (dy.type.ndim > 1 or dy.type.dtype not in tensor.float_dtypes): raise ValueError('dy must be {0,1}-d tensor of floats', dy.type) if (sm.type.ndim != 2 or sm.type.dtype not in tensor.float_dtypes): raise ValueError('sm must be 2-d tensor of floats', sm.type) if (y_idx.type.ndim != 1 or y_idx.type.dtype not in tensor.discrete_dtypes): raise ValueError('y_idx must be 1-d tensor of [u]ints', y_idx.type) return Apply(self, [dy, sm, y_idx], [sm.type()]) def perform(self, node, input_storage, output_storage): dy, sm, y_idx = input_storage if any(y_idx < 0): raise ValueError("y_i value out of bounds") dx = np.zeros_like(sm) if dy.ndim == 0: dy = dy[None] incr = int(dy.shape[0] > 1) for i in xrange(sm.shape[0]): dy_i = dy[i * incr] dx[i] = dy_i * sm[i] # vector scale dx[i, y_idx[i]] -= dy_i # scalar decrement output_storage[0][0] = dx def infer_shape(self, node, shapes): return [shapes[1]] def grad(self, inp, grads): dy, sm, y_idx = inp g_dx, = grads # TODO: currently we do not compute the gradient w.r.t. dy, because # advanced indexing is not working yet. When it works, do it to avoid # potentially misleading behavior in gradient computations! (although # typically we should not need the gradient w.r.t. dy). y_idx_range = tensor.arange(y_idx.shape[0]) g_dy = tensor.sum( g_dx * subtensor.AdvancedIncSubtensor()( sm, tensor.fill(dy, -1), y_idx_range, y_idx), axis=1) g_sm = dy.dimshuffle(0, 'x') * g_dx g_y_idx = grad_not_implemented(self, 2, y_idx) return [g_dy, g_sm, g_y_idx] def c_code_cache_version(self): return (6,) def c_code(self, node, name, inp, out, sub): dnll, sm, y_idx = inp dx, = out y_idx_type = node.inputs[2].type.dtype_specs()[1] return """ if ((PyArray_TYPE(%(dnll)s) != NPY_DOUBLE) && (PyArray_TYPE(%(dnll)s) != NPY_FLOAT)) { PyErr_SetString(PyExc_TypeError, "dnll type should be float32 or float64"); %(fail)s; } if ((PyArray_TYPE(%(sm)s) != NPY_DOUBLE) && (PyArray_TYPE(%(sm)s) != NPY_FLOAT)) { PyErr_SetString(PyExc_TypeError, "sm type should be float32 or float64"); %(fail)s; } // new scope because of variable declaration // TODO: proper indentation, but the diff will get messy { // Get `dnll.shape[0]` or set it to zero if `dnll` is a scalar. const npy_intp %(dnll)s_dims0 = (PyArray_NDIM(%(dnll)s) > 0 ? PyArray_DIMS(%(dnll)s)[0] : (npy_intp) 0); // Get `dnll.strides[0]` and set it to zero if `dnll` is a scalar // or a vector with just one element. const npy_intp %(dnll)s_strides0 = (%(dnll)s_dims0 > 1 ? PyArray_STRIDES(%(dnll)s)[0] : (npy_intp) 0); if ((PyArray_NDIM(%(dnll)s) > 1) || (PyArray_NDIM(%(sm)s) != 2) || (PyArray_NDIM(%(y_idx)s) != 1)) { PyErr_SetString(PyExc_ValueError, "rank error"); %(fail)s; } if (%(dnll)s_dims0 != PyArray_DIMS(%(sm)s)[0] && %(dnll)s_dims0 > 1) { PyErr_Format(PyExc_ValueError, "dnll.shape[0] (%%ld) != sm.shape[0] (%%ld)", (long int)%(dnll)s_dims0, (long int)PyArray_DIMS(%(sm)s)[0]); %(fail)s; } if (%(dnll)s_dims0 != PyArray_DIMS(%(y_idx)s)[0] && %(dnll)s_dims0 > 1) { PyErr_Format(PyExc_ValueError, "dnll.shape[0] (%%ld) != y_idx.shape[0] (%%ld)", (long int)%(dnll)s_dims0, (long int)PyArray_DIMS(%(y_idx)s)[0]); %(fail)s; } if (PyArray_DIMS(%(sm)s)[0] != PyArray_DIMS(%(y_idx)s)[0]) { PyErr_SetString(PyExc_ValueError, "sm.shape[0] != y_idx.shape[0]"); %(fail)s; } if ((NULL == %(dx)s) || (PyArray_DIMS(%(dx)s)[0] != PyArray_DIMS(%(sm)s)[0]) || (PyArray_DIMS(%(dx)s)[1] != PyArray_DIMS(%(sm)s)[1])) { if (NULL != %(dx)s) Py_XDECREF(%(dx)s); %(dx)s = (PyArrayObject*) PyArray_SimpleNew(2, PyArray_DIMS(%(sm)s), PyArray_TYPE(%(sm)s)); if(!%(dx)s) { PyErr_SetString(PyExc_MemoryError, "failed to alloc dx output"); %(fail)s } } for (size_t i = 0; i < PyArray_DIMS(%(dx)s)[0]; ++i) { const dtype_%(dnll)s dnll_i = ((dtype_%(dnll)s*)(PyArray_BYTES(%(dnll)s) + %(dnll)s_strides0 * i))[0]; const %(y_idx_type) s y_i = ((%(y_idx_type)s*)(PyArray_BYTES(%(y_idx)s) + PyArray_STRIDES(%(y_idx)s)[0] * i))[0]; const dtype_%(sm)s* __restrict__ sm_i = (dtype_%(sm)s*)(PyArray_BYTES(%(sm)s) + PyArray_STRIDES(%(sm)s)[0] * i); npy_intp Ssm = PyArray_STRIDES(%(sm)s)[1]/sizeof(dtype_%(sm)s); dtype_%(dx) s* __restrict__ dx_i = (dtype_%(dx)s*)(PyArray_BYTES(%(dx)s) + PyArray_STRIDES(%(dx)s)[0] * i); npy_intp Sdx = PyArray_STRIDES(%(dx)s)[1]/sizeof(dtype_%(dx)s); for (size_t j = 0; j < PyArray_DIMS(%(dx)s)[1]; ++j) { dx_i[j * Sdx] = dnll_i * sm_i[j * Ssm]; } if (y_i >= PyArray_DIMS(%(dx)s)[1] || (y_i < 0)) { PyErr_SetString(PyExc_ValueError, "y_i >= dx dimensions[1] or y_i < 0."); %(fail)s; } dx_i[y_i * Sdx] -= dnll_i; } } """ % dict(locals(), **sub) crossentropy_softmax_argmax_1hot_with_bias = \ CrossentropySoftmaxArgmax1HotWithBias() crossentropy_softmax_1hot_with_bias_dx = \ CrossentropySoftmax1HotWithBiasDx() def crossentropy_softmax_1hot_with_bias(x, b, y_idx, **kwargs): return crossentropy_softmax_argmax_1hot_with_bias(x, b, y_idx, **kwargs)[0:2] def crossentropy_softmax_1hot(x, y_idx, **kwargs): b = tensor.zeros_like(x[0, :]) return crossentropy_softmax_1hot_with_bias(x, b, y_idx, **kwargs) def crossentropy_softmax_max_and_argmax_1hot_with_bias(x, b, y_idx, **kwargs): """ Returns ------- object The cross-entropy, the softmax output, the max probability, and the argmax index. TODO: Since we are recomputing the argmax, we might as well assert that it is correct. TODO: Make this entire function is unnecessary? e.g. CrossentropySoftmaxArgmax1HotWithBias should return the appropriate information (i.e. the max probability)? """ (xent, softmax) = crossentropy_softmax_1hot_with_bias(x, b, y_idx, **kwargs) (max_pr, argmax) = tensor.max_and_argmax(softmax, axis=-1) return (xent, softmax, max_pr, argmax) def crossentropy_softmax_max_and_argmax_1hot(x, y_idx, **kwargs): b = tensor.zeros_like(x[0, :]) return crossentropy_softmax_max_and_argmax_1hot_with_bias(x, b, y_idx, **kwargs) class CrossentropyCategorical1HotGrad(gof.Op): __props__ = () def make_node(self, g_y, coding_dist, true_one_of_n): return Apply(self, [g_y, coding_dist, true_one_of_n], [coding_dist.type()]) def perform(self, node, inp, out): g_y, coding_dist, true_one_of_n = inp g_coding_strg, = out g_coding = np.zeros_like(coding_dist) for i in xrange(len(g_y)): g_coding[i, true_one_of_n[i]] = (-g_y[i] / coding_dist[i, true_one_of_n[i]]) g_coding_strg[0] = g_coding def infer_shape(self, node, in_shapes): return [in_shapes[1]] crossentropy_categorical_1hot_grad = CrossentropyCategorical1HotGrad() class CrossentropyCategorical1Hot(gof.Op): """ Compute the cross entropy between a coding distribution and a true distribution of the form [0, 0, ... 0, 1, 0, ..., 0]. .. math:: y[i] = - \log(coding_dist[i, one_of_n[i]) Notes ----- In the case that the coding distribution is the output of a softmax, an application of this Op will probably be optimized away in favour of one with a C implementation. """ __props__ = () def make_node(self, coding_dist, true_one_of_n): """ Parameters ---------- coding_dist : dense matrix true_one_of_n : lvector Returns ------- dvector """ _coding_dist = tensor.as_tensor_variable(coding_dist) _true_one_of_n = tensor.as_tensor_variable(true_one_of_n) if _coding_dist.type.ndim != 2: raise TypeError('matrix required for argument: coding_dist') if _true_one_of_n.type not in (tensor.lvector, tensor.ivector): raise TypeError( 'integer vector required for argument: true_one_of_n' '(got type: %s instead of: %s)' % (_true_one_of_n.type, tensor.lvector)) return Apply(self, [_coding_dist, _true_one_of_n], [tensor.Tensor(dtype=_coding_dist.dtype, broadcastable=[False])()]) def perform(self, node, inp, out): coding, one_of_n = inp y_out, = out y = np.zeros_like(coding[:, 0]) for i in xrange(len(y)): y[i] = -np.log(coding[i, one_of_n[i]]) y_out[0] = y def infer_shape(self, node, in_shapes): return [(in_shapes[0][0],)] def grad(self, inp, grads): coding, one_of_n = inp g_y, = grads return [crossentropy_categorical_1hot_grad(g_y, coding, one_of_n), grad_not_implemented(self, 1, one_of_n)] crossentropy_categorical_1hot = CrossentropyCategorical1Hot() @opt.register_stabilize('fast_compile_gpu') @opt.register_specialize('fast_compile_gpu') @gof.optimizer def crossentropy_to_crossentropy_with_softmax_with_bias(fgraph): """ This is a stabilization optimization. Notes ----- Not a local optimization because we are replacing outputs from several nodes at once. """ def search_make_one_sub(): for node in fgraph.toposort(): if node.op == crossentropy_categorical_1hot: nll, = node.outputs sm, one_of_n = node.inputs if sm.owner and sm.owner.op == softmax_with_bias: x, b = sm.owner.inputs new_nll, new_sm, new_am = crossentropy_softmax_argmax_1hot_with_bias( x, b, one_of_n) fgraph.replace_all_validate( [(nll, new_nll), (sm, new_sm)], reason="crossentropy_to_crossentropy_with_softmax_with_bias") return True return False while search_make_one_sub(): pass return @gof.optimizer def crossentropy_to_crossentropy_with_softmax(fgraph): """ This is a stabilization optimization that is more general than crossentropy_to_crossentropy_with_softmax_with_bias. It must be executed after local_softmax_with_bias optimization in specialize. TODO : This is a stabilization optimization! How to make this more cleanly? Notes ----- Not a local optimization because we are replacing outputs from several nodes at once. """ def search_make_one_sub(): for node in fgraph.toposort(): if node.op == crossentropy_categorical_1hot: nll, = node.outputs sm, one_of_n = node.inputs if sm.owner and sm.owner.op == softmax_op: x, = sm.owner.inputs new_nll, new_sm, new_am = crossentropy_softmax_argmax_1hot_with_bias( x, tensor.zeros_like(x[0]), one_of_n) fgraph.replace_all_validate( [(nll, new_nll), (sm, new_sm)], reason="crossentropy_to_crossentropy_with_softmax") return True if sm.owner and sm.owner.op == softmax_with_bias: x, b = sm.owner.inputs new_nll, new_sm, new_am = crossentropy_softmax_argmax_1hot_with_bias(x, b, one_of_n) fgraph.replace_all_validate( [(nll, new_nll), (sm, new_sm)], reason="crossentropy_to_crossentropy_with_softmax") return True return False while search_make_one_sub(): pass return optdb.register('crossentropy_to_crossentropy_with_softmax', crossentropy_to_crossentropy_with_softmax, 2.01, 'fast_run', 'xent', 'fast_compile_gpu') @opt.register_specialize( 'fast_compile_gpu', 'local_crossentropy_to_crossentropy_with_softmax_grad') # old name @gof.local_optimizer([softmax_grad]) def local_softmax_grad_to_crossentropy_with_softmax_grad(node): if node.op == softmax_grad: g_coding_dist, coding_dist = node.inputs if (g_coding_dist.owner and g_coding_dist.owner.op == crossentropy_categorical_1hot_grad): g_nll, coding_dist, true_one_of_n = g_coding_dist.owner.inputs dx = crossentropy_softmax_1hot_with_bias_dx(g_nll, coding_dist, true_one_of_n) copy_stack_trace(node.outputs[0], dx) return [dx] @opt.register_specialize('fast_compile_gpu') @gof.local_optimizer([tensor.MaxAndArgmax]) def local_argmax_pushdown(node): if isinstance(node.op, tensor.MaxAndArgmax) and node.inputs[0].owner and \ len(node.outputs[0].clients) > 0 and node.inputs[0].owner.op in \ (softmax_op, softplus, tensor.exp, tensor.log, tensor.tanh, sigmoid, softmax_with_bias): if theano.config.warn.argmax_pushdown_bug: logging.getLogger('theano.tensor.nnet.nnet').warn( "WARNING: there " "was a bug in Theano fixed on May 27th, 2010 in this case." " I.E. when we take the max of a softplus, softmax, exp, " "log, tanh, sigmoid, softmax_with_bias op, we were doing " "the max of the parent of the input. To remove this " "warning set the Theano flags 'warn.argmax_pushdown_bug' " "to False") if (isinstance(node.op, tensor.MaxAndArgmax) and node.inputs[0].owner and len(node.outputs[0].clients) == 0): x_max, x_argmax = node.outputs x = node.inputs[0] axis = node.op.get_params(node) # TODO: Make a list/set of monotonic ops... if x.owner and x.owner.op in (softmax_op, softplus, tensor.exp, tensor.log, tensor.tanh, sigmoid): pre_x, = x.owner.inputs ret = tensor.max_and_argmax(pre_x, axis) copy_stack_trace(x_max, ret) return ret if x.owner and x.owner.op == softmax_with_bias: pre_x, pre_bias = x.owner.inputs ret = tensor.max_and_argmax(pre_x + tensor.DimShuffle( pre_bias.broadcastable, ('x', 0))(pre_bias), axis) # copy both stack traces copy_stack_trace(x_max, ret) return ret # Utility function used by the two next optimizations def _check_rows_is_arange_len_labels(rows, labels): """ Check that 'rows' is the same node as T.arange(labels.shape[0]). Also considers the case where labels.shape[0] is constant and equal to 1, and T.arange(labels.shape[0]) has been constant-folded into 0. """ if labels.owner and hasattr(labels.owner.fgraph, 'shape_feature'): shape_of = labels.owner.fgraph.shape_feature.shape_of # TODO: consider cases where shape_of[labels] is constant, and # has a value different from 1. # This case is harder, as _is_const only accepts a scalar value # as second argument, so checking for # _is_const(rows, numpy.arange(...)) does not work for the moment. if len(shape_of[labels]) == 1 and _is_const(shape_of[labels][0], 1): return _is_const(rows, 0) if rows.owner and isinstance(rows.owner.op, tensor.ARange): start, stop, step = rows.owner.inputs if getattr(start, 'data', None) != 0: # constants will have data return False if getattr(step, 'data', None) != 1: # constant step will have data return False if not stop.owner: return False # Not sure if that case happens any more after the introduction of # ShapeOptimizer, but we keep it if ShapeOptimizer is not present if isinstance(stop.owner.op, subtensor.Subtensor): shape_subtensor = stop.owner if shape_subtensor.op.get_constant_idx(shape_subtensor.inputs, allow_partial=True) == [0]: shape_var = shape_subtensor.inputs[0] if shape_var.owner and shape_var.owner.op == tensor.shape: return shape_var.owner.inputs[0] is labels else: shape_of = stop.owner.fgraph.shape_feature.shape_of return shape_of[labels][0] is stop def _is_const(z, val, approx=False): try: maybe = opt.get_scalar_constant_value(z) except tensor.NotScalarConstantError: return False if approx: return np.allclose(maybe, val) else: return np.all(maybe == val) @opt.register_specialize('fast_compile_gpu') @gof.local_optimizer([subtensor.AdvancedSubtensor, tensor.log]) def local_advanced_indexing_crossentropy_onehot(node): log = None sm = None # First case: log(softmax(x))[rows, labels] if isinstance(node.op, subtensor.AdvancedSubtensor): try: log, rows, labels = node.inputs except Exception: pass if log and log.owner and log.owner.op == tensor.log: sm = log.owner.inputs[0] # Second case: log(softmax(x)[rows, labels]) elif node.op == tensor.log: pre_log = node.inputs[0].owner if pre_log and isinstance(pre_log.op, subtensor.AdvancedSubtensor): try: sm, rows, labels = pre_log.inputs except Exception: pass if sm is not None and sm.owner and sm.owner.op in (softmax_op, softmax_with_bias): sm_w_bias = local_softmax_with_bias.transform(sm.owner) if sm_w_bias: assert sm_w_bias[0].owner.op == softmax_with_bias x_var, b_var = sm_w_bias[0].owner.inputs else: x_var = sm.owner.inputs[0] b_var = tensor.zeros_like(x_var[0]) # Check that rows == arange(labels.shape[0]) if _check_rows_is_arange_len_labels(rows, labels): if labels.ndim == 1 and x_var.ndim == 2: minus_ret = crossentropy_softmax_argmax_1hot_with_bias(x_var, b_var, labels)[0] ret = -minus_ret copy_stack_trace(node.outputs[0], [minus_ret, ret]) return [ret] @opt.register_specialize('fast_compile_gpu') @gof.local_optimizer([softmax_grad]) def local_advanced_indexing_crossentropy_onehot_grad(node): if not (node.op == softmax_grad): return sm = None try: d_sm, sm = node.inputs except Exception: return if (sm is not None) and sm.owner and (sm.owner.op in (softmax_op, softmax_with_bias)): sm_w_bias = local_softmax_with_bias.transform(sm.owner) if sm_w_bias: assert sm_w_bias[0].owner.op == softmax_with_bias x_var, b_var = sm_w_bias[0].owner.inputs else: x_var = sm.owner.inputs[0] else: return # Two cases are supported: # 1. AdvancedIncSubtensor( # zeros_like(softmax(x)), # -out_grad / AdvancedSubtensor(softmax(x), arange(y.shape[0]), y), # arange(y.shape[0]), # y) # which arises from the gradient of log(softmax(x)[arange(y.shape[0]), y]) # # 2. AdvancedIncSubtensor( # zeros_like(log(softmax(x))), # -out_grad, # arange(y.shape[0]), # y) # / softmax(x) # which arises from the gradient of log(softmax(x))[arange(y.shape[0]), y] # # out_grad represents the gradient of the (final) cost wrt the output. # # N.B. Regarding clients -- This substitution is important for numerical stability, so we # perform the substitution even when intermediate values have multiple clients. # # First case. # After the check for AdvancedIncSubtensor, if anything does not fit with # the formula above, there's no way to fit it with the the second case, # so we return immediately. if d_sm.owner and isinstance(d_sm.owner.op, subtensor.AdvancedIncSubtensor): try: z, incr, rows, labels = d_sm.owner.inputs except Exception: return # Check that z == zeros_like(softmax(x)) # We know z has the right size because z has the same size as d_sm, # and d_sm and sm are both inputs of softmax_grad (so they have # the same size). if not _is_const(z, 0): return # In the base case (output gradient = 1), incr is -1./sm[arange(len(y)), y] # Here, we are looking for the AdvancedSubtensor term (sm[arange(len(y)), y]), # and constructing out_grad by incorporating the other terms. # out_grad will be constructed in 3 steps as follow: # out_grad = +/- 1. (according to sign) # out_grad *= -numerator # out_grad /= denominator # Then, if out_grad is a scalar, it will be allocated as a vector adv_subtensor = None out_grad = 1. # If there's a 'minus' sign before the whole expression, put it in # out_grad and iterate if incr.owner and incr.owner.op == tensor.neg: out_grad = - out_grad incr = incr.owner.inputs[0] if incr.owner and incr.owner.op == tensor.true_div: num, denom = incr.owner.inputs # set out_grad according to the numerator, it may be divided later # num should be a vector or a scalar if num.ndim == 1 or np.all(num.broadcastable): out_grad *= -num else: return if not denom.owner: return if isinstance(denom.owner.op, subtensor.AdvancedSubtensor): # Base case adv_subtensor = denom # out_grad /= 1. elif denom.owner.op == tensor.mul: # Try to find the AdvancedSubtensor node mentionned above, # and the output gradient for i, input in enumerate(denom.owner.inputs): if input.owner and isinstance(input.owner.op, subtensor.AdvancedSubtensor): other_inputs = [in_ for (j, in_) in enumerate(denom.owner.inputs) if j != i] if len(other_inputs) == 1: rest = other_inputs[0] else: rest = tensor.mul(*[other_inputs]) # Check that rest is a vector or a scalar if rest.ndim == 1 or np.all(rest.broadcastable): adv_subtensor = input out_grad /= rest break else: return # The output gradient needs to be a vector out_grad = tensor.fill(x_var[:, 0], out_grad) if adv_subtensor is not None: try: maybe_sm, maybe_rows, \ maybe_labels = adv_subtensor.owner.inputs except Exception: return if (not (maybe_sm is sm and maybe_rows is rows and maybe_labels is labels)): return # else: OK else: return else: return # Check that rows is arange(labels.shape[0]) if not _check_rows_is_arange_len_labels(rows, labels): return # else, arguments of AdvancedIncSubtensor are OK, # it was really case 1. # Second case elif d_sm.owner and d_sm.owner.op == tensor.true_div: # we're looking for # AdvIncSubtensor(zeros, grad_nll, arange(len(y)), y) / softmax try: num, denom = d_sm.owner.inputs except Exception: return if denom != sm: return # Check the numerator (AdvancedIncSubtensor) if num.owner and isinstance(num.owner.op, subtensor.AdvancedIncSubtensor): try: z, incr, rows, labels = num.owner.inputs except Exception: return # Check z is zeros_like(log(sm)) if not _is_const(z, 0): return if z.broadcastable not in [(False, False), (True, False)]: return # here we know that we are incrementing a matrix of zeros # (or a broadcasted vector). # Since d_sm and sm are the inputs of softmax_grad, # if the graph is valid, they have the same shape, so we # also know that z has the right shape. if incr.ndim != 1 or incr.dtype not in tensor.float_dtypes: return # here we know that we are incrementing some part of # matrix z by a vector # unless the user has taken care to mark that the data and # labels have the same number of rows, we cannot be sure # here that len(y) == len(z) However, in the common case # that these are predictions and labels it is true. We # leave it to the Op to crash (and the user to complain) # if this assumption is ever not true. out_grad = -incr # Check that rows is arange(labels.shape[0]) if not _check_rows_is_arange_len_labels(rows, labels): return # else, arguments of AdvancedIncSubtensor are OK else: return # numerator and denominator are OK, # it was really case 2. else: return # Dimension check before substitution if labels.ndim == 1 and x_var.ndim == 2: ret = crossentropy_softmax_1hot_with_bias_dx(out_grad, sm, labels) # The stack trace is not added to output_grad, sm and labels at # the moment but may need to be added at a future point copy_stack_trace(node.outputs[0], ret) return [ret] else: return @opt.register_specialize('fast_compile_gpu') @gof.local_optimizer([softmax_with_bias]) def graph_merge_softmax_with_crossentropy_softmax(node): if node.op == softmax_with_bias: x, b = node.inputs for x_client in x.clients: if x_client[0].op == crossentropy_softmax_argmax_1hot_with_bias: big_client = x_client[0] if big_client in [b_client[0] for b_client in b.clients]: xx, bb, ll = big_client.inputs mergeable_client = big_client.op(x, b, ll) copy_stack_trace(node.outputs[0], mergeable_client[1]) return [mergeable_client[1]] @opt.register_specialize @opt.register_stabilize @opt.register_canonicalize @gof.local_optimizer([CrossentropySoftmax1HotWithBiasDx]) def local_useless_crossentropy_softmax_1hot_with_bias_dx_alloc(node): """ Replace a CrossentropySoftmax1HotWithBiasDx op, whose incoming gradient is an `alloc` of a scalar variable or one that has either broadcastable or matching dimensions with the output variable, by one that skips the intermediate `alloc`. """ if isinstance(node.op, CrossentropySoftmax1HotWithBiasDx): dy, sm, y_idx = node.inputs # Those cases are directly handled by the internal broadcasting of the # `CrossentropySoftmax1HotWithBiasDx` op. if dy.ndim == 0: return False if dy.ndim == 1 and dy.broadcastable[0]: return False assert dy.ndim == 1 if dy.owner is not None and isinstance(dy.owner.op, tensor.Alloc): # dz is the input of the Alloc op, i.e. T.alloc(dz, ) dz = dy.owner.inputs[0] try: shape_feature = node.fgraph.shape_feature except AttributeError: # The shape feature may not be available in some mode, but we # need it for this optimization, so don't continue. return False shape_of = shape_feature.shape_of same_shape = shape_feature.same_shape # Build `dz_broad` explicitly to include extra implicit dimensions. dz_broad = (True,) * (dy.ndim - dz.ndim) + dz.broadcastable # If we can infer statically that the shape of `sm` and # `dy` are the same in dimension `k` or the shape of `dy` is equal # to 1 (which triggers the internal broadcasting in # `CrossentropySoftmax1HotWithBiasDx`) we do not need to # check it at runtime. if (dz_broad[0] and not same_shape(sm, dy, dim_x=0, dim_y=0) and shape_of[dy][0] != 1): # If `dz` is broadcastable, we need to check whether the shapes # of `dy` and `sm` are the same or whether the shape of `dy` is # equal to 1. cond = tensor.or_(tensor.eq(dy.shape[0], 1), tensor.eq(dy.shape[0], sm.shape[0])) msg = '`sm` and `dy` do not have the same shape.' dz = opt.Assert(msg)(dz, cond) ret = node.op(dz, sm, y_idx) copy_stack_trace(node.outputs[0], ret) return [ret] def binary_crossentropy(output, target): """ Compute the crossentropy of binary random variables. Output and target are each expectations of binary random variables; target may be exactly 0 or 1 but output must lie strictly between 0 and 1. Notes ----- We could use the x log y op to support output=0 and output=1. The gradient would still be undefined though. We do not sum, crossentropy is computed by component. TODO : Rewrite as a scalar, and then broadcast to tensor. """ return -(target * tensor.log(output) + (1.0 - target) * tensor.log(1.0 - output)) def sigmoid_binary_crossentropy(output, target): """ Compute the cross-entropy of binary random variables. `output` should be real-valued (range (-inf, +inf)); `sigmoid` will be applied to produce a (0, 1) valued input. `target` is assumed to be probabilities in [0, 1]. Notes ----- Mathematically equivalent to `binary_crossentropy(sigmoid(output), target)`, but with more efficient and numerically stable computation. """ def grad(inputs, out_grads): (output, target), (out_grad,) = inputs, out_grads g_output = out_grad * (sigmoid(output) - target) g_target = out_grad * (-output) return [g_output, g_target] inp = [output, target] outp = softplus(-abs(output)) + output * ((output > 0) - target) return theano.OpFromGraph(inp, [outp], grad_overrides=grad, inline=True, name='sigmoid_binary_crossentropy')(*inp) def categorical_crossentropy(coding_dist, true_dist): """ Return the cross-entropy between an approximating distribution and a true distribution. .. warning:: THIS FUNCTION IS UNNECESSARILY POLYMORPHIC. We ultimately don't want the polymorphism, and will move this function to pylearn.algorithms.cost. The 1hot version will be removed. The length of the documentation here is a form of code smell. The cross entropy between two probability distributions measures the average number of bits needed to identify an event from a set of possibilities, if a coding scheme is used based on a given probability distribution q, rather than the "true" distribution p. Mathematically it is defined as follows: .. math:: H(p,q) = - \sum_x p(x) \log(q(x)) Parameters ---------- coding_dist : a dense matrix Each slice along axis represents one distribution. true_dist : a dense matrix or sparse matrix or integer vector In the case of a matrix argument, each slice along axis represents one distribution. In the case of an integer vector argument, each element represents the position of the '1' in a 1-of-N encoding. Returns ------- tensor of rank one-less-than `coding_dist` The cross entropy between each coding and true distribution. Notes ----- axis : int The dimension over which each distribution runs (1 for row distributions, 0 for column distributions). """ if true_dist.ndim == coding_dist.ndim: return -tensor.sum(true_dist * tensor.log(coding_dist), axis=coding_dist.ndim - 1) elif true_dist.ndim == coding_dist.ndim - 1: return crossentropy_categorical_1hot(coding_dist, true_dist) else: raise TypeError('rank mismatch between coding and true distributions') class Prepend_scalar_constant_to_each_row(gof.Op): __props__ = () def __init__(self, val=0): if isinstance(val, float): val = scalar.constant(val) self.val = val def __str__(self): return '%s{%s}' % (self.__class__.__name__, self.val) def make_node(self, mat): # check type of input x = tensor.as_tensor_variable(mat) if not mat.type.broadcastable == (False, False): raise TypeError("Expected a matrix as input") y = tensor.as_tensor_variable(self.val) assert y.ndim == 0 if x.type.dtype != y.type.dtype: TypeError( "the value to prepend don't have the same type as the matrix") node = Apply(op=self, inputs=[mat], outputs=[mat.type()]) return node def perform(self, node, inp, out): mat, = inp output, = out new_shape = (mat.shape[0], mat.shape[1] + 1) if output[0] is None: output[0] = np.empty(new_shape, dtype=mat.dtype) out = output[0] else: if output[0].shape != new_shape: try: output[0].resize(new_shape) except Exception: output[0] = np.empty(new_shape, dtype=mat.dtype) out = output[0] out[:, 0].fill(self.val.data) out[:, 1:] = mat def infer_shape(self, node, in_shapes): shp = (in_shapes[0][0], in_shapes[0][1] + 1) return [shp] def grad(self, inp, grads): mat, = inp goutput, = grads return goutput[:, 1:] class Prepend_scalar_to_each_row(gof.Op): __props__ = () def make_node(self, val, mat): # check type of input x = tensor.as_tensor_variable(mat) if isinstance(val, float): val = scalar.constant(val) if not mat.type.broadcastable == (False, False): raise TypeError("Expected a matrix as input") y = tensor.as_tensor_variable(val) assert y.ndim == 0 if x.type.dtype != y.type.dtype: TypeError( "the value to prepend don't have the same type as the matrix") node = Apply(op=self, inputs=[val, mat], outputs=[mat.type()]) return node def perform(self, node, inp, out): val, mat = inp output, = out new_shape = (mat.shape[0], mat.shape[1] + 1) if output[0] is None: output[0] = np.empty(new_shape, dtype=mat.dtype) out = output[0] else: if output[0].shape != new_shape: try: output[0].resize(new_shape) except Exception: output[0] = np.empty(new_shape, dtype=mat.dtype) out = output[0] out[:, 0].fill(val) out[:, 1:] = mat def infer_shape(self, node, in_shapes): shp = (in_shapes[1][0], in_shapes[1][1] + 1) return [shp] def grad(self, inp, grads): val, mat = inp goutput, = grads return goutput[:, 0], goutput[:, 1:] prepend_scalar_to_each_row = Prepend_scalar_to_each_row() prepend_0_to_each_row = Prepend_scalar_constant_to_each_row(0.) prepend_1_to_each_row = Prepend_scalar_constant_to_each_row(1.) def relu(x, alpha=0): """ Compute the element-wise rectified linear activation function. .. versionadded:: 0.7.1 Parameters ---------- x : symbolic tensor Tensor to compute the activation function for. alpha : `scalar or tensor, optional` Slope for negative input, usually between 0 and 1. The default value of 0 will lead to the standard rectifier, 1 will lead to a linear activation function, and any value in between will give a leaky rectifier. A shared variable (broadcastable against `x`) will result in a parameterized rectifier with learnable slope(s). Returns ------- symbolic tensor Element-wise rectifier applied to `x`. Notes ----- This is numerically equivalent to ``T.switch(x > 0, x, alpha * x)`` (or ``T.maximum(x, alpha * x)`` for ``alpha < 1``), but uses a faster formulation or an optimized Op, so we encourage to use this function. """ # This is probably the fastest implementation for GPUs. Both the forward # pass and the gradient get compiled into a single GpuElemwise call. # TODO: Check if it's optimal for CPU as well; add an "if" clause if not. # TODO: Check if there's a faster way for the gradient; create an Op if so. if alpha == 0: return 0.5 * (x + abs(x)) else: # We can't use 0.5 and 1 for one and half. as if alpha is a # numpy dtype, they will be considered as float64, so would # cause upcast to float64. alpha = tensor.as_tensor_variable(alpha) f1 = 0.5 * (1 + alpha) f2 = 0.5 * (1 - alpha) return f1 * x + f2 * abs(x) def h_softmax(x, batch_size, n_outputs, n_classes, n_outputs_per_class, W1, b1, W2, b2, target=None): """ Two-level hierarchical softmax. This function implements a two-layer hierarchical softmax. It is commonly used as an alternative of the softmax when the number of outputs is important (it is common to use it for millions of outputs). See reference [1]_ for more information about the computational gains. The `n_outputs` outputs are organized in `n_classes` classes, each class containing the same number `n_outputs_per_class` of outputs. For an input `x` (last hidden activation), the first softmax layer predicts its class and the second softmax layer predicts its output among its class. If `target` is specified, it will only compute the outputs of the corresponding targets. Otherwise, if `target` is `None`, it will compute all the outputs. The outputs are grouped in classes in the same order as they are initially defined: if `n_outputs=10` and `n_classes=2`, then the first class is composed of the outputs labeled `{0,1,2,3,4}` while the second class is composed of `{5,6,7,8,9}`. If you need to change the classes, you have to re-label your outputs. .. versionadded:: 0.7.1 Parameters ---------- x: tensor of shape (batch_size, number of features) the minibatch input of the two-layer hierarchical softmax. batch_size: int the size of the minibatch input x. n_outputs: int the number of outputs. n_classes: int the number of classes of the two-layer hierarchical softmax. It corresponds to the number of outputs of the first softmax. See note at the end. n_outputs_per_class: int the number of outputs per class. See note at the end. W1: tensor of shape (number of features of the input x, n_classes) the weight matrix of the first softmax, which maps the input x to the probabilities of the classes. b1: tensor of shape (n_classes,) the bias vector of the first softmax layer. W2: tensor of shape (n_classes, number of features of the input x, n_outputs_per_class) the weight matrix of the second softmax, which maps the input x to the probabilities of the outputs. b2: tensor of shape (n_classes, n_outputs_per_class) the bias vector of the second softmax layer. target: tensor of shape either (batch_size,) or (batch_size, 1) (optional, default None) contains the indices of the targets for the minibatch input x. For each input, the function computes the output for its corresponding target. If target is None, then all the outputs are computed for each input. Returns ------- tensor of shape (`batch_size`, `n_outputs`) or (`batch_size`, 1) Output tensor of the two-layer hierarchical softmax for input `x`. Depending on argument `target`, it can have two different shapes. If `target` is not specified (`None`), then all the outputs are computed and the returned tensor has shape (`batch_size`, `n_outputs`). Otherwise, when `target` is specified, only the corresponding outputs are computed and the returned tensor has thus shape (`batch_size`, 1). Notes ----- The product of `n_outputs_per_class` and `n_classes` has to be greater or equal to `n_outputs`. If it is strictly greater, then the irrelevant outputs will be ignored. `n_outputs_per_class` and `n_classes` have to be the same as the corresponding dimensions of the tensors of `W1`, `b1`, `W2` and `b2`. The most computational efficient configuration is when `n_outputs_per_class` and `n_classes` are equal to the square root of `n_outputs`. Examples -------- The following example builds a simple hierarchical softmax layer. >>> import numpy as np >>> import theano >>> from theano import tensor >>> from theano.tensor.nnet import h_softmax >>> >>> # Parameters >>> batch_size = 32 >>> n_outputs = 100 >>> dim_x = 10 # dimension of the input >>> n_classes = int(np.ceil(np.sqrt(n_outputs))) >>> n_outputs_per_class = n_classes >>> output_size = n_outputs_per_class * n_outputs_per_class >>> >>> # First level of h_softmax >>> floatX = theano.config.floatX >>> W1 = theano.shared( ... np.random.normal(0, 0.001, (dim_x, n_classes)).astype(floatX)) >>> b1 = theano.shared(np.zeros((n_classes,), floatX)) >>> >>> # Second level of h_softmax >>> W2 = np.random.normal(0, 0.001, ... size=(n_classes, dim_x, n_outputs_per_class)).astype(floatX) >>> W2 = theano.shared(W2) >>> b2 = theano.shared(np.zeros((n_classes, n_outputs_per_class), floatX)) >>> >>> # We can now build the graph to compute a loss function, typically the >>> # negative log-likelihood: >>> >>> x = tensor.imatrix('x') >>> target = tensor.imatrix('target') >>> >>> # This only computes the output corresponding to the target. >>> # The complexity is O(n_classes + n_outputs_per_class). >>> y_hat_tg = h_softmax(x, batch_size, output_size, n_classes, ... n_outputs_per_class, W1, b1, W2, b2, target) >>> >>> negll = -tensor.mean(tensor.log(y_hat_tg)) >>> >>> # We may need to compute all the outputs (at test time usually): >>> >>> # This computes all the outputs. >>> # The complexity is O(n_classes * n_outputs_per_class). >>> output = h_softmax(x, batch_size, output_size, n_classes, ... n_outputs_per_class, W1, b1, W2, b2) References ---------- .. [1] J. Goodman, "Classes for Fast Maximum Entropy Training," ICASSP, 2001, `. """ # First softmax that computes the probabilities of belonging to each class class_probs = theano.tensor.nnet.softmax(tensor.dot(x, W1) + b1) if target is None: # Computes the probabilites of all the outputs # Second softmax that computes the output probabilities activations = tensor.tensordot(x, W2, (1, 1)) + b2 output_probs = theano.tensor.nnet.softmax( activations.reshape((-1, n_outputs_per_class))) output_probs = output_probs.reshape((batch_size, n_classes, -1)) output_probs = class_probs.dimshuffle(0, 1, 'x') * output_probs output_probs = output_probs.reshape((batch_size, -1)) # output_probs.shape[1] is n_classes * n_outputs_per_class, which might # be greater than n_outputs, so we ignore the potential irrelevant # outputs with the next line: output_probs = output_probs[:, :n_outputs] else: # Computes the probabilities of the outputs specified by the targets target = target.flatten() # Classes to which belong each target target_classes = target // n_outputs_per_class # Outputs to which belong each target inside a class target_outputs_in_class = target % n_outputs_per_class # Second softmax that computes the output probabilities activations = sparse_block_dot( W2.dimshuffle('x', 0, 1, 2), x.dimshuffle(0, 'x', 1), tensor.zeros((batch_size, 1), dtype='int32'), b2, target_classes.dimshuffle(0, 'x')) output_probs = theano.tensor.nnet.softmax(activations.dimshuffle(0, 2)) target_class_probs = class_probs[tensor.arange(batch_size), target_classes] output_probs = output_probs[tensor.arange(batch_size), target_outputs_in_class] output_probs = target_class_probs * output_probs return output_probs def elu(x, alpha=1): """ Compute the element-wise exponential linear activation function [2]_. .. versionadded:: 0.8.0 Parameters ---------- x : symbolic tensor Tensor to compute the activation function for. alpha : scalar Returns ------- symbolic tensor Element-wise exponential linear activation function applied to `x`. References ----- .. [2] Djork-Arne Clevert, Thomas Unterthiner, Sepp Hochreiter "Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)" `. """ return tensor.switch(x > 0, x, alpha * tensor.expm1(x)) def selu(x): """Compute the element-wise Scaled Exponential Linear unit [3]_. .. versionadded:: 0.9.0 Parameters ---------- x : symbolic tensor Tensor to compute the activation function for. Returns ------- symbolic tensor Element-wise scaled exponential linear activation function applied to `x`. References ---------- .. [3] Klambauer G, Unterthiner T, Mayr A, Hochreiter S. "Self-Normalizing Neural Networks" """ alpha = 1.6732632423543772848170429916717 scale = 1.0507009873554804934193349852946 return scale * elu(x, alpha) class ScalarSoftsign(theano.scalar.UnaryScalarOp): """ Softsign activation function :math:`\\varphi(\\mathbf{x}) = \\frac{1}{1+|x|}` """ @staticmethod def static_impl(x): return x / (1.0 + abs(x)) def impl(self, x): return ScalarSoftsign.static_impl(x) def grad(self, inp, grads): x, = inp gz, = grads if 'float' in x.type.dtype: d = (1.0 + abs(x)) return [gz / (d * d)] else: return NotImplemented def c_code(self, node, name, inp, out, sub): x, = inp z, = out if node.inputs[0].type in [theano.scalar.float32, theano.scalar.float64]: return "%(z)s = %(x)s / (1.0+fabs(%(x)s));" % locals() raise NotImplementedError('only floating point x is implemented') scalar_softsign = ScalarSoftsign(theano.scalar.upgrade_to_float, name='scalar_softsign') softsign = elemwise.Elemwise(scalar_softsign, name='softsign') def confusion_matrix(actual, pred): """ Computes the confusion matrix of given vectors containing actual observations and predicted observations. Parameters ---------- actual : 1-d tensor variable pred : 1-d tensor variable Returns ------- conf_mat : Confusion matrix of actual and predictions observations as shown below. | Predicted ___________|___________ Actual | | order : 1-d array of order of entries in rows and columns Examples -------- >>> import theano >>> from theano.tensor.nnet import confusion_matrix >>> x = theano.tensor.vector() >>> y = theano.tensor.vector() >>> f = theano.function([x, y], confusion_matrix(x, y)) >>> y_true = [2, 0, 2, 2, 0, 1] >>> y_pred = [0, 0, 2, 2, 0, 2] >>> print(f(y_true, y_pred)) [array([[2, 0, 0], [0, 0, 1], [1, 0, 2]]), array([ 0., 1., 2.])] """ if actual.ndim != 1: raise ValueError('actual must be 1-d tensor variable') if pred.ndim != 1: raise ValueError('pred must be 1-d tensor variable') order = extra_ops.Unique(False, False, False)(tensor.concatenate([actual, pred])) colA = actual.dimshuffle(0, 'x') colP = pred.dimshuffle(0, 'x') oneHotA = tensor.eq(colA, order).astype('int64') oneHotP = tensor.eq(colP, order).astype('int64') conf_mat = tensor.dot(oneHotA.T, oneHotP) return [conf_mat, order] Theano-1.0.0/theano/tensor/nnet/sigm.py00006000163562€ÿÿÿþ0000010222213203335532021333 0ustar boccoset00000000000000""" Ops and optimizations: sigmoid, softplus. These functions implement special cases of exp and log to improve numerical stability. """ from __future__ import absolute_import, print_function, division import warnings import numpy as np import theano from theano import config, gof, printing, scalar from theano.compat import imap from theano.printing import pprint from theano.tensor import basic as tensor from theano.tensor import elemwise, opt, NotScalarConstantError from theano.tensor.type import values_eq_approx_remove_inf from theano.gof.opt import copy_stack_trace ############ # # SCALAR OPS # class ScalarSigmoid(scalar.UnaryScalarOp): """ This is just speed opt. Not for stability. """ @staticmethod def st_impl(x): if x < -30.0: return 0.0 if x > 30.0: return 1.0 # If x is an int8 or uint8, numpy.exp will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return 1.0 / (1.0 + np.exp(-x, sig='f')) return 1.0 / (1.0 + np.exp(-x)) def impl(self, x): return ScalarSigmoid.st_impl(x) def grad(self, inp, grads): x, = inp gz, = grads y = scalar_sigmoid(x) rval = gz * y * (1.0 - y) assert rval.type.dtype.find('float') != -1 return [rval] def c_code(self, node, name, inp, out, sub): x, = inp z, = out # We add boundary checks prevent exp from generating inf or # 0. The reset of the logic always generate 0 or 1 in those # cases. This is a speed optimization. # The constants were obtained by looking at the output of # python commands like: # # import numpy, theano # dt='float32' # or float64 # for i in xrange(750): # print i, repr(theano._asarray(1.0, dtype=dt) / # (theano._asarray(1.0, dtype=dt) + # numpy.exp(-theano._asarray([i,-i], dtype=dt)))) # float16 limits: -11.0, 7.0f # We use the float32 limits for float16 for now as the # computation will happen in float32 anyway. if (node.inputs[0].type == scalar.float32 or node.inputs[0].type == scalar.float16): return """%(z)s = %(x)s < -88.0f ? 0.0 : %(x)s > 15.0f ? 1.0f : 1.0f /(1.0f + exp(-%(x)s));""" % locals() elif node.inputs[0].type == scalar.float64: return """%(z)s = %(x)s < -709.0 ? 0.0 : %(x)s > 19.0 ? 1.0 : 1.0 /(1.0+exp(-%(x)s));""" % locals() else: raise NotImplementedError('only floatingpoint is implemented') def c_code_cache_version(self): v = super(ScalarSigmoid, self).c_code_cache_version() if v: return (2,) + v else: return v # This fct is disabled as it is slower then the normal code! def c_code_contiguous_disabled(self, node, name, inp, out, sub): x, = inp z, = out if (not theano.config.lib.amdlibm or node.inputs[0].dtype != node.outputs[0].dtype): raise theano.gof.utils.MethodNotDefined() dtype = node.inputs[0].dtype if dtype == 'float32' and self.amd_float32 is not None: dtype = 'float' fct = "amd_vrsa_expf" elif dtype == 'float64' and self.amd_float64 is not None: dtype = 'double' fct = "amd_vrda_exp" else: raise theano.gof.utils.MethodNotDefined() return """ npy_intp n = PyArray_SIZE(%(z)s); %(dtype)s * x = (%(dtype)s*) PyArray_DATA(%(x)s); %(dtype)s * z = (%(dtype)s*) PyArray_DATA(%(z)s); // We block to keep the data in l1 // normal l1 size = 32k: 32k/2(input + output)/8(nb bytes of double)=2k // We stay bellow the 2k limit to let space for // This is faster than the not blocking version for(int i=0;i= 0: if x < 1.7: z = (1.5 * x / (1 + x)) elif x < 3: z = (0.935409070603099 + 0.0458812946797165 * (x - 1.7)) else: z = 0.99505475368673 else: xx = -x if xx < 1.7: z = (1.5 * xx / (1 + xx)) elif xx < 3: z = (0.935409070603099 + 0.0458812946797165 * (xx - 1.7)) else: z = 0.99505475368673 z = -z return 0.5 * (z + 1.) def impl(self, x): return UltraFastScalarSigmoid.st_impl(x) def c_code(self, node, name, inp, out, sub): x, = inp z, = out dtype = node.outputs[0].type.dtype_specs()[1] return """ %(dtype)s x = 0.5 * %(x)s; // The if is a tanh approximate. if(x>=0) { %(z)s = (x<1.7 ? (1.5*x/(1+x)) : (x<3 ? (0.935409070603099 + 0.0458812946797165*(x-1.7)): 0.99505475368673)); } else { %(dtype)s xx = -x; %(z)s = -(xx<1.7 ? (1.5*xx/(1+xx)) : (xx<3 ? (0.935409070603099 + 0.0458812946797165*(xx-1.7)): 0.99505475368673)); } //%(z)s = 0.5*(ultrafasttanh(0.5*x)+1.); %(z)s = 0.5*(%(z)s+1.); """ % locals() ultra_fast_scalar_sigmoid = UltraFastScalarSigmoid( scalar.upgrade_to_float, name='ultra_fast_scalar_sigmoid') ultra_fast_sigmoid = elemwise.Elemwise(ultra_fast_scalar_sigmoid, name='ultra_fast_sigmoid') ultra_fast_sigmoid_inplace = elemwise.Elemwise( UltraFastScalarSigmoid(scalar.transfer_type(0)), inplace_pattern={0: 0}, name='ultra_fast_sigmoid_inplace', ) pprint.assign(ultra_fast_sigmoid, printing.FunctionPrinter('ultra_fast_sigmoid')) # @opt.register_uncanonicalize @gof.local_optimizer([sigmoid]) def local_ultra_fast_sigmoid(node): """ When enabled, change all sigmoid to ultra_fast_sigmoid. For example do mode.including('local_ultra_fast_sigmoid') or use the Theano flag optimizer_including=local_ultra_fast_sigmoid. This speeds up the sigmoid op by using an approximation. This is done after the stabilization and specialize phases to avoid interacting with them. """ if (isinstance(node.op, tensor.Elemwise) and node.op.scalar_op == scalar_sigmoid): out = ultra_fast_sigmoid(node.inputs[0]) copy_stack_trace(node.outputs[0], out) def values_eq_approx_remove_low_prec(a, b): # atol is found by trial/error. # Other test could fail without good reason. return tensor.TensorType.values_eq_approx(a, b, atol=0.02) # Let DebugMode know that there this opt approx the values. out.tag.values_eq_approx = values_eq_approx_remove_low_prec return [out] theano.compile.optdb['uncanonicalize'].register("local_ultra_fast_sigmoid", local_ultra_fast_sigmoid) def hard_sigmoid(x): """ An approximation of sigmoid. More approximate and faster than ultra_fast_sigmoid. Approx in 3 parts: 0, scaled linear, 1. Removing the slope and shift does not make it faster. """ # Use the same dtype as determined by "upgrade_to_float", # and perform computation in that dtype. out_dtype = scalar.upgrade_to_float(scalar.Scalar(dtype=x.dtype))[0].dtype slope = tensor.constant(0.2, dtype=out_dtype) shift = tensor.constant(0.5, dtype=out_dtype) x = (x * slope) + shift x = tensor.clip(x, 0, 1) return x # @opt.register_uncanonicalize @gof.local_optimizer([sigmoid]) def local_hard_sigmoid(node): if (isinstance(node.op, tensor.Elemwise) and node.op.scalar_op == scalar_sigmoid): out = hard_sigmoid(node.inputs[0]) copy_stack_trace(node.outputs[0], out) def values_eq_approx_remove_low_prec(a, b): # atol is found by trial/error. # Other test could fail without good reason. return tensor.TensorType.values_eq_approx(a, b, atol=0.1) # Let DebugMode know that there this opt approx the values. out.tag.values_eq_approx = values_eq_approx_remove_low_prec return [out] theano.compile.optdb['uncanonicalize'].register("local_hard_sigmoid", local_hard_sigmoid) class ScalarSoftplus(scalar.UnaryScalarOp): """ This helps numerical stability. """ @staticmethod def static_impl(x): if x < -30.0: return 0.0 if x > 30.0: return x # If x is an int8 or uint8, numpy.exp will compute the result in # half-precision (float16), where we want float32. x_dtype = str(getattr(x, 'dtype', '')) if x_dtype in ('int8', 'uint8'): return np.log1p(np.exp(x, sig='f')) return np.log1p(np.exp(x)) def impl(self, x): return ScalarSoftplus.static_impl(x) def grad(self, inp, grads): x, = inp gz, = grads return [gz * scalar_sigmoid(x)] def c_code(self, node, name, inp, out, sub): x, = inp z, = out # These constants were obtained by looking at the output of # python commands like: # for i in xrange(750): # print i, repr(numpy.log1p(numpy.exp(theano._asarray([i,-i], dtype=dt)))) # the boundary checks prevent us from generating inf # float16 limits: -17.0, 6.0 # We use the float32 limits for float16 for now as the # computation will happen in float32 anyway. if (node.inputs[0].type == scalar.float32 or node.inputs[0].type == scalar.float16): return """%(z)s = %(x)s < -103.0f ? 0.0 : %(x)s > 14.0f ? %(x)s : log1p(exp(%(x)s));""" % locals() elif node.inputs[0].type == scalar.float64: return """%(z)s = %(x)s < -745.0 ? 0.0 : %(x)s > 16.0 ? %(x)s : log1p(exp(%(x)s));""" % locals() else: raise NotImplementedError('only floatingpoint is implemented') def c_code_cache_version(self): v = super(ScalarSoftplus, self).c_code_cache_version() if v: return (2,) + v else: return v scalar_softplus = ScalarSoftplus(scalar.upgrade_to_float, name='scalar_softplus') softplus = elemwise.Elemwise(scalar_softplus, name='softplus') pprint.assign(softplus, printing.FunctionPrinter('softplus')) def _skip_mul_1(r): if r.owner and r.owner.op == tensor.mul: not_is_1 = [i for i in r.owner.inputs if not _is_1(i)] if len(not_is_1) == 1: return not_is_1[0] logsigm_to_softplus = gof.PatternSub( (tensor.log, (sigmoid, 'x')), (tensor.neg, (softplus, (tensor.neg, 'x'))), allow_multiple_clients=True, values_eq_approx=values_eq_approx_remove_inf, skip_identities_fn=_skip_mul_1) def _is_1(expr): """ Returns ------- bool True iff expr is a constant close to 1. """ try: v = opt.get_scalar_constant_value(expr) return np.allclose(v, 1) except tensor.NotScalarConstantError: return False log1msigm_to_softplus = gof.PatternSub( (tensor.log, (tensor.sub, dict(pattern='y', constraint=_is_1), (sigmoid, 'x'))), (tensor.neg, (softplus, 'x')), allow_multiple_clients=True, values_eq_approx=values_eq_approx_remove_inf, skip_identities_fn=_skip_mul_1) log1pexp_to_softplus = gof.PatternSub( (tensor.log1p, (tensor.exp, 'x')), (softplus, 'x'), values_eq_approx=values_eq_approx_remove_inf, allow_multiple_clients=True) log1p_neg_sigmoid = gof.PatternSub( (tensor.log1p, (tensor.neg, (sigmoid, 'x'))), (tensor.neg, (softplus, 'x')), values_eq_approx=values_eq_approx_remove_inf, allow_multiple_clients=True) opt.register_stabilize(logsigm_to_softplus, name='logsigm_to_softplus') opt.register_stabilize(log1msigm_to_softplus, name='log1msigm_to_softplus') opt.register_stabilize(log1pexp_to_softplus, name='log1pexp_to_softplus') opt.register_stabilize(log1p_neg_sigmoid, name='log1p_neg_sigmoid,') def is_1pexp(t, only_process_constants=True): """ Returns ------- object If 't' is of the form (1+exp(x)), return (False, x). Else return None. """ if t.owner and t.owner.op == tensor.add: scalars, scalar_inputs, nonconsts = \ opt.scalarconsts_rest(t.owner.inputs, only_process_constants=only_process_constants) # scalar_inputs are potentially dimshuffled and filled with scalars if len(nonconsts) == 1: maybe_exp = nonconsts[0] if maybe_exp.owner and maybe_exp.owner.op == tensor.exp: # Verify that the constant terms sum to 1. if scalars: scal_sum = scalars[0] for s in scalars[1:]: scal_sum = scal_sum + s if np.allclose(scal_sum, 1): return False, maybe_exp.owner.inputs[0] # Before 7987b51 there used to be a bug where *any* constant # was considered as if it was equal to 1, and thus this # function would incorrectly identify it as (1 + exp(x)). if config.warn.identify_1pexp_bug: warnings.warn( 'Although your current code is fine, please note that ' 'Theano versions prior to 0.5 (more specifically, ' 'prior to commit 7987b51 on 2011-12-18) may have ' 'yielded an incorrect result. To remove this warning, ' 'either set the `warn.identify_1pexp_bug` config ' 'option to False, or `warn.ignore_bug_before` to at ' 'least \'0.4.1\'.') return None def is_exp(var): """ Match a variable with either of the `exp(x)` or `-exp(x)` patterns. Parameters ---------- var The Variable to analyze. Returns ------- tuple A pair (b, x) with `b` a boolean set to True if `var` is of the form `-exp(x)` and False if `var` is of the form `exp(x)`. If `var` cannot be cast into either form, then return `None`. """ neg = False neg_info = is_neg(var) if neg_info is not None: neg = True var = neg_info if var.owner and var.owner.op == tensor.exp: return neg, var.owner.inputs[0] def is_mul(var): """ Match a variable with `x * y * z * ...`. Parameters ---------- var The Variable to analyze. Returns ------- object A list [x, y, z, ...] if `var` is of the form `x * y * z * ...`, or None if `var` cannot be cast into this form. """ if var.owner and var.owner.op == tensor.mul: return var.owner.inputs else: return None def partition_num_or_denom(r, f): if r.owner and r.owner.op == tensor.mul: a = r.owner.inputs else: a = [r] # ugly 2.4-compatible thing f_terms = [] neg = False rest = [] for t in a: f_t = f(t) if f_t is None: rest.append(t) else: neg_t, f_t = f_t f_terms.append(f_t) neg ^= neg_t # bit flip if neg_t is true return f_terms, rest, neg def is_neg(var): """ Match a variable with the `-x` pattern. Parameters ---------- var The Variable to analyze. Returns ------- object `x` if `var` is of the form `-x`, or None otherwise. """ apply = var.owner if not apply: return None # First match against `tensor.neg`. if apply.op == tensor.neg: return apply.inputs[0] # Then match against a multiplication by -1. if apply.op == tensor.mul and len(apply.inputs) >= 2: for idx, mul_input in enumerate(apply.inputs): try: constant = opt.get_scalar_constant_value(mul_input) is_minus_1 = np.allclose(constant, -1) except NotScalarConstantError: is_minus_1 = False if is_minus_1: # Found a multiplication by -1. if len(apply.inputs) == 2: # Only return the other input. return apply.inputs[1 - idx] else: # Return the multiplication of all other inputs. return tensor.mul(*(apply.inputs[0:idx] + apply.inputs[idx + 1:])) # No match. return None @opt.register_stabilize @gof.local_optimizer([tensor.true_div]) def local_exp_over_1_plus_exp(node): """ exp(x)/(1+exp(x)) -> sigm(x) c/(1+exp(x)) -> c*sigm(-x) """ # this optimization should be done for numerical stability # so we don't care to check client counts if node.op == tensor.true_div: # find all the exp() terms in the numerator num, denom = node.inputs num_exp_x, num_rest, num_neg = partition_num_or_denom(num, is_exp) denom_1pexp, denom_rest, \ denom_neg = partition_num_or_denom(denom, is_1pexp) sigmoids = [] for t in denom_1pexp: if t in num_exp_x: # case: exp(x) /(1+exp(x)) sigmoids.append(sigmoid(t)) del num_exp_x[num_exp_x.index(t)] else: # case: 1/(1+exp(x)) sigmoids.append(sigmoid(-t)) copy_stack_trace(node.outputs[0], sigmoids[-1]) if not sigmoids: # we didn't find any. abort return # put the new numerator together new_num = sigmoids + [tensor.exp(t) for t in num_exp_x] + num_rest if len(new_num) == 1: new_num = new_num[0] else: new_num = tensor.mul(*new_num) if num_neg ^ denom_neg: new_num = -new_num copy_stack_trace(num, new_num) if len(denom_rest) == 0: return [new_num] elif len(denom_rest) == 1: out = new_num / denom_rest[0] else: out = new_num / tensor.mul(*denom_rest) copy_stack_trace(node.outputs[0], out) return [out] def parse_mul_tree(root): """ Parse a tree of multiplications starting at the given root. Parameters ---------- root The variable at the root of the tree. Returns ------- object A tree where each non-leaf node corresponds to a multiplication in the computation of `root`, represented by the list of its inputs. Each input is a pair [n, x] with `n` a boolean value indicating whether sub-tree `x` should be negated. Examples -------- x * y -> [False, [[False, x], [False, y]]] -(x * y) -> [True, [[False, x], [False, y]]] -x * y -> [False, [[True, x], [False, y]]] -x -> [True, x] (x * y) * -z -> [False, [[False, [[False, x], [False, y]]], [True, z]]] """ # Is it a multiplication? mul_info = is_mul(root) if mul_info is None: # Is it a negation? neg_info = is_neg(root) if neg_info is None: # Keep the root "as is". return [False, root] else: # Recurse, inverting the negation. neg, sub_tree = parse_mul_tree(neg_info) return [not neg, sub_tree] else: # Recurse into inputs. return [False, list(map(parse_mul_tree, mul_info))] def replace_leaf(arg, leaves, new_leaves, op, neg): """ Attempt to replace a leaf of a multiplication tree. We search for a leaf in `leaves` whose argument is `arg`, and if we find one, we remove it from `leaves` and add to `new_leaves` a leaf with argument `arg` and variable `op(arg)`. Parameters ---------- arg The argument of the leaf we are looking for. leaves List of leaves to look into. Each leaf should be a pair (x, l) with `x` the argument of the Op found in the leaf, and `l` the actual leaf as found in a multiplication tree output by `parse_mul_tree` (i.e. a pair [boolean, variable]). new_leaves If a replacement occurred, then the leaf is removed from `leaves` and added to the list `new_leaves` (after being modified by `op`). op A function that, when applied to `arg`, returns the Variable we want to replace the original leaf variable with. neg : bool If True, then the boolean value associated to the leaf should be swapped. If False, then this value should remain unchanged. Returns ------- bool True if a replacement occurred, or False otherwise. """ for idx, x in enumerate(leaves): if x[0] == arg: x[1][0] ^= neg x[1][1] = op(arg) leaves.pop(idx) new_leaves.append(x) return True return False def simplify_mul(tree): """ Simplify a multiplication tree. Parameters ---------- tree A multiplication tree (as output by `parse_mul_tree`). Returns ------- object A multiplication tree computing the same output as `tree` but without useless multiplications by 1 nor -1 (identified by leaves of the form [False, None] or [True, None] respectively). Useless multiplications (with less than two inputs) are also removed from the tree. """ neg, inputs = tree if isinstance(inputs, list): # Recurse through inputs. s_inputs = [] for s_i in imap(simplify_mul, inputs): if s_i[1] is None: # Multiplication by +/-1. neg ^= s_i[0] else: s_inputs.append(s_i) if not s_inputs: # The multiplication is empty. rval = [neg, None] elif len(s_inputs) == 1: # The multiplication has a single input. s_inputs[0][0] ^= neg rval = s_inputs[0] else: rval = [neg, s_inputs] else: rval = tree # print 'simplify_mul: %s -> %s' % (tree, rval) return rval def compute_mul(tree): """ Compute the Variable that is the output of a multiplication tree. This is the inverse of the operation performed by `parse_mul_tree`, i.e. compute_mul(parse_mul_tree(tree)) == tree. Parameters ---------- tree A multiplication tree (as output by `parse_mul_tree`). Returns ------- object A Variable that computes the multiplication represented by the tree. """ neg, inputs = tree if inputs is None: raise AssertionError( 'Function `compute_mul` found a missing leaf, did you forget to ' 'call `simplify_mul` on the tree first?') elif isinstance(inputs, list): # Recurse through inputs. rval = tensor.mul(*list(map(compute_mul, inputs))) else: rval = inputs if neg: rval = -rval return rval def perform_sigm_times_exp(tree, exp_x=None, exp_minus_x=None, sigm_x=None, sigm_minus_x=None, parent=None, child_idx=None, full_tree=None): """ Core processing of the `local_sigm_times_exp` optimization. This recursive function operates on a multiplication tree as output by `parse_mul_tree`. It walks through the tree and modifies it in-place by replacing matching pairs (exp, sigmoid) with the desired optimized version. Parameters ---------- tree The sub-tree to operate on. exp_x List of arguments x so that `exp(x)` exists somewhere in the whole multiplication tree. Each argument is a pair (x, leaf) with `x` the argument of the exponential, and `leaf` the corresponding leaf in the multiplication tree (of the form [n, exp(x)] -- see `parse_mul_tree`). If None, this argument is initialized to an empty list. exp_minus_x Similar to `exp_x`, but for `exp(-x)`. sigm_x Similar to `exp_x`, but for `sigmoid(x)`. sigm_minus_x Similar to `exp_x`, but for `sigmoid(-x)`. parent Parent of `tree` (None if `tree` is the global root). child_idx Index of `tree` in its parent's inputs (None if `tree` is the global root). full_tree The global multiplication tree (should not be set except by recursive calls to this function). Used for debugging only. Returns ------- bool True if a modification was performed somewhere in the whole multiplication tree, or False otherwise. """ if exp_x is None: exp_x = [] if exp_minus_x is None: exp_minus_x = [] if sigm_x is None: sigm_x = [] if sigm_minus_x is None: sigm_minus_x = [] if full_tree is None: full_tree = tree if False: # Debug code. print('') print(' full_tree = %s' % full_tree) print(' tree = %s' % tree) print(' exp_x = %s' % exp_x) print(' exp_minus_x = %s' % exp_minus_x) print(' sigm_x = %s' % sigm_x) print(' sigm_minus_x= %s' % sigm_minus_x) neg, inputs = tree if isinstance(inputs, list): # Recurse through inputs of the multiplication. rval = False for sub_idx, sub_tree in enumerate(inputs): rval |= perform_sigm_times_exp( tree=sub_tree, parent=tree, child_idx=sub_idx, exp_x=exp_x, exp_minus_x=exp_minus_x, sigm_x=sigm_x, sigm_minus_x=sigm_minus_x, full_tree=full_tree) return rval else: # Reached a leaf: if it is an exponential or a sigmoid, then we # first attempt to find a match in leaves already visited. # If there is such a match, we modify the already-visited leaf # accordingly: for instance if we visited a leaf sigmoid(x), then # find later a -exp(-x), we replace the previous leaf by # -sigmoid(-x) and remove the -exp(-x) from the tree. # If no match is found, then we register this leaf so that it can # be found later while walking the tree. var = inputs keep_it = False exp_info = is_exp(var) if exp_info is not None: exp_neg, exp_arg = exp_info neg ^= exp_neg neg_arg = is_neg(exp_arg) if neg_arg is None: if not replace_leaf(exp_arg, sigm_minus_x, sigm_x, sigmoid, neg): exp_x.append((exp_arg, tree)) keep_it = True else: if not replace_leaf(neg_arg, sigm_x, sigm_minus_x, lambda x: sigmoid(-x), neg): exp_minus_x.append((neg_arg, tree)) keep_it = True elif var.owner and var.owner.op == sigmoid: sigm_arg = var.owner.inputs[0] neg_arg = is_neg(sigm_arg) if neg_arg is None: if not replace_leaf(sigm_arg, exp_minus_x, sigm_minus_x, lambda x: sigmoid(-x), neg): sigm_x.append((sigm_arg, tree)) keep_it = True else: if not replace_leaf(neg_arg, exp_x, sigm_x, sigmoid, neg): sigm_minus_x.append((neg_arg, tree)) keep_it = True else: # It is not an exponential nor a sigmoid. keep_it = True if not keep_it: # Delete this leaf, i.e. replace it by [False, None] (corresponding # to a multiplication by 1). assert parent is not None parent[1][child_idx] = [False, None] return not keep_it @opt.register_stabilize @gof.local_optimizer([tensor.mul]) def local_sigm_times_exp(node): """ exp(x) * sigm(-x) -> sigm(x) exp(-x) * sigm(x) -> sigm(-x) todo: add stack traces to the intermediate variables """ # Bail early if it is not a multiplication. if node.op != tensor.mul: return None # Obtain tree of multiplications starting at this node. mul_tree = parse_mul_tree(node.outputs[0]) # Perform core optimization. did_something = perform_sigm_times_exp(mul_tree) if not did_something: # No change. return None # The optimization may have introduced multiplications by 1 in the tree: # get rid of them. mul_tree = simplify_mul(mul_tree) # Recompute final output based on the updated tree. out = compute_mul(mul_tree) # keep the stack trace copy_stack_trace(node.outputs[0], out) return [out] @opt.register_stabilize @gof.local_optimizer([tensor.inv]) def local_inv_1_plus_exp(node): """ 1/(1+exp(x)) -> sigm(-x) """ # this optimization should be done for numerical stability # so we don't care to check client counts if node.op == tensor.inv: inv_arg = node.inputs[0] if inv_arg.owner and inv_arg.owner.op == tensor.add: scalars, scalar_inputs, nonconsts = \ opt.scalarconsts_rest(inv_arg.owner.inputs, only_process_constants=True) # scalar_inputs are potentially dimshuffled and fill'd scalars if len(nonconsts) == 1: if nonconsts[0].owner and nonconsts[0].owner.op == tensor.exp: if scalars and np.allclose(np.sum(scalars), 1): out = opt._fill_chain( sigmoid( tensor.neg(nonconsts[0].owner.inputs[0])), scalar_inputs) # keep combined stack traces of # exp(x): nonconsts[0], # 1 + exp(x): inv_arg, # 1 / (1 + exp(x)): node.outputs[0] copy_stack_trace( [nonconsts[0], inv_arg, node.outputs[0]], out) return out # Registration is below, and conditional. @gof.local_optimizer([tensor.sub]) def local_1msigmoid(node): """ 1-sigm(x) -> sigm(-x) """ if node.op == tensor.sub: sub_l, sub_r = node.inputs if len(sub_r.clients) > 1: return # graph is using both sigm and 1-sigm if sub_r.owner and sub_r.owner.op == sigmoid: try: val_l = opt.get_scalar_constant_value(sub_l) except tensor.NotScalarConstantError: return if np.allclose(np.sum(val_l), 1): out = sigmoid(-sub_r.owner.inputs[0]) copy_stack_trace([sub_r, node.outputs[0]], out) return [out] register_local_1msigmoid = False # This is False because the Stabilize pattern above # is looking for 1-sigm. Also Canonizer turns neg into *(-1) and so # this optimization might set off an unwanted chain of things. # OTH - this transformation can be seen as pushing normal arithmetic either below or above the # sigmoidal nonlinearity... so if the canonicalized form had anything to say about that then it # would be a consideration... anyway leaving False for now. if register_local_1msigmoid: opt.register_canonicalize(local_1msigmoid) Theano-1.0.0/theano/tensor/nnet/abstract_conv.py00006000163562€ÿÿÿþ0000042440513203335532023236 0ustar boccoset00000000000000""" Abstract conv interface """ from __future__ import absolute_import, print_function, division import logging from six import reraise, integer_types import sys from fractions import gcd import theano from theano.tensor import as_tensor_variable, patternbroadcast from theano.tensor import get_scalar_constant_value, NotScalarConstantError from theano.tensor.opt import Assert from theano.gof import Apply, Op from six.moves import xrange import warnings import numpy as np try: from scipy.signal.signaltools import _valfrommode, _bvalfromboundary, convolve from scipy.signal.sigtools import _convolve2d imported_scipy_signal = True except ImportError: imported_scipy_signal = False __docformat__ = "restructuredtext en" _logger = logging.getLogger("theano.tensor.nnet.abstract_conv") def get_conv_output_shape(image_shape, kernel_shape, border_mode, subsample, filter_dilation=None): """ This function compute the output shape of convolution operation. Parameters ---------- image_shape: tuple of int (symbolic or numeric) corresponding to the input image shape. Its four (or five) element must correspond respectively to: batch size, number of input channels, height and width (and possibly depth) of the image. None where undefined. kernel_shape: tuple of int (symbolic or numeric) corresponding to the kernel shape. For a normal convolution, its four (for 2D convolution) or five (for 3D convolution) elements must correspond respectively to : number of output channels, number of input channels, height and width (and possibly depth) of the kernel. For an unshared 2D convolution, its six channels must correspond to : number of output channels, height and width of the output, number of input channels, height and width of the kernel. None where undefined. border_mode: string, int (symbolic or numeric) or tuple of int (symbolic or numeric) or pairs of ints. If it is a string, it must be 'valid', 'half' or 'full'. If it is a tuple, its two (or three) elements respectively correspond to the padding on height and width (and possibly depth) axis. For asymmetric padding, provide a pair of ints for each dimension. subsample: tuple of int (symbolic or numeric). Its two or three elements espectively correspond to the subsampling on height and width (and possibly depth) axis. filter_dilation: tuple of int (symbolic or numeric). Its two or three elements correspond respectively to the dilation on height and width axis. Note - The shape of the convolution output does not depend on the 'unshared' or the 'num_groups' parameters. Returns ------- output_shape: tuple of int corresponding to the output image shape. Its four element must correspond respectively to: batch size, number of output channels, height and width of the image. None where undefined. """ bsize, imshp = image_shape[0], image_shape[2:] convdim = len(image_shape) - 2 nkern, kshp = kernel_shape[0], kernel_shape[-convdim:] if filter_dilation is None: filter_dilation = np.ones(len(subsample), dtype='int') if isinstance(border_mode, tuple): out_shp = tuple(get_conv_shape_1axis( imshp[i], kshp[i], border_mode[i], subsample[i], filter_dilation[i]) for i in range(len(subsample))) else: out_shp = tuple(get_conv_shape_1axis( imshp[i], kshp[i], border_mode, subsample[i], filter_dilation[i]) for i in range(len(subsample))) return (bsize, nkern) + out_shp # filter dilation set by default to 1 # for compatibility with other tests. def get_conv_shape_1axis(image_shape, kernel_shape, border_mode, subsample, dilation=1): """ This function compute the output shape of convolution operation. Parameters ---------- image_shape: int or None. Corresponds to the input image shape on a given axis. None if undefined. kernel_shape: int or None. Corresponds to the kernel shape on a given axis. None if undefined. border_mode: string, int or tuple of 2 ints. If it is a string, it must be 'valid', 'half' or 'full'. If it is an integer, it must correspond to the padding on the considered axis. If it is a tuple, its two elements must correspond to the asymmetric padding (e.g., left and right) on the considered axis. subsample: int. It must correspond to the subsampling on the considered axis. dilation: int. It must correspond to the dilation on the considered axis. Returns ------- out_shp: int corresponding to the output image shape on the considered axis. None if undefined. """ if None in [image_shape, kernel_shape, border_mode, subsample, dilation]: return None # Implicit dilated kernel shape dil_kernel_shape = (kernel_shape - 1) * dilation + 1 if border_mode == "half": pad_l = pad_r = dil_kernel_shape // 2 elif border_mode == "full": pad_l = pad_r = dil_kernel_shape - 1 elif border_mode == "valid": pad_l = pad_r = 0 else: if isinstance(border_mode, tuple): pad_l, pad_r = border_mode else: pad_l = pad_r = border_mode if pad_l < 0 or pad_r < 0: raise ValueError("border_mode must be >= 0") # In case of symbolic shape, we want to build the smallest graph # (image_shape + 2 * pad - dil_kernel_shape) // subsample + 1 out_shp = (image_shape - dil_kernel_shape) if pad_l != 0: out_shp += pad_l if pad_r != 0: out_shp += pad_r if subsample != 1: out_shp = out_shp // subsample out_shp = out_shp + 1 return out_shp def get_conv_gradweights_shape(image_shape, top_shape, border_mode, subsample, filter_dilation=None, num_groups=1, unshared=False): """ This function tries to compute the kernel shape of convolution gradWeights. The weights shape can only be computed exactly when subsample is 1 and border_mode is not 'half'. If subsample is not 1 or border_mode is 'half', this function will return None. Parameters ---------- image_shape: tuple of int corresponding to the input image shape. Its four (or five) elements must correspond respectively to: batch size, number of output channels, height and width of the image. None where undefined. top_shape: tuple of int (symbolic or numeric) corresponding to the top image shape. Its four (or five) element must correspond respectively to: batch size, number of output channels, height and width (and possibly depth) of the image. None where undefined. border_mode: string, int (symbolic or numeric) or tuple of int (symbolic or numeric) or pairs of ints. If it is a string, it must be 'valid', 'half' or 'full'. If it is a tuple, its two (or three) elements respectively correspond to the padding on height and width (and possibly depth) axis. For asymmetric padding, provide a pair of ints for each dimension. subsample: tuple of int (symbolic or numeric). Its two or three elements respectively correspond to the subsampling on height and width (and possibly depth) axis. filter_dilation: tuple of int (symbolic or numeric). Its two or three elements correspond respectively to the dilation on height and width axis. num_groups: An int which specifies the number of separate groups to be divided into. unshared: Boolean value. If true, unshared convolution will be performed, where a different filter is applied to each area of the input. Returns ------- kernel_shape: tuple of int (symbolic or numeric) corresponding to the kernel shape. Its four (or five) elements correspond respectively to: number of output channels, number of input channels, height and width (and possibly depth) of the kernel. None where undefined. """ nkern, imshp = image_shape[1], image_shape[2:] nchan, topshp = top_shape[1], top_shape[2:] if filter_dilation is None: filter_dilation = np.ones(len(subsample), dtype='int') if num_groups > 1: nchan = nchan // num_groups if isinstance(border_mode, tuple): out_shp = tuple(get_conv_gradweights_shape_1axis( imshp[i], topshp[i], border_mode[i], subsample[i], filter_dilation[i]) for i in range(len(subsample))) else: out_shp = tuple(get_conv_gradweights_shape_1axis( imshp[i], topshp[i], border_mode, subsample[i], filter_dilation[i]) for i in range(len(subsample))) if unshared: return (nchan,) + top_shape[2:] + (nkern,) + out_shp else: return (nchan, nkern) + out_shp def get_conv_gradweights_shape_1axis(image_shape, top_shape, border_mode, subsample, dilation): """ This function tries to compute the image shape of convolution gradWeights. The weights shape can only be computed exactly when subsample is 1 and border_mode is not 'half'. If subsample is not 1 or border_mode is 'half', this function will return None. Parameters ---------- image_shape: int or None. Corresponds to the input image shape on a given axis. None if undefined. top_shape: int or None. Corresponds to the top shape on a given axis. None if undefined. border_mode: string, int or tuple of 2 ints. If it is a string, it must be 'valid', 'half' or 'full'. If it is an integer, it must correspond to the padding on the considered axis. If it is a tuple, its two elements must correspond to the asymmetric padding (e.g., left and right) on the considered axis. subsample: int. It must correspond to the subsampling on the considered axis. dilation: int. It must correspond to the dilation on the considered axis. Returns ------- kernel_shape: int or None. Corresponds to the kernel shape on a given axis. None if undefined. """ if None in [image_shape, top_shape, border_mode, subsample, dilation]: return None if subsample != 1 or border_mode == "half": return None if border_mode == "full": kernel_shape = top_shape - image_shape elif border_mode == "valid": kernel_shape = image_shape - top_shape else: if isinstance(border_mode, tuple): pad_l, pad_r = border_mode else: pad_l = pad_r = border_mode if pad_l < 0 or pad_r < 0: raise ValueError("border_mode must be >= 0") kernel_shape = (image_shape + pad_l + pad_r - top_shape) if dilation > 1: kernel_shape = kernel_shape / dilation return kernel_shape + 1 def get_conv_gradinputs_shape(kernel_shape, top_shape, border_mode, subsample, filter_dilation=None, num_groups=1): """ This function tries to compute the image shape of convolution gradInputs. The image shape can only be computed exactly when subsample is 1. If subsample for a dimension is not 1, this function will return None for that dimension. Parameters ---------- kernel_shape: tuple of int (symbolic or numeric) corresponding to the kernel shape. Its four (or five) elements must correspond respectively to: number of output channels, number of input channels, height and width (and possibly depth) of the kernel. None where undefined. top_shape: tuple of int (symbolic or numeric) corresponding to the top image shape. Its four (or five) element must correspond respectively to: batch size, number of output channels, height and width (and possibly depth) of the image. None where undefined. border_mode: string, int (symbolic or numeric) or tuple of int (symbolic or numeric) or pairs of ints. If it is a string, it must be 'valid', 'half' or 'full'. If it is a tuple, its two (or three) elements respectively correspond to the padding on height and width (and possibly depth) axis. For asymmetric padding, provide a pair of ints for each dimension. subsample: tuple of int (symbolic or numeric). Its two or three elements respectively correspond to the subsampling on height and width (and possibly depth) axis. filter_dilation: tuple of int (symbolic or numeric). Its two or three elements correspond respectively to the dilation on height and width axis. num_groups: An int which specifies the number of separate groups to be divided into. Note - The shape of the convolution output does not depend on the 'unshared' parameter. Returns ------- image_shape: tuple of int corresponding to the input image shape. Its four element must correspond respectively to: batch size, number of output channels, height and width of the image. None where undefined. """ bsize, topshp = top_shape[0], top_shape[2:] convdim = len(top_shape) - 2 nkern, kshp = kernel_shape[1], kernel_shape[-convdim:] if filter_dilation is None: filter_dilation = np.ones(len(subsample), dtype='int') if num_groups > 1: nkern = nkern * num_groups if isinstance(border_mode, tuple): out_shp = tuple(get_conv_gradinputs_shape_1axis( kshp[i], topshp[i], border_mode[i], subsample[i], filter_dilation[i]) for i in range(len(subsample))) else: out_shp = tuple(get_conv_gradinputs_shape_1axis( kshp[i], topshp[i], border_mode, subsample[i], filter_dilation[i]) for i in range(len(subsample))) return (bsize, nkern) + out_shp def get_conv_gradinputs_shape_1axis(kernel_shape, top_shape, border_mode, subsample, dilation): """ This function tries to compute the image shape of convolution gradInputs. The image shape can only be computed exactly when subsample is 1. If subsample is not 1, this function will return None. Parameters ---------- kernel_shape: int or None. Corresponds to the kernel shape on a given axis. None if undefined. top_shape: int or None. Corresponds to the top shape on a given axis. None if undefined. border_mode: string, int or tuple of 2 ints. If it is a string, it must be 'valid', 'half' or 'full'. If it is an integer, it must correspond to the padding on the considered axis. If it is a tuple, its two elements must correspond to the asymmetric padding (e.g., left and right) on the considered axis. subsample: int. It must correspond to the subsampling on the considered axis. dilation: int. It must correspond to the dilation on the considered axis. Returns ------- image_shape: int or None. Corresponds to the input image shape on a given axis. None if undefined. """ if None in [kernel_shape, top_shape, border_mode, subsample, dilation]: return None if subsample != 1: return None # Implicit dilated kernel shape dil_kernel_shape = (kernel_shape - 1) * dilation + 1 if border_mode == "half": pad_l = pad_r = dil_kernel_shape // 2 elif border_mode == "full": pad_l = pad_r = dil_kernel_shape - 1 elif border_mode == "valid": pad_l = pad_r = 0 else: if isinstance(border_mode, tuple): pad_l, pad_r = border_mode else: pad_l = pad_r = border_mode if pad_l < 0 or pad_r < 0: raise ValueError("border_mode must be >= 0") # In case of symbolic shape, we want to build the smallest graph # image_shape = (top_shape - 1) * s - 2 * pad + dil_kernel_shape + a # where 0 <= a < subsample, but we have checked that subsample == 1 image_shape = (top_shape + dil_kernel_shape - 1) if pad_l > 0: image_shape -= pad_l if pad_r > 0: image_shape -= pad_r return image_shape def check_conv_gradinputs_shape(image_shape, kernel_shape, output_shape, border_mode, subsample, filter_dilation=None): """ This function checks if the given image shapes are consistent. Parameters ---------- image_shape: tuple of int (symbolic or numeric) corresponding to the input image shape. Its four (or five) element must correspond respectively to: batch size, number of input channels, height and width (and possibly depth) of the image. None where undefined. kernel_shape: tuple of int (symbolic or numeric) corresponding to the kernel shape. Its four (or five) elements must correspond respectively to: number of output channels, number of input channels, height and width (and possibly depth) of the kernel. None where undefined. output_shape: tuple of int (symbolic or numeric) corresponding to the output shape. Its four (or five) elements must correspond respectively to: batch size, number of output channels, height and width (and possibly depth) of the output. None where undefined. border_mode: string, int (symbolic or numeric) or tuple of int (symbolic or numeric) or pairs of ints. If it is a string, it must be 'valid', 'half' or 'full'. If it is a tuple, its two (or three) elements respectively correspond to the padding on height and width (and possibly depth) axis. For asymmetric padding, provide a pair of ints for each dimension. subsample: tuple of int (symbolic or numeric). Its two or three elements respectively correspond to the subsampling on height and width (and possibly depth) axis. filter_dilation: tuple of int (symbolic or numeric). Its two or three elements correspond respectively to the dilation on height and width axis. Returns ------- Returns False if a convolution with the given input shape, kernel shape and parameters would not have produced the given output shape. Returns True in all other cases: if the given output shape matches the computed output shape, but also if the shape could not be checked because because the shape contains symbolic values. """ image_shape = tuple(image_shape) kernel_shape = tuple(kernel_shape) output_shape = tuple(output_shape) if len(image_shape) != len(kernel_shape) or len(image_shape) != len(output_shape): return False if len(image_shape) - 2 != len(subsample): return False if filter_dilation is not None and len(image_shape) - 2 != len(filter_dilation): return False # compute the predicted output shape computed_output_shape = get_conv_output_shape( image_shape, kernel_shape, border_mode, subsample, filter_dilation) # check if the given output shape matches the computed shape def check_dim(given, computed): if given is None or computed is None: return True try: given = get_scalar_constant_value(given) computed = get_scalar_constant_value(computed) return int(given) == int(computed) except NotScalarConstantError: # no answer possible, accept for now return True return all(check_dim(given, computed) for (given, computed) in zip(output_shape, computed_output_shape)) def assert_conv_shape(shape): """This function adds Assert nodes that check if shape is a valid convolution shape. The first two dimensions should be larger than or equal to zero. The convolution dimensions should be larger than zero. Parameters ---------- shape: tuple of int (symbolic or numeric) corresponding to the input, output or kernel shape of a convolution. For input and output, the first elements should should be the batch size and number of channels. For kernels, the first and second elements should contain the number of input and output channels. The remaining dimensions are the convolution dimensions. Returns ------- Returns a tuple similar to the given `shape`. For constant elements in `shape`, the function checks the value and raises a `ValueError` if the dimension is invalid. The elements that are not constant are wrapped in an `Assert` op that checks the dimension at run time. """ out_shape = [] for i, n in enumerate(shape): try: const_n = get_scalar_constant_value(n) if i < 2: if const_n < 0: raise ValueError('The convolution would produce an invalid shape (dim[%d]: %d < 0).' % (i, const_n)) else: if const_n <= 0: raise ValueError('The convolution would produce an invalid shape (dim[%d]: %d <= 0).' % (i, const_n)) out_shape.append(n) except NotScalarConstantError: if i < 2: assert_shp = Assert('The convolution would produce an invalid shape (dim[%d] < 0).' % i) out_shape.append(assert_shp(n, theano.tensor.ge(n, 0))) else: assert_shp = Assert('The convolution would produce an invalid shape (dim[%d] <= 0).' % i) out_shape.append(assert_shp(n, theano.tensor.gt(n, 0))) return tuple(out_shape) def assert_shape(x, expected_shape, msg='Unexpected shape.'): """Wraps `x` in an `Assert` to check its shape. Parameters ---------- x : Tensor x will be wrapped in an `Assert`. expected_shape : tuple or list The expected shape of `x`. The size of a dimension can be None, which means it will not be checked. msg : str The error message of the `Assert`. Returns ------- Tensor `x` wrapped in an `Assert`. At execution time, this will throw an AssertionError if the shape of `x` does not match `expected_shape`. If `expected_shape` is None or contains only Nones, the function will return `x` directly. """ if expected_shape is None or not theano.config.conv.assert_shape: return x shape = x.shape tests = [] for i in range(x.ndim): if expected_shape[i] is not None: tests.append(theano.tensor.eq(shape[i], expected_shape[i])) if tests: return Assert(msg)(x, *tests) else: return x def border_mode_to_pad(mode, convdim, kshp): """ Computes a tuple for padding given the border_mode parameter Parameters ---------- mode : int or tuple One of "valid", "full", "half", an integer, or a tuple where each member is either an integer or a tuple of 2 positive integers. convdim : int The dimensionality of the convolution. kshp : List/tuple of length 'convdim', indicating the size of the kernel in the spatial dimensions. Returns ------- A tuple containing 'convdim' elements, each of which is a tuple of two positive integers corresponding to the padding on the left and the right sides respectively. """ if isinstance(mode, tuple): if len(mode) != convdim: raise ValueError( 'invalid border_mode {} which must be a ' 'tuple of length {}'.format(mode, convdim)) border = () for m in mode: if isinstance(m, integer_types) and m >= 0: border += ((m, m),) elif isinstance(m, tuple) and min(m) >= 0 and \ all(isinstance(b, integer_types) for b in m): if len(m) != 2: raise NotImplementedError( 'Asymmetric padding not implemented ' 'for {}d'.format(len(m))) border += ((m[0], m[1]),) else: raise ValueError( 'invalid border mode {}. The tuple can only contain ' 'integers or tuples of length 2'.format(mode)) pad = border elif mode == 'full': pad = tuple((kshp[i] - 1,) * 2 for i in range(convdim)) elif mode == 'half': pad = tuple((kshp[i] // 2,) * 2 for i in range(convdim)) elif mode == 'valid': pad = ((0, 0),) * convdim else: raise ValueError( 'invalid border_mode {}, which must be either ' '"valid", "full", "half", an integer or a tuple ' 'of length {}'.format(mode, convdim)) return pad def conv2d(input, filters, input_shape=None, filter_shape=None, border_mode='valid', subsample=(1, 1), filter_flip=True, filter_dilation=(1, 1), num_groups=1, unshared=False): """This function will build the symbolic graph for convolving a mini-batch of a stack of 2D inputs with a set of 2D filters. The implementation is modelled after Convolutional Neural Networks (CNN). Refer to :func:`nnet.conv2d ` for a more detailed documentation. """ input = as_tensor_variable(input) filters = as_tensor_variable(filters) conv_op = AbstractConv2d(imshp=input_shape, kshp=filter_shape, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_groups, unshared=unshared) return conv_op(input, filters) def separable_conv2d(input, depthwise_filters, pointwise_filters, num_channels, input_shape=None, depthwise_filter_shape=None, pointwise_filter_shape=None, border_mode='valid', subsample=(1, 1), filter_flip=True, filter_dilation=(1, 1)): """ This function will build the symbolic graph for depthwise convolutions which act separately on the input channels followed by pointwise convolution which mixes channels. Parameters ---------- input: symbolic 4D tensor Mini-batch of feature map stacks, of shape (batch size, input channels, input rows, input columns). See the optional parameter ``input_shape``. depthwise_filters: symbolic 4D tensor Set of filters used depthwise convolution layer of shape (depthwise output channels, 1, filter rows, filter columns). pointwise_filters: symbolic 4D tensor Set of filters used pointwise convolution layer of shape (output channels, depthwise output channels, 1, 1). num_channels: int The number of channels of the input. Required for depthwise convolutions. input_shape: None, tuple/list of len 4 of int or Constant variable The shape of the input parameter. Optional, possibly used to choose an optimal implementation. You can give ``None`` for any element of the list to specify that this element is not known at compile time. depthwise_filter_shape: None, tuple/list of len 4 of int or Constant variable The shape of the depthwise filters parameter. Optional, possibly used to choose an optimal implementation. You can give ``None`` for any element of the list to specify that this element is not known at compile time. pointwise_filter_shape: None, tuple/list of len 4 of int or Constant variable The shape of the pointwise filters parameter. Optional, possibly used to choose an optimal implementation. You can give ``None`` for any element of the list to specify that this element is not known at compile time. border_mode: str, int or tuple of two int This applies only to depthwise convolutions Either of the following: ``'valid'``: apply filter wherever it completely overlaps with the input. Generates output of shape: input shape - filter shape + 1 ``'full'``: apply filter wherever it partly overlaps with the input. Generates output of shape: input shape + filter shape - 1 ``'half'``: pad input with a symmetric border of ``filter rows // 2`` rows and ``filter columns // 2`` columns, then perform a valid convolution. For filters with an odd number of rows and columns, this leads to the output shape being equal to the input shape. ``int``: pad input with a symmetric border of zeros of the given width, then perform a valid convolution. ``(int1, int2)``: pad input with a symmetric border of ``int1`` rows and ``int2`` columns, then perform a valid convolution. ``(int1, (int2, int3))`` or ``((int1, int2), int3)``: pad input with one symmetric border of `int1`` or ``int3``, and one asymmetric border of ``(int2, int3)`` or ``(int1, int2)``. ``((int1, int2), (int3, int4))``: pad input with an asymmetric border of ``(int1, int2)`` along one dimension and ``(int3, int4)`` along the second dimension. subsample: tuple of len 2 Factor by which to subsample the output. This applies only to depthwise convolutions filter_flip: bool If ``True``, will flip the filter rows and columns before sliding them over the input. This operation is normally referred to as a convolution, and this is the default. If ``False``, the filters are not flipped and the operation is referred to as a cross-correlation. filter_dilation: tuple of len 2 Factor by which to subsample (stride) the input. This applies only to depthwise convolutions Returns ------- Symbolic 4D tensor Set of feature maps generated by convolutional layer. Tensor is of shape (batch size, output channels, output rows, output columns) """ input = as_tensor_variable(input) depthwise_filters = as_tensor_variable(depthwise_filters) conv_op = AbstractConv2d(imshp=input_shape, kshp=depthwise_filter_shape, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_channels) if input_shape is None or depthwise_filter_shape is None: depthwise_op_shape = None else: depthwise_op_shape = conv_op.infer_shape(None, [input_shape, depthwise_filter_shape])[0] depthwise_op = conv_op(input, depthwise_filters) pointwise_op = conv2d(input=depthwise_op, filters=pointwise_filters, input_shape=depthwise_op_shape, filter_shape=pointwise_filter_shape, border_mode='valid', subsample=(1, 1), filter_flip=filter_flip, filter_dilation=(1, 1), num_groups=1) return pointwise_op def separable_conv3d(input, depthwise_filters, pointwise_filters, num_channels, input_shape=None, depthwise_filter_shape=None, pointwise_filter_shape=None, border_mode='valid', subsample=(1, 1, 1), filter_flip=True, filter_dilation=(1, 1, 1)): """ This function will build the symbolic graph for depthwise convolutions which act separately on the input channels followed by pointwise convolution which mixes channels. Parameters ---------- input: symbolic 5D tensor Mini-batch of feature map stacks, of shape (batch size, input channels, input depth, input rows, input columns). See the optional parameter ``input_shape``. depthwise_filters: symbolic 5D tensor Set of filters used depthwise convolution layer of shape (depthwise output channels, 1, filter_depth, filter rows, filter columns). pointwise_filters: symbolic 5D tensor Set of filters used pointwise convolution layer of shape (output channels, depthwise output channels, 1, 1, 1). num_channels: int The number of channels of the input. Required for depthwise convolutions. input_shape: None, tuple/list of len 5 of int or Constant variable The shape of the input parameter. Optional, possibly used to choose an optimal implementation. You can give ``None`` for any element of the list to specify that this element is not known at compile time. depthwise_filter_shape: None, tuple/list of len 5 of int or Constant variable The shape of the depthwise filters parameter. Optional, possibly used to choose an optimal implementation. You can give ``None`` for any element of the list to specify that this element is not known at compile time. pointwise_filter_shape: None, tuple/list of len 5 of int or Constant variable The shape of the pointwise filters parameter. Optional, possibly used to choose an optimal implementation. You can give ``None`` for any element of the list to specify that this element is not known at compile time. border_mode: str, int or tuple of three int This applies only to depthwise convolutions Either of the following: ``'valid'``: apply filter wherever it completely overlaps with the input. Generates output of shape: input shape - filter shape + 1 ``'full'``: apply filter wherever it partly overlaps with the input. Generates output of shape: input shape + filter shape - 1 ``'half'``: pad input with a symmetric border of ``filter // 2``, then perform a valid convolution. For filters with an odd number of slices, rows and columns, this leads to the output shape being equal to the input shape. ``int``: pad input with a symmetric border of zeros of the given width, then perform a valid convolution. ``(int1, int2, int3)`` pad input with a symmetric border of ``int1``, ``int2`` and ``int3`` columns, then perform a valid convolution. subsample: tuple of len 3 This applies only to depthwise convolutions Factor by which to subsample the output. Also called strides elsewhere. filter_flip: bool If ``True``, will flip the filter x, y and z dimensions before sliding them over the input. This operation is normally referred to as a convolution, and this is the default. If ``False``, the filters are not flipped and the operation is referred to as a cross-correlation. filter_dilation: tuple of len 3 Factor by which to subsample (stride) the input. Also called dilation elsewhere. Returns ------- Symbolic 5D tensor Set of feature maps generated by convolutional layer. Tensor is of shape (batch size, output channels, output_depth, output rows, output columns) """ input = as_tensor_variable(input) depthwise_filters = as_tensor_variable(depthwise_filters) conv_op = AbstractConv3d(imshp=input_shape, kshp=depthwise_filter_shape, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_channels) if input_shape is None or depthwise_filter_shape is None: depthwise_op_shape = None else: depthwise_op_shape = conv_op.infer_shape(None, [input_shape, depthwise_filter_shape])[0] depthwise_op = conv_op(input, depthwise_filters) pointwise_op = conv3d(input=depthwise_op, filters=pointwise_filters, input_shape=depthwise_op_shape, filter_shape=pointwise_filter_shape, border_mode='valid', subsample=(1, 1, 1), filter_flip=filter_flip, filter_dilation=(1, 1, 1), num_groups=1) return pointwise_op def conv3d(input, filters, input_shape=None, filter_shape=None, border_mode='valid', subsample=(1, 1, 1), filter_flip=True, filter_dilation=(1, 1, 1), num_groups=1): """ This function will build the symbolic graph for convolving a mini-batch of a stack of 3D inputs with a set of 3D filters. The implementation is modelled after Convolutional Neural Networks (CNN). Parameters ---------- input: symbolic 5D tensor Mini-batch of feature map stacks, of shape (batch size, input channels, input depth, input rows, input columns). See the optional parameter ``input_shape``. filters: symbolic 5D tensor Set of filters used in CNN layer of shape (output channels, input channels, filter depth, filter rows, filter columns). See the optional parameter ``filter_shape``. input_shape: None, tuple/list of len 5 of int or Constant variable The shape of the input parameter. Optional, possibly used to choose an optimal implementation. You can give ``None`` for any element of the list to specify that this element is not known at compile time. filter_shape: None, tuple/list of len 5 of int or Constant variable The shape of the filters parameter. Optional, possibly used to choose an optimal implementation. You can give ``None`` for any element of the list to specify that this element is not known at compile time. border_mode: str, int or tuple of three int Either of the following: ``'valid'``: apply filter wherever it completely overlaps with the input. Generates output of shape: input shape - filter shape + 1 ``'full'``: apply filter wherever it partly overlaps with the input. Generates output of shape: input shape + filter shape - 1 ``'half'``: pad input with a symmetric border of ``filter // 2``, then perform a valid convolution. For filters with an odd number of slices, rows and columns, this leads to the output shape being equal to the input shape. ``int``: pad input with a symmetric border of zeros of the given width, then perform a valid convolution. ``(int1, int2, int3)`` pad input with a symmetric border of ``int1``, ``int2`` and ``int3`` columns, then perform a valid convolution. subsample: tuple of len 3 Factor by which to subsample the output. Also called strides elsewhere. filter_flip: bool If ``True``, will flip the filter x, y and z dimensions before sliding them over the input. This operation is normally referred to as a convolution, and this is the default. If ``False``, the filters are not flipped and the operation is referred to as a cross-correlation. filter_dilation: tuple of len 3 Factor by which to subsample (stride) the input. Also called dilation elsewhere. num_groups : int Divides the image, kernel and output tensors into num_groups separate groups. Each which carry out convolutions separately Returns ------- Symbolic 5D tensor Set of feature maps generated by convolutional layer. Tensor is is of shape (batch size, output channels, output depth, output rows, output columns) Notes ----- If cuDNN is available, it will be used on the GPU. Otherwise, it is the *Corr3dMM* convolution that will be used "caffe style convolution". This is only supported in Theano 0.8 or the development version until it is released. """ input = as_tensor_variable(input) filters = as_tensor_variable(filters) conv_op = AbstractConv3d(imshp=input_shape, kshp=filter_shape, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_groups) return conv_op(input, filters) def conv2d_grad_wrt_inputs(output_grad, filters, input_shape, filter_shape=None, border_mode='valid', subsample=(1, 1), filter_flip=True, filter_dilation=(1, 1), num_groups=1, unshared=False): """Compute conv output gradient w.r.t its inputs This function builds the symbolic graph for getting the gradient of the output of a convolution (namely output_grad) w.r.t the input of the convolution, given a set of 2D filters used by the convolution, such that the output_grad is upsampled to the input_shape. Parameters ---------- output_grad : symbolic 4D tensor mini-batch of feature map stacks, of shape (batch size, input channels, input rows, input columns). This is the tensor that will be upsampled or the output gradient of the convolution whose gradient will be taken with respect to the input of the convolution. filters: symbolic 4D or 6D tensor Set of filters used in CNN layer of shape (output channels, input channels, filter rows, filter columns) for normal convolution and (output channels, output rows, output columns, input channels, filter rows, filter columns) for unshared convolution. See the optional parameter ``filter_shape``. input_shape : [None/int/Constant] * 2 + [Tensor/int/Constant] * 2 The shape of the input (upsampled) parameter. A tuple/list of len 4, with the first two dimensions being None or int or Constant and the last two dimensions being Tensor or int or Constant. Not Optional, since given the output_grad shape and the subsample values, multiple input_shape may be plausible. filter_shape : None or [None/int/Constant] * (4 or 6) The shape of the filters parameter. None or a tuple/list of len 4 or a tuple/list of len 6 (for unshared convolution) Optional, possibly used to choose an optimal implementation. You can give ``None`` for any element of the list to specify that this element is not known at compile time. border_mode: str, int or a tuple of two ints or pairs of ints Either of the following: ``'valid'`` apply filter wherever it completely overlaps with the input. Generates output of shape: input shape - filter shape + 1 ``'full'`` apply filter wherever it partly overlaps with the input. Generates output of shape: input shape + filter shape - 1 ``'half'`` pad input with a symmetric border of ``filter rows // 2`` rows and ``filter columns // 2`` columns, then perform a valid convolution. For filters with an odd number of rows and columns, this leads to the output shape being equal to the input shape. It is known as 'same' elsewhere. ``int`` pad input with a symmetric border of zeros of the given width, then perform a valid convolution. ``(int1, int2)`` pad input with a symmetric border of ``int1`` rows and ``int2`` columns, then perform a valid convolution. ``(int1, (int2, int3))`` or ``((int1, int2), int3)`` pad input with one symmetric border of `int1`` or ``int3``, and one asymmetric border of ``(int2, int3)`` or ``(int1, int2)``. ``((int1, int2), (int3, int4))`` pad input with an asymmetric border of ``(int1, int2)`` along one dimension and ``(int3, int4)`` along the second dimension. subsample : tuple of len 2 The subsampling used in the forward pass. Also called strides elsewhere. filter_flip : bool If ``True``, will flip the filter rows and columns before sliding them over the input. This operation is normally referred to as a convolution, and this is the default. If ``False``, the filters are not flipped and the operation is referred to as a cross-correlation. filter_dilation : tuple of len 2 The filter dilation used in the forward pass. Also known as input striding. num_groups : int Divides the image, kernel and output tensors into num_groups separate groups. Each which carry out convolutions separately unshared: bool If true, then unshared or 'locally connected' convolution will be performed. A different filter will be used for each region of the input. Returns ------- symbolic 4D tensor set of feature maps generated by convolutional layer. Tensor is of shape (batch size, output channels, output rows, output columns) Notes ----- :note: If cuDNN is available, it will be used on the GPU. Otherwise, it is the *CorrMM* convolution that will be used "caffe style convolution". :note: This is only supported in Theano 0.8 or the development version until it is released. """ filters = as_tensor_variable(filters) output_grad = as_tensor_variable(output_grad) # checking the type of input_shape for dim in [0, 1]: assert isinstance(input_shape[dim], (theano.tensor.TensorConstant, integer_types, type(None))) for dim in [2, 3]: assert isinstance(input_shape[dim], (theano.tensor.TensorVariable, theano.tensor.TensorConstant, integer_types)) # checking the type of filter_shape if filter_shape is not None: if unshared: expected_dim = 6 else: expected_dim = 4 assert len(filter_shape) == expected_dim for dim in range(expected_dim): assert isinstance(filter_shape[dim], (theano.tensor.TensorConstant, integer_types, type(None))) # setting the last two dimensions of input_shape to None, if # the type of these dimensions is TensorVariable. numerical_input_shape = list(input_shape) for dim in [2, 3]: if isinstance(input_shape[dim], theano.tensor.TensorVariable): numerical_input_shape[dim] = None grad_input_op = AbstractConv2d_gradInputs(imshp=numerical_input_shape, kshp=filter_shape, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_groups, unshared=unshared) return grad_input_op(filters, output_grad, input_shape[-2:]) def conv3d_grad_wrt_inputs(output_grad, filters, input_shape, filter_shape=None, border_mode='valid', subsample=(1, 1, 1), filter_flip=True, filter_dilation=(1, 1, 1), num_groups=1): """Compute conv output gradient w.r.t its inputs This function builds the symbolic graph for getting the gradient of the output of a convolution (namely output_grad) w.r.t the input of the convolution, given a set of 3D filters used by the convolution, such that the output_grad is upsampled to the input_shape. Parameters ---------- output_grad : symbolic 5D tensor mini-batch of feature map stacks, of shape (batch size, input channels, input depth, input rows, input columns). This is the tensor that will be upsampled or the output gradient of the convolution whose gradient will be taken with respect to the input of the convolution. filters : symbolic 5D tensor set of filters used in CNN layer of shape (output channels, input channels, filter depth, filter rows, filter columns). See the optional parameter ``filter_shape``. input_shape : [None/int/Constant] * 2 + [Tensor/int/Constant] * 2 The shape of the input (upsampled) parameter. A tuple/list of len 5, with the first two dimensions being None or int or Constant and the last three dimensions being Tensor or int or Constant. Not Optional, since given the output_grad shape and the subsample values, multiple input_shape may be plausible. filter_shape : None or [None/int/Constant] * 5 The shape of the filters parameter. None or a tuple/list of len 5. Optional, possibly used to choose an optimal implementation. You can give ``None`` for any element of the list to specify that this element is not known at compile time. border_mode : str, int or tuple of three int Either of the following: ``'valid'`` apply filter wherever it completely overlaps with the input. Generates output of shape: input shape - filter shape + 1 ``'full'`` apply filter wherever it partly overlaps with the input. Generates output of shape: input shape + filter shape - 1 ``'half'`` pad input with a symmetric border of ``filter // 2``, then perform a valid convolution. For filters with an odd number of slices, rows and columns, this leads to the output shape being equal to the input shape. It is known as 'same' elsewhere. ``int`` pad input with a symmetric border of zeros of the given width, then perform a valid convolution. ``(int1, int2, int3)`` pad input with a symmetric border of ``int1``, ``int2`` and ``int3`` columns, then perform a valid convolution. subsample : tuple of len 3 The subsampling used in the forward pass. Also called strides elsewhere. filter_flip : bool If ``True``, will flip the filter x, y and z dimensions before sliding them over the input. This operation is normally referred to as a convolution, and this is the default. If ``False``, the filters are not flipped and the operation is referred to as a cross-correlation. filter_dilation : tuple of len 3 The filter dilation used in the forward pass. Also known as input striding. num_groups : int Divides the image, kernel and output tensors into num_groups separate groups. Each which carry out convolutions separately Returns ------- symbolic 5D tensor set of feature maps generated by convolutional layer. Tensor is of shape (batch size, output channels, output depth, output rows, output columns) Notes ----- :note: If cuDNN is available, it will be used on the GPU. Otherwise, it is the *Corr3dMM* convolution that will be used "caffe style convolution". :note: This is only supported in Theano 0.8 or the development version until it is released. """ filters = as_tensor_variable(filters) output_grad = as_tensor_variable(output_grad) # checking the type of input_shape for dim in [0, 1]: assert isinstance(input_shape[dim], (theano.tensor.TensorConstant, integer_types, type(None))) for dim in [2, 3, 4]: assert isinstance(input_shape[dim], (theano.tensor.TensorVariable, theano.tensor.TensorConstant, integer_types)) # checking the type of filter_shape if filter_shape is not None: for dim in [0, 1, 2, 3, 4]: assert isinstance(filter_shape[dim], (theano.tensor.TensorConstant, integer_types, type(None))) # setting the last three dimensions of input_shape to None, if # the type of these dimensions is TensorVariable. numerical_input_shape = list(input_shape) for dim in [2, 3, 4]: if isinstance(input_shape[dim], theano.tensor.TensorVariable): numerical_input_shape[dim] = None grad_input_op = AbstractConv3d_gradInputs(imshp=numerical_input_shape, kshp=filter_shape, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_groups) return grad_input_op(filters, output_grad, input_shape[-3:]) def conv2d_grad_wrt_weights(input, output_grad, filter_shape, input_shape=None, border_mode='valid', subsample=(1, 1), filter_flip=True, filter_dilation=(1, 1), num_groups=1, unshared=False): """Compute conv output gradient w.r.t its weights This function will build the symbolic graph for getting the gradient of the output of a convolution (output_grad) w.r.t its wights. Parameters ---------- input : symbolic 4D tensor mini-batch of feature map stacks, of shape (batch size, input channels, input rows, input columns). This is the input of the convolution in the forward pass. output_grad : symbolic 4D tensor mini-batch of feature map stacks, of shape (batch size, input channels, input rows, input columns). This is the gradient of the output of convolution. filter_shape : [None/int/Constant] * (2 or 4) + [Tensor/int/Constant] * 2 The shape of the filter parameter. A tuple/list of len 4 or 6 (for unshared), with the first two dimensions being None or int or Constant and the last two dimensions being Tensor or int or Constant. Not Optional, since given the output_grad shape and the input_shape, multiple filter_shape may be plausible. input_shape : None or [None/int/Constant] * 4 The shape of the input parameter. None or a tuple/list of len 4. Optional, possibly used to choose an optimal implementation. You can give ``None`` for any element of the list to specify that this element is not known at compile time. border_mode: str, int or a tuple of two ints or pairs of ints Either of the following: ``'valid'`` apply filter wherever it completely overlaps with the input. Generates output of shape: input shape - filter shape + 1 ``'full'`` apply filter wherever it partly overlaps with the input. Generates output of shape: input shape + filter shape - 1 ``'half'`` pad input with a symmetric border of ``filter rows // 2`` rows and ``filter columns // 2`` columns, then perform a valid convolution. For filters with an odd number of rows and columns, this leads to the output shape being equal to the input shape. It is known as 'same' elsewhere. ``int`` pad input with a symmetric border of zeros of the given width, then perform a valid convolution. ``(int1, int2)`` pad input with a symmetric border of ``int1`` rows and ``int2`` columns, then perform a valid convolution. ``(int1, (int2, int3))`` or ``((int1, int2), int3)`` pad input with one symmetric border of `int1`` or ``int3``, and one asymmetric border of ``(int2, int3)`` or ``(int1, int2)``. ``((int1, int2), (int3, int4))`` pad input with an asymmetric border of ``(int1, int2)`` along one dimension and ``(int3, int4)`` along the second dimension. subsample : tuple of len 2 The subsampling used in the forward pass of the convolutional operation. Also called strides elsewhere. filter_flip : bool If ``True``, will flip the filter rows and columns before sliding them over the input. This operation is normally referred to as a convolution, and this is the default. If ``False``, the filters are not flipped and the operation is referred to as a cross-correlation. filter_dilation : tuple of len 2 The filter dilation used in the forward pass. Also known as input striding. num_groups : int Divides the image, kernel and output tensors into num_groups separate groups. Each which carry out convolutions separately unshared: bool If true, then unshared or 'locally connected' convolution will be performed. A different filter will be used for each region of the input. Returns ------- symbolic 4D tensor or 6D tensor set of feature maps generated by convolutional layer. Tensor is of shape (batch size, output channels, output rows, output columns) for normal convolution and (output channels, output rows, output columns, input channels, filter rows, filter columns) for unshared convolution Notes ----- :note: If cuDNN is available, it will be used on the GPU. Otherwise, it is the *CorrMM* convolution that will be used "caffe style convolution". :note: This is only supported in Theano 0.8 or the development version until it is released. """ input = as_tensor_variable(input) output_grad = as_tensor_variable(output_grad) # checking the type of filter_shape for dim in [0, 1]: assert isinstance(filter_shape[dim], (theano.tensor.TensorConstant, integer_types, type(None))) if unshared: for dim in [2, 3]: assert isinstance(filter_shape[dim], (theano.tensor.TensorConstant, integer_types, type(None))) for dim in [-2, -1]: assert isinstance(filter_shape[dim], (theano.tensor.TensorVariable, theano.tensor.TensorConstant, integer_types)) # checking the type of input_shape if input_shape is not None: for dim in [0, 1, 2, 3]: assert isinstance(input_shape[dim], (theano.tensor.TensorConstant, integer_types, type(None))) # setting the last two dimensions of filter_shape to None, if # the type of these dimensions is TensorVariable. numerical_filter_shape = list(filter_shape) for dim in [-2, -1]: if isinstance(filter_shape[dim], theano.tensor.TensorVariable): numerical_filter_shape[dim] = None gradWeight_op = AbstractConv2d_gradWeights(imshp=input_shape, kshp=numerical_filter_shape, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_groups, unshared=unshared) return gradWeight_op(input, output_grad, filter_shape[-2:]) def conv3d_grad_wrt_weights(input, output_grad, filter_shape, input_shape=None, border_mode='valid', subsample=(1, 1, 1), filter_flip=True, filter_dilation=(1, 1, 1), num_groups=1): """Compute conv output gradient w.r.t its weights This function will build the symbolic graph for getting the gradient of the output of a convolution (output_grad) w.r.t its weights. Parameters ---------- input : symbolic 5D tensor mini-batch of feature map stacks, of shape (batch size, input channels, input depth, input rows, input columns). This is the input of the convolution in the forward pass. output_grad : symbolic 5D tensor mini-batch of feature map stacks, of shape (batch size, input channels, input depth, input rows, input columns). This is the gradient of the output of convolution. filter_shape : [None/int/Constant] * 2 + [Tensor/int/Constant] * 2 The shape of the filter parameter. A tuple/list of len 5, with the first two dimensions being None or int or Constant and the last three dimensions being Tensor or int or Constant. Not Optional, since given the output_grad shape and the input_shape, multiple filter_shape may be plausible. input_shape : None or [None/int/Constant] * 5 The shape of the input parameter. None or a tuple/list of len 5. Optional, possibly used to choose an optimal implementation. You can give ``None`` for any element of the list to specify that this element is not known at compile time. border_mode : str, int or tuple of two ints Either of the following: ``'valid'`` apply filter wherever it completely overlaps with the input. Generates output of shape: input shape - filter shape + 1 ``'full'`` apply filter wherever it partly overlaps with the input. Generates output of shape: input shape + filter shape - 1 ``'half'`` pad input with a symmetric border of ``filter rows // 2`` rows and ``filter columns // 2`` columns, then perform a valid convolution. For filters with an odd number of rows and columns, this leads to the output shape being equal to the input shape. It is known as 'same' elsewhere. ``int`` pad input with a symmetric border of zeros of the given width, then perform a valid convolution. ``(int1, int2, int3)`` pad input with a symmetric border of ``int1``, ``int2`` and ``int3``, then perform a valid convolution. subsample : tuple of len 3 The subsampling used in the forward pass of the convolutional operation. Also called strides elsewhere. filter_flip : bool If ``True``, will flip the filters before sliding them over the input. This operation is normally referred to as a convolution, and this is the default. If ``False``, the filters are not flipped and the operation is referred to as a cross-correlation. filter_dilation : tuple of len 3 The filter dilation used in the forward pass. Also known as input striding. num_groups : int Divides the image, kernel and output tensors into num_groups separate groups. Each which carry out convolutions separately Returns ------- symbolic 5D tensor set of feature maps generated by convolutional layer. Tensor is of shape (batch size, output channels, output time, output rows, output columns) Notes ----- :note: If cuDNN is available, it will be used on the GPU. Otherwise, it is the *Corr3dMM* convolution that will be used "caffe style convolution". :note: This is only supported in Theano 0.8 or the development version until it is released. """ input = as_tensor_variable(input) output_grad = as_tensor_variable(output_grad) # checking the type of filter_shape for dim in [0, 1]: assert isinstance(filter_shape[dim], (theano.tensor.TensorConstant, integer_types, type(None))) for dim in [2, 3, 4]: assert isinstance(filter_shape[dim], (theano.tensor.TensorVariable, theano.tensor.TensorConstant, integer_types)) # checking the type of input_shape if input_shape is not None: for dim in [0, 1, 2, 3, 4]: assert isinstance(input_shape[dim], (theano.tensor.TensorConstant, integer_types, type(None))) # setting the last three dimensions of filter_shape to None, if # the type of these dimensions is TensorVariable. numerical_filter_shape = list(filter_shape) for dim in [2, 3, 4]: if isinstance(filter_shape[dim], theano.tensor.TensorVariable): numerical_filter_shape[dim] = None gradWeight_op = AbstractConv3d_gradWeights(imshp=input_shape, kshp=numerical_filter_shape, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_groups) return gradWeight_op(input, output_grad, filter_shape[-3:]) def causal_conv1d(input, filters, filter_shape, input_shape=None, subsample=1, filter_flip=True, filter_dilation=1, num_groups=1, unshared=False): """ Computes (dilated) causal convolution The output at time t depends only on the inputs till t-1. Used for modelling temporal data. See [WaveNet: A Generative Model for Raw Audio, section 2.1] (https://arxiv.org/abs/1609.03499). Parameters ---------- input : symbolic 3D tensor mini-batch of feature vector stacks, of shape (batch_size, input_channels, input_length) See the optional parameter ``input_shape`` filters : symbolic 3D tensor Set of filters used in the CNN, of shape (output_channels, input_channels, filter_length) filter_shape : [None/int/Constant] * 2 + [Tensor/int/Constant] The shape of the filters parameter. A tuple/list of len 3, with the first two dimensions being None or int or Constant and the last dimension being Tensor or int or Constant. Not optional, since the filter length is needed to calculate the left padding for causality. input_shape : None or [None/int/Constant] * 3 The shape of the input parameter. None, or a tuple/list of len 3. Optional, possibly used to choose an optimal implementation. subsample : int The factor by which to subsample the output. Also called strides elsewhere. filter_dilation : int Factor by which to subsample (stride) the input. Also called dilation factor. num_groups : int Divides the image, kernel and output tensors into num_groups separate groups. Each which carry out convolutions separately unshared : bool If true, then unshared or 'locally connected' convolution will be performed. A different filter will be used for each region of the input. Returns ------- Symbolic 3D tensor. Set of feature vectors generated by convolutional layer. Tensor is of shape (batch_size, output_channels, output_length) Notes ----- :note: Currently, this is implemented with the 2D convolution ops. """ input = as_tensor_variable(input) filters = as_tensor_variable(filters) if input.ndim != 3: raise ValueError('Input should be 3D for causal convolution.') if filters.ndim != 3: raise ValueError('Filters should be 3D for causal convolution') input = input.dimshuffle(0, 1, 2, 'x') filters = filters.dimshuffle(0, 1, 2, 'x') if input_shape is not None: assert(len(input_shape) == 3) input_shape = tuple(input_shape) input_shape += (1,) assert(len(filter_shape) == 3) filter_shape = tuple(filter_shape) filter_shape += (1,) left_pad = filter_dilation * (filter_shape[2] - 1) subsample = (subsample, 1) filter_dilation = (filter_dilation, 1) conv_op = AbstractConv2d(imshp=input_shape, kshp=filter_shape, border_mode=((left_pad, 0), 0), subsample=subsample, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_groups, unshared=unshared) output = conv_op(input, filters) return output[:, :, :, 0] def bilinear_kernel_2D(ratio, normalize=True): """Compute 2D kernel for bilinear upsampling This function builds the 2D kernel that can be used to upsample a tensor by the given ratio using bilinear interpolation. Parameters ---------- ratio: int or Constant/Scalar Theano tensor of int* dtype the ratio by which an image will be upsampled by the returned filter in the 2D space. normalize: bool param normalize: indicates whether to normalize the kernel or not. Default is True. Returns ------- symbolic 2D tensor the 2D kernels that can be applied to any given image to upsample it by the indicated ratio using bilinear interpolation in two dimensions. """ if isinstance(ratio, tuple): ratio_h = ratio[1] ratio_v = ratio[0] else: ratio_h = ratio ratio_v = ratio hkern = bilinear_kernel_1D(ratio=ratio_h, normalize=normalize).dimshuffle('x', 0) vkern = bilinear_kernel_1D(ratio=ratio_v, normalize=normalize).dimshuffle(0, 'x') kern = hkern * vkern return kern def bilinear_kernel_1D(ratio, normalize=True): """Compute 1D kernel for bilinear upsampling This function builds the 1D kernel that can be used to upsample a tensor by the given ratio using bilinear interpolation. Parameters ---------- ratio: int or Constant/Scalar Theano tensor of int* dtype the ratio by which an image will be upsampled by the returned filter in the 2D space. normalize: bool param normalize: indicates whether to normalize the kernel or not. Default is True. Returns ------- symbolic 1D tensor the 1D kernels that can be applied to any given image to upsample it by the indicated ratio using bilinear interpolation in one dimension. """ T = theano.tensor half_kern = T.arange(1, ratio + 1, dtype=theano.config.floatX) kern = T.concatenate([half_kern, half_kern[-2::-1]]) if normalize: kern /= ratio return kern def frac_bilinear_upsampling(input, frac_ratio): """Compute bilinear upsampling This function will build the symbolic graph for upsampling a tensor by the given ratio using bilinear interpolation. Parameters ---------- input: symbolic 4D tensor mini-batch of feature map stacks, of shape (batch size, input channels, input rows, input columns) that will be upsampled. frac_ratio: tuple of int or tuple of tuples of int The tuple defining the fractional ratio by which the input is upsampled in the 2D space. One fractional ratio should be represented as (numerator, denominator). If row and col ratios are different frac_ratio should be a tuple of fractional ratios, i.e a tuple of tuples. Returns ------- symbolic 4D tensor set of feature maps generated by bilinear upsampling. Tensor is of shape (batch size, num_input_channels, input row size * row ratio, input column size * column ratio). Each of these ratios can be fractional. Notes ----- :note: The kernel used for bilinear interpolation is fixed (not learned). :note: When the upsampling frac_ratio numerator is even, the last row and column is repeated one extra time compared to the first row and column which makes the upsampled tensor asymmetrical on both sides. This does not happen when it is odd. """ T = theano.tensor row, col = input.shape[2:] up_input = input.reshape((-1, 1, row, col)) # define the upsampling ratio depending on the case if not isinstance(frac_ratio, tuple): raise ValueError("frac_ratio must be a tuple") else: if isinstance(frac_ratio[0], tuple): f_r = [] for i, fr in enumerate(frac_ratio): p, q = fr div = gcd(p, q) f_r.append(tuple(np.array(fr) // div)) frac_ratio = tuple(f_r) ratio = (frac_ratio[0][0], frac_ratio[1][0]) subsample = (frac_ratio[0][1], frac_ratio[1][1]) else: p, q = frac_ratio div = gcd(p, q) frac_ratio = tuple(np.array(frac_ratio) // div) ratio = (frac_ratio[0], frac_ratio[0]) subsample = (frac_ratio[1], frac_ratio[1]) # duplicate borders of the input concat_mat = T.concatenate((up_input[:, :, :1, :], up_input, up_input[:, :, -1:, :]), axis=2) concat_mat = T.concatenate((concat_mat[:, :, :, :1], concat_mat, concat_mat[:, :, :, -1:]), axis=3) # add padding for the pyramidal kernel double_pad = (2 * T.as_tensor([row, col]) - 1) * np.array(ratio) + 1 pad = double_pad // 2 # build pyramidal kernel kern = bilinear_kernel_2D(ratio=ratio)[np.newaxis, np.newaxis, :, :].astype(theano.config.floatX) # add corresponding padding pad_kern = T.concatenate((T.zeros(tuple(kern.shape[:2]) + (pad[0], kern.shape[-1]), dtype=theano.config.floatX), kern, T.zeros(tuple(kern.shape[:2]) + (double_pad[0] - pad[0], kern.shape[-1]), dtype=theano.config.floatX)), axis=2) pad_kern = T.concatenate((T.zeros(tuple(pad_kern.shape[:3]) + (pad[1],), dtype=theano.config.floatX), pad_kern, T.zeros(tuple(pad_kern.shape[:3]) + (double_pad[1] - pad[1],), dtype=theano.config.floatX)), axis=3) # upsample the input by passing it as kernel of conv and using filter_dilation upsamp = T.nnet.conv2d(pad_kern, concat_mat, border_mode='valid', filter_dilation=ratio, subsample=subsample) up_img_sh = T.ceil(T.as_tensor([row, col]) * np.array(ratio) / np.array(subsample)).astype('int64') return upsamp.reshape((input.shape[0], input.shape[1], up_img_sh[0], up_img_sh[1])) def bilinear_upsampling(input, ratio=None, frac_ratio=None, batch_size=None, num_input_channels=None, use_1D_kernel=True): """Compute bilinear upsampling This function will build the symbolic graph for upsampling a tensor by the given ratio using bilinear interpolation. Parameters ---------- input: symbolic 4D tensor mini-batch of feature map stacks, of shape (batch size, input channels, input rows, input columns) that will be upsampled. ratio: `int or Constant or Scalar Tensor of int* dtype` the ratio by which the input is upsampled in the 2D space (row and col size). frac_ratio: None, tuple of int or tuple of tuples of int The tuple defining the fractional ratio by which the input is upsampled in the 2D space. One fractional ratio should be represented as (numerator, denominator). If row and col ratios are different frac_ratio should be a tuple of fractional ratios, i.e a tuple of tuples. use_1D_kernel: bool if set to true, row and column will be upsampled seperately by 1D kernels, otherwise they are upsampled together using a 2D kernel. The final result is the same, only the speed can differ, given factors such as upsampling ratio. Returns ------- symbolic 4D tensor set of feature maps generated by bilinear upsampling. Tensor is of shape (batch size, num_input_channels, input row size * row ratio, input column size * column ratio). Each of these ratios can be fractional. Notes ----- :note: The kernel used for bilinear interpolation is fixed (not learned). :note: When the upsampling ratio is even, the last row and column is repeated one extra time compared to the first row and column which makes the upsampled tensor asymmetrical on both sides. This does not happen when the upsampling ratio is odd. :note: This function must get either ratio or frac_ratio as parameter and never both at once. """ if ratio and frac_ratio: raise ValueError("can't use ratio and frac_ratio together") if not (ratio or frac_ratio): raise ValueError("No ratio (or frac_ratio) provided") if frac_ratio: if use_1D_kernel: raise ValueError('For fractional ratios 1D kernel ' 'method not implemented. You may want to pass ' 'use_1D_kernel as False') # case of fractional 2D upsampling return frac_bilinear_upsampling(input, frac_ratio=frac_ratio) # the remaining case if integer ratio with use_1D_kernel T = theano.tensor try: up_bs = batch_size * num_input_channels except TypeError: up_bs = None row, col = input.shape[2:] up_input = input.reshape((-1, 1, row, col)) # concatenating the first and last row and column # first and last row concat_mat = T.concatenate((up_input[:, :, :1, :], up_input, up_input[:, :, -1:, :]), axis=2) # first and last col concat_mat = T.concatenate((concat_mat[:, :, :, :1], concat_mat, concat_mat[:, :, :, -1:]), axis=3) concat_col = col + 2 pad = 2 * ratio - (ratio - 1) // 2 - 1 if use_1D_kernel: kern = bilinear_kernel_1D(ratio=ratio, normalize=True) # upsampling rows upsampled_row = conv2d_grad_wrt_inputs(output_grad=concat_mat, filters=kern[np.newaxis, np.newaxis, :, np.newaxis], input_shape=(up_bs, 1, row * ratio, concat_col), filter_shape=(1, 1, None, 1), border_mode=(pad, 0), subsample=(ratio, 1), filter_flip=True, filter_dilation=(1, 1)) # upsampling cols upsampled_mat = conv2d_grad_wrt_inputs(output_grad=upsampled_row, filters=kern[np.newaxis, np.newaxis, np.newaxis, :], input_shape=(up_bs, 1, row * ratio, col * ratio), filter_shape=(1, 1, 1, None), border_mode=(0, pad), subsample=(1, ratio), filter_flip=True, filter_dilation=(1, 1)) else: kern = bilinear_kernel_2D(ratio=ratio, normalize=True) upsampled_mat = conv2d_grad_wrt_inputs(output_grad=concat_mat, filters=kern[np.newaxis, np.newaxis, :, :], input_shape=(up_bs, 1, row * ratio, col * ratio), filter_shape=(1, 1, None, None), border_mode=(pad, pad), subsample=(ratio, ratio), filter_flip=True, filter_dilation=(1, 1)) return upsampled_mat.reshape((input.shape[0], input.shape[1], row * ratio, col * ratio)) class BaseAbstractConv(Op): """Base class for AbstractConv Parameters ---------- convdim: The number of convolution dimensions (2 or 3). imshp: None, tuple/list of len ``(2 + convdim)`` of int or Constant variable The shape of the input parameter. Optional, possibly used to choose an optimal implementation. You can give ``None`` for any element of the list to specify that this element is not known at compile time. imshp is defined w.r.t the forward conv. kshp: None, tuple/list of len ``(2 + convdim)`` or ``(2 + 2 * convdim)`` (for unshared) of int or Constant variable The shape of the filters parameter. Optional, possibly used to choose an optimal implementation. You can give ``None`` for any element of the list to specify that this element is not known at compile time. kshp is defined w.r.t the forward conv. border_mode: str, int or a tuple of two ints or pairs of ints Either of the following: ``'valid'``: apply filter wherever it completely overlaps with the input. Generates output of shape: input shape - filter shape + 1 ``'full'``: apply filter wherever it partly overlaps with the input. Generates output of shape: input shape + filter shape - 1 ``'half'``: pad input with a symmetric border of ``filter size // 2`` in each convolution dimension, then perform a valid convolution. For filters with an odd filter size, this leads to the output shape being equal to the input shape. ``int``: pad input with a symmetric border of zeros of the given width, then perform a valid convolution. ``(int1, int2)``: (for 2D) pad input with a symmetric border of ``int1``, ``int2``, then perform a valid convolution. ``(int1, (int2, int3))`` or ``((int1, int2), int3)``: (for 2D) pad input with one symmetric border of `int1`` or ``int3``, and one asymmetric border of ``(int2, int3)`` or ``(int1, int2)``. ``((int1, int2), (int3, int4))``: (for 2D) pad input with an asymmetric border of ``(int1, int2)`` along one dimension and ``(int3, int4)`` along the second dimension. ``(int1, int2, int3)``: (for 3D) pad input with a symmetric border of ``int1``, ``int2`` and ``int3``, then perform a valid convolution. subsample: tuple of len ``convdim`` Factor by which to subsample the output. Also called strides elsewhere. filter_flip: bool If ``True``, will flip the filter rows and columns before sliding them over the input. This operation is normally referred to as a convolution, and this is the default. If ``False``, the filters are not flipped and the operation is referred to as a cross-correlation. filter_dilation: tuple of len ``convdim`` Factor by which to subsample (stride) the input. Also called dilation factor. num_groups : int Divides the image, kernel and output tensors into num_groups separate groups. Each which carry out convolutions separately unshared: bool If true, then unshared or 'locally connected' convolution will be performed. A different filter will be used for each region of the input. """ check_broadcast = False __props__ = ('convdim', 'border_mode', 'subsample', 'filter_flip', 'imshp', 'kshp', 'filter_dilation', 'num_groups', 'unshared') def __init__(self, convdim, imshp=None, kshp=None, border_mode="valid", subsample=None, filter_flip=True, filter_dilation=None, num_groups=1, unshared=False): self.convdim = convdim if convdim not in (2, 3): raise ValueError( 'convolution dimension {} is not supported', convdim) if subsample is None: subsample = (1,) * convdim if filter_dilation is None: filter_dilation = (1,) * convdim if isinstance(border_mode, integer_types): if border_mode < 0: raise ValueError( 'invalid border_mode {}, which must be a ' 'non-negative integer'.format(border_mode)) border_mode = (border_mode,) * convdim elif isinstance(border_mode, tuple): if len(border_mode) != convdim: raise ValueError( 'invalid border_mode {}, which must be a ' 'tuple of length {}'.format(border_mode, convdim)) new_border_mode = () for mode in border_mode: if not((isinstance(mode, integer_types) and mode >= 0) or (isinstance(mode, tuple) and len(mode) == 2 and min(mode) >= 0 and all(isinstance(m, integer_types) for m in mode))): raise ValueError( 'invalid border mode {}. The tuple can only contain integers ' ' or pairs of integers'.format(border_mode)) if isinstance(mode, tuple): if convdim != 2: raise NotImplementedError( 'Asymmetric padding not implemented for {}D'.format(convdim)) if mode[0] == mode[1]: mode = mode[0] new_border_mode += (mode,) border_mode = new_border_mode elif border_mode not in ('valid', 'full', 'half'): raise ValueError( 'invalid border_mode {}, which must be either ' '"valid", "full", "half", an integer or a tuple ' 'of length {}'.format(border_mode, convdim)) if isinstance(border_mode, tuple) and \ all(mode == (0, 0) or mode == 0 for mode in border_mode): border_mode = 'valid' self.imshp = tuple(imshp) if imshp else (None,) * (2 + convdim) for imshp_i in self.imshp: if imshp_i is not None: # Components of imshp should be constant or ints try: get_scalar_constant_value(imshp_i, only_process_constants=True) except NotScalarConstantError: reraise(ValueError, ValueError("imshp should be None or a tuple of " "constant int values"), sys.exc_info()[2]) if kshp: self.kshp = tuple(kshp) else: self.kshp = (None,) * ((2 + 2 * convdim) if unshared else (2 + convdim)) for kshp_i in self.kshp: if kshp_i is not None: # Components of kshp should be constant or ints try: get_scalar_constant_value(kshp_i, only_process_constants=True) except NotScalarConstantError: reraise(ValueError, ValueError("kshp should be None or a tuple of " "constant int values"), sys.exc_info()[2]) self.border_mode = border_mode self.filter_flip = filter_flip if len(subsample) != convdim: raise ValueError("subsample must have {} elements".format(convdim)) self.subsample = tuple(subsample) if len(filter_dilation) != convdim: raise ValueError("filter_dilation must have {} elements".format(convdim)) self.filter_dilation = tuple(filter_dilation) if num_groups < 1: raise ValueError("num_groups must have value greater than zero") self.num_groups = num_groups if unshared and self.convdim != 2: raise NotImplementedError('Unshared convolution not implemented for %dD' % self.convdim) self.unshared = unshared def do_constant_folding(self, node): # Disable constant folding since there is no implementation. # This may change in the future. return False def flops(self, inp, outp): """ Useful with the hack in profiling to print the MFlops""" if self.convdim == 2: # if the output shape is correct, then this gives the correct # flops for any direction, sampling, padding, and border mode inputs, filters = inp outputs, = outp assert inputs[1] == (filters[1] * self.num_groups) # nb mul and add by output pixel flops = filters[2] * filters[3] * 2 # nb flops by output image flops *= outputs[2] * outputs[3] # nb patch multiplied flops *= inputs[1] * filters[0] * inputs[0] / self.num_groups return flops else: # TODO implement for convdim == 3 raise NotImplementedError( 'flops not implemented for convdim={}', self.convdim) def conv(self, img, kern, mode="valid", dilation=1, num_groups=1, unshared=False, direction="forward"): """ Basic slow Python 2D or 3D convolution for DebugMode """ if not imported_scipy_signal: raise NotImplementedError( "AbstractConv perform requires the python package" " for scipy.signal to be installed.") if not (mode in ('valid', 'full')): raise ValueError( 'invalid mode {}, which must be either ' '"valid" or "full"'.format(mode)) if isinstance(dilation, integer_types): dilation = (dilation,) * self.convdim if len(dilation) != self.convdim: raise ValueError( 'invalid dilation {}, expected {} values'.format(dilation, self.convdim)) if unshared and direction == "backprop weights": if mode != "valid": raise ValueError('conv mode for unshared backprop wrt weights must be "valid"') # To allow the same format for the call to 'unshared2d' for all three directions, # the out_shape is shuffled here. # We do a transpose in the 'perform' function to bring it to the required shape out_shape = (img.shape[0], kern.shape[0], kern.shape[2], kern.shape[3], img.shape[2] - kern.shape[2] + 1, img.shape[3] - kern.shape[3] + 1) else: out_shape = get_conv_output_shape(img.shape, kern.shape, mode, [1] * self.convdim, dilation) dil_kern_shp = kern.shape[:-self.convdim] + tuple( (kern.shape[-self.convdim + i] - 1) * dilation[i] + 1 for i in range(self.convdim)) dilated_kern = np.zeros(dil_kern_shp, dtype=kern.dtype) dilated_kern[(slice(None),) * (dilated_kern.ndim - self.convdim) + tuple(slice(None, None, dilation[i]) for i in range(self.convdim)) ] = kern out = np.zeros(out_shape, dtype=img.dtype) if img.shape[1] % self.num_groups != 0: raise ValueError( 'number of input channels must be divible by num_groups') if kern.shape[0] % self.num_groups != 0: raise ValueError( 'number of filters must be divisible by num_groups') if img.shape[1] // num_groups != kern.shape[1]: raise ValueError( 'the number of input channels in the kernel should ' 'specify the number of channels of 1 group') input_channel_offset = img.shape[1] // self.num_groups output_channel_offset = kern.shape[0] // self.num_groups if self.convdim == 2: val = _valfrommode(mode) bval = _bvalfromboundary('fill') with warnings.catch_warnings(): warnings.simplefilter('ignore', np.ComplexWarning) for b in xrange(img.shape[0]): for g in xrange(self.num_groups): for n in xrange(output_channel_offset): for im0 in xrange(input_channel_offset): if unshared: out[b, g * output_channel_offset + n, ...] += self.unshared2d(img[b, g * input_channel_offset + im0, ...], dilated_kern[g * output_channel_offset + n, im0, ...], out_shape[2:], direction) else: # some cast generates a warning here out[b, g * output_channel_offset + n, ...] += _convolve2d(img[b, g * input_channel_offset + im0, ...], dilated_kern[g * output_channel_offset + n, im0, ...], 1, val, bval, 0) elif self.convdim == 3: if unshared: raise NotImplementedError('Unshared 3D convolution is not implemented') for b in xrange(img.shape[0]): for g in xrange(self.num_groups): for n in xrange(output_channel_offset): for im0 in xrange(input_channel_offset): out[b, g * output_channel_offset + n, ...] += convolve(img[b, g * input_channel_offset + im0, ...], dilated_kern[g * output_channel_offset + n, im0, ...], mode) else: raise NotImplementedError('only 2D and 3D convolution are implemented') return out def unshared2d(self, inp, kern, out_shape, direction="forward"): ''' Basic slow Python unshared 2d convolution. ''' if self.convdim != 2: raise NotImplementedError('Unshared convolution not implemented for %dD' % self.convdim) out = np.zeros(out_shape, dtype=inp.dtype) if direction == "forward": for row in xrange(out_shape[0]): for col in xrange(out_shape[1]): out[row, col] = np.sum(np.multiply(inp[row:row + kern.shape[2], col:col + kern.shape[3]], kern[row, col, ::-1, ::-1])) elif direction == "backprop weights": for row in xrange(out_shape[0]): for col in xrange(out_shape[1]): out[row, col, ...] = kern[row, col] * \ inp[row:row + out_shape[2], col:col + out_shape[3]] elif direction == "backprop inputs": for row in xrange(kern.shape[0]): for col in xrange(kern.shape[1]): out[row:row + kern.shape[2], col:col + kern.shape[3]] += inp[row, col] * \ kern[row, col, ::-1, ::-1] else: raise ValueError("unshared2d: invalid value '{}' for 'direction'".format(direction)) return out class AbstractConv(BaseAbstractConv): """ Abstract Op for the forward convolution. Refer to :func:`BaseAbstractConv ` for a more detailed documentation. """ def __init__(self, convdim, imshp=None, kshp=None, border_mode="valid", subsample=None, filter_flip=True, filter_dilation=None, num_groups=1, unshared=False): super(AbstractConv, self).__init__(convdim=convdim, imshp=imshp, kshp=kshp, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_groups, unshared=unshared) def make_node(self, img, kern): # Make sure both inputs are Variables with the same Type if not isinstance(img, theano.Variable): img = as_tensor_variable(img) if not isinstance(kern, theano.Variable): kern = as_tensor_variable(kern) ktype = img.type.clone(dtype=kern.dtype, broadcastable=kern.broadcastable) kern = ktype.filter_variable(kern) if img.type.ndim != 2 + self.convdim: raise TypeError('img must be %dD tensor' % (2 + self.convdim)) if self.unshared: if kern.type.ndim != 2 + 2 * self.convdim: raise TypeError('kern must be %dD tensor for unshared convolution' % (2 + 2 * self.convdim)) else: if kern.type.ndim != 2 + self.convdim: raise TypeError('kern must be %dD tensor' % (2 + self.convdim)) img = assert_shape(img, self.imshp, 'AbstractConv shape mismatch: shape of ' 'image does not match given imshp.') kern = assert_shape(kern, self.kshp, 'AbstractConv shape mismatch: shape of ' 'filters does not match given kshp.') broadcastable = [img.broadcastable[0], kern.broadcastable[0]] + ([False] * self.convdim) output = img.type.clone(broadcastable=broadcastable)() return Apply(self, [img, kern], [output]) def perform(self, node, inp, out_): img, kern = inp img = np.asarray(img) kern = np.asarray(kern) dil_kernshp = tuple((kern.shape[-self.convdim + i] - 1) * self.filter_dilation[i] + 1 for i in range(self.convdim)) if self.unshared and self.convdim != 2: raise NotImplementedError('Unshared convolution not implemented for %dD' % self.convdim) o, = out_ mode = self.border_mode pad = border_mode_to_pad(mode, self.convdim, dil_kernshp) if any(p != (0, 0) for p in pad): mode = "valid" new_img = np.zeros((img.shape[0], img.shape[1]) + tuple(img.shape[i + 2] + pad[i][0] + pad[i][1] for i in range(self.convdim)), dtype=img.dtype) new_img[(slice(None), slice(None)) + tuple(slice(pad[i][0], img.shape[i + 2] + pad[i][0]) for i in range(self.convdim))] = img img = new_img if not self.filter_flip: kern = kern[(slice(None),) * (kern.ndim - self.convdim) + (slice(None, None, -1),) * self.convdim] if self.unshared: out_shape = get_conv_output_shape(img.shape, kern.shape, mode, self.subsample, self.filter_dilation) if kern.shape[1:1 + self.convdim] != out_shape[2:2 + self.convdim]: raise ValueError('Kernel shape {} does not match ' 'computed output size {}'.format(kern.shape[1:1 + self.convdim], out_shape[2:2 + self.convdim])) if any(self.subsample[i] > 1 for i in range(self.convdim)): # Expand regions in kernel to correct for subsampling out_shape = get_conv_output_shape(img.shape, kern.shape, mode, (1,) * self.convdim, self.filter_dilation) exp_kern_shp = kern.shape[:1] + out_shape[2:2 + self.convdim] + \ kern.shape[1 + self.convdim:] exp_kern = np.zeros(exp_kern_shp, dtype=kern.dtype) exp_kern[(slice(None),) + tuple(slice(None, None, self.subsample[i]) for i in range(self.convdim)) + (slice(None),) * (self.convdim + 1)] = kern kern = exp_kern # from (nFilters, out_rows, out_cols, nChannels, kH, kW) # to (nFilters, nChannels, out_rows, out_cols, kH, kW) axes_order = (0, 1 + self.convdim,) + tuple(range(1, 1 + self.convdim)) + \ tuple(range(2 + self.convdim, kern.ndim)) kern = kern.transpose(axes_order) conv_out = self.conv(img, kern, mode="valid", dilation=self.filter_dilation, num_groups=self.num_groups, unshared=self.unshared) conv_out = conv_out[(slice(None), slice(None)) + tuple(slice(None, None, self.subsample[i]) for i in range(self.convdim))] o[0] = node.outputs[0].type.filter(conv_out) def R_op(self, inputs, eval_points): if self.num_groups > 1: raise NotImplementedError( 'Rop not implemented for grouped convolutions') if self.unshared: raise NotImplementedError('Rop not implemented for unshared convolution') rval = None if eval_points[0] is not None: rval = self.make_node(eval_points[0], inputs[1]).outputs[0] if eval_points[1] is not None: if rval is None: rval = self.make_node(inputs[0], eval_points[1]).outputs[0] else: rval += self.make_node(inputs[0], eval_points[1]).outputs[0] return [rval] def infer_shape(self, node, input_shapes): imshp = input_shapes[0] kshp = input_shapes[1] # replace symbolic shapes with known constant shapes if self.imshp is not None: imshp = [imshp[i] if self.imshp[i] is None else self.imshp[i] for i in range(2 + self.convdim)] if self.kshp is not None: if self.unshared: kshp = [kshp[i] if self.kshp[i] is None else self.kshp[i] for i in range(2 + 2 * self.convdim)] else: kshp = [kshp[i] if self.kshp[i] is None else self.kshp[i] for i in range(2 + self.convdim)] res = get_conv_output_shape(imshp, kshp, self.border_mode, self.subsample, self.filter_dilation) return [res] class AbstractConv2d(AbstractConv): """ Abstract Op for the forward convolution. Refer to :func:`BaseAbstractConv ` for a more detailed documentation. """ def __init__(self, imshp=None, kshp=None, border_mode="valid", subsample=(1, 1), filter_flip=True, filter_dilation=(1, 1), num_groups=1, unshared=False): super(AbstractConv2d, self).__init__(convdim=2, imshp=imshp, kshp=kshp, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_groups, unshared=unshared) def grad(self, inp, grads): bottom, weights = inp top, = grads # Don't add the assert again, as it was already added in the forward. d_bottom = AbstractConv2d_gradInputs(self.imshp, self.kshp, self.border_mode, self.subsample, self.filter_flip, self.filter_dilation, num_groups=self.num_groups, unshared=self.unshared)( weights, top, bottom.shape[-2:], add_assert_shape=False) d_weights = AbstractConv2d_gradWeights(self.imshp, self.kshp, self.border_mode, self.subsample, self.filter_flip, self.filter_dilation, num_groups=self.num_groups, unshared=self.unshared)( bottom, top, weights.shape[-2:], add_assert_shape=False) # Make sure that the broadcastable pattern of the inputs is used # for the gradients, even if the grad opts are not able to infer # that the dimensions are broadcastable. # Also make sure that the gradient lives on the same device than # the corresponding input. d_bottom = patternbroadcast(d_bottom, bottom.broadcastable) d_bottom = bottom.type.filter_variable(d_bottom) d_weights = patternbroadcast(d_weights, weights.broadcastable) d_weights = weights.type.filter_variable(d_weights) return d_bottom, d_weights class AbstractConv3d(AbstractConv): """ Abstract Op for the forward convolution. Refer to :func:`BaseAbstractConv ` for a more detailed documentation. """ def __init__(self, imshp=None, kshp=None, border_mode="valid", subsample=(1, 1, 1), filter_flip=True, filter_dilation=(1, 1, 1), num_groups=1): super(AbstractConv3d, self).__init__(convdim=3, imshp=imshp, kshp=kshp, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_groups) def grad(self, inp, grads): bottom, weights = inp top, = grads d_bottom = AbstractConv3d_gradInputs(self.imshp, self.kshp, self.border_mode, self.subsample, self.filter_flip, self.filter_dilation, self.num_groups)( weights, top, bottom.shape[-3:]) d_weights = AbstractConv3d_gradWeights(self.imshp, self.kshp, self.border_mode, self.subsample, self.filter_flip, self.filter_dilation, self.num_groups)( bottom, top, weights.shape[-3:]) # Make sure that the broadcastable pattern of the inputs is used # for the gradients, even if the grad opts are not able to infer # that the dimensions are broadcastable. # Also make sure that the gradient lives on the same device than # the corresponding input. d_bottom = patternbroadcast(d_bottom, bottom.broadcastable) d_bottom = bottom.type.filter_variable(d_bottom) d_weights = patternbroadcast(d_weights, weights.broadcastable) d_weights = weights.type.filter_variable(d_weights) return d_bottom, d_weights class AbstractConv_gradWeights(BaseAbstractConv): """Gradient wrt. filters for `AbstractConv`. Refer to :func:`BaseAbstractConv ` for a more detailed documentation. :note: You will not want to use this directly, but rely on Theano's automatic differentiation or graph optimization to use it as needed. """ def __init__(self, convdim, imshp=None, kshp=None, border_mode="valid", subsample=None, filter_flip=True, filter_dilation=None, num_groups=1, unshared=False): super(AbstractConv_gradWeights, self).__init__(convdim=convdim, imshp=imshp, kshp=kshp, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_groups, unshared=unshared) # Update shape/height_width def make_node(self, img, topgrad, shape, add_assert_shape=True): # Make sure both inputs are Variables with the same Type if not isinstance(img, theano.Variable): img = as_tensor_variable(img) if not isinstance(topgrad, theano.Variable): topgrad = as_tensor_variable(topgrad) gtype = img.type.clone(dtype=topgrad.dtype, broadcastable=topgrad.broadcastable) topgrad = gtype.filter_variable(topgrad) if img.type.ndim != 2 + self.convdim: raise TypeError('img must be %dD tensor' % (2 + self.convdim)) if topgrad.type.ndim != 2 + self.convdim: raise TypeError('topgrad must be %dD tensor' % (2 + self.convdim)) if add_assert_shape: img = assert_shape(img, self.imshp, 'AbstractConv_gradWeights shape mismatch: shape of ' 'image does not match given imshp.') shape = as_tensor_variable(shape) if self.unshared: broadcastable = [topgrad.broadcastable[1]] + ([False] * self.convdim) + \ [img.broadcastable[1]] + ([False] * self.convdim) else: broadcastable = [topgrad.broadcastable[1], img.broadcastable[1]] + ([False] * self.convdim) output = img.type.clone(broadcastable=broadcastable)() return Apply(self, [img, topgrad, shape], [output]) def perform(self, node, inp, out_): img, topgrad, shape = inp img = np.asarray(img) topgrad = np.asarray(topgrad) o, = out_ if self.unshared and self.convdim != 2: raise NotImplementedError('Unshared convolution not implemented for %dD' % self.convdim) dil_shape = tuple((shape[i] - 1) * self.filter_dilation[i] + 1 for i in range(self.convdim)) pad = border_mode_to_pad(self.border_mode, self.convdim, dil_shape) if any(p != (0, 0) for p in pad): new_img = np.zeros((img.shape[0], img.shape[1]) + tuple(img.shape[i + 2] + pad[i][0] + pad[i][1] for i in range(self.convdim)), dtype=img.dtype) new_img[(slice(None), slice(None)) + tuple(slice(pad[i][0], img.shape[i + 2] + pad[i][0]) for i in range(self.convdim))] = img img = new_img if any(self.subsample[i] > 1 for i in range(self.convdim)): new_shape = ((topgrad.shape[0], topgrad.shape[1]) + tuple(img.shape[i + 2] - dil_shape[i] + 1 for i in range(self.convdim))) new_topgrad = np.zeros((new_shape), dtype=topgrad.dtype) new_topgrad[(slice(None), slice(None)) + tuple(slice(None, None, self.subsample[i]) for i in range(self.convdim))] = topgrad topgrad = new_topgrad axes_order = (1, 0) + tuple(range(2, self.convdim + 2)) topgrad = topgrad.transpose(axes_order) img = img.transpose(axes_order) def correct_for_groups(mat): mshp0 = mat.shape[0] // self.num_groups mshp1 = mat.shape[1] * self.num_groups mat = mat.reshape((self.num_groups, mshp0) + mat.shape[1:]) mat = mat.transpose((1, 0, 2) + tuple(range(3, 3 + self.convdim))) mat = mat.reshape((mshp0, mshp1) + mat.shape[-self.convdim:]) return mat if self.num_groups > 1: img = correct_for_groups(img) if self.unshared: flip_kern = ((slice(None),) * (2 + self.convdim) + (slice(None, None, -1),) * self.convdim) kern = self.conv(img, topgrad, mode="valid", num_groups=self.num_groups, unshared=True, direction="backprop weights") if any(self.subsample[i] > 1 for i in range(self.convdim)): sub_slice = (slice(None),) * 2 + \ tuple(slice(None, None, self.subsample[i]) for i in range(0, self.convdim)) + \ (slice(None),) * self.convdim kern = kern[sub_slice] # from (nChannels, nFilters, out_rows, out_cols, kH, kW) # to (nFilters, out_rows, out_cols, nChannels, kH, kW) kern_axes = (1,) + tuple(range(2, self.convdim + 2)) + (0,) + \ tuple(range(self.convdim + 2, kern.ndim)) else: flip_topgrad = flip_kern = ((slice(None), slice(None)) + (slice(None, None, -1),) * self.convdim) topgrad = topgrad[flip_topgrad] kern = self.conv(img, topgrad, mode="valid", num_groups=self.num_groups) kern_axes = (1, 0) + tuple(range(2, self.convdim + 2)) kern = kern.transpose(kern_axes) if any(self.filter_dilation[i] > 1 for i in range(self.convdim)): kern = kern[(slice(None),) * (kern.ndim - self.convdim) + tuple(slice(None, None, self.filter_dilation[i]) for i in range(self.convdim))] if self.filter_flip: kern = kern[flip_kern] o[0] = node.outputs[0].type.filter(kern) def connection_pattern(self, node): return [[1], [1], [0]] # no connection to height, width def infer_shape(self, node, input_shapes): # We use self.kshp (that was passed when creating the Op) if possible, # or fall back to the `shape` input of the node. # TODO: when there is no subsampling, try to infer the kernel shape # from the shapes of inputs. imshp = input_shapes[0] topshp = input_shapes[1] if self.kshp: kshp = self.kshp else: if self.unshared: kshp = [None] * (2 + 2 * self.convdim) else: kshp = [None] * (2 + self.convdim) if self.unshared: fallback_kshp = ([topshp[1], topshp[2], topshp[3], imshp[1] // self.num_groups] + [node.inputs[2][i] for i in range(self.convdim)]) kshp = [fallback_kshp[i] if kshp[i] is None else kshp[i] for i in range(2 + 2 * self.convdim)] else: fallback_kshp = ([topshp[1], imshp[1] // self.num_groups] + [node.inputs[2][i] for i in range(self.convdim)]) kshp = [fallback_kshp[i] if kshp[i] is None else kshp[i] for i in range(2 + self.convdim)] return [kshp] class AbstractConv2d_gradWeights(AbstractConv_gradWeights): """Gradient wrt. filters for `AbstractConv2d`. Refer to :func:`BaseAbstractConv ` for a more detailed documentation. :note: You will not want to use this directly, but rely on Theano's automatic differentiation or graph optimization to use it as needed. """ def __init__(self, imshp=None, kshp=None, border_mode="valid", subsample=(1, 1), filter_flip=True, filter_dilation=(1, 1), num_groups=1, unshared=False): super(AbstractConv2d_gradWeights, self).__init__(convdim=2, imshp=imshp, kshp=kshp, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_groups, unshared=unshared) def grad(self, inp, grads): bottom, top = inp[:2] weights, = grads d_bottom = AbstractConv2d_gradInputs(self.imshp, self.kshp, self.border_mode, self.subsample, self.filter_flip, self.filter_dilation, self.num_groups, self.unshared)(weights, top, bottom.shape[-2:]) d_top = AbstractConv2d(self.imshp, self.kshp, self.border_mode, self.subsample, self.filter_flip, self.filter_dilation, self.num_groups, self.unshared)(bottom, weights) # Make sure that the broadcastable pattern of the inputs is used # for the gradients, even if the grad opts are not able to infer # that the dimensions are broadcastable. # Also make sure that the gradient lives on the same device than # the corresponding input. d_bottom = patternbroadcast(d_bottom, bottom.broadcastable) d_bottom = bottom.type.filter_variable(d_bottom) d_top = patternbroadcast(d_top, top.broadcastable) d_top = top.type.filter_variable(d_top) d_height_width = (theano.gradient.DisconnectedType()(),) return (d_bottom, d_top) + d_height_width class AbstractConv3d_gradWeights(AbstractConv_gradWeights): """Gradient wrt. filters for `AbstractConv3d`. Refer to :func:`BaseAbstractConv ` for a more detailed documentation. :note: You will not want to use this directly, but rely on Theano's automatic differentiation or graph optimization to use it as needed. """ def __init__(self, imshp=None, kshp=None, border_mode="valid", subsample=(1, 1, 1), filter_flip=True, filter_dilation=(1, 1, 1), num_groups=1): super(AbstractConv3d_gradWeights, self).__init__(convdim=3, imshp=imshp, kshp=kshp, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_groups) def grad(self, inp, grads): bottom, top = inp[:2] weights, = grads d_bottom = AbstractConv3d_gradInputs(self.imshp, self.kshp, self.border_mode, self.subsample, self.filter_flip, self.filter_dilation, self.num_groups)(weights, top, bottom.shape[-3:]) d_top = AbstractConv3d(self.imshp, self.kshp, self.border_mode, self.subsample, self.filter_flip, self.filter_dilation, self.num_groups)(bottom, weights) # Make sure that the broadcastable pattern of the inputs is used # for the gradients, even if the grad opts are not able to infer # that the dimensions are broadcastable. # Also make sure that the gradient lives on the same device than # the corresponding input. d_bottom = patternbroadcast(d_bottom, bottom.broadcastable) d_bottom = bottom.type.filter_variable(d_bottom) d_top = patternbroadcast(d_top, top.broadcastable) d_top = top.type.filter_variable(d_top) d_depth_height_width = (theano.gradient.DisconnectedType()(),) return (d_bottom, d_top) + d_depth_height_width class AbstractConv_gradInputs(BaseAbstractConv): """Gradient wrt. inputs for `AbstractConv`. Refer to :func:`BaseAbstractConv ` for a more detailed documentation. :note: You will not want to use this directly, but rely on Theano's automatic differentiation or graph optimization to use it as needed. """ def __init__(self, convdim, imshp=None, kshp=None, border_mode="valid", subsample=None, filter_flip=True, filter_dilation=None, num_groups=1, unshared=False): super(AbstractConv_gradInputs, self).__init__(convdim=convdim, imshp=imshp, kshp=kshp, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_groups, unshared=unshared) # Update shape/height_width def make_node(self, kern, topgrad, shape, add_assert_shape=True): # Make sure both inputs are Variables with the same Type if not isinstance(kern, theano.Variable): kern = as_tensor_variable(kern) if not isinstance(topgrad, theano.Variable): topgrad = as_tensor_variable(topgrad) gtype = kern.type.clone(dtype=topgrad.dtype, broadcastable=topgrad.broadcastable) topgrad = gtype.filter_variable(topgrad) if self.unshared: if self.convdim != 2: raise NotImplementedError('Unshared convolution not implemented for %dD' % self.convdim) elif kern.type.ndim != 2 + 2 * self.convdim: raise TypeError('kern must be %dD tensor for unshared convolution' % (2 + 2 * self.convdim)) else: if kern.type.ndim != 2 + self.convdim: raise TypeError('kern must be %dD tensor' % (2 + self.convdim)) if topgrad.type.ndim != 2 + self.convdim: raise TypeError('topgrad must be %dD tensor' % (2 + self.convdim)) if add_assert_shape: kern = assert_shape(kern, self.kshp, 'AbstractConv_gradInputs shape mismatch: shape of ' 'filters does not match given kshp.') shape = as_tensor_variable(shape) if self.num_groups > 1: broadcastable = [topgrad.type.broadcastable[0], False] + ([False] * self.convdim) else: broadcastable = [topgrad.type.broadcastable[0], kern.type.broadcastable[-self.convdim - 1]] + ([False] * self.convdim) output = kern.type.clone(broadcastable=broadcastable)() return Apply(self, [kern, topgrad, shape], [output]) def perform(self, node, inp, out_): kern, topgrad, shape = inp kern = np.asarray(kern) topgrad = np.asarray(topgrad) o, = out_ if self.unshared and self.convdim != 2: raise NotImplementedError('Unshared convolution not implemented for %dD' % self.convdim) dil_kernshp = tuple((kern.shape[-self.convdim + i] - 1) * self.filter_dilation[i] + 1 for i in range(self.convdim)) pad = border_mode_to_pad(self.border_mode, self.convdim, dil_kernshp) imshp = self.imshp[:] if self.imshp is not None else [None] * (2 + self.convdim) fallback_imshp = ([topgrad.shape[0], kern.shape[-self.convdim - 1]] + [shape[i] for i in range(self.convdim)]) imshp = [fallback_imshp[i] if imshp[i] is None else imshp[i] for i in range(2 + self.convdim)] expected_topgrad_shape = get_conv_output_shape( imshp, kern.shape, self.border_mode, self.subsample, self.filter_dilation) if not tuple(expected_topgrad_shape) == tuple(topgrad.shape): raise ValueError( 'invalid input_shape for gradInputs: the given input_shape ' 'would produce an output of shape {}, but the given topgrad ' 'has shape {}'.format(tuple(expected_topgrad_shape), tuple(topgrad.shape))) if any(self.subsample[i] > 1 for i in range(self.convdim)): new_shape = ((topgrad.shape[0], topgrad.shape[1]) + tuple(shape[i] + pad[i][0] + pad[i][1] - dil_kernshp[i] + 1 for i in range(self.convdim))) new_topgrad = np.zeros((new_shape), dtype=topgrad.dtype) new_topgrad[(slice(None), slice(None)) + tuple(slice(None, None, self.subsample[i]) for i in range(self.convdim))] = topgrad topgrad = new_topgrad if self.unshared: # Expand regions in kernel to correct for subsampling exp_kern_shp = kern.shape[:1] + topgrad.shape[2:] + kern.shape[1 + self.convdim:] exp_kern = np.zeros(exp_kern_shp, dtype=kern.dtype) exp_kern[(slice(None),) + tuple(slice(None, None, self.subsample[i]) for i in range(self.convdim)) + (slice(None),) * (self.convdim + 1)] = kern kern = exp_kern def correct_for_groups(mat): mshp0 = mat.shape[0] // self.num_groups mshp1 = mat.shape[-self.convdim - 1] * self.num_groups mat = mat.reshape((self.num_groups, mshp0) + mat.shape[1:]) if self.unshared: # for 2D -> (1, 2, 3, 0, 4, 5, 6) mat = mat.transpose(tuple(range(1, 2 + self.convdim)) + (0,) + tuple(range(2 + self.convdim, mat.ndim))) mat = mat.reshape((mshp0,) + mat.shape[1:1 + self.convdim] + (mshp1,) + mat.shape[-self.convdim:]) else: mat = mat.transpose((1, 0, 2) + tuple(range(3, 3 + self.convdim))) mat = mat.reshape((mshp0, mshp1) + mat.shape[-self.convdim:]) return mat kern = correct_for_groups(kern) if self.unshared: # from (nFilters, out_rows, out_cols, nChannels, kH, kW) # to (nChannels, nFilters, out_rows, out_cols, kH, kW) axes_order = (1 + self.convdim, 0,) + tuple(range(1, 1 + self.convdim)) + \ tuple(range(2 + self.convdim, kern.ndim)) kern = kern.transpose(axes_order) if not self.filter_flip: kern = kern[(slice(None),) * (kern.ndim - self.convdim) + (slice(None, None, -1),) * self.convdim] img = self.conv(topgrad, kern, mode="full", dilation=self.filter_dilation, num_groups=self.num_groups, unshared=True, direction="backprop inputs") else: axes_order = (1, 0) + tuple(range(2, 2 + self.convdim)) kern = kern.transpose(axes_order) flip_filters = ((slice(None), slice(None)) + (slice(None, None, -1),) * self.convdim) if self.filter_flip: topgrad = topgrad[flip_filters] img = self.conv(topgrad, kern, mode="full", dilation=self.filter_dilation, num_groups=self.num_groups) if self.filter_flip: img = img[flip_filters] if any(p != (0, 0) for p in pad): img = img[(slice(None), slice(None)) + tuple(slice(pad[i][0], img.shape[i + 2] - pad[i][1]) for i in range(self.convdim))] o[0] = node.outputs[0].type.filter(img) def connection_pattern(self, node): return [[1], [1], [0]] # no connection to height, width def infer_shape(self, node, input_shapes): # We use self.imshp (that was passed when creating the Op) if possible, # or fall back to the `shape` input of the node. # TODO: when there is no subsampling, try to infer the image shape # from the shapes of inputs. kshp = input_shapes[0] topshp = input_shapes[1] imshp = self.imshp[:] if self.imshp is not None else [None] * (2 + self.convdim) if self.num_groups > 1: fallback_imshp = ([topshp[0], kshp[-self.convdim - 1] * self.num_groups] + [node.inputs[2][i] for i in range(self.convdim)]) else: fallback_imshp = ([topshp[0], kshp[-self.convdim - 1]] + [node.inputs[2][i] for i in range(self.convdim)]) imshp = [fallback_imshp[i] if imshp[i] is None else imshp[i] for i in range(2 + self.convdim)] return [imshp] class AbstractConv2d_gradInputs(AbstractConv_gradInputs): """Gradient wrt. inputs for `AbstractConv2d`. Refer to :func:`BaseAbstractConv ` for a more detailed documentation. :note: You will not want to use this directly, but rely on Theano's automatic differentiation or graph optimization to use it as needed. """ def __init__(self, imshp=None, kshp=None, border_mode="valid", subsample=(1, 1), filter_flip=True, filter_dilation=(1, 1), num_groups=1, unshared=False): super(AbstractConv2d_gradInputs, self).__init__(convdim=2, imshp=imshp, kshp=kshp, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_groups, unshared=unshared) def grad(self, inp, grads): weights, top = inp[:2] bottom, = grads d_weights = AbstractConv2d_gradWeights(self.imshp, self.kshp, self.border_mode, self.subsample, self.filter_flip, self.filter_dilation, self.num_groups, self.unshared)( bottom, top, weights.shape[-2:]) d_top = AbstractConv2d(self.imshp, self.kshp, self.border_mode, self.subsample, self.filter_flip, self.filter_dilation, self.num_groups, self.unshared)(bottom, weights) # Make sure that the broadcastable pattern of the inputs is used # for the gradients, even if the grad opts are not able to infer # that the dimensions are broadcastable. # Also make sure that the gradient lives on the same device than # the corresponding input. d_weights = patternbroadcast(d_weights, weights.broadcastable) d_weights = weights.type.filter_variable(d_weights) d_top = patternbroadcast(d_top, top.broadcastable) d_top = top.type.filter_variable(d_top) d_height_width = (theano.gradient.DisconnectedType()(),) return (d_weights, d_top) + d_height_width class AbstractConv3d_gradInputs(AbstractConv_gradInputs): """Gradient wrt. inputs for `AbstractConv3d`. Refer to :func:`BaseAbstractConv ` for a more detailed documentation. :note: You will not want to use this directly, but rely on Theano's automatic differentiation or graph optimization to use it as needed. """ def __init__(self, imshp=None, kshp=None, border_mode="valid", subsample=(1, 1, 1), filter_flip=True, filter_dilation=(1, 1, 1), num_groups=1): super(AbstractConv3d_gradInputs, self).__init__(convdim=3, imshp=imshp, kshp=kshp, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_groups) def grad(self, inp, grads): weights, top = inp[:2] bottom, = grads d_weights = AbstractConv3d_gradWeights(self.imshp, self.kshp, self.border_mode, self.subsample, self.filter_flip, self.filter_dilation, self.num_groups)(bottom, top, weights.shape[-3:]) d_top = AbstractConv3d(self.imshp, self.kshp, self.border_mode, self.subsample, self.filter_flip, self.filter_dilation, self.num_groups)(bottom, weights) # Make sure that the broadcastable pattern of the inputs is used # for the gradients, even if the grad opts are not able to infer # that the dimensions are broadcastable. # Also make sure that the gradient lives on the same device than # the corresponding input. d_weights = patternbroadcast(d_weights, weights.broadcastable) d_weights = weights.type.filter_variable(d_weights) d_top = patternbroadcast(d_top, top.broadcastable) d_top = top.type.filter_variable(d_top) d_depth_height_width = (theano.gradient.DisconnectedType()(),) return (d_weights, d_top) + d_depth_height_width Theano-1.0.0/theano/tensor/nnet/ctc.py00006000163562€ÿÿÿþ0000001777013203335532021162 0ustar boccoset00000000000000from __future__ import (division, absolute_import, print_function) import os import sys import theano.tensor as T from theano import config from theano import gof from theano.gof import local_optimizer from theano.gof.cmodule import GCC_compiler from theano.tensor.opt import register_canonicalize from theano.tensor.extra_ops import cpu_contiguous from theano.gradient import grad_undefined def _ctc_find_lib(): """ Find the directory that contains libwarpctc.so """ if config.ctc.root != '': for lib_dir in ["build", "lib", "lib64"]: lib_path = os.path.join(config.ctc.root, lib_dir) if os.path.isdir(lib_path) and os.path.exists(lib_path): lib_found = os.path.exists(os.path.join(lib_path, "libwarpctc.so")) if lib_found: return lib_path return None def _ctc_check_compile(ctc_lib_path): preambule = """ #include #include "ctc.h" """ body = """ ctcOptions options; memset(&options, 0, sizeof(ctcOptions)); options.loc = CTC_CPU; options.num_threads = 1; """ params = ['-I%s' % (os.path.dirname(__file__))] if ctc_lib_path is not None: params.extend(["-I%s" % (os.path.join(config.ctc.root, "include"))]) params.extend(["-L%s" % (ctc_lib_path)]) params.extend(["-l", "warpctc"]) compiler_res = GCC_compiler.try_flags( params, preambule=preambule, body=body, try_run=False, output=True) avail, out, err = compiler_res if isinstance(compiler_res, tuple) else (compiler_res, None, None) if not avail: return False, ("cannot compile with warp-ctc. " "We got this error:\n" + str(err)) return True, None def ctc_present(): if ctc_present.avail is not None: return ctc_present.avail ctc_lib_path = _ctc_find_lib() ctc_present.path = ctc_lib_path ctc_present.avail, ctc_present.msg = _ctc_check_compile(ctc_present.path) return ctc_present.avail ctc_present.avail = None ctc_present.msg = None ctc_present.path = None def ctc_available(): if os.name == 'nt': ctc_available.msg = 'Windows platforms are currently not supported ', 'by underlying CTC library (warp-ctc).' return False elif not ctc_present(): ctc_available.msg = ctc_present.msg return False ctc_available.path = ctc_present.path return True ctc_available.msg = None ctc_available.path = None class ConnectionistTemporalClassification(gof.COp, gof.OpenMPOp): """ CTC loss function wrapper. Notes ----- Using the wrapper requires that Baidu's warp-ctc library is installed. If the warp-ctc library is not on your compiler's default library path, you must set the configuration variable ``config.ctc.root`` appropriately. Parameters ---------- compute_grad If set to True, enables the computation of gradients of the CTC loss function. """ __props__ = ('compute_grad',) _cop_num_inputs = 3 _cop_num_outputs = 2 func_file = os.path.join('c_code', 'ctc_wrapper.c') func_name = "APPLY_SPECIFIC(ctc_cost_cpu)" def __init__(self, compute_grad=True, openmp=None): if not ctc_available(): raise RuntimeError('Baidu CTC is not available and ' 'ConnectionistTemporalClassification Op ' 'can not be constructed.') gof.COp.__init__(self, self.func_file, self.func_name) gof.OpenMPOp.__init__(self, openmp=openmp) self.compute_grad = compute_grad # Return only the cost. Gradient will be returned by grad() self.default_output = 0 def c_lib_dirs(self): lib_dirs = [] if ctc_available.path is not None: lib_dirs += [ctc_available.path] return lib_dirs def c_compile_args(self): if ctc_available.path is not None: if sys.platform != 'darwin' and ' ' in ctc_available.path: return ['-Wl,-rpath,"' + ctc_available.path + '"'] else: return ['-Wl,-rpath,' + ctc_available.path] return [] def c_libraries(self): return ["warpctc"] def c_header_dirs(self): header_dirs = [] if config.ctc.root != '': # We assume here that the header is available at the include directory # of the CTC root directory. header_dirs += [os.path.join(config.ctc.root, "include")] return header_dirs def c_headers(self): return ["ctc.h"] + gof.OpenMPOp.c_headers(self) def make_node(self, activations, labels, input_lengths): t_activations = T.as_tensor_variable(activations) # Ensure activations array is C-contiguous t_activations = cpu_contiguous(t_activations) t_labels = T.as_tensor_variable(labels) t_input_lengths = T.as_tensor_variable(input_lengths) if t_activations.type.dtype != 'float32': raise TypeError('activations must use the float32 type!') if t_activations.ndim != 3: raise ValueError('activations must have 3 dimensions.') if t_labels.type.dtype != 'int32': raise TypeError('labels must use the int32 type!') if t_labels.ndim != 2: raise ValueError('labels must have 2 dimensions.') if t_input_lengths.type.dtype != 'int32': raise TypeError('input_lengths must use the int32 type!') if t_input_lengths.ndim != 1: raise ValueError('input_lengths must have 1 dimension.') costs = T.fvector(name="ctc_cost") outputs = [costs] if self.compute_grad: gradients = T.ftensor3(name="ctc_grad") outputs += [gradients] return gof.Apply(self, inputs=[t_activations, t_labels, t_input_lengths], outputs=outputs) def L_op(self, inputs, outputs, output_grads): assert self.compute_grad and len(outputs) == 2 gradients = outputs[1] assert gradients is not None grad_op = output_grads[0] total_grad = T.basic.batched_dot(grad_op, gradients.dimshuffle(1, 0, 2)).dimshuffle(1, 0, 2) return [total_grad, grad_undefined(self, 1, inputs[1]), grad_undefined(self, 2, inputs[2])] def ctc(activations, labels, input_lengths): """ Compute CTC loss function. Notes ----- Using the loss function requires that the Baidu's warp-ctc library be installed. If the warp-ctc library is not on the compiler's default library path, the configuration variable ``config.ctc.root`` must be properly set. Parameters ---------- activations Three-dimensional tensor, which has a shape of (t, m, p), where t is the time index, m is the minibatch index, and p is the index over the probabilities of each symbol in the alphabet. The memory layout is assumed to be in C-order, which consists in the slowest to the fastest changing dimension, from left to right. In this case, p is the fastest changing dimension. labels A 2-D tensor of all the labels for the minibatch. In each row, there is a sequence of target labels. Negative values are assumed to be padding, and thus are ignored. Blank symbol is assumed to have index 0 in the alphabet. input_lengths A 1-D tensor with the number of time steps for each sequence in the minibatch. Returns ------- 1-D array Cost of each example in the minibatch. """ return ConnectionistTemporalClassification()(activations, labels, input_lengths) # Disable gradient computation if not needed @register_canonicalize('fast_compile') @local_optimizer([ConnectionistTemporalClassification]) def local_ctc_no_grad(node): if isinstance(node.op, ConnectionistTemporalClassification): if len(node.outputs) > 1: if len(node.outputs[1].clients) == 0: # gradient is not used return [ConnectionistTemporalClassification(compute_grad=False)(*node.inputs), None] return False Theano-1.0.0/theano/tensor/nnet/neighbours.py00006000163562€ÿÿÿþ0000010443413203335532022550 0ustar boccoset00000000000000""" TODO: implement Images2Neibs.infer_shape() methods """ from __future__ import absolute_import, print_function, division import numpy as np import theano from theano import Op, Apply from theano.gof import EnumList import theano.tensor as T from theano.gradient import grad_not_implemented from theano.gradient import grad_undefined class Images2Neibs(Op): """ Reshapes the input as a 2D tensor where each row is an pooling example. Parameters ---------- mode : {'valid', 'ignore_borders', 'wrap_centered'} - 'valid' : Requires an input that is a multiple of the pooling factor (in each direction). - 'half' : Equivalent to 'valid' if we pre-pad with zeros the input on each side by (neib_shape[0]//2, neib_shape[1]//2) - 'full' : Equivalent to 'valid' if we pre-pad with zeros the input on each side by (neib_shape[0] - 1, neib_shape[1] - 1) - 'ignore_borders' : Same as valid, but will ignore the borders if the shape(s) of the input is not a multiple of the pooling factor(s). - 'wrap_centered' : ?? TODO comment """ __props__ = ("mode",) BORDER_MODE = EnumList(('MODE_VALID', 'valid'), ('MODE_HALF', 'half'), ('MODE_FULL', 'full'), ('MODE_WRAP_CENTERED', 'wrap_centered'), ('MODE_IGNORE_BORDERS', 'ignore_borders')) params_type = BORDER_MODE def get_params(self, node): return self.mode def __init__(self, mode='valid'): implemented_modes = self.BORDER_MODE.get_aliases() if mode not in implemented_modes: raise NotImplementedError("Only modes %s have been implemented for %s" % (', '.join(implemented_modes), type(self).__name__)) self.mode = mode def __str__(self): return self.__class__.__name__ + "{%s}" % self.mode def __setstate__(self, d): self.__dict__.update(d) if not hasattr(self, "mode"): self.mode = 'valid' def make_node(self, ten4, neib_shape, neib_step=None): """ Parameters ---------- ten4 : a list of lists of images ten4 is of shape (list 1 dim, list 2 dim, row, col). neib_shape (r,c) where r is the height of the neighborhood in rows and c is the width of the neighborhood in columns. neib_step (dr,dc) where dr is the number of rows to skip between patch and dc is the number of columns. When None, this is the same as neib_shape (patch are disjoint). Returns ------- matrix A 2D matrix, written using the following pattern:: idx = 0 for i in xrange(list 1 dim) for j in xrange(list 2 dim) for k in for l in output[idx,:] = flattened version of ten4[i,j,l:l+r,k:k+c] idx += 1 .. note:: The op isn't necessarily implemented internally with these for loops, they're just the easiest way to describe the output pattern. """ ten4 = T.as_tensor_variable(ten4) neib_shape = T.as_tensor_variable(neib_shape) if neib_step is None: neib_step = neib_shape else: neib_step = T.as_tensor_variable(neib_step) assert ten4.ndim == 4 assert neib_shape.ndim == 1 assert neib_step.ndim == 1 return Apply(self, [ten4, neib_shape, neib_step], [T.matrix(dtype=ten4.type.dtype)]) def grad(self, inp, grads): x, neib_shape, neib_step = inp gz, = grads if self.mode in ['valid', 'ignore_borders']: if (neib_shape is neib_step or neib_shape == neib_step or # Theano Constant == do not compare the data # the equals function do that. (hasattr(neib_shape, "equals") and neib_shape.equals(neib_step))): return [neibs2images(gz, neib_shape, x.shape, mode=self.mode), grad_undefined(self, 1, neib_shape), grad_undefined(self, 2, neib_step)] if self.mode in ['valid']: # Iterate over neighborhood positions, summing contributions. def pos2map(pidx, pgz, prior_result, neib_shape, neib_step): ''' Helper function that adds gradient contribution from a single neighborhood position i,j. pidx = Index of position within neighborhood. pgz = Gradient of shape (batch_size*num_channels*neibs) prior_result = Shape (batch_size, num_channnels, rows, cols) neib_shape = Number of rows, cols in a neighborhood. neib_step = Step sizes from image2neibs. ''' nrows, ncols = neib_shape rstep, cstep = neib_step batch_size, num_channels, rows, cols = prior_result.shape i = pidx // ncols j = pidx - (i * ncols) # This position does not touch some img pixels in valid mode. result_indices = prior_result[:, :, i:(rows - nrows + i + 1):rstep, j:(cols - ncols + j + 1):cstep] newshape = (batch_size, num_channels) + \ ((rows - nrows) // rstep + 1,) + \ ((cols - ncols) // cstep + 1,) return T.inc_subtensor(result_indices, pgz.reshape(newshape)) indices = T.arange(neib_shape[0] * neib_shape[1]) pgzs = gz.dimshuffle((1, 0)) result, _ = theano.scan(fn=pos2map, sequences=[indices, pgzs], outputs_info=T.zeros(x.shape), non_sequences=[neib_shape, neib_step]) grad_input = result[-1] return [grad_input, grad_undefined(self, 1, neib_shape), grad_undefined(self, 2, neib_step)] return [grad_not_implemented(self, 0, x), grad_undefined(self, 1, neib_shape), grad_undefined(self, 2, neib_step)] def c_code_cache_version(self): return (10,) def perform(self, node, inp, out_, params): ten4, neib_shape, neib_step = inp z, = out_ # GpuImages2Neibs should not run this perform in DebugMode if type(self) != Images2Neibs: raise theano.gof.utils.MethodNotDefined() def CEIL_INTDIV(a, b): if a % b: return (a // b) + 1 else: return a // b grid_c = -1 # number of patch in height grid_d = -1 # number of patch in width assert ten4.ndim == 4 assert neib_shape.ndim == 1 assert neib_shape.shape[0] == 2 assert neib_step.ndim == 1 assert neib_step.shape[0] == 2 c, d = neib_shape step_x, step_y = neib_step mode = self.mode if step_x <= 0 or step_y <= 0: raise ValueError( "neib_step wrong step ; values <= 0. Got " + str(neib_step)) if c <= 0 or d <= 0: raise ValueError( "neib_shape values <=0. Got " + str(neib_shape)) if mode == "wrap_centered": if (c % 2 != 1) or (d % 2 != 1): raise TypeError( "Images2Neibs:" " in mode wrap_centered need patch with odd shapes") if (ten4.shape[2] < c) or (ten4.shape[3] < d): raise TypeError( "Images2Neibs: in wrap_centered mode, don't support" " image shapes smaller then the patch shapes:" " neib_shape=(%d,%d), ten4[2:]=[%d,%d]" % (c, d, ten4.shape[2], ten4.shape[3])) grid_c = CEIL_INTDIV(ten4.shape[2], step_x) grid_d = CEIL_INTDIV(ten4.shape[3], step_y) elif mode == "valid": if (ten4.shape[2] < c) or (((ten4.shape[2] - c) % step_x) != 0): raise TypeError( "neib_shape[0]=%d, neib_step[0]=%d and" " ten4.shape[2]=%d not consistent" % (c, step_x, ten4.shape[2])) if (ten4.shape[3] < d) or (((ten4.shape[3] - d) % step_y) != 0): raise TypeError( "neib_shape[1]=%d, neib_step[1]=%d and" " ten4.shape[3]=%d not consistent" % (d, step_y, ten4.shape[3])) # number of patch in height grid_c = 1 + ((ten4.shape[2] - c) // step_x) # number of patch in width grid_d = 1 + ((ten4.shape[3] - d) // step_y) elif mode == "ignore_borders": # number of patch in height grid_c = 1 + ((ten4.shape[2] - c) // step_x) # number of patch in width grid_d = 1 + ((ten4.shape[3] - d) // step_y) elif mode == "half": # This is equivalent to 'valid' with padding (c // 2, d // 2) on both sides # Thus the expanded image will have size (h + 2 * (c // 2), w + 2 * (d // 2)) # Plugging these in the equation for 'valid' we get # h + 2 * (c // 2) - c = h - (c % 2) # w + 2 * (d // 2) - c = w - (d % 2) if (ten4.shape[2] < c) or (((ten4.shape[2] - (c % 2)) % step_x) != 0): raise TypeError( "neib_shape[0]=%d, neib_step[0]=%d and" " ten4.shape[2]=%d not consistent" % (c, step_x, ten4.shape[2])) if (ten4.shape[3] < d) or (((ten4.shape[3] - (d % 2)) % step_y) != 0): raise TypeError( "neib_shape[1]=%d, neib_step[1]=%d and" " ten4.shape[3]=%d not consistent" % (d, step_y, ten4.shape[3])) # number of patch in height grid_c = 1 + ((ten4.shape[2] - (c % 2)) // step_x) # number of patch in width grid_d = 1 + ((ten4.shape[3] - (d % 2)) // step_y) elif mode == "full": # This is equivalent to 'valid' with padding (c - 1, d - 1) on both sides # Thus the expanded image will have size (h + 2 * (c - 1), w + 2 * (d - 1)) # Plugging these in the equation for 'valid' we get # h + 2 * (c - 1) - c = h + c - 2 # w + 2 * (d - 1) - c = w + d - 2 if (ten4.shape[2] < c) or (((ten4.shape[2] + c - 2) % step_x) != 0): raise TypeError( "neib_shape[0]=%d, neib_step[0]=%d and" " ten4.shape[2]=%d not consistent" % (c, step_x, ten4.shape[2])) if (ten4.shape[3] < d) or (((ten4.shape[3] + d - 2) % step_y) != 0): raise TypeError( "neib_shape[1]=%d, neib_step[1]=%d and" " ten4.shape[3]=%d not consistent" % (d, step_y, ten4.shape[3])) # number of patch in height grid_c = 1 + ((ten4.shape[2] + c - 2) // step_x) # number of patch in width grid_d = 1 + ((ten4.shape[3] + d - 2) // step_y) else: raise TypeError("Images2Neibs: unknow mode '%s'" % mode) z_dim0 = grid_c * grid_d * ten4.shape[1] * ten4.shape[0] z_dim1 = c * d z[0] = np.empty((z_dim0, z_dim1), dtype=node.outputs[0].dtype) nb_batch = ten4.shape[0] nb_stack = ten4.shape[1] height = ten4.shape[2] width = ten4.shape[3] wrap_centered_half_idx_shift_x = c // 2 wrap_centered_half_idx_shift_y = d // 2 for n in range(nb_batch): for s in range(nb_stack): # loop over the number of patch in height for a in range(grid_c): # loop over the number of patch in width for b in range(grid_d): z_row = b + grid_d * (a + grid_c * (s + nb_stack * n)) for i in range(c): ten4_2 = i + a * step_x if mode == "wrap_centered": ten4_2 -= wrap_centered_half_idx_shift_x if ten4_2 < 0: ten4_2 += height elif ten4_2 >= height: ten4_2 -= height elif mode == "half": ten4_2 -= wrap_centered_half_idx_shift_x elif mode == "full": ten4_2 -= c - 1 if ten4_2 < 0 or ten4_2 >= height: z[0][z_row, d * i: d * i + d] = 0 else: for j in range(d): ten4_3 = j + b * step_y if mode == "wrap_centered": ten4_3 -= wrap_centered_half_idx_shift_y if ten4_3 < 0: ten4_3 += width elif ten4_3 >= width: ten4_3 -= width elif mode == "half": ten4_3 -= wrap_centered_half_idx_shift_y elif mode == "full": ten4_3 -= d - 1 z_col = j + d * i if ten4_3 < 0 or ten4_3 >= width: z[0][z_row, z_col] = 0 else: z[0][z_row, z_col] = ten4[n, s, ten4_2, ten4_3] def infer_shape(self, node, input_shape): in_shape = input_shape[0] c, d = node.inputs[1] step_x, step_y = node.inputs[2] if self.mode == 'wrap_centered': grid_c = T.ceil_intdiv(in_shape[2], step_x) grid_d = T.ceil_intdiv(in_shape[3], step_y) elif self.mode == 'valid': grid_c = 1 + ((in_shape[2] - c) // step_x) grid_d = 1 + ((in_shape[3] - d) // step_y) elif self.mode == 'ignore_borders': grid_c = 1 + ((in_shape[2] - c) // step_x) grid_d = 1 + ((in_shape[3] - d) // step_y) elif self.mode == 'half': grid_c = 1 + ((in_shape[2] - (c % 2)) // step_x) grid_d = 1 + ((in_shape[3] - (d % 2)) // step_y) elif self.mode == 'full': grid_c = 1 + ((in_shape[2] + c - 2) // step_x) grid_d = 1 + ((in_shape[3] + d - 2) // step_y) else: raise TypeError("Images2Neibs: unknow mode '%s'" % self.mode) z_dim0 = grid_c * grid_d * in_shape[1] * in_shape[0] z_dim1 = c * d return [(z_dim0, z_dim1)] def c_code(self, node, name, inp, out, sub): return """ #ifndef CEIL_INTDIV #define CEIL_INTDIV(a, b) ((a/b) + ((a %% b) ? 1: 0)) #endif int grid_c = -1; //number of patch in height int grid_d = -1; //number of patch in width { if (PyArray_NDIM(%(ten4)s) != 4) { PyErr_Format(PyExc_TypeError, "ten4 wrong rank"); %(fail)s; } if (PyArray_NDIM(%(neib_shape)s) != 1) { PyErr_Format(PyExc_TypeError, "neib_shape wrong rank"); %(fail)s; } if ( (PyArray_DIMS(%(neib_shape)s))[0] != 2) { PyErr_Format(PyExc_TypeError, "neib_shape wrong shape ; has to" " contain 2 elements"); %(fail)s; } if (PyArray_NDIM(%(neib_step)s) != 1) { PyErr_Format(PyExc_TypeError, "neib_step wrong rank"); %(fail)s; } if ( (PyArray_DIMS(%(neib_step)s))[0] != 2) { PyErr_Format(PyExc_TypeError, "neib_step wrong step ; has to contain 2 elements"); %(fail)s; } // (c,d) = neib_shape const npy_intp c = (npy_intp) *(dtype_%(neib_shape)s*) PyArray_GETPTR1(%(neib_shape)s, 0); const npy_intp d = (npy_intp) *(dtype_%(neib_shape)s*) PyArray_GETPTR1(%(neib_shape)s, 1); // (step_x,step_y) = neib_step const dtype_%(neib_step)s step_x = *(dtype_%(neib_step)s*) PyArray_GETPTR1(%(neib_step)s, 0); const dtype_%(neib_step)s step_y = *(dtype_%(neib_step)s*) PyArray_GETPTR1(%(neib_step)s, 1); if (step_x <=0 || step_y <=0) { PyErr_Format(PyExc_ValueError, "neib_step wrong step ; values <= 0. Got %%lld %%lld.", (long long) step_x, (long long) step_y); %(fail)s; } if (c <=0 || d <=0) { PyErr_Format(PyExc_ValueError, "neib_shape values <= 0. Got %%lld %%lld.", (long long)c, (long long)d); %(fail)s; } if (%(mode)s == MODE_WRAP_CENTERED) { if (c%%2!=1 || d%%2!=1){ PyErr_Format(PyExc_TypeError, "Images2Neibs: in mode wrap_centered" " need patch with odd shapes"); %(fail)s; } if ( (PyArray_DIMS(%(ten4)s))[2] < c || (PyArray_DIMS(%(ten4)s))[3] < d) { PyErr_Format(PyExc_TypeError, "Images2Neibs: in wrap_centered mode, don't support image" " shapes smaller then the patch shapes:" " neib_shape=(%%ld,%%ld), ten4[2:]=[%%ld,%%ld]", (long int)c, (long int)d, (long int)(PyArray_DIMS(%(ten4)s)[2]), (long int)(PyArray_DIMS(%(ten4)s)[3])); %(fail)s; } grid_c = CEIL_INTDIV(((PyArray_DIMS(%(ten4)s))[2]),step_x); grid_d = CEIL_INTDIV(((PyArray_DIMS(%(ten4)s))[3]),step_y); } else if (%(mode)s == MODE_VALID) { if ( ((PyArray_DIMS(%(ten4)s))[2] < c) || ( (((PyArray_DIMS(%(ten4)s))[2]-c) %% step_x)!=0)) { PyErr_Format(PyExc_TypeError, "neib_shape[0]=%%ld, neib_step[0]=%%ld and" " ten4.shape[2]=%%ld not consistent", (long int)c, (long int)step_x, (long int)(PyArray_DIMS(%(ten4)s)[2])); %(fail)s; } if ( ((PyArray_DIMS(%(ten4)s))[3] < d) || ( (((PyArray_DIMS(%(ten4)s))[3]-d) %% step_y)!=0)) { PyErr_Format(PyExc_TypeError, "neib_shape[1]=%%ld, neib_step[1]=%%ld and" " ten4.shape[3]=%%ld not consistent", (long int)d, (long int)step_y, (long int)(PyArray_DIMS(%(ten4)s)[3])); %(fail)s; } //number of patch in height grid_c = 1+(((PyArray_DIMS(%(ten4)s))[2]-c)/step_x); //number of patch in width grid_d = 1+(((PyArray_DIMS(%(ten4)s))[3]-d)/step_y); } else if (%(mode)s == MODE_IGNORE_BORDERS) { //number of patch in height grid_c = 1+(((PyArray_DIMS(%(ten4)s))[2]-c)/step_x); //number of patch in width grid_d = 1+(((PyArray_DIMS(%(ten4)s))[3]-d)/step_y); } else if (%(mode)s == MODE_HALF) { if ( ((PyArray_DIMS(%(ten4)s))[2] < c) || ( (((PyArray_DIMS(%(ten4)s))[2]-(c%%2)) %% step_x)!=0)) { PyErr_Format(PyExc_TypeError, "neib_shape[0]=%%ld, neib_step[0]=%%ld and" " ten4.shape[2]=%%ld not consistent", (long int)c, (long int)step_x, (long int)(PyArray_DIMS(%(ten4)s)[2])); %(fail)s; } if ( ((PyArray_DIMS(%(ten4)s))[3] < d) || ( (((PyArray_DIMS(%(ten4)s))[3]-(d%%2)) %% step_y)!=0)) { PyErr_Format(PyExc_TypeError, "neib_shape[1]=%%ld, neib_step[1]=%%ld and" " ten4.shape[3]=%%ld not consistent", (long int)d, (long int)step_y, (long int)(PyArray_DIMS(%(ten4)s)[3])); %(fail)s; } //number of patch in height grid_c = 1+(((PyArray_DIMS(%(ten4)s))[2]-(c%%2))/step_x); //number of patch in width grid_d = 1+(((PyArray_DIMS(%(ten4)s))[3]-(d%%2))/step_y); } else if (%(mode)s == MODE_FULL) { if ( ((PyArray_DIMS(%(ten4)s))[2] < c) || ( (((PyArray_DIMS(%(ten4)s))[2]+c-2) %% step_x)!=0)) { PyErr_Format(PyExc_TypeError, "neib_shape[0]=%%ld, neib_step[0]=%%ld and" " ten4.shape[2]=%%ld not consistent", (long int)c, (long int)step_x, (long int)(PyArray_DIMS(%(ten4)s)[2])); %(fail)s; } if ( ((PyArray_DIMS(%(ten4)s))[3] < d) || ( (((PyArray_DIMS(%(ten4)s))[3]+d-2) %% step_y)!=0)) { PyErr_Format(PyExc_TypeError, "neib_shape[1]=%%ld, neib_step[1]=%%ld and" " ten4.shape[3]=%%ld not consistent", (long int)d, (long int)step_y, (long int)(PyArray_DIMS(%(ten4)s)[3])); %(fail)s; } //number of patch in height grid_c = 1+(((PyArray_DIMS(%(ten4)s))[2]+c-2)/step_x); //number of patch in width grid_d = 1+(((PyArray_DIMS(%(ten4)s))[3]+d-2)/step_y); } else { PyErr_Format(PyExc_TypeError, "Images2Neibs: unknow mode %%d", %(mode)s); %(fail)s; } // new dimensions for z const npy_intp z_dim1 = c * d; const npy_intp z_dim0 = grid_c * grid_d * (PyArray_DIMS(%(ten4)s))[1] * (PyArray_DIMS(%(ten4)s))[0]; if ((NULL == %(z)s) || ((PyArray_DIMS(%(z)s))[0] != z_dim0 ) || ((PyArray_DIMS(%(z)s))[1] != z_dim1 ) ) { Py_XDECREF(%(z)s); npy_intp dims[2]; dims[0] = z_dim0; dims[1] = z_dim1; %(z)s = (PyArrayObject*) PyArray_EMPTY(2, dims, PyArray_TYPE((PyArrayObject*) py_%(ten4)s), 0); if (!%(z)s) { PyErr_SetString(PyExc_MemoryError, "failed to alloc z output"); %(fail)s; } } } { // NESTED SCOPE const int nb_batch = (PyArray_DIMS(%(ten4)s))[0]; const int nb_stack = (PyArray_DIMS(%(ten4)s))[1]; const int height = (PyArray_DIMS(%(ten4)s))[2]; const int width = (PyArray_DIMS(%(ten4)s))[3]; // (c,d) = neib_shape const npy_intp c = (npy_intp) *(dtype_%(neib_shape)s*) PyArray_GETPTR1(%(neib_shape)s, 0); const npy_intp d = (npy_intp) *(dtype_%(neib_shape)s*) PyArray_GETPTR1(%(neib_shape)s, 1); // (step_x,step_y) = neib_step const npy_intp step_x = (npy_intp) *(dtype_%(neib_step)s*) PyArray_GETPTR1(%(neib_step)s, 0); const npy_intp step_y = (npy_intp) *(dtype_%(neib_step)s*) PyArray_GETPTR1(%(neib_step)s, 1); const int wrap_centered_half_idx_shift_x = c/2; const int wrap_centered_half_idx_shift_y = d/2; // Oh this is messed up... for (int n = 0; n < nb_batch; n++) // loop over batches for (int s = 0; s < nb_stack; s++) // loop over stacks for (int a = 0; a < grid_c; a++) // loop over the number of patch in height for (int b = 0; b < grid_d; b++) // loop over the number of patch in width { int z_row = b + grid_d*(a + grid_c*(s + nb_stack*n)); for (int i = 0; i < c; i++) // loop over c { int ten4_2 = i + a * step_x; if (%(mode)s == MODE_WRAP_CENTERED) { ten4_2 -= wrap_centered_half_idx_shift_x; if ( ten4_2 < 0 ) ten4_2 += height; else if (ten4_2 >= height) ten4_2 -= height; } else if (%(mode)s == MODE_HALF) { ten4_2 -= wrap_centered_half_idx_shift_x; } else if (%(mode)s == MODE_FULL) { ten4_2 -= c - 1; } if (ten4_2 < 0 | ten4_2 >= height) { dtype_%(z)s* curr_z = (dtype_%(z)s*) PyArray_GETPTR2(%(z)s, z_row, d * i); memset(curr_z, 0, d*sizeof(*curr_z)); } else { for (int j = 0; j < d; j++) // loop over d { int ten4_3 = j + b * step_y; if (%(mode)s == MODE_WRAP_CENTERED) { ten4_3 -= wrap_centered_half_idx_shift_y; if ( ten4_3 < 0 ) ten4_3 += width; else if (ten4_3 >= width) ten4_3 -= width; } else if (%(mode)s == MODE_HALF) { ten4_3 -= wrap_centered_half_idx_shift_y; } else if (%(mode)s == MODE_FULL) { ten4_3 -= d - 1; } int z_col = j + d * i; dtype_%(z)s* curr_z = (dtype_%(z)s*) PyArray_GETPTR2(%(z)s, z_row, z_col); if (ten4_3 < 0 | ten4_3 >= width) { *curr_z = 0; } else { *curr_z = *( (dtype_%(ten4)s*) PyArray_GETPTR4(%(ten4)s, n, s, ten4_2, ten4_3)); } } } } } } // END NESTED SCOPE """ % dict(ten4=inp[0], neib_shape=inp[1], neib_step=inp[2], z=out[0], fail=sub['fail'], mode=sub['params']) def images2neibs(ten4, neib_shape, neib_step=None, mode='valid'): """ Function :func:`images2neibs ` allows to apply a sliding window operation to a tensor containing images or other two-dimensional objects. The sliding window operation loops over points in input data and stores a rectangular neighbourhood of each point. It is possible to assign a step of selecting patches (parameter `neib_step`). Parameters ---------- ten4 : A 4d tensor-like A 4-dimensional tensor which represents a list of lists of images. It should have shape (list 1 dim, list 2 dim, row, col). The first two dimensions can be useful to store different channels and batches. neib_shape : A 1d tensor-like of 2 values A tuple containing two values: height and width of the neighbourhood. It should have shape (r,c) where r is the height of the neighborhood in rows and c is the width of the neighborhood in columns. neib_step : A 1d tensor-like of 2 values (dr,dc) where dr is the number of rows to skip between patch and dc is the number of columns. The parameter should be a tuple of two elements: number of rows and number of columns to skip each iteration. Basically, when the step is 1, the neighbourhood of every first element is taken and every possible rectangular subset is returned. By default it is equal to `neib_shape` in other words, the patches are disjoint. When the step is greater than `neib_shape`, some elements are omitted. When None, this is the same as neib_shape (patch are disjoint). mode : {'valid', 'ignore_borders', 'wrap_centered', 'half'} ``valid`` Requires an input that is a multiple of the pooling factor (in each direction). ``half`` Equivalent to 'valid' if we pre-pad with zeros the input on each side by (neib_shape[0]//2, neib_shape[1]//2) ``full`` Equivalent to 'valid' if we pre-pad with zeros the input on each side by (neib_shape[0] - 1, neib_shape[1] - 1) ``ignore_borders`` Same as valid, but will ignore the borders if the shape(s) of the input is not a multiple of the pooling factor(s). ``wrap_centered`` ?? TODO comment Returns ------- object Reshapes the input as a 2D tensor where each row is an pooling example. Pseudo-code of the output: .. code-block:: python idx = 0 for i in xrange(list 1 dim): for j in xrange(list 2 dim): for k in : for l in : output[idx,:] = flattened version of ten4[i,j,l:l+r,k:k+c] idx += 1 .. note:: The operation isn't necessarily implemented internally with these for loops, they're just the easiest way to describe the output pattern. Notes ----- .. note:: Currently the step size should be chosen in the way that the corresponding dimension :math:`i` (width or height) is equal to :math:`n * step\_size_i + neib\_shape_i` for some :math:`n`. Examples -------- .. code-block:: python # Defining variables images = T.tensor4('images') neibs = images2neibs(images, neib_shape=(5, 5)) # Constructing theano function window_function = theano.function([images], neibs) # Input tensor (one image 10x10) im_val = np.arange(100.).reshape((1, 1, 10, 10)) # Function application neibs_val = window_function(im_val) .. note:: The underlying code will construct a 2D tensor of disjoint patches 5x5. The output has shape 4x25. """ return Images2Neibs(mode)(ten4, neib_shape, neib_step) def neibs2images(neibs, neib_shape, original_shape, mode='valid'): """ Function :func:`neibs2images ` performs the inverse operation of :func:`images2neibs `. It inputs the output of :func:`images2neibs ` and reconstructs its input. Parameters ---------- neibs : 2d tensor Like the one obtained by :func:`images2neibs `. neib_shape `neib_shape` that was used in :func:`images2neibs `. original_shape Original shape of the 4d tensor given to :func:`images2neibs ` Returns ------- object Reconstructs the input of :func:`images2neibs `, a 4d tensor of shape `original_shape`. Notes ----- Currently, the function doesn't support tensors created with `neib_step` different from default value. This means that it may be impossible to compute the gradient of a variable gained by :func:`images2neibs ` w.r.t. its inputs in this case, because it uses :func:`images2neibs ` for gradient computation. Examples -------- Example, which uses a tensor gained in example for :func:`images2neibs `: .. code-block:: python im_new = neibs2images(neibs, (5, 5), im_val.shape) # Theano function definition inv_window = theano.function([neibs], im_new) # Function application im_new_val = inv_window(neibs_val) .. note:: The code will output the initial image array. """ neibs = T.as_tensor_variable(neibs) neib_shape = T.as_tensor_variable(neib_shape) original_shape = T.as_tensor_variable(original_shape) new_neib_shape = T.stack([original_shape[-1] // neib_shape[1], neib_shape[1]]) output_2d = images2neibs(neibs.dimshuffle('x', 'x', 0, 1), new_neib_shape, mode=mode) if mode == 'ignore_borders': # We use set_subtensor to accept original_shape we can't infer # the shape and still raise error when it don't have the right # shape. valid_shape = original_shape valid_shape = T.set_subtensor( valid_shape[2], (valid_shape[2] // neib_shape[0]) * neib_shape[0]) valid_shape = T.set_subtensor( valid_shape[3], (valid_shape[3] // neib_shape[1]) * neib_shape[1]) output_4d = output_2d.reshape(valid_shape, ndim=4) # padding the borders with zeros for d in [2, 3]: pad_shape = list(output_4d.shape) pad_shape[d] = original_shape[d] - valid_shape[d] output_4d = T.concatenate([output_4d, T.zeros(pad_shape)], axis=d) elif mode == 'valid': # TODO: we do not implement all mode with this code. # Add a check for the good cases. output_4d = output_2d.reshape(original_shape, ndim=4) else: raise NotImplementedError("neibs2images do not support mode=%s" % mode) return output_4d Theano-1.0.0/theano/tensor/nnet/__init__.py00006000163562€ÿÿÿþ0000002662213203335532022144 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division from .nnet import ( CrossentropyCategorical1Hot, CrossentropyCategorical1HotGrad, CrossentropySoftmax1HotWithBiasDx, CrossentropySoftmaxArgmax1HotWithBias, LogSoftmax, Prepend_scalar_constant_to_each_row, Prepend_scalar_to_each_row, Softmax, SoftmaxGrad, SoftmaxWithBias, binary_crossentropy, sigmoid_binary_crossentropy, categorical_crossentropy, crossentropy_categorical_1hot, crossentropy_categorical_1hot_grad, crossentropy_softmax_1hot, crossentropy_softmax_1hot_with_bias, crossentropy_softmax_1hot_with_bias_dx, crossentropy_softmax_argmax_1hot_with_bias, crossentropy_softmax_max_and_argmax_1hot, crossentropy_softmax_max_and_argmax_1hot_with_bias, crossentropy_to_crossentropy_with_softmax, crossentropy_to_crossentropy_with_softmax_with_bias, graph_merge_softmax_with_crossentropy_softmax, h_softmax, logsoftmax, logsoftmax_op, prepend_0_to_each_row, prepend_1_to_each_row, prepend_scalar_to_each_row, relu, softmax, softmax_grad, softmax_graph, softmax_op, softmax_simplifier, softmax_with_bias, elu, selu, confusion_matrix, softsign) from . import opt from .conv import ConvOp from .sigm import (softplus, sigmoid, sigmoid_inplace, scalar_sigmoid, ultra_fast_sigmoid, hard_sigmoid) from .bn import batch_normalization import warnings from .abstract_conv import conv2d as abstract_conv2d from .abstract_conv import conv2d_grad_wrt_inputs from .abstract_conv import conv3d from .abstract_conv import separable_conv2d def conv2d(input, filters, input_shape=None, filter_shape=None, border_mode='valid', subsample=(1, 1), filter_flip=True, image_shape=None, filter_dilation=(1, 1), num_groups=1, unshared=False, **kwargs): """ This function will build the symbolic graph for convolving a mini-batch of a stack of 2D inputs with a set of 2D filters. The implementation is modelled after Convolutional Neural Networks (CNN). Parameters ---------- input: symbolic 4D tensor Mini-batch of feature map stacks, of shape (batch size, input channels, input rows, input columns). See the optional parameter ``input_shape``. filters: symbolic 4D or 6D tensor Set of filters used in CNN layer of shape (output channels, input channels, filter rows, filter columns) for normal convolution and (output channels, output rows, output columns, input channels, filter rows, filter columns) for unshared convolution. See the optional parameter ``filter_shape``. input_shape: None, tuple/list of len 4 or 6 of int or Constant variable The shape of the input parameter. Optional, possibly used to choose an optimal implementation. You can give ``None`` for any element of the list to specify that this element is not known at compile time. filter_shape: None, tuple/list of len 4 or 6 of int or Constant variable The shape of the filters parameter. Optional, possibly used to choose an optimal implementation. You can give ``None`` for any element of the list to specify that this element is not known at compile time. border_mode: str, int or a tuple of two ints or pairs of ints Either of the following: ``'valid'``: apply filter wherever it completely overlaps with the input. Generates output of shape: input shape - filter shape + 1 ``'full'``: apply filter wherever it partly overlaps with the input. Generates output of shape: input shape + filter shape - 1 ``'half'``: pad input with a symmetric border of ``filter rows // 2`` rows and ``filter columns // 2`` columns, then perform a valid convolution. For filters with an odd number of rows and columns, this leads to the output shape being equal to the input shape. ``int``: pad input with a symmetric border of zeros of the given width, then perform a valid convolution. ``(int1, int2)``: (for 2D) pad input with a symmetric border of ``int1``, ``int2``, then perform a valid convolution. ``(int1, (int2, int3))`` or ``((int1, int2), int3)``: (for 2D) pad input with one symmetric border of `int1`` or ``int3``, and one asymmetric border of ``(int2, int3)`` or ``(int1, int2)``. subsample: tuple of len 2 Factor by which to subsample the output. Also called strides elsewhere. filter_flip: bool If ``True``, will flip the filter rows and columns before sliding them over the input. This operation is normally referred to as a convolution, and this is the default. If ``False``, the filters are not flipped and the operation is referred to as a cross-correlation. image_shape: None, tuple/list of len 4 of int or Constant variable Deprecated alias for input_shape. filter_dilation: tuple of len 2 Factor by which to subsample (stride) the input. Also called dilation elsewhere. num_groups : int Divides the image, kernel and output tensors into num_groups separate groups. Each which carry out convolutions separately unshared: bool If true, then unshared or 'locally connected' convolution will be performed. A different filter will be used for each region of the input. kwargs: Any other keyword arguments are accepted for backwards compatibility, but will be ignored. Returns ------- Symbolic 4D tensor Set of feature maps generated by convolutional layer. Tensor is of shape (batch size, output channels, output rows, output columns) Notes ----- If cuDNN is available, it will be used on the GPU. Otherwise, it is the *CorrMM* convolution that will be used "caffe style convolution". This is only supported in Theano 0.8 or the development version until it is released. The parameter filter_dilation is an implementation of `dilated convolution `_. """ if 'imshp_logical' in kwargs or 'kshp_logical' in kwargs: raise ValueError( "Keyword arguments 'imshp_logical' and 'kshp_logical' for conv2d " "are not supported anymore (and have not been a reliable way to " "perform upsampling). That feature is still available by calling " "theano.tensor.nnet.conv.conv2d() for the time being.") if len(kwargs.keys()) > 0: warnings.warn(str(kwargs.keys()) + " are now deprecated in " "`tensor.nnet.abstract_conv.conv2d` interface" " and will be ignored.", stacklevel=2) if image_shape is not None: warnings.warn("The `image_shape` keyword argument to " "`tensor.nnet.conv2d` is deprecated, it has been " "renamed to `input_shape`.", stacklevel=2) if input_shape is None: input_shape = image_shape else: raise ValueError("input_shape and image_shape should not" " be provided at the same time.") return abstract_conv2d(input, filters, input_shape, filter_shape, border_mode, subsample, filter_flip, filter_dilation, num_groups, unshared) def conv2d_transpose(input, filters, output_shape, filter_shape=None, border_mode='valid', input_dilation=(1, 1), filter_flip=True, filter_dilation=(1, 1), num_groups=1, unshared=False): """ This function will build the symbolic graph for applying a transposed convolution over a mini-batch of a stack of 2D inputs with a set of 2D filters. Parameters ---------- input: symbolic 4D tensor Mini-batch of feature map stacks, of shape (batch size, input channels, input rows, input columns). See the optional parameter ``input_shape``. filters: symbolic 4D tensor Set of filters used in CNN layer of shape (input channels, output channels, filter rows, filter columns). See the optional parameter ``filter_shape``. **Note: the order for ``output_channels`` and ``input_channels`` is reversed with respect to ``conv2d``.** output_shape: tuple/list of len 4 of int or Constant variable The shape of the output of ``conv2d_transpose``. The last two elements are allowed to be ``tensor.scalar`` variables. filter_shape: None, tuple/list of len 4 of int or Constant variable The shape of the filters parameter. Optional, possibly used to choose an optimal implementation. You can give ``None`` for any element of the list to specify that this element is not known at compile time. border_mode: str, int or tuple of two int Refers to the ``border_mode`` argument of the corresponding forward (non-transposed) convolution. See the argument description in ``conv2d``. What was ``padding`` for the forward convolution means ``cropping`` the output of the transposed one. ``valid`` corresponds to no cropping, ``full`` to maximal cropping. input_dilation: tuple of len 2 Corresponds to ``subsample`` (also called strides elsewhere) in the non-transposed convolution. filter_flip: bool If ``True``, will flip the filter rows and columns before sliding them over the input. This operation is normally referred to as a convolution, and this is the default. If ``False``, the filters are not flipped and the operation is referred to as a cross-correlation. filter_dilation: tuple of len 2 Factor by which to subsample (stride) the input. Also called dilation elsewhere. num_groups : int Divides the image, kernel and output tensors into num_groups separate groups. Each which carry out convolutions separately unshared: bool If true, then unshared or 'locally connected' convolution will be performed. A different filter will be used for each region of the input. Grouped unshared convolution is supported. Returns ------- Symbolic 4D tensor Set of feature maps generated by the transposed convolution. Tensor is of shape (batch size, output channels, output rows, output columns) Notes ----- If cuDNN is available, it will be used on the GPU. Otherwise, it is the *CorrMM* convolution that will be used "caffe style convolution". This operation is also sometimes called "deconvolution". The parameter filter_dilation is an implementation of `dilated convolution `_. """ return conv2d_grad_wrt_inputs(output_grad=input, filters=filters, input_shape=output_shape, filter_shape=filter_shape, border_mode=border_mode, subsample=input_dilation, filter_flip=filter_flip, filter_dilation=filter_dilation, num_groups=num_groups, unshared=unshared) Theano-1.0.0/theano/tensor/nnet/corr3d.py00006000163562€ÿÿÿþ0000010647713203335532021610 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import os import logging from six import integer_types import theano from theano import Apply from theano import gof from theano.gof import ParamsType, EnumList from theano.scalar import int64 from theano.tensor import as_tensor_variable, TensorType from theano.tensor.nnet.abstract_conv import get_conv_output_shape from theano.tensor import blas_headers from theano.tensor.blas import ldflags, blas_header_version _logger = logging.getLogger(__name__) class BaseCorr3dMM(gof.OpenMPOp): """ Base class for `Corr3dMM`, `Corr3dMM_gradWeights` and `Corr3dMM_gradInputs`. Cannot be used directly. Every sub-class must define internal attribute ``_direction`` out of __init__(). ``_direction`` must take one of following values: - "forward" to correlate bottom with weights and store results in top. - "backprop weights" to do a valid convolution of bottom with top (swapping the first two dimensions) and store results in weights. - "backprop inputs" to do a full convolution of top with weights (swapping the first two dimensions) and store results in bottom. Parameters ---------- border_mode : {'valid', 'full', 'half'} Additionally, the padding size could be directly specified by an integer or a tuple of three of integers subsample Perform subsampling of the output (default: (1, 1, 1)). filter_dilation Perform dilated correlation (default: (1, 1, 1)) num_groups Perform grouped convolutions (default: 1) """ check_broadcast = False __props__ = ('border_mode', 'subsample', 'filter_dilation', 'num_groups') _direction = None params_type = ParamsType(direction=EnumList(('DIRECTION_FORWARD', 'forward'), # 0 ('DIRECTION_BACKPROP_WEIGHTS', 'backprop weights'), # 1 ('DIRECTION_BACKPROP_INPUTS', 'backprop inputs')), # 2 dH=int64, dW=int64, dD=int64, dilH=int64, dilW=int64, dilD=int64, padH=int64, padW=int64, padD=int64, num_groups=int64) def __init__(self, border_mode="valid", subsample=(1, 1, 1), filter_dilation=(1, 1, 1), openmp=None, num_groups=1): super(BaseCorr3dMM, self).__init__(openmp=openmp) if isinstance(border_mode, integer_types): if border_mode < 0: raise ValueError( 'invalid border_mode {}, which must be a ' 'non-negative integer'.format(border_mode)) border_mode = (border_mode, border_mode, border_mode) if isinstance(border_mode, tuple): if len(border_mode) != 3 or min(border_mode) < 0: raise ValueError( 'invalid border_mode {}, which must be a tuple of ' 'three non-negative integers'.format(border_mode)) pad_h, pad_w, pad_d = map(int, border_mode) border_mode = (pad_h, pad_w, pad_d) if not ((isinstance(border_mode, tuple) and min(border_mode) >= 0) or border_mode in ('valid', 'full', 'half')): raise ValueError( 'invalid border_mode {}, which must be either ' '"valid", "full", "half", an integer or a tuple of three' ' integers'.format(border_mode)) self.border_mode = border_mode if len(subsample) != 3: raise ValueError("subsample must have three elements") if len(filter_dilation) != 3: raise ValueError("filter_dilation must have three elements") self.subsample = tuple(subsample) self.filter_dilation = tuple(filter_dilation) if num_groups < 1: raise ValueError("Number of groups should be greater than 0") self.num_groups = num_groups if not theano.config.blas.ldflags: # Theano will use a NumPy C implementation of [sd]gemm_ instead. self.blas_type = '' else: if 'openblas' in theano.config.blas.ldflags: self.blas_type = 'openblas' elif 'mkl' in theano.config.blas.ldflags: self.blas_type = 'mkl' else: self.blas_type = '' if self._direction not in ["forward", "backprop weights", "backprop inputs"]: raise ValueError("_direction must be one of 'forward', " "'backprop weights', 'backprop inputs'") @property def pad(self): if self.border_mode == "half": return (-1, -1, -1) elif self.border_mode == "full": return (-2, -2, -2) elif isinstance(self.border_mode, tuple): return self.border_mode else: assert self.border_mode == "valid" return (0, 0, 0) # Direction should be converted to real enum value, # as it is compared to integer later in c_code_helper(). direction = property(lambda self: self.params_type.enum_from_alias(self._direction)) dH = property(lambda self: self.subsample[0]) dW = property(lambda self: self.subsample[1]) dD = property(lambda self: self.subsample[2]) dilH = property(lambda self: self.filter_dilation[0]) dilW = property(lambda self: self.filter_dilation[1]) dilD = property(lambda self: self.filter_dilation[2]) padH = property(lambda self: self.pad[0]) padW = property(lambda self: self.pad[1]) padD = property(lambda self: self.pad[2]) def __str__(self): return '%s{%s, %s, %s, %s}' % ( self.__class__.__name__, self.border_mode, str(self.subsample), str(self.filter_dilation), str(self.num_groups)) @staticmethod def as_common_dtype(in1, in2): """ Upcast input variables if neccesary. """ dtype = theano.scalar.upcast(in1.dtype, in2.dtype) return in1.astype(dtype), in2.astype(dtype) def __setstate__(self, d): self.__dict__.update(d) if not hasattr(self, 'num_groups'): self.num_groups = 1 def c_support_code(self): ccodes = blas_headers.blas_header_text() if self.blas_type == 'openblas': ccodes += blas_headers.openblas_threads_text() elif self.blas_type == 'mkl': ccodes += blas_headers.mkl_threads_text() return ccodes def c_libraries(self): return ldflags() def c_compile_args(self): compile_args = ldflags(libs=False, flags=True) compile_args += super(BaseCorr3dMM, self).c_compile_args() return compile_args def c_lib_dirs(self): return ldflags(libs=False, libs_dir=True) def c_header_dirs(self): return ldflags(libs=False, include_dir=True) def c_headers(self): headers = [''] headers += super(BaseCorr3dMM, self).c_headers() return headers def c_code_cache_version(self): # raise this whenever modifying any of the support_code_files return (8, self.openmp, blas_header_version()) def c_support_code_apply(self, node, nodename): # REMEMBER TO RAISE c_code_cache_version when changing any of # these files sub = {} dtype = str(node.__dict__['inputs'][0].dtype) assert dtype in ('float32', 'float64') if dtype == 'float32': sub['gemm'] = 'sgemm_' sub['float_type'] = 'npy_float' sub['float_typenum'] = 'NPY_FLOAT' sub['n_bytes'] = 4 sub['c_float_type'] = 'float' else: sub['gemm'] = 'dgemm_' sub['float_type'] = 'npy_double' sub['float_typenum'] = 'NPY_DOUBLE' sub['n_bytes'] = 8 sub['c_float_type'] = 'double' if self.openmp: sub['omp_flags'] = '#pragma omp parallel for schedule(static)' sub['omp_get_max_threads'] = 'omp_get_max_threads()' sub['omp_get_thread_num'] = 'omp_get_thread_num()' if self.blas_type == 'openblas': sub['blas_set_num_threads'] = 'openblas_set_num_threads' sub['blas_get_num_threads'] = 'openblas_get_num_threads()' elif self.blas_type == 'mkl': sub['blas_set_num_threads'] = 'mkl_set_num_threads' sub['blas_get_num_threads'] = 'mkl_get_max_threads()' else: sub['blas_set_num_threads'] = '' sub['blas_get_num_threads'] = '0' else: sub['omp_flags'] = '' sub['omp_get_max_threads'] = '1' sub['omp_get_thread_num'] = '0' sub['blas_set_num_threads'] = '' sub['blas_get_num_threads'] = '0' files = [os.path.join('c_code', 'corr3d_gemm.c')] codes = [open(os.path.join(os.path.split(__file__)[0], f)).read() for f in files] final_code = '' for code in codes: final_code += code return final_code % sub def c_code_helper(self, bottom, weights, top, sub, height=None, width=None, depth=None): """ This generates the C code for Corr3dMM (direction="forward"), Corr3dMM_gradWeights (direction="backprop weights"), and Corr3dMM_gradInputs (direction="backprop inputs"). Depending on the direction, one of bottom, weights, top will receive the output, while the other two serve as inputs. :param bottom: Variable name of the input images in the forward pass, or the gradient of the input images in backprop wrt. inputs :param weights: Variable name of the filters in the forward pass, or the gradient of the filters in backprop wrt. weights :param top: Variable name of the output images / feature maps in the forward pass, or the gradient of the outputs in the backprop passes :param sub: Dictionary of substitutions useable to help generating the C code. :param height: If self.subsample[0] != 1, a variable giving the height of the filters for direction="backprop weights" or the height of the input images for direction="backprop inputs". If self.border_mode == 'half', a variable giving the height of the filters for direction="backprop weights". Ignored otherwise. :param width: If self.subsample[1] != 1, a variable giving the width of the filters for direction="backprop weights" or the width of the input images for direction="backprop inputs". If self.border_mode == 'half', a variable giving the width of the filters for direction="backprop weights". Ignored otherwise. :param depth: If self.subsample[1] != 1, a variable giving the depth of the filters for direction="backprop weights" or the depth of the input images for direction="backprop inputs". If self.border_mode == 'half', a variable giving the depth of the filters for direction="backprop weights". Ignored otherwise. """ # When subsampling, we cannot unambiguously infer the height and width # of bottom and weights from top, so we require them to be given. # Similarly, when border_mode="half", we cannot infer the weight size. if height: height = '(*(npy_int64 *)(PyArray_DATA(%s)))' % height else: if ((self.direction != 0) and (self.dH != 1)) or ((self.direction == 1) and (self.padH == -1)): raise ValueError("height must be given for backprop with vertical sampling or border_mode='half'") height = '-1' if width: width = '(*(npy_int64 *)(PyArray_DATA(%s)))' % width else: if ((self.direction != 0) and (self.dW != 1)) or ((self.direction == 1) and (self.padW == -1)): raise ValueError("width must be given for backprop with horizontal sampling or border_mode='half'") width = '-1' if depth: depth = '(*(npy_int64 *)(PyArray_DATA(%s)))' % depth else: if ((self.direction != 0) and (self.dD != 1)) or ((self.direction == 1) and (self.padD == -1)): raise ValueError("depth must be given for backprop with depth sampling or border_mode='half'") depth = '-1' return """ // Mandatory args int direction = %(params)s->direction; // forward, bprop weights, bprop inputs // Optional args int dH = %(params)s->dH; int dW = %(params)s->dW; int dD = %(params)s->dD; int dilH = %(params)s->dilH; int dilW = %(params)s->dilW; int dilD = %(params)s->dilD; int padH = %(params)s->padH; int padW = %(params)s->padW; int padD = %(params)s->padD; int numgroups = %(params)s->num_groups; PyArrayObject * bottom = %(bottom)s; PyArrayObject * weights = %(weights)s; PyArrayObject * top = %(top)s; PyArrayObject * out2 = NULL; PyArrayObject **out = NULL; switch(%(params)s->direction) { case DIRECTION_FORWARD: out = &%(top)s; break; case DIRECTION_BACKPROP_WEIGHTS: out = &%(weights)s; break; case DIRECTION_BACKPROP_INPUTS: out = &%(bottom)s; break; default: PyErr_SetString(PyExc_ValueError, "CPU Corr3dMM: Invalid direction."); {%(fail)s} break; } // Obtain or infer kernel width, height and depth // (we need to know it early to be able to handle auto-padding) int kH, kW, kD, dil_kH, dil_kW, dil_kD; if (direction != 1) { // weight is an input variable, we can just read its shape kH = PyArray_DIMS(weights)[2]; kW = PyArray_DIMS(weights)[3]; kD = PyArray_DIMS(weights)[4]; } else { if (%(height)s != -1) { // kernel height is specified (perhaps vertical subsampling or half padding) kH = %(height)s; } else if (padH == -2) { // vertical full padding, we can infer the kernel height kH = (2 - PyArray_DIMS(bottom)[2] + (PyArray_DIMS(top)[2] - 1) * dH - 1)/ dilH + 1; } else { // explicit padding, we can infer the kernel height kH = (PyArray_DIMS(bottom)[2] + 2*padH - (PyArray_DIMS(top)[2] - 1) * dH - 1) / dilH +1; } if (%(width)s != -1) { kW = %(width)s; } else if (padW == -2) { kW = (2 - PyArray_DIMS(bottom)[3] + (PyArray_DIMS(top)[3] - 1) * dW - 1) / dilW + 1; } else { kW = (PyArray_DIMS(bottom)[3] + 2*padW - (PyArray_DIMS(top)[3] - 1) * dW - 1) / dilW + 1; } if (%(depth)s != -1) { kD = %(depth)s; } else if (padD == -2) { kD = (2 - PyArray_DIMS(bottom)[4] + (PyArray_DIMS(top)[4] - 1) * dD - 1) / dilD + 1; } else { kD = (PyArray_DIMS(bottom)[4] + 2*padD - (PyArray_DIMS(top)[4] - 1) * dD - 1) / dilD + 1; } } // Implicit dilated kernel size dil_kH = (kH - 1) * dilH + 1; dil_kW = (kW - 1) * dilW + 1; dil_kD = (kD - 1) * dilD + 1; // Auto-padding if requested if (padH == -1) { // vertical half padding padH = dil_kH / 2; } else if (padH == -2) { // vertical full padding padH = dil_kH - 1; } else if (padH < 0) { PyErr_SetString(PyExc_ValueError, "BaseCorr3dMM: padH must be >= -2"); %(fail)s } if (padW == -1) { // horizontal half padding padW = dil_kW / 2; } else if (padW == -2) { // horizontal full padding padW = dil_kW - 1; } else if (padW < 0) { PyErr_SetString(PyExc_ValueError, "BaseCorr3dMM: padW must be >= -2"); %(fail)s } if (padD == -1) { // depth half padding padD = dil_kD / 2; } else if (padD == -2) { // depth full padding padD = dil_kD - 1; } else if (padD < 0) { PyErr_SetString(PyExc_ValueError, "BaseCorr3dMM: padD must be >= -2"); %(fail)s } // Infer output shape npy_intp out_dim[5]; switch(direction) { case 0: // forward pass // output is top: (batchsize, num_filters, height, width, depth) // height and width: top = (bottom + 2*pad - ((weight-1)*dil + 1)) / sample + 1 out_dim[0] = (npy_intp)PyArray_DIMS(bottom)[0]; out_dim[1] = (npy_intp)PyArray_DIMS(weights)[0]; out_dim[2] = (npy_intp)((PyArray_DIMS(bottom)[2] + 2*padH - ((PyArray_DIMS(weights)[2]-1)*dilH + 1)) / dH + 1); out_dim[3] = (npy_intp)((PyArray_DIMS(bottom)[3] + 2*padW - ((PyArray_DIMS(weights)[3]-1)*dilW + 1)) / dW + 1); out_dim[4] = (npy_intp)((PyArray_DIMS(bottom)[4] + 2*padD - ((PyArray_DIMS(weights)[4]-1)*dilD + 1)) / dD + 1); if (out_dim[0] < 0 || out_dim[1] < 0 || out_dim[2] <= 0 || out_dim[3] <= 0 || out_dim[4] <= 0) { PyErr_Format(PyExc_ValueError, "Corr3dMM: impossible output shape\\n" " bottom shape: %%ld x %%ld x %%ld x %%ld x %%ld\\n" " weights shape: %%ld x %%ld x %%ld x %%ld x %%ld\\n" " top shape: %%ld x %%ld x %%ld x %%ld x %%ld\\n", (long int)PyArray_DIMS(bottom)[0], (long int)PyArray_DIMS(bottom)[1], (long int)PyArray_DIMS(bottom)[2], (long int)PyArray_DIMS(bottom)[3], (long int)PyArray_DIMS(bottom)[4], (long int)PyArray_DIMS(weights)[0], (long int)PyArray_DIMS(weights)[1], (long int)PyArray_DIMS(weights)[2], (long int)PyArray_DIMS(weights)[3], (long int)PyArray_DIMS(weights)[4], (long int)out_dim[0], (long int)out_dim[1], (long int)out_dim[2], (long int)out_dim[3], (long int)out_dim[4]); %(fail)s } break; case 1: // backprop wrt. weights // output is weights: (num_filters, num_channels, height, width, depth) // height and width: weights = (bottom + 2*pad - (top - 1) * sample - 1) / dil + 1 out_dim[0] = (npy_intp)PyArray_DIMS(top)[1]; out_dim[1] = (npy_intp)PyArray_DIMS(bottom)[1] / numgroups; out_dim[2] = (npy_intp)kH; // already inferred further above out_dim[3] = (npy_intp)kW; // how convenient out_dim[4] = (npy_intp)kD; if (out_dim[0] < 0 || out_dim[1] < 0 || out_dim[2] <= 0 || out_dim[3] <= 0 || out_dim[4] <= 0) { PyErr_Format(PyExc_ValueError, "Corr3dMM backprop wrt. weights: impossible output shape\\n" " bottom shape: %%ld x %%ld x %%ld x %%ld x %%ld\\n" " weights shape: %%ld x %%ld x %%ld x %%ld x %%ld\\n" " top shape: %%ld x %%ld x %%ld x %%ld x %%ld\\n", (long int)PyArray_DIMS(bottom)[0], (long int)PyArray_DIMS(bottom)[1], (long int)PyArray_DIMS(bottom)[2], (long int)PyArray_DIMS(bottom)[3], (long int)PyArray_DIMS(bottom)[4], (long int)out_dim[0], (long int)out_dim[1], (long int)out_dim[2], (long int)out_dim[3], (long int)out_dim[4], (long int)PyArray_DIMS(top)[0], (long int)PyArray_DIMS(top)[1], (long int)PyArray_DIMS(top)[2], (long int)PyArray_DIMS(top)[3], (long int)PyArray_DIMS(top)[4]); %(fail)s } break; case 2: // backprop wrt. inputs // output is bottom: (batchsize, num_channels, height, width, depth) // height and width: bottom = (top - 1) * sample + (weights-1)*dil + 1 - 2*pad out_dim[0] = (npy_intp)PyArray_DIMS(top)[0]; out_dim[1] = (npy_intp)PyArray_DIMS(weights)[1] * numgroups; out_dim[2] = (npy_intp)((%(height)s != -1) ? %(height)s : (PyArray_DIMS(top)[2] - 1) * dH + (PyArray_DIMS(weights)[2]-1)*dilH + 1 - 2*padH); out_dim[3] = (npy_intp)((%(width)s != -1) ? %(width)s : (PyArray_DIMS(top)[3] - 1) * dW + (PyArray_DIMS(weights)[3]-1)*dilW + 1 - 2*padW); out_dim[4] = (npy_intp)((%(depth)s != -1) ? %(depth)s : (PyArray_DIMS(top)[4] - 1) * dD + (PyArray_DIMS(weights)[4]-1)*dilD + 1 - 2*padD); if (out_dim[0] < 0 || out_dim[1] < 0 || out_dim[2] <= 0 || out_dim[3] <= 0 || out_dim[4] <= 0) { PyErr_Format(PyExc_ValueError, "Corr3dMM backprop wrt. inputs: impossible output shape\\n" " bottom shape: %%ld x %%ld x %%ld x %%ld x %%ld\\n" " weights shape: %%ld x %%ld x %%ld x %%ld x %%ld\\n" " top shape: %%ld x %%ld x %%ld x %%ld x %%ld\\n", (long int)out_dim[0], (long int)out_dim[1], (long int)out_dim[2], (long int)out_dim[3], (long int)out_dim[4], (long int)PyArray_DIMS(weights)[0], (long int)PyArray_DIMS(weights)[1], (long int)PyArray_DIMS(weights)[2], (long int)PyArray_DIMS(weights)[3], (long int)PyArray_DIMS(weights)[4], (long int)PyArray_DIMS(top)[0], (long int)PyArray_DIMS(top)[1], (long int)PyArray_DIMS(top)[2], (long int)PyArray_DIMS(top)[3], (long int)PyArray_DIMS(top)[4]); %(fail)s } break; default: PyErr_SetString(PyExc_ValueError, "BaseCorr3dMM: direction must be 0, 1, or 2\\n"); %(fail)s } // Prepare output array int typenum; if ( !(*out && PyArray_NDIM(*out)==4 && PyArray_IS_C_CONTIGUOUS(*out) && PyArray_DIMS(*out)[0]==out_dim[0] && PyArray_DIMS(*out)[1]==out_dim[1] && PyArray_DIMS(*out)[2]==out_dim[2] && PyArray_DIMS(*out)[3]==out_dim[3] && PyArray_DIMS(*out)[4]==out_dim[4])) { Py_XDECREF(*out); if (direction != 1) { typenum = PyArray_TYPE(weights); } else { typenum = PyArray_TYPE(bottom); } //Change to PyArray_ZEROS which is faster than PyArray_EMPTY. *out = (PyArrayObject*)PyArray_ZEROS(5, out_dim, typenum, 0); if (NULL == *out) { PyErr_Format(PyExc_RuntimeError, "BaseCorr3dMM: Failed to allocate output of %%lld x %%lld x %%lld x %%lld x %%lld", (long long)out_dim[0], (long long)out_dim[1], (long long)out_dim[2], (long long)out_dim[3], (long long)out_dim[4]); %(fail)s } } // Call corr3dMM code out2 = corr3dMM(%(bottom)s, %(weights)s, %(top)s, direction, dH, dW, dD, dilH, dilW, dilD, padH, padW, padD, numgroups); if (out2==NULL){ %(fail)s } assert (out2 == *out); """ % dict(bottom=bottom, weights=weights, top=top, height=height, width=width, depth=depth, fail=sub['fail'], params=sub['params']) class Corr3dMM(BaseCorr3dMM): """ CPU correlation implementation using Matrix Multiplication. Parameters ---------- border_mode The width of a border of implicit zeros to pad the input with. Must be a tuple with 3 elements giving the width of the padding on each side, or a single integer to pad the same on all sides, or a string shortcut setting the padding at runtime: ``'valid'`` for ``(0, 0, 0)`` (valid convolution, no padding), ``'full'`` for ``(kernel_rows - 1, kernel_columns - 1, kernel_depth - 1)`` (full convolution), ``'half'`` for ``(kernel_rows // 2, kernel_columns // 2, kernel_depth // 2)`` (same convolution for odd-sized kernels). Note that the three widths are each applied twice, once per side (left and right, top and bottom, front and back). subsample The subsample operation applied to each output image. Should be a tuple with 3 elements. Set to `(1, 1, 1)` to disable subsampling. filter_dilation The filter dilation operation applied to each input image. Should be a tuple with 3 elements. Set to `(1, 1, 1)` to disable filter dilation. num_groups Perform grouped convolutions (default: 1) """ _direction = "forward" def make_node(self, img, kern): img = as_tensor_variable(img) kern = as_tensor_variable(kern) img, kern = self.as_common_dtype(img, kern) if img.type.ndim != 5: raise TypeError('img must be 5D tensor') if kern.type.ndim != 5: raise TypeError('kern must be 5D tensor') broadcastable = [img.type.broadcastable[0], kern.type.broadcastable[0], False, False, False] dtype = img.type.dtype return Apply(self, [img, kern], [TensorType(dtype, broadcastable)()]) def infer_shape(self, node, input_shape): imshp = input_shape[0] kshp = input_shape[1] res = get_conv_output_shape( imshp, kshp, self.border_mode, self.subsample, self.filter_dilation) return [res] def c_code(self, node, nodename, inp, out_, sub): bottom, weights = inp top, = out_ return super(Corr3dMM, self).c_code_helper(bottom, weights, top, sub) def grad(self, inp, grads): bottom, weights = inp top, = grads d_bottom = Corr3dMM_gradInputs(self.border_mode, self.subsample, self.filter_dilation, num_groups=self.num_groups)(weights, top, bottom.shape[-3:]) d_weights = Corr3dMM_gradWeights(self.border_mode, self.subsample, self.filter_dilation, num_groups=self.num_groups)(bottom, top, weights.shape[-3:]) return d_bottom, d_weights class Corr3dMM_gradWeights(BaseCorr3dMM): """ Gradient wrt. filters for `Corr3dMM`. Notes ----- You will not want to use this directly, but rely on Theano's automatic differentiation or graph optimization to use it as needed. """ _direction = "backprop weights" def make_node(self, img, topgrad, shape=None): img = as_tensor_variable(img) topgrad = as_tensor_variable(topgrad) img, topgrad = self.as_common_dtype(img, topgrad) if img.type.ndim != 5: raise TypeError('img must be 5D tensor') if topgrad.type.ndim != 5: raise TypeError('topgrad must be 5D tensor') if shape is None: if self.subsample != (1, 1, 1) or self.border_mode == "half": raise ValueError('shape must be given if subsample != (1, 1, 1)' ' or border_mode == "half"') height_width_depth = [] else: height_width_depth = [as_tensor_variable(shape[0]).astype('int64'), as_tensor_variable(shape[1]).astype('int64'), as_tensor_variable(shape[2]).astype('int64')] broadcastable = [topgrad.type.broadcastable[1], img.type.broadcastable[1], False, False, False] dtype = img.type.dtype return Apply(self, [img, topgrad] + height_width_depth, [TensorType(dtype, broadcastable)()]) def infer_shape(self, node, input_shape): if self.border_mode == "half": padH = padW = padD = -1 elif self.border_mode == "full": padH = padW = padD = -2 elif isinstance(self.border_mode, tuple): padH, padW, padD = self.border_mode else: assert self.border_mode == "valid" padH = padW = padD = 0 dH, dW, dD = self.subsample imshp = input_shape[0] topshp = input_shape[1] ssize, imshp = imshp[1], list(imshp[2:]) ssize = ssize // self.num_groups nkern, topshp = topshp[1], list(topshp[2:]) height_width_depth = node.inputs[-3:] if ((dH != 1) or (padH == -1)): # vertical subsampling or half padding, kernel height is specified kH = height_width_depth[0] elif padH == -2: # vertical full padding, we can infer the kernel height kH = 2 - imshp[0] + (topshp[0] - 1) * dH else: # explicit padding, we can infer the kernel height kH = imshp[0] + 2 * padH - (topshp[0] - 1) * dH if ((dW != 1) or (padW == -1)): kW = height_width_depth[1] elif (padW == -2): kW = 2 - imshp[1] + (topshp[1] - 1) * dW else: kW = imshp[1] + 2 * padW - (topshp[1] - 1) * dW if ((dD != 1) or (padD == -1)): kD = height_width_depth[2] elif (padD == -2): kD = 2 - imshp[2] + (topshp[2] - 1) * dD else: kD = imshp[2] + 2 * padD - (topshp[2] - 1) * dD return [(nkern, ssize, kH, kW, kD)] def c_code(self, node, nodename, inp, out_, sub): bottom, top = inp[:2] height, width, depth = inp[2:] or (None, None, None) weights, = out_ return super(Corr3dMM_gradWeights, self).c_code_helper(bottom, weights, top, sub, height, width, depth) def grad(self, inp, grads): bottom, top = inp[:2] weights, = grads d_bottom = Corr3dMM_gradInputs(self.border_mode, self.subsample, self.filter_dilation, num_groups=self.num_groups)(weights, top, bottom.shape[-3:]) d_top = Corr3dMM(self.border_mode, self.subsample, self.filter_dilation, num_groups=self.num_groups)(bottom, weights) d_height_width_depth = ((theano.gradient.DisconnectedType()(),) * 3 if len(inp) == 5 else ()) return (d_bottom, d_top) + d_height_width_depth def connection_pattern(self, node): if node.nin == 2: return [[1], [1]] else: return [[1], [1], [0], [0], [0]] # no connection to height, width, depth class Corr3dMM_gradInputs(BaseCorr3dMM): """ Gradient wrt. inputs for `Corr3dMM`. Notes ----- You will not want to use this directly, but rely on Theano's automatic differentiation or graph optimization to use it as needed. """ _direction = "backprop inputs" def make_node(self, kern, topgrad, shape=None): kern = as_tensor_variable(kern) topgrad = as_tensor_variable(topgrad) kern, topgrad = self.as_common_dtype(kern, topgrad) if kern.type.ndim != 5: raise TypeError('kern must be 5D tensor') if topgrad.type.ndim != 5: raise TypeError('topgrad must be 5D tensor') if shape is None: if self.subsample != (1, 1, 1): raise ValueError('shape must be given if subsample != (1, 1, 1)') height_width_depth = [] else: height_width_depth = [as_tensor_variable(shape[0]).astype('int64'), as_tensor_variable(shape[1]).astype('int64'), as_tensor_variable(shape[2]).astype('int64')] if self.num_groups > 1: broadcastable = [topgrad.type.broadcastable[0], False, False, False, False] else: broadcastable = [topgrad.type.broadcastable[0], kern.type.broadcastable[1], False, False, False] dtype = kern.type.dtype return Apply(self, [kern, topgrad] + height_width_depth, [TensorType(dtype, broadcastable)()]) def infer_shape(self, node, input_shape): if self.border_mode == "half": padH = padW = padD = -1 elif self.border_mode == "full": padH = padW = padD = -2 elif isinstance(self.border_mode, tuple): padH, padW, padD = self.border_mode else: assert self.border_mode == "valid" padH = padW = padD = 0 dH, dW, dD = self.subsample kshp = input_shape[0] topshp = input_shape[1] ssize, kshp = kshp[1], list(kshp[2:]) ssize = ssize * self.num_groups bsize, topshp = topshp[0], list(topshp[2:]) height_width_depth = node.inputs[-3:] if padH == -1: padH = kshp[0] // 2 elif padH == -2: padH = kshp[0] - 1 elif padH < -2: raise ValueError('Corr3dMM_gradInputs: border_mode must be >= 0.') if padW == -1: padW = kshp[1] // 2 elif padW == -2: padW = kshp[1] - 1 elif padW < -2: raise ValueError('Corr3dMM_gradInputs: border_mode must be >= 0.') if padD == -1: padD = kshp[2] // 2 elif padD == -2: padD = kshp[2] - 1 elif padD < -2: raise ValueError('Corr3dMM_gradInputs: border_mode must be >= 0.') if dH != 1: out_shp0 = height_width_depth[0] else: out_shp0 = (topshp[0] - 1) * dH + kshp[0] - 2 * padH if dW != 1: out_shp1 = height_width_depth[1] else: out_shp1 = (topshp[1] - 1) * dW + kshp[1] - 2 * padW if dD != 1: out_shp2 = height_width_depth[2] else: out_shp2 = (topshp[2] - 1) * dD + kshp[2] - 2 * padD out_shp = (out_shp0, out_shp1, out_shp2) return [(bsize, ssize) + out_shp] def c_code(self, node, nodename, inp, out_, sub): weights, top = inp[:2] height, width, depth = inp[2:] or (None, None, None) bottom, = out_ return super(Corr3dMM_gradInputs, self).c_code_helper(bottom, weights, top, sub, height, width, depth) def grad(self, inp, grads): weights, top = inp[:2] bottom, = grads d_weights = Corr3dMM_gradWeights(self.border_mode, self.subsample, self.filter_dilation, num_groups=self.num_groups)(bottom, top, weights.shape[-3:]) d_top = Corr3dMM(self.border_mode, self.subsample, self.filter_dilation, num_groups=self.num_groups)(bottom, weights) d_height_width_depth = ((theano.gradient.DisconnectedType()(),) * 3 if len(inp) == 5 else ()) return (d_weights, d_top) + d_height_width_depth def connection_pattern(self, node): if node.nin == 2: return [[1], [1]] else: return [[1], [1], [0], [0], [0]] # no connection to height, width, depth Theano-1.0.0/theano/tensor/nnet/blocksparse.py00006000163562€ÿÿÿþ0000002145213203335532022711 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import numpy as np import theano from theano import Op, Apply from theano.tensor import discrete_dtypes from theano.gradient import grad_undefined class SparseBlockGemv(Op): """ This op computes the dot product of specified pieces of vectors and matrices, returning pieces of vectors:: for b in range(batch_size): for j in range(o.shape[1]): for i in range(h.shape[1]): o[b, j, :] += numpy.dot(h[b, i], W[iIdx[b, i], oIdx[b, j]]) where b, h, W, o iIdx, oIdx are defined in the docstring of make_node. .. image:: ../../../images/blocksparse.png :scale: 50 % """ __props__ = ('inplace',) registered_opts = [] def __init__(self, inplace=False): self.inplace = inplace if self.inplace: self.destroy_map = {0: [0]} def make_node(self, o, W, h, inputIdx, outputIdx): """ Compute the dot product of the specified pieces of vectors and matrices. The parameter types are actually their expected shapes relative to each other. Parameters ---------- o : batch, oWin, oSize output vector W : iBlocks, oBlocks, iSize, oSize weight matrix h : batch, iWin, iSize input from lower layer (sparse) inputIdx : batch, iWin indexes of the input blocks outputIdx : batch, oWin indexes of the output blocks Returns ------- (batch, oWin, oSize) dot(W[i, j], h[i]) + o[j] Notes ----- - `batch` is the number of examples in a minibatch (batch size). - `iBlocks` is the total number of blocks in the input (from lower layer). - `iSize` is the size of each of these input blocks. - `iWin` is the number of blocks that will be used as inputs. Which blocks will be used is specified in `inputIdx`. - `oBlocks` is the number or possible output blocks. - `oSize` is the size of each of these output blocks. - `oWin` is the number of output blocks that will actually be computed. Which blocks will be computed is specified in `outputIdx`. """ o = theano.tensor.as_tensor_variable(o) W = theano.tensor.as_tensor_variable(W) h = theano.tensor.as_tensor_variable(h) inputIdx = theano.tensor.as_tensor_variable(inputIdx) outputIdx = theano.tensor.as_tensor_variable(outputIdx) if o.ndim != 3: raise TypeError('The output o must be a 2D tensor') if W.ndim != 4: raise TypeError('The weight matrix W must be a 4D tensor') if h.ndim != 3: raise TypeError('The input h must be a 3D tensor') if inputIdx.ndim != 2: raise TypeError('The input indices inputIdx must be a 2D tensor') if outputIdx.ndim != 2: raise TypeError('The output indices outputIdx must be a 2D tensor') assert inputIdx.type.dtype in discrete_dtypes assert outputIdx.type.dtype in discrete_dtypes return Apply(self, [o, W, h, inputIdx, outputIdx], [o.type()]) def perform(self, node, inp, out_): o, W, h, iIdx, oIdx = inp[:5] if not self.inplace: o = o.copy() for b in range(o.shape[0]): for j in range(o.shape[1]): outputIdx = oIdx[b, j] for i in range(h.shape[1]): inputIdx = iIdx[b, i] w = W[inputIdx, outputIdx] o[b, j, :] += np.dot(h[b, i], w) out_[0][0] = o def infer_shape(self, node, input_shapes): return [input_shapes[0]] def grad(self, inputs, grads): o, W, h, inputIdx, outputIdx = inputs go = grads[0] outer_fun = SparseBlockOuter(self.inplace) gemv_fun = SparseBlockGemv(self.inplace) Wgrad = outer_fun(W.zeros_like(), h, go, inputIdx, outputIdx) hgrad = gemv_fun(h.zeros_like(), W.dimshuffle((1, 0, 3, 2)), go, outputIdx, inputIdx) return [go, Wgrad, hgrad, grad_undefined(self, 3, inputIdx, "grad of inputIdx makes no sense"), grad_undefined(self, 4, outputIdx, "grad of outputIdx makes no sense")] class SparseBlockOuter(Op): """ This computes the outer product of two sets of pieces of vectors updating a full matrix with the results:: for b in range(batch_size): o[xIdx[b, i], yIdx[b, j]] += (alpha * outer(x[b, i], y[b, j])) This op is involved in the gradient of SparseBlockGemv. """ __props__ = ('inplace',) registered_opts = [] def __init__(self, inplace=False): self.inplace = inplace if self.inplace: self.destroy_map = {0: [0]} def make_node(self, o, x, y, xIdx, yIdx, alpha=None): """ Compute the dot product of the specified pieces of vectors and matrices. The parameter types are actually their expected shapes relative to each other. Parameters ---------- o : xBlocks, yBlocks, xSize, ySize x : batch, xWin, xSize y : batch, yWin, ySize xIdx : batch, iWin indexes of the x blocks yIdx : batch, oWin indexes of the y blocks Returns ------- (xBlocks, yBlocks, xSize, ySize) outer(x[i], y[j]) + o[i, j] Notes ----- - `batch` is the number of examples in a minibatch (batch size). - `xBlocks` is the total number of blocks in x. - `xSize` is the size of each of these x blocks. - `xWin` is the number of blocks that will be used as x. Which blocks will be used is specified in `xIdx`. - `yBlocks` is the number or possible y blocks. - `ySize` is the size of each of these y blocks. - `yWin` is the number of y blocks that will actually be computed. Which blocks will be computed is specified in `yIdx`. """ one = theano.tensor.constant(np.asarray(1.0, dtype='float32')) o = theano.tensor.as_tensor_variable(o) x = theano.tensor.as_tensor_variable(x) y = theano.tensor.as_tensor_variable(y) if alpha is None: alpha = one return Apply(self, [o, x, y, xIdx, yIdx, alpha], [o.type()]) def infer_shape(self, node, input_shapes): return [input_shapes[0]] def perform(self, node, inp, out_): o, x, y, xIdx, yIdx, alpha = inp[:6] if not self.inplace: o = o.copy() for b in range(x.shape[0]): for i in range(xIdx.shape[1]): for j in range(yIdx.shape[1]): o[xIdx[b, i], yIdx[b, j]] += np.outer(x[b, i], y[b, j, :]) out_[0][0] = o sparse_block_gemv = SparseBlockGemv(False) sparse_block_gemv_inplace = SparseBlockGemv(True) sparse_block_outer = SparseBlockOuter(False) sparse_block_outer_inplace = SparseBlockOuter(True) def sparse_block_dot(W, h, inputIdx, b, outputIdx): """ Compute the dot product (plus bias) of the specified pieces of vectors and matrices. See SparseBlockGemv to get more information. The parameter types are actually their expected shapes relative to each other. Parameters ---------- W : iBlocks, oBlocks, iSize, oSize weight matrix h : batch, iWin, iSize input from lower layer (sparse) inputIdx : batch, iWin indexes of the input blocks b : oBlocks, oSize bias vector outputIdx : batch, oWin indexes of the output blocks Returns ------- (batch, oWin, oSize) dot(W[i, j], h[i]) + b[j] but b[j] is only added once Notes ----- - `batch` is the number of examples in a minibatch (batch size). - `iBlocks` is the total number of blocks in the input (from lower layer). - `iSize` is the size of each of these input blocks. - `iWin` is the number of blocks that will be used as inputs. Which blocks will be used is specified in `inputIdx`. - `oBlocks` is the number or possible output blocks. - `oSize` is the size of each of these output blocks. - `oWin` is the number of output blocks that will actually be computed. Which blocks will be computed is specified in `outputIdx`. """ assert inputIdx.ndim == h.ndim - 1 assert outputIdx.ndim == inputIdx.ndim if h.ndim == 2: h = h.dimshuffle('x', 0, 1) inputIdx = inputIdx.dimshuffle('x', 0) outputIdx = outputIdx.dimshuffle('x', 0) return SparseBlockGemv()(b.take(outputIdx, axis=0), W, h, inputIdx, outputIdx) Theano-1.0.0/theano/tensor/nnet/conv3d2d.py00006000163562€ÿÿÿþ0000002270113203335532022021 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import theano from theano.gradient import DisconnectedType from theano.gof import Op, Apply, TopoOptimizer from theano.gof.opt import copy_stack_trace from theano import tensor def get_diagonal_subtensor_view(x, i0, i1): """ Helper function for DiagonalSubtensor and IncDiagonalSubtensor. Notes ----- It returns a partial view of x, not a partial copy. """ # We have to cast i0 and i0 to int because python # do not support indexing with 0-dim, 'int*' ndarrays. i0 = int(i0) i1 = int(i1) if x.shape[i0] < x.shape[i1]: raise NotImplementedError('is this allowed?') idx = [slice(None)] * x.ndim idx[i0] = slice(x.shape[i1] - 1, None, None) xview = x.__getitem__(tuple(idx)) strides = list(xview.strides) if x.shape[i1] != 1: strides[i1] -= strides[i0] xview.strides = strides return xview class DiagonalSubtensor(Op): """ Return a form a nd diagonal subtensor. Parameters ---------- x n-d tensor i0 Axis index in x i1 Axis index in x Notes ----- Work on the GPU. Extended summary ---------------- ``x`` is some n-dimensional tensor, but this Op only deals with a matrix-shaped slice, using axes i0 and i1. Without loss of generality, suppose that ``i0`` picks out our ``row`` dimension, and i1 the ``column`` dimension. So the relevant part of ``x`` is some matrix ``u``. Suppose it has 7 rows and 4 columns:: [ 0 0 0 0 ] [ 0 0 0 0 ] [ 0 0 0 0 ] [ 0 0 0 0 ] [ 0 0 0 0 ] [ 0 0 0 0 ] The view returned by this function is also a matrix. It's a thick, diagonal ``stripe`` across u that discards the lower left triangle and the upper right triangle: [ x 0 0 0 ] [ x x 0 0 ] [ x x x 0 ] [ 0 x x x ] [ 0 0 x x ] [ 0 0 0 x ] In this case the return value would be this view of shape 3x4. The returned view has the same number of dimensions as the input ``x``, and the only difference is that the shape along dimension ``i0`` has been reduced by ``shape[i1] - 1`` because of the triangles that got chopped out. The NotImplementedError is meant to catch the case where shape[i0] is too small for the stripe to reach across the matrix, in which case it's not clear what this function should do. Maybe always raise an error. I'd look back to the call site in the Conv3D to see what's necessary at that point. """ __props__ = ("inplace",) def __str__(self): if self.inplace: return "%s{inplace}" % self.__class__.__name__ return "%s" % self.__class__.__name__ def __init__(self, inplace=False): self.inplace = inplace if inplace: self.view_map = {0: [0]} def make_node(self, x, i0, i1): _i0 = tensor.as_tensor_variable(i0) _i1 = tensor.as_tensor_variable(i1) return Apply(self, [x, _i0, _i1], [x.type()]) def perform(self, node, inputs, output_storage): xview = get_diagonal_subtensor_view(*inputs) if self.inplace: output_storage[0][0] = xview else: output_storage[0][0] = xview.copy() def grad(self, inputs, g_outputs): z = tensor.zeros_like(inputs[0]) gx = inc_diagonal_subtensor(z, inputs[1], inputs[2], g_outputs[0]) return [gx, DisconnectedType()(), DisconnectedType()()] def connection_pattern(self, node): rval = [[True], [False], [False]] return rval diagonal_subtensor = DiagonalSubtensor(False) class IncDiagonalSubtensor(Op): """ The gradient of DiagonalSubtensor. """ __props__ = ("inplace",) def __str__(self): if self.inplace: return "%s{inplace}" % self.__class__.__name__ return "%s" % self.__class__.__name__ def __init__(self, inplace=False): self.inplace = inplace if inplace: self.destroy_map = {0: [0]} def make_node(self, x, i0, i1, amt): _i0 = tensor.as_tensor_variable(i0) _i1 = tensor.as_tensor_variable(i1) return Apply(self, [x, _i0, _i1, amt], [x.type()]) def perform(self, node, inputs, output_storage): x, i0, i1, amt = inputs if not self.inplace: x = x.copy() xview = get_diagonal_subtensor_view(x, i0, i1) xview += amt output_storage[0][0] = x def grad(self, inputs, g_outputs): x, i0, i1, amt = inputs gy = g_outputs[0] return [gy, DisconnectedType()(), DisconnectedType()(), diagonal_subtensor(gy, i0, i1)] def connection_pattern(self, node): rval = [[True], [False], [False], [True]] return rval inc_diagonal_subtensor = IncDiagonalSubtensor(False) def conv3d(signals, filters, signals_shape=None, filters_shape=None, border_mode='valid'): """ Convolve spatio-temporal filters with a movie. It flips the filters. Parameters ---------- signals Timeseries of images whose pixels have color channels. Shape: [Ns, Ts, C, Hs, Ws]. filters Spatio-temporal filters. Shape: [Nf, Tf, C, Hf, Wf]. signals_shape None or a tuple/list with the shape of signals. filters_shape None or a tuple/list with the shape of filters. border_mode One of 'valid', 'full' or 'half'. Notes ----- Another way to define signals: (batch, time, in channel, row, column) Another way to define filters: (out channel,time,in channel, row, column) For the GPU, use nnet.conv3d. See Also -------- Someone made a script that shows how to swap the axes between both 3d convolution implementations in Theano. See the last `attachment `_ """ if isinstance(border_mode, str): border_mode = (border_mode, border_mode, border_mode) if signals_shape is None: _signals_shape_5d = signals.shape else: _signals_shape_5d = signals_shape if filters_shape is None: _filters_shape_5d = filters.shape else: _filters_shape_5d = filters_shape Ns, Ts, C, Hs, Ws = _signals_shape_5d Nf, Tf, C, Hf, Wf = _filters_shape_5d _signals_shape_4d = (Ns * Ts, C, Hs, Ws) _filters_shape_4d = (Nf * Tf, C, Hf, Wf) if border_mode[1] != border_mode[2]: raise NotImplementedError('height and width bordermodes must match') conv2d_signal_shape = _signals_shape_4d conv2d_filter_shape = _filters_shape_4d if signals_shape is None: conv2d_signal_shape = None if filters_shape is None: conv2d_filter_shape = None out_4d = tensor.nnet.conv2d( signals.reshape(_signals_shape_4d), filters.reshape(_filters_shape_4d), input_shape=conv2d_signal_shape, filter_shape=conv2d_filter_shape, border_mode=border_mode[1]) # ignoring border_mode[2] # compute the intended output size if border_mode[1] == 'valid': Hout = Hs - Hf + 1 Wout = Ws - Wf + 1 elif border_mode[1] == 'full': Hout = Hs + Hf - 1 Wout = Ws + Wf - 1 elif border_mode[1] == 'half': Hout = Hs - (Hf % 2) + 1 Wout = Ws - (Wf % 2) + 1 elif border_mode[1] == 'same': raise NotImplementedError() else: raise ValueError('invalid border mode', border_mode[1]) # reshape the temporary output to restore its original size out_tmp = out_4d.reshape((Ns, Ts, Nf, Tf, Hout, Wout)) # now sum out along the Tf to get the output # but we have to sum on a diagonal through the Tf and Ts submatrix. if Tf == 1: # for Tf==1, no sum along Tf, the Ts-axis of the output is unchanged! out_5d = out_tmp.reshape((Ns, Ts, Nf, Hout, Wout)) else: # for some types of convolution, pad out_tmp with zeros if border_mode[0] == 'valid': Tpad = 0 elif border_mode[0] == 'full': Tpad = Tf - 1 elif border_mode[0] == 'half': Tpad = Tf // 2 elif border_mode[0] == 'same': raise NotImplementedError() else: raise ValueError('invalid border mode', border_mode[0]) if Tpad == 0: out_5d = diagonal_subtensor(out_tmp, 1, 3).sum(axis=3) else: # pad out_tmp with zeros before summing over the diagonal out_tmp_padded = tensor.zeros(dtype=out_tmp.dtype, shape=( Ns, Ts + 2 * Tpad, Nf, Tf, Hout, Wout )) out_tmp_padded = tensor.set_subtensor( out_tmp_padded[:, Tpad:(Ts + Tpad), :, :, :, :], out_tmp) out_5d = diagonal_subtensor(out_tmp_padded, 1, 3).sum(axis=3) return out_5d @theano.gof.local_optimizer([DiagonalSubtensor, IncDiagonalSubtensor]) def local_inplace_DiagonalSubtensor(node): """Also work for IncDiagonalSubtensor.""" if (isinstance(node.op, (DiagonalSubtensor, IncDiagonalSubtensor)) and not node.op.inplace): new_op = node.op.__class__(inplace=True) new_node = new_op(*node.inputs) copy_stack_trace(node.outputs[0], new_node) return [new_node] return False theano.compile.optdb.register( 'local_inplace_DiagonalSubtensor', TopoOptimizer( local_inplace_DiagonalSubtensor, failure_callback=TopoOptimizer.warn_inplace), 60, 'fast_run', 'inplace') Theano-1.0.0/theano/tensor/nnet/corr.py00006000163562€ÿÿÿþ0000011676213203335532021357 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import os import logging from six import integer_types import theano from theano import Apply from theano import gof from theano.gof import ParamsType, EnumList from theano.scalar import int64, int8 from theano.tensor import as_tensor_variable, TensorType from theano.tensor.nnet.abstract_conv import get_conv_output_shape from theano.tensor import blas_headers from theano.tensor.blas import ldflags, blas_header_version _logger = logging.getLogger(__name__) class BaseCorrMM(gof.OpenMPOp): """ Base class for `CorrMM`, `CorrMM_gradWeights` and `CorrMM_gradInputs`. Cannot be used directly. Every sub-class must define internal attribute ``_direction`` out of __init__(). ``_direction`` must take one of following values: - "forward" to correlate bottom with weights and store results in top. - "backprop weights" to do a valid convolution of bottom with top (swapping the first two dimensions) and store results in weights. - "backprop inputs" to do a full convolution of top with weights (swapping the first two dimensions) and store results in bottom. Parameters ---------- border_mode : {'valid', 'full', 'half'} Additionally, the padding size could be directly specified by an integer, a pair of integers, or two pairs of integers. subsample Perform subsampling of the output (default: (1, 1)). filter_dilation Perform dilated correlation (default: (1,1)) num_groups Perform grouped convolutions (default: 1) unshared Perform unshared correlation (default: False) """ check_broadcast = False __props__ = ('border_mode', 'subsample', 'filter_dilation', 'num_groups', 'unshared') _direction = None params_type = ParamsType(direction=EnumList(('DIRECTION_FORWARD', 'forward'), # 0 ('DIRECTION_BACKPROP_WEIGHTS', 'backprop weights'), # 1 ('DIRECTION_BACKPROP_INPUTS', 'backprop inputs')), # 2 dH=int64, dW=int64, dilH=int64, dilW=int64, padH_l=int64, padH_r=int64, padW_l=int64, padW_r=int64, num_groups=int64, unshared=int8) def __init__(self, border_mode="valid", subsample=(1, 1), filter_dilation=(1, 1), num_groups=1, unshared=False, openmp=None): super(BaseCorrMM, self).__init__(openmp=openmp) if isinstance(border_mode, integer_types): if border_mode < 0: raise ValueError( 'invalid border_mode {}, which must be a ' 'non-negative integer'.format(border_mode)) border_mode = ((border_mode, border_mode),) * 2 elif isinstance(border_mode, tuple): if len(border_mode) != 2: raise ValueError( 'invalid border_mode {} which must be a ' 'tuple of length 2'.format(border_mode)) border = () for mode in border_mode: if isinstance(mode, tuple) and len(mode) == 2 and \ min(mode) >= 0: border += ((int(mode[0]), int(mode[1])),) elif mode >= 0: border += ((int(mode), int(mode)),) else: raise ValueError( 'invalid border mode {}. The tuple can only contain ' 'integers or tuples of length 2'.format(border_mode)) border_mode = border elif border_mode not in ('valid', 'full', 'half'): raise ValueError( 'invalid border_mode {}, which must be either ' '"valid", "full", "half", an integer or a tuple ' 'of two integers or a pair of integers'.format(border_mode)) self.border_mode = border_mode if len(subsample) != 2: raise ValueError("subsample must have two elements") if len(filter_dilation) != 2: raise ValueError("filter_dilation must have two elements") self.subsample = tuple(subsample) self.filter_dilation = tuple(filter_dilation) self.unshared = unshared if not theano.config.blas.ldflags: # Theano will use a NumPy C implementation of [sd]gemm_ instead. self.blas_type = '' else: if 'openblas' in theano.config.blas.ldflags: self.blas_type = 'openblas' elif 'mkl' in theano.config.blas.ldflags: self.blas_type = 'mkl' else: self.blas_type = '' if self._direction not in ["forward", "backprop weights", "backprop inputs"]: raise ValueError("_direction must be one of 'forward', " "'backprop weights', 'backprop inputs'") if num_groups < 1: raise ValueError("Number of groups should be greater than 0") self.num_groups = num_groups @property def pad(self): if self.border_mode == "half": return ((-1, -1),) * 2 elif self.border_mode == "full": return ((-2, -2),) * 2 elif isinstance(self.border_mode, tuple): return self.border_mode else: assert self.border_mode == "valid" return ((0, 0),) * 2 # Direction should be converted to real enum value, # as it is compared to integer later in c_code_helper(). direction = property(lambda self: self.params_type.enum_from_alias(self._direction)) dH = property(lambda self: self.subsample[0]) dW = property(lambda self: self.subsample[1]) dilH = property(lambda self: self.filter_dilation[0]) dilW = property(lambda self: self.filter_dilation[1]) padH_l = property(lambda self: self.pad[0][0]) padH_r = property(lambda self: self.pad[0][1]) padW_l = property(lambda self: self.pad[1][0]) padW_r = property(lambda self: self.pad[1][1]) def __str__(self): return '%s{%s, %s, %s, %s %s}' % ( self.__class__.__name__, self.border_mode, str(self.subsample), str(self.filter_dilation), str(self.num_groups), str(self.unshared)) @staticmethod def as_common_dtype(in1, in2): """ Upcast input variables if neccesary. """ dtype = theano.scalar.upcast(in1.dtype, in2.dtype) return in1.astype(dtype), in2.astype(dtype) def __setstate__(self, d): self.__dict__.update(d) if not hasattr(self, 'num_groups'): self.num_groups = 1 def c_support_code(self): ccodes = blas_headers.blas_header_text() if self.blas_type == 'openblas': ccodes += blas_headers.openblas_threads_text() elif self.blas_type == 'mkl': ccodes += blas_headers.mkl_threads_text() return ccodes def c_libraries(self): return ldflags() def c_compile_args(self): compile_args = ldflags(libs=False, flags=True) compile_args += super(BaseCorrMM, self).c_compile_args() return compile_args def c_lib_dirs(self): return ldflags(libs=False, libs_dir=True) def c_header_dirs(self): return ldflags(libs=False, include_dir=True) def c_headers(self): headers = [''] headers += super(BaseCorrMM, self).c_headers() return headers def c_code_cache_version(self): # raise this whenever modifying any of the support_code_files return (10, self.openmp, blas_header_version()) def c_support_code_apply(self, node, nodename): # REMEMBER TO RAISE c_code_cache_version when changing any of # these files sub = {} dtype = str(node.__dict__['inputs'][0].dtype) assert dtype in ('float32', 'float64') if dtype == 'float32': sub['gemm'] = 'sgemm_' sub['gemv'] = 'sgemv_' sub['float_type'] = 'npy_float' sub['float_typenum'] = 'NPY_FLOAT' sub['n_bytes'] = 4 sub['c_float_type'] = 'float' else: sub['gemm'] = 'dgemm_' sub['gemv'] = 'dgemv_' sub['float_type'] = 'npy_double' sub['float_typenum'] = 'NPY_DOUBLE' sub['n_bytes'] = 8 sub['c_float_type'] = 'double' if self.openmp: sub['omp_flags'] = '#pragma omp parallel for schedule(static)' sub['omp_get_max_threads'] = 'omp_get_max_threads()' sub['omp_get_thread_num'] = 'omp_get_thread_num()' if self.blas_type == 'openblas': sub['blas_set_num_threads'] = 'openblas_set_num_threads' sub['blas_get_num_threads'] = 'openblas_get_num_threads()' elif self.blas_type == 'mkl': sub['blas_set_num_threads'] = 'mkl_set_num_threads' sub['blas_get_num_threads'] = 'mkl_get_max_threads()' else: sub['blas_set_num_threads'] = '' sub['blas_get_num_threads'] = '0' else: sub['omp_flags'] = '' sub['omp_get_max_threads'] = '1' sub['omp_get_thread_num'] = '0' sub['blas_set_num_threads'] = '' sub['blas_get_num_threads'] = '0' files = [os.path.join('c_code', 'corr_gemm.c')] codes = [open(os.path.join(os.path.split(__file__)[0], f)).read() for f in files] final_code = '' for code in codes: final_code += code return final_code % sub def c_code_helper(self, bottom, weights, top, sub, height=None, width=None): """ This generates the C code for CorrMM (direction="forward"), CorrMM_gradWeights (direction="backprop weights"), and CorrMM_gradInputs (direction="backprop inputs"). Depending on the direction, one of bottom, weights, top will receive the output, while the other two serve as inputs. :param bottom: Variable name of the input images in the forward pass, or the gradient of the input images in backprop wrt. inputs :param weights: Variable name of the filters in the forward pass, or the gradient of the filters in backprop wrt. weights :param top: Variable name of the output images / feature maps in the forward pass, or the gradient of the outputs in the backprop passes :param sub: Dictionary of substitutions useable to help generating the C code. :param height: If self.subsample[0] != 1, a variable giving the height of the filters for direction="backprop weights" or the height of the input images for direction="backprop inputs". If self.border_mode == 'half', a variable giving the height of the filters for direction="backprop weights". Ignored otherwise. :param width: If self.subsample[1] != 1, a variable giving the width of the filters for direction="backprop weights" or the width of the input images for direction="backprop inputs". If self.border_mode == 'half', a variable giving the width of the filters for direction="backprop weights". Ignored otherwise. """ # When subsampling, we cannot unambiguously infer the height and width # of bottom and weights from top, so we require them to be given. # Similarly, when border_mode="half", we cannot infer the weight size. if height: height = '(*(npy_int64 *)(PyArray_DATA(%s)))' % height else: if ((self.direction != 0) and (self.dH != 1)) or ((self.direction == 1) and (self.padH_l == -1 or self.padH_r == -1)): raise ValueError("height must be given for backprop with vertical sampling or border_mode='half'") height = '-1' if width: width = '(*(npy_int64 *)(PyArray_DATA(%s)))' % width else: if ((self.direction != 0) and (self.dW != 1)) or ((self.direction == 1) and (self.padW_l == -1 or self.padW_r == -1)): raise ValueError("width must be given for backprop with horizontal sampling or border_mode='half'") width = '-1' return """ // Mandatory args int direction = %(params)s->direction; // forward, bprop weights, bprop inputs // Optional args int dH = %(params)s->dH; int dW = %(params)s->dW; int dilH = %(params)s->dilH; int dilW = %(params)s->dilW; int padH_l = %(params)s->padH_l; int padH_r = %(params)s->padH_r; int padW_l = %(params)s->padW_l; int padW_r = %(params)s->padW_r; int numgroups = %(params)s->num_groups; int unshared = %(params)s->unshared; PyArrayObject * bottom = %(bottom)s; PyArrayObject * weights = %(weights)s; PyArrayObject * top = %(top)s; PyArrayObject * out2 = NULL; PyArrayObject **out = NULL; switch(%(params)s->direction) { case DIRECTION_FORWARD: out = &%(top)s; break; case DIRECTION_BACKPROP_WEIGHTS: out = &%(weights)s; break; case DIRECTION_BACKPROP_INPUTS: out = &%(bottom)s; break; default: PyErr_SetString(PyExc_ValueError, "CPU CorrMM: Invalid direction."); {%(fail)s} break; } int wdim, odim; wdim = unshared ? 6 : 4; odim = 4; //Can be set to 6 later for unshared backprop wrt weights // Obtain or infer kernel width and height // (we need to know it early to be able to handle auto-padding) int kH, kW, dil_kH, dil_kW; if (direction != 1) { // weight is an input variable, we can just read its shape kH = PyArray_DIMS(weights)[wdim-2]; kW = PyArray_DIMS(weights)[wdim-1]; } else { if (%(height)s != -1) { // kernel height is specified (perhaps vertical subsampling or half padding) kH = %(height)s; } else if (padH_l == -2 || padH_r == -2) { // vertical full padding, we can infer the kernel height kH = (2 - PyArray_DIMS(bottom)[2] + (PyArray_DIMS(top)[2] - 1) * dH - 1)/ dilH + 1; } else { // explicit padding, we can infer the kernel height kH = (PyArray_DIMS(bottom)[2] + padH_l + padH_r - (PyArray_DIMS(top)[2] - 1) * dH - 1) / dilH +1; } if (%(width)s != -1) { // kernel width is specified (perhaps horizontal subsampling or half padding) kW = %(width)s; } else if (padW_l == -2 || padW_r == -2) { kW = (2 - PyArray_DIMS(bottom)[3] + (PyArray_DIMS(top)[3] - 1) * dW - 1) / dilW + 1; } else { kW = (PyArray_DIMS(bottom)[3] + padW_l + padW_r - (PyArray_DIMS(top)[3] - 1) * dW - 1) / dilW + 1; } } // Implicit dilated kernel size dil_kH = (kH - 1) * dilH + 1; dil_kW = (kW - 1) * dilW + 1; // Auto-padding if requested if (padH_l == -1 || padH_r == -1) { // vertical half padding padH_l = padH_r = dil_kH / 2; } else if (padH_l == -2 || padH_r == -2) { // vertical full padding padH_l = padH_r = dil_kH - 1; } else if (padH_l < -2 || padH_r < -2) { PyErr_SetString(PyExc_ValueError, "BaseCorrMM: padH_l and padH_r must be >= -2"); %(fail)s } if (padW_l == -1 || padW_r == -1) { // horizontal half padding padW_l = padW_r = dil_kW / 2; } else if (padW_l == -2 || padW_r == -2) { // horizontal full padding padW_l = padW_r = dil_kW - 1; } else if (padW_l < -2 || padW_r < -2) { PyErr_SetString(PyExc_ValueError, "BaseCorrMM: padW_l and padW_r must be >= -2"); %(fail)s } // Infer output shape npy_intp out_dim[6]; out_dim[4] = out_dim[5] = 0; //Only used for unshared backprop wrt weights switch(direction) { case 0: // forward pass // output is top: (batchsize, num_filters, height, width) // height and width: top = (bottom + pad_l + pad_r - ((weight-1)*dil + 1)) / sample + 1 out_dim[0] = (npy_intp)PyArray_DIMS(bottom)[0]; out_dim[1] = (npy_intp)PyArray_DIMS(weights)[0]; out_dim[2] = (npy_intp)((PyArray_DIMS(bottom)[2] + padH_l + padH_r - ((PyArray_DIMS(weights)[wdim-2]-1)*dilH + 1)) / dH + 1); out_dim[3] = (npy_intp)((PyArray_DIMS(bottom)[3] + padW_l + padW_r - ((PyArray_DIMS(weights)[wdim-1]-1)*dilW + 1)) / dW + 1); if (out_dim[0] < 0 || out_dim[1] < 0 || out_dim[2] <= 0 || out_dim[3] <= 0) { if (unshared) { PyErr_Format(PyExc_ValueError, "CorrMM: impossible output shape\\n" " bottom shape: %%ld x %%ld x %%ld x %%ld\\n" " weights shape: %%ld x %%ld x %%ld x %%ld x %%ld x %%ld\\n" " top shape: %%ld x %%ld x %%ld x %%ld\\n", (long int)PyArray_DIMS(bottom)[0], (long int)PyArray_DIMS(bottom)[1], (long int)PyArray_DIMS(bottom)[2], (long int)PyArray_DIMS(bottom)[3], (long int)PyArray_DIMS(weights)[0], (long int)PyArray_DIMS(weights)[1], (long int)PyArray_DIMS(weights)[2], (long int)PyArray_DIMS(weights)[3], (long int)PyArray_DIMS(weights)[4], (long int)PyArray_DIMS(weights)[5], (long int)out_dim[0], (long int)out_dim[1], (long int)out_dim[2], (long int)out_dim[3]); } else { PyErr_Format(PyExc_ValueError, "CorrMM: impossible output shape\\n" " bottom shape: %%ld x %%ld x %%ld x %%ld\\n" " weights shape: %%ld x %%ld x %%ld x %%ld\\n" " top shape: %%ld x %%ld x %%ld x %%ld\\n", (long int)PyArray_DIMS(bottom)[0], (long int)PyArray_DIMS(bottom)[1], (long int)PyArray_DIMS(bottom)[2], (long int)PyArray_DIMS(bottom)[3], (long int)PyArray_DIMS(weights)[0], (long int)PyArray_DIMS(weights)[1], (long int)PyArray_DIMS(weights)[2], (long int)PyArray_DIMS(weights)[3], (long int)out_dim[0], (long int)out_dim[1], (long int)out_dim[2], (long int)out_dim[3]); } %(fail)s } break; case 1: // backprop wrt. weights // output is weights: (num_filters, num_channels, height, width) // height and width: weights = (bottom + pad_l + pad_r - (top - 1) * sample - 1) / dil + 1 out_dim[0] = (npy_intp)PyArray_DIMS(top)[1]; if (unshared){ odim = 6; out_dim[1] = (npy_intp)PyArray_DIMS(top)[2]; out_dim[2] = (npy_intp)PyArray_DIMS(top)[3]; } out_dim[wdim-3] = (npy_intp)PyArray_DIMS(bottom)[1] / numgroups; out_dim[wdim-2] = (npy_intp)kH; // already inferred further above out_dim[wdim-1] = (npy_intp)kW; // how convenient if (unshared) { if (out_dim[0] < 0 || out_dim[1] <= 0 || out_dim[2] <= 0 || out_dim[3] < 0 || out_dim[4] <= 0 || out_dim[5] <= 0){ PyErr_Format(PyExc_ValueError, "CorrMM backprop wrt. weights: impossible output shape\\n" " bottom shape: %%ld x %%ld x %%ld x %%ld\\n" " weights shape: %%ld x %%ld x %%ld x %%ld x %%ld x %%ld\\n" " top shape: %%ld x %%ld x %%ld x %%ld\\n", (long int)PyArray_DIMS(bottom)[0], (long int)PyArray_DIMS(bottom)[1], (long int)PyArray_DIMS(bottom)[2], (long int)PyArray_DIMS(bottom)[3], (long int)out_dim[0], (long int)out_dim[1], (long int)out_dim[2], (long int)out_dim[3], (long int)out_dim[4], (long int)out_dim[5], (long int)PyArray_DIMS(top)[0], (long int)PyArray_DIMS(top)[1], (long int)PyArray_DIMS(top)[2], (long int)PyArray_DIMS(top)[3]); %(fail)s } } else { if (out_dim[0] < 0 || out_dim[1] < 0 || out_dim[2] <= 0 || out_dim[3] <= 0) { PyErr_Format(PyExc_ValueError, "CorrMM backprop wrt. weights: impossible output shape\\n" " bottom shape: %%ld x %%ld x %%ld x %%ld\\n" " weights shape: %%ld x %%ld x %%ld x %%ld\\n" " top shape: %%ld x %%ld x %%ld x %%ld\\n", (long int)PyArray_DIMS(bottom)[0], (long int)PyArray_DIMS(bottom)[1], (long int)PyArray_DIMS(bottom)[2], (long int)PyArray_DIMS(bottom)[3], (long int)out_dim[0], (long int)out_dim[1], (long int)out_dim[2], (long int)out_dim[3], (long int)PyArray_DIMS(top)[0], (long int)PyArray_DIMS(top)[1], (long int)PyArray_DIMS(top)[2], (long int)PyArray_DIMS(top)[3]); %(fail)s } } break; case 2: // backprop wrt. inputs // output is bottom: (batchsize, num_channels, height, width) // height and width: bottom = (top - 1) * sample + (weights-1)*dil + 1 - 2*pad out_dim[0] = (npy_intp)PyArray_DIMS(top)[0]; out_dim[1] = (npy_intp)PyArray_DIMS(weights)[wdim-3] * numgroups; out_dim[2] = (npy_intp)((%(height)s != -1) ? %(height)s : (PyArray_DIMS(top)[2] - 1) * dH + (PyArray_DIMS(weights)[wdim-2]-1)*dilH + 1 - padH_l - padH_r); out_dim[3] = (npy_intp)((%(width)s != -1) ? %(width)s : (PyArray_DIMS(top)[3] - 1) * dW + (PyArray_DIMS(weights)[wdim-1]-1)*dilW + 1 - padW_l - padW_r); if (unshared) { if (out_dim[0] < 0 || out_dim[1] < 0 || out_dim[2] <= 0 || out_dim[3] <= 0) { PyErr_Format(PyExc_ValueError, "CorrMM backprop wrt. inputs: impossible output shape\\n" " bottom shape: %%ld x %%ld x %%ld x %%ld\\n" " weights shape: %%ld x %%ld x %%ld x %%ld x %%ld x %%ld\\n" " top shape: %%ld x %%ld x %%ld x %%ld\\n", (long int)out_dim[0], (long int)out_dim[1], (long int)out_dim[2], (long int)out_dim[3], (long int)PyArray_DIMS(weights)[0], (long int)PyArray_DIMS(weights)[1], (long int)PyArray_DIMS(weights)[2], (long int)PyArray_DIMS(weights)[3], (long int)PyArray_DIMS(weights)[4], (long int)PyArray_DIMS(weights)[5], (long int)PyArray_DIMS(top)[0], (long int)PyArray_DIMS(top)[1], (long int)PyArray_DIMS(top)[2], (long int)PyArray_DIMS(top)[3]); %(fail)s } } else { if (out_dim[0] < 0 || out_dim[1] < 0 || out_dim[2] <= 0 || out_dim[3] <= 0) { PyErr_Format(PyExc_ValueError, "CorrMM backprop wrt. inputs: impossible output shape\\n" " bottom shape: %%ld x %%ld x %%ld x %%ld\\n" " weights shape: %%ld x %%ld x %%ld x %%ld\\n" " top shape: %%ld x %%ld x %%ld x %%ld\\n", (long int)out_dim[0], (long int)out_dim[1], (long int)out_dim[2], (long int)out_dim[3], (long int)PyArray_DIMS(weights)[0], (long int)PyArray_DIMS(weights)[1], (long int)PyArray_DIMS(weights)[2], (long int)PyArray_DIMS(weights)[3], (long int)PyArray_DIMS(top)[0], (long int)PyArray_DIMS(top)[1], (long int)PyArray_DIMS(top)[2], (long int)PyArray_DIMS(top)[3]); %(fail)s } } break; default: PyErr_SetString(PyExc_ValueError, "BaseCorrMM: direction must be 0, 1, or 2\\n"); %(fail)s } // Prepare output array int typenum; int failure; failure = !(*out && PyArray_NDIM(*out)==odim && PyArray_IS_C_CONTIGUOUS(*out) && PyArray_DIMS(*out)[0]==out_dim[0] && PyArray_DIMS(*out)[1]==out_dim[1] && PyArray_DIMS(*out)[2]==out_dim[2] && PyArray_DIMS(*out)[3]==out_dim[3]); if (odim == 6){ failure = failure || !(PyArray_DIMS(*out)[4]==out_dim[4] && PyArray_DIMS(*out)[5]==out_dim[5]); } if ( failure ) { Py_XDECREF(*out); if (direction != 1) { typenum = PyArray_TYPE(weights); } else { typenum = PyArray_TYPE(bottom); } //Change to PyArray_ZEROS which is faster than PyArray_EMPTY. *out = (PyArrayObject*)PyArray_ZEROS(odim, out_dim, typenum, 0); if (NULL == *out) { if (odim == 4) { PyErr_Format(PyExc_RuntimeError, "BaseCorrMM: Failed to allocate output of %%lld x %%lld x %%lld x %%lld", (long long)out_dim[0], (long long)out_dim[1], (long long)out_dim[2], (long long)out_dim[3]); } if (odim == 6) { PyErr_Format(PyExc_RuntimeError, "BaseCorrMM: Failed to allocate output of %%lld x %%lld x %%lld x %%lld %%lld %%lld", (long long)out_dim[0], (long long)out_dim[1], (long long)out_dim[2], (long long)out_dim[3], (long long)out_dim[4], (long long)out_dim[5]); } %(fail)s } } // Call corrMM code out2 = corrMM(%(bottom)s, %(weights)s, %(top)s, direction, dH, dW, dilH, dilW, padH_l, padH_r, padW_l, padW_r, numgroups, unshared); if (out2==NULL){ %(fail)s } assert (out2 == *out); """ % dict(bottom=bottom, weights=weights, top=top, height=height, width=width, fail=sub['fail'], params=sub['params']) class CorrMM(BaseCorrMM): """ CPU correlation implementation using Matrix Multiplication. Parameters ---------- border_mode The width of a border of implicit zeros to pad the input with. Must be a tuple with 2 elements giving the numbers of rows and columns to pad on each side, or a single integer to pad the same on all sides, or a string shortcut setting the padding at runtime: ``'valid'`` for ``(0, 0)`` (valid convolution, no padding), ``'full'`` for ``(kernel_rows - 1, kernel_columns - 1)`` (full convolution), ``'half'`` for ``(kernel_rows // 2, kernel_columns // 2)`` (same convolution for odd-sized kernels). If it is a tuple containing 2 pairs of integers, then these specify the padding to be applied on each side ((left, right), (top, bottom)). Otherwise, each width is applied twice, once per side (left and right, top and bottom). subsample The subsample operation applied to each output image. Should be a tuple with 2 elements. `(sv, sh)` is equivalent to `CorrMM(...)(...)[:,:,::sv, ::sh]`, but faster. Set to `(1, 1)` to disable subsampling. filter_dilation The filter dilation operation applied to each input image. Should be a tuple with 2 elements. Set to `(1, 1)` to disable filter dilation. num_groups Divides the image, kernel and output tensors into num_groups separate groups. Each which carry out convolutions separately. Should be an integer. unshared Boolean value. If true, then a different filter will be applied to each region of the input image. """ _direction = "forward" def make_node(self, img, kern): img = as_tensor_variable(img) kern = as_tensor_variable(kern) img, kern = self.as_common_dtype(img, kern) if img.type.ndim != 4: raise TypeError('img must be 4D tensor') if self.unshared is True: if kern.type.ndim != 6: raise TypeError('kern must be 6D tensor') else: if kern.type.ndim != 4: raise TypeError('kern must be 4D tensor') broadcastable = [img.type.broadcastable[0], kern.type.broadcastable[0], False, False] dtype = img.type.dtype return Apply(self, [img, kern], [TensorType(dtype, broadcastable)()]) def infer_shape(self, node, input_shape): imshp = input_shape[0] kshp = input_shape[1] res = get_conv_output_shape( imshp, kshp, self.border_mode, self.subsample, self.filter_dilation) return [res] def c_code(self, node, nodename, inp, out_, sub): bottom, weights = inp top, = out_ return super(CorrMM, self).c_code_helper(bottom, weights, top, sub) def grad(self, inp, grads): bottom, weights = inp top, = grads d_bottom = CorrMM_gradInputs(self.border_mode, self.subsample, self.filter_dilation, self.num_groups, self.unshared)(weights, top, bottom.shape[-2:]) d_weights = CorrMM_gradWeights(self.border_mode, self.subsample, self.filter_dilation, self.num_groups, self.unshared)(bottom, top, weights.shape[-2:]) return d_bottom, d_weights class CorrMM_gradWeights(BaseCorrMM): """ Gradient wrt. filters for `CorrMM`. Notes ----- You will not want to use this directly, but rely on Theano's automatic differentiation or graph optimization to use it as needed. """ _direction = "backprop weights" def make_node(self, img, topgrad, shape=None): img = as_tensor_variable(img) topgrad = as_tensor_variable(topgrad) img, topgrad = self.as_common_dtype(img, topgrad) if img.type.ndim != 4: raise TypeError('img must be 4D tensor') if topgrad.type.ndim != 4: raise TypeError('topgrad must be 4D tensor') if shape is None: if self.subsample != (1, 1) or self.border_mode == "half": raise ValueError('shape must be given if subsample != (1, 1)' ' or border_mode == "half"') height_width = [] else: height_width = [as_tensor_variable(shape[0]).astype('int64'), as_tensor_variable(shape[1]).astype('int64')] if self.unshared is True: broadcastable = [topgrad.type.broadcastable[1], False, False, img.type.broadcastable[1], False, False] else: broadcastable = [topgrad.type.broadcastable[1], img.type.broadcastable[1], False, False] dtype = img.type.dtype return Apply(self, [img, topgrad] + height_width, [TensorType(dtype, broadcastable)()]) def infer_shape(self, node, input_shape): if self.border_mode == "half": padH_l = padH_r = padW_l = padW_r = -1 elif self.border_mode == "full": padH_l = padH_r = padW_l = padW_r = -2 elif isinstance(self.border_mode, tuple): border = () for mode in self.border_mode: if isinstance(mode, tuple): border += ((int(mode[0]), int(mode[1])),) else: border += ((int(mode), int(mode)),) (padH_l, padH_r), (padW_l, padW_r) = border else: assert self.border_mode == "valid" padH_l = padH_r = padW_l = padW_r = 0 dH, dW = self.subsample imshp = input_shape[0] topshp = input_shape[1] ssize, imshp = imshp[1], list(imshp[2:]) ssize = ssize // self.num_groups nkern, topshp = topshp[1], list(topshp[2:]) height_width = node.inputs[-2:] if ((dH != 1) or (padH_l == -1) or (padH_r == -1)): # vertical subsampling or half padding, kernel height is specified kH = height_width[0] elif (padH_l == -2) or (padH_r == -2): # vertical full padding, we can infer the kernel height kH = 2 - imshp[0] + (topshp[0] - 1) * dH else: # explicit padding, we can infer the kernel height kH = imshp[0] + padH_l + padH_r - (topshp[0] - 1) * dH if ((dW != 1) or (padW_l == -1) or (padW_r == -1)): kW = height_width[1] elif (padW_l == -2) or (padW_r == -2): kW = 2 - imshp[1] + (topshp[1] - 1) * dW else: kW = imshp[1] + padW_l + padW_r - (topshp[1] - 1) * dW if self.unshared is True: return [(nkern, topshp[0], topshp[1], ssize, kH, kW)] else: return [(nkern, ssize, kH, kW)] def c_code(self, node, nodename, inp, out_, sub): bottom, top = inp[:2] height, width = inp[2:] or (None, None) weights, = out_ return super(CorrMM_gradWeights, self).c_code_helper(bottom, weights, top, sub, height, width) def grad(self, inp, grads): bottom, top = inp[:2] weights, = grads d_bottom = CorrMM_gradInputs(self.border_mode, self.subsample, self.filter_dilation, self.num_groups, self.unshared)(weights, top, bottom.shape[-2:]) d_top = CorrMM(self.border_mode, self.subsample, self.filter_dilation, self.num_groups, self.unshared)(bottom, weights) d_height_width = ((theano.gradient.DisconnectedType()(),) * 2 if len(inp) == 4 else ()) return (d_bottom, d_top) + d_height_width def connection_pattern(self, node): if node.nin == 2: return [[1], [1]] else: return [[1], [1], [0], [0]] # no connection to height, width class CorrMM_gradInputs(BaseCorrMM): """ Gradient wrt. inputs for `CorrMM`. Notes ----- You will not want to use this directly, but rely on Theano's automatic differentiation or graph optimization to use it as needed. """ _direction = "backprop inputs" def make_node(self, kern, topgrad, shape=None): kern = as_tensor_variable(kern) topgrad = as_tensor_variable(topgrad) kern, topgrad = self.as_common_dtype(kern, topgrad) if self.unshared is True: if kern.type.ndim != 6: raise TypeError('kern must be 6D tensor') else: if kern.type.ndim != 4: raise TypeError('kern must be 4D tensor') if topgrad.type.ndim != 4: raise TypeError('topgrad must be 4D tensor') if shape is None: if self.subsample != (1, 1): raise ValueError('shape must be given if subsample != (1, 1)') height_width = [] else: height_width = [as_tensor_variable(shape[0]).astype('int64'), as_tensor_variable(shape[1]).astype('int64')] if self.num_groups > 1: broadcastable = [topgrad.type.broadcastable[0], False, False, False] else: broadcastable = [topgrad.type.broadcastable[0], kern.type.broadcastable[-3], False, False] dtype = kern.type.dtype return Apply(self, [kern, topgrad] + height_width, [TensorType(dtype, broadcastable)()]) def infer_shape(self, node, input_shape): if self.border_mode == "half": padH_l = padH_r = padW_l = padW_r = -1 elif self.border_mode == "full": padH_l = padH_r = padW_l = padW_r = -2 elif isinstance(self.border_mode, tuple): border = () for mode in self.border_mode: if isinstance(mode, tuple): border += ((int(mode[0]), int(mode[1])),) else: border += ((int(mode), int(mode)),) (padH_l, padH_r), (padW_l, padW_r) = border else: assert self.border_mode == "valid" padH_l = padH_r = padW_l = padW_r = 0 dH, dW = self.subsample kshp = input_shape[0] topshp = input_shape[1] ssize, kshp = kshp[-3], list(kshp[-2:]) ssize = ssize * self.num_groups bsize, topshp = topshp[0], list(topshp[2:]) height_width = node.inputs[-2:] if padH_l == -1 or padH_r == -1: padH_l = padH_r = kshp[0] // 2 elif padH_l == -2 or padH_r == -2: padH_l = padH_r = kshp[0] - 1 elif padH_l < -2 or padH_r < -2: raise ValueError('CorrMM_gradInputs: border_mode must be >= 0.') if padW_l == -1 or padW_r == -1: padW_l = padW_r = kshp[1] // 2 elif padW_l == -2 or padW_r == -2: padW_l = padW_r = kshp[1] - 1 elif padW_l < -2 or padW_r < -2: raise ValueError('CorrMM_gradInputs: border_mode must be >= 0.') if dH != 1: out_shp0 = height_width[0] else: out_shp0 = (topshp[0] - 1) * dH + kshp[0] - padH_l - padH_r if dW != 1: out_shp1 = height_width[1] else: out_shp1 = (topshp[1] - 1) * dW + kshp[1] - padW_l - padW_r out_shp = (out_shp0, out_shp1) return [(bsize, ssize) + out_shp] def c_code(self, node, nodename, inp, out_, sub): weights, top = inp[:2] height, width = inp[2:] or (None, None) bottom, = out_ return super(CorrMM_gradInputs, self).c_code_helper(bottom, weights, top, sub, height, width) def grad(self, inp, grads): weights, top = inp[:2] bottom, = grads d_weights = CorrMM_gradWeights(self.border_mode, self.subsample, self.filter_dilation, self.num_groups, self.unshared)(bottom, top, weights.shape[-2:]) d_top = CorrMM(self.border_mode, self.subsample, self.filter_dilation, self.num_groups, self.unshared)(bottom, weights) d_height_width = ((theano.gradient.DisconnectedType()(),) * 2 if len(inp) == 4 else ()) return (d_weights, d_top) + d_height_width def connection_pattern(self, node): if node.nin == 2: return [[1], [1]] else: return [[1], [1], [0], [0]] # no connection to height, width Theano-1.0.0/theano/tensor/nnet/conv.py00006000163562€ÿÿÿþ0000026714013203335532021354 0ustar boccoset00000000000000""" Contains an Op for convolving input images with a set of filters. This was developed especially for Convolutional Neural Networks. For related ops, including downsampling and subsampling, see tensor.signal and tensor.signal.pool. See especially conv2d(). """ from __future__ import absolute_import, print_function, division import logging import numpy as np from six.moves import xrange import warnings import theano from theano import OpenMPOp from theano.tensor import (as_tensor_variable, blas, get_scalar_constant_value, patternbroadcast, NotScalarConstantError) from theano.gof import Apply from theano.tensor.nnet.abstract_conv import (get_conv_output_shape, get_conv_shape_1axis) try: # TODO: move these back out to global scope when they no longer # cause an atexit error from scipy.signal.signaltools import _valfrommode, _bvalfromboundary from scipy.signal.sigtools import _convolve2d imported_scipy_signal = True except ImportError: imported_scipy_signal = False __docformat__ = "restructuredtext en" _logger = logging.getLogger("theano.tensor.nnet.conv") def conv2d(input, filters, image_shape=None, filter_shape=None, border_mode='valid', subsample=(1, 1), **kargs): """ Deprecated, old conv2d interface. This function will build the symbolic graph for convolving a stack of input images with a set of filters. The implementation is modelled after Convolutional Neural Networks (CNN). It is simply a wrapper to the ConvOp but provides a much cleaner interface. Parameters ---------- input : symbolic 4D tensor Mini-batch of feature map stacks, of shape (batch size, stack size, nb row, nb col) see the optional parameter image_shape filters: symbolic 4D tensor Set of filters used in CNN layer of shape (nb filters, stack size, nb row, nb col) see the optional parameter filter_shape border_mode : {'valid', 'full'} 'valid'only apply filter to complete patches of the image. Generates output of shape: image_shape - filter_shape + 1. 'full' zero-pads image to multiple of filter shape to generate output of shape: image_shape + filter_shape - 1. subsample: tuple of len 2 Factor by which to subsample the output. Also called strides elsewhere. image_shape: None, tuple/list of len 4 of int, None or Constant variable The shape of the input parameter. Optional, used for optimization like loop unrolling You can put None for any element of the list to tell that this element is not constant. filter_shape : None, tuple/list of len 4 of int, None or Constant variable Optional, used for optimization like loop unrolling You can put None for any element of the list to tell that this element is not constant. kwargs Kwargs are passed onto ConvOp. Can be used to set the following: unroll_batch, unroll_kern, unroll_patch, openmp (see ConvOp doc). openmp: By default have the same value as config.openmp. For small image, filter, batch size, nkern and stack size, it can be faster to disable manually openmp. A fast and incomplete test show that with image size 6x6, filter size 4x4, batch size==1, n kern==1 and stack size==1, it is faster to disable it in valid mode. But if we grow the batch size to 10, it is faster with openmp on a core 2 duo. Returns ------- symbolic 4D tensor Set of feature maps generated by convolutional layer. Tensor is of shape (batch size, nb filters, output row, output col). """ warnings.warn("theano.tensor.nnet.conv.conv2d is deprecated." " Use theano.tensor.nnet.conv2d instead.") # accept Constant value for image_shape and filter_shape. if image_shape is not None: image_shape = list(image_shape) for i in xrange(len(image_shape)): if image_shape[i] is not None: try: image_shape[i] = get_scalar_constant_value( as_tensor_variable(image_shape[i])) except NotScalarConstantError: raise NotScalarConstantError( "The convolution need that the shape" " information are constant values. We got" " %s for the image_shape parameter" % image_shape[i]) assert image_shape[i].dtype in theano.tensor.discrete_dtypes image_shape[i] = int(image_shape[i]) if filter_shape is not None: filter_shape = list(filter_shape) for i in xrange(len(filter_shape)): if filter_shape[i] is not None: try: filter_shape[i] = get_scalar_constant_value( as_tensor_variable(filter_shape[i])) except NotScalarConstantError: raise NotScalarConstantError( "The convolution need that the shape" " information are constant values. We got" " %s for the filter_shape " "parameter" % filter_shape[i]) assert filter_shape[i].dtype in theano.tensor.discrete_dtypes filter_shape[i] = int(filter_shape[i]) if image_shape and filter_shape: try: if image_shape[1] is not None and filter_shape[1] is not None: assert image_shape[1] == filter_shape[1] except Exception: print('image ', image_shape, ' filters ', filter_shape) raise if filter_shape is not None: nkern = filter_shape[0] kshp = filter_shape[2:] else: nkern, kshp = None, None if image_shape is not None: bsize = image_shape[0] imshp = image_shape[1:] else: bsize, imshp = None, None op = ConvOp(output_mode=border_mode, dx=subsample[0], dy=subsample[1], imshp=imshp, kshp=kshp, nkern=nkern, bsize=bsize, **kargs) return op(input, filters) class ConvOp(OpenMPOp): """ This Op serves a dual purpose: it can implement a vanilla 2D convolution (as taught in any signal processing class) or implement the convolutional layers found in Convolutional Neural Networks. In this setting, a set of 3D images is convolved with a set of 3D kernels, with the particularity that their leading dimensions are of equal length. Vanilla 2D convolution is treated as a special case of this. The input parameter represents a mini-batch of multiple images. Its shape is: batch size x num. input feature maps x image height x image width The kernel parameter represents a set of 3D kernels. Its shape is: number of filters x num. input images x filter height x filter width The output of ConvOp is a 4D tensor, generated as follows: output[b,k,:,:] = \sum_i input[b,i,:,:] * filter[k,i,:,:] \forall b,k where b is the mini-batch index, k the filter index and * is the convolution operator. The constructor initializes a ConvOp with given output_mode (full/valid). All other parameters are optional and are only used to generate more optimized c code, or to enable graph optimizers to optimally replace the ConvOp. NOTES ON OPTIMIZATION: There are two types of optimization. The first is the selection of the fastest algo when bsize and nkern are provided with imshp and kshp. By default we try to select the fastest version. You can specify it with the unroll_batch, unroll_kern, and unroll_patch parameter. The second type of optimization is hardcoding some dimensions into the code when all shape are know. This make a significant difference for the 'full' output_mode. Sometimes, the fastest implementation on x86-64 uses {unroll_batch=4, unroll_kern=4, unroll_patch=False} with all other shape parameters being provided. For optimizing other architectures, see: Kazushige Goto and Robert A. Van De Geijn, Anatomy of High-Performance Matrix Multiplication, (mr x nr). ACM Transactions on Mathematical Software, May 2008. Figure 12: (mr x nr). For x86 use 2x4, itanium 8x8, etc. Parameters ---------- output_mode : {'valid', 'full'} 'valid' gives an output smaller then the image. 'full' gives an output bigger then the image. See 'border_mode' in conv2d's doc. Optional parameters: (will generate more optimal c code) imshp : tuple of len 2 or 3: 2 for 2d image, 3 for a stack of 2d images. Stacksize, nb image row, nb image col. kshp : tuple of len 2 Nb kernel row, nb kernel col. nkern : int The number of kernel. bsize : int The size of the minibatch. dx : int Patch stride rows. dy : int Patch stride cols Params which select the version of code used: unroll_patch : bool Use a version of c_code that unroll the patch loop that don't request all shape information to work, but if all shape information are present, will use it to hardcode the value in the code for faster code. unroll_batch : int Use a version of c_code that unroll the batch (by unroll_batch) and the nkern (by unroll_kern) loop. The size must by a multiple of bsize or nkern respectively. unroll_kern : int Use a version of c_code that unroll the batch (by unroll_batch) and the nkern(by unroll_kern) loop. The size must by a multiple of bsize or nkern respectively. verbose : int Passed to GpuConv. version: int or str Passed to GpuConv, if version='no_fft', fft optimization will be desactivated at the op level. direction_hint: {'forward', 'bprop weights', 'bprop inputs'} Passed to GpuConv, used by graph optimizers to aid algorithm choice. The 3 following parameters are used internally when we generate the gradient when dx!=1 or dy!=1. imshp_logical Default None. None value is equivalent to imshp value. When imshp_logical != imshp, it tell we need to insert 0 in the image before we do the convolution. For example, when dx==dy==2 and the image is [[1, 2], [3, 4]], we should make as if the image was [[1, 0, 2, 0], [0, 0, 0, 0], [3, 0, 4, 0], [0, 0, 0, 0]]. Our python code insert the zero, but the c code optimize it. imshp_logical != imshp when taking the grad again the weights or the image when the output_mode is full and `dx != 1` or `dy != 1`. kshp_logical Idem but for kshp and used for the grad again the weights when the output_mode is valid and `dx != 1` or `dy != 1`. kshp_logical_top_aligned Used in the same case. Default to True. Set to False in the grad again the weight when the output_mode is full. """ __attrnames = ['imshp', 'kshp', 'nkern', 'bsize', 'dx', 'dy', 'out_mode', 'unroll_batch', 'unroll_kern', 'unroll_patch', 'imshp_logical', 'kshp_logical', 'kshp_logical_top_aligned'] """These attributes uniquely identify the behaviour of this op for given inputs. Do not set openmp here. """ # the value of speed_unroll_batch_kern,speed_unroll_patch_noshape,speed_unroll_patch_shape # have bean calculated on maggie36 when their is only 1 session logged on and only this was running. # It is an Intel(R) Xeon(R) CPU E5430 @ 2.66GHz. It is computer with theano/tensor/nnet/tests/speed_test_conv.py # and took 5 minutes to run. # TODO: we should compute this table for each computer/os as this can change. # I saw on one computer that the speed with the shape can be slower than without! # using the real shape and the same dtype could also help. # unroll_batch, unroll_kern, valid time, full time speed_unroll_batch_kern = [(1, 1, 2.4661250114440918, 6.5472931861877441), (1, 2, 1.5869178771972656, 5.1499760150909424), (1, 3, 1.4270510673522949, 3.6593470573425293), (1, 4, 1.3373479843139648, 3.3451821804046631), (1, 5, 1.2818830013275146, 3.1444568634033203), (1, 6, 1.2521560192108154, 3.0256359577178955), (1, 10, 1.2134110927581787, 2.9174180030822754), (2, 1, 1.657214879989624, 4.5261678695678711), (2, 2, 1.2123160362243652, 2.9747390747070312), (2, 3, 1.0758891105651855, 2.5690360069274902), (2, 4, 1.0683329105377197, 2.4233770370483398), (2, 5, 1.0955719947814941, 2.3999948501586914), (2, 6, 1.5935721397399902, 2.6878271102905273), (2, 10, 1.8511250019073486, 3.2417428493499756), (3, 1, 1.5948119163513184, 3.631148099899292), (3, 2, 1.0761330127716064, 2.6011371612548828), (3, 3, 1.0551531314849854, 2.4200370311737061), (3, 4, 1.3930759429931641, 2.5211219787597656), (3, 5, 1.4330689907073975, 2.5704989433288574), (3, 6, 1.362138032913208, 2.5964410305023193), (3, 10, 1.6582000255584717, 2.9907989501953125), (4, 1, 1.4793620109558105, 3.3473429679870605), (4, 2, 1.0671560764312744, 2.4171769618988037), (4, 3, 1.2569692134857178, 2.2807950973510742), (4, 4, 1.3456289768218994, 2.6219108104705811), (4, 5, 1.4055080413818359, 2.4606490135192871), (4, 6, 1.372107982635498, 2.551663875579834), (4, 10, 1.599470853805542, 2.9172940254211426), (5, 1, 1.4115700721740723, 3.2077109813690186), (5, 2, 1.0635769367218018, 2.2648060321807861), (5, 3, 1.3842809200286865, 2.6135518550872803), (5, 4, 1.3470511436462402, 2.3852400779724121), (5, 5, 1.3539440631866455, 2.5245928764343262), (5, 6, 1.4037849903106689, 2.5985310077667236), (5, 10, 1.6120610237121582, 2.8127608299255371), (6, 1, 1.3623628616333008, 3.021122932434082), (6, 2, 1.1697649955749512, 2.6285450458526611), (6, 3, 1.2980999946594238, 2.4746189117431641), (6, 4, 1.3739941120147705, 2.5579929351806641), (6, 5, 1.3967819213867188, 2.5522029399871826), (6, 6, 1.4279270172119141, 2.6127138137817383), (6, 10, 1.605496883392334, 2.864037036895752), (10, 1, 1.6401121616363525, 2.970099925994873), (10, 2, 1.46710205078125, 2.7231831550598145), (10, 3, 1.4193780422210693, 2.6087639331817627), (10, 4, 1.4657118320465088, 2.6246678829193115), (10, 5, 1.5052611827850342, 2.6542458534240723), (10, 6, 1.5214400291442871, 2.7243161201477051), (10, 10, 1.6116268634796143, 2.956165075302124)] # valid time, full time speed_unroll_patch_noshape = [2.0109100341796875, 5.8175678253173828] # valid time, full time speed_unroll_patch_shape = [1.2967290878295898, 5.5283889770507812] @staticmethod def has_all_shape(imshp, kshp, nkern=1, bsize=1): return (nkern is not None and bsize is not None and all(shp is not None for shp in imshp) and all(shp is not None for shp in kshp)) @staticmethod def getOutputShape(inshp, kshp, stride=(1, 1), mode='valid'): """ Computes the output dimensions of convolving an image of shape "inshp" with kernels of shape "kshp". Accepts symbolic or integer shapes. Propagates `None`s (for unknown shapes). Parameters ---------- inshp (rows,cols) of input image. kshp (rows,cols) of filters. mode: {'valid', 'full'} See 'border_mode' in conv2d's doc. Returns ------- object (rows,cols) of output image. """ # The formula would be ceil((i + s * k - s * 1) / float(d)), # with s=1 for mode=='full' and s=-1 for mode=='valid'. # To support symbolic shapes, we express this with integer arithmetics. warnings.warn("The method `getOutputShape` is deprecated use" "`get_conv_output_shape` instead.", stacklevel=2) return tuple(get_conv_shape_1axis(i, k, mode, d) for i, k, d in zip(inshp, kshp, stride)) def __init__(self, imshp=None, kshp=None, nkern=None, bsize=None, dx=1, dy=1, output_mode='valid', unroll_batch=None, unroll_kern=None, unroll_patch=None, imshp_logical=None, kshp_logical=None, kshp_logical_top_aligned=True, verbose=0, version=-1, direction_hint='forward', openmp=None): # Deactivate fft_optimization at the op level if specified if version == "no_fft": self.fft_opt = False version = -1 else: self.fft_opt = True # Expand unknown image / kernel shapes into tuples of Nones if imshp is None: imshp = (None, None, None) else: imshp = tuple(imshp) if kshp is None: kshp = (None, None) else: kshp = tuple(kshp) # Check imshp and kshp dimensionality if len(imshp) == 2: imshp = (1,) + imshp elif len(imshp) != 3: raise ValueError("len(imshp) must be 2 or 3, got %d" % len(imshp)) if len(kshp) != 2: raise ValueError("len(kshp) must be 2, got %d" % len(kshp)) # We must continue to consider None as 1 for backward compatibility. if dx is None: dx = 1 if dy is None: dy = 1 if int(dx) != dx: raise TypeError('ConvOp.__init__ param dx must be an int', dx) dx = int(dx) if int(dy) != dy: raise TypeError('ConvOp.__init__ param dy must be an int', dy) dy = int(dy) all_shape = self.has_all_shape(imshp, kshp, nkern, bsize) if (unroll_batch or unroll_kern) and not all_shape: raise Exception("In ConvOp, when using unroll_batch and" " unroll_nkern, all shape are needed") # Init the openmp attribute super(ConvOp, self).__init__(openmp=openmp) if not all_shape or self.openmp: # Only this version is parallelized unroll_patch = True self.imshp = imshp self.kshp = kshp self.nkern = nkern self.bsize = bsize self.dx = dx self.dy = dy self.verbose = verbose self.version = version self.direction_hint = direction_hint # a triple if imshp_logical is None: self.imshp_logical = self.imshp else: imshp_logical = tuple(imshp_logical) if len(imshp_logical) != 3: raise ValueError("len(imshp_logical) must be 3, got %d" % len(imshp_logical)) self.imshp_logical = imshp_logical # a pair if kshp_logical is None: self.kshp_logical = self.kshp else: kshp_logical = tuple(kshp_logical) if len(kshp_logical) != 2: raise ValueError("len(kshp_logical) must be 2, got %d" % len(kshp_logical)) self.kshp_logical = kshp_logical # a bool self.kshp_logical_top_aligned = kshp_logical_top_aligned self.unroll_batch = unroll_batch self.unroll_kern = unroll_kern self.unroll_patch = unroll_patch if self.unroll_batch and not self.unroll_kern: self.unroll_kern = 1 if self.unroll_kern and not self.unroll_batch: self.unroll_batch = 1 # downcast unroll_batch if not a divisor of batch size if self.unroll_batch is not None and self.unroll_batch > 0 and self.bsize % self.unroll_batch != 0: if self.bsize <= self.unroll_batch: self.unroll_batch = self.bsize else: # find the maximum value under unroll_batch that would work new = self.unroll_batch assert(new >= 1) while self.bsize % new != 0: new -= 1 warnstr = ("OPTIMISATION WARNING: in ConvOp.__init__() " "unroll_batch(%i) must be 0 or a divisor of" " bsize(%i). We revert it to %i. This" " won't change the result, but may make it slower.") _logger.warn(warnstr, self.unroll_batch, self.bsize, new) self.unroll_batch = new # downcast unroll_kern if not a divisor of nb of kernel if self.unroll_kern is not None and self.unroll_kern > 0 and self.nkern % self.unroll_kern != 0: if self.nkern <= self.unroll_kern: self.unroll_kern = self.nkern else: # find the maximum value under unroll_kern that would work new = self.unroll_kern assert(new >= 1) while self.nkern % new != 0: new -= 1 warnstr = ("OPTIMISATION WARNING: in ConvOp.__init__()" " unroll_kern(%i) should be 0 or a divisor of" " nkern(%i). We revert it to %i. This" " won't change the result, but may make it slower.") _logger.warn(warnstr, self.unroll_kern, self.nkern, new) self.unroll_kern = new self.outshp = get_conv_output_shape( (None,) + self.imshp_logical, (None, None,) + self.kshp_logical, output_mode, (dx, dy))[2:] self.fulloutshp = get_conv_output_shape( (None,) + self.imshp_logical, (None, None,) + self.kshp_logical, output_mode, (1, 1))[2:] self.out_mode = output_mode if self.out_mode not in ["valid", "full"]: raise Exception("Mode %s not implemented" % str(self.out_mode)) if any((shp is not None) and (shp <= 0) for shp in self.outshp): raise Exception("Bad size for the output shape. Verify that [post-" "supersampling] input shape (%s) and kern" " shape(%s) are ok. (Hint: kerns must fit inside" " image in valid mode)" % (self.imshp_logical, self.kshp_logical)) if (self.unroll_kern is None and self.unroll_batch is None and self.unroll_patch is None): # no version specified. Find the faster we have if self.bsize is None and self.nkern is None: self.unroll_patch = True elif self.bsize is not None and self.nkern is not None: bsize = self.bsize nkern = self.nkern mode_idx = 0 if self.out_mode != "valid": mode_idx = 1 if self.has_all_shape(self.imshp, self.kshp): time_unroll_patch = self.speed_unroll_patch_shape[mode_idx] else: time_unroll_patch = self.speed_unroll_patch_noshape[ mode_idx] time_unroll_batch_kern = 9999999 for i in xrange(len(self.speed_unroll_batch_kern)): if (bsize % self.speed_unroll_batch_kern[i][0] == 0 and nkern % self.speed_unroll_batch_kern[i][1] == 0): if self.speed_unroll_batch_kern[i][2 + mode_idx] < time_unroll_batch_kern: time_unroll_batch_kern = self.speed_unroll_batch_kern[i][2 + mode_idx] time_unroll_batch_kern_idx = i if time_unroll_patch < time_unroll_batch_kern: self.unroll_patch = True else: self.unroll_batch = self.speed_unroll_batch_kern[ time_unroll_batch_kern_idx][0] self.unroll_kern = self.speed_unroll_batch_kern[ time_unroll_batch_kern_idx][1] self.unroll_patch = False _logger.debug("AUTO FIND VERSION OF C_CODE OF CONV OP " "%s %s %s %s %s %s %s", self.unroll_batch, self.unroll_kern, self.unroll_patch, self.bsize, self.nkern, time_unroll_patch, time_unroll_batch_kern) self._rehash() def __eq__(self, other): if type(self) != type(other): return False for a in self.__attrnames: if getattr(self, a) != getattr(other, a): return False return True def __setstate__(self, d): super(ConvOp, self).__setstate__(d) self.direction_hint = d.get("direction_hint", None) self._rehash() def _rehash(self): hashval = hash(type(self)) for a in self.__attrnames: hashval = hashval ^ hash(getattr(self, a)) self.__hashval = hashval def __hash__(self): return self.__hashval def __str__(self): return "ConvOp{" + ",".join(str((a, getattr(self, a))) for a in self.__attrnames) + "}" def flops(self, inputs, outputs): """ Useful with the hack in profiling to print the MFlops. """ images, kerns = inputs out, = outputs assert images[1] == kerns[1] flops = 0 if self.out_mode == "valid": # nb mul and add by output pixel flops = kerns[2] * kerns[3] * 2 # nb flops by output image flops *= out[2] * out[3] # nb patch multiplied flops *= images[1] * kerns[0] * images[0] else: flops = (images[0] * kerns[0] * images[1] * kerns[2] * kerns[3] * images[2] * images[3] * 2) return flops def make_node(self, inputs, kerns): # TODO: find a way to make ConvOp work for N-D (after NIPS09) """ Parameters ---------- inputs 4 dim: batches x stacksize x rows x cols. kerns 4 dim: nkern x stackidx x rows x cols. """ _inputs = as_tensor_variable(inputs) _kerns = as_tensor_variable(kerns) # TODO: lift this restriction by upcasting either inputs or kerns if _inputs.ndim != 4: raise TypeError('ConvOp (make_node) requires input be a 4D tensor;' ' received "%s" (%i dims)' % (inputs, _inputs.ndim)) if _kerns.ndim != 4: raise TypeError('make_node requires 4D tensor of kernels') if _inputs.type.dtype != _kerns.type.dtype: raise NotImplementedError( "The image and the kernel must have the same type." "inputs(%s), kerns(%s)" % (_inputs.dtype, _kerns.dtype)) bcastable23 = [self.outshp[0] == 1, self.outshp[1] == 1] output = theano.tensor.tensor(dtype=_inputs.type.dtype, broadcastable=[_inputs.broadcastable[0], _kerns.broadcastable[0]] + bcastable23) return Apply(self, [_inputs, _kerns], [output]) def infer_shape(self, node, input_shapes): imshp = input_shapes[0] # 4D image shape kshp = input_shapes[1] # 4D filter shape bsize, imshp = imshp[0], list(imshp[1:]) nkern, kshp = kshp[0], list(kshp[2:]) # replace symbolic shapes with known shapes if self.bsize is not None: bsize = self.bsize for i in [0, 1, 2]: if self.imshp_logical[i] is not None: imshp[i] = self.imshp_logical[i] if self.nkern is not None: nkern = self.nkern for i in [0, 1]: if self.kshp_logical[i] is not None: kshp[i] = self.kshp_logical[i] # infer output shape from what we have res = get_conv_output_shape( (bsize,) + tuple(imshp), (nkern, None,) + tuple(kshp), self.out_mode, (self.dx, self.dy)) return [res] def perform(self, node, inp, out): """ By default if len(img2d.shape)==3, we TODO """ img2d, filtersflipped = inp z, = out if not imported_scipy_signal: raise theano.gof.utils.MethodNotDefined( "c_headers", type(self), self.__class__.__name__, "Need the python package for scipy.signal to be installed " "for the python implementation. You can use the C" " implementation instead.") # TODO: move these back out to global scope when they no longer # cause an atexit error imshp = self.imshp if any(x is None for x in imshp): imshp = tuple(img2d.shape[1:]) if imshp != img2d.shape[1:]: raise ValueError("The image shape provided at build time " "is different from the one passed at run time", imshp, img2d.shape[1:]) kshp = self.kshp if any(x is None for x in kshp): kshp = tuple(filtersflipped.shape[2:]) if kshp != filtersflipped.shape[2:]: raise ValueError("The filter shape provided at build time " "is different from the one passed at run time", kshp, filtersflipped.shape[2:]) bsize = self.bsize if bsize is None: bsize = img2d.shape[0] elif bsize != img2d.shape[0]: raise ValueError("The batch size provided at build time " "is different from the one passed at run time", bsize, img2d.shape[0]) nkern = self.nkern if nkern is None: nkern = filtersflipped.shape[0] elif nkern != filtersflipped.shape[0]: raise ValueError("The number of filters provided at build time " "is different from the one passed at run time", nkern, filtersflipped.shape[0]) imshp_logical = self.imshp_logical if imshp_logical[0] is None: imshp_logical = (imshp[0],) + imshp_logical[1:] if imshp_logical[1] is None: imshp_logical = (imshp_logical[0], imshp[1], imshp_logical[2]) if imshp_logical[2] is None: imshp_logical = imshp_logical[:2] + (imshp[2],) assert all(x is not None for x in imshp_logical) kshp_logical = self.kshp_logical if kshp_logical[0] is None: kshp_logical = (kshp[0], kshp_logical[1]) if kshp_logical[1] is None: kshp_logical = (kshp_logical[0], kshp[1]) assert all(x is not None for x in kshp_logical) if all(shp is not None for shp in self.fulloutshp): fulloutshp = tuple(self.fulloutshp) else: fulloutshp = get_conv_output_shape( (None,) + imshp_logical, (None, None,) + kshp_logical, self.out_mode, (1, 1))[2:] if z[0] is None or z[0].shape != (bsize, nkern,) + fulloutshp: z[0] = np.zeros((bsize, nkern,) + fulloutshp, dtype=img2d.dtype) zz = z[0] stacklen = imshp[0] img2d = img2d.reshape((bsize,) + imshp) filtersflipped = filtersflipped.reshape((nkern, stacklen) + kshp) if self.imshp != self.imshp_logical: # assuming that to get from imshp to imshp logical we insert zeros in missing spots rstride = int(np.ceil(imshp_logical[1] / float(imshp[1]))) cstride = int(np.ceil(imshp_logical[2] / float(imshp[2]))) buf = np.zeros((bsize,) + imshp_logical, dtype=img2d.dtype) buf[:, :, ::rstride, ::cstride] = img2d img2d = buf del buf, rstride, cstride if kshp != kshp_logical: rstride = int(np.ceil(kshp_logical[0] / float(kshp[0]))) cstride = int(np.ceil(kshp_logical[1] / float(kshp[1]))) buf = np.zeros((nkern, stacklen) + self.kshp_logical, dtype=filtersflipped.dtype) if self.kshp_logical_top_aligned: roffset = coffset = 0 else: roffset = (kshp_logical[0] - (kshp[0] * rstride) - 1 + rstride) % rstride coffset = (kshp_logical[1] - (kshp[1] * cstride) - 1 + cstride) % cstride assert roffset >= 0 assert coffset >= 0 buf[:, :, roffset::rstride, coffset::cstride] = filtersflipped filtersflipped = buf del buf, rstride, cstride val = _valfrommode(self.out_mode) bval = _bvalfromboundary('fill') with warnings.catch_warnings(): warnings.simplefilter('ignore', np.ComplexWarning) for b in xrange(bsize): for n in xrange(nkern): zz[b, n, ...].fill(0) for im0 in xrange(stacklen): # some cast generates a warning here zz[b, n, ...] += _convolve2d(img2d[b, im0, ...], filtersflipped[n, im0, ...], 1, val, bval, 0) if False: if False and self.out_mode == "full": img2d2 = np.zeros((bsize, stacklen, imshp[1] + 2 * kshp[0] - 2, imshp[2] + 2 * kshp[1] - 2)) img2d2[:, :, kshp[0] - 1:kshp[0] - 1 + imshp[1], kshp[1] - 1:kshp[1] - 1 + imshp[2]] = img2d img2d = img2d2 # N_image_shape = image_data.shape for b in xrange(bsize): for n in xrange(nkern): zz[b, n, ...].fill(0) for im0 in xrange(stacklen): for row in xrange(0, zz.shape[2], self.dx): for col in xrange(0, zz.shape[3], self.dy): zz[b, n, row, col] += ( img2d[b, im0, row:row + kshp[0], col:col + kshp[1]] * filtersflipped[n, im0, ::-1, ::-1]).sum() # We copy it to remove the Stride mismatch warning from DEBUG_MODE. # The copy make that we return an object with the same stride as the c version. # The copy don't affect the performence during our experience as in that case we # execute the c version which is much faster. if self.dx > 1 or self.dy > 1: zz = zz[:, :, 0::self.dx, 0::self.dy].copy() z[0] = zz def R_op(self, inputs, eval_points): rval = None if eval_points[0] is not None: rval = self.make_node(eval_points[0], inputs[1]).outputs[0] if eval_points[1] is not None: if rval is None: rval = self.make_node(inputs[0], eval_points[1]).outputs[0] else: rval += self.make_node(inputs[0], eval_points[1]).outputs[0] return [rval] def grad(self, inp, grads): inputs, kerns = inp gz, = grads if self.imshp != self.imshp_logical or self.kshp != self.kshp_logical: raise NotImplementedError('todo') if self.out_mode == 'valid' and (self.dx, self.dy) != (1, 1): raise NotImplementedError( "ERROR: ConvOp.grad is now disabled for 'valid' convolutions with" " stride != (1, 1); call theano.tensor.nnet.conv2d() instead.") if self.dx not in (1, 2) or self.dy not in (1, 2): raise NotImplementedError( "ERROR: We disable ConvOp.grad now when output_mode is not" " 'valid' and dx or dy are greater than 2, as there is a bug" " in it. See `abstract_conv2d <>`_ for a version that support this.") all_shape = self.has_all_shape(self.imshp, self.kshp, self.nkern, self.bsize) if not all_shape and (self.dx != 1 or self.dy != 1): raise Exception("ConvOp.grad when dx!=1 or dy!=1 we must have all " "the optional shape information") # Determine gradient on kernels ######## assert inputs.ndim == 4 and kerns.ndim == 4 newin = inputs.dimshuffle((1, 0, 2, 3)) newgz = gz.dimshuffle((1, 0, 2, 3)) if self.out_mode == 'valid': (img, filters) = (newin, newgz) kshp_logical = self.fulloutshp kshp_logical_top_aligned = False imshp_logical = None (bsize, nkern) = (self.imshp[0], self.nkern) imshp = (self.bsize, self.imshp[1], self.imshp[2]) kshp = self.outshp elif self.out_mode == 'full': (img, filters) = (newgz, newin) kshp_logical = None kshp_logical_top_aligned = True imshp_logical = (self.bsize, self.fulloutshp[0], self.fulloutshp[1]) (bsize, nkern) = (self.nkern, self.imshp[0]) imshp = (self.bsize, self.outshp[0], self.outshp[1]) kshp = self.imshp[1:] else: raise NotImplementedError( 'Only [full,valid] modes are currently supported.') filters = filters[:, :, ::-1, ::-1] # flip them dw = ConvOp(imshp, kshp, nkern, bsize, 1, 1, output_mode='valid', unroll_batch=None, unroll_kern=None, unroll_patch=None, imshp_logical=imshp_logical, kshp_logical=kshp_logical, kshp_logical_top_aligned=kshp_logical_top_aligned, version=self.version, direction_hint='bprop weights', verbose=self.verbose) dw = dw(img, filters) if all_shape: assert all(o == k for o, k in zip(dw.owner.op.outshp, self.kshp)) if self.out_mode == 'valid': # before DimShuffle, dw is of shape visdim x nkern x kshp[0] x kshp[1] dw = dw.dimshuffle((1, 0, 2, 3)) dw = dw[:, :, ::-1, ::-1] # Determine gradient on inputs ######## mode = 'valid' if not self.out_mode == 'full': mode = 'full' filters = kerns.dimshuffle((1, 0, 2, 3)) filters = filters[:, :, ::-1, ::-1] nkern = self.imshp[0] imshp = (self.nkern, self.outshp[0], self.outshp[1]) imshp_logical = (self.nkern, self.fulloutshp[0], self.fulloutshp[1]) din = ConvOp(imshp, self.kshp, nkern, self.bsize, 1, 1, output_mode=mode, unroll_batch=None, unroll_kern=None, unroll_patch=None, imshp_logical=imshp_logical, kshp_logical=None, version=-1, # we we change the mode, we don't forward the version. direction_hint='bprop inputs', verbose=self.verbose) din = din(gz, filters) assert all(o is None or o == i for o, i in zip(din.owner.op.outshp, self.imshp[1:])) # din and dw should have the same broadcasting pattern as the # parameters they are the gradient of (resp. inputs and kerns). din = patternbroadcast(din, inputs.broadcastable) dw = patternbroadcast(dw, kerns.broadcastable) return [din, dw] def c_headers(self): return ['', '', ''] def c_code_cache_version(self): return (15, self.openmp, blas.blas_header_version()) def c_support_code(self): return """ #define STRIDES(arr) (PyArray_STRIDES(arr)) #define FULL 2 #define SAME 1 #define VALID 0 #define MOD % using namespace std; """ + blas.blas_header_text() def use_blas(self): """ Return True if we will generate code that use gemm. """ # the gemm version only support that case if self.out_mode == 'valid' and self.dx == 0 and self.dy == 0: # We use a faster version in those case. if (self.imshp != self.imshp_logical or self.kshp != self.kshp_logical or self.unroll_patch or self.unroll_batch > 0 or self.unroll_kern > 0): return False return True return False def c_libraries(self): if self.use_blas(): return blas.ldflags() return [] def c_no_compile_args(self): # when the ksph==(1,1) gcc 4.3.0 segfault during the # compilation with -O3. This don't happen at -O2 if (theano.gof.cmodule.gcc_version() in ['4.3.0'] and self.kshp == (1, 1)): return ['-O3'] else: return [] def c_compile_args(self): ret = [] if self.use_blas(): ret = blas.ldflags(libs=False, flags=True) if (theano.gof.cmodule.gcc_version() in ['4.3.0'] and self.kshp == (1, 1)): ret += ['-O2'] # Add the -fopenmp flags ret += super(ConvOp, self).c_compile_args() return ret def c_lib_dirs(self): if self.use_blas(): return blas.ldflags(libs=False, libs_dir=True) return [] def c_header_dirs(self): if self.use_blas(): return blas.ldflags(libs=False, include_dir=True) return [] def c_code(self, node, name, inp, out, sub): img2d, filtersflipped = inp z, = out if node.inputs[0].type.dtype != node.inputs[1].type.dtype: raise NotImplementedError() assert node.inputs[0].type.dtype == node.inputs[1].type.dtype d = locals() d.update(sub) all_shape = (self.has_all_shape(self.imshp, self.kshp, self.nkern, self.bsize) and self.has_all_shape(self.imshp_logical, self.kshp_logical)) d["self_out_mode"] = self.out_mode d["self_dx"] = self.dx d["self_dy"] = self.dy d["mode"] = self.out_mode.upper() d["affectation"] = "=" # Default values, will be overrided if the shape info is provided d["self_bsize"] = "PyArray_DIMS(%(img2d)s)[0]" % d d["self_nkern"] = "PyArray_DIMS(%(filtersflipped)s)[0]" % d d["self_outshp0"] = "-1" d["self_outshp1"] = "-1" d["self_imshp0"] = "PyArray_DIMS(%(img2d)s)[1]" % d d["self_imshp1"] = "PyArray_DIMS(%(img2d)s)[2]" % d d["self_imshp2"] = "PyArray_DIMS(%(img2d)s)[3]" % d d["self_kshp0"] = "PyArray_DIMS(%(filtersflipped)s)[2]" % d d["self_kshp1"] = "PyArray_DIMS(%(filtersflipped)s)[3]" % d d["assert_size"] = "" # Override the default value if we have it if self.kshp[0] is not None: expected = d["self_kshp0"] value = self.kshp[0] d["assert_size"] += """ if(%(value)s != %(expected)s){ PyErr_Format(PyExc_ValueError, "The hardcoded shape for the number of rows in the filter " "(%%ld) isn't the run time shape (%%ld).", (long)%(value)s, (long)%(expected)s); %(fail)s; } """ % dict(expected=expected, value=value, **sub) d["self_kshp0"] = self.kshp[0] if self.kshp[1] is not None: expected = d["self_kshp1"] value = self.kshp[1] d["assert_size"] += """ if(%(value)s != %(expected)s){ PyErr_Format(PyExc_ValueError, "The hardcoded shape for the number of columns in the filter " "(%%ld) isn't the run time shape (%%ld).", (long)%(value)s, (long)%(expected)s); %(fail)s; } """ % dict(expected=expected, value=value, **sub) d["self_kshp1"] = self.kshp[1] if self.outshp[0] is not None: expected = "dim_zz[0]" value = self.outshp[0] d["assert_size"] += """ if(%(value)s != %(expected)s){ PyErr_Format(PyExc_ValueError, "The hardcoded shape for the number of rows in the output " "(%%ld) isn't the run time shape (%%ld).", (long)%(value)s, (long)%(expected)s); %(fail)s; } """ % dict(expected=expected, value=value, **sub) d["self_outshp0"] = self.outshp[0] if self.outshp[1] is not None: expected = "dim_zz[1]" value = self.outshp[1] d["assert_size"] += """ if(%(value)s != %(expected)s){ PyErr_Format(PyExc_ValueError, "The hardcoded shape for the number of columns in the output " "(%%ld) isn't the run time shape (%%ld).", (long)%(value)s, (long)%(expected)s); %(fail)s; } """ % dict(expected=expected, value=value, **sub) d["self_outshp1"] = self.outshp[1] if self.imshp[0] is not None: expected = d["self_imshp0"] value = self.imshp[0] d["assert_size"] += """ if(%(value)s != %(expected)s){ PyErr_Format(PyExc_ValueError, "The hardcoded shape for the image stack size (%%ld) " "isn't the run time shape (%%ld).", (long)%(value)s, (long)%(expected)s); %(fail)s; } """ % dict(expected=expected, value=value, **sub) expected = "kerns_dim[1]" value = self.imshp[0] d["assert_size"] += """ if(%(value)s != %(expected)s){ PyErr_Format(PyExc_ValueError, "The hardcoded shape for the kernel stack size (%%ld) " "isn't the run time shape (%%ld).", (long)%(value)s, (long)%(expected)s); %(fail)s; } """ % dict(expected=expected, value=value, **sub) d["self_imshp0"] = self.imshp[0] if self.imshp[1] is not None: expected = d["self_imshp1"] value = self.imshp[1] d["assert_size"] += """ if(%(value)s != %(expected)s){ PyErr_Format(PyExc_ValueError, "The hardcoded shape for the number of rows in the image " "(%%ld) isn't the run time shape (%%ld).", (long)%(value)s, (long)%(expected)s); %(fail)s; } """ % dict(expected=expected, value=value, **sub) d["self_imshp1"] = self.imshp[1] if self.imshp[2] is not None: expected = d["self_imshp2"] value = self.imshp[2] d["assert_size"] += """ if(%(value)s != %(expected)s){ PyErr_Format(PyExc_ValueError, "The hardcoded shape for the number of columns in the image " "(%%ld) isn't the run time shape (%%ld).", (long)%(value)s, (long)%(expected)s); %(fail)s; } """ % dict(expected=expected, value=value, **sub) d["self_imshp2"] = self.imshp[2] if self.bsize is not None: expected = d["self_bsize"] value = self.bsize d["assert_size"] += """ if(%(value)s != %(expected)s){ PyErr_Format(PyExc_ValueError, "The hardcoded shape for the batch size (%%ld) " "isn't the run time shape (%%ld).", (long)%(value)s, (long)%(expected)s); %(fail)s; } """ % dict(expected=expected, value=value, **sub) d["self_bsize"] = self.bsize if self.nkern is not None: expected = d["self_nkern"] value = self.nkern d["assert_size"] += """ if(%(value)s != %(expected)s){ PyErr_Format(PyExc_ValueError, "The hardcoded shape for the number of kernels in the filter " "(%%ld) isn't the run time shape (%%ld).", (long)%(value)s, (long)%(expected)s); %(fail)s; } """ % dict(expected=expected, value=value, **sub) d["self_nkern"] = self.nkern # Other hard coded stuff only if we have all shapes if all_shape: d["self_kshp_logical_r"] = self.kshp_logical[0] d["self_kshp_logical_c"] = self.kshp_logical[1] d["self_kshp_logical_stride_r"] = int(np.ceil( self.kshp_logical[0] / float(self.kshp[0]))) d["self_kshp_logical_stride_c"] = int(np.ceil( self.kshp_logical[1] / float(self.kshp[1]))) d["self_imshp_logical_r"] = self.imshp_logical[1] # numpy.B. 1 not 0 d["self_imshp_logical_c"] = self.imshp_logical[2] # numpy.B. 2 not 1 d["self_imshp_logical_stride_r"] = int(np.ceil( self.imshp_logical[1] / float(self.imshp[1]))) d["self_imshp_logical_stride_c"] = int(np.ceil( self.imshp_logical[2] / float(self.imshp[2]))) if not self.imshp[0] == 1: d["affectation"] = "+=" d["all_shape"] = "1" d["dim_zz_const"] = "const" d["dim_zz_affect"] = "" else: d["affectation"] = "+=" d["all_shape"] = "0" d["dim_zz_const"] = "" d["dim_zz_affect"] = """ if (mode == FULL) { dim_zz[0] = (int)ceil((dim_im[0]+dim_ker0-1)/float(%(self_dx)s)); dim_zz[1] = (int)ceil((dim_im[1]+dim_ker1-1)/float(%(self_dy)s)); } else { dim_zz[0] = (int)ceil((dim_im[0]-dim_ker0+1)/float(%(self_dx)s)); dim_zz[1] = (int)ceil((dim_im[1]-dim_ker1+1)/float(%(self_dy)s)); } """ % d d["assert_size"] += """ // Check the stack size of the filter and images are equals if(kerns_dim[1] != img2d_dim[1]){ PyErr_Format(PyExc_ValueError, "the filter stack size (%%ld) and image stack size (%%ld) differ", (long)kerns_dim[1], (long)img2d_dim[1]); %(fail)s; } """ % sub if self.kshp_logical_top_aligned: d["self_kshp_logical_offset_r"] = 0 d["self_kshp_logical_offset_c"] = 0 elif all_shape: rstride = d["self_kshp_logical_stride_r"] cstride = d["self_kshp_logical_stride_c"] d["self_kshp_logical_offset_r"] = (self.kshp_logical[0] - (self.kshp[0] * rstride) - 1 + rstride) % rstride d["self_kshp_logical_offset_c"] = (self.kshp_logical[1] - (self.kshp[1] * cstride) - 1 + cstride) % cstride del rstride, cstride if node.inputs[0].type.dtype == "float32": d["type"] = "float" elif node.inputs[0].type.dtype == "float64": d["type"] = "double" else: raise Exception("Type %s not implemented" % node.inputs[0].type.dtype) d["gemm"] = 'dgemm_' if not d["type"] == "double": d["gemm"] = 'sgemm_' if self.imshp != self.imshp_logical or self.kshp != self.kshp_logical: if self.verbose: _logger.debug("return imshp!=imshp_logical or" " self.kshp != self.kshp_logical shape version") return _conv_op_code_a % d if self.unroll_patch: if self.verbose: _logger.debug("return unroll patch version. all_shape=%s", all_shape) return _conv_op_code_unroll_patch % d if ((self.unroll_batch is not None and self.unroll_batch > 0) or (self.unroll_kern is not None and self.unroll_kern > 0)): assert self.unroll_batch > 0 assert self.unroll_kern > 0 if self.verbose: _logger.debug("return unrolled batch (%s) and kern code (%s)", str(self.unroll_batch), str(self.unroll_kern)) return gen_conv_code_unroll_batch_kern(d, self.unroll_batch, self.unroll_kern) # TODO: should we choose the unroll size automatically with the bigger divisor under 5? if self.out_mode == 'valid' and self.dx == 0 and self.dy == 0: if self.verbose: _logger.debug("return gemm version") return _conv_op_code_valid_gemm % d else: if self.verbose: _logger.debug("return no gemm version") return _conv_op_code_a % d _conv_op_code_a = """ const int mode=%(mode)s; int typenum=0, typenum_f=0; PyArrayObject *ain1=NULL, *ain2=NULL; PyArrayObject *filtersflipped_arr=NULL, *img2d_arr=NULL, *z_arr=NULL; const %(type)s fill_value = 0; int type_im=PyArray_TYPE(%(img2d)s); int type_ker=PyArray_TYPE(%(filtersflipped)s); npy_intp dim_zz[2]={%(self_outshp0)s,%(self_outshp1)s}; npy_intp dim_im_phys[2]={%(self_imshp1)s,%(self_imshp2)s}; npy_intp dim_im_log[2]={%(self_imshp_logical_r)s,%(self_imshp_logical_c)s}; npy_intp dim_ker_phys[2]={%(self_kshp0)s,%(self_kshp1)s}; npy_intp dim_ker_log[2]={%(self_kshp_logical_r)s,%(self_kshp_logical_c)s}; PyArray_Dims img2d_shape; npy_intp img2d_dim[4]={1,1,0,0}; img2d_shape.ptr=img2d_dim; img2d_shape.len=4; PyArray_Dims kerns_shape; npy_intp kerns_dim[4]={1,1,0,0}; kerns_shape.ptr=kerns_dim; kerns_shape.len=4; PyObject *img2d=NULL, *contig, *filtersflipped=NULL; if(PyArray_NDIM(%(img2d)s)==2){ img2d_dim[3]=PyArray_DIMS(%(img2d)s)[1]; img2d_dim[2]=PyArray_DIMS(%(img2d)s)[0]; }else if(PyArray_NDIM(%(img2d)s)==3){ img2d_dim[3]=PyArray_DIMS(%(img2d)s)[2]; img2d_dim[2]=PyArray_DIMS(%(img2d)s)[1]; img2d_dim[0]=PyArray_DIMS(%(img2d)s)[0]; }else if(PyArray_NDIM(%(img2d)s)==4){ img2d_dim[3]=PyArray_DIMS(%(img2d)s)[3]; img2d_dim[2]=PyArray_DIMS(%(img2d)s)[2]; img2d_dim[1]=PyArray_DIMS(%(img2d)s)[1]; img2d_dim[0]=PyArray_DIMS(%(img2d)s)[0]; }else { PyErr_SetString(PyExc_ValueError, "img don't have a good shape"); %(fail)s; } if(PyArray_NDIM(%(filtersflipped)s)==3){ kerns_dim[3]=PyArray_DIMS(%(filtersflipped)s)[2]; kerns_dim[2]=PyArray_DIMS(%(filtersflipped)s)[1]; kerns_dim[0]=PyArray_DIMS(%(filtersflipped)s)[0]; }else if(PyArray_NDIM(%(filtersflipped)s)==4){ kerns_dim[3]=PyArray_DIMS(%(filtersflipped)s)[3]; kerns_dim[2]=PyArray_DIMS(%(filtersflipped)s)[2]; kerns_dim[1]=PyArray_DIMS(%(filtersflipped)s)[1]; kerns_dim[0]=PyArray_DIMS(%(filtersflipped)s)[0]; }else{ std::stringstream temp; temp << "nddim="<= dim_im_log[0]){ // the current row of the kernel is off the image }else{ int k = max((int)(pos_n-dim_im_log[1])+1,0); int max_k=min(pos_n+1,(int)dim_ker_log[1]); const %(type)s * idx_in=&in[ind0_phys*dim_im_phys[1]]; for (int ind1_log=pos_n-k; k= PyArray_DIMS(%(z)s)[0]) %(fail)s; if (kernel_idx >= PyArray_DIMS(%(z)s)[1]) %(fail)s; if (img_row >= PyArray_DIMS(%(z)s)[2]) %(fail)s; if (img_col >= PyArray_DIMS(%(z)s)[3]) %(fail)s; } z_p[0] += kbuf[img_row * kbufstride + kernel_idx]; } } } } } free(kbuf); } Py_XDECREF(img2d); """ def gen_conv_code_unroll_batch_kern(d, unroll_bsize=1, unroll_ksize=1): """ c_code for ConvOp that unroll the batch size loop. """ assert unroll_bsize > 0 and unroll_ksize > 0 if "unroll_bsize" in d or "unroll_ksize" in d or "unroll_iter" in d or "unroll_biter" in d or "unroll_kiter" in d: raise Exception("We can't use this dictionnary as we will overwrite some of its containt") d = d.copy() d["unroll_bsize"] = unroll_bsize d["unroll_ksize"] = unroll_ksize def my_dup(st, size): s = "" for i in xrange(size): d["unroll_iter"] = i s += st % d return s + "\n" def my_dup2(st): s = "" iter = 0 for i in xrange(unroll_bsize): d["unroll_biter"] = i for j in xrange(unroll_ksize): d["unroll_kiter"] = j d["unroll_iter"] = iter iter += 1 s += st % d return s + "\n" ret = """ const int mode=%(mode)s; int typenum=0, typenum_f=0; PyArrayObject *ain1=NULL, *ain2=NULL, *filtersflipped_arr=NULL, *img2d_arr=NULL, *z_arr=NULL;; const %(type)s fill_value = 0; int type_im=PyArray_TYPE(%(img2d)s); int type_ker=PyArray_TYPE(%(filtersflipped)s); npy_intp dim_zz[2]={%(self_outshp0)s,%(self_outshp1)s}; npy_intp dim_im[2]={%(self_imshp1)s,%(self_imshp2)s}; const npy_intp dim_ker0=%(self_kshp0)s; const npy_intp dim_ker1=%(self_kshp1)s; PyArray_Dims img2d_shape; npy_intp img2d_dim[4]={1,1,0,0}; img2d_shape.ptr=img2d_dim; img2d_shape.len=4; PyArray_Dims kerns_shape; npy_intp kerns_dim[4]={1,1,0,0}; kerns_shape.ptr=kerns_dim; kerns_shape.len=4; PyObject *img2d=NULL, *contig, *filtersflipped=NULL; if(PyArray_NDIM(%(img2d)s)==2){ img2d_dim[3]=PyArray_DIMS(%(img2d)s)[1]; img2d_dim[2]=PyArray_DIMS(%(img2d)s)[0]; }else if(PyArray_NDIM(%(img2d)s)==3){ img2d_dim[3]=PyArray_DIMS(%(img2d)s)[2]; img2d_dim[2]=PyArray_DIMS(%(img2d)s)[1]; img2d_dim[0]=PyArray_DIMS(%(img2d)s)[0]; }else if(PyArray_NDIM(%(img2d)s)==4){ img2d_dim[3]=PyArray_DIMS(%(img2d)s)[3]; img2d_dim[2]=PyArray_DIMS(%(img2d)s)[2]; img2d_dim[1]=PyArray_DIMS(%(img2d)s)[1]; img2d_dim[0]=PyArray_DIMS(%(img2d)s)[0]; }else { std::stringstream temp; temp << "nddim="<= dim_im[0]){ if(fill_value!=0) for (int k=0; k < dim_ker1; k++) { """ % d ret += my_dup2("sum%(unroll_iter)s += idx_hvals%(unroll_kiter)s[k] * fill_value;") ret += """ } }else{ //do the part where kernel is to the right of the img int k=0,max_k=max((int)(pos_n-dim_im[1])+1,0); if(fill_value!=0){ for(k=0;k 1) // We merge the 2 loop into one to make it easier to parallelize on both // This is the equivalent of those 2 lines. //for(int b=0;b< %(self_bsize)s;b++){ // for(int n_kern=0;n_kern<%(self_nkern)s;n_kern++){ for(int batch_kern_idx=0; batch_kern_idx < %(self_bsize)s * %(self_nkern)s; batch_kern_idx++){ int b = batch_kern_idx / %(self_nkern)s; int n_kern = batch_kern_idx %% %(self_nkern)s; %(type)s * __restrict__ out=(%(type)s *)(PyArray_GETPTR2(z_arr,b,n_kern)); for (int i = 0; i < dim_zz[0]*dim_zz[1]; ++i) out[i] = 0; for(int stack_size=0;stack_size<%(self_imshp0)s;stack_size++){ const %(type)s * __restrict__ in=(%(type)s *)(PyArray_GETPTR2(img2d_arr,b,stack_size)); const %(type)s * __restrict__ hvals=(%(type)s *)(PyArray_GETPTR2(filtersflipped_arr,n_kern,stack_size)); int new_m; for (int iter_m=0; iter_m < dim_zz[0]; iter_m++) { // Reposition index into input image based on requested output size int pos_m = iter_m*%(self_dx)s;//The position of the patch in the image if (mode == FULL) new_m = pos_m ; else new_m = (pos_m+dim_ker0-1); for (int iter_n=0; iter_n < dim_zz[1]; iter_n++) { // loop over columns int pos_n=iter_n*%(self_dy)s; %(type)s sum=0; %(type)s sum2=0; %(type)s sum3=0; %(type)s sum4=0; int nb_sum=0; // Sum over kernel, if index into image is out of bounds // fill with the value for (int j=0; j < dim_ker0; j++) { int ind0 = (new_m-j); if(mode==FULL){ const %(type)s * idx_hvals=&hvals[j*dim_ker1]; if(ind0 < 0 || ind0 >= dim_im[0]){ if(fill_value!=0) for (int k=0; k < dim_ker1; k++) { sum+= idx_hvals[k] * fill_value; } }else{ //do the part where kernel is to the right of the img int k=0,max_k=max((int)(pos_n-dim_im[1])+1,0); if(fill_value!=0){ for(k=0;kdim_ker1-1 && iter_ndim_ker1-1 && iter_n=0; k--,im_idx++) { sum+=idx_hvals[k]*idx_in[im_idx]; sum2+=idx_hvals[k]*idx_in[im_idx+%(self_dy)s]; sum3+=idx_hvals[k]*idx_in[im_idx+2*%(self_dy)s]; sum4+=idx_hvals[k]*idx_in[im_idx+3*%(self_dy)s]; } }else if(iter_n + 2*%(self_dy)s < dim_zz[1]){ nb_sum=2; for (int k=dim_ker1-1,im_idx=pos_n; k >=0; k--,im_idx++) { sum+=idx_hvals[k]*idx_in[im_idx]; sum2+=idx_hvals[k]*idx_in[im_idx+%(self_dy)s]; } }else{ nb_sum=1; for (int k=dim_ker1-1,im_idx=pos_n; k >=0; k--,im_idx++) { sum+=idx_hvals[k]*idx_in[im_idx]; } } }//else valid mode }//for j switch(nb_sum){ case 4: out[iter_m*dim_zz[1]+iter_n+3] %(affectation)s sum4; case 3: out[iter_m*dim_zz[1]+iter_n+2] %(affectation)s sum3; case 2: out[iter_m*dim_zz[1]+iter_n+1] %(affectation)s sum2; case 1: out[iter_m*dim_zz[1]+iter_n] %(affectation)s sum; } iter_n+=nb_sum-1; }//for iter_n }//for iter_m }//for stack_size }//for b and n_kern Py_XDECREF(img2d); Py_XDECREF(filtersflipped); """ Theano-1.0.0/theano/tensor/nnet/bn.py00006000163562€ÿÿÿþ0000010364013203335532021000 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import numpy as np import theano from theano import Apply, Op from theano.gof import local_optimizer from theano.gof.opt import copy_stack_trace from theano.tensor import as_tensor_variable, TensorType from theano.tensor import basic as T from theano.tensor.opt import register_specialize_device from theano.scalar import Composite, as_common_dtype from theano.scalar import add, sub, true_div, mul class BNComposite(Composite): init_param = ('dtype',) @theano.change_flags(compute_test_value='off') def __init__(self, dtype): self.dtype = dtype x = theano.scalar.Scalar(dtype=dtype).make_variable() mean = theano.scalar.Scalar(dtype=dtype).make_variable() std = theano.scalar.Scalar(dtype=dtype).make_variable() gamma = theano.scalar.Scalar(dtype=dtype).make_variable() beta = theano.scalar.Scalar(dtype=dtype).make_variable() o = add(mul(true_div(sub(x, mean), std), gamma), beta) inputs = [x, mean, std, gamma, beta] outputs = [o] super(BNComposite, self).__init__(inputs, outputs) def grad(self, inps, grads): x, mean, std, gamma, beta = inps top, = grads top_gamma = top * gamma x_mean = x - mean dx = top_gamma / std dmean = -dx dstd = -(top_gamma * x_mean) / (std * std) dgamma = top * x_mean / std return [dx, dmean, dstd, dgamma, top] def batch_normalization(inputs, gamma, beta, mean, std, mode='low_mem'): """ This function will build the symbolic graph for applying batch normalization to a set of activations. Also works on GPUs, but is not optimized using cuDNN. .. versionadded:: 0.7.1 Parameters ---------- inputs : symbolic tensor Mini-batch of activations gamma: symbolic tensor BN scale parameter, must be of same dimensionality as inputs and broadcastable against it beta: symbolic tensor BN shift parameter, must be of same dimensionality as inputs and broadcastable against it mean: symbolic tensor inputs means, must be of same dimensionality as inputs and broadcastable against it std: symbolic tensor inputs standard deviation, must be of same dimensionality as inputs and broadcastable against it mode: 'low_mem' or 'high_mem' Specify which batch_normalization implementation that will be used. As no intermediate representations are stored for the back-propagation, 'low_mem' implementation lower the memory usage, however, it is 5-10% slower than 'high_mem' implementation. Note that 5-10% computation time difference compare the batch_normalization operation only, time difference between implementation is likely to be less important on the full model fprop/bprop. """ if mode == 'low_mem': elm_bn = theano.tensor.elemwise.Elemwise(scalar_op=BNComposite(dtype=inputs.dtype)) rval = elm_bn(inputs, mean, std, gamma, beta) elif mode == 'high_mem': rval = (inputs - mean) * (gamma / std) + beta else: raise ValueError( 'mode must be either "low_mem", "high_mem"') return rval def _prepare_batch_normalization_axes(axes, ndim): if axes == 'per-activation': axes = (0,) elif axes == 'spatial': axes = (0,) + tuple(range(2, ndim)) elif isinstance(axes, (tuple, list, np.ndarray)): axes = tuple(int(a) for a in axes) else: raise ValueError('invalid axes: %s', str(axes)) axes = tuple(sorted(axes)) if len(axes) == 0: raise ValueError('there should be at least one normalization axis') if min(axes) < 0 or max(axes) >= ndim: raise ValueError('axes should be less than ndim (<%d), but %s given' % (ndim, str(axes))) non_bc_axes = tuple(i for i in range(ndim) if i not in axes) return axes, non_bc_axes def batch_normalization_train(inputs, gamma, beta, axes='per-activation', epsilon=1e-4, running_average_factor=0.1, running_mean=None, running_var=None): """ Performs batch normalization of the given inputs, using the mean and variance of the inputs. Parameters ---------- axes : 'per-activation', 'spatial' or a tuple of ints The axes along which the input should be normalized. ``'per-activation'`` normalizes per activation and is equal to ``axes=(0,)``. ``'spatial'`` shares normalization factors across spatial dimensions (i.e., all dimensions past the second), which for 4D inputs would be equal to ``axes=(0, 2, 3)``. gamma : tensor Learnable scale factors. The shape must match the shape of `inputs`, except for the axes in `axes`. These axes should be set to 1 or be skipped altogether (such that `gamma.ndim == inputs.ndim - len(axes)`). beta : tensor Learnable biases. Must match the tensor layout of `gamma`. epsilon : float Epsilon value used in the batch normalization formula. Minimum allowed value is 1e-5 (imposed by cuDNN). running_average_factor : float Factor for updating the values or `running_mean` and `running_var`. If the factor is close to one, the running averages will update quickly, if the factor is close to zero it will update slowly. running_mean : tensor or None Previous value of the running mean. If this is given, the new value ``running_mean * (1 - r_a_factor) + batch mean * r_a_factor`` will be returned as one of the outputs of this function. `running_mean` and `running_var` should either both be given or both be None. The shape should match that of `gamma` and `beta`. running_var : tensor or None Previous value of the running variance. If this is given, the new value ``running_var * (1 - r_a_factor) + (m / (m - 1)) * batch var * r_a_factor`` will be returned as one of the outputs of this function, where `m` is the product of lengths of the averaged-over dimensions. `running_mean` and `running_var` should either both be given or both be None. The shape should match that of `gamma` and `beta`. Returns ------- out : tensor Batch-normalized inputs. mean : tensor Means of `inputs` across the normalization axes. invstd : tensor Inverse standard deviations of `inputs` across the normalization axes. new_running_mean : tensor New value of the running mean (only if both `running_mean` and `running_var` were given). new_running_var : tensor New value of the running variance (only if both `running_var` and `running_mean` were given). Notes ----- If per-activation or spatial normalization is selected, this operation will use the cuDNN implementation. (This requires cuDNN 5 or newer.) The returned values are equivalent to: .. code-block:: python # for per-activation normalization axes = (0,) # for spatial normalization axes = (0,) + tuple(range(2, inputs.ndim)) mean = inputs.mean(axes, keepdims=True) var = inputs.var(axes, keepdims=True) invstd = T.inv(T.sqrt(var + epsilon)) out = (inputs - mean) * gamma * invstd + beta m = T.cast(T.prod(inputs.shape) / T.prod(mean.shape), 'float32') running_mean = running_mean * (1 - running_average_factor) + \\ mean * running_average_factor running_var = running_var * (1 - running_average_factor) + \\ (m / (m - 1)) * var * running_average_factor """ ndim = inputs.ndim axes, non_bc_axes = _prepare_batch_normalization_axes(axes, ndim) # have the parameter tensors been broadcasted yet? if gamma.ndim == ndim: params_ndim = ndim else: params_ndim = len(non_bc_axes) params_dimshuffle_pattern = ['x'] * ndim for i, axis in enumerate(non_bc_axes): params_dimshuffle_pattern[axis] = i if gamma.ndim != params_ndim or beta.ndim != params_ndim: raise ValueError("gamma and beta dimensionality must match the " "number of non-normalized axes, or have the " "same number of dimensions as the inputs; " "got %d and %d instead of %d" % (gamma.ndim, beta.ndim, params_ndim)) if (running_mean is None) != (running_var is None): raise ValueError("running_mean and running_var must either both be " "given or both be None") if running_mean is not None and running_mean.ndim != params_ndim: raise ValueError("running_mean must be of the same dimensionality " "as gamma and beta; got %d instead of %d" % (running_mean.ndim, params_ndim)) if running_var is not None and running_var.ndim != params_ndim: raise ValueError("running_var must be of the same dimensionality " "as gamma and beta; got %d instead of %d" % (running_var.ndim, params_ndim)) # epsilon will be converted to floatX later. we need to check # for rounding errors now, since numpy.float32(1e-5) < 1e-5. epsilon = np.cast[theano.config.floatX](epsilon) if epsilon < 1e-5: raise ValueError("epsilon must be at least 1e-5, got %s" % str(epsilon)) inputs = as_tensor_variable(inputs) gamma = as_tensor_variable(gamma) beta = as_tensor_variable(beta) if params_ndim != ndim: gamma = gamma.dimshuffle(params_dimshuffle_pattern) beta = beta.dimshuffle(params_dimshuffle_pattern) else: gamma = T.addbroadcast(gamma, *axes) beta = T.addbroadcast(beta, *axes) batchnorm_op = AbstractBatchNormTrain(axes=axes) if running_mean is not None and running_var is not None: running_mean = as_tensor_variable(running_mean) running_var = as_tensor_variable(running_var) if params_ndim != ndim: running_mean = running_mean.dimshuffle(params_dimshuffle_pattern) running_var = running_var.dimshuffle(params_dimshuffle_pattern) else: running_mean = T.addbroadcast(running_mean, *axes) running_var = T.addbroadcast(running_var, *axes) out, mean, invstd, new_running_mean, new_running_var = batchnorm_op( inputs, gamma, beta, epsilon=epsilon, running_average_factor=running_average_factor, running_mean=running_mean, running_var=running_var) if new_running_mean.broadcastable != running_mean.broadcastable: new_running_mean = T.patternbroadcast(new_running_mean, running_mean.broadcastable) if new_running_var.broadcastable != running_var.broadcastable: new_running_var = T.patternbroadcast(new_running_var, running_var.broadcastable) results = (out, mean, invstd, new_running_mean, new_running_var) else: results = batchnorm_op(inputs, gamma, beta, epsilon=epsilon) if params_ndim != ndim: # remove the broadcasted dimensions (except from the output) results = ([results[0]] + [r.dimshuffle(non_bc_axes) for r in results[1:]]) return tuple(results) def batch_normalization_test(inputs, gamma, beta, mean, var, axes='per-activation', epsilon=1e-4): """ Performs batch normalization of the given inputs, using the given mean and variance. Parameters ---------- axes : 'per-activation', 'spatial' or a tuple of ints The axes along which the input should be normalized. ``'per-activation'`` normalizes per activation and is equal to ``axes=(0,)``. ``'spatial'`` shares normalization factors across spatial dimensions (i.e., all dimensions past the second), which for 4D inputs would be equal to ``axes=(0, 2, 3)``. gamma : tensor Scale factors. The shape must match the shape of `inputs`, except for the axes in `axes`. These axes should be set to 1 or be skipped altogether (such that `gamma.ndim == inputs.ndim - len(axes)`). beta : tensor Biases. Must match the tensor layout of `gamma`. mean : tensor Means. Usually these are running averages computed during training. Must match the tensor layout of `gamma`. var : tensor Variances. Usually these are running averages computed during training. Must match the tensor layout of `gamma`. epsilon : float Epsilon value used in the batch normalization formula. Minimum allowed value is 1e-5 (imposed by cuDNN). Returns ------- out : tensor Batch-normalized inputs. Notes ----- If per-activation or spatial normalization is selected, this operation will use the cuDNN implementation. (This requires cuDNN 5 or newer.) The returned value is equivalent to: .. code-block:: python # for per-activation normalization axes = (0,) # for spatial normalization axes = (0,) + tuple(range(2, inputs.ndim)) gamma, beta, mean, var = (T.addbroadcast(t, *axes) for t in (gamma, beta, mean, var)) out = (inputs - mean) * gamma / T.sqrt(var + epsilon) + beta """ ndim = inputs.ndim axes, non_bc_axes = _prepare_batch_normalization_axes(axes, ndim) # have the parameter tensors been broadcasted yet? if gamma.ndim == ndim: params_ndim = ndim else: params_ndim = len(non_bc_axes) params_dimshuffle_pattern = ['x'] * ndim for i, axis in enumerate(non_bc_axes): params_dimshuffle_pattern[axis] = i if gamma.ndim != params_ndim or beta.ndim != params_ndim: raise ValueError("gamma and beta dimensionality must match the " "number of non-normalized axes, or have the " "same number of dimensions as the inputs; " "got %d and %d instead of %d" % (gamma.ndim, beta.ndim, params_ndim)) if mean.ndim != params_ndim or var.ndim != params_ndim: raise ValueError("mean and var must be of the same dimensionality " "as gamma and beta; got %d and %d instead of %d" % (mean.ndim, var.ndim, params_ndim)) # epsilon will be converted to floatX later. we need to check # for rounding errors now, since numpy.float32(1e-5) < 1e-5. epsilon = np.cast[theano.config.floatX](epsilon) if epsilon < 1e-5: raise ValueError("epsilon must be at least 1e-5, got %s" % str(epsilon)) gamma = as_tensor_variable(gamma) beta = as_tensor_variable(beta) mean = as_tensor_variable(mean) var = as_tensor_variable(var) if params_ndim != ndim: gamma = gamma.dimshuffle(params_dimshuffle_pattern) beta = beta.dimshuffle(params_dimshuffle_pattern) mean = mean.dimshuffle(params_dimshuffle_pattern) var = var.dimshuffle(params_dimshuffle_pattern) else: gamma = T.addbroadcast(gamma, *axes) beta = T.addbroadcast(beta, *axes) mean = T.addbroadcast(mean, *axes) var = T.addbroadcast(var, *axes) batchnorm_op = AbstractBatchNormInference(axes=axes) return batchnorm_op(inputs, gamma, beta, mean, var, epsilon=epsilon) class AbstractBatchNormTrain(Op): """ Abstract Op for Batch Normalization. Parameters ---------- axes : a tuple of ints The axes along which the input should be normalized. x : tensor The input to be normalized along `axes`. scale : tensor `scale` should have the same number of dimensions as `x`. All dimensions listed in `axes` should have length 1. bias : tensor `bias` should have the same number of dimensions as `x`. All dimensions listed in `axes` should have length 1. epsilon Epsilon value used in the batch normalization formula. Minimum allowed value is 1e-5 (imposed by cuDNN). running_average_factor : float Factor for updating the values or `running_mean` and `running_var`. If the factor is close to one, the running averages will update quickly, if the factor is close to zero it will update slowly. running_mean : tensor or None Previous value of the running mean. If this is given, the new value ``running_mean * (1 - running_average_factor) + batch mean * running_average_factor`` will be returned as one of the outputs of this function. `running_mean` and `running_var` should either both be given or both be None. running_var : tensor or None Previous value of the running variance. If this is given, the new value ``running_var * (1 - running_average_factor) + (m / (m - 1)) * batch var * running_average_factor`` will be returned as one of the outputs of this function, where `m` is the product of lengths of the averaged-over dimensions. `running_mean` and `running_var` should either both be given or both be None. """ __props__ = ('axes',) def __init__(self, axes=(0,)): assert isinstance(axes, (tuple, list)) assert len(axes) > 0 axes = tuple(int(a) for a in axes) self.axes = axes def infer_shape(self, node, shape): return [shape[0]] + [shape[1]] * (len(node.outputs) - 1) def make_node(self, x, scale, bias, epsilon=1e-4, running_average_factor=0.1, running_mean=None, running_var=None): x = as_tensor_variable(x) scale = as_tensor_variable(scale) bias = as_tensor_variable(bias) epsilon = as_tensor_variable(epsilon) running_average_factor = as_tensor_variable(running_average_factor) if running_mean is not None: running_mean = as_tensor_variable(running_mean) if running_var is not None: running_var = as_tensor_variable(running_var) assert x.ndim == scale.ndim == bias.ndim assert ((running_mean is None and running_var is None) or (running_mean is not None and running_var is not None)) assert (running_mean is None or running_mean.ndim == x.ndim) assert (running_var is None or running_var.ndim == x.ndim) # Upcast to common dtype on the non-scalar # Keep as is dtype of scalar (epsilon and running_average_factor) if running_mean: x, scale, bias, running_mean, running_var = as_common_dtype( x, scale, bias, running_mean, running_var) else: x, scale, bias = as_common_dtype(x, scale, bias) inputs = [x, scale, bias, epsilon, running_average_factor] output_types = [x.type(), scale.type(), scale.type()] if running_mean is not None and running_var is not None: inputs.append(running_mean) inputs.append(running_var) output_types.append(scale.type()) output_types.append(scale.type()) return Apply(self, inputs, output_types) def L_op(self, inputs, outputs, grads): x, scale, bias, epsilon, running_average_factor = inputs[:5] dy = grads[0] _, x_mean, x_invstd = outputs[:3] disconnected_outputs = [ theano.gradient.DisconnectedType()(), # epsilon theano.gradient.DisconnectedType()()] # running_average_factor # Optional running_mean and running_var. for i in range(5, len(inputs)): disconnected_outputs.append(theano.gradient.DisconnectedType()()) return AbstractBatchNormTrainGrad(self.axes)( x, dy, scale, x_mean, x_invstd, epsilon) + disconnected_outputs def connection_pattern(self, node): # Specificy that epsilon and running_average_factor are not connected to outputs. patterns = [[True, True, True], # x [True, True, True], # scale [True, True, True], # bias [False, False, False], # epsilon [False, False, False]] # running_average_factor # Optional running_mean and running_var are only # connected to their new values. for i in range(5, len(node.inputs)): patterns[0].append(True) for pattern in patterns[1:]: pattern.append(False) patterns.append([False] * (3 + i - 5) + [True]) return patterns def perform(self, node, inputs, output_storage): x, scale, bias, epsilon, running_average_factor = inputs[:5] axes = self.axes if min(axes) < 0 or max(axes) >= x.ndim: raise ValueError('axes should be less than ndim (<%d), but %s given' % (x.ndim, str(axes))) mean = x.mean(axes, keepdims=True) var = x.var(axes, keepdims=True) invstd = 1.0 / np.sqrt(var + epsilon) out = (x - mean) * (scale * invstd) + bias output_storage[0][0] = out output_storage[1][0] = mean output_storage[2][0] = invstd if len(inputs) > 5: running_mean = inputs[5] running_mean = running_mean * (1.0 - running_average_factor) + \ mean * running_average_factor output_storage[3][0] = running_mean if len(inputs) > 6: m = float(np.prod(x.shape) / np.prod(scale.shape)) running_var = inputs[6] running_var = running_var * (1.0 - running_average_factor) + \ (m / (m - 1)) * var * running_average_factor output_storage[4][0] = running_var class AbstractBatchNormInference(Op): """ Abstract Op for Batch Normalization. Parameters ---------- axes : a tuple of ints The axes along which the input is normalized. epsilon Epsilon value used in the batch normalization formula. Minimum allowed value is 1e-5 (imposed by cuDNN). """ __props__ = ('axes',) def __init__(self, axes=(0,)): assert isinstance(axes, (tuple, list)) assert len(axes) > 0 axes = tuple(int(a) for a in axes) self.axes = axes def infer_shape(self, node, shape): return [shape[0]] def make_node(self, x, scale, bias, estimated_mean, estimated_variance, epsilon=1e-4): x = as_tensor_variable(x) scale = as_tensor_variable(scale) bias = as_tensor_variable(bias) estimated_mean = as_tensor_variable(estimated_mean) estimated_variance = as_tensor_variable(estimated_variance) epsilon = as_tensor_variable(epsilon) # Upcast to common dtype on the non-scalar # Keep as is dtype of scalar (epsilon) x, scale, bias, estimated_mean, estimated_variance = as_common_dtype( x, scale, bias, estimated_mean, estimated_variance) assert x.ndim == scale.ndim == bias.ndim == estimated_mean.ndim == estimated_variance.ndim return Apply(self, [x, scale, bias, estimated_mean, estimated_variance, epsilon], [x.type()]) def grad(self, inputs, grads): x, scale, bias, est_mean, est_var, epsilon = inputs dy = grads[0] axes = self.axes if min(axes) < 0 or max(axes) >= x.ndim: raise ValueError('axes should be less than ndim (<%d), but %s given' % (x.ndim, str(axes))) scale, bias, est_mean, est_var = (theano.tensor.addbroadcast(t, *axes) for t in (scale, bias, est_mean, est_var)) # define helper expressions est_var_eps = est_var + epsilon est_std = theano.tensor.sqrt(est_var_eps) two = theano.tensor.constant(2.) # define and return gradients dx = dy * (scale / est_std) dscale = (dy * (x - est_mean)).sum(axes, keepdims=True) / est_std dbias = dy.sum(axes, keepdims=True) dmean = -dy.sum(axes, keepdims=True) * (scale / est_std) dvar = -(dy * (x - est_mean)).sum(axes, keepdims=True) * (scale / (two * est_var_eps * est_std)) return [dx, dscale, dbias, dmean, dvar, theano.gradient.DisconnectedType()()] def connection_pattern(self, node): # Specificy that epsilon is not connected to outputs. return [[True], [True], [True], [True], [True], [False]] def perform(self, node, inputs, output_storage): x, scale, bias, estimated_mean, estimated_variance, epsilon = inputs out = (x - estimated_mean) * (scale / np.sqrt(estimated_variance + epsilon)) + bias output_storage[0][0] = out class AbstractBatchNormTrainGrad(Op): __props__ = ('axes',) def __init__(self, axes=(0,)): assert isinstance(axes, (tuple, list)) assert len(axes) > 0 axes = tuple(int(a) for a in axes) self.axes = axes def make_node(self, x, dy, scale, x_mean, x_invstd, epsilon=1e-4): x = as_tensor_variable(x) dy = as_tensor_variable(dy) scale = as_tensor_variable(scale) x_mean = as_tensor_variable(x_mean) x_invstd = as_tensor_variable(x_invstd) epsilon = as_tensor_variable(epsilon) # Upcast to common dtype on the non-scalar # Keep as is dtype of scalar (epsilon) x, dy, scale, x_mean, x_invstd = as_common_dtype( x, dy, scale, x_mean, x_invstd) assert x.ndim == dy.ndim == scale.ndim == x_mean.ndim == x_invstd.ndim return Apply(self, [x, dy, scale, x_mean, x_invstd, epsilon], [x.type(), scale.type(), scale.type()]) def grad(self, inp, grads): x, dy, scale, x_mean, x_invstd, epsilon = inp ddinputs, ddscale, ddbias = grads x_diff = x - x_mean mean_dy_x_diff = T.mean(dy * x_diff, axis=self.axes, keepdims=True) # compute gradients given each of the output gradients g_wrt_x = 0 g_wrt_dy = 0 g_wrt_scale = 0 g_wrt_x_mean = 0 g_wrt_x_invstd = 0 if not isinstance(ddinputs.type, theano.gradient.DisconnectedType): ccc = scale * (ddinputs - T.mean(ddinputs, axis=self.axes, keepdims=True)) ddd = (x_invstd ** 3) * (ccc * T.mean(dy * x_diff, axis=self.axes, keepdims=True) + dy * T.mean(ccc * x_diff, axis=self.axes, keepdims=True)) g_wrt_x = g_wrt_x - ddd g_wrt_dy = g_wrt_dy + ((ccc * x_invstd) - ((x_invstd ** 3) * x_diff * T.mean(ccc * x_diff, axis=self.axes, keepdims=True))) eee = (dy * x_invstd) - ((x_invstd ** 3) * x_diff * mean_dy_x_diff) g_wrt_scale = g_wrt_scale + T.sum(ddinputs * (eee - T.mean(eee, axis=self.axes, keepdims=True)), axis=self.axes, keepdims=True) g_wrt_x_mean = g_wrt_x_mean + T.sum(ddd, axis=self.axes, keepdims=True) g_wrt_x_invstd = g_wrt_x_invstd + T.sum(ccc * (dy - 3 * (x_invstd ** 2) * x_diff * mean_dy_x_diff), axis=self.axes, keepdims=True) if not isinstance(ddscale.type, theano.gradient.DisconnectedType): g_wrt_x = g_wrt_x + (x_invstd * ddscale * dy) g_wrt_dy = g_wrt_dy + (x_invstd * ddscale * x_diff) g_wrt_x_mean = g_wrt_x_mean - (x_invstd * ddscale * T.sum(dy, axis=self.axes, keepdims=True)) g_wrt_x_invstd = g_wrt_x_invstd + (ddscale * T.sum(dy * x_diff, axis=self.axes, keepdims=True)) if not isinstance(ddbias.type, theano.gradient.DisconnectedType): g_wrt_dy = g_wrt_dy + T.fill(dy, ddbias) # depending on which output gradients are given, # some inputs should be disconnected results = [g_wrt_x, g_wrt_dy, g_wrt_scale, g_wrt_x_mean, g_wrt_x_invstd, theano.gradient.DisconnectedType()()] return [theano.gradient.DisconnectedType()() if r is 0 else r for r in results] def connection_pattern(self, node): return [[True, True, False], # x [True, True, True], # dy [True, False, False], # scale [True, True, False], # x_mean [True, True, False], # x_invstd [False, False, False]] # epsilon def infer_shape(self, node, shape): return [shape[0], shape[2], shape[2]] def perform(self, node, inputs, output_storage): x, dy, scale, x_mean, x_invstd, epsilon = inputs axes = self.axes if min(axes) < 0 or max(axes) >= x.ndim: raise ValueError('axes should be less than ndim (<%d), but %s given' % (x.ndim, str(axes))) x_diff = x - x_mean mean_dy_x_diff = np.mean(dy * x_diff, axis=axes, keepdims=True) c = (dy * x_invstd) - (x_diff * mean_dy_x_diff * (x_invstd ** 3)) g_wrt_inputs = scale * (c - np.mean(c, axis=axes, keepdims=True)) g_wrt_scale = np.sum(dy * x_invstd * x_diff, axis=axes, keepdims=True) g_wrt_bias = np.sum(dy, axis=axes, keepdims=True) output_storage[0][0] = g_wrt_inputs output_storage[1][0] = g_wrt_scale output_storage[2][0] = g_wrt_bias @local_optimizer([AbstractBatchNormTrain]) def local_abstract_batch_norm_train(node): if not isinstance(node.op, AbstractBatchNormTrain): return None x, scale, bias, epsilon, running_average_factor = node.inputs[:5] axes = node.op.axes if min(axes) < 0 or max(axes) > x.ndim: return None if not isinstance(x.type, TensorType) or \ not isinstance(scale.type, TensorType) or \ not isinstance(bias.type, TensorType) or \ not isinstance(epsilon.type, TensorType) or \ not isinstance(running_average_factor.type, TensorType): return None # optional running_mean and running_var if len(node.inputs) > 5 and not isinstance(node.inputs[5].type, TensorType): return None if len(node.inputs) > 6 and not isinstance(node.inputs[6].type, TensorType): return None mean = x.mean(axes, keepdims=True) var = x.var(axes, keepdims=True) # The epsilon should not upcast the dtype. if var.dtype == 'float32' and epsilon.dtype == 'float64': epsilon = epsilon.astype('float32') invstd = T.inv(T.sqrt(var + epsilon)) out = (x - mean) * (scale * invstd) + bias results = [out, mean, invstd] if len(node.inputs) > 5: running_mean = node.inputs[5] running_mean = running_mean * (1.0 - running_average_factor) + \ mean * running_average_factor results.append(running_mean) if len(node.inputs) > 6: m = T.cast(T.prod(x.shape) / T.prod(scale.shape), theano.config.floatX) running_var = node.inputs[6] running_var = running_var * (1.0 - running_average_factor) + \ (m / (m - 1)) * var * running_average_factor results.append(running_var) results = [T.patternbroadcast(r, r_orig.broadcastable) for (r, r_orig) in zip(results, node.outputs)] for var in theano.gof.graph.variables(node.inputs, results): if var not in node.inputs: copy_stack_trace(node.outputs[0], var) return results @local_optimizer([AbstractBatchNormTrainGrad]) def local_abstract_batch_norm_train_grad(node): if not isinstance(node.op, AbstractBatchNormTrainGrad): return None x, dy, scale, x_mean, x_invstd, epsilon = node.inputs axes = node.op.axes if min(axes) < 0 or max(axes) > x.ndim: return None if not isinstance(x.type, TensorType) or \ not isinstance(dy.type, TensorType) or \ not isinstance(scale.type, TensorType) or \ not isinstance(x_mean.type, TensorType) or \ not isinstance(x_invstd.type, TensorType) or \ not isinstance(epsilon.type, TensorType): return None x_diff = x - x_mean mean_dy_x_diff = T.mean(dy * x_diff, axis=axes, keepdims=True) c = (dy * x_invstd) - x_diff * (mean_dy_x_diff * (x_invstd ** 3)) g_wrt_inputs = scale * (c - T.mean(c, axis=axes, keepdims=True)) g_wrt_scale = T.sum(dy * x_invstd * x_diff, axis=axes, keepdims=True) g_wrt_bias = T.sum(dy, axis=axes, keepdims=True) results = [g_wrt_inputs, g_wrt_scale, g_wrt_bias] results = [T.patternbroadcast(r, r_orig.broadcastable) for (r, r_orig) in zip(results, node.outputs)] for var in theano.gof.graph.variables(node.inputs, results): if var not in node.inputs: copy_stack_trace(node.outputs[0], var) return results @local_optimizer([AbstractBatchNormInference]) def local_abstract_batch_norm_inference(node): if not isinstance(node.op, AbstractBatchNormInference): return None x, scale, bias, estimated_mean, estimated_variance, epsilon = node.inputs if not isinstance(x.type, TensorType) or \ not isinstance(scale.type, TensorType) or \ not isinstance(bias.type, TensorType) or \ not isinstance(estimated_mean.type, TensorType) or \ not isinstance(estimated_variance.type, TensorType) or \ not isinstance(epsilon.type, TensorType): return None # The epsilon should not upcast the dtype. if estimated_variance.dtype == 'float32' and epsilon.dtype == 'float64': epsilon = epsilon.astype('float32') result = (x - estimated_mean) * (scale / T.sqrt(estimated_variance + epsilon)) + bias result = T.patternbroadcast(result, node.outputs[0].broadcastable) for var in theano.gof.graph.variables(node.inputs, [result]): if var not in node.inputs: copy_stack_trace(node.outputs[0], var) return [result] # Register Cpu Optmization bn_groupopt = theano.gof.optdb.LocalGroupDB() bn_groupopt.__name__ = 'batchnorm_opts' register_specialize_device(bn_groupopt, 'fast_compile', 'fast_run') bn_groupopt.register('local_abstract_batch_norm_train', local_abstract_batch_norm_train, 30, 'fast_compile', 'fast_run') bn_groupopt.register('local_abstract_batch_norm_train_grad', local_abstract_batch_norm_train_grad, 30, 'fast_compile', 'fast_run') bn_groupopt.register('local_abstract_batch_norm_inference', local_abstract_batch_norm_inference, 30, 'fast_compile', 'fast_run') Theano-1.0.0/theano/tensor/nnet/opt.py00006000163562€ÿÿÿþ0000004502113203335532021201 0ustar boccoset00000000000000""" Optimizations addressing the ops in nnet root directory """ from __future__ import absolute_import, print_function, division import theano from theano import compile, gof from theano.compile import optdb from theano.gof import local_optimizer from theano.gof.opt import copy_stack_trace from theano.tensor.nnet.corr import ( CorrMM, CorrMM_gradInputs, CorrMM_gradWeights) from theano.tensor.nnet.corr3d import ( Corr3dMM, Corr3dMM_gradInputs, Corr3dMM_gradWeights) from theano.tensor.nnet.blocksparse import ( SparseBlockGemv, SparseBlockOuter, sparse_block_gemv_inplace, sparse_block_outer_inplace) from theano.tensor.nnet.abstract_conv import (AbstractConv2d, AbstractConv2d_gradWeights, AbstractConv2d_gradInputs) from theano.tensor.nnet.abstract_conv import (AbstractConv3d, AbstractConv3d_gradWeights, AbstractConv3d_gradInputs) from theano.tensor.nnet.abstract_conv import get_conv_output_shape from theano.tensor.opt import register_specialize_device from theano.tensor import TensorType from theano.tensor import opt # Cpu implementation from theano.tensor.nnet.conv import conv2d, ConvOp @gof.local_optimizer([SparseBlockGemv], inplace=True) def local_inplace_sparse_block_gemv(node): """ SparseBlockGemv(inplace=False) -> SparseBlockGemv(inplace=True) """ if isinstance(node.op, SparseBlockGemv) and not node.op.inplace: new_node = sparse_block_gemv_inplace(*node.inputs) copy_stack_trace(node.outputs[0], new_node) return [new_node] return False compile.optdb.register('local_inplace_sparse_block_gemv', gof.TopoOptimizer( local_inplace_sparse_block_gemv, failure_callback=gof.TopoOptimizer.warn_inplace), 60, 'fast_run', 'inplace') # DEBUG @gof.local_optimizer([SparseBlockOuter], inplace=True) def local_inplace_sparse_block_outer(node): """ SparseBlockOuter(inplace=False) -> SparseBlockOuter(inplace=True) """ if isinstance(node.op, SparseBlockOuter) and not node.op.inplace: new_node = sparse_block_outer_inplace(*node.inputs) copy_stack_trace(node.outputs[0], new_node) return [new_node] return False compile.optdb.register('local_inplace_sparse_block_outer', gof.TopoOptimizer( local_inplace_sparse_block_outer, failure_callback=gof.TopoOptimizer.warn_inplace), 60, 'fast_run', 'inplace') # DEBUG # Conv opts @local_optimizer([AbstractConv2d]) def local_abstractconv_gemm(node): # If theano.config.blas.ldflags is empty, Theano will use # a NumPy C implementation of [sd]gemm_. if theano.config.cxx == "" or node.inputs[0].dtype == 'float16': return if not isinstance(node.op, AbstractConv2d): return None img, kern = node.inputs if not isinstance(img.type, TensorType) or \ not isinstance(kern.type, TensorType): return None # need to flip the kernel if necessary if node.op.filter_flip: flip = (slice(None),) * (kern.ndim - 2) + \ (slice(None, None, -1),) * 2 kern = kern[flip] rval = CorrMM(border_mode=node.op.border_mode, subsample=node.op.subsample, filter_dilation=node.op.filter_dilation, num_groups=node.op.num_groups, unshared=node.op.unshared)(img, kern) copy_stack_trace(node.outputs[0], rval) return [rval] @local_optimizer([AbstractConv3d]) def local_abstractconv3d_gemm(node): # If theano.config.blas.ldflags is empty, Theano will use # a NumPy C implementation of [sd]gemm_. if theano.config.cxx == "" or node.inputs[0].dtype == 'float16': return if not isinstance(node.op, AbstractConv3d): return None img, kern = node.inputs if not isinstance(img.type, TensorType) or \ not isinstance(kern.type, TensorType): return None # need to flip the kernel if necessary if node.op.filter_flip: kern = kern[:, :, ::-1, ::-1, ::-1] rval = Corr3dMM(border_mode=node.op.border_mode, subsample=node.op.subsample, filter_dilation=node.op.filter_dilation, num_groups=node.op.num_groups)(img, kern) copy_stack_trace(node.outputs[0], rval) return [rval] @local_optimizer([AbstractConv2d_gradWeights]) def local_abstractconv_gradweight_gemm(node): # If theano.config.blas.ldflags is empty, Theano will use # a NumPy C implementation of [sd]gemm_. if theano.config.cxx == "" or node.inputs[0].dtype == 'float16': return if not isinstance(node.op, AbstractConv2d_gradWeights): return None img, topgrad, shape = node.inputs if not isinstance(img.type, TensorType) or \ not isinstance(topgrad.type, TensorType): return None rval = CorrMM_gradWeights(border_mode=node.op.border_mode, subsample=node.op.subsample, filter_dilation=node.op.filter_dilation, num_groups=node.op.num_groups, unshared=node.op.unshared)(img, topgrad, shape) copy_stack_trace(node.outputs[0], rval) # need to flip the kernel if necessary if node.op.filter_flip: flip = (slice(None),) * (rval.ndim - 2) + \ (slice(None, None, -1),) * 2 rval = rval[flip] rval = theano.tensor.patternbroadcast(rval, node.outputs[0].broadcastable) copy_stack_trace(node.outputs[0], rval) return [rval] @local_optimizer([AbstractConv3d_gradWeights]) def local_abstractconv3d_gradweight_gemm(node): # If theano.config.blas.ldflags is empty, Theano will use # a NumPy C implementation of [sd]gemm_. if theano.config.cxx == "" or node.inputs[0].dtype == 'float16': return if not isinstance(node.op, AbstractConv3d_gradWeights): return None img, topgrad, shape = node.inputs if not isinstance(img.type, TensorType) or \ not isinstance(topgrad.type, TensorType): return None rval = Corr3dMM_gradWeights(border_mode=node.op.border_mode, subsample=node.op.subsample, filter_dilation=node.op.filter_dilation, num_groups=node.op.num_groups)(img, topgrad, shape) copy_stack_trace(node.outputs[0], rval) # need to flip the kernel if necessary if node.op.filter_flip: rval = rval[:, :, ::-1, ::-1, ::-1] rval = theano.tensor.patternbroadcast(rval, node.outputs[0].broadcastable) copy_stack_trace(node.outputs[0], rval) return [rval] @local_optimizer([AbstractConv2d_gradInputs]) def local_abstractconv_gradinputs_gemm(node): # If theano.config.blas.ldflags is empty, Theano will use # a NumPy C implementation of [sd]gemm_. if theano.config.cxx == "" or node.inputs[0].dtype == 'float16': return if not isinstance(node.op, AbstractConv2d_gradInputs): return None kern, topgrad, shape = node.inputs if not isinstance(kern.type, TensorType) or \ not isinstance(topgrad.type, TensorType): return None # need to flip the kernel if necessary if node.op.filter_flip: flip = (slice(None),) * (kern.ndim - 2) + \ (slice(None, None, -1),) * 2 kern = kern[flip] rval = CorrMM_gradInputs(border_mode=node.op.border_mode, subsample=node.op.subsample, filter_dilation=node.op.filter_dilation, num_groups=node.op.num_groups, unshared=node.op.unshared)(kern, topgrad, shape) copy_stack_trace(node.outputs[0], rval) return [rval] @local_optimizer([AbstractConv3d_gradInputs]) def local_abstractconv3d_gradinputs_gemm(node): # If theano.config.blas.ldflags is empty, Theano will use # a NumPy C implementation of [sd]gemm_. if theano.config.cxx == "" or node.inputs[0].dtype == 'float16': return if not isinstance(node.op, AbstractConv3d_gradInputs): return None kern, topgrad, shape = node.inputs if not isinstance(kern.type, TensorType) or \ not isinstance(topgrad.type, TensorType): return None # need to flip the kernel if necessary if node.op.filter_flip: kern = kern[:, :, ::-1, ::-1, ::-1] rval = Corr3dMM_gradInputs(border_mode=node.op.border_mode, subsample=node.op.subsample, filter_dilation=node.op.filter_dilation, num_groups=node.op.num_groups)(kern, topgrad, shape) copy_stack_trace(node.outputs[0], rval) return [rval] @local_optimizer([AbstractConv2d]) def local_conv2d_cpu(node): if (not isinstance(node.op, AbstractConv2d) or node.inputs[0].dtype == 'float16'): return None img, kern = node.inputs if ((not isinstance(img.type, TensorType) or not isinstance(kern.type, TensorType))): return None if node.op.border_mode not in ['full', 'valid']: return None if not node.op.filter_flip: # Not tested yet return None if node.op.num_groups > 1 or node.op.unshared: return None if node.op.filter_dilation != (1, 1): return None rval = conv2d(img, kern, node.op.imshp, node.op.kshp, border_mode=node.op.border_mode, subsample=node.op.subsample) copy_stack_trace(node.outputs[0], rval) return [rval] @local_optimizer([AbstractConv2d_gradWeights]) def local_conv2d_gradweight_cpu(node): if (not isinstance(node.op, AbstractConv2d_gradWeights) or node.inputs[0].dtype == 'float16'): return None img, topgrad, shape = node.inputs if ((not isinstance(img.type, TensorType) or not isinstance(topgrad.type, TensorType))): return None if node.op.border_mode not in ['full', 'valid']: return None if not node.op.filter_flip: # Not tested yet return if node.op.num_groups > 1 or node.op.unshared: return None if node.op.border_mode == 'valid' and \ (node.op.subsample != (1, 1)): return None dx, dy = node.op.subsample if dx not in (1, 2) or dy not in (1, 2): # Not implemented in the gradient of ConvOp return None if node.op.imshp is None: op_imshp = (None, None, None, None) else: op_imshp = node.op.imshp if node.op.kshp is None: op_kshp = (None, None, None, None) else: op_kshp = node.op.kshp if None in op_imshp or None in op_kshp: if (dx, dy) != (1, 1): # We cannot infer the shapes return None # Determine gradient on kernels assert len(op_imshp) == 4 and len(op_kshp) == 4 outshp = get_conv_output_shape(op_imshp, op_kshp, node.op.border_mode, node.op.subsample, node.op.filter_dilation)[2:] fulloutshp = get_conv_output_shape(op_imshp, op_kshp, node.op.border_mode, (1, 1))[2:] newimg = img.dimshuffle((1, 0, 2, 3)) newtopgrad = topgrad.dimshuffle((1, 0, 2, 3)) if node.op.border_mode == 'valid': (img, filters) = (newimg, newtopgrad) kshp_logical = fulloutshp kshp_logical_top_aligned = False imshp_logical = None (bsize, nkern) = (op_imshp[1], op_kshp[0]) imshp = (op_imshp[0], op_imshp[2], op_imshp[3]) kshp = outshp elif node.op.border_mode == 'full': (img, filters) = (newtopgrad, newimg) kshp_logical = None kshp_logical_top_aligned = True imshp_logical = (op_imshp[0], fulloutshp[0], fulloutshp[1]) (bsize, nkern) = (op_kshp[0], op_imshp[1]) imshp = (op_imshp[0], outshp[0], outshp[1]) kshp = op_imshp[2:] else: raise NotImplementedError( 'Only [full,valid] modes are currently supported.') # Flip the kernels filters = filters[:, :, ::-1, ::-1] dw = ConvOp(imshp, kshp, nkern, bsize, 1, 1, output_mode='valid', unroll_batch=None, unroll_kern=None, unroll_patch=None, imshp_logical=imshp_logical, kshp_logical=kshp_logical, kshp_logical_top_aligned=kshp_logical_top_aligned, direction_hint='bprop weights') res = dw(img, filters) copy_stack_trace(node.outputs[0], res) if node.op.border_mode == 'valid': res = res.dimshuffle((1, 0, 2, 3)) res = res[:, :, ::-1, ::-1] res = theano.tensor.patternbroadcast(res, node.outputs[0].broadcastable) copy_stack_trace(node.outputs[0], res) return [res] @local_optimizer([AbstractConv2d_gradInputs]) def local_conv2d_gradinputs_cpu(node): if (not isinstance(node.op, AbstractConv2d_gradInputs) or node.inputs[0].dtype == 'float16'): return None kern, topgrad, shape = node.inputs if ((not isinstance(kern.type, TensorType) or not isinstance(topgrad.type, TensorType))): return None if node.op.border_mode not in ['full', 'valid']: return None if not node.op.filter_flip: # Not tested yet return None if node.op.num_groups > 1 or node.op.unshared: return None # Conv 3d implementation, needed when subsample > 2 if node.op.border_mode == 'valid' and node.op.subsample != (1, 1): # The op don't support that anymore. return False # Conv2d Implementation dx, dy = node.op.subsample if dx not in (1, 2) or dy not in (1, 2): # Not implemented in the gradient of ConvOp return None if node.op.imshp is None: op_imshp = (None, None, None, None) else: op_imshp = node.op.imshp if node.op.kshp is None: op_kshp = (None, None, None, None) else: op_kshp = node.op.kshp if None in op_imshp or None in op_kshp: if (dx, dy) != (1, 1): return None mode = 'valid' if not node.op.border_mode == 'full': mode = 'full' filters = kern.dimshuffle((1, 0, 2, 3)) filters = filters[:, :, ::-1, ::-1] outshp = get_conv_output_shape(op_imshp, op_kshp, node.op.border_mode, node.op.subsample, node.op.filter_dilation)[2:] fulloutshp = get_conv_output_shape(op_imshp, op_kshp, node.op.border_mode, (1, 1))[2:] nkern = op_imshp[1] imshp = (op_kshp[0], outshp[0], outshp[1]) imshp_logical = (op_kshp[0], fulloutshp[0], fulloutshp[1]) din = ConvOp(imshp, op_kshp[2:], nkern, op_imshp[0], 1, 1, output_mode=mode, unroll_batch=None, unroll_kern=None, unroll_patch=None, imshp_logical=imshp_logical, kshp_logical=None, version=-1, direction_hint='bprop inputs') din = din(topgrad, filters) copy_stack_trace(node.outputs[0], din) din = theano.tensor.patternbroadcast(din, node.outputs[0].broadcastable) copy_stack_trace(node.outputs[0], din) return [din] # Register Cpu Optmization conv_groupopt = theano.gof.optdb.LocalGroupDB() conv_groupopt.__name__ = "conv_opts" register_specialize_device(conv_groupopt, 'fast_compile', 'fast_run') # GEMM-based convolution # It can be disabled by excluding 'conv_gemm'. conv_groupopt.register('local_abstractconv_gemm', local_abstractconv_gemm, 30, 'conv_gemm', 'fast_compile', 'fast_run') conv_groupopt.register('local_abstractconv_gradweight_gemm', local_abstractconv_gradweight_gemm, 30, 'conv_gemm', 'fast_compile', 'fast_run') conv_groupopt.register('local_abstractconv_gradinputs_gemm', local_abstractconv_gradinputs_gemm, 30, 'conv_gemm', 'fast_compile', 'fast_run') conv_groupopt.register('local_abstractconv3d_gemm', local_abstractconv3d_gemm, 30, 'conv_gemm', 'fast_compile', 'fast_run') conv_groupopt.register('local_abstractconv3d_gradweight_gemm', local_abstractconv3d_gradweight_gemm, 30, 'conv_gemm', 'fast_compile', 'fast_run') conv_groupopt.register('local_abstractconv3d_gradinputs_gemm', local_abstractconv3d_gradinputs_gemm, 30, 'conv_gemm', 'fast_compile', 'fast_run') # Legacy convolution conv_groupopt.register('local_conv2d_cpu', local_conv2d_cpu, 40, 'fast_compile', 'fast_run') conv_groupopt.register('local_conv2d_gradweight_cpu', local_conv2d_gradweight_cpu, 40, 'fast_compile', 'fast_run') conv_groupopt.register('local_conv2d_gradinputs_cpu', local_conv2d_gradinputs_cpu, 40, 'fast_compile', 'fast_run') # Verify that no AbstractConv are present in the graph @local_optimizer([AbstractConv2d, AbstractConv2d_gradWeights, AbstractConv2d_gradInputs, AbstractConv3d, AbstractConv3d_gradWeights, AbstractConv3d_gradInputs]) def local_abstractconv_check(node): if isinstance(node.op, (AbstractConv2d, AbstractConv2d_gradWeights, AbstractConv2d_gradInputs, AbstractConv3d, AbstractConv3d_gradWeights, AbstractConv3d_gradInputs)): raise AssertionError( '%s Theano optimization failed: there is no implementation ' 'available supporting the requested options. Did you exclude ' 'both "conv_dnn" and "conv_gemm" from the optimizer? If on GPU, ' 'is cuDNN available and does the GPU support it? If on CPU, ' 'do you have a BLAS library installed Theano can link against? ' 'On the CPU we do not support float16.' % node.op.__class__.__name__) optdb.register('AbstractConvCheck', opt.in2out(local_abstractconv_check, name="AbstractConvCheck"), 48.7, 'fast_compile', 'fast_run') Theano-1.0.0/theano/tensor/utils.py00006000163562€ÿÿÿþ0000000611013203335532020567 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import numpy as np import theano from theano.compat import izip from theano.gof.utils import hash_from_code def hash_from_ndarray(data): """ Return a hash from an ndarray. It takes care of the data, shapes, strides and dtype. """ # We need to hash the shapes and strides as hash_from_code only hashes # the data buffer. Otherwise, this will cause problem with shapes like: # (1, 0) and (2, 0) and problem with inplace transpose. # We also need to add the dtype to make the distinction between # uint32 and int32 of zeros with the same shape and strides. # python hash are not strong, so use sha256 (md5 is not # FIPS compatible). To not have too long of hash, I call it again on # the concatenation of all parts. if not data.flags["C_CONTIGUOUS"]: # hash_from_code needs a C-contiguous array. data = np.ascontiguousarray(data) return hash_from_code(hash_from_code(data) + hash_from_code(str(data.shape)) + hash_from_code(str(data.strides)) + hash_from_code(str(data.dtype))) def shape_of_variables(fgraph, input_shapes): """ Compute the numeric shape of all intermediate variables given input shapes. Parameters ---------- fgraph The theano.FunctionGraph in question. input_shapes : dict A dict mapping input to shape. Returns ------- shapes : dict A dict mapping variable to shape .. warning:: This modifies the fgraph. Not pure. Examples -------- >>> import theano >>> x = theano.tensor.matrix('x') >>> y = x[512:]; y.name = 'y' >>> fgraph = theano.FunctionGraph([x], [y], clone=False) >>> d = shape_of_variables(fgraph, {x: (1024, 1024)}) >>> d[y] (array(512), array(1024)) >>> d[x] (array(1024), array(1024)) """ if not hasattr(fgraph, 'shape_feature'): fgraph.attach_feature(theano.tensor.opt.ShapeFeature()) input_dims = [dimension for inp in fgraph.inputs for dimension in fgraph.shape_feature.shape_of[inp]] output_dims = [dimension for shape in fgraph.shape_feature.shape_of.values() for dimension in shape] compute_shapes = theano.function(input_dims, output_dims) if any([i not in fgraph.inputs for i in input_shapes.keys()]): raise ValueError( "input_shapes keys aren't in the fgraph.inputs. FunctionGraph()" " interface changed. Now by default, it clones the graph it receives." " To have the old behavior, give it this new parameter `clone=False`.") numeric_input_dims = [dim for inp in fgraph.inputs for dim in input_shapes[inp]] numeric_output_dims = compute_shapes(*numeric_input_dims) sym_to_num_dict = dict(izip(output_dims, numeric_output_dims)) l = {} for var in fgraph.shape_feature.shape_of: l[var] = tuple(sym_to_num_dict[sym] for sym in fgraph.shape_feature.shape_of[var]) return l Theano-1.0.0/theano/tensor/c_code/00007500163562€ÿÿÿþ0000000000013203336027020276 5ustar boccoset00000000000000Theano-1.0.0/theano/tensor/c_code/alt_blas_common.h00006000163562€ÿÿÿþ0000000160013203335532023572 0ustar boccoset00000000000000/** C Implementation (with NumPy back-end) of BLAS functions used in Theano. * Used instead of BLAS when Theano flag ``blas.ldflags`` is empty. * This file contains some useful header code not templated. * File alt_blas_template.c currently contains template code for: * - [sd]gemm_ * - [sd]gemv_ * - [sd]dot_ **/ #define alt_fatal_error(message) { if (PyErr_Occurred()) PyErr_Print(); if(message != NULL) fprintf(stderr, message); exit(-1); } #define alt_trans_to_bool(trans) (*trans != 'N' && *trans != 'n') /**Template code for BLAS functions follows in file alt_blas_template.c * (as Python string to be used with old formatting). * PARAMETERS: * float_type: "float" or "double". * float_size: 4 for float32 (sgemm_), 8 for float64 (dgemm_). * npy_float: "NPY_FLOAT32" or "NPY_FLOAT64". * precision: "s" for single, "d" for double. * See blas_headers.py for current use.**/ Theano-1.0.0/theano/tensor/c_code/dimshuffle.c00006000163562€ÿÿÿþ0000000704113203335532022567 0ustar boccoset00000000000000#section support_code_apply int APPLY_SPECIFIC(cpu_dimshuffle)(PyArrayObject* input, PyArrayObject** res, PARAMS_TYPE* params) { npy_bool* input_broadcastable; npy_int64* new_order; npy_intp nd_in; npy_intp nd_out; PyArrayObject* basename; npy_intp* dimensions; npy_intp* strides; if (!PyArray_IS_C_CONTIGUOUS(params->input_broadcastable)) { PyErr_SetString(PyExc_RuntimeError, "DimShuffle: param input_broadcastable must be C-contiguous."); return 1; } if (!PyArray_IS_C_CONTIGUOUS(params->_new_order)) { PyErr_SetString(PyExc_RuntimeError, "DimShuffle: param _new_order must be C-contiguous."); return 1; } input_broadcastable = (npy_bool*) PyArray_DATA(params->input_broadcastable); new_order = (npy_int64*) PyArray_DATA(params->_new_order); nd_in = PyArray_SIZE(params->input_broadcastable); nd_out = PyArray_SIZE(params->_new_order); /* check_input_nd */ if (PyArray_NDIM(input) != nd_in) { PyErr_SetString(PyExc_NotImplementedError, "input nd"); return 1; } /* clear_output */ if (*res) Py_XDECREF(*res); /* get_base */ if (params->inplace) { basename = input; Py_INCREF((PyObject*)basename); } else { basename = (PyArrayObject*)PyArray_FromAny((PyObject*)input, NULL, 0, 0, NPY_ARRAY_ALIGNED|NPY_ARRAY_ENSURECOPY, NULL); } /* shape_statements and strides_statements */ dimensions = (npy_intp*) malloc(nd_out * sizeof(npy_intp)); strides = (npy_intp*) malloc(nd_out * sizeof(npy_intp)); if (dimensions == NULL || strides == NULL) { PyErr_NoMemory(); free(dimensions); free(strides); return 1; }; for (npy_intp i = 0; i < nd_out; ++i) { if (new_order[i] != -1) { dimensions[i] = PyArray_DIMS(basename)[new_order[i]]; strides[i] = PyArray_DIMS(basename)[new_order[i]] == 1 ? 0 : PyArray_STRIDES(basename)[new_order[i]]; } else { dimensions[i] = 1; strides[i] = 0; } } /* set the strides of the broadcasted dimensions. * This algorithm is from numpy: PyArray_Newshape() in * cvs/numpy/numpy/core/src/multiarraymodule.c */ if (nd_out > 0) { if (strides[nd_out - 1] == 0) strides[nd_out - 1] = PyArray_DESCR(basename)->elsize; for (npy_intp i = nd_out - 2; i > -1; --i) { if (strides[i] == 0) strides[i] = strides[i + 1] * dimensions[i + 1]; } } /* close_bracket */ // create a new array. *res = (PyArrayObject*)PyArray_New(&PyArray_Type, nd_out, dimensions, PyArray_TYPE(basename), strides, PyArray_DATA(basename), PyArray_ITEMSIZE(basename), // borrow only the writable flag from the base // the NPY_OWNDATA flag will default to 0. (NPY_ARRAY_WRITEABLE * PyArray_ISWRITEABLE(basename)), NULL); if (*res == NULL) { free(dimensions); free(strides); return 1; } // recalculate flags: CONTIGUOUS, FORTRAN, ALIGNED PyArray_UpdateFlags(*res, NPY_ARRAY_UPDATE_ALL); // we are making a view in both inplace and non-inplace cases PyArray_SetBaseObject(*res, (PyObject*)basename); free(strides); free(dimensions); return 0; } Theano-1.0.0/theano/tensor/c_code/alt_blas_template.c00006000163562€ÿÿÿþ0000003755213203335532024127 0ustar boccoset00000000000000/** Alternative template NumPy-based implementation of BLAS functions used in Theano. **/ /* Compute matrix[i][j] = scalar for every position (i, j) in matrix. */ void alt_numpy_memset_inplace_%(float_type)s(PyArrayObject* matrix, const %(float_type)s* scalar) { if (PyArray_IS_C_CONTIGUOUS(matrix) && *scalar == (char)(*scalar)) { // This will use memset. PyArray_FILLWBYTE(matrix, (char)(*scalar)); return; } NpyIter* iterator = NpyIter_New(matrix, NPY_ITER_READWRITE | NPY_ITER_EXTERNAL_LOOP | NPY_ITER_REFS_OK, NPY_KEEPORDER, NPY_NO_CASTING, NULL); if(iterator == NULL) alt_fatal_error("Unable to iterate over a matrix for a memory assignation."); NpyIter_IterNextFunc* get_next = NpyIter_GetIterNext(iterator, NULL); char** data_ptr = NpyIter_GetDataPtrArray(iterator); npy_intp* stride_ptr = NpyIter_GetInnerStrideArray(iterator); npy_intp* innersize_ptr = NpyIter_GetInnerLoopSizePtr(iterator); do { char* data = *data_ptr; npy_intp stride = *stride_ptr; npy_intp count = *innersize_ptr; while(count) { *((%(float_type)s*)data) = *scalar; data += stride; --count; } } while(get_next(iterator)); NpyIter_Deallocate(iterator); } /* Scalar * Matrix function. * Computes: matrix = scalar * matrix. */ void alt_numpy_scale_matrix_inplace_%(float_type)s(const %(float_type)s* scalar, PyArrayObject* matrix) { if (*scalar == 1) return; if (*scalar == 0) { alt_numpy_memset_inplace_%(float_type)s(matrix, scalar); return; } NpyIter* iterator = NpyIter_New(matrix, NPY_ITER_READWRITE | NPY_ITER_EXTERNAL_LOOP | NPY_ITER_REFS_OK, NPY_KEEPORDER, NPY_NO_CASTING, NULL); if(iterator == NULL) alt_fatal_error("Unable to iterate over a matrix " "for a scalar * matrix operation."); NpyIter_IterNextFunc* get_next = NpyIter_GetIterNext(iterator, NULL); char** data_ptr = NpyIter_GetDataPtrArray(iterator); npy_intp* stride_ptr = NpyIter_GetInnerStrideArray(iterator); npy_intp* innersize_ptr = NpyIter_GetInnerLoopSizePtr(iterator); do { char* data = *data_ptr; npy_intp stride = *stride_ptr; npy_intp count = *innersize_ptr; while(count) { *((%(float_type)s*)data) *= *scalar; data += stride; --count; } } while(get_next(iterator)); NpyIter_Deallocate(iterator); } /* Matrix + Matrix function. * Computes: matrix2 = (scalar1 * matrix1) + (scalar2 * matrix2) */ void alt_numpy_matrix_extended_sum_inplace_%(float_type)s( const %(float_type)s* scalar1, PyArrayObject* matrix1, const %(float_type)s* scalar2, PyArrayObject* matrix2 ) { if (*scalar1 == 0 && *scalar2 == 0) { alt_numpy_memset_inplace_%(float_type)s(matrix2, scalar2); return; } if (*scalar1 == 0) { alt_numpy_scale_matrix_inplace_%(float_type)s(scalar2, matrix2); return; } PyArrayObject* op[2] = {matrix1, matrix2}; npy_uint32 op_flags[2] = {NPY_ITER_READONLY, NPY_ITER_READWRITE}; npy_uint32 flags = 0; NpyIter* iterators = NpyIter_MultiNew( 2, op, flags, NPY_CORDER, NPY_NO_CASTING, op_flags, NULL); if(iterators == NULL) alt_fatal_error("Unable to iterate over some matrices " "for matrix + matrix operation."); NpyIter_IterNextFunc* get_next = NpyIter_GetIterNext(iterators, NULL); char** data_ptr_array = NpyIter_GetDataPtrArray(iterators); if (*scalar2 == 0) { do { %(float_type)s* from_matrix1 = (%(float_type)s*)data_ptr_array[0]; %(float_type)s* from_matrix2 = (%(float_type)s*)data_ptr_array[1]; *from_matrix2 = (*scalar1)*(*from_matrix1); } while(get_next(iterators)); } else { do { %(float_type)s* from_matrix1 = (%(float_type)s*)data_ptr_array[0]; %(float_type)s* from_matrix2 = (%(float_type)s*)data_ptr_array[1]; *from_matrix2 = (*scalar1)*(*from_matrix1) + (*scalar2)*(*from_matrix2); } while(get_next(iterators)); } NpyIter_Deallocate(iterators); } /* NumPy Wrapping function. Wraps a data into a NumPy's PyArrayObject. * By default, data is considered as Fortran-style array (column by column). * If to_transpose, data will be considered as C-style array (row by row) * with dimensions reversed. */ PyObject* alt_op_%(float_type)s(int to_transpose, %(float_type)s* M, int nrow, int ncol, int LDM, int numpyFlags) { npy_intp dims[2]; npy_intp strides[2]; if(to_transpose) { dims[0] = ncol; dims[1] = nrow; strides[0] = LDM * %(float_size)d; strides[1] = %(float_size)d; } else { dims[0] = nrow; dims[1] = ncol; strides[0] = %(float_size)d; strides[1] = LDM * %(float_size)d; } return PyArray_New(&PyArray_Type, 2, dims, %(npy_float)s, strides, M, 0, numpyFlags, NULL); } /* Special wrapping case used for matrix C in gemm_ implementation. */ inline PyObject* alt_wrap_fortran_writeable_matrix_%(float_type)s( %(float_type)s* matrix, const int* nrow, const int* ncol, const int* LD ) { npy_intp dims[2] = {*nrow, *ncol}; npy_intp strides[2] = {%(float_size)d, (*LD) * %(float_size)d}; return PyArray_New(&PyArray_Type, 2, dims, %(npy_float)s, strides, matrix, 0, NPY_ARRAY_WRITEABLE, NULL); } /* gemm */ void %(precision)sgemm_( char* TRANSA, char* TRANSB, const int* M, const int* N, const int* K, const %(float_type)s* ALPHA, %(float_type)s* A, const int* LDA, %(float_type)s* B, const int* LDB, const %(float_type)s* BETA, %(float_type)s* C, const int* LDC ) { if(*M < 0 || *N < 0 || *K < 0 || *LDA < 0 || *LDB < 0 || *LDC < 0) alt_fatal_error("The integer arguments passed to %(precision)sgemm_ must all be at least 0."); /* If M or N is null, there is nothing to do with C, * as C should contain M*N == 0 items. */ if(*M == 0 || *N == 0) return; int nrowa, ncola, nrowb, ncolb; int to_transpose_A = alt_trans_to_bool(TRANSA); int to_transpose_B = alt_trans_to_bool(TRANSB); if(to_transpose_A) { nrowa = *K; ncola = *M; } else { nrowa = *M; ncola = *K; } if(to_transpose_B) { nrowb = *N; ncolb = *K; } else { nrowb = *K; ncolb = *N; } int computation_flags; void* computation_pointer; npy_intp* computation_strides; npy_intp computation_dims[2] = {*N, *M}; npy_intp default_computation_strides[2] = {(*LDC) * %(float_size)d, %(float_size)d}; if(*BETA == 0 && *LDC == *M) { /* BETA == 0, so C is never read. * LDC == M, so C is contiguous in memory * (that condition is needed for dot operation, se below). * Then we can compute ALPHA*op(A)*op(B) directly in C. */ computation_flags = NPY_ARRAY_WRITEABLE; computation_pointer = C; computation_strides = default_computation_strides; } else { /* Either BETA != 0 (C will be read) * or LDC != M (C is not read but is not contiguous in memory). * Then in both cases, we need to allocate a new memory * to compute ALPHA*op(A)*op(B). */ computation_flags = 0; computation_pointer = NULL; computation_strides = NULL; } /* The memory buffer used to compute op(A)*op(B) (either C or * new allocated buffer) will be considered as C-contiguous because * the 3rd parameter of PyArray_MatrixProduct2 (used below) * expects a C-contiguous array. * Also, to avoid some memory copy, transposition conditions * for A and B will be reversed, so that the buffer will contain * C-contiguous opB_transposed * opA_transposed (N*M matrix). * After that, the code that uses the buffer (either the code calling * this function, or this function if BETA != 0) just has to * consider the buffer as a F-contiguous M*N matrix, so that * it will get the transposed of op_B_transposed * op_A_transposed, * that is op_A * op_B (M*N matrix) as expected. */ PyObject* opA_transposed = alt_op_%(float_type)s(!to_transpose_A, A, nrowa, ncola, *LDA, 0); PyObject* opB_transposed = alt_op_%(float_type)s(!to_transpose_B, B, nrowb, ncolb, *LDB, 0); PyObject* opB_trans_dot_opA_trans = PyArray_New(&PyArray_Type, 2, computation_dims, %(npy_float)s, computation_strides, computation_pointer, 0, computation_flags, NULL); PyArray_MatrixProduct2(opB_transposed, opA_transposed, (PyArrayObject*)opB_trans_dot_opA_trans); /* PyArray_MatrixProduct2 adds a reference to the output array, * which we need to remove to avoid a memory leak. */ Py_XDECREF(opB_trans_dot_opA_trans); if(*BETA == 0) { if(*ALPHA != 1.0) alt_numpy_scale_matrix_inplace_%(float_type)s(ALPHA, (PyArrayObject*)opB_trans_dot_opA_trans); if(*LDC != *M) { /* A buffer has been created to compute ALPHA*op(A)*op(B), * so we must copy it to the real output, that is C. */ PyObject* matrix_C = alt_wrap_fortran_writeable_matrix_%(float_type)s(C, M, N, LDC); PyObject* alpha_opA_dot_opB = PyArray_Transpose((PyArrayObject*)opB_trans_dot_opA_trans, NULL); if(0 != PyArray_CopyInto((PyArrayObject*)matrix_C, (PyArrayObject*)alpha_opA_dot_opB)) alt_fatal_error("NumPy %(precision)sgemm_ implementation: unable to copy ALPHA*op(A)*op(B) into C when BETA == 0."); Py_XDECREF(alpha_opA_dot_opB); Py_XDECREF(matrix_C); } } else { /* C is read, so we must consider it as Fortran-style matrix. */ PyObject* matrix_C = alt_wrap_fortran_writeable_matrix_%(float_type)s(C, M, N, LDC); PyObject* opA_dot_opB = PyArray_Transpose((PyArrayObject*)opB_trans_dot_opA_trans, NULL); alt_numpy_matrix_extended_sum_inplace_%(float_type)s(ALPHA, (PyArrayObject*)opA_dot_opB, BETA, (PyArrayObject*)matrix_C); Py_XDECREF(opA_dot_opB); Py_XDECREF(matrix_C); } Py_XDECREF(opB_trans_dot_opA_trans); Py_XDECREF(opB_transposed); Py_XDECREF(opA_transposed); } /* gemv */ void %(precision)sgemv_( char* TRANS, const int* M, const int* N, const %(float_type)s* ALPHA, %(float_type)s* A, const int* LDA, %(float_type)s* x, const int* incx, const %(float_type)s* BETA, %(float_type)s* y, const int* incy ) { /** If TRANS is 'n' or 'N', computes: y = ALPHA * A * x + BETA * y Else, computes: y = ALPHA * A.T * x + BETA * y A is a M*N matrix, A.T is A transposed x, y are vectors ALPHA, BETA are scalars **/ // If alpha == 0 and beta == 1, we have nothing to do, as alpha*A*x + beta*y == y. if (*ALPHA == 0 && *BETA == 1) return; if (*M < 0 || *N < 0 || *LDA < 0) alt_fatal_error("NumPy %(precision)sgemv_ implementation: M, N and LDA must be at least 0."); if (*incx == 0 || *incy == 0) alt_fatal_error("NumPy %(precision)sgemv_ implementation: incx and incy must not be 0."); int transpose = alt_trans_to_bool(TRANS); int size_x = 0, size_y = 0; if (transpose) { size_x = *M; size_y = *N; } else { size_x = *N; size_y = *M; } if (*M == 0 || *N == 0) { /* A contains M * N == 0 values. y should be empty too, and we have nothing to do. */ if (size_y != 0) alt_fatal_error("NumPy %(precision)sgemv_ implementation: the output vector should be empty."); return; } /* Vector pointers points to the begining of memory (see function `theano.tensor.blas_c.gemv_c_code`). * NumPy seems to expect that pointers points to the first element of the array. */ if (*incx < 0) x += (size_x - 1) * (-*incx); if (*incy < 0) y += (size_y - 1) * (-*incy); PyObject* matrixA = alt_op_%(float_type)s(transpose, A, *M, *N, *LDA, 0); PyObject* matrixX = alt_op_%(float_type)s(1, x, 1, size_x, *incx, 0); PyObject* matrixY = alt_op_%(float_type)s(1, y, 1, size_y, *incy, NPY_ARRAY_WRITEABLE); if (matrixA == NULL || matrixX == NULL || matrixY == NULL) alt_fatal_error("NumPy %(precision)sgemv_ implementation: unable to wrap A, x or y arrays.") if (*ALPHA == 0) { // Just BETA * y alt_numpy_scale_matrix_inplace_%(float_type)s(BETA, (PyArrayObject*)matrixY); } else if (*BETA == 0) { // We can directly compute alpha * A * x into y if y is C-contiguous. if (PyArray_IS_C_CONTIGUOUS((PyArrayObject*)matrixY)) { PyArray_MatrixProduct2(matrixA, matrixX, (PyArrayObject*)matrixY); // PyArray_MatrixProduct2 adds an extra reference to the output array. Py_XDECREF(matrixY); alt_numpy_scale_matrix_inplace_%(float_type)s(ALPHA, (PyArrayObject*)matrixY); } else { // If y is not contiguous, we need a temporar workspace. PyObject* tempAX = PyArray_MatrixProduct(matrixA, matrixX); if (tempAX == NULL) alt_fatal_error("NumPy %(precision)sgemv_ implementation: Unable to get matrix product."); alt_numpy_scale_matrix_inplace_%(float_type)s(ALPHA, (PyArrayObject*)tempAX); if(0 != PyArray_CopyInto((PyArrayObject*)matrixY, (PyArrayObject*)tempAX)) { alt_fatal_error("NumPy %(precision)sgemv_ implementation: unable to update output."); } Py_XDECREF(tempAX); } } else { // We must perform full computation. PyObject* tempAX = PyArray_MatrixProduct(matrixA, matrixX); if (tempAX == NULL) alt_fatal_error("NumPy %(precision)sgemv_ implementation: unable to get matrix product."); // ALPHA * (A * x) + BETA * y. alt_numpy_matrix_extended_sum_inplace_%(float_type)s(ALPHA, (PyArrayObject*)tempAX, BETA, (PyArrayObject*)matrixY); Py_XDECREF(tempAX); } Py_XDECREF(matrixY); Py_XDECREF(matrixX); Py_XDECREF(matrixA); } /* dot */ %(float_type)s %(precision)sdot_( const int* N, %(float_type)s *SX, const int *INCX, %(float_type)s *SY, const int *INCY ) { if (*N < 0) alt_fatal_error("NumPy %(precision)sdot_ implementation: N must be at least 0."); if (*INCX == 0 || *INCY == 0) alt_fatal_error("NumPy %(precision)sdot_ implementation: INCX and INCY must not be 0."); %(float_type)s result = 0; int one = 1; /* Vector pointers points to the begining of memory (see function `theano.tensor.blas_c.gemv_c_code`). * NumPy seems to expect that pointers points to the first element of the array. */ if (*INCX < 0) SX += (*N - 1) * (-*INCX); if (*INCY < 0) SY += (*N - 1) * (-*INCY); // Create vector_x with shape (1, N) PyObject* vector_x = alt_op_%(float_type)s(0, SX, 1, *N, *INCX, 0); // Create vector_y with shape (N, 1) PyObject* vector_y = alt_op_%(float_type)s(1, SY, 1, *N, *INCY, 0); // Create output scalar z with shape (1, 1) to wrap `result`. PyArrayObject* dot_product = (PyArrayObject*)alt_wrap_fortran_writeable_matrix_%(float_type)s(&result, &one, &one, &one); if (vector_x == NULL || vector_y == NULL || dot_product == NULL) alt_fatal_error("NumPy %(precision)sdot_ implementation: unable to wrap x, y or output arrays."); // Compute matrix product: (1, N) * (N, 1) => (1, 1) PyArray_MatrixProduct2(vector_x, vector_y, dot_product); // PyArray_MatrixProduct2 adds an extra reference to the output array. Py_XDECREF(dot_product); if (PyErr_Occurred()) alt_fatal_error("NumPy %(precision)sdot_ implementation: unable to compute dot."); Py_XDECREF(dot_product); Py_XDECREF(vector_y); Py_XDECREF(vector_x); return result; } Theano-1.0.0/theano/tensor/opt_uncanonicalize.py00006000163562€ÿÿÿþ0000002242713203335532023324 0ustar boccoset00000000000000""" This file implement specialization optimization that break the canonization form of the graph. Currently there is problem with the order of optimization and the definition of definition of canonized graph. Right now there is a canonization optimization phase that try to make all equivalent graph identical. This is not always the case, but it do many of the basic stuff canonical. We need to extend the definition of canonization to make this true more often. The problem this file indent to fix in the future is that in the "Equilibrium" specialization optimization phase, there is optimization that request that the graph is canonical, some other request that this is not true, and some other that break the canonicalization for some optimization. As we can't control the order of those optimization, there is case that some optimization requesting a canonical graph won't be applied as optimization that break the canonicalization form of the graph executed before. To fix this, we need to split the specialization phase into a phase where optimization can't break the canonicalization form and one where this is allowed. This is also needed for the stabilized optimization phase, but as it happen before the specialization phase, this cause less problem. Also, we should make the fgraph refuse optimization that break the canonization of the graph in the optimizations phases where the graph is supposed to be canonical. """ from __future__ import absolute_import, print_function, division # TODO: intelligent merge for mul/add # TODO: 0*x -> 0 import logging from theano import gof from theano.tensor.elemwise import CAReduce from theano.tensor import basic as T from theano.tensor import DimShuffle, Subtensor from theano.tensor.opt import register_uncanonicalize from theano import scalar as scal from theano.gof.opt import copy_stack_trace _logger = logging.getLogger('theano.tensor.opt') @register_uncanonicalize @gof.local_optimizer([T.MaxAndArgmax]) def local_max_and_argmax(node): """ If we don't use the argmax, change it to a max only. """ if isinstance(node.op, T.MaxAndArgmax): axis = node.op.get_params(node) if len(node.outputs[1].clients) == 0: new = CAReduce(scal.maximum, axis)(node.inputs[0]) copy_stack_trace(node.outputs[0], new) return [new, None] if len(node.outputs[0].clients) == 0: new = T.Argmax(axis)(node.inputs[0]) copy_stack_trace(node.outputs[0], new) return [None, new] @register_uncanonicalize @gof.local_optimizer([T.neg]) def local_max_to_min(node): """ Change -(max(-x)) to min. This is tested in tensor/tests/test_basic.py:test_min_max. Notes ----- We don't need an opt that will do the reverse as by default the interface put only MaxAndArgmax into the graph. """ if node.op == T.neg and node.inputs[0].owner: max = node.inputs[0] if (max.owner and isinstance(max.owner.op, CAReduce) and max.owner.op.scalar_op == scal.maximum): neg = max.owner.inputs[0] if neg.owner and neg.owner.op == T.neg: new = CAReduce(scal.minimum, max.owner.op.axis)(neg.owner.inputs[0]) return [copy_stack_trace(node.outputs[0], new)] return False @register_uncanonicalize @gof.local_optimizer([T.Alloc]) def local_alloc_dimshuffle(node): """ If a dimshuffle is inside an alloc and only adds dimension to the left, remove it. Alloc(DimShuffle(x), ...) - > Alloc(x, ...) """ if isinstance(node.op, T.Alloc): input_ = node.inputs[0] if input_.owner and isinstance(input_.owner.op, DimShuffle): # check if it only adds dimension to the left new_order = input_.owner.op.new_order expected_new_order = ('x',) * (input_.ndim - input_.owner.inputs[0].ndim) + \ tuple(range(input_.owner.inputs[0].ndim)) if new_order != expected_new_order: return False return [T.alloc(input_.owner.inputs[0], *node.inputs[1:])] return False @register_uncanonicalize @gof.local_optimizer([T.Reshape]) def local_reshape_dimshuffle(node): """ If a dimshuffle is inside a reshape and does not change the order of dimensions, remove it. Reshape(Dimshuffle(x), shp) -> Reshape(x, shp) """ if isinstance(node.op, T.Reshape): input_ = node.inputs[0] if input_.owner and isinstance(input_.owner.op, DimShuffle): new_order = input_.owner.op.new_order offset = 0 for dim in new_order: if dim == 'x': continue elif dim != offset: return False else: offset += 1 return [T.reshape(input_.owner.inputs[0], node.inputs[1], ndim=node.outputs[0].ndim)] return False @register_uncanonicalize @gof.local_optimizer([DimShuffle]) def local_dimshuffle_alloc(node): """ If an alloc is inside a dimshuffle which only adds dimension to the left, scrap the dimshuffle and adds 1 into the alloc dimshuffle{x, 0, 1}(alloc([3 4], 3, 2) => alloc([3 4], 1, 3, 2) """ if isinstance(node.op, DimShuffle) and node.inputs[0].owner: input_ = node.inputs[0] if isinstance(input_.owner.op, T.Alloc): # check if it only adds dimension to the left new_order = node.op.new_order expected_new_order = ('x',) * (len(new_order) - input_.ndim) + \ tuple(range(input_.ndim)) if new_order != expected_new_order: return False # count numbers of 'x' nb_new_dims = len(new_order) - input_.ndim new_shape_input = (1,) * nb_new_dims + tuple(input_.owner.inputs[1:]) return [T.alloc(input_.owner.inputs[0], *new_shape_input)] return False @register_uncanonicalize @gof.local_optimizer([DimShuffle]) def local_dimshuffle_subtensor(node): """If a subtensor is inside a dimshuffle which only drop broadcastable dimensions, scrap the dimshuffle and index the subtensor with 0 x[i:j, :, k:l].dimshuffle(0, 2) => x[i:j, 0, k:l] if x.broadcastable == (False, True, False) """ if isinstance(node.op, DimShuffle) and node.inputs[0].owner: # the dimshuffle can only drop dimensions (cannot reshape nor add 'x') if 'x' in node.op.new_order: return False new_order = node.op.new_order # new order could be empty # Verif that we don't change dimensions order. if len(new_order) > 1: past_dim = new_order[0] for dim in new_order[1:]: if not dim > past_dim: return False else: past_dim = dim input_ = node.inputs[0] if isinstance(input_.owner.op, Subtensor): # the arguments missing from the dimshuffles must be dims # that are broadcastable broadcastable = input_.broadcastable missing_dims = list(range(input_.ndim)) for dim in new_order: missing_dims.remove(dim) if not all([broadcastable[i] for i in missing_dims]): return False # create a new idx_list for a new Subtensor object # have to loop on idx_list and inputs # inputs has the length of sum of non None elements of idx_list # (check in slice!). # len(missing_dims) can be < len(idx_list), this happens if # tensor was indexed such as x[scalar, :, :], check that as well new_idx_list = list(input_.owner.op.idx_list) new_inputs = [input_.owner.inputs[0]] zero = T.constant(0) slice_attr_list = ['start', 'stop', 'step'] j = 0 slice_i = -1 subtensor_removed_dims = 0 for i, idx in enumerate(input_.owner.op.idx_list): if isinstance(idx, slice): past_j = j slice_i += 1 for slice_attr in slice_attr_list: if getattr(idx, slice_attr) is not None: new_inputs += [input_.owner.inputs[1 + j]] j += 1 # if past_j == j indicates a slice(None, None, None), # that's where we want to index with 0 if it is also at # the same spot of a missing dim if past_j == j and slice_i in missing_dims: new_idx_list[i] = zero new_inputs += [zero] else: new_inputs += [input_.owner.inputs[1 + j]] j += 1 subtensor_removed_dims += 1 # Verify the trailing dimensions the subtensor didn't look at. for idx in range(len(input_.owner.op.idx_list), new_inputs[0].ndim): if (idx - subtensor_removed_dims) in missing_dims: while len(new_idx_list) < idx: new_idx_list.append(slice(None)) new_idx_list.append(zero) new_inputs.append(zero) return [Subtensor(new_idx_list)(*new_inputs)] return False Theano-1.0.0/theano/tensor/slinalg.py00006000163562€ÿÿÿþ0000004151013203335532021063 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import logging import warnings from six.moves import xrange import numpy as np try: import scipy.linalg imported_scipy = True except ImportError: # some ops (e.g. Cholesky, Solve, A_Xinv_b) won't work imported_scipy = False from theano import tensor import theano.tensor from theano.tensor import as_tensor_variable from theano.gof import Op, Apply logger = logging.getLogger(__name__) MATRIX_STRUCTURES = ( 'general', 'symmetric', 'lower_triangular', 'upper_triangular', 'hermitian', 'banded', 'diagonal', 'toeplitz') class Cholesky(Op): """ Return a triangular matrix square root of positive semi-definite `x`. L = cholesky(X, lower=True) implies dot(L, L.T) == X. Parameters ---------- lower : bool, default=True Whether to return the lower or upper cholesky factor on_error : ['raise', 'nan'] If on_error is set to 'raise', this Op will raise a `scipy.linalg.LinAlgError` if the matrix is not positive definite. If on_error is set to 'nan', it will return a matrix containing nans instead. """ # TODO: inplace # TODO: for specific dtypes # TODO: LAPACK wrapper with in-place behavior, for solve also __props__ = ('lower', 'destructive', 'on_error') def __init__(self, lower=True, on_error='raise'): self.lower = lower self.destructive = False if on_error not in ['raise', 'nan']: raise ValueError('on_error must be one of "raise" or ""nan"') self.on_error = on_error def infer_shape(self, node, shapes): return [shapes[0]] def make_node(self, x): assert imported_scipy, ( "Scipy not available. Scipy is needed for the Cholesky op") x = as_tensor_variable(x) assert x.ndim == 2 return Apply(self, [x], [x.type()]) def perform(self, node, inputs, outputs): x = inputs[0] z = outputs[0] try: z[0] = scipy.linalg.cholesky(x, lower=self.lower).astype(x.dtype) except scipy.linalg.LinAlgError: if self.on_error == 'raise': raise else: z[0] = (np.zeros(x.shape) * np.nan).astype(x.dtype) def L_op(self, inputs, outputs, gradients): """ Cholesky decomposition reverse-mode gradient update. Symbolic expression for reverse-mode Cholesky gradient taken from [#]_ References ---------- .. [#] I. Murray, "Differentiation of the Cholesky decomposition", http://arxiv.org/abs/1602.07527 """ dz = gradients[0] chol_x = outputs[0] # Replace the cholesky decomposition with 1 if there are nans # or solve_upper_triangular will throw a ValueError. if self.on_error == 'nan': ok = ~tensor.any(tensor.isnan(chol_x)) chol_x = tensor.switch(ok, chol_x, 1) dz = tensor.switch(ok, dz, 1) # deal with upper triangular by converting to lower triangular if not self.lower: chol_x = chol_x.T dz = dz.T def tril_and_halve_diagonal(mtx): """Extracts lower triangle of square matrix and halves diagonal.""" return tensor.tril(mtx) - tensor.diag(tensor.diagonal(mtx) / 2.) def conjugate_solve_triangular(outer, inner): """Computes L^{-T} P L^{-1} for lower-triangular L.""" return solve_upper_triangular( outer.T, solve_upper_triangular(outer.T, inner.T).T) s = conjugate_solve_triangular( chol_x, tril_and_halve_diagonal(chol_x.T.dot(dz))) if self.lower: grad = tensor.tril(s + s.T) - tensor.diag(tensor.diagonal(s)) else: grad = tensor.triu(s + s.T) - tensor.diag(tensor.diagonal(s)) if self.on_error == 'nan': return [tensor.switch(ok, grad, np.nan)] else: return [grad] cholesky = Cholesky() class CholeskyGrad(Op): """ """ __props__ = ('lower', 'destructive') def __init__(self, lower=True): self.lower = lower self.destructive = False def make_node(self, x, l, dz): x = as_tensor_variable(x) l = as_tensor_variable(l) dz = as_tensor_variable(dz) assert x.ndim == 2 assert l.ndim == 2 assert dz.ndim == 2 assert l.owner.op.lower == self.lower, ( "lower/upper mismatch between Cholesky op and CholeskyGrad op" ) return Apply(self, [x, l, dz], [x.type()]) def perform(self, node, inputs, outputs): """ Implements the "reverse-mode" gradient [#]_ for the Cholesky factorization of a positive-definite matrix. References ---------- .. [#] S. P. Smith. "Differentiation of the Cholesky Algorithm". Journal of Computational and Graphical Statistics, Vol. 4, No. 2 (Jun.,1995), pp. 134-147 http://www.jstor.org/stable/1390762 """ x = inputs[0] L = inputs[1] dz = inputs[2] dx = outputs[0] N = x.shape[0] if self.lower: F = np.tril(dz) for k in xrange(N - 1, -1, -1): for j in xrange(k + 1, N): for i in xrange(j, N): F[i, k] -= F[i, j] * L[j, k] F[j, k] -= F[i, j] * L[i, k] for j in xrange(k + 1, N): F[j, k] /= L[k, k] F[k, k] -= L[j, k] * F[j, k] F[k, k] /= (2 * L[k, k]) else: F = np.triu(dz) for k in xrange(N - 1, -1, -1): for j in xrange(k + 1, N): for i in xrange(j, N): F[k, i] -= F[j, i] * L[k, j] F[k, j] -= F[j, i] * L[k, i] for j in xrange(k + 1, N): F[k, j] /= L[k, k] F[k, k] -= L[k, j] * F[k, j] F[k, k] /= (2 * L[k, k]) dx[0] = F def infer_shape(self, node, shapes): return [shapes[0]] class Solve(Op): """ Solve a system of linear equations. For on CPU and GPU. """ __props__ = ('A_structure', 'lower', 'overwrite_A', 'overwrite_b') def __init__(self, A_structure='general', lower=False, overwrite_A=False, overwrite_b=False): if A_structure not in MATRIX_STRUCTURES: raise ValueError('Invalid matrix structure argument', A_structure) self.A_structure = A_structure self.lower = lower self.overwrite_A = overwrite_A self.overwrite_b = overwrite_b def __repr__(self): return 'Solve{%s}' % str(self._props()) def make_node(self, A, b): assert imported_scipy, ( "Scipy not available. Scipy is needed for the Solve op") A = as_tensor_variable(A) b = as_tensor_variable(b) assert A.ndim == 2 assert b.ndim in [1, 2] # infer dtype by solving the most simple # case with (1, 1) matrices o_dtype = scipy.linalg.solve( np.eye(1).astype(A.dtype), np.eye(1).astype(b.dtype)).dtype x = tensor.tensor( broadcastable=b.broadcastable, dtype=o_dtype) return Apply(self, [A, b], [x]) def perform(self, node, inputs, output_storage): A, b = inputs if self.A_structure == 'lower_triangular': rval = scipy.linalg.solve_triangular( A, b, lower=True) elif self.A_structure == 'upper_triangular': rval = scipy.linalg.solve_triangular( A, b, lower=False) else: rval = scipy.linalg.solve(A, b) output_storage[0][0] = rval # computes shape of x where x = inv(A) * b def infer_shape(self, node, shapes): Ashape, Bshape = shapes rows = Ashape[1] if len(Bshape) == 1: # b is a Vector return [(rows,)] else: cols = Bshape[1] # b is a Matrix return [(rows, cols)] def L_op(self, inputs, outputs, output_gradients): """ Reverse-mode gradient updates for matrix solve operation c = A \\\ b. Symbolic expression for updates taken from [#]_. References ---------- .. [#] M. B. Giles, "An extended collection of matrix derivative results for forward and reverse mode automatic differentiation", http://eprints.maths.ox.ac.uk/1079/ """ A, b = inputs c = outputs[0] c_bar = output_gradients[0] trans_map = { 'lower_triangular': 'upper_triangular', 'upper_triangular': 'lower_triangular' } trans_solve_op = Solve( # update A_structure and lower to account for a transpose operation A_structure=trans_map.get(self.A_structure, self.A_structure), lower=not self.lower ) b_bar = trans_solve_op(A.T, c_bar) # force outer product if vector second input A_bar = -tensor.outer(b_bar, c) if c.ndim == 1 else -b_bar.dot(c.T) if self.A_structure == 'lower_triangular': A_bar = tensor.tril(A_bar) elif self.A_structure == 'upper_triangular': A_bar = tensor.triu(A_bar) return [A_bar, b_bar] solve = Solve() """ Solves the equation ``a x = b`` for x, where ``a`` is a matrix and ``b`` can be either a vector or a matrix. Note Parameters ---------- a : `(M, M) symbolix matrix` A square matrix b : `(M,) or (M, N) symbolic vector or matrix` Right hand side matrix in ``a x = b`` Returns ------- x : `(M, ) or (M, N) symbolic vector or matrix` x will have the same shape as b """ # lower and upper triangular solves solve_lower_triangular = Solve(A_structure='lower_triangular', lower=True) """Optimized implementation of :func:`theano.tensor.slinalg.solve` when A is lower triangular.""" solve_upper_triangular = Solve(A_structure='upper_triangular', lower=False) """Optimized implementation of :func:`theano.tensor.slinalg.solve` when A is upper triangular.""" # symmetric solves solve_symmetric = Solve(A_structure='symmetric') """Optimized implementation of :func:`theano.tensor.slinalg.solve` when A is symmetric.""" # TODO: Optimizations to replace multiplication by matrix inverse # with solve() Op (still unwritten) class Eigvalsh(Op): """ Generalized eigenvalues of a Hermitian positive definite eigensystem. """ __props__ = ('lower',) def __init__(self, lower=True): assert lower in [True, False] self.lower = lower def make_node(self, a, b): assert imported_scipy, ( "Scipy not available. Scipy is needed for the Eigvalsh op") if b == theano.tensor.NoneConst: a = as_tensor_variable(a) assert a.ndim == 2 out_dtype = theano.scalar.upcast(a.dtype) w = theano.tensor.vector(dtype=out_dtype) return Apply(self, [a], [w]) else: a = as_tensor_variable(a) b = as_tensor_variable(b) assert a.ndim == 2 assert b.ndim == 2 out_dtype = theano.scalar.upcast(a.dtype, b.dtype) w = theano.tensor.vector(dtype=out_dtype) return Apply(self, [a, b], [w]) def perform(self, node, inputs, outputs): (w,) = outputs if len(inputs) == 2: w[0] = scipy.linalg.eigvalsh(a=inputs[0], b=inputs[1], lower=self.lower) else: w[0] = scipy.linalg.eigvalsh(a=inputs[0], b=None, lower=self.lower) def grad(self, inputs, g_outputs): a, b = inputs gw, = g_outputs return EigvalshGrad(self.lower)(a, b, gw) def infer_shape(self, node, shapes): n = shapes[0][0] return [(n,)] class EigvalshGrad(Op): """ Gradient of generalized eigenvalues of a Hermitian positive definite eigensystem. """ # Note: This Op (EigvalshGrad), should be removed and replaced with a graph # of theano ops that is constructed directly in Eigvalsh.grad. # But this can only be done once scipy.linalg.eigh is available as an Op # (currently the Eigh uses numpy.linalg.eigh, which doesn't let you # pass the right-hand-side matrix for a generalized eigenproblem.) See the # discussion on github at # https://github.com/Theano/Theano/pull/1846#discussion-diff-12486764 __props__ = ('lower',) def __init__(self, lower=True): assert lower in [True, False] self.lower = lower if lower: self.tri0 = np.tril self.tri1 = lambda a: np.triu(a, 1) else: self.tri0 = np.triu self.tri1 = lambda a: np.tril(a, -1) def make_node(self, a, b, gw): assert imported_scipy, ( "Scipy not available. Scipy is needed for the GEigvalsh op") a = as_tensor_variable(a) b = as_tensor_variable(b) gw = as_tensor_variable(gw) assert a.ndim == 2 assert b.ndim == 2 assert gw.ndim == 1 out_dtype = theano.scalar.upcast(a.dtype, b.dtype, gw.dtype) out1 = theano.tensor.matrix(dtype=out_dtype) out2 = theano.tensor.matrix(dtype=out_dtype) return Apply(self, [a, b, gw], [out1, out2]) def perform(self, node, inputs, outputs): (a, b, gw) = inputs w, v = scipy.linalg.eigh(a, b, lower=self.lower) gA = v.dot(np.diag(gw).dot(v.T)) gB = - v.dot(np.diag(gw * w).dot(v.T)) # See EighGrad comments for an explanation of these lines out1 = self.tri0(gA) + self.tri1(gA).T out2 = self.tri0(gB) + self.tri1(gB).T outputs[0][0] = np.asarray(out1, dtype=node.outputs[0].dtype) outputs[1][0] = np.asarray(out2, dtype=node.outputs[1].dtype) def infer_shape(self, node, shapes): return [shapes[0], shapes[1]] def eigvalsh(a, b, lower=True): return Eigvalsh(lower)(a, b) def kron(a, b): """ Kronecker product. Same as scipy.linalg.kron(a, b). Parameters ---------- a: array_like b: array_like Returns ------- array_like with a.ndim + b.ndim - 2 dimensions Notes ----- numpy.kron(a, b) != scipy.linalg.kron(a, b)! They don't have the same shape and order when a.ndim != b.ndim != 2. """ a = tensor.as_tensor_variable(a) b = tensor.as_tensor_variable(b) if (a.ndim + b.ndim <= 2): raise TypeError('kron: inputs dimensions must sum to 3 or more. ' 'You passed %d and %d.' % (a.ndim, b.ndim)) o = tensor.outer(a, b) o = o.reshape(tensor.concatenate((a.shape, b.shape)), a.ndim + b.ndim) shf = o.dimshuffle(0, 2, 1, * list(range(3, o.ndim))) if shf.ndim == 3: shf = o.dimshuffle(1, 0, 2) o = shf.flatten() else: o = shf.reshape((o.shape[0] * o.shape[2], o.shape[1] * o.shape[3]) + tuple(o.shape[i] for i in xrange(4, o.ndim))) return o class Expm(Op): """ Compute the matrix exponential of a square array. """ __props__ = () def make_node(self, A): assert imported_scipy, ( "Scipy not available. Scipy is needed for the Expm op") A = as_tensor_variable(A) assert A.ndim == 2 expm = theano.tensor.matrix(dtype=A.dtype) return Apply(self, [A, ], [expm, ]) def perform(self, node, inputs, outputs): (A,) = inputs (expm,) = outputs expm[0] = scipy.linalg.expm(A) def grad(self, inputs, outputs): (A,) = inputs (g_out,) = outputs return [ExpmGrad()(A, g_out)] def infer_shape(self, node, shapes): return [shapes[0]] class ExpmGrad(Op): """ Gradient of the matrix exponential of a square array. """ __props__ = () def make_node(self, A, gw): assert imported_scipy, ( "Scipy not available. Scipy is needed for the Expm op") A = as_tensor_variable(A) assert A.ndim == 2 out = theano.tensor.matrix(dtype=A.dtype) return Apply(self, [A, gw], [out, ]) def infer_shape(self, node, shapes): return [shapes[0]] def perform(self, node, inputs, outputs): # Kalbfleisch and Lawless, J. Am. Stat. Assoc. 80 (1985) Equation 3.4 # Kind of... You need to do some algebra from there to arrive at # this expression. (A, gA) = inputs (out,) = outputs w, V = scipy.linalg.eig(A, right=True) U = scipy.linalg.inv(V).T exp_w = np.exp(w) X = np.subtract.outer(exp_w, exp_w) / np.subtract.outer(w, w) np.fill_diagonal(X, exp_w) Y = U.dot(V.T.dot(gA).dot(U) * X).dot(V.T) with warnings.catch_warnings(): warnings.simplefilter("ignore", np.ComplexWarning) out[0] = Y.astype(A.dtype) expm = Expm() Theano-1.0.0/theano/tensor/basic.py00006000163562€ÿÿÿþ0000070673713203335532020536 0ustar boccoset00000000000000"""A `Type` and `Op` classes to work with numpy.ndarrays symbolically.""" from __future__ import absolute_import, print_function, division from six.moves import builtins import sys import warnings import numpy as np from six import integer_types from six.moves import xrange import numbers import theano from theano.compat import izip from theano import config from theano import gof from theano.gof import Apply, Constant, Op, Variable, ParamsType from theano.gof.type import Generic from theano.scalar import int32 as int32_t from theano.tensor import elemwise from theano.tensor.var import (AsTensorError, TensorVariable, TensorConstant, TensorConstantSignature, _tensor_py_operators) from theano.tensor.type import TensorType, values_eq_approx_always_true from theano.tensor.type_other import NoneConst from theano import scalar as scal from functools import partial from theano import compile, printing from theano.printing import pprint, min_informative_str # For history from theano.compile import Rebroadcast, Shape, shape from theano.scalar import int32 # We use these exceptions as well. import theano.scalar.sharedvar from theano.gradient import grad_undefined from theano.gradient import grad_not_implemented from theano.gradient import DisconnectedType # set up the external interface from theano.tensor.elemwise import Elemwise, DimShuffle, CAReduce, Sum import logging _logger = logging.getLogger("theano.tensor.basic") __docformat__ = "restructuredtext en" # This is needed as we will hide it later python_complex = complex python_any = any python_all = all # Define common subsets of dtypes (as strings). complex_dtypes = list(map(str, scal.complex_types)) continuous_dtypes = list(map(str, scal.continuous_types)) float_dtypes = list(map(str, scal.float_types)) integer_dtypes = list(map(str, scal.integer_types)) discrete_dtypes = list(map(str, scal.discrete_types)) all_dtypes = list(map(str, scal.all_types)) int_dtypes = list(map(str, scal.int_types)) uint_dtypes = list(map(str, scal.uint_types)) class ShapeError(Exception): """Raised when the shape cannot be computed.""" pass def check_equal_numpy(x, y): """ Return True iff x and y are equal. Checks the dtype and shape if x and y are numpy.ndarray instances. """ if isinstance(x, np.ndarray) and isinstance(y, np.ndarray): return (x.dtype == y.dtype and x.shape == y.shape and np.all(abs(x - y) < 1e-10)) elif (isinstance(x, np.random.RandomState) and isinstance(y, np.random.RandomState)): return python_all(np.all(a == b) for a, b in izip(x.__getstate__(), y.__getstate__())) else: return x == y compile.register_checker(check_equal_numpy) __oplist_constructor_list = [] """List of functions to be listed as op constructors in the oplist (`gen_oplist`, doc/oplist.txt).""" def constructor(f): """Add `f` to :doc:`oplist`. Make `f` appear as a constructor in the oplist (`gen_oplist`, doc/oplist.txt). """ __oplist_constructor_list.append(f) return f def __oplist_tag(thing, tag): tags = getattr(thing, '__oplist_tags', []) tags.append(tag) thing.__oplist_tags = tags def as_tensor_variable(x, name=None, ndim=None): """Return `x`, transformed into a `TensorType`. This function is often used by `make_node` methods of `Op` subclasses to turn ndarrays, numbers, `Scalar` instances, `Apply` instances and `TensorType` instances into valid input list elements. Parameters ---------- x : Apply instance, Variable instance, numpy.ndarray, or number This thing will be transformed into a `Variable` in a sensible way. An ndarray argument will not be copied, but a list of numbers will be copied to make an ndarray. name : str or None If a new `Variable` instance is created, it will be named with this string. ndim : None or integer Return a Variable with this many dimensions. Raises ------ ValueError If an `Apply` with more than one output is fetched or if `x` cannot be made into a Variable with `ndim` dimensions. AsTensorError If `x` cannot be converted to a TensorType Variable. """ if hasattr(x, '_as_TensorVariable'): return x._as_TensorVariable() # TODO: pass name and ndim arguments if isinstance(x, gof.Apply): # use Apply's default output mechanism if (x.op.default_output is None) and (len(x.outputs) != 1): raise ValueError( "It is ambiguous which output of a multi-output Op has" " to be fetched.", x) x = x.default_output() if isinstance(x, Variable): if isinstance(x.type, scal.Scalar): x = tensor_from_scalar(x) if not isinstance(x.type, TensorType): raise AsTensorError( "Variable type field must be a TensorType.", x, x.type) if ndim is None: return x else: if (x.type.ndim > ndim): # strip off leading broadcastable dimensions first_non_broadcastable = [idx for idx in xrange(x.ndim) if not x.broadcastable[idx]][0] x = x.dimshuffle(list(range(x.ndim))[first_non_broadcastable:]) if x.ndim > ndim: raise ValueError( 'TensorType could not be cast to have %i dimensions' % ndim, x.type ) return x elif (x.type.ndim < ndim): return shape_padleft(x, n_ones=(ndim - x.type.ndim)) else: return x if isinstance(x, (tuple, list)) and python_any(isinstance(xi, Variable) for xi in x): try: return stack(x) except (TypeError, ValueError): pass if isinstance(x, bool): raise AsTensorError( "Cannot cast True or False as a tensor variable. Please use " "np.array(True) or np.array(False) if you need these constants. " "This error might be caused by using the == operator on " "Variables. v == w does not do what you think it does, " "use theano.tensor.eq(v, w) instead.") try: return constant(x, name=name, ndim=ndim) except TypeError: try: str_x = str(x) except Exception: str_x = repr(x) raise AsTensorError("Cannot convert %s to TensorType" % str_x, type(x)) # this has a different name, because _as_tensor_variable is the # function which ops use to upcast their arguments... this # internal-use function is a good place to put debugging stuff, better # than the global astensor. _as_tensor_variable = as_tensor_variable as_tensor = as_tensor_variable def constant(x, name=None, ndim=None, dtype=None): """Return a symbolic `Constant` with value `x`. Raises ------ TypeError `x` could not be converted to a numpy.ndarray. ValueError `x` could not be expanded to have ndim dimensions. Note ---- We create a small cache of frequently used constant. This speed up the Merge optimization for big graph. We want to cache all scalar to don't merge as frequently constants. But we don't want to cache too much stuff. So we cache integer with dtype [u]int and float where the value is between -10 and 10. We cache all broadcast pattern for scalar. """ x_ = scal.convert(x, dtype=dtype) bcastable = [d == 1 for d in x_.shape] if ndim is not None: if len(bcastable) < ndim: bcastable = [True] * (ndim - len(bcastable)) + bcastable elif len(bcastable) > ndim: # TODO: strip off dimensions of size 1 raise ValueError( 'ndarray could not be cast to constant with %i dimensions' % ndim) assert len(bcastable) == ndim try: ttype = TensorType(dtype=x_.dtype, broadcastable=bcastable) if not constant.enable: return TensorConstant(ttype, x_, name=name) sig = TensorConstantSignature((ttype, x_)) if sig in constant_cache: return constant_cache[sig] ret = TensorConstant(ttype, x_, name=name) if (x_.size == 1 and (-10) <= x_ <= 10 and (x_.dtype in int_dtypes or x_.dtype in uint_dtypes or (x_.dtype in float_dtypes and # Limit the size of the cache. len(constant_cache) < 10000))): constant_cache[sig] = ret # This is needed to raise a good error to the user. ret.cached = True return ret except Exception: raise TypeError("Could not convert %s to TensorType" % x, type(x)) constant.enable = True constant_cache = {} def _obj_is_wrappable_as_tensor(x): try: constant(x) return True except TypeError: return False if int(config.tensor.cmp_sloppy) > 1: # This config variable is a quick-and-dirty way to get low-precision # comparisons. For a more precise setting of these tolerances set # them explicitly in your user code by assigning, for example, # "theano.tensor.basic.float32_atol = ..." # When config.tensor.cmp_sloppy>1 we are even more sloppy. This is # useful to test the GPU as they don't use extended precision and # this cause some difference bigger then the normal sloppy. float16_atol = 1e-2 float16_rtol = 5e-2 float32_atol = 5e-4 float32_rtol = 1e-3 float64_rtol = 1e-4 float64_atol = 1e-3 elif int(config.tensor.cmp_sloppy): float16_atol = 5e-3 float16_rtol = 1e-2 float32_atol = 1e-4 float32_rtol = 1e-3 float64_rtol = 1e-4 float64_atol = 1e-3 else: # If you change those value in test don't forget to put them back # when the test end. Don't forget the case when the test fail. float16_atol = 1e-3 float16_rtol = 1e-3 float32_atol = 1e-5 float32_rtol = 1e-5 # defaults in numpy.allclose # Don't be more strict then numpy rtol # It cause useless error. float64_rtol = 1.0000000000000001e-05 float64_atol = 1e-8 def _get_atol_rtol(a, b): tiny = ('float16',) narrow = ('float32', 'complex64') if (str(a.dtype) in tiny) or (str(b.dtype) in tiny): atol = float16_atol rtol = float16_rtol elif (str(a.dtype) in narrow) or (str(b.dtype) in narrow): atol = float32_atol rtol = float32_rtol else: atol = float64_atol rtol = float64_rtol return atol, rtol def _allclose(a, b, rtol=None, atol=None): a = np.asarray(a) b = np.asarray(b) atol_, rtol_ = _get_atol_rtol(a, b) if rtol is not None: rtol_ = rtol if atol is not None: atol_ = atol return np.allclose(a, b, atol=atol_, rtol=rtol_) class NotScalarConstantError(Exception): """ Raised by get_scalar_constant_value if called on something that is not a scalar constant. """ class EmptyConstantError(NotScalarConstantError): """ Raised by get_scalar_const_value if called on something that is a zero dimensional constant. """ def numpy_scalar(data): """ Return a scalar stored in a numpy ndarray. Raises ------ NotScalarConstantError If the numpy ndarray is not a scalar. """ # handle case where data is numpy.array([]) if (data.ndim > 0 and (len(data.shape) == 0 or builtins.max(data.shape) == 0)): assert np.all(np.array([]) == data) raise EmptyConstantError() try: np.complex(data) # works for all numeric scalars return data except Exception: raise NotScalarConstantError( 'v.data is non-numeric, non-scalar, or has more than one' ' unique value', data) get_scalar_constant_value_elemwises = ( scal.Cast, scal.Switch, scal.NEQ, scal.EQ, scal.LT, scal.GT, scal.LE, scal.GE, scal.Sub, scal.Add, scal.Mod, scal.Mul, scal.IntDiv, scal.TrueDiv, scal.Minimum, scal.Maximum) def get_scalar_constant_value(orig_v, elemwise=True, only_process_constants=False, max_recur=10): """Return the constant scalar(0-D) value underlying variable `v`. If `v` is the output of dimshuffles, fills, allocs, rebroadcasts, cast, OutputGuard, DeepCopyOp, ScalarFromTensor, ScalarOp, Elemwise and some pattern with Subtensor, this function digs through them. If `v` is not some view of constant scalar data, then raise a NotScalarConstantError. Parameters ---------- elemwise : bool If False, we won't try to go into elemwise. So this call is faster. But we still investigate in Second Elemwise (as this is a substitute for Alloc) only_process_constants : bool If True, we only attempt to obtain the value of `orig_v` if it's directly constant and don't try to dig through dimshuffles, fills, allocs, and other to figure out its value. max_recur : int The maximum number of recursion. Notes ----- There may be another function similar to this one in the code, but I'm not sure where it is. """ v = orig_v while True: if v is None: # None is not a scalar (and many uses of this function seem # to depend on passing it None) raise NotScalarConstantError() if isinstance(v, (np.integer, integer_types, float)): return np.asarray(v) if isinstance(v, np.ndarray): return numpy_scalar(v).copy() if isinstance(v, Constant): if getattr(v.tag, 'unique_value', None) is not None: data = v.tag.unique_value else: data = v.data return numpy_scalar(data).copy() if (not only_process_constants and getattr(v, 'owner', None) and max_recur > 0): max_recur -= 1 if isinstance(v.owner.op, (Alloc, DimShuffle, Rebroadcast, # outputguard is only used in debugmode but we # keep it here to avoid problems with old pickels. compile.ops.OutputGuard, compile.DeepCopyOp)): v = v.owner.inputs[0] continue elif isinstance(v.owner.op, theano.compile.ops.Shape_i): i = v.owner.op.i inp = v.owner.inputs[0] if isinstance(inp, Constant): return np.asarray(inp.data.shape[i]) # The shape of a broadcastable dimension is 1 if (hasattr(inp.type, 'broadcastable') and inp.type.broadcastable[i]): return np.asarray(1) # Don't act as the constant_folding optimization here as this # fct is used too early in the optimization phase. This would # mess with the stabilization optimization and be too slow. # We put all the scalar Ops used by get_canonical_form_slice() # to allow it to determine the broadcast pattern correctly. elif isinstance(v.owner.op, (ScalarFromTensor, TensorFromScalar)): v = v.owner.inputs[0] continue elif isinstance(v.owner.op, theano.tensor.opt.Assert): # check if all conditions are constant and true cond = [get_scalar_constant_value(c, max_recur=max_recur) for c in v.owner.inputs[1:]] if builtins.all([0 == c.ndim and c != 0 for c in cond]): v = v.owner.inputs[0] continue elif isinstance(v.owner.op, scal.ScalarOp): if isinstance(v.owner.op, scal.Second): # We don't need both input to be constant for second shp, val = v.owner.inputs v = val continue if isinstance(v.owner.op, get_scalar_constant_value_elemwises): const = [get_scalar_constant_value(i, max_recur=max_recur) for i in v.owner.inputs] ret = [[None]] v.owner.op.perform(v.owner, const, ret) return ret[0][0].copy() # In fast_compile, we don't enable local_fill_to_alloc, so # we need to investigate Second as Alloc. So elemwise # don't disable the check for Second. elif isinstance(v.owner.op, Elemwise): if isinstance(v.owner.op.scalar_op, scal.Second): # We don't need both input to be constant for second shp, val = v.owner.inputs v = val continue elif elemwise and isinstance( v.owner.op.scalar_op, get_scalar_constant_value_elemwises): const = [get_scalar_constant_value(i, max_recur=max_recur) for i in v.owner.inputs] ret = [[None]] v.owner.op.perform(v.owner, const, ret) return ret[0][0].copy() elif (isinstance(v.owner.op, theano.tensor.subtensor.Subtensor) and v.ndim == 0): if isinstance(v.owner.inputs[0], TensorConstant): cdata = tuple(v.owner.op.get_constant_idx(v.owner.inputs)) try: return v.owner.inputs[0].data.__getitem__(cdata).copy() except IndexError: raise IndexError( str(tuple(v.owner.op.idx_list)) + " is not a valid index into " + str(v.owner.inputs[0].data)) # The index list 'idx_list' should have length the same # shape as the input. # TODO: implement the case where we take a scalar in a matrix assert len(v.owner.op.idx_list) == v.owner.inputs[0].ndim # Needed to make better graph in this test in # theano/tensor/tests/test_sharedvar.py: # test_shared_options.test_specify_shape_partial if ((v.owner.inputs[0].owner and isinstance(v.owner.inputs[0].owner.op, Join) and len(v.owner.op.idx_list) == 1)): # Ensure the Join is joining only scalar variables (so that # the constant value can be found at the same index as the # one used in the sub-tensor). if python_all(var.ndim == 0 for var in v.owner.inputs[0].owner.inputs[1:]): idx = v.owner.op.idx_list[0] if isinstance(idx, gof.Type): idx = get_scalar_constant_value(v.owner.inputs[1], max_recur=max_recur) # Note the '+ 1' is because the first argument to Join # is the axis. ret = v.owner.inputs[0].owner.inputs[idx + 1] ret = get_scalar_constant_value(ret, max_recur=max_recur) # join can cast implicitly its input in some case. return theano._asarray(ret, dtype=v.type.dtype) if python_all(var.ndim == 1 for var in v.owner.inputs[0].owner.inputs[1:]): idx = v.owner.op.idx_list[0] if isinstance(idx, gof.Type): idx = get_scalar_constant_value(v.owner.inputs[1], max_recur=max_recur) try: # TODO: assert joined axis is 0. length = 0 loop = False for joined in v.owner.inputs[0].owner.inputs[1:]: ll = get_vector_length(joined) if idx < length + ll: v = joined[idx - length] loop = True break length += ll if loop: continue except TypeError: pass except ValueError: pass elif (v.owner.inputs[0].owner and isinstance(v.owner.inputs[0].owner.op, theano.tensor.opt.MakeVector) and # MakeVector normally accept only scalar as input. # We put this check in case there is change in the future python_all(var.ndim == 0 for var in v.owner.inputs[0].owner.inputs) and len(v.owner.op.idx_list) == 1): idx = v.owner.op.idx_list[0] if isinstance(idx, gof.Type): idx = get_scalar_constant_value(v.owner.inputs[1], max_recur=max_recur) # Python 2.4 does not support indexing with numpy.integer # So we cast it. idx = int(idx) ret = v.owner.inputs[0].owner.inputs[idx] ret = get_scalar_constant_value(ret, max_recur=max_recur) # MakeVector can cast implicitly its input in some case. return theano._asarray(ret, dtype=v.type.dtype) # This is needed when we take the grad as the Shape op # are not already changed into MakeVector owner = v.owner leftmost_parent = owner.inputs[0] if (leftmost_parent.owner and isinstance(leftmost_parent.owner.op, theano.tensor.Shape)): op = owner.op idx_list = op.idx_list idx = idx_list[0] if isinstance(idx, gof.Type): idx = get_scalar_constant_value(owner.inputs[1], max_recur=max_recur) grandparent = leftmost_parent.owner.inputs[0] gp_broadcastable = grandparent.type.broadcastable ndim = grandparent.type.ndim if grandparent.owner and isinstance(grandparent.owner.op, Rebroadcast): ggp_broadcastable = grandparent.owner.inputs[0].broadcastable l = [b1 or b2 for b1, b2 in zip(ggp_broadcastable, gp_broadcastable)] gp_broadcastable = tuple(l) assert ndim == len(gp_broadcastable) if not (idx < len(gp_broadcastable)): msg = ("get_scalar_constant_value detected " + "deterministic IndexError: x.shape[%d] " + "when x.ndim=%d.") % (idx, ndim) if config.exception_verbosity == 'high': msg += ' x=%s' % min_informative_str(v) else: msg += ' x=%s' % str(v) raise ValueError(msg) if gp_broadcastable[idx]: return np.asarray(1) raise NotScalarConstantError(v) # Easy constructors def tensor(*args, **kwargs): name = kwargs.pop('name', None) return TensorType(*args, **kwargs)(name=name) def _multi(*fns): def f2(f, *names): if names and isinstance(names[0], integer_types): if names == 1: return f() else: return [f() for i in xrange(names[0])] if isinstance(names, tuple): if len(names) == 1: names = names[0] if len(names) == 1: return f(names) else: return [f(name) for name in names] if len(fns) == 1: return partial(f2, fns) else: return [partial(f2, f) for f in fns] cscalar = TensorType('complex64', ()) zscalar = TensorType('complex128', ()) fscalar = TensorType('float32', ()) dscalar = TensorType('float64', ()) bscalar = TensorType('int8', ()) wscalar = TensorType('int16', ()) iscalar = TensorType('int32', ()) lscalar = TensorType('int64', ()) def scalar(name=None, dtype=None): """Return a symbolic scalar variable. Parameters ---------- dtype: numeric None means to use theano.config.floatX. name A name to attach to this variable. """ if dtype is None: dtype = config.floatX type = TensorType(dtype, ()) return type(name) scalars, fscalars, dscalars, iscalars, lscalars = _multi( scalar, fscalar, dscalar, iscalar, lscalar) int_types = bscalar, wscalar, iscalar, lscalar float_types = fscalar, dscalar complex_types = cscalar, zscalar int_scalar_types = int_types float_scalar_types = float_types complex_scalar_types = complex_types cvector = TensorType('complex64', (False, )) zvector = TensorType('complex128', (False, )) fvector = TensorType('float32', (False, )) dvector = TensorType('float64', (False, )) bvector = TensorType('int8', (False,)) wvector = TensorType('int16', (False,)) ivector = TensorType('int32', (False, )) lvector = TensorType('int64', (False, )) def vector(name=None, dtype=None): """Return a symbolic vector variable. Parameters ---------- dtype: numeric None means to use theano.config.floatX. name A name to attach to this variable """ if dtype is None: dtype = config.floatX type = TensorType(dtype, (False, )) return type(name) vectors, fvectors, dvectors, ivectors, lvectors = _multi( vector, fvector, dvector, ivector, lvector) int_vector_types = bvector, wvector, ivector, lvector float_vector_types = fvector, dvector complex_vector_types = cvector, zvector cmatrix = TensorType('complex64', (False, False)) zmatrix = TensorType('complex128', (False, False)) fmatrix = TensorType('float32', (False, False)) dmatrix = TensorType('float64', (False, False)) bmatrix = TensorType('int8', (False, False)) wmatrix = TensorType('int16', (False, False)) imatrix = TensorType('int32', (False, False)) lmatrix = TensorType('int64', (False, False)) def matrix(name=None, dtype=None): """Return a symbolic matrix variable. Parameters ---------- dtype: numeric None means to use theano.config.floatX. name A name to attach to this variable. """ if dtype is None: dtype = config.floatX type = TensorType(dtype, (False, False)) return type(name) matrices, fmatrices, dmatrices, imatrices, lmatrices = _multi( matrix, fmatrix, dmatrix, imatrix, lmatrix) int_matrix_types = bmatrix, wmatrix, imatrix, lmatrix float_matrix_types = fmatrix, dmatrix complex_matrix_types = cmatrix, zmatrix crow = TensorType('complex64', (True, False)) zrow = TensorType('complex128', (True, False)) frow = TensorType('float32', (True, False)) drow = TensorType('float64', (True, False)) brow = TensorType('int8', (True, False)) wrow = TensorType('int16', (True, False)) irow = TensorType('int32', (True, False)) lrow = TensorType('int64', (True, False)) def row(name=None, dtype=None): """Return a symbolic row variable (ndim=2, broadcastable=[True,False]). Parameters ---------- dtype: numeric type None means to use theano.config.floatX. name A name to attach to this variable. """ if dtype is None: dtype = config.floatX type = TensorType(dtype, (True, False)) return type(name) rows, frows, drows, irows, lrows = _multi(row, frow, drow, irow, lrow) ccol = TensorType('complex64', (False, True)) zcol = TensorType('complex128', (False, True)) fcol = TensorType('float32', (False, True)) dcol = TensorType('float64', (False, True)) bcol = TensorType('int8', (False, True)) wcol = TensorType('int16', (False, True)) icol = TensorType('int32', (False, True)) lcol = TensorType('int64', (False, True)) def col(name=None, dtype=None): """Return a symbolic column variable (ndim=2, broadcastable=[False,True]). Parameters ---------- dtype : numeric None means to use theano.config.floatX. name A name to attach to this variable. """ if dtype is None: dtype = config.floatX type = TensorType(dtype, (False, True)) return type(name) cols, fcols, dcols, icols, lcols = _multi(col, fcol, dcol, icol, lcol) ctensor3 = TensorType('complex64', ((False,) * 3)) ztensor3 = TensorType('complex128', ((False,) * 3)) ftensor3 = TensorType('float32', ((False,) * 3)) dtensor3 = TensorType('float64', ((False,) * 3)) btensor3 = TensorType('int8', ((False,) * 3)) wtensor3 = TensorType('int16', ((False,) * 3)) itensor3 = TensorType('int32', ((False,) * 3)) ltensor3 = TensorType('int64', ((False,) * 3)) def tensor3(name=None, dtype=None): """Return a symbolic 3-D variable. Parameters ---------- dtype: numeric type None means to use theano.config.floatX. name A name to attach to this variable. """ if dtype is None: dtype = config.floatX type = TensorType(dtype, (False, False, False)) return type(name) tensor3s, ftensor3s, dtensor3s, itensor3s, ltensor3s = _multi( tensor3, ftensor3, dtensor3, itensor3, ltensor3) ctensor4 = TensorType('complex64', ((False,) * 4)) ztensor4 = TensorType('complex128', ((False,) * 4)) ftensor4 = TensorType('float32', ((False,) * 4)) dtensor4 = TensorType('float64', ((False,) * 4)) btensor4 = TensorType('int8', ((False,) * 4)) wtensor4 = TensorType('int16', ((False,) * 4)) itensor4 = TensorType('int32', ((False,) * 4)) ltensor4 = TensorType('int64', ((False,) * 4)) def tensor4(name=None, dtype=None): """Return a symbolic 4-D variable. Parameters ---------- dtype: numeric type None means to use theano.config.floatX. name A name to attach to this variable. """ if dtype is None: dtype = config.floatX type = TensorType(dtype, (False, False, False, False)) return type(name) tensor4s, ftensor4s, dtensor4s, itensor4s, ltensor4s = _multi( tensor4, ftensor4, dtensor4, itensor4, ltensor4) ctensor5 = TensorType('complex64', ((False,) * 5)) ztensor5 = TensorType('complex128', ((False,) * 5)) ftensor5 = TensorType('float32', ((False,) * 5)) dtensor5 = TensorType('float64', ((False,) * 5)) btensor5 = TensorType('int8', ((False,) * 5)) wtensor5 = TensorType('int16', ((False,) * 5)) itensor5 = TensorType('int32', ((False,) * 5)) ltensor5 = TensorType('int64', ((False,) * 5)) def tensor5(name=None, dtype=None): """Return a symbolic 5-D variable. Parameters ---------- dtype: numeric type None means to use theano.config.floatX. name A name to attach to this variable. """ if dtype is None: dtype = config.floatX type = TensorType(dtype, (False, False, False, False, False)) return type(name) tensor5s, ftensor5s, dtensor5s, itensor5s, ltensor5s = _multi( tensor5, ftensor5, dtensor5, itensor5, ltensor5) ctensor6 = TensorType('complex64', ((False,) * 6)) ztensor6 = TensorType('complex128', ((False,) * 6)) ftensor6 = TensorType('float32', ((False,) * 6)) dtensor6 = TensorType('float64', ((False,) * 6)) btensor6 = TensorType('int8', ((False,) * 6)) wtensor6 = TensorType('int16', ((False,) * 6)) itensor6 = TensorType('int32', ((False,) * 6)) ltensor6 = TensorType('int64', ((False,) * 6)) def tensor6(name=None, dtype=None): """Return a symbolic 6-D variable. Parameters ---------- dtype: numeric type None means to use theano.config.floatX. name A name to attach to this variable. """ if dtype is None: dtype = config.floatX type = TensorType(dtype, (False,) * 6) return type(name) tensor6s, ftensor6s, dtensor6s, itensor6s, ltensor6s = _multi( tensor6, ftensor6, dtensor6, itensor6, ltensor6) ctensor7 = TensorType('complex64', ((False,) * 7)) ztensor7 = TensorType('complex128', ((False,) * 7)) ftensor7 = TensorType('float32', ((False,) * 7)) dtensor7 = TensorType('float64', ((False,) * 7)) btensor7 = TensorType('int8', ((False,) * 7)) wtensor7 = TensorType('int16', ((False,) * 7)) itensor7 = TensorType('int32', ((False,) * 7)) ltensor7 = TensorType('int64', ((False,) * 7)) def tensor7(name=None, dtype=None): """Return a symbolic 7-D variable. Parameters ---------- dtype: numeric type None means to use theano.config.floatX. name A name to attach to this variable. """ if dtype is None: dtype = config.floatX type = TensorType(dtype, (False,) * 7) return type(name) tensor7s, ftensor7s, dtensor7s, itensor7s, ltensor7s = _multi( tensor7, ftensor7, dtensor7, itensor7, ltensor7) Tensor = TensorType # This bizarre push-import avoids a circular dependency. elemwise.as_tensor_variable = as_tensor_variable elemwise.TensorType = TensorType elemwise.TensorVariable = TensorVariable elemwise.TensorConstant = TensorConstant ######################### # Utilities ######################### def _scal_elemwise_with_nfunc(nfunc, nin, nout): """ Replace a symbol definition with an elementwise version of the corresponding scalar Op. If it is not None, the nfunc argument should be a string such that getattr(numpy, nfunc) implements a vectorized version of the elemwise operation. nin is the number of inputs expected by that function, and nout is the number of **destination** inputs it takes. That is, the function should take nin+nout inputs. nout == 0 means that the numpy function does not take a numpy array argument to put its result in. """ def construct(symbol): symbolname = symbol.__name__ inplace = symbolname.endswith('_inplace') if inplace: msg = "inplace" else: msg = "no_inplace" n = "Elemwise{%s,%s}" % (symbolname, msg) if inplace: scalar_op = getattr(scal, symbolname[:-len('_inplace')]) inplace_scalar_op = scalar_op.__class__(scal.transfer_type(0)) rval = elemwise.Elemwise(inplace_scalar_op, {0: 0}, name=n, nfunc_spec=(nfunc and (nfunc, nin, nout))) else: scalar_op = getattr(scal, symbolname) rval = elemwise.Elemwise(scalar_op, name=n, nfunc_spec=(nfunc and (nfunc, nin, nout))) if getattr(symbol, '__doc__', False): rval.__doc__ = symbol.__doc__ + '\n' + rval.__doc__ # for the meaning of this see the ./epydoc script # it makes epydoc display rval as if it were a function, not an object rval.__epydoc_asRoutine = symbol rval.__module__ = 'tensor' pprint.assign(rval, printing.FunctionPrinter(symbolname)) return rval return construct _scal_elemwise = _scal_elemwise_with_nfunc(None, None, None) def _pack(x): """ Convert x to a list if it is an iterable, otherwise wrap it in a list. """ try: return list(x) except TypeError: return [x] def check_and_normalize_axes(x, axis): """ Check axes, normalize and convert them to a Python list of integers. Return an empty list if argument is None. Parameters ---------- x: Tensor variable axis = Integer, tuple or list of integers Returns ------- axis: list of integers """ x = as_tensor_variable(x) if axis is None: axis = [] elif (isinstance(axis, (integer_types, np.integer)) or (isinstance(axis, np.ndarray) and axis.ndim == 0)): axis = [int(axis)] elif isinstance(axis, (tuple, list, np.ndarray)): axis = [int(i) for i in axis] elif isinstance(axis, Variable): if NoneConst.equals(axis): axis = [] elif not isinstance(axis, TensorConstant): raise TypeError("Computation needs a constant axis. Got %s" % axis) else: assert axis.dtype in integer_dtypes if (isinstance(axis.data, (integer_types, np.integer)) or (isinstance(axis.data, np.ndarray) and axis.data.ndim == 0)): axis = [int(axis.data)] elif isinstance(axis.data, (list, np.ndarray)): axis = [int(i) for i in axis.data] else: raise TypeError("Axis must be an integer, tuple, list of integers or a TensorVariable. Got %s" % axis) if len(axis) > 0: for i in range(len(axis)): if axis[i] < 0: axis[i] += x.type.ndim if axis[i] < 0 or axis[i] >= x.type.ndim: raise ValueError("Computation needs a valid axis number for %d-D tensor. Got %d" % (x.type.ndim, axis[i])) axis = list(set(axis)) axis.sort() return axis ######################### # Casting Operations ######################### class TensorFromScalar(Op): __props__ = () def make_node(self, s): assert isinstance(s.type, scal.Scalar) return Apply(self, [s], [tensor(dtype=s.type.dtype, broadcastable=())]) def perform(self, node, inp, out_): s, = inp out, = out_ out[0] = np.asarray(s) def infer_shape(self, node, in_shapes): return [()] def grad(self, inp, grads): s, = inp dt, = grads if s.type.dtype in float_dtypes: assert dt.type.dtype in float_dtypes return [scalar_from_tensor(dt)] # If the input dtype is an integer, then so is the output dtype, # and the "zero" gradient can be represented in that int dtype. # Currently, theano.grad insists that the dtype of the returned # gradient has a float dtype, so we use floatX. if s.type.dtype in discrete_dtypes: return [s.zeros_like().astype(theano.config.floatX)] raise NotImplementedError("grad not implemented for complex dtypes") tensor_from_scalar = TensorFromScalar() class ScalarFromTensor(Op): __props__ = () def make_node(self, t): assert isinstance(t.type, TensorType) assert t.type.broadcastable == () return Apply(self, [t], [scal.get_scalar_type(dtype=t.type.dtype).make_variable()] ) def perform(self, node, inp, out_): s, = inp out, = out_ out[0] = s.flatten()[0] def infer_shape(self, node, in_shapes): return [()] def grad(self, inp, grads): s, = inp dt, = grads return [tensor_from_scalar(dt)] def R_op(self, inputs, eval_points): if None in eval_points: return [None] return self.make_node(*eval_points).outputs def c_code(self, node, name, inputs, outputs, sub): x, = inputs z, = outputs fail = sub['fail'] return """ %(z)s = ((dtype_%(x)s*)(PyArray_DATA(%(x)s)))[0]; """ % locals() def c_code_cache_version(self): return (1,) scalar_from_tensor = ScalarFromTensor() # to be removed as we get the epydoc routine-documenting thing going # -JB 20080924 def _conversion(real_value, name): __oplist_tag(real_value, 'casting') real_value.__module__ = 'tensor.basic' pprint.assign(real_value, printing.FunctionPrinter(name)) return real_value # These _conver_to_ functions have leading underscores to indicate that # they should not be called directly. They do not perform sanity checks about # what types you are casting to what. That logic is implemented by the # `cast()` function below. _convert_to_bool = _conversion( elemwise.Elemwise(scal.convert_to_bool), 'bool') """Cast to boolean""" _convert_to_int8 = _conversion( elemwise.Elemwise(scal.convert_to_int8), 'int8') """Cast to 8-bit integer""" _convert_to_int16 = _conversion( elemwise.Elemwise(scal.convert_to_int16), 'int16') """Cast to 16-bit integer""" _convert_to_int32 = _conversion( elemwise.Elemwise(scal.convert_to_int32), 'int32') """Cast to 32-bit integer""" _convert_to_int64 = _conversion( elemwise.Elemwise(scal.convert_to_int64), 'int64') """Cast to 64-bit integer""" _convert_to_uint8 = _conversion( elemwise.Elemwise(scal.convert_to_uint8), 'uint8') """Cast to unsigned 8-bit integer""" _convert_to_uint16 = _conversion( elemwise.Elemwise(scal.convert_to_uint16), 'uint16') """Cast to unsigned 16-bit integer""" _convert_to_uint32 = _conversion( elemwise.Elemwise(scal.convert_to_uint32), 'uint32') """Cast to unsigned 32-bit integer""" _convert_to_uint64 = _conversion( elemwise.Elemwise(scal.convert_to_uint64), 'uint64') """Cast to unsigned 64-bit integer""" _convert_to_float16 = _conversion( elemwise.Elemwise(scal.convert_to_float16), 'float16') """Cast to half-precision floating point""" _convert_to_float32 = _conversion( elemwise.Elemwise(scal.convert_to_float32), 'float32') """Cast to single-precision floating point""" _convert_to_float64 = _conversion( elemwise.Elemwise(scal.convert_to_float64), 'float64') """Cast to double-precision floating point""" _convert_to_complex64 = _conversion( elemwise.Elemwise(scal.convert_to_complex64), 'complex64') """Cast to single-precision complex""" _convert_to_complex128 = _conversion( elemwise.Elemwise(scal.convert_to_complex128), 'complex128') """Cast to double-precision complex""" _cast_mapping = { 'bool': _convert_to_bool, 'int8': _convert_to_int8, 'int16': _convert_to_int16, 'int32': _convert_to_int32, 'int64': _convert_to_int64, 'uint8': _convert_to_uint8, 'uint16': _convert_to_uint16, 'uint32': _convert_to_uint32, 'uint64': _convert_to_uint64, 'float16': _convert_to_float16, 'float32': _convert_to_float32, 'float64': _convert_to_float64, 'complex64': _convert_to_complex64, 'complex128': _convert_to_complex128} @constructor def cast(x, dtype): """Symbolically cast `x` to a Tensor of type `dtype`.""" if dtype == 'floatX': dtype = config.floatX _x = as_tensor_variable(x) if _x.type.dtype == dtype: return _x if _x.type.dtype.startswith('complex') and not dtype.startswith('complex'): raise TypeError(( 'Casting from complex to real is ambiguous: consider real(), ' 'imag(), angle() or abs()')) return _cast_mapping[dtype](x) ########################## # Unary Operations ########################## class MaxAndArgmax(Op): """ Calculate the max and argmax over a given axis or over all axes. """ nin = 2 # tensor, axis nout = 2 # max val, max idx E_axis = 'invalid axis' params_type = Generic() __props__ = ('axis',) _f16_ok = True def __init__(self, axis): assert isinstance(axis, list) self.axis = tuple(axis) def get_params(self, node): return self.axis def make_node(self, x): x = _as_tensor_variable(x) # We keep the original broadcastable flags for dimensions on which # we do not perform the max / argmax. all_axes = set(self.axis) broadcastable = [b for i, b in enumerate(x.type.broadcastable) if i not in all_axes] inputs = [x] outputs = [tensor(x.type.dtype, broadcastable, name='max'), tensor('int64', broadcastable, name='argmax')] return Apply(self, inputs, outputs) def perform(self, node, inp, outs, params): x = inp[0] axes = params max, max_idx = outs if axes is None: axes = tuple(range(x.ndim)) else: axes = tuple(int(ax) for ax in axes) max[0] = theano._asarray(np.max(x, axes), dtype=node.outputs[0].dtype) # Numpy does not support multiple axes for argmax # Work around keep_axes = np.array([i for i in range(x.ndim) if i not in axes], dtype='int64') # Not-reduced axes in front transposed_x = np.transpose(x, np.concatenate((keep_axes, axes))) kept_shape = transposed_x.shape[:len(keep_axes)] reduced_shape = transposed_x.shape[len(keep_axes):] # Numpy.prod returns 1.0 when arg is empty, so we cast it to int64 # Otherwise reshape would complain citing float arg new_shape = kept_shape + (np.prod(reduced_shape, dtype='int64'),) reshaped_x = transposed_x.reshape(new_shape) max_idx[0] = theano._asarray(np.argmax(reshaped_x, axis=-1), dtype='int64') def c_code(self, node, name, inp, out, sub): if len(self.axis) != 1 and len(self.axis) != node.inputs[0].ndim: raise NotImplementedError("NumPy C-API can compute max and argmax only for 1 axis or for all axes.") x = inp[0] axis = sub['params'] max, argmax = out fail = sub["fail"] ret = """ #if PY_MAJOR_VERSION >= 3 #ifndef PyInt_AS_LONG #define PyInt_AS_LONG PyLong_AS_LONG #endif #endif int axis; if (PyTuple_GET_SIZE(%(axis)s) == PyArray_NDIM(%(x)s)) { axis = NPY_MAXDIMS; } else if(PyTuple_GET_SIZE(%(axis)s) == 1) { PyObject* axis_object = PyTuple_GET_ITEM(%(axis)s, 0); axis = (int)PyInt_AS_LONG(axis_object); if (axis > PyArray_NDIM(%(x)s)-1 || axis < -PyArray_NDIM(%(x)s)) { PyErr_SetString(PyExc_ValueError, "MaxAndArgmax: bad axis argument"); %(fail)s } } else { PyErr_SetString(PyExc_NotImplementedError, "MaxAndArgmax: NumPy C-API can compute max and argmax only for 1 axis or for all axes."); %(fail)s } Py_CLEAR(%(max)s); Py_CLEAR(%(argmax)s);//todo pass them as out parameter. %(max)s = (PyArrayObject*)PyArray_Max(%(x)s, axis, NULL); if (%(max)s == NULL) { %(fail)s; } if (!PyArray_CheckExact(%(max)s)) { %(max)s = (PyArrayObject*)PyArray_FromAny((PyObject*)%(max)s, NULL, 0, 0, NPY_ARRAY_ENSUREARRAY, NULL); if(%(max)s == NULL){ %(fail)s; } } %(argmax)s = (PyArrayObject*)PyArray_ArgMax(%(x)s, axis, NULL); if (%(argmax)s == NULL) { Py_CLEAR(%(max)s); %(fail)s; } if (!PyArray_CheckExact(%(argmax)s)) { %(argmax)s = (PyArrayObject*)PyArray_FromAny((PyObject*)%(argmax)s, NULL, 0, 0, NPY_ARRAY_ENSUREARRAY, NULL); if(%(argmax)s == NULL){ %(fail)s; } } if (PyArray_TYPE(%(argmax)s) != NPY_INT64) { PyObject * tmp = PyArray_Cast(%(argmax)s, NPY_INT64); if (NULL == tmp){ %(fail)s; } Py_DECREF(%(argmax)s); %(argmax)s = (PyArrayObject*)tmp; } """ return ret % locals() def c_code_cache_version(self): return (5,) def infer_shape(self, node, shapes): ishape = shapes[0] rval = tuple(ishape[i] for (i, b) in enumerate( node.inputs[0].type.broadcastable) if i not in self.axis) return [rval, rval] def R_op(self, inputs, eval_points): if eval_points[0] is None: return [None, None] if len(self.axis) != 1: raise ValueError(('R_op supported for arg_max only for ' 'one axis!')) if self.axis[0] > 1: raise ValueError(('R_op supported for arg_max only when ' ' axis is 0 or 1')) if inputs[0].ndim != 2: raise ValueError(('R_op supported for arg_max only when ' ' input is a matrix')) max_vals, max_pos = self.make_node(*inputs).outputs if self.axis[0] == 0: return [eval_points[0][max_pos, arange(eval_points[0].shape[1])], None] else: return [eval_points[0][arange(eval_points[0].shape[0]), max_pos], None] def grad(self, inp, grads): # The strict sense mathematical gradient of the maximum function is # not calculated here for it is not defined at every point where some # coordinates are identical. However, since the latter set has null # Lebesgue measure, the result may be interpreted as weak gradient. # @note: This function should work correctly for L{vector}s. # (x, y), (gz, gw) # gz*dz/dx + gw*dw/dx, gz*dz/dy + gw*dw/dy # gMax * dMax/dx + gArgMax * dArgMax/dx, # gMax * dMax/daxis + gArgMax * dArgMax/daxis # g_max has one less dimension than x, so you need to complete # g_max to x's shape when axis=0 the broadcasting mechanism # does it automatically x = inp[0] axis = _as_tensor_variable(self.axis) g_max, g_max_idx = grads g_max_disconnected = isinstance(g_max.type, DisconnectedType) g_max_idx_disconnected = isinstance(g_max_idx.type, DisconnectedType) # if the op is totally disconnected, so are its inputs if g_max_disconnected and g_max_idx_disconnected: return [DisconnectedType()(), DisconnectedType()()] # if the max is disconnected but the argmax is not, # the gradient on its inputs is zero if g_max_disconnected: return [x.zeros_like()] if NoneConst.equals(axis): axis_ = list(range(x.ndim)) else: axis_ = axis xmax = max(x, axis_) # Raise the g_max and xmax to the same number of dim as the input. pattern = [] out_dim = 0 if NoneConst.equals(axis): # We are taking the max/argmax over all dimensions. axis = None for i in xrange(x.ndim): if axis is None or i in axis.data: pattern.append('x') else: pattern.append(out_dim) out_dim += 1 g_max_pad = DimShuffle(g_max.broadcastable, pattern)(g_max) xmax_pad = DimShuffle(xmax.broadcastable, pattern)(xmax) # Set the grad to the correct position. g_x = eq(xmax_pad, x) * g_max_pad return g_x, class Argmax(Op): """ Calculate the argmax over a given axis or over all axes. """ nin = 2 # tensor, axis nout = 1 E_axis = 'invalid axis' __props__ = ('axis',) _f16_ok = True params_type = ParamsType(c_axis=scal.int64) def __init__(self, axis): if axis is not None: axis = tuple(axis) self.axis = tuple(axis) def get_params(self, node): if self.axis is not None and len(self.axis) == 1: c_axis = np.int64(self.axis[0]) else: # The value here doesn't matter, it won't be used c_axis = np.int64(-1) return self.params_type.get_params(c_axis=c_axis) def make_node(self, x, axis=None): x = _as_tensor_variable(x) if self.axis is None: all_axes = list(range(x.ndim)) else: all_axes = self.axis inputs = [x] # We keep the original broadcastable flags for dimensions on which # we do not perform the argmax. broadcastable = [b for i, b in enumerate(x.type.broadcastable) if i not in all_axes] outputs = [tensor('int64', broadcastable, name='argmax')] return Apply(self, inputs, outputs) def prepare_node(self, node, storage_map, compute_map, impl): if len(node.inputs) == 2: raise ValueError('You are trying to compile a graph with an old Argmax node. Either reoptimize your graph or rebuild it to get the new node format.') def perform(self, node, inp, outs, params): x, = inp axes = self.axis max_idx, = outs if axes is None: axes = tuple(range(x.ndim)) # Numpy does not support multiple axes for argmax # Work around keep_axes = np.array([i for i in range(x.ndim) if i not in axes], dtype='int64') # Not-reduced axes in front transposed_x = np.transpose(x, np.concatenate((keep_axes, axes))) kept_shape = transposed_x.shape[:len(keep_axes)] reduced_shape = transposed_x.shape[len(keep_axes):] new_shape = kept_shape + (np.prod(reduced_shape),) reshaped_x = transposed_x.reshape(new_shape) max_idx[0] = theano._asarray(np.argmax(reshaped_x, axis=-1), dtype='int64') def c_code(self, node, name, inp, out, sub): x, = inp argmax, = out fail = sub["fail"] params = sub["params"] if self.axis is None: axis_code = "axis = NPY_MAXDIMS;" else: if len(self.axis) > 1: raise NotImplementedError() # params is only used here for now axis_code = """ axis = %(params)s->c_axis; if(axis > PyArray_NDIM(%(x)s)-1 || axis < -PyArray_NDIM(%(x)s)){ PyErr_SetString(PyExc_ValueError, "Argmax, bad axis argument"); %(fail)s } """ % locals() ret = """ int axis; Py_CLEAR(%(argmax)s);//todo pass them as out parameter. %(axis_code)s %(argmax)s = (PyArrayObject*)PyArray_ArgMax(%(x)s, axis, NULL); if(%(argmax)s == NULL){ %(fail)s; } if(!PyArray_CheckExact(%(argmax)s)){ %(argmax)s = (PyArrayObject*)PyArray_FromAny((PyObject*)%(argmax)s, NULL, 0, 0, NPY_ARRAY_ENSUREARRAY, NULL); if(%(argmax)s == NULL){ %(fail)s; } } if(PyArray_TYPE(%(argmax)s) != NPY_INT64){ PyObject * tmp = PyArray_Cast(%(argmax)s, NPY_INT64); if (NULL == tmp){ %(fail)s; } Py_DECREF(%(argmax)s); %(argmax)s = (PyArrayObject*)tmp; } """ return ret % locals() def c_code_cache_version(self): return (1,) def infer_shape(self, node, shapes): ishape, = shapes if self.axis is None: return [()] rval = tuple([ishape[i] for (i, b) in enumerate( node.inputs[0].type.broadcastable) if i not in self.axis]) return [rval] def grad(self, inp, grads): x, = inp return [x.zeros_like()] def makeKeepDims(x, y, axis): """ Reintroduces in y with length one the axes of x which have been left out in a prior reduction of x. With this option, the resulting tensor will broadcast correctly against the original tensor x. """ x = as_tensor_variable(x) y = as_tensor_variable(y) if axis is None: axis = list(range(x.type.ndim)) elif isinstance(axis, (integer_types, np.integer)): axis = [axis] elif isinstance(axis, np.ndarray) and axis.ndim == 0: axis = [int(axis)] else: axis = [int(a) for a in axis] newaxis = [] for a in axis: if not isinstance(a, integer_types): raise ValueError( "keepdims option can be used only with constant axis") if a < 0: a += x.type.ndim newaxis.append(a) i = 0 new_dims = [] for j, _ in enumerate(x.type.broadcastable): if j in newaxis: new_dims.append('x') else: new_dims.append(i) i += 1 return DimShuffle(y.type.broadcastable, new_dims)(y) @constructor def max_and_argmax(a, axis=None, keepdims=False): """ Returns maximum elements and their indices obtained by iterating over given axis. When axis is None (the default value), the max is performed over the flattened tensor. Parameters ---------- keepdims : bool If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, the result will broadcast correctly against the original tensor. """ # Check axis and convert it to a Python list of integers. # Axis will be used as an op param of MaxAndArgmax. a = as_tensor_variable(a) axis = check_and_normalize_axes(a, axis) if len(axis) == 0: axis = list(range(a.type.ndim)) out, argout = MaxAndArgmax(axis)(a) if keepdims: out = makeKeepDims(a, out, axis) argout = makeKeepDims(a, argout, axis) return [out, argout] @constructor def max(x, axis=None, keepdims=False): """ Returns maximum elements obtained by iterating over given axis. When axis is None (the default value), the max is performed over the flattened tensor. Parameters ---------- keepdims: bool If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, the result will broadcast correctly against the original tensor. Notes ----- We return an error as numpy when we reduce a dim with a shape of 0. """ # We have a choice of implementing this call with the # CAReduce op or the MaxAndArgmax op. # MaxAndArgmax supports grad and Rop, so we prefer to use that. # CAReduce is faster, but optimizations will replace MaxAndArgmax[0] # with CAReduce at compile time, so at this stage the important # thing is supporting all user interface features, not speed. # Some cases can be implemented only with CAReduce. # We thus prefer to use MaxAndArgmax, if possible. It does not # support all axis arguments, so we may need to fall back to CAReduce. try: out = max_and_argmax(x, axis)[0] except Exception: out = CAReduce(scal.maximum, axis)(x) if keepdims: out = makeKeepDims(x, out, axis) return out @constructor def argmax(x, axis=None, keepdims=False): """ Returns indices of maximum elements obtained by iterating over given axis. When axis is None (the default value), the argmax is performed over the flattened tensor. Parameters ---------- keepdims : bool If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, the result will broadcast correctly against the original tensor. """ argout = max_and_argmax(x, axis)[1] if keepdims: argout = makeKeepDims(x, argout, axis) return argout @constructor def min(x, axis=None, keepdims=False): """ Returns minimum elements obtained by iterating over given axis. When axis is None (the default value), the min is performed over the flattened tensor. Parameters ---------- keepdims: bool If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, the result will broadcast correctly against the original tensor. """ x = as_tensor_variable(x) str_x_type = str(x.dtype) if str_x_type.startswith('float') or str_x_type in int_dtypes: return -max(-x, axis=axis, keepdims=keepdims) elif str_x_type in uint_dtypes: itype = np.iinfo(x.dtype) max_val = np.array(itype.max, dtype=itype.dtype) return max_val - max(max_val - x, axis=axis, keepdims=keepdims) elif str_x_type == 'bool': return ~max(~x, axis=axis, keepdims=keepdims) else: # Be careful about unsigned integers, complex raise NotImplementedError() @constructor def argmin(x, axis=None, keepdims=False): """ Returns indices of minimum elements obtained by iterating over given axis. When axis is None (the default value), the argmin is performed over the flattened tensor. Parameters ---------- keepdims: bool If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, the result will broadcast correctly against the original tensor. """ x = as_tensor_variable(x) str_x_type = str(x.dtype) if str_x_type.startswith('float') or str_x_type in int_dtypes: return argmax(-x, axis=axis, keepdims=keepdims) elif str_x_type in uint_dtypes: itype = np.iinfo(x.dtype) return argmax(itype.max - x, axis=axis, keepdims=keepdims) elif str_x_type == 'bool': return argmax(~x, axis=axis, keepdims=keepdims) else: # Be careful about unsigned integers, complex raise NotImplementedError() @constructor def smallest(*args): """ Return the [elementwise] smallest of a variable number of arguments. Like python's min. """ if len(args) == 2: a, b = args return switch(a < b, a, b) else: return min(stack(args), axis=0) @constructor def largest(*args): """ Return the [elementwise] largest of a variable number of arguments. Like python's max. """ if len(args) == 2: a, b = args return switch(a > b, a, b) else: return max(stack(args), axis=0) ########################## # Comparison ########################## @_scal_elemwise def lt(a, b): """a < b""" @_scal_elemwise def gt(a, b): """a > b""" @_scal_elemwise def le(a, b): """a <= b""" @_scal_elemwise def ge(a, b): """a >= b""" @_scal_elemwise def eq(a, b): """a == b""" @_scal_elemwise def neq(a, b): """a != b""" @_scal_elemwise def isnan(a): """isnan(a)""" # Rename isnan to isnan_ to allow to bypass it when not needed. # glibc 2.23 don't allow isnan on int, so we remove it from the graph. isnan_ = isnan def isnan(a): """isnan(a)""" a = as_tensor_variable(a) if a.dtype in discrete_dtypes: return alloc(np.asarray(False, dtype="bool"), *[a.shape[i] for i in range(a.ndim)]) return isnan_(a) @_scal_elemwise def isinf(a): """isinf(a)""" # Rename isnan to isnan_ to allow to bypass it when not needed. # glibc 2.23 don't allow isnan on int, so we remove it from the graph. isinf_ = isinf def isinf(a): """isinf(a)""" a = as_tensor_variable(a) if a.dtype in discrete_dtypes: return alloc(np.asarray(False, dtype="bool"), *[a.shape[i] for i in range(a.ndim)]) return isinf_(a) def allclose(a, b, rtol=1.e-5, atol=1.e-8, equal_nan=False): """ Implement Numpy's ``allclose`` on tensors. ``absolute(a - b) <= (atol + rtol * absolute(b))`` Parameters ---------- a : tensor Input to compare. b : tensor Input to compare. rtol : float The relative tolerance parameter. atol : float The absolute tolerance parameter. equal_nan: bool Whether to consider nan's in the same place to be close. Returns ------- bool A boolean value (of type int8 returned by the tensor elementwise `all` function) whether all elements in a and b are in the tolerance range defined above. Notes ----- Not a symmetric equation. See Numpy's documentation. """ return all(isclose(a, b, rtol, atol, equal_nan)) def isclose(a, b, rtol=1.e-5, atol=1.e-8, equal_nan=False): """ Implements Numpy's ``isclose`` on tensors. The tolerance values are positive, typically very small numbers. The relative difference (`rtol` * abs(`b`)) and the absolute difference `atol` are added together to compare against the absolute difference between `a` and `b`. ``absolute(a - b) <= (atol + rtol * absolute(b))`` Parameters ---------- a : tensor Input to compare. b : tensor Input to compare. rtol : float The relative tolerance parameter. atol : float The absolute tolerance parameter. equal_nan : bool Whether to consider nan's in the same place to be close Returns ------- int8 A boolean (int8) array where two arrays are element-wise equal within a tolerance. Notes ----- Not a symmetric equation. See Numpy's documentation. Examples -------- >>> import theano >>> import numpy as np >>> a = theano._asarray([1e10, 1e-7], dtype="float64") >>> b = theano._asarray([1.00001e10, 1e-8], dtype="float64") >>> theano.tensor.isclose(a, b).eval() array([1, 0], dtype=int8) >>> a = theano._asarray([1e10, 1e-8], dtype="float64") >>> b = theano._asarray([1.00001e10, 1e-9], dtype="float64") >>> theano.tensor.isclose(a, b).eval() array([1, 1], dtype=int8) >>> a = theano._asarray([1e10, 1e-8], dtype="float64") >>> b = theano._asarray([1.0001e10, 1e-9], dtype="float64") >>> theano.tensor.isclose(a, b).eval() array([0, 1], dtype=int8) >>> a = theano._asarray([1.0, np.nan], dtype="float64") >>> b = theano._asarray([1.0, np.nan], dtype="float64") >>> theano.tensor.isclose(a, b).eval() array([1, 0], dtype==int8) >>> a = theano._asarray([1.0, np.nan], dtype="float64") >>> b = theano._asarray([1.0, np.nan], dtype="float64") >>> theano.tensor.isclose(a, b, equal_nan=True).eval() array([1, 1], dtype==int8) >>> a = theano._asarray([1.0, np.inf], dtype="float64") >>> b = theano._asarray([1.0, -np.inf], dtype="float64") >>> theano.tensor.isclose(a, b).eval() array([1, 0], dtype==int8) >>> a = theano._asarray([1.0, np.inf], dtype="float64") >>> b = theano._asarray([1.0, np.inf], dtype="float64") >>> theano.tensor.isclose(a, b).eval() array([1, 1], dtype==int8) """ # close will be an int8 array of 1 where within tolerance # and 0 where not within tolerance or there was a nan or inf value. diff = abs(a - b) tolerance = atol + rtol * abs(b) close_prelim = le(diff, tolerance) a_nan = isnan(a) b_nan = isnan(b) nans = bitwise_or(a_nan, b_nan) a_inf = isinf(a) b_inf = isinf(b) infs = bitwise_or(a_inf, b_inf) nans_or_infs = bitwise_or(nans, infs) # close is now an array of 0's except where elements are not nan or inf # and are withing the tolerance. close = bitwise_and(close_prelim, bitwise_not(nans_or_infs)) # deal with signed inf values. this will make an array inf_eq of 0's # except where inf values have the same sign. both_infs = bitwise_and(a_inf, b_inf) inf_signs_eq = eq(a_inf * sgn(a), b_inf * sgn(b)) inf_eq = bitwise_and(both_infs, inf_signs_eq) # now create the potential result combining close and inf_eq close_with_infs = bitwise_or(close, inf_eq) # deal with comparing nan's. if equal_nan: both_nans = bitwise_and(a_nan, b_nan) return bitwise_or(close_with_infs, both_nans) # otherwise nan's aren't considered close. else: return close_with_infs ########################## # Condition ########################## @_scal_elemwise def switch(cond, ift, iff): """if cond then ift else iff""" where = switch ########################## # Bit-wise ########################## @_scal_elemwise def and_(a, b): """bitwise a & b""" bitwise_and = and_ # numpy name for it @_scal_elemwise def or_(a, b): """bitwise a | b""" bitwise_or = or_ # numpy name for it @_scal_elemwise def xor(a, b): """bitwise a ^ b""" bitwise_xor = xor # numpy name for it @_scal_elemwise def invert(a): """bitwise ~a""" bitwise_not = invert # numpy alias for it ########################## # Math ########################## @_scal_elemwise def abs_(a): """|`a`| TensorVariable overloads the `TensorVariable.__abs__` operator so that this function is called when you type abs(a). """ pprint.assign(abs_, printing.PatternPrinter(('|%(0)s|', -1000))) @_scal_elemwise def exp(a): """e^`a`""" @_scal_elemwise def exp2(a): """2^`a`""" @_scal_elemwise def expm1(a): """e^`a` - 1""" @_scal_elemwise def neg(a): """-a""" # numpy.reciprocal does integer division on integer inputs # (which is not very interesting) @_scal_elemwise def inv(a): """1.0/a""" @_scal_elemwise def log(a): """base e logarithm of a""" @_scal_elemwise def log2(a): """base 2 logarithm of a""" @_scal_elemwise def log10(a): """base 10 logarithm of a""" @_scal_elemwise def log1p(a): """log(1+a)""" @_scal_elemwise def sgn(a): """sign of a""" @_scal_elemwise def ceil(a): """ceiling of a""" @_scal_elemwise def floor(a): """floor of a""" @_scal_elemwise def trunc(a): """trunc of a""" @constructor def iround(a, mode=None): """cast(round(a,mode),'int64')""" return cast(round(a, mode), 'int64') @constructor def round(a, mode=None): """round_mode(a) with mode in [half_away_from_zero, half_to_even]. Default to half_to_even.""" if mode is None: mode = "half_to_even" if config.warn.round: warnings.warn( "theano.tensor.round() changed its default from" " `half_away_from_zero` to `half_to_even` to have" " the same default as NumPy. Use the Theano flag" " `warn.round=False` to disable this warning.") if mode == "half_away_from_zero": return round_half_away_from_zero(a) elif mode == "half_to_even": return round_half_to_even(a) else: raise Exception("round mode %s is not implemented." % mode) @_scal_elemwise def round_half_to_even(a): """round_half_to_even(a)""" @_scal_elemwise def round_half_away_from_zero(a): """round_half_away_from_zero(a)""" @_scal_elemwise def sqr(a): """square of a""" # alias to sqr, included to maintain similarity with numpy interface square = sqr def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, aweights=None): """Calculate the covariance matrix. Covariance indicates the level to which two variables vary together. If we examine N-dimensional samples, :math:`m = [x_1, x_2, ... x_N]^T`, then the covariance matrix element :math:`C_{ij}` is the covariance of :math:`x_i` and :math:`x_j`. The element :math:`C_{ii}` is the variance of :math:`x_i`. Code and docstring ported from numpy. ---------- m : array_like A 2-D array containing multiple variables and observations. Each row of `m` represents a variable, and each column is observations of all those variables. y : array_like, optional An additional set of variables and observations. `y` has the same form as that of `m`. rowvar : bool, optional If `rowvar` is True (default), then each row represents a variable, with observations in the columns. Otherwise, the relationship is transposed: each column represents a variable, while the rows contain observations. bias : bool, optional Default normalization (False) is by ``(N - 1)``, where ``N`` is the number of observations given (unbiased estimate). If `bias` is True, then normalization is by ``N``. These values can be overridden by using the keyword ``ddof``. ddof : int, optional If not ``None`` the default value implied by `bias` is overridden. The default value is ``None``. Returns ------- out : The covariance matrix of the variables. """ if fweights is not None: raise NotImplementedError('fweights are not implemented') if aweights is not None: raise NotImplementedError('aweights are not implemented') if not rowvar and m.shape[0] != 1: m = m.T if y is not None: if not rowvar and y.shape[0] != 1: y = y.T m = theano.tensor.concatenate((m, y), axis=0) if ddof is None: if not bias: ddof = 1 else: ddof = 0 # Determine the normalization fact = m.shape[1] - ddof m -= m.mean(axis=1, keepdims=1) c = m.dot(m.T) c *= theano.tensor.constant(1) / fact return c.squeeze() @_scal_elemwise def sqrt(a): """square root of a""" @_scal_elemwise def deg2rad(a): """convert degree a to radian""" @_scal_elemwise def rad2deg(a): """convert radian a to degree""" @_scal_elemwise def cos(a): """cosine of a""" @_scal_elemwise def arccos(a): """arccosine of a""" @_scal_elemwise def sin(a): """sine of a""" @_scal_elemwise def arcsin(a): """arcsine of a""" @_scal_elemwise def tan(a): """tangent of a""" @_scal_elemwise def arctan(a): """arctangent of a""" @_scal_elemwise def arctan2(a, b): """arctangent of a / b""" @_scal_elemwise def cosh(a): """hyperbolic cosine of a""" @_scal_elemwise def arccosh(a): """hyperbolic arc cosine of a""" @_scal_elemwise def sinh(a): """hyperbolic sine of a""" @_scal_elemwise def arcsinh(a): """hyperbolic arc sine of a""" @_scal_elemwise def tanh(a): """hyperbolic tangent of a""" @_scal_elemwise def arctanh(a): """hyperbolic arc tangent of a""" @_scal_elemwise def erf(a): """error function""" @_scal_elemwise def erfc(a): """complementary error function""" @_scal_elemwise def erfcx(a): """scaled complementary error function""" @_scal_elemwise def erfinv(a): """inverse error function""" @_scal_elemwise def erfcinv(a): """inverse complementary error function""" @_scal_elemwise def gamma(a): """gamma function""" @_scal_elemwise def gammaln(a): """log gamma function""" @_scal_elemwise def psi(a): """derivative of log gamma function""" @_scal_elemwise def tri_gamma(a): """second derivative of the log gamma function""" @_scal_elemwise def chi2sf(x, k): """chi squared survival function""" @_scal_elemwise def j0(x): """Bessel function of the first kind of order 0.""" @_scal_elemwise def j1(x): """Bessel function of the first kind of order 1.""" @_scal_elemwise def jv(v, x): """Bessel function of the first kind of order v (real).""" @_scal_elemwise def i0(x): """Modified Bessel function of the first kind of order 0.""" @_scal_elemwise def i1(x): """Modified Bessel function of the first kind of order 1.""" @_scal_elemwise def iv(v, x): """Modified Bessel function of the first kind of order v (real).""" @_scal_elemwise def real(z): """Return real component of complex-valued tensor `z`""" _tensor_py_operators.real = property(real) @_scal_elemwise def imag(z): """Return imaginary component of complex-valued tensor `z`""" _tensor_py_operators.imag = property(imag) @_scal_elemwise def angle(z): """Return polar-coordinate angle of complex-valued tensor `z`""" @_scal_elemwise # numpy.complex cannot build tensors def complex(real, imag): """Return complex-valued tensor with `real` and `imag` components""" @_scal_elemwise def conj(z): """Return the complex conjugate of `z`.""" @_scal_elemwise def complex_from_polar(abs, angle): """Return complex-valued tensor from polar coordinate specification.""" ########################## # Misc ########################## # fill, _fill_inplace = _elemwise(scal.second, 'fill', # """fill WRITEME (elemwise)""") @_scal_elemwise def second(a, b): """Create a matrix by filling the shape of a with b""" fill = second pprint.assign(fill, printing.FunctionPrinter('fill')) @constructor def ones_like(model, dtype=None, opt=False): """equivalent of numpy.ones_like Parameters ---------- model : tensor dtype : data-type, optional opt : If True, we will return a constant instead of a graph when possible. Useful for Theano optimization, not for user building a graph as this have the consequence that model isn't always in the graph. Returns ------- tensor tensor the shape of model containing ones of the type of dtype. """ if dtype is None: dtype = model.type.dtype ret = constant(1.0, dtype=dtype) if opt and ret.type == model.type: return ret return fill(model, ret) @constructor def zeros_like(model, dtype=None, opt=False): """equivalent of numpy.zeros_like Parameters ---------- model : tensor dtype : data-type, optional opt : If True, we will return a constant instead of a graph when possible. Useful for Theano optimization, not for user building a graph as this have the consequence that model isn't always in the graph. Returns ------- tensor tensor the shape of model containing zeros of the type of dtype. """ if dtype is None: dtype = model.type.dtype ret = constant(0.0, dtype=dtype) if opt and ret.type == model.type: return ret return fill(model, ret) def zeros(shape, dtype=None): """ Create a Tensor filled with zeros, closer to Numpy's syntax than ``alloc``. """ if not isinstance(shape, (list, tuple, TensorVariable)): shape = [shape] if dtype is None: dtype = config.floatX return alloc(np.array(0, dtype=dtype), *shape) def ones(shape, dtype=None): """ Create a Tensor filled with ones, closer to Numpy's syntax than ``alloc``. """ if not isinstance(shape, (list, tuple, TensorVariable)): shape = [shape] if dtype is None: dtype = config.floatX return alloc(np.array(1, dtype=dtype), *shape) class Nonzero(gof.Op): """ Return the indices of the elements that are non-zero. Returns a matrix of shape (ndim, number of nonzero elements) such that element (i,j) is the index in the ith dimension of the jth non-zero element. Note this is different than NumPy, which returns a tuple of arrays, one for each dimension of the input array. Parameters ---------- a : array_like Input array. Returns ------- matrix Matrix containing the indices of the non-zero elements of a. See Also -------- nonzero_values : Return the non-zero elements of the input array flatnonzero : Return the indices of the non-zero elements of the flattened input array. """ __props__ = () def make_node(self, a): a = as_tensor_variable(a) if a.ndim == 0: raise ValueError('Nonzero only supports non-scalar arrays.') output = [TensorType(dtype='int64', broadcastable=(False, False))()] return gof.Apply(self, [a], output) def perform(self, node, inp, out_): a = inp[0] out, = out_ result_tuple = np.nonzero(a) if len(result_tuple[0]) > 0: result = np.vstack(result_tuple) else: result = np.zeros((len(result_tuple), 0)) out[0] = result.astype('int64') def grad(self, inp, grads): return [grad_undefined(self, 0, inp[0])] _nonzero = Nonzero() def nonzero(a, return_matrix=False): """ Returns one of the following: If return_matrix is False (default, same as NumPy): A tuple of vector arrays such that the ith element of the jth array is the index of the ith non-zero element of the input array in the jth dimension. If return_matrix is True (same as Theano Op): Returns a matrix of shape (ndim, number of nonzero elements) such that element (i,j) is the index in the ith dimension of the jth non-zero element. Parameters ---------- a : array_like Input array. return_matrix : bool If True, returns a symbolic matrix. If False, returns a tuple of arrays. Defaults to False. Returns ------- tuple of vectors or matrix See Also -------- nonzero_values : Return the non-zero elements of the input array flatnonzero : Return the indices of the non-zero elements of the flattened input array. """ matrix_result = _nonzero(a) if return_matrix: return matrix_result else: if a.ndim > 0: tuple_result = tuple([matrix_result[i] for i in xrange(a.ndim)]) else: tuple_result = tuple([matrix_result[0]]) return tuple_result def flatnonzero(a): """ Return a vector of indices that are non-zero in the flattened version of a. This is equivalent to nonzero(a.flatten(), return_matrix=True)[0] Parameters ---------- a : tensor Input tensor Returns ------- vector Output vector, containing the indices of the elements of `a.flatten()` that are non-zero. See Also -------- nonzero : Return the indices of the non-zero elements of the input array. nonzero_values : Return the non-zero elements of the input array """ if a.ndim == 0: raise ValueError('Nonzero only supports non-scalar arrays.') return nonzero(a.flatten(), return_matrix=True)[0] def nonzero_values(a): """ Return a vector of non-zero elements contained in the input array. The following behavior works to extract non-zero elements from an array in NumPy but is *NOT* supported by Theano: a[numpy.nonzero(a)] Instead, the nonzero_values function or method should be used: tensor.nonzero_values(a) a.nonzero_values() This is equivalent to the following: a.flatten()[tensor.flatnonzero(a)] Parameters ---------- a : tensor Input tensor Returns ------- vector Output vector, containing the non-zero elements of a. See Also -------- nonzero : Return the indices of the non-zero elements of the input array. flatnonzero : Return the indices of the non-zero elements of the flattened input array. """ return a.flatten()[flatnonzero(a)] class Tri(gof.Op): __props__ = ("dtype",) def __init__(self, dtype=None): if dtype is None: dtype = config.floatX self.dtype = dtype def make_node(self, N, M, k): N = as_tensor_variable(N) M = as_tensor_variable(M) k = as_tensor_variable(k) return gof.Apply( self, [N, M, k], [TensorType(dtype=self.dtype, broadcastable=(False, False))()]) def perform(self, node, inp, out_): N, M, k = inp out, = out_ out[0] = np.tri(N, M, k, dtype=self.dtype) def infer_shape(self, node, in_shapes): out_shape = [node.inputs[0], node.inputs[1]] return [out_shape] def grad(self, inp, grads): return [grad_undefined(self, i, inp[i]) for i in xrange(3)] def tri(N, M=None, k=0, dtype=None): """ An array with ones at and below the given diagonal and zeros elsewhere. Parameters ---------- N : int Number of rows in the array. M : int, optional Number of columns in the array. By default, `M` is taken equal to `N`. k : int, optional The sub-diagonal at and below which the array is filled. `k` = 0 is the main diagonal, while `k` < 0 is below it, and `k` > 0 is above. The default is 0. dtype : dtype, optional Data type of the returned array. The default is float. Returns ------- Array of shape (N, M) Array with its lower triangle filled with ones and zero elsewhere; in other words ``T[i,j] == 1`` for ``i <= j + k``, 0 otherwise. """ if dtype is None: dtype = config.floatX if M is None: M = N op = Tri(dtype) return op(N, M, k) def tril(m, k=0): """ Lower triangle of an array. Return a copy of an array with elements above the `k`-th diagonal zeroed. Parameters ---------- m : array_like, shape (M, N) Input array. k : int, optional Diagonal above which to zero elements. `k = 0` (the default) is the main diagonal, `k < 0` is below it and `k > 0` is above. Returns ------- array, shape (M, N) Lower triangle of `m`, of same shape and data-type as `m`. See Also -------- triu : Same thing, only for the upper triangle. """ return m * tri(m.shape[0], m.shape[1], k=k, dtype=m.dtype) def triu(m, k=0): """ Upper triangle of an array. Return a copy of a matrix with the elements below the `k`-th diagonal zeroed. Please refer to the documentation for `tril` for further details. See Also -------- tril : Lower triangle of an array. """ return m * (1 - tri(m.shape[0], m.shape[1], k=k - 1, dtype=m.dtype)) class Eye(gof.Op): __props__ = ("dtype", ) def __init__(self, dtype=None): if dtype is None: dtype = config.floatX self.dtype = dtype def make_node(self, n, m, k): n = as_tensor_variable(n) m = as_tensor_variable(m) k = as_tensor_variable(k) assert n.ndim == 0 assert m.ndim == 0 assert k.ndim == 0 return gof.Apply( self, [n, m, k], [TensorType(dtype=self.dtype, broadcastable=(False, False))()]) def perform(self, node, inp, out_): n, m, k = inp out, = out_ out[0] = np.eye(n, m, k, dtype=self.dtype) def infer_shape(self, node, in_shapes): out_shape = [node.inputs[0], node.inputs[1]] return [out_shape] def grad(self, inp, grads): return [grad_undefined(self, i, inp[i]) for i in xrange(3)] def eye(n, m=None, k=0, dtype=None): """Return a 2-D array with ones on the diagonal and zeros elsewhere. Parameters ---------- n : int Number of rows in the output. m : int, optional Number of columns in the output. If None, defaults to `N`. k : int, optional Index of the diagonal: 0 (the default) refers to the main diagonal, a positive value refers to an upper diagonal, and a negative value to a lower diagonal. dtype : data-type, optional Data-type of the returned array. Returns ------- ndarray of shape (N,M) An array where all elements are equal to zero, except for the `k`-th diagonal, whose values are equal to one. """ if dtype is None: dtype = config.floatX if m is None: m = n localop = Eye(dtype) return localop(n, m, k) def identity_like(x): return eye(x.shape[0], x.shape[1], k=0, dtype=x.dtype) def alloc_validate_shape(shape): sh = [as_tensor_variable(s) for s in shape] bcast = [] for i, s in enumerate(sh): def err_str(): if config.exception_verbosity == 'high': return '\n' + min_informative_str(s) else: return str(s) if s.type.dtype not in integer_dtypes: s_as_str = err_str() raise TypeError('Shape arguments to Alloc must be integers, ' 'but argument %s is not for apply node: %s' % (i, s_as_str)) if s.ndim != 0: s_as_str = err_str() raise TypeError( "Each shape dimension to Alloc must be a scalar, ", 'but dimension %s have %d dimensions for apply node: %s' % (i, s.ndim, s_as_str)) # if s is constant 1, then we're broadcastable in that dim try: const_shp = get_scalar_constant_value(s) except NotScalarConstantError: const_shp = None bcast.append(1 == const_shp) return sh, bcast class Alloc(gof.Op): """Create a Tensor from an initial value and a desired shape. alloc(value, shape0, shape1, ..., shapeN) Returns an N-dimensional tensor initialized by `value` using something equivalent to z = numpy.zeros(shape, value.dtype) z += value The result has N dimensions, has the dtype of `value` and is obtained by broadcasting value over the output ndarray. This Op is used to replace fill() during optimizations because after shapes are lifted, the first argument to fill can often be pruned from the graph. """ _f16_ok = True __props__ = () def validate_shape(self, shape): return alloc_validate_shape(shape) def make_node(self, value, *shape): v = as_tensor_variable(value) sh, bcast = alloc_validate_shape(shape) if v.ndim > len(sh): raise TypeError("The Alloc value to use has more dimensions" " than the specified dimensions", v.ndim, len(sh)) otype = TensorType(dtype=v.dtype, broadcastable=bcast) return gof.Apply(self, [v] + sh, [otype()]) def perform(self, node, inputs, out_): out, = out_ v = inputs[0] sh = tuple([int(i) for i in inputs[1:]]) if out[0] is None or out[0].shape != sh: if v.size == 1 and v.item() == 0: out[0] = np.zeros(sh, dtype=v.dtype) else: out[0] = np.empty(sh, dtype=v.dtype) out[0][...] = v # broadcast v to fill us up else: # reuse the allocated memory. out[0][...] = v # broadcast v to fill us up def c_code(self, node, name, inp, out, sub): vv = inp[0] ndim = len(inp[1:]) zz, = out fail = sub['fail'] code = """ npy_intp shape[%(ndim)s]; """ % dict(ndim=ndim) # Initialize shape for i, shp_i in enumerate(inp[1:]): code += """ shape[%(i)s] = ((dtype_%(shp_i)s*) PyArray_DATA(%(shp_i)s))[0]; """ % dict(i=i, shp_i=shp_i) code += """ int need_new_out = (NULL == %(zz)s); for (int i = 0; i < %(ndim)s; i++) need_new_out = (need_new_out || (PyArray_DIMS(%(zz)s)[i] != shape[i])); if (need_new_out) { Py_XDECREF(%(zz)s); %(zz)s = (PyArrayObject*) PyArray_SimpleNew(%(ndim)s, shape, PyArray_TYPE((PyArrayObject*) py_%(vv)s)); if (!%(zz)s) { PyErr_SetString(PyExc_MemoryError, "alloc failed"); %(fail)s } } // This function takes care of broadcasting if (PyArray_CopyInto(%(zz)s, %(vv)s) == -1) %(fail)s """ % dict(vv=vv, ndim=ndim, zz=zz, fail=fail) return code def c_code_cache_version(self): return (2,) def infer_shape(self, node, input_shapes): return [node.inputs[1:]] def connection_pattern(self, node): rval = [[True]] for ipt in node.inputs[1:]: rval.append([False]) return rval def grad(self, inputs, grads): x = inputs[0] gz = grads[0] n_axes_to_sum = gz.ndim - x.ndim # The number of dimensions added axis = list(range(n_axes_to_sum)) # The broadcasted dimensions axis_broadcasted = [] axis_kept = [] for i, (ib, gb) in enumerate( zip(inputs[0].broadcastable, # We need the dimensions corresponding to x grads[0].broadcastable[-inputs[0].ndim:])): if ib and not gb: axis_broadcasted.append(i + n_axes_to_sum) else: axis_kept.append(i) gx = gz.sum(axis=axis + axis_broadcasted) if axis_broadcasted: new_order = ['x'] * x.ndim for idx, axis in enumerate(axis_kept): new_order[axis] = idx gx = gx.dimshuffle(new_order) # Dimshuffle to add back the broadcasted dims # The *elements* of the output are not connected to # the inputs that specify the shape. If you grow the # shape by epsilon, the existing elements do not # change. return [gx] + [DisconnectedType()() for i in inputs[1:]] def __call__(self, val, *shapes, **kwargs): """ If the alloc would be useless, this function returns val. If this function is called outside of a graph optimization context (for instance, it is manually called by a user building a graph), then we always return an Alloc node, to allow for DebugMode to check for size mismatches. If you always want an Alloc node, call make_node. """ ret = super(Alloc, self).__call__(val, *shapes, **kwargs) try: # It makes optimization difficult when useless allocs are thrown # into the graph at every stage of optimization. This little logic # tries to help at least in some cases. if hasattr(val, 'fgraph') and (val.type == ret.type): return val except AttributeError: pass return ret def R_op(self, inputs, eval_points): if eval_points[0] is None: return [None] return self(eval_points[0], *inputs[1:], **dict(return_list=True)) def do_constant_folding(self, node): if not getattr(node.outputs[0], 'clients', []): # If there are no clients then there is no point doing constant # folding. return False for client in node.outputs[0].clients: if client[0] == 'output': # If the output is a constant, it will have to be deepcopied # each time the function is called. So we do not fold. return False elif ( # The following ops work inplace of their input id 0. client[1] == 0 and isinstance(client[0].op, ( # Ops that will work inplace on the Alloc. So if they # get constant_folded, they would copy the # constant and this is less efficients. # Not doing the constant folding could also lower # the peak memory usage, as we the "constant" won't # always exists. theano.tensor.subtensor.IncSubtensor, theano.tensor.subtensor.AdvancedIncSubtensor1, theano.tensor.subtensor.AdvancedIncSubtensor, theano.tensor.blas.Gemv, theano.tensor.blas_c.CGemv, theano.tensor.blas.Ger, theano.tensor.blas_c.CGer, theano.tensor.blas_scipy.ScipyGer))): return False # If the clients is a transfer to the GPU, we don't want to # fold. We let the Alloc being moved to the GPU, then we # let the GPU algo decide if it need to fold it or not. elif client[0].op.__class__.__name__.lower().startswith("gpu"): return False return True alloc = Alloc() pprint.assign(alloc, printing.FunctionPrinter('alloc')) def transfer(var, target): """ Return a version of `var` transferred to `target`. `cpu` mean a TensorType (on the CPU). Other types may define additional targets. Parameters ---------- var : variable A theano variable target : str The target of the transfer """ if target == 'cpu': return as_tensor_variable(var) else: for trans in transfer._others: res = trans(var, target) if res is not None: return res raise ValueError("Can't transfer to target %s" % (target,)) transfer._others = [] def register_transfer(fn): """ Register a transfer function for alternative targets. Parameters ---------- fn : callable """ transfer._others.append(fn) """Create a duplicate of `a` (with duplicated storage)""" tensor_copy = elemwise.Elemwise(scal.identity) pprint.assign(tensor_copy, printing.IgnorePrinter()) @constructor def sum(input, axis=None, dtype=None, keepdims=False, acc_dtype=None): """ Computes the sum along the given axis(es) of a tensor `input`. When axis is None (the default value), the sum is performed over the flattened tensor. For full documentation see ``tensor.elemwise.Sum``. In particular please pay attention to the important warning when using a custom acc_dtype. Parameters ---------- keepdims: bool If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, the result will broadcast correctly against the original tensor. """ out = elemwise.Sum(axis=axis, dtype=dtype, acc_dtype=acc_dtype)(input) if keepdims: out = makeKeepDims(input, out, axis) return out pprint.assign(Sum(), printing.FunctionPrinter('sum')) @constructor def prod(input, axis=None, dtype=None, keepdims=False, acc_dtype=None, no_zeros_in_input=False): """ Computes the product along the given axis(es) of a tensor `input`. When axis is None (the default value), the product is performed over the flattened tensor. For full documentation see ``tensor.elemwise.Prod``. Parameters ---------- keepdims: bool If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, the result will broadcast correctly against the original tensor. """ out = elemwise.Prod(axis, dtype=dtype, acc_dtype=acc_dtype, no_zeros_in_input=no_zeros_in_input)(input) if keepdims: out = makeKeepDims(input, out, axis) return out class Mean(elemwise.CAReduce): def __init__(self, axis=None): elemwise.CAReduce.__init__(self, scal.add, axis) assert self.axis is None or len(self.axis) == 1 def __str__(self): if self.axis is not None: return "Mean{%s}" % (", ".join(str(x) for x in self.axis)) else: return "Mean" def _output_dtype(self, idtype): # we want to protect against overflow return 'float64' def perform(self, node, inp, out): input, = inp output, = out if self.axis is None: axis = None else: axis = self.axis[0] # numpy.asarray is needed as otherwise we can end up with a # numpy scalar. output[0] = np.asarray(np.mean(input, dtype='float64', axis=axis)) def c_code(self, node, name, inames, onames, sub): if self.axis is not None: return super(Op, self).c_code(node, name, inames, onames, sub) ret = elemwise.CAReduce.c_code(self, node, name, inames, onames, sub) # TODO: c_code perform support only axis is None return ret + """ *((double *)PyArray_DATA(%s)) /= PyArray_SIZE(%s); """ % (onames[0], inames[0]) # TODO: implement the grad. When done and tested, you can make this the default # version. # def grad(self, (x,), (gout,)): # import pdb;pdb.set_trace() # return grad(mean(x, self.axis, op=False),[x]) @constructor def mean(input, axis=None, dtype=None, op=False, keepdims=False, acc_dtype=None): """ Computes the mean value along the given axis(es) of a tensor `input`. Parameters ---------- axis : None or int or (list of int) (see `Sum`) Compute the mean along this axis of the tensor. None means all axes (like numpy). dtype: None or string Dtype to cast the result of the inner summation into. For instance, by default, a sum of a float32 tensor will be done in float64 (acc_dtype would be float64 by default), but that result will be casted back in float32. keepdims: bool If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, the result will broadcast correctly against the original tensor. acc_dtype: None or string Dtype to use for the inner summation. This will not necessarily be the dtype of the output (in particular if it is a discrete (int/uint) dtype, the output will be in a float type). If None, then we use the same rules as `sum()`. Notes ----- For gpu, if you specify dtype=float32, everything will be done on the gpu. """ input = as_tensor_variable(input) if op: if dtype not in (None, 'float64'): raise NotImplementedError( 'The Mean op does not support the dtype argument, ' 'and will always use float64. If you want to specify ' 'the dtype, call tensor.mean(..., op=False).', dtype) if acc_dtype not in (None, 'float64'): raise NotImplementedError( 'The Mean op does not support the acc_dtype argument, ' 'and will always use float64. If you want to specify ' 'acc_dtype, call tensor.mean(..., op=False).', dtype) out = Mean(axis)(input) if keepdims: out = makeKeepDims(input, out, axis) return out if dtype is not None: # The summation will be done with the specified dtype. # sum() will complain if it is not suitable. sum_dtype = dtype else: sum_dtype = None # float16 overflows on the cast way too often if input.dtype == 'float16': sum_dtype = 'float32' s = sum(input, axis=axis, dtype=sum_dtype, keepdims=keepdims, acc_dtype=acc_dtype) shp = shape(input) # Cast shp into a float type # TODO Once we have a consistent casting policy, we could simply # use true_div. if s.dtype in ('float16', 'float32', 'complex64'): shp = cast(shp, 'float32') else: shp = cast(shp, 'float64') if axis is None: axis = list(range(input.ndim)) elif isinstance(axis, (integer_types, np.integer)): axis = [axis] elif isinstance(axis, np.ndarray) and axis.ndim == 0: axis = [int(axis)] else: axis = [int(a) for a in axis] # This sequential division will possibly be optimized by Theano: for i in axis: s = true_div(s, shp[i]) # This can happen when axis is an empty list/tuple if s.dtype != shp.dtype and s.dtype in discrete_dtypes: s = cast(s, shp.dtype) if dtype == 'float16' or (dtype is None and input.dtype == 'float16'): s = cast(s, 'float16') s.name = 'mean' return s @constructor def var(input, axis=None, ddof=0, keepdims=False, corrected=False): """ Computes the variance along the given axis(es) of a tensor `input`. Parameters ---------- axis: None or int or (list of int) (see `Sum`) Compute the variance along this axis of the tensor. None means all axes (like numpy). ddof: Degrees of freedom; 0 would compute the ML estimate, 1 would compute the unbiased estimate. keepdims : bool If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, the result will broadcast correctly against the original tensor. corrected : bool If this is set to True, the 'corrected_two_pass' algorithm is used to compute the variance. Refer : http://www.cs.yale.edu/publications/techreports/tr222.pdf Notes ----- Default uses the two-pass algorithm (reference below). https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Two-pass_algorithm Also supports 'corrected_two_pass' algorithm (using the 'corrected' flag) which is numerically more stable. There exist other implementations that offer better stability, but probably slower. """ if isinstance(ddof, (bool)): raise ValueError('Parameter keepdims is now at index 3: (input, \ axis=None, ddof=0, keepdims=False, corrected=False)') input_ndim = input.type.ndim if axis is None: axis = list(range(input_ndim)) elif isinstance(axis, (integer_types, np.integer)): axis = [axis] elif isinstance(axis, np.ndarray) and axis.ndim == 0: axis = [int(axis)] else: axis = [int(a) for a in axis] # compute the axis-wise mean mean_input = mean(input, axis, keepdims=True) # center the input centered_input = input - mean_input # return the mean sqr two = constant(2, dtype=centered_input.dtype) if ddof == 0: v = mean((centered_input ** two), axis, keepdims=keepdims) else: shp = shape(input) - ddof v = sum((centered_input ** two), axis=axis, keepdims=keepdims) for i in axis: v = true_div(v, shp[i]) # use 'corrected_two_pass' algorithm if corrected: if ddof == 0: error = mean(centered_input, axis, keepdims=keepdims) ** 2 else: shp = shape(input) - ddof shp_inp = shape(input) error = sum(centered_input, axis=axis, keepdims=keepdims) ** 2 for i in axis: error = true_div(error, shp[i] * shp_inp[i]) v = v - error v.name = 'var' return v @constructor def std(input, axis=None, ddof=0, keepdims=False, corrected=False): """ Computes the standard deviation along the given axis(es) of a tensor `input`. Parameters ---------- axis: None or int or (list of int) (see `Sum`) Compute the variance along this axis of the tensor. None means all axes (like numpy). ddof: Degrees of freedom; 0 would compute the ML estimate, 1 would compute the unbiased estimate. keepdims : bool If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, the result will broadcast correctly against the original tensor. corrected : bool If this is set to True, the 'corrected_two_pass' algorithm is used to compute the variance. Refer : http://www.cs.yale.edu/publications/techreports/tr222.pdf Notes ----- It calls 'var()' and 'var()' uses the two-pass algorithm (reference below). https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Two-pass_algorithm Function 'var()' also supports 'corrected_two_pass' algorithm (using the 'corrected' flag) which is numerically more stable. There exist other implementations that offer better stability, but probably slower. """ if isinstance(ddof, (bool)): raise ValueError('Parameter keepdims is now at index 3: (input, \ axis=None, ddof=0, keepdims=False, corrected=False)') ret = sqrt(var(input=input, axis=axis, ddof=ddof, keepdims=keepdims, corrected=corrected)) ret.name = 'std' return ret class Default(gof.Op): """ Takes an input x and a default value. If the input is not None, a reference to it is returned. If the input is None, a copy of the default value is returned instead. The input and the default must have exactly the same type. """ view_map = {0: [0]} __props__ = () def make_node(self, x, default): x, default = as_tensor_variable(x), as_tensor_variable(default) if x.type != default.type: raise TypeError('Both default() arguments must have same type', x, default) return gof.Apply(self, [x, default], [default.type()]) def perform(self, node, inp, out_): x, default = inp out, = out_ if x is None: # why copy? Theano can't yet understand out[0] being a view of # either x or y, so we can be a view of x, but only a copy of y. out[0] = default.copy() else: out[0] = x default = Default() setdefault = default # legacy ########################## # Arithmetics ########################## @_scal_elemwise def maximum(x, y): """elemwise maximum. See max for the maximum in one tensor""" # see decorator for function body @_scal_elemwise def minimum(x, y): """elemwise minimum. See min for the minimum in one tensor""" # see decorator for function body def div_proxy(x, y): """Proxy for either true_div or int_div, depending on types of x, y.""" f = scal.int_or_true_div( as_tensor_variable(x).dtype in discrete_dtypes, as_tensor_variable(y).dtype in discrete_dtypes) if f is scal.int_div: return int_div(x, y) else: return true_div(x, y) def divmod(x, y): """elementvise divmod, using floor_div and mod_check""" return floor_div(x, y), mod_check(x, y) @_scal_elemwise def add(a, *other_terms): """elementwise addition""" # see decorator for function body @_scal_elemwise def sub(a, b): """elementwise subtraction""" # see decorator for function body @_scal_elemwise def mul(a, *other_terms): """elementwise multiplication""" # see decorator for function body @_scal_elemwise def true_div(a, b): """elementwise [true] division (inverse of multiplication)""" # see decorator for function body @_scal_elemwise def int_div(a, b): """elementwise [floor] division (inverse of multiplication)""" # see decorator for function body # floor_div and int_div are the same thing floor_div = int_div def ceil_intdiv(a, b): """ Safely compute ceil(float_division(a, b)). Works for all dtypes, but mostly useful when a and b are int. """ # If a and b are int with not many significant bits, we could # cast them to float to avoid doing the modulo. We do not know if this # is faster or not. But this is not safe for int64 as the cast will # lose precision. # e.g.: cast(cast(a, scalar.upcast(a, 'float32')) / b, scal.upcast(a, b)) # We cast for the case when a and b are uint*. Otherwise neq will # force their upcast to int. div = int_div(a, b) ret = cast(neq(a % b, 0), div.dtype) + div assert ret.dtype == scal.upcast(div.owner.inputs[0], div.owner.inputs[1]) return ret def mod_check(x, y): """Make sure we do not try to use complex numbers.""" if ((as_tensor_variable(x).dtype in complex_dtypes or as_tensor_variable(y).dtype in complex_dtypes)): # Currently forbidden. raise scal.Mod.complex_error else: return mod(x, y) @_scal_elemwise def mod(a, b): """elementwise modulo""" # see decorator for function body @_scal_elemwise def pow(a, b): """elementwise power""" # see decorator for function body @_scal_elemwise def clip(x, min, max): """ Clip x to be between min and max. Notes ----- When `x` is equal to the boundaries, the output is considered to be `x`, so at these points, the gradient of the cost wrt the output will be propagated to `x`, not to `min` nor `max`. In other words, on these points, the gradient wrt `x` will be equal to the gradient wrt the output, and the gradient wrt `min` and `max` will be zero. """ # see decorator for function body # for grep: clamp, bound pprint.assign(add, printing.OperatorPrinter('+', -2, 'either')) pprint.assign(mul, printing.OperatorPrinter('*', -1, 'either')) pprint.assign(sub, printing.OperatorPrinter('-', -2, 'left')) pprint.assign(neg, printing.OperatorPrinter('-', 0, 'either')) pprint.assign(true_div, printing.OperatorPrinter('/', -1, 'left')) pprint.assign(int_div, printing.OperatorPrinter('//', -1, 'left')) pprint.assign(pow, printing.OperatorPrinter('**', 1, 'right')) ########################## # View Operations ########################## def extract_constant(x, elemwise=True, only_process_constants=False): """ This function is basically a call to tensor.get_scalar_constant_value. The main difference is the behaviour in case of failure. While get_scalar_constant_value raises an TypeError, this function returns x, as a tensor if possible. If x is a ScalarVariable from a scalar_from_tensor, we remove the conversion. If x is just a ScalarVariable, we convert it to a tensor with tensor_from_scalar. """ try: x = get_scalar_constant_value(x, elemwise, only_process_constants) except NotScalarConstantError: pass if ((isinstance(x, scal.ScalarVariable) or isinstance(x, scal.sharedvar.ScalarSharedVariable))): if x.owner and isinstance(x.owner.op, ScalarFromTensor): x = x.owner.inputs[0] else: x = tensor_from_scalar(x) return x def transpose(x, axes=None): """ Reorder the dimensions of x. (Default: reverse them) This is a macro around dimshuffle that matches the numpy.transpose function. """ if axes is None: axes = list(range((x.ndim - 1), -1, -1)) ret = DimShuffle(x.broadcastable, axes)(x) if x.name and axes == list(range((x.ndim - 1), -1, -1)): ret.name = x.name + '.T' return ret def batched_dot(a, b): """ Compute the batched dot product of two variables: batched_dot(a, b)[i] = dot(a[i], b[i]) Note that this batched_dot function does one of three things, in the following sequence: 1. If either a or b is a vector, it returns the batched elementwise product without calling the Theano BatchedDot op. 2. If both a and b have either 2 or 3 dimensions, it calls Theano's BatchedDot op on a and b. 3. If either a or b has more than 3 dimensions, it calls Theano's batched_tensordot function with appropriate axes. The batched_tensordot function expresses high-dimensional batched dot products in terms of batched matrix-matrix dot products, so it may be possible to futherize optimize for performance. """ a, b = as_tensor_variable(a), as_tensor_variable(b) if a.ndim == 0: raise TypeError("a must have at least one (batch) axis") elif b.ndim == 0: raise TypeError("b must have at least one (batch) axis") elif a.ndim == 1: return a.dimshuffle(*([0] + ["x"] * (b.ndim - 1))) * b elif b.ndim == 1: return a * b.dimshuffle(*([0] + ["x"] * (a.ndim - 1))) elif a.ndim > 3 or b.ndim > 3: return batched_tensordot( a, b, [[a.ndim - 1], [np.maximum(1, b.ndim - 2)]]) else: # avoid circular import return theano.tensor.blas.BatchedDot()(a, b) def batched_tensordot(x, y, axes=2): """ Compute a batched tensordot product. A hybrid of batched_dot and tensordot, this function computes the tensordot product between the two tensors, by iterating over the first dimension to perform a sequence of tensordots. Parameters ---------- x : tensor A Tensor with sizes e.g.: for 3D (dim1, dim3, dim2) y : tensor A Tensor with sizes e.g.: for 3D (dim1, dim2, dim4) axes: int or array-like of length 2 If an integer, the number of axes to sum over. If an array, it must have two array elements containing the axes to sum over in each tensor. If an integer i, it is converted to an array containing the last i dimensions of the first tensor and the first i dimensions of the second tensor (excluding the first (batch) dimension): axes = [list(range(a.ndim - i, b.ndim)), list(range(1,i+1))] If an array, its two elements must contain compatible axes of the two tensors. For example, [[1, 2], [2, 4]] means sum over the 2nd and 3rd axes of a and the 3rd and 5th axes of b. (Remember axes are zero-indexed!) The 2nd axis of a and the 3rd axis of b must have the same shape; the same is true for the 3rd axis of a and the 5th axis of b. Like tensordot, this function uses a series of dimshuffles and reshapes to reduce the tensor dot product to a matrix or vector dot product. Finally, it calls batched_dot to compute the result. """ return _tensordot_as_dot(x, y, axes, dot=batched_dot, batched=True) def split(x, splits_size, n_splits, axis=0): the_split = Split(n_splits) return the_split(x, axis, splits_size) class Split(Op): """Partition a `TensorVariable` along some axis. Examples -------- >>> x = vector() >>> splits = lvector() You have to declare right away how many split_points there will be. >>> ra, rb, rc = split(x, splits, n_splits = 3, axis = 0) >>> f = function([x, splits], [ra, rb, rc]) >>> a, b, c = f([0,1,2,3,4,5], [3, 2, 1]) a == [0,1,2] b == [3, 4] c == [5] """ len_splits = None """A Split instance will have this many outputs, and require that the splits argument to `perform` have exactly this many elements. """ __props__ = ("len_splits",) def __init__(self, len_splits): self.len_splits = int(len_splits) def __str__(self): return self.__class__.__name__ + "{%s}" % self.len_splits def make_node(self, x, axis, splits): """WRITEME""" x = as_tensor_variable(x) axis = as_tensor_variable(axis) splits = as_tensor_variable(splits) if splits.type not in int_vector_types: raise TypeError('splits must have type tensor.lvector', splits.type) if axis.type not in int_types: raise TypeError('axis must have type lscalar', axis.type) # # The following lines are necessary if we allow splits of zero # if isinstance(axis, gof.Constant): # x = unbroadcast(x, int(axis.data)) # else: # x = unbroadcast(x, *range(x.type.ndim)) inputs = [x, axis, splits] outputs = [x.type() for i in xrange(self.len_splits)] return Apply(self, inputs, outputs) def perform(self, node, inputs, outputs): """WRITEME""" x, axis, splits = inputs # in python 2.4, x.shape[numpy.asarray(1)] don't work. if sys.version_info[0:2] == (2, 4) and axis.size == 1: axis = int(axis) try: len_along_axis = x.shape[axis] except: raise ValueError('Split.perform() with axis=(%s) is invalid' ' for x.shape==(%s)' % (axis, x.shape)) if len(splits) != self.len_splits: raise ValueError('In Split.perform(), len(splits) != len_splits.', (len(splits), self.len_splits)) if np.sum(splits) != len_along_axis: raise ValueError('The splits sum to %s, expected %s' % (np.sum(splits), len_along_axis)) if python_any([nb < 0 for nb in splits]): raise ValueError('Split: you tried to make an ndarray with a ' 'negative number of elements.') # Checking is done, let's roll the splitting algorithm! # Basically we step along the given axis of x, extracting # subtensors of size splits[i] as we go along. general_key = [slice(None, None, None) for s in x.shape] lower_idx = 0 for i in xrange(self.len_splits): upper_idx = lower_idx + splits[i] general_key[axis] = slice(lower_idx, upper_idx, None) outputs[i][0] = x.__getitem__(tuple(general_key)).copy() lower_idx = upper_idx def infer_shape(self, node, in_shapes): axis = node.inputs[1] splits = node.inputs[2] shp_x, shp_axis, shp_splits = in_shapes out_shapes = [] for i in xrange(self.len_splits): temp = as_tensor_variable(shp_x) temp = theano.tensor.subtensor.set_subtensor(temp[axis], splits[i]) temp = [temp[i] for i in xrange(len(shp_x))] out_shapes.append(temp) return out_shapes def grad(self, inputs, g_outputs): """Join the gradients along the axis that was used to split x.""" x, axis, n = inputs outputs = self(*inputs, **dict(return_list=True)) # If all the output gradients are disconnected, then so are the inputs if python_all([isinstance(g.type, DisconnectedType) for g in g_outputs]): return [DisconnectedType()(), grad_undefined(self, 1, axis), grad_undefined(self, 2, n)] # Else, we have to make them zeros before joining them new_g_outputs = [] for o, g in zip(outputs, g_outputs): if isinstance(g.type, DisconnectedType): new_g_outputs.append(o.zeros_like()) else: new_g_outputs.append(g) return [join(axis, *new_g_outputs), grad_undefined(self, 1, axis), grad_undefined(self, 2, n)] def R_op(self, inputs, eval_points): if eval_points[0] is None: return [None for i in self.len_splits] return self.make_node(eval_points[0], *inputs[1:]).outputs def c_code_cache_version(self): return (2,) def c_support_code(self): return """ /* Return 1 if output has the correct shape. */ int split_output_shape_is_correct ( PyArrayObject* output, PyArrayObject* array_to_split, int axis_to_split, npy_intp split_size ) { return PyArray_NDIM(output) == PyArray_NDIM(array_to_split) && memcmp( PyArray_DIMS(output), PyArray_DIMS(array_to_split), axis_to_split * sizeof(npy_intp) ) == 0 && memcmp( PyArray_DIMS(output) + axis_to_split + 1, PyArray_DIMS(array_to_split) + axis_to_split + 1, (PyArray_NDIM(array_to_split) - axis_to_split - 1) * sizeof(npy_intp) ) == 0 && split_size == PyArray_DIM(output, axis_to_split); } """ def c_code(self, node, name, inputs, outputs, sub): if self.len_splits == 0: # There are no outputs, then nothing to do. return '' # outputs_pointers lists the addresses of the pointers to the outputs. outputs_pointers = '&' + (', &'.join(outputs)) x, axis, splits = inputs fail = sub['fail'] x_typenum = np.dtype(node.inputs[0].dtype).num x_itemsize = np.dtype(node.inputs[0].dtype).itemsize axis_dtype = node.inputs[1].type.dtype_specs()[1] splits_dtype = node.inputs[2].type.dtype_specs()[1] expected_splits_count = self.len_splits return """ int ndim = PyArray_NDIM(%(x)s); int axis = (int)(*(%(axis_dtype)s*)PyArray_GETPTR1(%(axis)s, 0)); int splits_count = PyArray_DIM(%(splits)s, 0); npy_intp len_along_axis, sum_of_splits = 0, current_split_length = 0, current_split_start = 0; npy_intp* split_dims = NULL; PyObject* split_view = NULL; npy_intp data_offset; int i; PyArrayObject** outputs[] = {%(outputs_pointers)s}; /* Check inputs. */ if (splits_count != %(expected_splits_count)s) { PyErr_Format(PyExc_ValueError, "Split: splits count (%%d) != expected count (%%d).", splits_count, %(expected_splits_count)s); %(fail)s } if (axis < 0) { axis += ndim; } if (axis < 0 || axis >= ndim) { PyErr_Format(PyExc_IndexError, "Split: invalid axis %%d for a %%d-D array.", axis, ndim); %(fail)s } len_along_axis = PyArray_DIM(%(x)s, axis); for (i = 0; i < splits_count; ++i) { current_split_length = (npy_intp)(*(%(splits_dtype)s*)PyArray_GETPTR1(%(splits)s, i)); if (current_split_length < 0) { PyErr_Format(PyExc_ValueError, "Split: you try to take a negative number (%%ld) of elements.", current_split_length); %(fail)s } sum_of_splits += current_split_length; } if (sum_of_splits != len_along_axis) { PyErr_Format(PyExc_ValueError, "Split: the splits sums to %%ld, expected %%ld.", sum_of_splits, len_along_axis); %(fail)s } /* Check outputs. */ split_dims = (npy_intp*) malloc(ndim * sizeof(npy_intp)); if (split_dims == NULL) { PyErr_NoMemory(); %(fail)s } memcpy(split_dims, PyArray_DIMS(%(x)s), ndim * sizeof(npy_intp)); for (i = 0; i < splits_count; ++i) { PyArrayObject** output = outputs[i]; current_split_length = (npy_intp) (* (%(splits_dtype)s*) PyArray_GETPTR1(%(splits)s, i)); if (*output == NULL || !split_output_shape_is_correct(*output, %(x)s, axis, current_split_length)) { Py_XDECREF(*output); split_dims[axis] = current_split_length; *output = (PyArrayObject*)PyArray_EMPTY(ndim, split_dims, %(x_typenum)s, PyArray_IS_F_CONTIGUOUS(%(x)s)); if (outputs == NULL) { PyErr_SetString(PyExc_RuntimeError, "Split: unable to allocate an output."); free(split_dims); %(fail)s } } } /* Compute split. */ for (i = 0; i < splits_count; ++i) { current_split_length = (npy_intp) (* (%(splits_dtype)s*) PyArray_GETPTR1(%(splits)s, i)); data_offset = PyArray_STRIDE(%(x)s, axis) * current_split_start; split_dims[axis] = current_split_length; split_view = PyArray_New(&PyArray_Type, ndim, split_dims, %(x_typenum)s, PyArray_STRIDES(%(x)s), PyArray_BYTES(%(x)s) + data_offset, %(x_itemsize)s, PyArray_FLAGS(%(x)s), NULL); if (split_view == NULL) { PyErr_SetString(PyExc_RuntimeError, "Split: unable to create a view for a split."); free(split_dims); %(fail)s } if (PyArray_CopyInto(*outputs[i], (PyArrayObject*)split_view) != 0) { PyErr_SetString(PyExc_RuntimeError, "Split: unable to copy a split view into the output."); Py_XDECREF(split_view); free(split_dims); %(fail)s } Py_XDECREF(split_view); current_split_start += current_split_length; } free(split_dims); """ % locals() def addbroadcast(x, *axes): """ Make the input broadcastable in the specified axes. For example, addbroadcast(x, 0) will make the first dimension of x broadcastable. When performing the function, if the length of x along that dimension is not 1, a ValueError will be raised. We apply the opt here not to pollute the graph especially during the gpu optimization Parameters ---------- x : tensor_like Input theano tensor. axis : an int or an iterable object such as list or tuple of int values The dimension along which the tensor x should be broadcastable. If the length of x along these dimensions is not 1, a ValueError will be raised. Returns ------- tensor A theano tensor, which is broadcastable along the specified dimensions. """ rval = Rebroadcast(*[(axis, True) for axis in axes])(x) return theano.tensor.opt.apply_rebroadcast_opt(rval) def unbroadcast(x, *axes): """ Make the input impossible to broadcast in the specified axes. For example, addbroadcast(x, 0) will make the first dimension of x broadcastable. When performing the function, if the length of x along that dimension is not 1, a ValueError will be raised. We apply the opt here not to pollute the graph especially during the gpu optimization Parameters ---------- x : tensor_like Input theano tensor. axis : an int or an iterable object such as list or tuple of int values The dimension along which the tensor x should be unbroadcastable. If the length of x along these dimensions is not 1, a ValueError will be raised. Returns ------- tensor A theano tensor, which is unbroadcastable along the specified dimensions. """ rval = Rebroadcast(*[(axis, False) for axis in axes])(x) return theano.tensor.opt.apply_rebroadcast_opt(rval) def patternbroadcast(x, broadcastable): """ Make the input adopt a specific broadcasting pattern. Broadcastable must be iterable. For example, patternbroadcast(x, (True, False)) will make the first dimension of x broadcastable and the second dimension not broadcastable, so x will now be a row. We apply the opt here not to pollute the graph especially during the gpu optimization. Parameters ---------- x : tensor_like Input theano tensor. broadcastable : an iterable object such as list or tuple of bool values A set of boolean values indicating whether a dimension should be broadcastable or not. If the length of x along these dimensions is not 1, a ValueError will be raised. Returns ------- tensor A theano tensor, which is unbroadcastable along the specified dimensions. """ rval = Rebroadcast(*[(i, broadcastable[i]) for i in xrange(len(broadcastable))])(x) return theano.tensor.opt.apply_rebroadcast_opt(rval) class Join(Op): """ Concatenate multiple `TensorVariable`s along some axis. The axis must be given as first argument. All tensors must have the same shape along all dimensions other than this axis. Of course, TensorVariable instances do not have a shape, so this error cannot be caught until runtime. See `perform()`. See Also -------- stack : For joins involving scalar values Examples -------- >>> x, y, z = tensor.matrix(), tensor.matrix(), tensor.matrix() >>> u = tensor.vector() >>> r = join(0, x, y, z) >>> c = join(1, x, y, z) >>> join(2, x, y, z) # WRONG: the axis has to be an index into the shape >>> join(0, x, u) # WRONG: joined tensors must have the same rank """ check_input = False __props__ = ("view",) def __init__(self, view=-1): self.view = view if view != -1: # since the first input is always the axis, the tensors # start from index 1. self.view_map = {0: [1 + view]} def __str__(self): if self.view == -1: return self.__class__.__name__ else: return "%s{%s}" % ( self.__class__.__name__, ", ".join("%s=%r" % (p, getattr(self, p)) for p in self.__props__)) def __setstate__(self, d): self.__dict__.update(d) if not hasattr(self, "view"): self.view = -1 def make_node(self, *axis_and_tensors): """ Parameters ---------- axis: an Int or integer-valued Variable tensors A variable number (but not zero) of tensors to concatenate along the specified axis. These tensors must have the same shape along all dimensions other than this axis. Returns ------- A symbolic Variable It has the same ndim as the input tensors, and the most inclusive dtype. """ axis, tensors = axis_and_tensors[0], axis_and_tensors[1:] if not tensors: raise ValueError('Cannot join an empty list of tensors') as_tensor_variable_args = [as_tensor_variable(x) for x in tensors] dtypes = [x.type.dtype for x in as_tensor_variable_args] out_dtype = scal.upcast(*dtypes) def output_maker(bcastable): return tensor(dtype=out_dtype, broadcastable=bcastable) return self._make_node_internal( axis, tensors, as_tensor_variable_args, output_maker) def _make_node_internal(self, axis, tensors, as_tensor_variable_args, output_maker): if not python_all(targs.type.ndim for targs in as_tensor_variable_args): raise TypeError('Join cannot handle arguments of dimension 0.' ' For joining scalar values, see @stack') # Handle single-tensor joins immediately. if len(as_tensor_variable_args) == 1: bcastable = list(as_tensor_variable_args[0].type.broadcastable) else: # When the axis is fixed, a dimension should be # broadcastable if at least one of the inputs is # broadcastable on that dimension (see justification below), # except for the axis dimension. # Initialize bcastable all false, and then fill in some trues with # the loops. bcastable = [False] * len( as_tensor_variable_args[0].type.broadcastable) ndim = len(bcastable) # Axis can also be a constant if not isinstance(axis, integer_types): try: # Note : `get_scalar_constant_value` returns a ndarray not # an int axis = int(get_scalar_constant_value(axis)) except NotScalarConstantError: pass if isinstance(axis, integer_types): # Basically, broadcastable -> length 1, but the # converse does not hold. So we permit e.g. T/F/T # joins, and if they fail at runtime they fail, but if # they don't then it means that the argument where # that broadcastable flag was False had length 1 along # this dimension, and therefore this dimension should # be broadcastable for the output. if axis < -ndim: raise IndexError("Join axis %d out of bounds [0, %d)" % (axis, ndim)) if axis < 0: axis += ndim for x in as_tensor_variable_args: for current_axis, bflag in enumerate(x.type.broadcastable): # Constant negative axis can no longer be negative at # this point. It safe to compare this way. if current_axis == axis: continue if bflag: bcastable[current_axis] = True try: bcastable[axis] = False except IndexError: raise ValueError('Join argument "axis" is out of range' ' (given input dimensions)') else: # When the axis may vary, no dimension can be guaranteed to be # broadcastable. bcastable = [False] * len( as_tensor_variable_args[0].type.broadcastable) if not python_all([x.ndim == len(bcastable) for x in as_tensor_variable_args[1:]]): raise TypeError("Join() can only join tensors with the same " "number of dimensions.") inputs = [as_tensor_variable(axis)] + list(as_tensor_variable_args) if inputs[0].type not in int_types: raise TypeError('Axis could not be cast to an integer type', axis, inputs[0].type, int_types) outputs = [output_maker(bcastable)] node = Apply(self, inputs, outputs) return node def perform(self, node, axis_and_tensors, out_): out, = out_ view = self.view axis, tensors = axis_and_tensors[0], axis_and_tensors[1:] # we check these tensors for being empty. if (view != -1) and np.all( [tensor.shape[axis] == 0 for tensor in tensors[0:view] + tensors[view + 1:]]): out[0] = tensors[view] else: ndim = tensors[0].ndim if axis < -ndim: raise IndexError("Join axis %d out of bounds [0, %d)" % (axis, ndim)) out[0] = theano._asarray(np.concatenate(tensors, axis=axis), dtype=node.outputs[0].type.dtype) def c_code_cache_version(self): return (5,) def c_code(self, node, name, inputs, outputs, sub): axis, tensors = inputs[0], inputs[1:] view = self.view non_empty_tensor = tensors[view] input_1 = tensors[0] l = len(tensors) out, = outputs fail = sub['fail'] adtype = node.inputs[0].type.dtype_specs()[1] copy_to_list = [] for i, inp in enumerate(tensors): copy_to_list.append( """Py_INCREF(%s); PyList_SetItem(list, %s, (PyObject*)%s);""" % (inp, i, inp)) copy_inputs_to_list = '\n'.join(copy_to_list) n = len(tensors) code = """ int axis = ((%(adtype)s *)PyArray_DATA(%(axis)s))[0]; PyObject* list = PyList_New(%(l)s); %(copy_inputs_to_list)s int tensors_lens_sum; if(%(view)s != -1) { tensors_lens_sum = 0; for(int i=0; i < %(n)s; i++){ tensors_lens_sum += PyArray_DIM((PyArrayObject *)(PyList_GetItem(list, i)), axis); } tensors_lens_sum -= PyArray_DIM(%(non_empty_tensor)s, axis); } if(%(view)s != -1 && tensors_lens_sum == 0) { Py_XDECREF(%(out)s); Py_INCREF(%(non_empty_tensor)s); %(out)s = %(non_empty_tensor)s; }else{ //PyObject* PyArray_Concatenate(PyObject* obj, int axis) int ndim = PyArray_NDIM(%(input_1)s); if( axis < -ndim ){ PyErr_Format(PyExc_IndexError, "Join axis %%d out of bounds [0, %%d)", axis, ndim); %(fail)s } Py_XDECREF(%(out)s); %(out)s = (PyArrayObject *)PyArray_Concatenate(list, axis); Py_DECREF(list); if(!%(out)s){ %(fail)s } } """ % locals() return code def R_op(self, inputs, eval_points): if None in eval_points[1:]: return [None] return self.make_node(inputs[0], *eval_points[1:]).outputs def grad(self, axis_and_tensors, grads): """ The gradient wrt a join op is a `Split`, used to partition the gradient along the `axis` which was used for joining. """ gz, = grads axis, tensors = axis_and_tensors[0], axis_and_tensors[1:] rval = [grad_undefined(self, 0, axis)] dtypes = [as_tensor_variable(x).type.dtype for x in tensors] out_dtype = scal.upcast(*dtypes) if 'float' in out_dtype or 'complex' in out_dtype: # assume that this is differentiable split = Split(len(tensors)) split_gz = split(gz, axis, stack([shape(x)[axis] for x in tensors])) # If there is only one split, it might not be in a list. if not isinstance(split_gz, list): split_gz = [split_gz] # Split.make_node isn't always able to infer the right # broadcast. As the grad need to keep the information, # read it if needed. split_gz = [patternbroadcast(g, t.broadcastable) for t, g in zip(tensors, split_gz)] rval = rval + split_gz else: # the output has integer type, so the gradient through it # is 0 rval = rval + [tensor.zeros_like(dtype=config.floatX) for tensor in tensors] return rval def infer_shape(self, node, ishapes): # ishapes[0] contains the size of the axis on which we join # Join op should get at least one input to join assert len(ishapes) > 1 n_dim = len(ishapes[1]) for shp in ishapes[1:]: assert shp is not None assert len(shp) == n_dim # The joining dimension could be negative, but we need it to be # in [0, n_dim) in the loop below. # An axis < -n_dim or >= ndim would be invalid, but this is # not checked here. An Assert op would be a way of addressing that, # but it may disrupt optimizations. join_dim = switch(ge(node.inputs[0], 0), node.inputs[0], node.inputs[0] + n_dim) out_shapes = [] for dim in xrange(n_dim): # we have to deal with 2 possible cases in here : # a) we are dealing with the dimension for which we join # (called t_side from true side of the if, where the if # compares current dimension with the joining dimension) # b) a non joining dimension ( in which maybe a symbolic # assertion can be used to make sure all tensors have # the same number of elements on this non-joined dimension # this is f_side # initialize t_side = ishapes[1][dim] f_side = ishapes[1][dim] # loop over tensors and sum for the joining dimension for shp in ishapes[2:]: t_side = t_side + shp[dim] # return the dimensions found out_shapes.append(switch(eq(dim, join_dim), t_side, f_side)) return [tuple(out_shapes)] join_ = Join() pprint.assign(Join, printing.FunctionPrinter('join')) def join(axis, *tensors_list): """ Convenience function to concatenate `TensorType`s along the given axis. This function will not add the op in the graph when it is not useful. For example, in the case that the list of tensors to be concatenated is one, it will just return the tensor. Parameters ---------- tensors : list of tensors (or list-like) A list of tensors to be concatenated along the given axis. The shapes of the tensors to be concatenated must be all identical, except in the dimension (`axis`) on which they are to be joined. axis : int (symbolic or literal) On which dimension should the tensors be joined? The `axis` must be a valid index into the shape of the tensors to be concatenated. The `axis` parameter may either be an integer or an object that can be converted to a scalar using `as_scalar`(`axis`). In the former case, the axis is fixed at construction, while in the latter it may vary over time depending on the value of the `axis` variable. """ if len(tensors_list) == 1: return tensors_list[0] else: return join_(axis, *tensors_list) def roll(x, shift, axis=None): """ Convenience function to roll TensorTypes along the given axis. Syntax copies numpy.roll function. Parameters ---------- x : tensor_like Input tensor. shift : int (symbolic or literal) The number of places by which elements are shifted. axis : int (symbolic or literal), optional The axis along which elements are shifted. By default, the array is flattened before shifting, after which the original shape is restored. Returns ------- tensor Output tensor, with the same shape as ``x``. """ if axis is None: if x.ndim > 1: y = x.flatten() return roll(y, shift, axis=0).reshape(x.shape) else: axis = 0 if axis < 0: axis += x.ndim # Shift may be larger than the size of the axis. If so, since the # roll operation is cyclic, we can take the shift modulo the size # of the axis shift = shift % x.shape[axis] # A slice of all elements in a dimension ':' allslice = slice(None) # List of slices describing the front half [:, :, shift:, :] front_slice = slice(-shift, None) front_list = ([allslice] * axis + [front_slice] + [allslice] * (x.ndim - axis - 1)) # List of slices describing the back half [:, :, :shift, :] end_slice = slice(0, -shift) end_list = ([allslice] * axis + [end_slice] + [allslice] * (x.ndim - axis - 1)) return join(axis, x.__getitem__(tuple(front_list)), x.__getitem__(tuple(end_list))) @constructor def shape_padleft(t, n_ones=1): """Reshape `t` by left-padding the shape with `n_ones` 1s. See Also -------- shape_padaxis shape_padright Dimshuffle """ _t = as_tensor_variable(t) pattern = ['x'] * n_ones + [i for i in xrange(_t.type.ndim)] return DimShuffle(_t.broadcastable, pattern)(_t) @constructor def shape_padright(t, n_ones=1): """Reshape `t` by right-padding the shape with `n_ones` 1s. See Also -------- shape_padaxis shape_padleft Dimshuffle """ _t = as_tensor_variable(t) pattern = [i for i in xrange(_t.type.ndim)] + ['x'] * n_ones return DimShuffle(_t.broadcastable, pattern)(_t) @constructor def shape_padaxis(t, axis): """Reshape `t` by inserting 1 at the dimension `axis`. Example ------- >>> tensor = theano.tensor.tensor3() >>> theano.tensor.shape_padaxis(tensor, axis=0) DimShuffle{x,0,1,2}.0 >>> theano.tensor.shape_padaxis(tensor, axis=1) DimShuffle{0,x,1,2}.0 >>> theano.tensor.shape_padaxis(tensor, axis=3) DimShuffle{0,1,2,x}.0 >>> theano.tensor.shape_padaxis(tensor, axis=-1) DimShuffle{0,1,2,x}.0 See Also -------- shape_padleft shape_padright Dimshuffle """ _t = as_tensor_variable(t) ndim = _t.ndim + 1 if not -ndim <= axis < ndim: msg = 'axis {0} is out of bounds [-{1}, {1})'.format(axis, ndim) raise IndexError(msg) if axis < 0: axis += ndim pattern = [i for i in xrange(_t.type.ndim)] pattern.insert(axis, 'x') return DimShuffle(_t.broadcastable, pattern)(_t) @constructor def stack(*tensors, **kwargs): """Stack tensors in sequence on given axis (default is 0). Take a sequence of tensors and stack them on given axis to make a single tensor. The size in dimension `axis` of the result will be equal to the number of tensors passed. Note: The interface stack(*tensors) is deprecated, you should use stack(tensors, axis=0) insted. Parameters ---------- tensors : list or tuple of tensors A list of tensors to be stacked. axis : int The index of the new axis. Default value is 0. Examples -------- >>> a = theano.tensor.scalar() >>> b = theano.tensor.scalar() >>> c = theano.tensor.scalar() >>> x = theano.tensor.stack([a, b, c]) >>> x.ndim # x is a vector of length 3. 1 >>> a = theano.tensor.tensor4() >>> b = theano.tensor.tensor4() >>> c = theano.tensor.tensor4() >>> x = theano.tensor.stack([a, b, c]) >>> x.ndim # x is a 5d tensor. 5 >>> rval = x.eval(dict((t, np.zeros((2, 2, 2, 2))) for t in [a, b, c])) >>> rval.shape # 3 tensors are stacked on axis 0 (3, 2, 2, 2, 2) >>> x = theano.tensor.stack([a, b, c], axis=3) >>> x.ndim 5 >>> rval = x.eval(dict((t, np.zeros((2, 2, 2, 2))) for t in [a, b, c])) >>> rval.shape # 3 tensors are stacked on axis 3 (2, 2, 2, 3, 2) >>> x = theano.tensor.stack([a, b, c], axis=-2) >>> x.ndim 5 >>> rval = x.eval(dict((t, np.zeros((2, 2, 2, 2))) for t in [a, b, c])) >>> rval.shape # 3 tensors are stacked on axis -2 (2, 2, 2, 3, 2) """ # ---> Remove this when moving to the new interface: if not tensors and not kwargs: raise Exception('theano.tensor.stack(tensors, axis) must have at least' ' one parameter') if not kwargs and not isinstance(tensors[0], (list, tuple)): warnings.warn('stack(*tensors) interface is deprecated, use' ' stack(tensors, axis=0) instead.', DeprecationWarning, stacklevel=3) axis = 0 elif 'tensors' in kwargs: tensors = kwargs['tensors'] if 'axis' in kwargs: axis = kwargs['axis'] else: axis = 0 else: if len(tensors) == 2: axis = tensors[1] elif 'axis' in kwargs: axis = kwargs['axis'] else: axis = 0 tensors = tensors[0] # <--- Until here. if len(tensors) == 0: raise Exception('tensors is empty. You should at least provide one' ' tensor to theano.tensor.stack(tensors, axis).') # If all tensors are scalars of the same type, call make_vector. # It makes the graph simpler, by not adding DimShuffles and Rebroadcasts # This should be an optimization! # Doing it here make the graph less canonicalized # (more type need to be understood by all optimization) # And DebugMode can't detect error in this code as it is not in an # optimization. # See ticket #660 if np.all( [ # in case there is direct int in tensors. isinstance(t, (np.number, float, integer_types, python_complex)) or (isinstance(t, Variable) and isinstance(t.type, TensorType) and t.ndim == 0) for t in tensors]): # in case there is direct int tensors = list(map(as_tensor_variable, tensors)) dtype = scal.upcast(*[i.dtype for i in tensors]) return theano.tensor.opt.MakeVector(dtype)(*tensors) return join(axis, *[shape_padaxis(t, axis) for t in tensors]) @constructor def concatenate(tensor_list, axis=0): """Alias for `join`(axis, *tensor_list). This function is similar to `join`, but uses the signature of numpy's concatenate function. Raises ------ TypeError The tensor_list must be a tuple or list. """ # Check someone did not make the common mistake to do something like: # c = concatenate(x, y) # instead of # c = concatenate((x, y)) if not isinstance(tensor_list, (tuple, list)): raise TypeError( "The 'tensors' argument must be either a tuple " "or a list, make sure you did not forget () or [] around " "arguments of concatenate.", tensor_list) return join(axis, *tensor_list) def get_vector_length(v): """Return the run-time length of a symbolic vector. Parameters ---------- v A rank-1 TensorType variable. Raises ------ TypeError `v` hasn't the proper type. ValueError No special case applies, the length is not known. In general this is not possible, but for a number of special cases the length can be determined at compile / graph-construction time. This function implements these special cases. """ v = as_tensor_variable(v) if v.ndim != 1: raise TypeError("argument must be symbolic vector, got '%s'" % v) if v.type.broadcastable[0]: return 1 if isinstance(v, gof.Constant) and v.type.ndim == 1: return len(v.data) if v.owner and isinstance(v.owner.op, theano.tensor.opt.MakeVector): return len(v.owner.inputs) if v.owner and isinstance(v.owner.op, Shape): return v.owner.inputs[0].type.ndim # If we take a slice, we know how many elements it will result in if ((v.owner and isinstance(v.owner.op, theano.tensor.subtensor.Subtensor) and isinstance(v.owner.op.idx_list[0], slice) and v.owner.inputs[0].owner and isinstance(v.owner.inputs[0].owner.op, theano.compile.ops.Shape))): start = extract_constant(theano.tensor.subtensor.get_idx_list( v.owner.inputs, v.owner.op.idx_list)[0].start) stop = extract_constant(theano.tensor.subtensor.get_idx_list( v.owner.inputs, v.owner.op.idx_list)[0].stop) step = extract_constant(theano.tensor.subtensor.get_idx_list( v.owner.inputs, v.owner.op.idx_list)[0].step) ndim = v.owner.inputs[0].owner.inputs[0].ndim types = (numbers.Integral, np.integer) if start is None: start = 0 elif isinstance(start, types) and start < 0: start += ndim if start < 0: start = 0 if stop is None: stop = ndim elif isinstance(stop, types): if stop > ndim: stop = ndim elif stop < 0: stop += ndim if step is None: step = 1 if (isinstance(stop, types) and isinstance(start, types) and isinstance(step, types) and start >= 0 and stop >= 0 and step > 0 and stop >= start): return (stop - start - 1) // step + 1 if isinstance(v, Variable): msg = theano.printing.debugprint(v, file='str') else: msg = str(v) raise ValueError("length not known: %s" % msg) @constructor def horizontal_stack(*args): """ Horizontally stack two L{TensorType}s. Stack two L{TensorType}s along the second axis (column wise). These L{TensorType}s must have the same shape along all dimensions but the second. """ # Note: 'horizontal_stack' and 'vertical_stack' do not behave exactly like # Numpy's hstack and vstack functions. This is intended, because Numpy's # functions have potentially confusing/incoherent behavior (try them on 1D # arrays). If this is fixed in a future version of Numpy, it may be worth # trying to get closer to Numpy's way of doing things. In the meantime, # better keep different names to emphasize the implementation divergences. assert len(args) >= 2 for arg in args: assert arg.type.ndim == 2 return concatenate(args, axis=1) @constructor def vertical_stack(*args): assert len(args) >= 2 for arg in args: assert arg.type.ndim == 2 return concatenate(args, axis=0) class Reshape(Op): """Perform a reshape operation of the input x to the new shape shp. The number of dimensions to which to reshape to (ndim) must be known at graph build time. """ view_map = {0: [0]} # output 0 is potentially aliased to inputs [0] _f16_ok = True check_input = False __props__ = ("ndim",) params_type = ParamsType(ndim=int32) # name does not participate because it doesn't affect computations def __init__(self, ndim, name=None): self.ndim = int(ndim) if ndim < 0: raise ValueError("The output dimensions after reshape must be 0 or greater") assert name is None, 'name attribute for Reshape has been deprecated' def __str__(self): return '%s{%s}' % (self.__class__.__name__, self.ndim) def make_node(self, x, shp): x = as_tensor_variable(x) shp_orig = shp shp = as_tensor_variable(shp, ndim=1) if not (shp.dtype in int_dtypes or (isinstance(shp, TensorConstant) and shp.data.size == 0)): # It raises an error if shp is not of integer type, # except when shp is constant and empty # (in this case, shp.dtype does not matter anymore). raise TypeError("Shape must be integers", shp, shp.dtype) assert shp.ndim == 1 if isinstance(shp, TensorConstant): bcast = [s == 1 for s in shp.data] return gof.Apply(self, [x, shp], [tensor(x.type.dtype, bcast)]) else: bcasts = [False] * self.ndim shp_list = shp_orig if hasattr(shp_orig, "ndim") and shp_orig.ndim == 0: shp_list = [shp_orig] for index in xrange(self.ndim): y = shp_list[index] y = as_tensor_variable(y) # Try to see if we can infer that y has a constant value of 1. # If so, that dimension should be broadcastable. try: bcasts[index] = ( hasattr(y, 'get_scalar_constant_value') and y.get_scalar_constant_value() == 1) except NotScalarConstantError: pass return gof.Apply(self, [x, shp], [tensor(x.type.dtype, bcasts)]) def perform(self, node, inp, out_, params): x, shp = inp out, = out_ if (len(shp) != self.ndim): raise ValueError('shape argument to Reshape.perform has incorrect' ' length %i' ', should be %i' % (len(shp), self.ndim), shp) try: out[0] = np.reshape(x, shp) except Exception: raise ValueError('Cannot reshape input of shape %s to shape %s' % (x.shape, shp)) def connection_pattern(self, node): return [[True], [False]] def grad(self, inp, grads): x, shp = inp g_out, = grads return [reshape(g_out, shape(x), ndim=x.ndim), DisconnectedType()()] def R_op(self, inputs, eval_points): if eval_points[0] is None: return [None] return self(eval_points[0], *inputs[1:], **dict(return_list=True)) def infer_shape(self, node, ishapes): # inputs[1] can contain at most one value of '-1', meaning the actual # shape of the output will be automatically computed by reshape, so # that the total number of elements stays the same. # TODO: Maybe put that formula here? # It's not trivial, because we would have to check if the product of # all the non-minus-one shapes is a divisor of the product of the # original shapes. # The following expression leads to cycles in feature_shape, # because it tries to replace the Shape_i node by the switch # statement, which depends on Shape_i. # return [tuple([switch(eq(node.inputs[1][i], -1), # theano.tensor.opt.Shape_i(i)(node.outputs[0]), # node.inputs[1][i]) # for i in xrange(self.ndim)] # )] # Here, we only simplify if the shape (node.inputs[1]) is a constant, # ideally it would suffice to check that it is always non-negative. # If current variable is a scalar and its dimensionality should # change to self.ndim, then use size 1 for all new dimensions. if len(ishapes[0]) == 0: return [(1,) * self.ndim] requ = node.inputs[1] input_size = mul(*ishapes[0]) if isinstance(requ, theano.tensor.TensorConstant): requ = list(requ.data) requ_part = [ele for ele in requ if ele != -1] crit = len(requ) - len(requ_part) if crit == 1 and len(requ_part) > 0: # If there are both 0 and -1 in requ_size, it is impossible # to determine a right output, but we can at least prevent # a division by 0. We do not want to keep a negative # size here as it could lead to further weird errors # after other optimizations. requ_size = mul(*requ_part) missing = input_size // (1 if requ_size == 0 else requ_size) for i, ele in enumerate(requ): if ele == -1: requ[i] = missing elif crit == 1: # we reshape to -1 requ = [input_size] if ishapes[0] else [1] elif crit > 1: raise ValueError('shape argument to Reshape.perform' ' must have at most one entry equal to -1') return [requ] else: requ = [requ[i] for i in xrange(self.ndim)] # since new_dims can have negative value (-1), the # multiplication of all values should be negated # to give a positive value. # To avoid optimization complexity, we avoid checking # for the case when there are two or more '-1' values. if self.ndim: requ_size = -mul(*requ) # If there are both 0 and -1 in requ_size, it is impossible # to determine a right output, but we can at least prevent # a division by 0. We do not want to keep a negative # size here as it could lead to further weird errors # after other optimizations. rest_size = input_size // maximum(requ_size, 1) return [tuple([switch(eq(requ[i], -1), rest_size, requ[i]) for i in xrange(self.ndim)])] def c_code_cache_version(self): return (8,) def c_code(self, node, name, inputs, outputs, sub): if isinstance(node.inputs[0], TensorVariable): x, shp = inputs z, = outputs sdtype = node.inputs[1].type.dtype_specs()[1] fail = sub['fail'] params = sub['params'] return """ assert (PyArray_NDIM(%(shp)s) == 1); npy_intp new_dims[%(params)s->ndim]; PyArray_Dims newshape; newshape.ptr = new_dims; newshape.len = %(params)s->ndim; for (int ii = 0; ii < %(params)s->ndim; ++ii) { // -- We do not want an explicit cast here. the shp can be any // -- int* dtype. The compiler will explicitly upcast it, but // -- will err if this will downcast. This could happen if the // -- user pass an int64 dtype, but npy_intp endup being int32. new_dims[ii] = ((%(sdtype)s*)( PyArray_BYTES(%(shp)s) + ii * PyArray_STRIDES(%(shp)s)[0]))[0]; } Py_XDECREF(%(z)s); %(z)s = (PyArrayObject *) PyArray_Newshape(%(x)s, &newshape, NPY_CORDER); if (!%(z)s) { //The error message should have been set by PyArray_Newshape %(fail)s; } """ % locals() else: return Op.c_code(self, node, name, inputs, outputs, sub) def reshape(x, newshape, ndim=None): if ndim is None: newshape = as_tensor_variable(newshape) if newshape.ndim != 1: raise TypeError( "New shape in reshape must be a vector or a list/tuple of" " scalar. Got %s after conversion to a vector." % newshape) try: ndim = get_vector_length(newshape) except ValueError: raise ValueError( "The length of the provided shape (%s) cannot " "be automatically determined, so Theano is not able " "to know what the number of dimensions of the reshaped " "variable will be. You can provide the 'ndim' keyword " "argument to 'reshape' to avoid this problem." % newshape) op = Reshape(ndim) rval = op(x, newshape) return rval class Flatten(Op): """ Flatten a tensor. Flattens a tensor to `outdim` dimensions by preserving the leading outdim - 1 shape components. .. note:: The interface Flatten(Op) is deprecated, you should use flatten. """ view_map = {0: [0]} check_input = False __props__ = ("outdim",) def __init__(self, outdim=1): warnings.warn( "Flatten class is deprecated, " "please use flatten method instead.", DeprecationWarning, stacklevel=4) self.outdim = int(outdim) def __str__(self): return '%s{%s}' % (self.__class__.__name__, self.outdim) def make_node(self, x): t_x = as_tensor_variable(x) if self.outdim < 1 or (x.ndim and self.outdim > x.ndim): raise ValueError('invalid output ndimensions (%i) for tensor of ' 'rank %i' % (self.outdim, t_x.ndim)) # Infer the broadcastable pattern of the output. For every dimension # unaffected by the flatten, the broadcast flag should be unchanged. # For the dimension resulting from the collapse of other dimensions, # it should be broadcastable iff all the collapsed dimensions were # broadcastable. bcast_kept_dims = x.broadcastable[:self.outdim - 1] bcast_new_dim = python_all(x.broadcastable[self.outdim - 1:]) broadcastable = bcast_kept_dims + (bcast_new_dim,) return gof.Apply(self, [t_x], [tensor(x.type.dtype, broadcastable)]) def perform(self, node, inp, out_): x, = inp out, = out_ outdim = self.outdim if outdim == 1: try: out[0] = x.reshape(x.size) except AttributeError: out[0] = x.reshape((np.prod(x.shape),)) elif outdim == len(x.shape): out[0] = x else: newshape = (x.shape[:outdim - 1] + (np.prod(x.shape[outdim - 1:]),)) out[0] = x.reshape(newshape) def infer_shape(self, node, in_shapes): in_shp, = in_shapes part1 = in_shp[:self.outdim - 1] part2 = in_shp[self.outdim - 1:] if len(part2) > 1: part2 = (prod(part2, dtype='int64'),) elif len(part2) == 1: # We do not want to force an upcast of part2 if its length is 1 pass else: if len(in_shp) == 0 and self.outdim == 1: part2 = (1,) else: raise ValueError('invalid output ndimensions (%i) for tensor ' 'of rank %i' % (self.outdim, len(in_shp))) out_shape = (part1 + part2) return [out_shape] def grad(self, inp, grads): x, = inp g_out, = grads return [reshape(g_out, shape(x), x.ndim)] def R_op(self, inputs, eval_points): if None in eval_points: return [None] return self.make_node(*eval_points).outputs def c_code_cache_version(self): return (1, 1) def c_code(self, node, name, inputs, outputs, sub): x, = inputs out, = outputs outdim = self.outdim fail = sub['fail'] return """ if (%(outdim)s == PyArray_NDIM(%(x)s)) { Py_XDECREF(%(out)s); Py_XINCREF(%(x)s); %(out)s = %(x)s; } else { Py_XDECREF(%(out)s); if (%(outdim)s == 1) { npy_intp size = PyArray_SIZE(%(x)s); PyArray_Dims newshape; newshape.ptr = &size; newshape.len = 1; %(out)s = (PyArrayObject*)PyArray_Newshape(%(x)s, &newshape, NPY_CORDER); } else { npy_intp *oldshape = PyArray_DIMS(%(x)s); npy_intp newshape_dims[%(outdim)s]; int i; for (i = 0; i < %(outdim)s - 1; ++i) newshape_dims[i] = oldshape[i]; newshape_dims[i] = 1; for (int j = %(outdim)s - 1; j < PyArray_NDIM(%(x)s); ++j) newshape_dims[i] *= oldshape[j]; PyArray_Dims newshape; newshape.ptr = newshape_dims; newshape.len = %(outdim)s; %(out)s = (PyArrayObject*)PyArray_Newshape(%(x)s, &newshape, NPY_CORDER); } } if (!%(out)s) { //The error message should have been set by // PyArray_Newshape %(fail)s; } """ % locals() def is_flat(var, ndim=None, outdim=None): """ Verifies the dimensionality of the var is equal to outdim. This method is usually called after flatten method on a variable, where the first outdim-1 dimension size(s) of the variable is kept intact, and the last dimension size of the variable is made equal to the multiplication of its remaining dimension size(s), such that the variable would end up with as many dimension as outdim. Parameters ---------- var : theano.tensor.var.TensorVariable the theano var on which the dimensionality is checked. outdim : int the expected dimensionality of var. Returns ------- bool the comparison result of var's dim and the expected outdim. """ if outdim is None and ndim is None: ndim = 1 elif outdim is not None and ndim is not None: raise ValueError("You should only specify ndim") elif outdim is not None: warnings.warn( "flatten outdim parameter is deprecated, use ndim instead.") ndim = outdim return var.ndim == ndim def flatten(x, ndim=None, outdim=None): """ Reshapes the variable x by keeping the first outdim-1 dimension size(s) of x the same, and making the last dimension size of x equal to the multiplication of its remaining dimension size(s). Parameters ---------- x : theano.tensor.var.TensorVariable the variable that should be reshaped. ndim : int the number of dimensions of the returned variable Default 1. outdim : int DEPRECATED synonym for ndim Returns ------- theano.tensor.var.TensorVariable the flattend variable with dimensionality of outdim """ if outdim is None and ndim is None: ndim = 1 elif outdim is not None and ndim is not None: raise ValueError("You should only specify ndim") elif outdim is not None: warnings.warn( "flatten outdim parameter is deprecated, use ndim instead.") ndim = outdim # Any input variable can be flattened to have ndim of 1, # even if it's a scalar. Otherwise, ndim must be positive # and smaller than x.ndim. if ndim < 1 or (ndim > 1 and ndim > x.ndim): raise ValueError('ndim %s out of bound [1, %d)' % (ndim, x.ndim + 1)) if ndim > 1: dims = tuple(x.shape[:ndim - 1]) + (-1,) else: dims = (-1,) x_reshaped = x.reshape(dims) bcast_kept_dims = x.broadcastable[:ndim - 1] bcast_new_dim = python_all(x.broadcastable[ndim - 1:]) broadcastable = bcast_kept_dims + (bcast_new_dim,) x_reshaped = theano.tensor.addbroadcast( x_reshaped, *filter(lambda i: broadcastable[i], range(ndim))) return x_reshaped # class TileGrad(Op): # """ # Calculates the gradient of the Tile Op. # """ # # this is so weird, I can't think of how to make this a general thing. # def make_node(self, x, reps, g_out): # return gof.Apply(self, [x, reps, g_out], [x.type()]) # # def perform(self, node, inp, out): # x, reps, g_out = inp # gx, = out # xsh = x.shape # if len(reps) == 2 and reps[1] == 1 and len(x.shape) == 1: # gx[0] = numpy.sum(g_out, axis=0) # else: # raise NotImplementedError('x.shape, reps combination not ' # 'supported', (x.shape, reps)) # # tilegrad = TileGrad() class Tile(Op): """ Construct an array by repeating the input x according to reps pattern. .. note:: Deprecated Use tile() instead. Tiles its input according to reps. The length of reps is the number of dimension of x and contains the number of times to tile x in each dimension. See Also -------- numpy.tile : http://docs.scipy.org/doc/numpy/reference/generated/numpy.tile.html """ __props__ = ("ndim",) def __init__(self, ndim): self.ndim = ndim def __str__(self): return self.__class__.__name__ + "{ndim=%d}" % self.ndim def make_node(self, x, reps): warnings.warn(( "Tile op is deprecated, use tile function instead."), stacklevel=3) x = as_tensor_variable(x) reps = as_tensor_variable(reps) return gof.Apply(self, [x, reps], [tensor(x.type.dtype, [False] * self.ndim)]) def perform(self, node, inp, out_): x, reps = inp out, = out_ res = np.tile(x, reps) if res.ndim != self.ndim: raise ValueError( 'Tile.perform produced incorrect number of dimensions') if (np.asarray(reps) == 1).all(): # In that case, some NumPy version return a view! As this # op isn't declared as inplace, we need to check that and # copy the data. if np.may_share_memory(res, x): res = res.copy() out[0] = res def infer_shape(self, node, in_shapes): # Note: in contrast with numpy, it is assumed that x.shape and reps # have equal length; see also tile function below # Note: if reps were to be allowed not to be a constant and x.shape # and reps to be unequal, the following block of code could be used: # prepend 1 to x.shape if needed # if self.ndim > x.ndim: # shp = concatenate(ones(self.ndim - x.ndim), shp) # prepend 1 to reps if needed # reps = concatenate(ones(self.ndim - reps.shape[0]), reps) x, reps = node.inputs shp = in_shapes[0] tiled_shp = shp * reps out_shape = [] for i in xrange(self.ndim): out_shape.append(tiled_shp[i]) return [out_shape] def grad(self, inp, grads): x, reps = inp g_out, = grads # return [tilegrad(x, reps, g_out), None] raise NotImplementedError() def tile(x, reps, ndim=None): """ Tile input array `x` according to `reps`. See the docstring of `numpy.tile` for details. 'reps' can be constant integer (e.g. 3), constant vector(e.g. [2 3]), symbolic scalar (e.g. tensor.iscalar()), symbolic vector (e.g. tensor.ivector()) or a list of symbolic scalar (e.g. [tensor.iscalar(), tensor.iscalar()]). ndim is the number of the dimensions of the output, if it is provided, ndim should be equal or larger than x.ndim and len(reps), otherwise, we will use max(x.ndim, len(reps)) as ndim. If reps is symbolic vector, the ndim has to be provided. """ if ndim is not None and ndim < x.ndim: raise ValueError("ndim should be equal or larger than x.ndim") # if reps is tensor.scalar, integer or tensor.vector, we convert it to a list. if not isinstance(reps, (list, tuple)): reps_astensor = as_tensor_variable(reps) ndim_check = reps_astensor.ndim if reps_astensor.dtype not in theano.tensor.discrete_dtypes: raise ValueError("elements of reps must be integer dtype") # tensor.scalar/integer case if ndim_check == 0: reps = [reps] # tensor.vector case elif ndim_check == 1: if ndim is None: raise ValueError("if reps is tensor.vector, you should specify " "the ndim") else: offset = ndim - reps.shape[0] # assert that reps.shape[0] does not exceed ndim offset = theano.tensor.opt.assert_(offset, ge(offset, 0)) # if reps.ndim is less than x.ndim, we pad the reps with # "1" so that reps will have the same ndim as x. reps_ = [switch(i < offset, 1, reps[i - offset]) for i in range(ndim)] reps = reps_ # other raise error else: raise ValueError("the dimension of reps should not exceed 1") else: if ndim is not None and len(reps) > ndim: raise ValueError("len(reps) should be equal or less than ndim") if not np.all([isinstance(r, integer_types) or (isinstance(r, TensorVariable) and r.dtype in theano.tensor.discrete_dtypes) for r in reps]): raise ValueError("elements of reps must be scalars of integer dtype") # if reps.ndim is less than x.ndim, we pad the reps with # "1" so that reps will have the same ndim as x. reps = list(reps) if ndim is None: ndim = builtins.max(len(reps), x.ndim) if len(reps) < ndim: reps = [1] * (ndim - len(reps)) + reps shape = [1] * (ndim - x.ndim) + [x.shape[i] for i in xrange(x.ndim)] alloc_shape = reps + shape y = alloc(x, *alloc_shape) shuffle_ind = np.arange(ndim * 2).reshape(2, ndim) shuffle_ind = shuffle_ind.transpose().flatten() y = y.dimshuffle(*shuffle_ind) new_shapes = [sh * reps[i] for i, sh in enumerate(shape)] y = y.reshape(new_shapes) return y class ARange(Op): """Create an array containing evenly spaced values within a given interval. Parameters and behaviour are the same as numpy.arange(). """ __props__ = ("dtype",) def __init__(self, dtype): self.dtype = dtype def make_node(self, start, stop, step): start, stop, step = map(as_tensor_variable, (start, stop, step)) assert start.ndim == 0 assert stop.ndim == 0 assert step.ndim == 0 inputs = [start, stop, step] outputs = [tensor(self.dtype, (False,))] return Apply(self, inputs, outputs) @theano.configparser.change_flags(warn_float64='ignore') def infer_shape(self, node, i_shapes): # Note start, stop and step can be float numbers. start, stop, step = node.inputs def is_constant_value(var, value): try: v = get_scalar_constant_value(var) return np.all(v == value) except NotScalarConstantError: pass return False def upcast(var): if (var.dtype in integer_dtypes and # We do not want to cast uint64 to int64 as this can # loose information. If we upcast uint64 with int64, # this give float64. This is safer then checking for # uint64 in case we support [u]int128 or other in the # future. scal.upcast(var.dtype, 'int64') == 'int64'): return cast(var, 'int64') return var if is_constant_value(step, 1): if is_constant_value(start, 0): return [(cast(stop, 'int64'),)] else: stop = upcast(stop) start = upcast(start) return [(maximum(cast(stop - start, 'int64'), 0),)] else: stop = upcast(stop) start = upcast(start) return [(maximum(cast(ceil(cast((stop - start), 'float64') / step), 'int64'), 0),)] def perform(self, node, inp, out_): start, stop, step = inp out, = out_ start = start.item() stop = stop.item() step = step.item() out[0] = np.arange(start, stop, step, dtype=self.dtype) def connection_pattern(self, node): return [[True], [False], [True]] def L_op(self, inputs, outputs, grads): start, stop, step = inputs gz, = grads # `start` and `step` affect the output values # but the outputs are integers so there's # no gradient through them. # When they are not integers, the gradients are # as expressed below. # `stop` does not affect the output values, # just the output shape, so it is disconnected. if self.dtype in discrete_dtypes: return [start.zeros_like(dtype=config.floatX), DisconnectedType()(), step.zeros_like(dtype=config.floatX)] else: num_steps_taken = outputs[0].shape[0] return [gz.sum(), DisconnectedType()(), (gz * arange(num_steps_taken, dtype=self.dtype)).sum()] def R_op(self, inputs, eval_points): return [None] _arange = {} def arange(start, stop=None, step=1, dtype=None): # If only one argument is provided, it is in fact the "stop" argument, # and start is 0. if stop is None: start, stop = 0, start start, stop, step = map(as_tensor_variable, (start, stop, step)) # If dtype is not provided, infer it from the other arguments if dtype is None: dtype = scal.upcast(start.type.dtype, stop.type.dtype, step.type.dtype) # don't try to be stingy and byte-optimize, this leads to # overflow problems. if dtype in int_dtypes: dtype = 'int64' if dtype in uint_dtypes: dtype = 'uint64' if config.cast_policy in ('numpy', 'numpy+floatX'): # We enforce numpy semantics, except in the special case where # `config.cast_policy` is 'numpy+floatX' and we want to use float32 # rather than float64. # As an example, if `start`, `stop` and `step` are all int32, # `numpy.arange` returns an int64 array (on 64-bit platforms), # while the upcast above returns int32. numpy_dtype = np.arange( start=np.array(0, dtype=start.dtype), stop=np.array(1, dtype=stop.dtype), step=np.array(1, dtype=step.dtype)).dtype if numpy_dtype != dtype: if (config.cast_policy == 'numpy+floatX' and config.floatX == 'float32' and numpy_dtype == 'float64' and # No explicit float64 in the three arguments? python_all( dt != 'float64' for dt in [s.dtype for s in (start, stop, step)])): # We use float32 instead. assert dtype != 'float64' dtype = 'float32' else: # We use the same dtype as numpy instead of the result of # the upcast. dtype = str(numpy_dtype) if dtype not in _arange: _arange[dtype] = ARange(dtype) return _arange[dtype](start, stop, step) class _nd_grid(object): """Create a dense n-dimensional 'meshgrid' with equally spaced points. Used to create the instance ``mgrid`` and ``ogrid`` which act similarly to their numpy equivalents. Parameters ---------- sparse : boolean, optional, default=True Specifying False leads to the equivalent of numpy's mgrid functionality. Specifying True leads to the equivalent of ogrid. Examples -------- >>> a = T.mgrid[0:5, 0:3] >>> a[0].eval() array([[0, 0, 0], [1, 1, 1], [2, 2, 2], [3, 3, 3], [4, 4, 4]], dtype=int8) >>> a[1].eval() array([[0, 1, 2], [0, 1, 2], [0, 1, 2], [0, 1, 2], [0, 1, 2]], dtype=int8) >>> b = T.ogrid[0:5, 0:3] >>> b[0].eval() array([[0], [1], [2], [3], [4]], dtype=int8) >>> b[1].eval() array([[0, 1, 2, 3]], dtype=int8) """ def __init__(self, sparse=False): self.sparse = sparse def __getitem__(self, *args): ndim = len(args[0]) for sl in args[0]: if isinstance(sl.step, python_complex): raise NotImplementedError("Not implemented for slices " "whose step is complex") ranges = [arange(sl.start or 0, sl.stop, sl.step or 1) for sl in args[0]] shapes = [tuple([1] * j + [r.shape[0]] + [1] * (ndim - 1 - j)) for j, r in enumerate(ranges)] ranges = [r.reshape(shape) for r, shape in zip(ranges, shapes)] if self.sparse: grids = ranges else: grids = [] ones = [ones_like(r) for r in ranges] for i in range(ndim): grid = 1 for j in range(ndim): if j == i: grid = grid * ranges[j] else: grid = grid * ones[j] grids.append(grid) return grids mgrid = _nd_grid() ogrid = _nd_grid(sparse=True) class PermuteRowElements(Op): """Permute the elements of each row (inner-most dim) of a tensor. A permutation will be applied to every row (vector) of the input tensor x. Depending on the dimensionality of x and the permutation tensor y, different cases are possible. If y.ndim = 1, y is a single permutation, that will be applied to every vector of x. For instance, if x is a matrix, the same permutation will be applied to each row of x. If x.ndim = y.ndim, each row of x corresponds to a row of y, containing a permutation that will be applied to that row. For instance, if x and y are two matrices, a different permutation will be applied to each row of x. If x.ndim > y.ndim, y will be broadcasted to fit x, then each row (vector) of x will be reordered according to the corresponding row of y. (This is a generalization of the first case). If x.ndim = 1, every permutation in y will be applied to x, and the output will contain all the results. If x.ndim < y.ndim, x will be broadcasted to fit y, and different permutations contained in y will be applied to each vector in x. (This is a generalization of the previous case). If the "inverse" argument is True, the Op will perform the inverse permutation instead. """ __props__ = () def make_node(self, x, y, inverse): x = as_tensor_variable(x) y = as_tensor_variable(y) if inverse: # as_tensor_variable does not accept booleans inverse = as_tensor_variable(1) else: inverse = as_tensor_variable(0) # y should contain integers assert y.type.dtype in integer_dtypes # Inverse should be an integer scalar assert (inverse.type.ndim == 0 and inverse.type.dtype in integer_dtypes) # Match shapes of x and y x_dim = x.type.ndim y_dim = y.type.ndim if x_dim > y_dim: y = shape_padleft(y, n_ones=(x_dim - y_dim)) elif x_dim < y_dim: x = shape_padleft(x, n_ones=(y_dim - x_dim)) # Compute the broadcastable pattern of the output out_broadcastable = [xb and yb for xb, yb in izip(x.type.broadcastable, y.type.broadcastable)] out_type = tensor(dtype=x.type.dtype, broadcastable=out_broadcastable) inputlist = [x, y, inverse] outputlist = [out_type] return Apply(self, inputlist, outputlist) def _rec_perform(self, node, x, y, inverse, out, curdim): """Perform the permutation by doing a recursion over the input dimensions. For every dimension, starting with the leftmost, the right set of indices is determined (depending if broadcasting or not), then the function is recursively called on the appropriate subtensors. The terminal case is reached when the current tensors are vector, then the permutation contained in y is applied to x. Parameters ---------- x : tensor The input tensor, on which the permutation is applied. y : tensor Tensor containing the permutations to apply. out : tensor Tensor storing the output result. curdim : int Counter of the current depth of recursion. inverse Wether to apply permutations or their inverse. """ if len(x.shape) == 1: # Numpy advanced indexing works in this case if inverse: out[y] = x[:] else: out[:] = x[y] else: xs0 = x.shape[0] ys0 = y.shape[0] if xs0 == ys0: for i in xrange(xs0): self._rec_perform(node, x[i], y[i], inverse, out[i], curdim + 1) elif ys0 == 1 and node.inputs[1].type.broadcastable[curdim]: # Broadcast y for i in xrange(xs0): self._rec_perform(node, x[i], y[0], inverse, out[i], curdim + 1) elif xs0 == 1 and node.inputs[0].type.broadcastable[curdim]: # Broadcast x for i in xrange(ys0): self._rec_perform(node, x[0], y[i], inverse, out[i], curdim + 1) else: raise ValueError('Dimension mismatch: %s, %s' % (xs0, ys0)) def perform(self, node, inp, out): x, y, inverse = inp outs, = out x_s = x.shape y_s = y.shape assert len(x_s) == len(y_s) # Make sure the output is big enough out_s = [] for xdim, ydim in izip(x_s, y_s): if xdim == ydim: outdim = xdim elif xdim == 1: outdim = ydim elif ydim == 1: outdim = xdim else: raise ValueError('Dimension mismatch: %s, %s' % (xdim, ydim)) out_s.append(outdim) if outs[0] is None or outs[0].shape != out_s: outs[0] = np.empty(out_s, dtype=x.dtype) self._rec_perform(node, x, y, inverse, outs[0], curdim=0) def infer_shape(self, node, in_shapes): shp_x = in_shapes[0] shp_y = in_shapes[1] assert len(shp_x) == len(shp_y) out_shape = [] for i in xrange(len(shp_x)): out_shape.append(maximum(shp_x[i], shp_y[i])) return [out_shape] def grad(self, inp, grads): x, y, inverse = inp gz, = grads # First, compute the gradient wrt the broadcasted x. # If 'inverse' is False (0), apply the inverse of y on gz. # Else, apply y on gz. gx = permute_row_elements(gz, y, eq(inverse, 0)) # If x has been broadcasted along some axes, we need to sum # the gradient over these axes, but keep the dimension (as # broadcastable) broadcasted_dims = [dim for dim in xrange(gz.type.ndim) if x.type.broadcastable[dim] and not gz.type.broadcastable[dim]] gx = Sum(axis=broadcasted_dims)(gx) # Sum(...) removed the dimensions in broadcasted_dims, # so we need to put them back. newdims = [] i = 0 for dim in xrange(gz.type.ndim): if dim in broadcasted_dims: newdims.append('x') else: newdims.append(i) i += 1 gx = DimShuffle(gx.type.broadcastable, newdims)(gx) assert gx.type.broadcastable == x.type.broadcastable # if x is an integer type, then so is the output. # this means f(x+eps) = f(x) so the gradient with respect # to x is zero if x.type.dtype in discrete_dtypes: gx = x.zeros_like() # The elements of y and of inverse both affect the output, # so they are connected to the output, # and the transformation isn't defined if their values # are non-integer, so the gradient with respect to them is # undefined return [gx, grad_undefined(self, 1, y), grad_undefined(self, 1, inverse)] _permute_row_elements = PermuteRowElements() def permute_row_elements(x, y, inverse=0): return _permute_row_elements(x, y, inverse) def inverse_permutation(perm): """Computes the inverse of permutations. Each row of input should contain a permutation of the first integers. """ return permute_row_elements( arange(perm.shape[-1], dtype=perm.dtype), perm, inverse=True) ######################### # Linalg : Dot ######################### # # For BLAS-related ops see blas.py # # TODO: Dotinv should go here, Eigs, Svd, etc. class Dot(Op): """ Computes the dot product of two variables. For two matrices, this is equivalent to matrix multiplication. For two vectors, this is the inner product. Notes ----- Matrix-matrix products are sometimes optimized to Dot22 or Gemm ops (see tensor.blas). Vector-vector products are sometimes optimized to Ger or CGer (see tensor.blas). Matrix-vector products are sometimes optimized to Gemv, CGemv (see tensor.blas). """ __props__ = () # the rationale for Dot22 is related to getting GEMM Ops into the # graph. See Dot22 in tensor.blas for details. def make_node(self, *inputs): inputs = list(map(as_tensor_variable, inputs)) if len(inputs) != 2: raise TypeError( 'theano.tensor.Dot: 2 arguments required, %d given ' % len(inputs)) if inputs[0].ndim not in (1, 2): raise TypeError( 'theano.tensor.Dot: input 0 (0-indexed) must have ndim of ' '1 or 2, %d given. Consider calling theano.tensor.dot ' 'instead.' % inputs[0].ndim) if inputs[1].ndim not in (1, 2): raise TypeError( 'theano.tensor.Dot: input 1 (0-indexed) must have ndim of ' '1 or 2, %d given. Consider calling theano.tensor.dot ' 'instead.' % inputs[1].ndim) i_broadcastables = [input.type.broadcastable for input in inputs] bx, by = i_broadcastables if len(by) == 2: # y is a matrix bz = bx[:-1] + by[-1:] elif len(by) == 1: # y is vector bz = bx[:-1] i_dtypes = [input.type.dtype for input in inputs] outputs = [tensor(scal.upcast(*i_dtypes), bz)] return Apply(self, inputs, outputs) def perform(self, node, inp, out): x, y = inp z, = out # the asarray is here because dot between two vectors # gives a numpy float object but we need to return a 0d # ndarray z[0] = np.asarray(np.dot(x, y)) def grad(self, inp, grads): x, y = inp gz, = grads xdim, ydim, gdim = x.type.ndim, y.type.ndim, gz.type.ndim # grad is scalar, so x is vector and y is vector if gdim == 0: xgrad = gz * y ygrad = gz * x # x is vector, y is matrix, grad is vector elif xdim == 1 and ydim == 2: xgrad = dot(gz, y.T) ygrad = outer(x.T, gz) # x is matrix, y is vector, grad is vector elif xdim == 2 and ydim == 1: xgrad = outer(gz, y.T) ygrad = dot(x.T, gz) # x is matrix, y is matrix, grad is matrix elif xdim == ydim == 2: xgrad = dot(gz, y.T) ygrad = dot(x.T, gz) # If x or y contain broadcastable dimensions but only one of # them know that a matching dimensions is broadcastable, the # above code don't always return the right broadcast pattern. # This cause problem down the road. See gh-1461. if xgrad.broadcastable != x.broadcastable: xgrad = patternbroadcast(xgrad, x.broadcastable) if ygrad.broadcastable != y.broadcastable: ygrad = patternbroadcast(ygrad, y.broadcastable) rval = xgrad, ygrad for elem in rval: assert elem.dtype.find('float') != -1 return rval def R_op(self, inputs, eval_points): # R_op for a \dot b evaluted at c for a and d for b is # simply c \dot b + a \dot d assert len(inputs) == 2 assert len(eval_points) == 2 if eval_points[0] is None and eval_points[1] is None: return [None] if eval_points[0]: t1 = self(eval_points[0], inputs[1]) if eval_points[1]: t2 = self(inputs[0], eval_points[1]) if eval_points[0] and eval_points[1]: return [t1 + t2] elif eval_points[0]: return [t1] else: return [t2] def infer_shape(self, node, shapes): xshp, yshp = shapes x, y = node.inputs # vector / vector if x.ndim == 1 and y.ndim == 1: return [()] # matrix / vector if x.ndim == 2 and y.ndim == 1: return [xshp[:-1]] # vector / matrix if x.ndim == 1 and y.ndim == 2: return [yshp[-1:]] # matrix / matrix if x.ndim == 2 and y.ndim == 2: return [xshp[:-1] + yshp[-1:]] raise NotImplementedError() def __str__(self): return "dot" _dot = Dot() pprint.assign(_dot, printing.OperatorPrinter(printing.special['middle_dot'], -1, 'left')) def dot(a, b): """ Computes the dot product of two variables. For two matrices, this is equivalent to matrix multiplication. For two vectors, this is the inner product. When one variable is a scalar, this is like elementwise multiplication. For N dimensions, this is a sum product over the last axis of the first array and the second-to-last axis of the second array: dot(a, b)[i,j,k,m] = sum(a[i,j,:] * b[k,:,m]) Note that this dot function does one of three things, in the following sequence: 1. If either a or b is scalar, it returns the elementwise product without calling the Theano Dot op. 2. If either a or b has more than 2 dimensions, it calls Theano's tensordot function with appropriate axes. The tensordot function expresses high-dimensional dot products in terms of 2D matrix multiplications, so it may be possible to futherize optimize for performance. 3. If both a and b have either 1 or 2 dimensions, it calls Theano's Dot op on a and b. Notes ----- Matrix-matrix products are sometimes optimized to Dot22 or Gemm ops (see tensor.blas). Vector-vector products are sometimes optimized to Ger or CGer (see tensor.blas). Matrix-vector products are sometimes optimized to Gemv, CGemv (see tensor.blas). """ a, b = as_tensor_variable(a), as_tensor_variable(b) if a.ndim == 0 or b.ndim == 0: return a * b elif a.ndim > 2 or b.ndim > 2: return tensordot(a, b, [[a.ndim - 1], [np.maximum(0, b.ndim - 2)]]) else: return _dot(a, b) ######################### # Linalg : TensorDot ######################### def _tensordot_as_dot(a, b, axes, dot, batched): """ Reduces a tensor dot product to a matrix or vector dot product. Based on code from Tijmen Tieleman's gnumpy (http://www.cs.toronto.edu/~tijmen/gnumpy.html). Please see the documentation of tensordot for the meaning of the a, b and axes arguments. :param dot: a function that accepts two symbolic variables and computes the appropriate dot product (e.g. dot, batched_dot) :type dot: function :param batched: whether to treat the first axis of a and b as a batch axis. If so, this axis will be preserved in the output, allowing this function to be used also for batched tensor dot products. :type batched: boolean :returns: a tensor with shape equal to the concatenation of a's shape (less any dimensions that were summed over) and b's shape (less the first dimension and any dimensions that were summed over). :rtype: symbolic tensor """ a, b = as_tensor_variable(a), as_tensor_variable(b) if not np.isscalar(axes) and len(axes) != 2: raise ValueError('Axes should be an integer or a ' 'list/tuple of len 2 (%s was provided)' % str(axes)) # if 'axes' is a number of axes to multiply and sum over (trailing axes # of a, leading axes of b), we can just reshape and use dot. elif np.isscalar(axes): axes = int(axes) for operand_name, operand in (("a", a), ("b", b)): if axes > operand.ndim: raise ValueError( 'axes can not be larger than the dimension of %s ' '(%s.ndim=%i, axes=%i)' % (operand_name, operand_name, operand.ndim, axes)) if batched and axes == operand.ndim: raise ValueError( 'axes to sum over must not include the batch axis ' 'of %s (%s.ndim=%i, axes=%i)' % (operand_name, operand_name, operand.ndim, axes)) batch_axes = 1 if batched else 0 a_outaxes = slice(0, a.ndim - axes) b_outaxes = slice(batch_axes + axes, b.ndim) outshape = concatenate([a.shape[a_outaxes], b.shape[b_outaxes]]) outbcast = a.broadcastable[a_outaxes] + b.broadcastable[b_outaxes] outndim = len(outbcast) a_shape = [1] * 2 b_shape = [1] * 2 # compute total size of summed axes for i in xrange(0, axes): a_shape[1] *= a.shape[-(i + 1)] b_shape[0] *= b.shape[batch_axes + i] # compute total size of other axes for i in xrange(0, a.ndim - axes - batch_axes): a_shape[0] *= a.shape[batch_axes + i] for i in xrange(0, b.ndim - axes - batch_axes): b_shape[1] *= b.shape[-(i + 1)] if batched: a_shape.insert(0, a.shape[0]) b_shape.insert(0, b.shape[0]) a_reshaped = a.reshape(a_shape) b_reshaped = b.reshape(b_shape) out_reshaped = dot(a_reshaped, b_reshaped) out = out_reshaped.reshape(outshape, outndim) # Make sure the broadcastable pattern of the result is correct, # since some shape information can be lost in the reshapes. return patternbroadcast(out, outbcast) # if 'axes' is a list, transpose a and b such that the summed axes of a # are last and the summed axes of b are first. else: axes = [_pack(axes_) for axes_ in axes] if len(axes[0]) != len(axes[1]): raise ValueError('Axes elements must have the same length.') for i, (operand_name, operand) in enumerate((("a", a), ("b", b))): if len(axes[i]) > operand.ndim: raise ValueError( 'axes[%i] should be array_like with length less than ' 'the dimensions of %s (%s.ndim=%i, len(axes[0])=%i).' % (i, operand_name, operand_name, operand.ndim, len(axes[i]))) if len(axes[i]) > 0 and np.max(axes[i]) >= operand.ndim: raise ValueError( 'axes[%i] contains dimensions greater than or equal ' 'to %s.ndim (%s.ndim=%i, max(axes[0])=%i).' % (i, operand_name, operand_name, operand.ndim, np.max(np.array(axes[i])))) if batched and 0 in axes[i]: raise ValueError( 'axes to sum over must not contain the batch axis ' '(axes[%i]=%s)' % (i, axes[i])) batch_axes = [0] if batched else [] other_axes = [[x for x in xrange(operand.ndim) if x not in axes[i] and x not in batch_axes] for i, operand in enumerate((a, b))] a_shuffled = a.dimshuffle(batch_axes + other_axes[0] + axes[0]) b_shuffled = b.dimshuffle(batch_axes + axes[1] + other_axes[1]) # now that a and b are in the right order, recur with integer axes return _tensordot_as_dot(a_shuffled, b_shuffled, len(axes[0]), dot=dot, batched=batched) def tensordot(a, b, axes=2): """ Compute a generalized dot product over provided axes. Given two tensors a and b, tensordot computes a generalized dot product over the provided axes. Theano's implementation reduces all expressions to matrix or vector dot products and is based on code from Tijmen Tieleman's gnumpy (http://www.cs.toronto.edu/~tijmen/gnumpy.html). Parameters ---------- a: symbolic tensor The first tensor variable. b: symbolic tensor The second tensor variable axes: int or array-like of length 2 If an integer, the number of axes to sum over. If an array, it must have two array elements containing the axes to sum over in each tensor. Note that the default value of 2 is not guaranteed to work for all values of a and b, and an error will be raised if that is the case. The reason for keeping the default is to maintain the same signature as numpy's tensordot function (and np.tensordot raises analogous errors for non-compatible inputs). If an integer i, it is converted to an array containing the last i dimensions of the first tensor and the first i dimensions of the second tensor: axes = [list(range(a.ndim - i, b.ndim)), list(range(i))] If an array, its two elements must contain compatible axes of the two tensors. For example, [[1, 2], [2, 0]] means sum over the 2nd and 3rd axes of a and the 3rd and 1st axes of b. (Remember axes are zero-indexed!) The 2nd axis of a and the 3rd axis of b must have the same shape; the same is true for the 3rd axis of a and the 1st axis of b. Returns ------- symbolic tensor A tensor with shape equal to the concatenation of a's shape (less any dimensions that were summed over) and b's shape (less any dimensions that were summed over). Examples -------- It may be helpful to consider an example to see what tensordot does. Theano's implementation is identical to NumPy's. Here a has shape (2, 3, 4) and b has shape (5, 6, 4, 3). The axes to sum over are [[1, 2], [3, 2]] -- note that a.shape[1] == b.shape[3] and a.shape[2] == b.shape[2]; these axes are compatible. The resulting tensor will have shape (2, 5, 6) -- the dimensions that are not being summed: >>> a = np.random.random((2,3,4)) >>> b = np.random.random((5,6,4,3)) #tensordot >>> c = np.tensordot(a, b, [[1,2],[3,2]]) #loop replicating tensordot >>> a0, a1, a2 = a.shape >>> b0, b1, _, _ = b.shape >>> cloop = np.zeros((a0,b0,b1)) #loop over non-summed indices -- these exist #in the tensor product. >>> for i in range(a0): ... for j in range(b0): ... for k in range(b1): ... #loop over summed indices -- these don't exist ... #in the tensor product. ... for l in range(a1): ... for m in range(a2): ... cloop[i,j,k] += a[i,l,m] * b[j,k,m,l] >>> np.allclose(c, cloop) true This specific implementation avoids a loop by transposing a and b such that the summed axes of a are last and the summed axes of b are first. The resulting arrays are reshaped to 2 dimensions (or left as vectors, if appropriate) and a matrix or vector dot product is taken. The result is reshaped back to the required output dimensions. In an extreme case, no axes may be specified. The resulting tensor will have shape equal to the concatenation of the shapes of a and b: >>> c = np.tensordot(a, b, 0) >>> print(a.shape) (2,3,4) >>> print(b.shape) (5,6,4,3) >>> print(c.shape) (2,3,4,5,6,4,3) See the documentation of numpy.tensordot for more examples. """ return _tensordot_as_dot(a, b, axes, dot=dot, batched=False) def outer(x, y): """Return vector-vector outer product. If an input isn't a vector, we flatten it first. """ if x.ndim != 1: x = x.flatten() if y.ndim != 1: y = y.flatten() return dot( x.dimshuffle(0, 'x'), y.dimshuffle('x', 0)) def any(x, axis=None, keepdims=False): out = elemwise.Any(axis)(x) if keepdims: out = makeKeepDims(x, out, axis) return out def all(x, axis=None, keepdims=False): out = elemwise.All(axis)(x) if keepdims: out = makeKeepDims(x, out, axis) return out # Some NumPy version like 1.9.2 return a view for numpy.diagonal x = np.zeros((4, 4)) numpy_diagonal_return_view = np.may_share_memory(np.diagonal(x), x) del x class ExtractDiag(Op): """ Return specified diagonals. If x is 2-D, returns the diagonal of x with the given offset, i.e., the collection of elements of the form x[i, i+offset]. If x has more than two dimensions, then the axes specified by axis1 and axis2 are used to determine the 2-D sub-array whose diagonal is returned. The shape of the resulting array can be determined by removing axis1 and axis2 and appending an index to the right equal to the size of the resulting diagonals. Parameters ---------- x: A tensor variable with x.ndim >= 2. offset: Offset of the diagonal from the main diagonal. Can be positive or negative. Defaults to main diagonal (0). axis1: Axis to be used as the first axis of the 2-D sub-arrays from which the diagonals should be taken. Defaults to first axis (0). axis2: Axis to be used as the second axis of the 2-D sub-arrays from which the diagonals should be taken. Defaults to second axis (1). Returns ------- array_of_diagonals: If x is 2-D, a 1-D array of the same type as a containing the diagonal is returned. If the dimension of x is greater than two, then an array of diagonals is returned, "packed" from left-most dimension to right-most (e.g., if x is 3-D, then the diagonals are "packed" along rows). Raises ------ ValueError If the dimension of x is less than 2. See Also -------- numpy.diagonal: https://docs.scipy.org/doc/numpy-dev/reference/generated/numpy.diagonal.html """ __props__ = ("offset", "axis1", "axis2", "view") def __init__(self, offset=0, axis1=0, axis2=1, view=False): self.view = view if self.view and not numpy_diagonal_return_view: warnings.warn("View will forced to False. ExtractDiag property view is " "set to True but numpy version %s and prior versions of " "numpy.diagonal() do not return a view. Update " "numpy to use ExtractDiag(view=True)" % np.version.version) self.view = False if self.view: self.view_map = {0: [0]} self.offset = offset self.axis1 = axis1 self.axis2 = axis2 def make_node(self, x): x = as_tensor_variable(x) if x.ndim < 2: raise ValueError('ExtractDiag needs an input with 2 or more ' 'dimensions', x) return Apply(self, [x], [x.type.__class__( dtype=x.dtype, broadcastable=[False] * (x.ndim - 1))()]) def perform(self, node, inputs, outputs): (x,) = inputs (z,) = outputs z[0] = x.diagonal(self.offset, self.axis1, self.axis2) if not self.view: z[0] = z[0].copy() def grad(self, inputs, gout): (x,) = inputs (gz,) = gout if x.ndim == 2: x = theano.tensor.zeros_like(x) xdiag = theano.tensor.AllocDiag(offset=self.offset)(gz) return [theano.tensor.set_subtensor( x[:xdiag.shape[0], :xdiag.shape[1]], xdiag)] else: warnings.warn("gradient of theano.tensor.basic.ExtractDiag only" "works for matrices.") return [grad_not_implemented(self, 0, x)] def infer_shape(self, node, shapes): in_shape, = shapes dim1 = in_shape[self.axis1] dim2 = in_shape[self.axis2] out_shape = [d for i, d in enumerate(in_shape) if i not in (self.axis1, self.axis2)] # The following logic is inspired by C code of PyArray_Diagonal(). offset = self.offset if offset > 0: diag_size = clip(dim2 - offset, 0, dim1) elif offset < 0: diag_size = clip(dim1 + offset, 0, dim2) else: diag_size = minimum(dim1, dim2) out_shape.append(diag_size) return [tuple(out_shape)] def __setstate__(self, state): self.__dict__.update(state) if self.view and not numpy_diagonal_return_view: warnings.warn("View will forced to False. ExtractDiag property view is " "set to True but numpy version %s and prior versions of " "numpy.diagonal() do not return a view. Update " "numpy to use ExtractDiag(view=True)" % np.version.version) self.view = False if self.view: self.view_map = {0: [0]} if "offset" not in state: self.offset = 0 if "axis1" not in state: self.axis1 = 0 if "axis2" not in state: self.axis2 = 1 def diagonal(a, offset=0, axis1=0, axis2=1): """ A helper function for `theano.tensor.ExtractDiag`. It accepts tensor with `ndim >= 2` as input. The name `diagonal` is just meant to keep it consistent with numpy. Parameters ---------- a : symbolic tensor offset : int offset axis1 : int axis2 : int Returns ------- tensor : symbolic tensor """ return ExtractDiag(offset, axis1, axis2)(a) class AllocDiag(Op): """ An op that copies a vector to the diagonal of an empty matrix. It does the inverse of ExtractDiag. Usage: T.AllocDiag()(x) `x` should be a tensor vector. The parenthesis in the front should indicate which main diagonal the vector value goes into. By default it is set to `0`, which corresponds to setting the values of x to the main diagonal in the returned matrix. Parameters ---------- axis1: Axis to be used as the first axis of the 2-D sub-arrays to which the diagonals will be allocated. Defaults to first axis (0). axis2: Axis to be used as the second axis of the 2-D sub-arrays to which the diagonals will be allocated. Defaults to second axis (1). offset: Offset of the diagonal from the main diagonal defined by `axis1` and `axis2`. Can be positive or negative. Defaults to main diagonal (0). x: symbolic vector A tensor vector consists of diagonal values. Returns ------- tensor : symbolic tenstor A tensor with passed tensor values at their corresponding diagonals. """ __props__ = ("offset", "axis1", "axis2") def __init__(self, offset=0, axis1=0, axis2=1): self.offset = offset self.axis1 = axis1 self.axis2 = axis2 def make_node(self, diag): diag = as_tensor_variable(diag) if diag.type.ndim < 1: raise ValueError('AllocDiag needs an input with 1 or more ' 'dimensions', diag.type) return Apply( self, [diag], [diag.type.__class__( dtype=diag.dtype, broadcastable=[False] * (diag.ndim + 1))()] ) def perform(self, node, inputs, outputs): (x,) = inputs (z,) = outputs axis1 = np.minimum(self.axis1, self.axis2) axis2 = np.maximum(self.axis1, self.axis2) offset = self.offset # Create array with one extra dimension for resulting matrix result_shape = x.shape[:-1] + (x.shape[-1] + abs(offset),) * 2 result = np.zeros(result_shape, dtype=x.dtype) # Create slice for diagonal in final 2 axes idxs = np.arange(x.shape[-1]) diagonal_slice = ((len(result_shape) - 2) * [slice(None)] + [idxs + np.maximum(0, -offset), idxs + np.maximum(0, offset)]) # Fill in final 2 axes with x result[diagonal_slice] = x if len(x.shape) > 1: # Re-order axes so they correspond to diagonals at axis1, axis2 axes = list(range(len(x.shape[:-1]))) last_idx = axes[-1] axes = axes[:axis1] + [last_idx + 1] + axes[axis1:] axes = axes[:axis2] + [last_idx + 2] + axes[axis2:] result = result.transpose(axes) z[0] = result def grad(self, inputs, gout): (gz,) = gout return [diagonal( gz, offset=self.offset, axis1=self.axis1, axis2=self.axis2 )] def infer_shape(self, nodes, shapes): (x_shape,) = shapes axis1 = np.minimum(self.axis1, self.axis2) axis2 = np.maximum(self.axis1, self.axis2) result_shape = list(x_shape[:-1]) diag_shape = x_shape[-1] + abs(self.offset) result_shape = result_shape[:axis1] + [diag_shape] + result_shape[axis1:] result_shape = result_shape[:axis2] + [diag_shape] + result_shape[axis2:] return [tuple(result_shape)] def __setstate__(self, state): if "view_map" in state: del state["view_map"] self.__dict__.update(state) if "offset" not in state: self.offset = 0 if "axis1" not in state: self.axis1 = 0 if "axis2" not in state: self.axis2 = 1 def diag(v, k=0): """ A helper function for two ops: `theano.tensor.ExtractDiag` and `theano.tensor.AllocDiag`. The name `diag` is meant to keep it consistent with numpy. It both accepts tensor vector and tensor matrix. While the passed tensor variable `v` has `v.ndim>=2`, it builds a `ExtractDiag` instance, and returns a vector with its entries equal to `v`'s main diagonal; otherwise if `v.ndim` is `1`, it builds an `AllocDiag` instance, and returns a matrix with `v` at its k-th diaogonal. Parameters ---------- v : symbolic tensor k : int offset Returns ------- tensor : symbolic tensor """ if v.ndim == 1: return AllocDiag(k)(v) elif v.ndim >= 2: return diagonal(v, offset=k) else: raise ValueError("Input must has v.ndim >= 1.") def stacklists(arg): """ Recursively stack lists of tensors to maintain similar structure. This function can create a tensor from a shaped list of scalars: Examples -------- >>> from theano.tensor import stacklists, scalars, matrices >>> from theano import function >>> a, b, c, d = scalars('abcd') >>> X = stacklists([[a, b], [c, d]]) >>> f = function([a, b, c, d], X) >>> f(1, 2, 3, 4) array([[ 1., 2.], [ 3., 4.]], dtype=float32) We can also stack arbitrarily shaped tensors. Here we stack matrices into a 2 by 2 grid: >>> from numpy import ones >>> a, b, c, d = matrices('abcd') >>> X = stacklists([[a, b], [c, d]]) >>> f = function([a, b, c, d], X) >>> x = ones((4, 4), 'float32') >>> f(x, x, x, x).shape (2, 2, 4, 4) """ if isinstance(arg, (tuple, list)): return stack(list(map(stacklists, arg))) else: return arg def ptp(a, axis=None): """ Range of values (maximum - minimum) along an axis. The name of the function comes from the acronym for peak to peak. Parameters ---------- a Input tensor. axis Axis along which to find the peaks. By default, flatten the array. Returns ------- array A new array holding the result. """ a = as_tensor_variable(a) out = max(a, axis) - min(a, axis) return out def power(x, y): return x ** y def swapaxes(y, axis1, axis2): "swap axes of inputted tensor" y = as_tensor_variable(y) ndim = y.ndim li = list(range(0, ndim)) li[axis1], li[axis2] = li[axis2], li[axis1] return y.dimshuffle(li) def choose(a, choices, out=None, mode='raise'): """ Construct an array from an index array and a set of arrays to choose from. First of all, if confused or uncertain, definitely look at the Examples - in its full generality, this function is less simple than it might seem from the following code description (below ndi = numpy.lib.index_tricks): np.choose(a,c) == np.array([c[a[I]][I] for I in ndi.ndindex(a.shape)]). But this omits some subtleties. Here is a fully general summary: Given an ``index`` array (a) of integers and a sequence of n arrays (choices), a and each choice array are first broadcast, as necessary, to arrays of a common shape; calling these Ba and Bchoices[i], i = 0,...,n-1 we have that, necessarily, Ba.shape == Bchoices[i].shape for each i. Then, a new array with shape Ba.shape is created as follows: - if mode=raise (the default), then, first of all, each element of a (and thus Ba) must be in the range [0, n-1]; now, suppose that i (in that range) is the value at the (j0, j1, ..., jm) position in Ba - then the value at the same position in the new array is the value in Bchoices[i] at that same position; - if mode=wrap, values in a (and thus Ba) may be any (signed) integer; modular arithmetic is used to map integers outside the range [0, n-1] back into that range; and then the new array is constructed as above; - if mode=clip, values in a (and thus Ba) may be any (signed) integer; negative integers are mapped to 0; values greater than n-1 are mapped to n-1; and then the new array is constructed as above. Parameters ---------- a : int array This array must contain integers in [0, n-1], where n is the number of choices, unless mode=wrap or mode=clip, in which cases any integers are permissible. choices : sequence of arrays Choice arrays. a and all of the choices must be broadcastable to the same shape. If choices is itself an array (not recommended), then its outermost dimension (i.e., the one corresponding to choices.shape[0]) is taken as defining the ``sequence``. out : array, optional If provided, the result will be inserted into this array. It should be of the appropriate shape and dtype. mode : {``raise`` (default), ``wrap``, ``clip``}, optional Specifies how indices outside [0, n-1] will be treated: ``raise`` : an exception is raised ``wrap`` : value becomes value mod n ``clip`` : values < 0 are mapped to 0, values > n-1 are mapped to n-1 Returns ------- merged_array - array The merged result. Raises ------ ValueError - shape mismatch If a and each choice array are not all broadcastable to the same shape. """ # This is done to keep the same function signature then NumPy. assert out is None return Choose(mode)(a, choices) class Choose(Op): __props__ = ('mode',) def __init__(self, mode): assert mode in ("raise", "wrap", "clip") self.mode = mode def infer_shape(self, node, shapes): if isinstance(node.inputs[1], TensorVariable): # We have padded node.inputs[0] to the right number of # dimensions for the output l = [] for sh1, sh2, b1 in zip(shapes[0], shapes[1][1:], node.inputs[0].broadcastable): if b1: l.append(sh2) else: l.append(sh1) return [tuple(l)] else: import theano.typed_list assert isinstance(node.inputs[1], theano.typed_list.TypedListVariable) raise ShapeError("Case not implemented") shape = shapes[0] for i in xrange(len(shapes[0]) - 1): shape[i] = shapes[1][i] return [(shape)] def make_node(self, a, choices): # Import here as it isn't imported by default and we can't # import at the top as it would cause circular import. import theano.typed_list a = as_tensor_variable(a) if a.dtype not in theano.tensor.discrete_dtypes: raise TypeError( 'choose first argument must have an [u]int* dtype. Got %s.' % a.dtype) if isinstance(choices, (tuple, list, theano.typed_list.TypedListVariable)): choice = theano.typed_list.make_list(choices) choice_ndim = choice.ttype.ndim choice_bcast = choice.ttype.broadcastable else: choice = as_tensor_variable(choices) choice_ndim = choice.ndim - 1 choice_bcast = choice.broadcastable[1:] out_ndim = np.max([a.ndim, choice_ndim]) # Make explicit all added broadcastable dimensions. a = shape_padleft(a, out_ndim - a.ndim) if len(choice_bcast) != out_ndim: if isinstance(choice.type, TensorType): choice = choice.dimshuffle(0, *(('x',) * (out_ndim - choice_ndim) + tuple(range(1, choice.ndim)))) choice_ndim = choice.ndim - 1 choice_bcast = choice.broadcastable[1:] else: raise NotImplementedError( "We currently didn't implemented that case. " "To make it work, explicitly add dimensions " "of size one for dimensions that will be broadcasted") bcast = [False] * out_ndim for idx, (b1, b2) in enumerate( zip(a.broadcastable, (True,) * (out_ndim - choice_ndim) + choice_bcast)): if b1 and b2: bcast[idx] = True o = TensorType(choice.dtype, bcast) return Apply(self, [a, choice], [o()]) def perform(self, node, inputs, outputs): (z,) = outputs a = inputs[0] choice = inputs[1] # TODO reuse out? z[0] = np.choose(a, choice, mode=self.mode) class AllocEmpty(gof.Op): """Implement Alloc on the cpu, but without initializing memory.""" __props__ = ("dtype", ) params_type = ParamsType(typecode=int32_t) # specify the type of the data def __init__(self, dtype): assert isinstance(dtype, str), dtype self.dtype = dtype.lower() @property def typecode(self): return np.dtype(self.dtype).num def make_node(self, *shape): shape, bcast = alloc_validate_shape(shape) otype = TensorType(dtype=self.dtype, broadcastable=bcast) output = otype() output.tag.values_eq_approx = values_eq_approx_always_true # The outut can contain nan/inf. output.type is a new # instance, so we can do this only for that variable. output.type.filter_checks_isfinite = False # We can't reuse filter_checks_isfinite as by default it is # False and it is set to true only in DebugMode. # We can't set it in the type as other make_node can reuse the type. # We can't set it in the variable as it isn't copied when we copy # the variale. So we set it in the tag. output.tag.nan_guard_mode_check = False return Apply(self, shape, [output]) def debug_perform(self, node, inputs, out_, params): self.perform(node, inputs, out_, params) out_[0][0].fill(-123456789) def perform(self, node, inputs, out_, params): out, = out_ sh = tuple([int(i) for i in inputs]) if out[0] is None or out[0].shape != sh: out[0] = np.empty(sh, dtype=self.dtype) def c_code(self, node, name, inputs, out_, sub): out, = out_ fail = sub['fail'] shps = inputs nd = len(shps) params = sub['params'] str = "npy_intp dims[%(nd)s];\n" % locals() for idx, sh in enumerate(shps): str += "dims[%(idx)s] =" \ "((npy_intp)((dtype_%(sh)s*)" \ " PyArray_DATA(%(sh)s))[0]);\n" % locals() # Validate that the output storage exists str += "if(%(out)s==NULL\n" % locals() for idx, sh in enumerate(shps): str += "||PyArray_DIMS(%(out)s)[%(idx)s]!=dims[%(idx)s]" % locals() str += """){ /* Reference received to invalid output variable. Decrease received reference's ref count and allocate new output variable */ Py_XDECREF(%(out)s); %(out)s = (PyArrayObject*)PyArray_EMPTY(%(nd)s, dims, %(params)s->typecode, 0); if (!%(out)s) { PyErr_SetString(PyExc_MemoryError, "alloc failed"); %(fail)s; } } """ % locals() return str def infer_shape(self, node, input_shapes): return [node.inputs] def c_code_cache_version(self): return (4,) def do_constant_folding(self, node): return False def connection_pattern(self, node): return [[False] for i in node.inputs] def grad(self, inputs, grads): return [DisconnectedType()() for i in inputs] def R_op(self, inputs, eval_points): return [zeros(inputs, self.dtype)] Theano-1.0.0/theano/tensor/blas_scipy.py00006000163562€ÿÿÿþ0000000553613203335532021572 0ustar boccoset00000000000000""" Implementations of BLAS Ops based on scipy's BLAS bindings. """ from __future__ import absolute_import, print_function, division import numpy as np from theano.tensor.blas import Ger, ger, ger_destructive, have_fblas from theano.tensor.blas import blas_optdb, optdb, local_optimizer from theano.tensor.opt import in2out if have_fblas: from theano.tensor.blas import fblas _blas_ger_fns = { np.dtype('float32'): fblas.sger, np.dtype('float64'): fblas.dger, np.dtype('complex64'): fblas.cgeru, np.dtype('complex128'): fblas.zgeru, } class ScipyGer(Ger): def prepare_node(self, node, storage_map, compute_map, impl): if impl == 'py': node.tag.local_ger = _blas_ger_fns[np.dtype( node.inputs[0].type.dtype)] def perform(self, node, inputs, output_storage): cA, calpha, cx, cy = inputs cZ, = output_storage # N.B. some versions of scipy (e.g. mine) don't actually work # in-place on a, even when I tell it to. A = cA local_ger = node.tag.local_ger if A.size == 0: # We don't have to compute anything, A is empty. # We need this special case because Numpy considers it # C-contiguous, wich is confusing. if not self.destructive: # Sometimes numpy thinks empty matrices can share memory, # so here to stop DebugMode from complaining. A = A.copy() elif A.flags['C_CONTIGUOUS']: A = local_ger(calpha, cy, cx, a=A.T, overwrite_a=int(self.destructive)).T else: A = local_ger(calpha, cx, cy, a=A, overwrite_a=int(self.destructive)) cZ[0] = A scipy_ger_no_inplace = ScipyGer(False) scipy_ger_inplace = ScipyGer(True) @local_optimizer([ger, ger_destructive]) def use_scipy_ger(node): if node.op == ger: return [scipy_ger_no_inplace(*node.inputs)] @local_optimizer([scipy_ger_no_inplace]) def make_ger_destructive(node): if node.op == scipy_ger_no_inplace: return [scipy_ger_inplace(*node.inputs)] use_scipy_blas = in2out(use_scipy_ger) make_scipy_blas_destructive = in2out(make_ger_destructive) if have_fblas: # scipy_blas is scheduled in the blas_optdb very late, because scipy sortof # sucks, but it is almost always present. # C implementations should be scheduled earlier than this, so that they take # precedence. Once the original Ger is replaced, then these optimizations # have no effect. blas_optdb.register('scipy_blas', use_scipy_blas, 100, 'fast_run') # this matches the InplaceBlasOpt defined in blas.py optdb.register('make_scipy_blas_destructive', make_scipy_blas_destructive, 70.0, 'fast_run', 'inplace') Theano-1.0.0/theano/tensor/type.py00006000163562€ÿÿÿþ0000007450513203335532020425 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import logging import warnings import numpy as np import theano from theano import config from theano.gof import hashtype, Type, Variable from theano import scalar as scal _logger = logging.getLogger("theano.tensor.type") class TensorType(Type): """ Symbolic `Type` representing a numpy.ndarray value. Initialize self.dtype and self.broadcastable. Parameters ---------- dtype: str Corresponding to numpy dtype (e.g., 'int64') The value (ndarray) associated to a `Variable` of this `Type` will have this dtype. broadcastable: tuple, list, or array of boolean values This argument serves two purposes. First, the True elements of this list indicate the dimensions where the shape of an associated value must be 1. Secondly, the length of this list is the number of dimensions that an associated value must have. See doc:`broadcasting` for an explanation of how this list is used. name : str Optional name for this type. """ context_name = 'cpu' filter_checks_isfinite = False """ When this is True, strict filtering rejects data containing NaN or Inf entries. (Used in `DebugMode`) """ def __init__(self, dtype, broadcastable, name=None, sparse_grad=False): self.dtype = str(dtype) if self.dtype == 'floatX': self.dtype = config.floatX # broadcastable is immutable, and all elements are either # True or False self.broadcastable = tuple(bool(b) for b in broadcastable) self.dtype_specs() # error checking is done there self.name = name self.numpy_dtype = np.dtype(self.dtype) self.sparse_grad = sparse_grad if sparse_grad: warnings.warn( "DEPRECATION WARNING: You use an old interface to" " AdvancedSubtensor1 sparse_grad. Now use" " theano.sparse_grad(a_tensor[an_int_vector]).") def clone(self, dtype=None, broadcastable=None): """ Return a copy of the type optionally with a new dtype or broadcastable pattern. """ if dtype is None: dtype = self.dtype if broadcastable is None: broadcastable = self.broadcastable return self.__class__(dtype, broadcastable, name=self.name, sparse_grad=self.sparse_grad) def filter(self, data, strict=False, allow_downcast=None): """ Convert `data` to something which can be associated to a `TensorVariable`. This function is not meant to be called in user code. It is for `Linker` instances to use when running a compiled graph. """ # Explicit error message when one accidentally uses a Variable as # input (typical mistake, especially with shared variables). if isinstance(data, Variable): raise TypeError( 'Expected an array-like object, but found a Variable: ' 'maybe you are trying to call a function on a (possibly ' 'shared) variable instead of a numeric array?') if ((type(data) is np.ndarray) and (data.dtype == self.numpy_dtype)): if data.dtype.num != self.numpy_dtype.num: data = theano._asarray(data, dtype=self.dtype) # -- now fall through to ndim check elif ((type(data) is np.memmap) and (data.dtype == self.numpy_dtype)): # numpy.memmap is a "safe" subclass of ndarray, # so we can use it whereever we expect a base ndarray. # however, casting it would defeat the purpose of not # loading the whole data into memory pass elif strict: # If any of the two conditions above was not met, # we raise a meaningful TypeError. if not (type(data) is np.ndarray): raise TypeError("%s expected a ndarray object." % self, data, type(data)) if data.dtype != self.numpy_dtype: raise TypeError(("%s expected a ndarray object with " "dtype = %s (got %s).") % (self, self.numpy_dtype, data.dtype)) assert False, "This point should never be reached." else: if allow_downcast: # Convert to self.dtype, regardless of the type of data data = theano._asarray(data, dtype=self.dtype) # TODO: consider to pad shape with ones to make it consistent # with self.broadcastable... like vector->row type thing else: if isinstance(data, np.ndarray): # Check if self.dtype can accurately represent data # (do not try to convert the data) up_dtype = scal.upcast(self.dtype, data.dtype) if up_dtype == self.dtype: # Bug in the following line when data is a # scalar array, see # http://projects.scipy.org/numpy/ticket/1611 # data = data.astype(self.dtype) data = theano._asarray(data, dtype=self.dtype) if up_dtype != self.dtype: err_msg = ( '%s cannot store a value of dtype %s without ' 'risking loss of precision. If you do not mind ' 'this loss, you can: ' '1) explicitly cast your data to %s, or ' '2) set "allow_input_downcast=True" when calling ' '"function". Value: "%s"' % (self, data.dtype, self.dtype, repr(data))) raise TypeError(err_msg) elif (allow_downcast is None and type(data) is float and self.dtype == theano.config.floatX): # Special case where we allow downcasting of Python float # literals to floatX, even when floatX=='float32' data = theano._asarray(data, self.dtype) else: # data has to be converted. # Check that this conversion is lossless converted_data = theano._asarray(data, self.dtype) # We use the `values_eq` static function from TensorType # to handle NaN values. if TensorType.values_eq(np.asarray(data), converted_data, force_same_dtype=False): data = converted_data else: # Do not print a too long description of data # (ndarray truncates it, but it's not sure for data) str_data = str(data) if len(str_data) > 80: str_data = str_data[:75] + '(...)' err_msg = ( '%s cannot store accurately value %s, ' 'it would be represented as %s. ' 'If you do not mind this precision loss, you can: ' '1) explicitly convert your data to a numpy array ' 'of dtype %s, or ' '2) set "allow_input_downcast=True" when calling ' '"function".' % (self, data, converted_data, self.dtype)) raise TypeError(err_msg, data) if self.ndim != data.ndim: raise TypeError("Wrong number of dimensions: expected %s," " got %s with shape %s." % (self.ndim, data.ndim, data.shape)) if not data.flags.aligned: try: msg = "object buffer" + str(data.data) except AttributeError: msg = "" raise TypeError("The numpy.ndarray object is not aligned." " Theano C code does not support that.", msg, "object shape", data.shape, "object strides", data.strides, "object dtype", data.dtype) i = 0 for b in self.broadcastable: if b and data.shape[i] != 1: raise TypeError("Non-unit value on shape on a broadcastable" " dimension.", data.shape, self.broadcastable) i += 1 if (self.filter_checks_isfinite and not np.all(np.isfinite(data))): raise ValueError("non-finite elements not allowed") return data def filter_variable(self, other, allow_convert=True): """ Convert a symbolic Variable into a TensorType, if compatible. For the moment, only a TensorType and GpuArrayType will be converted, provided they have the same number of dimensions and dtype and have "compatible" broadcastable pattern. """ if hasattr(other, '_as_TensorVariable'): other = other._as_TensorVariable() if not isinstance(other, Variable): # The value is not a Variable: we cast it into # a Constant of the appropriate Type. other = self.Constant(type=self, data=other) if other.type == self: return other if allow_convert: # Attempt safe broadcast conversion. other2 = self.convert_variable(other) if other2 is not None and other2.type == self: return other2 raise TypeError( 'Cannot convert Type %(othertype)s ' '(of Variable %(other)s) into Type %(self)s. ' 'You can try to manually convert %(other)s into a %(self)s.' % dict(othertype=other.type, other=other, self=self)) def value_validity_msg(self, a): try: self.filter(a, strict=True) except Exception as e: return str(e) return "value is valid" def dtype_specs(self): """ Return a tuple (python type, c type, numpy typenum) that corresponds to self.dtype. This function is used internally as part of C code generation. """ # TODO: add more type correspondances for e.g. int32, int64, float32, # complex64, etc. try: return { 'float16': (float, 'npy_float16', 'NPY_FLOAT16'), 'float32': (float, 'npy_float32', 'NPY_FLOAT32'), 'float64': (float, 'npy_float64', 'NPY_FLOAT64'), 'bool': (bool, 'npy_bool', 'NPY_BOOL'), 'uint8': (int, 'npy_uint8', 'NPY_UINT8'), 'int8': (int, 'npy_int8', 'NPY_INT8'), 'uint16': (int, 'npy_uint16', 'NPY_UINT16'), 'int16': (int, 'npy_int16', 'NPY_INT16'), 'uint32': (int, 'npy_uint32', 'NPY_UINT32'), 'int32': (int, 'npy_int32', 'NPY_INT32'), 'uint64': (int, 'npy_uint64', 'NPY_UINT64'), 'int64': (int, 'npy_int64', 'NPY_INT64'), 'complex128': (complex, 'theano_complex128', 'NPY_COMPLEX128'), 'complex64': (complex, 'theano_complex64', 'NPY_COMPLEX64') }[self.dtype] except KeyError: raise TypeError("Unsupported dtype for %s: %s" % (self.__class__.__name__, self.dtype)) def to_scalar_type(self): return scal.get_scalar_type(dtype=self.dtype) def __eq__(self, other): """ Compare True iff other is the same kind of TensorType. """ return type(self) == type(other) and other.dtype == self.dtype \ and other.broadcastable == self.broadcastable def convert_variable(self, var): if (type(self) == type(var.type) and # noqa self.dtype == var.type.dtype and self.ndim == var.type.ndim and all(sb == ob or ob for sb, ob in zip(self.broadcastable, var.type.broadcastable))): return theano.tensor.patternbroadcast(var, self.broadcastable) @staticmethod def may_share_memory(a, b): # This is a method of TensorType, so both a and b should be ndarrays if isinstance(a, np.ndarray) and isinstance(b, np.ndarray): return np.may_share_memory(a, b) else: return False @staticmethod def values_eq(a, b, force_same_dtype=True): # TODO: check to see if the shapes must match # for now, we err on safe side... if a.shape != b.shape: return False if force_same_dtype and a.dtype != b.dtype: return False a_eq_b = (a == b) r = np.all(a_eq_b) if r: return True # maybe the trouble is that there are NaNs a_missing = np.isnan(a) if a_missing.any(): b_missing = np.isnan(b) return np.all(a_eq_b + (a_missing == b_missing)) else: return False @staticmethod def values_eq_approx(a, b, allow_remove_inf=False, allow_remove_nan=False, rtol=None, atol=None): return values_eq_approx(a, b, allow_remove_inf, allow_remove_nan, rtol, atol) def __hash__(self): """Hash equal for same kinds of TensorType""" return hashtype(self) ^ hash(self.dtype) ^ hash(self.broadcastable) ndim = property(lambda self: len(self.broadcastable), doc="number of dimensions") """ Number of dimensions. This read-only property is the preferred way to get the number of dimensions of a `TensorType`. """ def make_variable(self, name=None): """ Return a `TensorVariable` of this type. Parameters ---------- name : str A pretty name to identify this `Variable` when printing and debugging """ return self.Variable(self, name=name) def __str__(self): if self.name: return self.name else: b = self.broadcastable named_broadcastable = {(): 'scalar', (False,): 'vector', (False, True): 'col', (True, False): 'row', (False, False): 'matrix'} if b in named_broadcastable: bcast = named_broadcastable[b] else: if any(b): bcast = str(b) else: bcast = '%iD' % len(b) return "TensorType(%s, %s)" % (str(self.dtype), bcast) def __repr__(self): return str(self) def c_element_type(self): return self.dtype_specs()[1] def c_declare(self, name, sub, check_input=True): """ Override `CLinkerType.c_declare`. """ if(check_input): check = """ typedef %(dtype)s dtype_%(name)s; """ % dict(sub, name=name, dtype=self.dtype_specs()[1]) else: check = "" declaration = """ PyArrayObject* %(name)s; """ % dict(sub, name=name, dtype=self.dtype_specs()[1]) return declaration + check def c_init(self, name, sub): """ Override `CLinkerType.c_init`. """ return """ %(name)s = NULL; """ % dict(sub, name=name, type_num=self.dtype_specs()[2]) def c_extract(self, name, sub, check_input=True): """ Override `CLinkerType.c_extract`. """ if(check_input): check = """ %(name)s = NULL; if (py_%(name)s == Py_None) { // We can either fail here or set %(name)s to NULL and rely on Ops // using tensors to handle the NULL case, but if they fail to do so // they'll end up with nasty segfaults, so this is public service. PyErr_SetString(PyExc_ValueError, "expected an ndarray, not None"); %(fail)s } if (!PyArray_Check(py_%(name)s)) { PyErr_SetString(PyExc_ValueError, "expected an ndarray"); %(fail)s } // We expect %(type_num)s if (!PyArray_ISALIGNED((PyArrayObject*) py_%(name)s)) { PyArrayObject * tmp = (PyArrayObject*) py_%(name)s; PyErr_Format(PyExc_NotImplementedError, "expected an aligned array of type %%ld " "(%(type_num)s), got non-aligned array of type %%ld" " with %%ld dimensions, with 3 last dims " "%%ld, %%ld, %%ld" " and 3 last strides %%ld %%ld, %%ld.", (long int) %(type_num)s, (long int) PyArray_TYPE((PyArrayObject*) py_%(name)s), (long int) PyArray_NDIM(tmp), (long int) (PyArray_NDIM(tmp) >= 3 ? PyArray_DIMS(tmp)[PyArray_NDIM(tmp)-3] : -1), (long int) (PyArray_NDIM(tmp) >= 2 ? PyArray_DIMS(tmp)[PyArray_NDIM(tmp)-2] : -1), (long int) (PyArray_NDIM(tmp) >= 1 ? PyArray_DIMS(tmp)[PyArray_NDIM(tmp)-1] : -1), (long int) (PyArray_NDIM(tmp) >= 3 ? PyArray_STRIDES(tmp)[PyArray_NDIM(tmp)-3] : -1), (long int) (PyArray_NDIM(tmp) >= 2 ? PyArray_STRIDES(tmp)[PyArray_NDIM(tmp)-2] : -1), (long int) (PyArray_NDIM(tmp) >= 1 ? PyArray_STRIDES(tmp)[PyArray_NDIM(tmp)-1] : -1) ); %(fail)s } // This is a TypeError to be consistent with DEBUG_MODE // Note: DEBUG_MODE also tells the name of the container if (PyArray_TYPE((PyArrayObject*) py_%(name)s) != %(type_num)s) { PyErr_Format(PyExc_TypeError, "expected type_num %%d (%(type_num)s) got %%d", %(type_num)s, PyArray_TYPE((PyArrayObject*) py_%(name)s)); %(fail)s } """ % dict(sub, name=name, type_num=self.dtype_specs()[2]) else: check = "" return check + """ %(name)s = (PyArrayObject*)(py_%(name)s); Py_XINCREF(%(name)s); """ % dict(sub, name=name, type_num=self.dtype_specs()[2]) def c_cleanup(self, name, sub): """ Override `CLinkerType.c_cleanup`. """ return """ if (%(name)s) { Py_XDECREF(%(name)s); } """ % locals() def c_sync(self, name, sub): """ Override `CLinkerType.c_sync`. """ fail = sub['fail'] type_num = self.dtype_specs()[2] return """ {Py_XDECREF(py_%(name)s);} if (!%(name)s) { Py_INCREF(Py_None); py_%(name)s = Py_None; } else if ((void*)py_%(name)s != (void*)%(name)s) { py_%(name)s = (PyObject*)%(name)s; } {Py_XINCREF(py_%(name)s);} if (%(name)s && !PyArray_ISALIGNED((PyArrayObject*) py_%(name)s)) { PyErr_Format(PyExc_NotImplementedError, "c_sync: expected an aligned array, got non-aligned array of type %%ld" " with %%ld dimensions, with 3 last dims " "%%ld, %%ld, %%ld" " and 3 last strides %%ld %%ld, %%ld.", (long int) PyArray_TYPE((PyArrayObject*) py_%(name)s), (long int) PyArray_NDIM(%(name)s), (long int) (PyArray_NDIM(%(name)s) >= 3 ? PyArray_DIMS(%(name)s)[PyArray_NDIM(%(name)s)-3] : -1), (long int) (PyArray_NDIM(%(name)s) >= 2 ? PyArray_DIMS(%(name)s)[PyArray_NDIM(%(name)s)-2] : -1), (long int) (PyArray_NDIM(%(name)s) >= 1 ? PyArray_DIMS(%(name)s)[PyArray_NDIM(%(name)s)-1] : -1), (long int) (PyArray_NDIM(%(name)s) >= 3 ? PyArray_STRIDES(%(name)s)[PyArray_NDIM(%(name)s)-3] : -1), (long int) (PyArray_NDIM(%(name)s) >= 2 ? PyArray_STRIDES(%(name)s)[PyArray_NDIM(%(name)s)-2] : -1), (long int) (PyArray_NDIM(%(name)s) >= 1 ? PyArray_STRIDES(%(name)s)[PyArray_NDIM(%(name)s)-1] : -1) ); %(fail)s } """ % locals() def c_headers(self, c_compiler): """ Override `CLinkerObject.c_headers`. """ return scal.get_scalar_type(self.dtype).c_headers(c_compiler) def c_libraries(self, c_compiler): return scal.get_scalar_type(self.dtype).c_libraries(c_compiler) def c_compile_args(self, c_compiler): return scal.get_scalar_type(self.dtype).c_compile_args(c_compiler) def c_support_code(self): """ Override `CLinkerObject.c_support_code`. """ return scal.get_scalar_type(self.dtype).c_support_code() def c_init_code(self): return scal.get_scalar_type(self.dtype).c_init_code() def c_code_cache_version(self): scalar_version = scal.get_scalar_type(self.dtype).c_code_cache_version() if scalar_version: return (11,) + scalar_version else: return () def value_zeros(self, shape): """ Create an numpy ndarray full of 0 values. """ return np.zeros(shape, dtype=self.dtype) def get_shape_info(self, obj): """ Return the information needed to compute the memory size of ``obj``. The memory size is only the data, so this excludes the container. For an ndarray, this is the data, but not the ndarray object and other data structures such as shape and strides. ``get_shape_info()`` and ``get_size()`` work in tandem for the memory profiler. ``get_shape_info()`` is called during the execution of the function. So it is better that it is not too slow. ``get_size()`` will be called on the output of this function when printing the memory profile. Parameters ---------- obj The object that this Type represents during execution. Returns ------- object Python object that ``self.get_size()`` understands. """ return obj.shape def get_size(self, shape_info): """ Number of bytes taken by the object represented by shape_info. Parameters ---------- shape_info The output of the call to get_shape_info(). Returns ------- int The number of bytes taken by the object described by ``shape_info``. """ if shape_info: return np.prod(shape_info) * np.dtype(self.dtype).itemsize else: # a scalar return np.dtype(self.dtype).itemsize theano.compile.ops.expandable_types += (TensorType,) def values_eq_approx(a, b, allow_remove_inf=False, allow_remove_nan=False, rtol=None, atol=None): """ Parameters ---------- allow_remove_inf If True, when there is an inf in a, we allow any value in b in that position. Event -inf allow_remove_nan If True, when there is a nan in a, we allow any value in b in that position. Event +-inf rtol Relative tolerance, passed to _allclose. atol Absolute tolerance, passed to _allclose. """ if isinstance(a, np.ndarray) and isinstance(b, np.ndarray): if a.shape != b.shape: return False if a.dtype != b.dtype: return False if str(a.dtype) not in theano.tensor.continuous_dtypes: return np.all(a == b) else: cmp = theano.tensor.basic._allclose(a, b, rtol=rtol, atol=atol) if cmp: # Numpy claims they are close, this is good enough for us. return True # Numpy is unhappy, but it does not necessarily mean that a and # b are different. Indeed, Numpy does not like missing values # and will return False whenever some are found in a or b. # The proper way would be to use the MaskArray stuff available # in Numpy. However, it looks like it has been added to Numpy's # core recently, so it may not be available to everyone. Thus, # for now we use a home-made recipe, that should probably be # revisited in the future. a_missing = np.isnan(a) a_inf = np.isinf(a) if not (a_missing.any() or (allow_remove_inf and a_inf.any())): # There are no missing values in a, thus this is not the # reason why numpy.allclose(a, b) returned False. _logger.info( 'numpy allclose failed for abs_err %f and rel_err %f', np.max(abs(a - b)), np.max(abs(a - b) / (abs(a) + abs(b)))) return False # The following line is what numpy.allclose bases its decision # upon, according to its documentation. rtol = 1.0000000000000001e-05 atol = 1e-8 cmp_elemwise = (np.absolute(a - b) <= (atol + rtol * np.absolute(b))) # Find places where both a and b have missing values. both_missing = a_missing * np.isnan(b) # Find places where both a and b have inf of the same sign. both_inf = a_inf * np.isinf(b) # cmp_elemwise is weird when we have inf and -inf. # set it to False cmp_elemwise = np.where( both_inf & cmp_elemwise, a == b, cmp_elemwise) # check the sign of the inf both_inf = np.where(both_inf, (a == b), both_inf) if allow_remove_inf: both_inf += a_inf if allow_remove_nan: both_missing += a_missing # Combine all information. return (cmp_elemwise + both_missing + both_inf).all() return False def values_eq_approx_remove_inf(a, b): return values_eq_approx(a, b, True) def values_eq_approx_remove_nan(a, b): return values_eq_approx(a, b, False, True) def values_eq_approx_remove_inf_nan(a, b): return values_eq_approx(a, b, True, True) def values_eq_approx_always_true(a, b): return True # Register TensorType C code for ViewOp. theano.compile.register_view_op_c_code( TensorType, """ Py_XDECREF(%(oname)s); %(oname)s = %(iname)s; Py_XINCREF(%(oname)s); """, version=1) # Register TensorType C code for Shape Op. theano.compile.register_shape_c_code( TensorType, """ npy_intp shape[] = {PyArray_NDIM(%(iname)s)}; if(%(oname)s == NULL || (PyArray_DIMS(%(oname)s)[0] != shape[0])) { Py_XDECREF(%(oname)s); %(oname)s = (PyArrayObject*) PyArray_SimpleNew(1, shape, NPY_INT64); } for(int i=0;i=PyArray_NDIM(%(iname)s)){ PyErr_SetString(PyExc_TypeError, "Number of dimensions lower than expected"); %(fail)s } """, version=3) # Register TensorType C code for DeepCopyOp theano.compile.register_deep_copy_op_c_code( TensorType, """ int alloc = %(oname)s == NULL; for(int i=0; !alloc && i 2, with first dimension batches' % self.__class__.__name__) if s is None: s = a.shape[1:] s = T.as_tensor_variable(s) else: s = T.as_tensor_variable(s) if s.dtype not in T.integer_dtypes: raise TypeError('%s: length of the transformed axis must be' ' of type integer' % self.__class__.__name__) return gof.Apply(self, [a, s], [self.output_type(a)()]) def perform(self, node, inputs, output_storage): a = inputs[0] s = inputs[1] A = np.fft.rfftn(a, s=tuple(s)) # Format output with two extra dimensions for real and imaginary # parts. out = np.zeros(A.shape + (2,), dtype=a.dtype) out[..., 0], out[..., 1] = np.real(A), np.imag(A) output_storage[0][0] = out def grad(self, inputs, output_grads): gout, = output_grads s = inputs[1] # Divide the last dimension of the output gradients by 2, they are # double-counted by the real-IFFT due to symmetry, except the first # and last elements (for even transforms) which are unique. idx = [slice(None)] * (gout.ndim - 2) \ + [slice(1, (s[-1] // 2) + (s[-1] % 2))] + [slice(None)] gout = T.set_subtensor(gout[idx], gout[idx] * 0.5) return [irfft_op(gout, s), DisconnectedType()()] def connection_pattern(self, node): # Specificy that shape input parameter has no connection to graph and gradients. return [[True], [False]] rfft_op = RFFTOp() class IRFFTOp(gof.Op): __props__ = () def output_type(self, inp): # remove extra dim for real/imag return T.TensorType(inp.dtype, broadcastable=[False] * (inp.type.ndim - 1)) def make_node(self, a, s=None): a = T.as_tensor_variable(a) if a.ndim < 3: raise TypeError('%s: input must have dimension >= 3, with ' % self.__class__.__name__ + 'first dimension batches and last real/imag parts') if s is None: s = a.shape[1:-1] s = T.set_subtensor(s[-1], (s[-1] - 1) * 2) s = T.as_tensor_variable(s) else: s = T.as_tensor_variable(s) if s.dtype not in T.integer_dtypes: raise TypeError('%s: length of the transformed axis must be' ' of type integer' % self.__class__.__name__) return gof.Apply(self, [a, s], [self.output_type(a)()]) def perform(self, node, inputs, output_storage): a = inputs[0] s = inputs[1] # Reconstruct complex array from two float dimensions inp = a[..., 0] + 1j * a[..., 1] out = np.fft.irfftn(inp, s=tuple(s)) # Remove numpy's default normalization # Cast to input type (numpy outputs float64 by default) output_storage[0][0] = (out * s.prod()).astype(a.dtype) def grad(self, inputs, output_grads): gout, = output_grads s = inputs[1] gf = rfft_op(gout, s) # Multiply the last dimension of the gradient by 2, they represent # both positive and negative frequencies, except the first # and last elements (for even transforms) which are unique. idx = [slice(None)] * (gf.ndim - 2) \ + [slice(1, (s[-1] // 2) + (s[-1] % 2))] + [slice(None)] gf = T.set_subtensor(gf[idx], gf[idx] * 2) return [gf, DisconnectedType()()] def connection_pattern(self, node): # Specificy that shape input parameter has no connection to graph and gradients. return [[True], [False]] irfft_op = IRFFTOp() def rfft(inp, norm=None): """ Performs the fast Fourier transform of a real-valued input. The input must be a real-valued variable of dimensions (m, ..., n). It performs FFTs of size (..., n) on m batches. The output is a tensor of dimensions (m, ..., n//2+1, 2). The second to last dimension of the output contains the n//2+1 non-trivial elements of the real-valued FFTs. The real and imaginary parts are stored as a pair of float arrays. Parameters ---------- inp Array of floats of size (m, ..., n), containing m inputs of size (..., n). norm : {None, 'ortho', 'no_norm'} Normalization of transform. Following numpy, default *None* normalizes only the inverse transform by n, 'ortho' yields the unitary transform (:math:`1/\sqrt n` forward and inverse). In addition, 'no_norm' leaves the transform unnormalized. """ s = inp.shape[1:] cond_norm = _unitary(norm) scaling = 1 if cond_norm == "ortho": scaling = T.sqrt(s.prod().astype(inp.dtype)) return rfft_op(inp, s) / scaling def irfft(inp, norm=None, is_odd=False): """ Performs the inverse fast Fourier Transform with real-valued output. The input is a variable of dimensions (m, ..., n//2+1, 2) representing the non-trivial elements of m real-valued Fourier transforms of initial size (..., n). The real and imaginary parts are stored as a pair of float arrays. The output is a real-valued variable of dimensions (m, ..., n) giving the m inverse FFTs. Parameters ---------- inp Array of size (m, ..., n//2+1, 2), containing m inputs with n//2+1 non-trivial elements on the last dimension and real and imaginary parts stored as separate real arrays. norm : {None, 'ortho', 'no_norm'} Normalization of transform. Following numpy, default *None* normalizes only the inverse transform by n, 'ortho' yields the unitary transform (:math:`1/\sqrt n` forward and inverse). In addition, 'no_norm' leaves the transform unnormalized. is_odd : {True, False} Set to True to get a real inverse transform output with an odd last dimension of length (N-1)*2 + 1 for an input last dimension of length N. """ if is_odd not in (True, False): raise ValueError("Invalid value %s for id_odd, must be True or False" % is_odd) s = inp.shape[1:-1] if is_odd: s = T.set_subtensor(s[-1], (s[-1] - 1) * 2 + 1) else: s = T.set_subtensor(s[-1], (s[-1] - 1) * 2) cond_norm = _unitary(norm) scaling = 1 # Numpy's default normalization is 1/N on the inverse transform. if cond_norm is None: scaling = s.prod().astype(inp.dtype) elif cond_norm == "ortho": scaling = T.sqrt(s.prod().astype(inp.dtype)) return irfft_op(inp, s) / scaling def _unitary(norm): if norm not in (None, "ortho", "no_norm"): raise ValueError("Invalid value %s for norm, must be None, 'ortho' or " "'no norm'" % norm) return norm Theano-1.0.0/theano/tensor/type_other.py00006000163562€ÿÿÿþ0000000746713203335532021631 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division # # Slice type and Op. None Type and NoneConst. # import numpy as np import theano from theano.gof import Apply, Constant, Generic, Op, Type, hashtype from theano.gradient import DisconnectedType def as_int_none_variable(x): if x is None: return NoneConst elif NoneConst.equals(x): return x x = theano.tensor.as_tensor_variable(x, ndim=0) if x.type.dtype not in theano.tensor.integer_dtypes: raise TypeError('index must be integers') return x class MakeSlice(Op): __props__ = () def make_node(self, slc, stop=None, step=None): # We need to accept and handle in make_node inputs the node # inputs to allow redoing a new op elsewhere in the graph by # optimization. if isinstance(slc, slice): assert stop is None assert step is None inp = [slc.start, slc.stop, slc.step] else: inp = [slc, stop, step] return Apply(self, list(map(as_int_none_variable, inp)), [slicetype()]) def perform(self, node, inp, out_): out, = out_ out[0] = slice(*inp) def grad(self, inputs, grads): return [DisconnectedType()() for i in inputs] make_slice = MakeSlice() class SliceType(Type): def filter(self, x, strict=False, allow_downcast=None): if isinstance(x, slice): return x else: raise TypeError('Expected a slice!') def __str__(self): return "slice" def __eq__(self, other): return type(self) == type(other) def __hash__(self): return hashtype(self) @staticmethod def may_share_memory(a, b): # Slices never shared memory between object return isinstance(a, slice) and a is b slicetype = SliceType() class SliceConstant(Constant): def __init__(self, type, data, name=None): assert isinstance(data, slice) # Numpy ndarray aren't hashable, so get rid of them. if isinstance(data.start, np.ndarray): assert data.start.ndim == 0 assert str(data.start.dtype) in theano.tensor.integer_dtypes data = slice(int(data.start), data.stop, data.step) elif isinstance(data.stop, np.ndarray): assert data.stop.ndim == 0 assert str(data.stop.dtype) in theano.tensor.integer_dtypes data = slice(data.start, int(data.stop), data.step) elif isinstance(data.step, np.ndarray): assert data.step.ndim == 0 assert str(data.step.dtype) in theano.tensor.integer_dtypes data = slice(data.start, int(data.stop), data.step) Constant.__init__(self, type, data, name) def signature(self): return (SliceConstant, self.data.start, self.data.stop, self.data.step) def __str__(self): return "%s{%s, %s, %s}" % (self.__class__.__name__, self.data.start, self.data.stop, self.data.step) SliceType.Constant = SliceConstant class NoneTypeT(Generic): """ Inherit from Generic to have c code working. """ def filter(self, x, strict=False, allow_downcast=None): if x is None: return x else: raise TypeError('Expected None!') @staticmethod def may_share_memory(a, b): # None never share memory between object, in the sence of DebugMode. # Python None are singleton return False none_type_t = NoneTypeT() # This is a variable instance. It can be used only once per fgraph. # So use NoneConst.clone() before using it in a Theano graph. # Use NoneConst.equals(x) to check if two variable are NoneConst. NoneConst = Constant(none_type_t, None, name='NoneConst') Theano-1.0.0/theano/tensor/subtensor.py00006000163562€ÿÿÿþ0000025235713203335532021473 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import sys from textwrap import dedent import collections import warnings import logging import numpy as np from six import integer_types from six.moves import xrange import theano from theano.compat import izip from theano.gradient import DisconnectedType from theano import gof from theano.gof import Apply, hashtype, Op, Type, MethodNotDefined, ParamsType from theano.printing import pprint from theano import scalar as scal from theano.tensor.basic import alloc from theano.tensor.basic import (addbroadcast, clip, get_scalar_constant_value, TensorType, NotScalarConstantError) from theano.tensor.elemwise import DimShuffle from theano.tensor.type_other import NoneConst, SliceType, NoneTypeT, make_slice from theano import config from .inc_code import inc_code _logger = logging.getLogger("theano.tensor.subtensor") # Do a lazy import of the sparse module sparse_module_ref = None class AdvancedIndexingError(TypeError): """ Raised when Subtensor is asked to perform advanced indexing. """ pass class AdvancedBooleanIndexingError(TypeError): """ Raised when Subtensor is asked to perform advanced indexing with boolean masks. """ pass ########## # Helpful functions to deal with Subtensor and IncSubtensor ########## def make_constant(args): """ Convert python litterals to theano constants in subtensor arguments. """ def conv(a): if a is None: return a elif isinstance(a, slice): return slice(conv(a.start), conv(a.stop), conv(a.step)) elif isinstance(a, (integer_types, np.integer)): return scal.ScalarConstant(scal.int64, a) else: return a return tuple(map(conv, args)) def get_idx_list(inputs, idx_list, get_count=False): """ Given a list of inputs to the subtensor and its idx_list reorders the inputs according to the idx list to get the right values. If get_counts=True, instead returns the number of inputs consumed during this process. """ # The number of indices n = len(inputs) - 1 # The subtensor (or idx_list) does not depend on the inputs. if n == 0: return tuple(idx_list) indices = list(reversed(list(inputs[1:]))) # General case def convert(entry): if isinstance(entry, gof.Type): return indices.pop() elif isinstance(entry, slice): return slice(convert(entry.start), convert(entry.stop), convert(entry.step)) else: return entry cdata = tuple(map(convert, idx_list)) if get_count: return n - len(indices) else: return cdata def get_canonical_form_slice(theslice, length): """ Given a slice [start:stop:step] transform it into a canonical form that respects the conventions imposed by python and numpy. In a canonical form a slice is represented by a canonical form slice, in which 0 <= start <= stop <= length and step > 0, and a flag which says if the resulting set of numbers needs to be reversed or not. """ from theano.tensor import switch, lt, ge, sgn if isinstance(theslice, slice): def analyze(x): try: x_constant = get_scalar_constant_value(x) is_constant = True except theano.tensor.NotScalarConstantError: x_constant = theano.tensor.extract_constant(x) is_constant = False return x_constant, is_constant start, is_start_constant = analyze(theslice.start) stop, is_stop_constant = analyze(theslice.stop) step, is_step_constant = analyze(theslice.step) length, is_length_constant = analyze(length) if step is None: step = 1 is_step_constant = True # First handle the easier and common case where `step` is 1 and # either `start` or `stop` is a range boundary. More specializations # could be added later. This makes the resulting graph smaller than # in the generic case below. if step == 1: is_start_0 = ( start is None or start == 0 or (is_start_constant and is_length_constant and start < 0 and start + length <= 0)) is_stop_length = ( stop is None or stop in [length, sys.maxsize] or (is_stop_constant and is_length_constant and stop >= length)) if is_start_0: # 0:stop:1 if is_stop_length: # Full slice. return slice(0, length, 1), 1 if is_stop_constant and stop >= 0: return (slice(0, switch(lt(stop, length), stop, length), 1), 1) stop_plus_len = stop + length stop = switch( lt(stop, 0), # stop < 0 switch( lt(stop_plus_len, 0), # stop + len < 0 0, # stop + len >= 0 stop_plus_len), # stop >= 0: use min(stop, length) switch(lt(stop, length), stop, length)) return slice(0, stop, 1), 1 elif is_stop_length: # start:length:1 if is_start_constant and start >= 0: return slice(switch(lt(start, length), start, length), length, 1), 1 start_plus_len = start + length start = switch( lt(start, 0), # start < 0 switch( lt(start_plus_len, 0), # start + len < 0 0, # start + len >= 0 start_plus_len), # start >= 0: use min(start, length) switch(lt(start, length), start, length)) return slice(start, length, 1), 1 # This is the generic case. if is_step_constant: # When we know the sign of `step`, the graph can be made simpler. assert step != 0 if step > 0: def switch_neg_step(a, b): return b abs_step = step sgn_step = 1 else: def switch_neg_step(a, b): return a abs_step = -step sgn_step = -1 else: is_step_neg = lt(step, 0) def switch_neg_step(a, b): return switch(is_step_neg, a, b) abs_step = abs(step) sgn_step = sgn(step) defstart = switch_neg_step(length - 1, 0) defstop = switch_neg_step(-1, length) if start is None: start = defstart else: start = switch(lt(start, 0), start + length, start) start = switch(lt(start, 0), switch_neg_step(-1, 0), start) start = switch(ge(start, length), switch_neg_step(length - 1, length), start) if stop is None or stop == sys.maxsize: # The special "maxsize" case is probably not needed here, # as slices containing maxsize are not generated by # __getslice__ anymore. stop = defstop else: stop = switch(lt(stop, 0), stop + length, stop) stop = switch(lt(stop, 0), -1, stop) stop = switch(ge(stop, length), length, stop) nw_stop = switch_neg_step(start + 1, stop) slice_len = (start - stop - 1) // abs_step + 1 slice_len = switch(lt(slice_len, 0), 0, slice_len) neg_start = nw_stop - (slice_len - 1) * abs_step - 1 neg_start = switch(lt(neg_start, 0), (nw_stop - 1), neg_start) nw_start = switch_neg_step(neg_start, start) nw_start = switch(lt(nw_start, 0), 0, nw_start) nw_stop = switch(lt(nw_stop, 0), 0, nw_stop) # Ensure start <= stop. nw_start = switch(lt(nw_start, nw_stop), nw_start, nw_stop) nw_step = abs_step if step != 1: reverse = sgn_step return slice(nw_start, nw_stop, nw_step), reverse else: return slice(nw_start, nw_stop, nw_step), 1 else: value = theano.tensor.extract_constant(theslice) value = switch(lt(value, 0), (value + length), value) return value, 1 class Subtensor(Op): """ Return a subtensor view. The inputs array is the tensor x, followed by scalar integer types. TODO: WRITEME: how are the scalar integer variables formatted? This class uses a relatively complex internal representation of the inputs to remember how the input tensor x should be sliced. idx_list: instance variable TODO: WRITEME: is this a list or a tuple? (old docstring gives two conflicting descriptions) elements are either integers, theano scalar types, or slices. one element per "explicitly named dimension" TODO: WRITEME: what is an "explicitly named dimension" ? if integer: indexes into the inputs array if slice: start/stop/step members of each slice are integer indices into the inputs array or None integer indices be actual integers or theano scalar types Note that the idx_list defines the Op, so two Subtensor instances are considered to be different Ops if they have different idx_list fields. This means that the entries in it are theano Types, not theano Variables. @todo: add support for advanced tensor indexing (in Subtensor_dx too). """ e_subslice = 'nested slicing is not supported' e_indextype = "Invalid index type or slice for Subtensor" debug = 0 check_input = False view_map = {0: [0]} _f16_ok = True __props__ = ("idx_list",) @staticmethod def collapse(idxs, cond): """ Parameters ---------- idxs : a list of indices or slices. cond : a callable that returns a bool Returns ------- list idxs, with the slices flattened out into a list. If cond is true for an entry, does not flatten it. """ ret = [] def helper(entry): if cond(entry): ret.append(entry) elif isinstance(entry, slice): helper(entry.start) helper(entry.stop) helper(entry.step) for idx in idxs: helper(idx) return ret @staticmethod def convert(entry, slice_ok=True): """ Change references to Variables into references to Types. The "idx_list" field is unique to each Subtensor instance. It is not unique to each Apply node, so it should not refer to specific Variables. TODO: WRITEME: This method also accepts "entry" already being a Type; when would that happen? """ invalid_scal_types = [scal.float64, scal.float32, scal.float16] scal_types = [scal.int64, scal.int32, scal.int16, scal.int8] tensor_types = [theano.tensor.lscalar, theano.tensor.iscalar, theano.tensor.wscalar, theano.tensor.bscalar] invalid_tensor_types = [theano.tensor.fscalar, theano.tensor.dscalar, theano.tensor.cscalar, theano.tensor.zscalar] if (isinstance(entry, (np.ndarray, theano.tensor.Variable)) and hasattr(entry, 'dtype') and entry.dtype == 'bool'): raise AdvancedBooleanIndexingError(Subtensor.e_indextype, entry) if (isinstance(entry, gof.Variable) and (entry.type in invalid_scal_types or entry.type in invalid_tensor_types)): raise TypeError("Expected an integer") if isinstance(entry, gof.Variable) and entry.type in scal_types: return entry.type elif isinstance(entry, gof.Type) and entry in scal_types: return entry if (isinstance(entry, gof.Variable) and entry.type in tensor_types and np.all(entry.type.broadcastable)): return scal.get_scalar_type(entry.type.dtype) elif (isinstance(entry, gof.Type) and entry in tensor_types and np.all(entry.broadcastable)): return scal.get_scalar_type(entry.dtype) elif slice_ok and isinstance(entry, slice): a = entry.start b = entry.stop c = entry.step if a is not None: slice_a = Subtensor.convert(a, False) else: slice_a = None if b is not None and b != sys.maxsize: # The special "maxsize" case is probably not needed here, # as slices containing maxsize are not generated by # __getslice__ anymore. slice_b = Subtensor.convert(b, False) else: slice_b = None if c is not None: slice_c = Subtensor.convert(c, False) else: slice_c = None return slice(slice_a, slice_b, slice_c) elif isinstance(entry, (integer_types, np.integer)): # Disallow the use of python scalars in idx_list raise TypeError("Python scalar in idx_list." "Please report this error to theano-dev.") else: raise AdvancedIndexingError(Subtensor.e_indextype, entry) def get_constant_idx(self, inputs, allow_partial=False, only_process_constants=False, elemwise=True): """ Return the idx_list with constant inputs replaced by their python scalar equivalent. May raise `theano.tensor.NotScalarConstantError` if the idx contains non-constant entries. If allow_partial is True, then entries that are not constant will stay as their input variable rather than raising an exception. None entries are always left as-is. Parameters ---------- only_process_constants If True, we only attempt to obtain the value of an index/slice if it's directly constant and don't try to dig through dimshuffles, fills, allocs, and other to figure out its value. Examples -------- Example usage where v, a are appropriately typed theano variables : >>> b = a[v, 1:3] >>> b.owner.op.idx_list (Scalar(int64), slice(Scalar(int64), Scalar(int64), None)) >>> b.owner.op.get_constant_idx(b.owner.inputs, allow_partial=True) [v, slice(1, 3, None)] >>> b.owner.op.get_constant_idx(b.owner.inputs) NotScalarConstantError: v """ real_idx = get_idx_list(inputs, self.idx_list) def conv(val): if val is None: return None elif isinstance(val, slice): return slice(conv(val.start), conv(val.stop), conv(val.step)) else: try: return get_scalar_constant_value( val, only_process_constants=only_process_constants, elemwise=elemwise) except theano.tensor.NotScalarConstantError: if allow_partial: return val else: raise return list(map(conv, real_idx)) def __init__(self, idx_list): self.idx_list = tuple(map(self.convert, idx_list)) @staticmethod def my_as_scalar(a): # Since scal.as_scalar does not know about tensor types (it would # create a circular import) , this method converts either a # TensorVariable or a ScalarVariable to a scalar. if isinstance(a, gof.Variable) and isinstance(a.type, TensorType): return theano.tensor.scalar_from_tensor(a) else: return scal.as_scalar(a) def make_node(self, x, *inputs): """ Parameters ---------- x The tensor to take a subtensor of. inputs A list of theano Scalars. """ x = theano.tensor.as_tensor_variable(x) inputs = tuple(self.my_as_scalar(a) for a in inputs) idx_list = list(self.idx_list) if len(idx_list) > x.type.ndim: raise IndexError('too many indices for array') input_types = Subtensor.collapse(idx_list, lambda entry: isinstance(entry, gof.Type)) if len(inputs) != len(input_types): raise IndexError( "Not enough inputs to fill in the Subtensor template.", inputs, idx_list) for input, expected_type in izip(inputs, input_types): if input.type != expected_type: raise TypeError( "Wrong type for Subtensor template. Expected %s, got %s." % (input.type, expected_type)) # infer the broadcasting pattern padded = (self.get_constant_idx((None,) + inputs, allow_partial=True) + [slice(None, None, None)] * (x.type.ndim - len(idx_list))) broadcastable = [] for i, (p, bc) in enumerate(izip(padded, x.type.broadcastable)): if isinstance(p, slice): if bc: start = p.start try: start = get_scalar_constant_value(start) except NotScalarConstantError: pass if start is None or start == 0: start = p.start if start is None: start = 0 if (p.stop is None or (isinstance(p.stop, (integer_types, np.integer, np.ndarray)) and p.stop > start)): broadcastable.append(True) continue broadcastable.append(False) return gof.Apply(self, (x, ) + inputs, [theano.tensor.tensor(dtype=x.type.dtype, broadcastable=broadcastable)]) def perform(self, node, inputs, out_): out, = out_ x = inputs[0] cdata = get_idx_list(inputs, self.idx_list) if len(cdata) == 1: cdata = cdata[0] out[0] = np.asarray(x.__getitem__(cdata)) def infer_shape(self, node, shapes): xshp = shapes[0] assert len(xshp) == node.inputs[0].ndim outshp = [] actual_idx_list = list(get_idx_list(node.inputs, self.idx_list)) padded = (actual_idx_list + [slice(None, None, None)] * (len(xshp) - len(self.idx_list))) i = 0 for idx, xl in izip(padded, xshp): if isinstance(idx, slice): # If it is the default (None, None, None) slice, or a variant, # the shape will be xl if ((idx.start in [None, 0]) and (idx.stop in [None, sys.maxsize]) and (idx.step is None or idx.step == 1)): outshp.append(xl) else: cnf = get_canonical_form_slice(idx, xl)[0] if cnf.step == 1: length = cnf.stop - cnf.start else: length = (cnf.stop - cnf.start - 1) // cnf.step + 1 outshp.append(length) i += 1 else: # That dimension is dropped pass assert i == node.outputs[0].ndim assert len(outshp) == node.outputs[0].ndim return [outshp] def grad(self, inputs, grads): gz, = grads x = inputs[0] rest = inputs[1:] if x.dtype in theano.tensor.discrete_dtypes: first = x.zeros_like().astype(theano.config.floatX) else: # For best optimization, we let this as an inc. # This allow the opt local_IncSubtensor_serialize to apply first. # We have an optimization that will convert this to a # set subtensor here at: # theano/tensor/opt.py:local_incsubtensor_of_zeros_to_setsubtensor() first = IncSubtensor(self.idx_list)(x.zeros_like(), gz, *rest) return ([first] + [DisconnectedType()()] * len(rest)) def connection_pattern(self, node): rval = [[True]] for ipt in node.inputs[1:]: rval.append([False]) return rval def __hash__(self): # TODO: optimize by cache this hash value msg = [] for entry in self.idx_list: if isinstance(entry, slice): msg += [(entry.start, entry.stop, entry.step)] else: msg += [entry] idx_list = tuple(msg) # backport # idx_list = tuple((entry.start, entry.stop, entry.step) # if isinstance(entry, slice) # else entry # for entry in self.idx_list) return hash(idx_list) @staticmethod def str_from_slice(entry): msg = [] for x in [entry.start, entry.stop, entry.step]: if x is None: msg.append("") else: msg.append(str(x)) return ":".join(msg) def __str__(self): indices = [] for entry in self.idx_list: if isinstance(entry, slice): indices.append(self.str_from_slice(entry)) else: indices.append(str(entry)) return "%s{%s}" % (self.__class__.__name__, ", ".join(indices)) @staticmethod def default_helper_c_code_args(): """ Returns a dictionary of default arguments to helper_c_code. """ return {"c_prefix": "PyArray", "strides_mul": 1} @staticmethod def helper_c_code(node, name, inputs, outputs, sub, idx_list, view_ndim, c_prefix=None, strides_mul=None): """ The parameters c_prefix are there to allow reusing this function on PyArray and GpuArray object. This fct take as input the x. """ default_args = Subtensor.default_helper_c_code_args() if strides_mul is None: strides_mul = default_args['strides_mul'] if c_prefix is None: c_prefix = default_args['c_prefix'] # # two arrays are created in C code: # is_slice: len == ndim, 0 means int, 1 means slice # subtensor_spec: len = n_ints + 3 * n_slices # fail = sub['fail'] init_cmds = [] # initialization for subtensor_spec is_slice = [] # TODO: change that, it might lead to unexpected results, # see assembla-#767 NONE_CODE = sys.maxsize - 1 pos = [0, 1] # annoying version of global variable for init_entry def inc_spec_pos(amt): pos[0] += amt def inc_input_pos(amt): pos[1] += amt def spec_pos(): return pos[0] def input_pos(): return pos[1] def init_entry(entry, depth=0): if isinstance(entry, (np.integer, integer_types)): init_cmds.append( "subtensor_spec[%i] = %i;" % (spec_pos(), entry)) inc_spec_pos(1) if depth == 0: is_slice.append(0) elif isinstance(entry, Type): init_cmds.append( "subtensor_spec[%i] = %s;" % (spec_pos(), inputs[input_pos()])) inc_spec_pos(1) inc_input_pos(1) if depth == 0: is_slice.append(0) elif entry is None: init_cmds.append( "subtensor_spec[%i] = %i;" % (spec_pos(), NONE_CODE)) inc_spec_pos(1) if depth == 0: is_slice.append(0) elif depth == 0 and isinstance(entry, slice): init_entry(entry.start, depth + 1) init_entry(entry.stop, depth + 1) init_entry(entry.step, depth + 1) is_slice.append(1) else: assert 0, entry for entry in idx_list: init_entry(entry) # make sure we used all inputs assert input_pos() == len(inputs), input_pos() assert len(is_slice) <= node.inputs[0].ndim, node.inputs[0].ndim len_is_slice = len(is_slice) len_subtensor_spec = spec_pos() subensor_spec = "npy_intp subtensor_spec[%(len_subtensor_spec)s];" % locals() if len_subtensor_spec == 0: subensor_spec = "npy_intp * subtensor_spec = NULL;" if is_slice: is_slice_init = "int is_slice[] = {" + ",".join([str(s) for s in is_slice]) + "};" else: is_slice_init = "int* is_slice = NULL;" subtensor_init = "\n".join(init_cmds) x, = inputs[:1] z, = outputs if view_ndim: rval = """ // Argument of the view npy_intp xview_dims[%(view_ndim)s]; npy_intp xview_strides[%(view_ndim)s]; """ % locals() else: rval = """ // Argument of the view npy_intp* xview_dims = NULL; npy_intp* xview_strides = NULL; """ rval += """ // One more argument of the view npy_intp xview_offset = 0; // The subtensor is created by iterating over the dimensions // and updating stride, shape, and data pointers %(is_slice_init)s %(subensor_spec)s %(subtensor_init)s; int spec_pos = 0; //position in subtensor_spec int inner_ii = 0; // the current dimension of zview int outer_ii = 0; // current dimension of z for (; outer_ii < %(len_is_slice)s; ++outer_ii) { if (is_slice[outer_ii]) { npy_intp length = %(c_prefix)s_DIMS(%(x)s)[outer_ii]; npy_intp slicelength; npy_intp start = subtensor_spec[spec_pos+0]; npy_intp stop = subtensor_spec[spec_pos+1]; npy_intp step = subtensor_spec[spec_pos+2]; if (step == %(NONE_CODE)s) step = 1; npy_intp defstart = step < 0 ? length-1 : 0; npy_intp defstop = step < 0 ? -1 : length; // logic adapted from // PySlice_GetIndicesEx in python source if (!step) { PyErr_Format(PyExc_ValueError, "slice step cannot be zero"); %(fail)s; } if (start == %(NONE_CODE)s) { start = defstart; } else { if (start < 0) start += length; if (start < 0) start = (step < 0) ? -1 : 0; if (start >= length) start = (step < 0) ? length - 1 : length; } if (stop == %(NONE_CODE)s) { stop = defstop; } else { if (stop < 0) stop += length; if (stop < 0) stop = (step < 0) ? -1 : 0; if (stop >= length) stop = (step < 0) ? length - 1 : length; } if ((step < 0 && stop >= start) || (step > 0 && start >= stop)) { slicelength = 0; } else if (step < 0) { slicelength = (stop-start+1)/step+1; } else { slicelength = (stop-start-1)/step+1; } if (0){ fprintf(stdout, "start %%zi\\n", start); fprintf(stdout, "stop %%zi\\n", stop); fprintf(stdout, "step %%zi\\n", step); fprintf(stdout, "length %%zi\\n", length); fprintf(stdout, "slicelength %%zi\\n", slicelength); } assert (slicelength <= length); xview_offset += (npy_intp)%(c_prefix)s_STRIDES(%(x)s)[outer_ii] * start * %(strides_mul)s; xview_dims[inner_ii] = slicelength; xview_strides[inner_ii] = (npy_intp)%(c_prefix)s_STRIDES(%(x)s)[outer_ii] * step; inner_ii += 1; spec_pos += 3; } else // tuple coord `outer_ii` is an int { int idx = subtensor_spec[spec_pos]; if (idx < 0) idx += %(c_prefix)s_DIMS(%(x)s)[outer_ii]; if (idx >= 0) { if (idx < %(c_prefix)s_DIMS(%(x)s)[outer_ii]) { xview_offset += (npy_intp)%(c_prefix)s_STRIDES(%(x)s)[outer_ii] * idx * %(strides_mul)s; } else { PyErr_Format(PyExc_IndexError,"index out of bounds"); %(fail)s; } } else { PyErr_Format(PyExc_IndexError,"index out of bounds"); %(fail)s; } spec_pos += 1; } } assert (inner_ii <= %(view_ndim)s); while (inner_ii < %(view_ndim)s) { assert (outer_ii < %(c_prefix)s_NDIM(%(x)s)); xview_dims[inner_ii] = %(c_prefix)s_DIMS(%(x)s)[outer_ii]; xview_strides[inner_ii] = %(c_prefix)s_STRIDES(%(x)s)[outer_ii]; inner_ii += 1; outer_ii += 1; } """ % locals() # print rval return rval @staticmethod def helper_c_code_cache_version(): return (9,) def c_code(self, node, name, inputs, outputs, sub): # DEBUG if not isinstance(node.inputs[0].type, theano.tensor.TensorType): raise NotImplementedError() x = inputs[0] z, = outputs ndim = node.inputs[0].ndim view_ndim = node.outputs[0].ndim fail = sub['fail'] decl = "PyArrayObject * xview = NULL;" checkNDim = """ if (PyArray_NDIM(%(x)s) != %(ndim)s){ PyErr_SetString(PyExc_ValueError, "Expected %(ndim)s dimensions input" ); %(fail)s } """ % locals() get_xview = self.helper_c_code(node, name, inputs, outputs, sub, self.idx_list, view_ndim) build_view = """ //TODO: give this Op a second output so that this view can be cached //TODO: alternatively, fix the memory leak on failure Py_INCREF(PyArray_DESCR(%(x)s)); xview = (PyArrayObject*)PyArray_NewFromDescr( &PyArray_Type, PyArray_DESCR(%(x)s), %(view_ndim)s, xview_dims, xview_strides, PyArray_BYTES(%(x)s) + xview_offset, PyArray_FLAGS(%(x)s), NULL); assert (PyArray_NDIM(xview) == %(view_ndim)s); if (!xview) { %(fail)s; } """ % locals() finish_view = """ Py_XDECREF(%(z)s); Py_INCREF(py_%(x)s); PyArray_SetBaseObject(xview, py_%(x)s); assert(py_%(x)s == (PyObject*)%(x)s); %(z)s = xview; """ % locals() return (decl + checkNDim + "{" + get_xview + build_view + finish_view + "}") def c_code_cache_version(self): hv = self.helper_c_code_cache_version() # If `helper_c_code_cache_version` is not versioned we do not want to # have a versioned version of this op's C code. if len(hv) == 0: return () return (4, hv) def R_op(self, inputs, eval_points): # Subtensor is not differentiable wrt to its indices, therefore we # do not even need to consider the eval_points provided for those # (they should be defaulted to zeros_like by the global R_op) if eval_points[0] is None: return [None] return self(eval_points[0], *inputs[1:], **dict(return_list=True)) class SubtensorPrinter: def process(self, r, pstate): if r.owner is None: raise TypeError("Can only print Subtensor.") elif isinstance(r.owner.op, Subtensor): idxs = r.owner.op.idx_list inputs = list(r.owner.inputs) input = inputs.pop(0) sidxs = [] old_precedence = getattr(pstate, 'precedence', None) try: pstate.precedence = -1000 for entry in idxs: if isinstance(entry, integer_types): sidxs.append(str(entry)) elif isinstance(entry, scal.Scalar): sidxs.append(pstate.pprinter.process(inputs.pop())) elif isinstance(entry, slice): if entry.start is None or entry.start == 0: msg1 = "" else: msg1 = entry.start if entry.stop is None or entry.stop == sys.maxsize: msg2 = "" else: msg2 = entry.stop if entry.step is None: msg3 = "" else: msg3 = ":%s" % entry.step sidxs.append("%s:%s%s" % (msg1, msg2, msg3)) finally: pstate.precedence = old_precedence try: pstate.precedence = 1000 sub = pstate.pprinter.process(input, pstate) finally: pstate.precedence = old_precedence return "%s[%s]" % (sub, ", ".join(sidxs)) else: raise TypeError("Can only print Subtensor.") pprint.assign(Subtensor, SubtensorPrinter()) def set_subtensor(x, y, inplace=False, tolerate_inplace_aliasing=False): """ Return x with the given subtensor overwritten by y. Parameters ---------- x Symbolic variable for the lvalue of = operation. y Symbolic variable for the rvalue of = operation. tolerate_inplace_aliasing See inc_subtensor for documentation. Examples -------- To replicate the numpy expression "r[10:] = 5", type >>> r = ivector() >>> new_r = set_subtensor(r[10:], 5) """ return inc_subtensor(x, y, inplace, set_instead_of_inc=True, tolerate_inplace_aliasing=tolerate_inplace_aliasing) def inc_subtensor(x, y, inplace=False, set_instead_of_inc=False, tolerate_inplace_aliasing=False): """ Return x with the given subtensor incremented by y. Parameters ---------- x The symbolic result of a Subtensor operation. y The amount by which to increment the subtensor in question. inplace Don't use. Theano will do it when possible. set_instead_of_inc If True, do a set_subtensor instead. tolerate_inplace_aliasing: Allow x and y to be views of a single underlying array even while working inplace. For correct results, x and y must not be overlapping views; if they overlap, the result of this Op will generally be incorrect. This value has no effect if inplace=False. Examples -------- To replicate the numpy expression "r[10:] += 5", type >>> r = ivector() >>> new_r = inc_subtensor(r[10:], 5) """ # First of all, y cannot have a higher dimension than x, # nor have non-broadcastable dimensions where x is broadcastable. x = theano.tensor.as_tensor_variable(x) y = theano.tensor.as_tensor_variable(y) if y.ndim > x.ndim: raise TypeError(("Trying to increment a %d-dimensional " "subtensor with a %d-dimensional value.") % (x.ndim, y.ndim)) dim_offset = x.ndim - y.ndim for dim in xrange(y.ndim): if (x.broadcastable[dim + dim_offset] and not y.broadcastable[dim]): # It is acceptable to try to increment a subtensor with a # broadcastable dim with a tensor that is not broadcastable # on that dimension. However, its length must then be 1. # We insert a Rebroadcast Op to make sure it is the case. y = addbroadcast(y, dim) if not x.owner: raise TypeError('x must be the result of a subtensor operation') # retrieve idx_list from x.owner if isinstance(x.owner.op, Subtensor): if tolerate_inplace_aliasing: destroyhandler_tolerate_aliased = [[0, 1]] else: destroyhandler_tolerate_aliased = [] the_op = IncSubtensor( x.owner.op.idx_list, inplace, set_instead_of_inc, destroyhandler_tolerate_aliased=destroyhandler_tolerate_aliased) real_x = x.owner.inputs[0] real_idxargs = x.owner.inputs[1:] return the_op(real_x, y, *real_idxargs) elif isinstance(x.owner.op, AdvancedSubtensor1): real_x = x.owner.inputs[0] ilist = x.owner.inputs[1] the_op = AdvancedIncSubtensor1(inplace, set_instead_of_inc=set_instead_of_inc) return the_op(real_x, y, ilist) elif isinstance(x.owner.op, AdvancedSubtensor): real_x = x.owner.inputs[0] ilist = x.owner.inputs[1:] the_op = AdvancedIncSubtensor(inplace, set_instead_of_inc=set_instead_of_inc) return the_op(real_x, y, *ilist) elif isinstance(x.owner.op, AdvancedBooleanSubtensor): real_x = x.owner.inputs[0] ilist = x.owner.inputs[1:] the_op = AdvancedBooleanIncSubtensor(inplace, set_instead_of_inc=set_instead_of_inc) return the_op(real_x, y, *ilist) elif isinstance(x.owner.op, DimShuffle): inner_x = x.owner.inputs[0] # In the dimshuffle case, there are in fact two dimshuffles: # one to make the indexed dimension the last one, # and one to put it back where it was. So, in the case where we have # inc_subtensor(x[:,i], y), the graph is actually # inc_subtensor((x.T)[i].T, y). # We could get all the way to x, and then get rid of the dimshuffles # completely, but the problem is that advanced_inc_subtensor1 can only # work on the first (outer-most, left-most) dimension of x, # just like advanced_subtensor1. # So we call advanced_inc_subtensor1(x.T, i, y.T) (as we also need to # transpose y if it is not a scalar or a vector), but then we need to # return something that has the same shape as x, not as x.T (inner_x). # So re-apply the outer dimshuffle on the new inc_subtensor, # and return advanced_inc_subtensor1(x.T, i, y.T).T. # Get the dimshuffle pattern to apply to y. x_order = x.owner.op.new_order y_order = ['x'] * x.ndim for i, v in enumerate(x_order): if v != 'x' and (v - dim_offset) >= 0: y_order[v - dim_offset] = i # Warn if this code path would have produced wrong results in the past if config.warn.inc_set_subtensor1: # Dimshuffle pattern for y that would be equivalent to past code prev_y_order = ['x'] * (dim_offset) + list(range(y.ndim)) if y_order != prev_y_order: warnings.warn( 'Although your current code is fine, please note that ' 'earlier versions prior to 0.7 (or this development ' 'version) may have yielded an incorrect result in ' 'this `inc_subtensor` or `set_subtensor` operation. ' 'To remove this warning, you can either set the ' '`warn.inc_set_subtensor1` config option to `False`, ' 'or `warn.ignore_bug_before` to at least "0.7".', stacklevel=2) inner_incsubtensor = inc_subtensor( inner_x, y.dimshuffle(y_order), inplace=inplace, set_instead_of_inc=set_instead_of_inc, tolerate_inplace_aliasing=tolerate_inplace_aliasing) # The broadcastable pattern of inner_x may not be the same as # the one of x, so we have to build a new dimshuffle here, # instead of reusing x.owner.op(). return inner_incsubtensor.dimshuffle(x.owner.op.new_order) elif isinstance(x.owner.op, theano.tensor.Reshape): # This case happens when the indices are not arranged as a vector, but # as a higher-dimensional array. This is handled by the subtensor # by flattening this list, taking the subtensor, then reshaping the # result. inner_x = x.owner.inputs[0] # Try to apply inc_subtensor on inner_x. # If it works, there is no need to reshape, as the inc_subtensor # will have the same shape as inner_x, which is what we want. # We also explicitly duplicate y to its broadcasted shape # before we partially flatten it to inner_x dimension. This is # not strictly needed in all cases, but it is easier this way. if y.ndim > 0: # This if is needed to prevent some useless warning about # old code bug. expanded_y = alloc(y, *[x.shape[i] for i in xrange(x.ndim)]) flattened_y = expanded_y.reshape(inner_x.shape) else: flattened_y = y # Warn if this code path would have produced wrong results in the past if config.warn.inc_set_subtensor1: if inner_x.ndim > 1 and sum(y.broadcastable) > 0: warnings.warn( 'Although your current code is fine, please note that ' 'earlier versions prior to 0.7 (or this development ' 'version) may have yielded an incorrect result in ' 'this `inc_subtensor` or `set_subtensor` operation. ' 'To remove this warning, you can either set the ' '`warn.inc_set_subtensor1` config option to `False`, ' 'or `warn.ignore_bug_before` to at least "0.7".', stacklevel=2) inner_incsubtensor = inc_subtensor( inner_x, flattened_y, inplace=inplace, set_instead_of_inc=set_instead_of_inc, tolerate_inplace_aliasing=tolerate_inplace_aliasing) return inner_incsubtensor else: raise TypeError('x must be the result of a subtensor operation') class IncSubtensor(Op): """ Increment a subtensor. This is like numpy's x[i,j,k] += y It is used internally to implement the gradient on SubTensor. Parameters ---------- set_instead_of_inc If True set the subtensor to the value instead of incrementing it by that value. """ check_input = False __props__ = ("idx_list", "inplace", "set_instead_of_inc") def __init__(self, idx_list, inplace=False, set_instead_of_inc=False, destroyhandler_tolerate_aliased=None): if destroyhandler_tolerate_aliased is None: destroyhandler_tolerate_aliased = [] self.idx_list = list(map(Subtensor.convert, idx_list)) self.inplace = inplace if inplace: self.destroy_map = {0: [0]} self.destroyhandler_tolerate_aliased = list( destroyhandler_tolerate_aliased) self.set_instead_of_inc = set_instead_of_inc def __hash__(self): msg = [] for entry in self.idx_list: if isinstance(entry, slice): msg += [(entry.start, entry.stop, entry.step)] else: msg += [entry] idx_list = tuple(msg) # backport # idx_list = tuple((entry.start, entry.stop, entry.step) # if isinstance(entry, slice) # else entry # for entry in self.idx_list) return (hashtype(self) ^ hash(idx_list) ^ hash(self.inplace) ^ hash(self.set_instead_of_inc)) def __str__(self): indices = [] for entry in self.idx_list: if isinstance(entry, slice): indices.append(Subtensor.str_from_slice(entry)) else: indices.append(str(entry)) if self.inplace: msg = 'Inplace' else: msg = '' if not self.set_instead_of_inc: msg += 'Inc' else: msg += 'Set' return "%s{%s;%s}" % ( self.__class__.__name__, msg, ", ".join(indices)) def make_node(self, x, y, *inputs): """ Parameters ---------- x The tensor to increment. y The value to increment by. inputs: TODO WRITEME """ x, y = map(theano.tensor.as_tensor_variable, [x, y]) if y.ndim > x.ndim: raise ValueError(("Trying to increment a %d-dimensional " "subtensor with a %d-dimensional value.") % ( x.ndim, y.ndim)) inputs = tuple(map(Subtensor.my_as_scalar, inputs)) idx_list = list(self.idx_list) if len(idx_list) > x.type.ndim: raise IndexError('too many indices for array') input_types = Subtensor.collapse( idx_list, lambda entry: isinstance(entry, gof.Type)) if len(inputs) != len(input_types): raise IndexError( "Not enough inputs to fill in the Subtensor template.", inputs, idx_list) for input, expected_type in izip(inputs, input_types): if input.type != expected_type: raise TypeError( "Wrong type for Subtensor template. Expected %s, got %s." % (input.type, expected_type)) return gof.Apply(self, (x, y) + inputs, [x.type()]) def decl_view(self): return "PyArrayObject * zview = NULL;" def perform(self, node, inputs, out_): out, = out_ x, y = inputs[:2] indices = list(reversed(inputs[2:])) def convert(entry): if isinstance(entry, gof.Type): rval = indices.pop() if sys.version_info < (2, 5): # Before Python 2.5, PySlice_GetIndicesEx requires # Python int to be passed. rval_ = int(rval) if rval_ != rval: raise IndexError(( "Invalid value for indexing: %s. " "That value may be too big.") % rval) return rval_ return rval elif isinstance(entry, slice): return slice(convert(entry.start), convert(entry.stop), convert(entry.step)) else: return entry cdata = tuple(map(convert, self.idx_list)) if len(cdata) == 1: cdata = cdata[0] if not self.inplace: x = x.copy() sub_x = x.__getitem__(cdata) if sub_x.shape: # we've sliced out an N-D tensor with N > 0 if not self.set_instead_of_inc: sub_x += y else: # sub_x += -sub_x + y x.__setitem__(cdata, y) else: # scalar case if not self.set_instead_of_inc: x.__setitem__(cdata, sub_x + y) else: x.__setitem__(cdata, y) out[0] = x def c_code(self, node, name, inputs, outputs, sub): # This method delegates much of the work to helper # methods. This method implements the main logic # but subclasses may override the helper methods # to change the particulars, e.g. GpuIncSubtensor # turns the view/copy operations on numpy arrays # into the same operations on gpu arrays. self.do_type_checking(node) if self.inplace: # convert bool to int inplace = 1 else: inplace = 0 x = inputs[0] y = inputs[1] z, = outputs if self.set_instead_of_inc: # convert bool to int op_is_set = 1 else: op_is_set = 0 fail = sub['fail'] view_ndim = (node.inputs[0].ndim - np.sum([not isinstance(idx, slice) for idx in self.idx_list])) copy_of_x = self.copy_of_x(x) copy_input_if_necessary = """ if (%(inplace)s) { if (%(x)s != %(z)s) { Py_XDECREF(%(z)s); Py_INCREF(%(x)s); %(z)s = %(x)s; } } else { Py_XDECREF(%(z)s); %(z)s = %(copy_of_x)s; if (!%(z)s) { // Exception already set %(fail)s } } """ % locals() # get info needed to make zview: a view of %(z)s helper_args = self.get_helper_c_code_args() get_zview = Subtensor.helper_c_code( node=node, name=name, inputs=outputs[:1] + inputs[2:], outputs=outputs, sub=sub, idx_list=self.idx_list, view_ndim=view_ndim, ** helper_args ) # Make a view on the output, as we will write into it. alloc_zview = self.make_view_array(z, view_ndim) build_view = """ //TODO: give this Op a second output so that this view can be cached //TODO: alternatively, fix the memory leak on failure %(alloc_zview)s; if (!zview) { %(fail)s; } """ % locals() copy_into = self.copy_into("zview", y) add_to_zview = self.add_to_zview(name, y, fail) make_modification = """ if (%(op_is_set)s) { if (%(copy_into)s) // does broadcasting { Py_DECREF(zview); %(fail)s; } } else { %(add_to_zview)s } """ % locals() return (self.decl_view() + copy_input_if_necessary + "{" + get_zview + build_view + make_modification + "Py_DECREF(zview);" + "}" ) def do_type_checking(self, node): """ Should raise NotImplementedError if c_code does not support the types involved in this node. """ if not isinstance(node.inputs[0].type, theano.tensor.TensorType): raise NotImplementedError() def c_code_cache_version(self): hv = Subtensor.helper_c_code_cache_version() if hv: return (3, hv) else: return () def copy_of_x(self, x): """ Parameters ---------- x A string giving the name of a C variable pointing to an array. Returns ------- object C code expression to make a copy of x. Base class uses PyArrayObject *, subclasses may override for different types of arrays. """ # Parameters of PyArrary_FromAny are: # array # dtype: we pass NULL to say any dtype is acceptable, so the existing # dtype will be copied # min_depth: we pass 0 to have this parameter ignored # max_depth: we pass 0 to have this parameter ignored # requirements: here we pass NPY_ARRAY_ENSURECOPY to force a copy # context: this is almost always NULL, I'm not sure what it's used for return """(PyArrayObject*)PyArray_FromAny(py_%(x)s, NULL, 0, 0, NPY_ARRAY_ENSURECOPY, NULL)""" % locals() def make_view_array(self, x, view_ndim): """ Parameters ---------- x A string identifying an array to be viewed. view_ndim A string specifying the number of dimensions to have in the view. This doesn't need to actually set up the view with the right indexing; we'll do that manually later. """ return """Py_INCREF(PyArray_DESCR(%(x)s)); zview = (PyArrayObject*)PyArray_NewFromDescr( &PyArray_Type, PyArray_DESCR(%(x)s), %(view_ndim)s, xview_dims, //PyArray_DIMS(%(x)s), xview_strides, //PyArray_STRIDES(%(x)s), PyArray_BYTES(%(x)s) + xview_offset, //PyArray_DATA(%(x)s), PyArray_FLAGS(%(x)s), NULL); """ % locals() def get_helper_c_code_args(self): """ Return a dictionary of arguments to pass to helper_c_code. """ return Subtensor.default_helper_c_code_args() def copy_into(self, view, source): """ Parameters ---------- view : string C code expression for an array. source : string C code expression for an array. Returns ------- object C code expression to copy source into view, and 0 on success. """ return """PyArray_CopyInto(%(view)s, %(source)s)""" % locals() def add_to_zview(self, name, x, fail): """ Return C code to add x to zview. Should DECREF zview if the add fails. """ return """ PyArrayObject * add_rval = (PyArrayObject*)PyNumber_InPlaceAdd( (PyObject*)zview, py_%(x)s); if (add_rval) { assert (PyArray_Check((PyObject*)add_rval)); assert (PyArray_DATA(add_rval) == PyArray_DATA(zview)); Py_DECREF(add_rval); } else { Py_DECREF(zview); %(fail)s; }""" % locals() def infer_shape(self, node, shapes): return [shapes[0]] def R_op(self, inputs, eval_points): if eval_points[0] is None or eval_points[1] is None: return [None] # Again we ignore eval points for indices because incsubtensor is # not differentiable wrt to those return self(eval_points[0], eval_points[1], *inputs[2:], **dict(return_list=True)) def connection_pattern(self, node): rval = [[True], [True]] for ipt in node.inputs[2:]: rval.append([False]) return rval def grad(self, inputs, grads): g_output, = grads x, y = inputs[:2] idx_list = inputs[2:] if x.dtype in theano.tensor.discrete_dtypes: # The output dtype is the same as x gx = x.zeros_like(dtype=theano.config.floatX) if y.dtype in theano.tensor.discrete_dtypes: gy = y.zeros_like(dtype=theano.config.floatX) else: gy = y.zeros_like() elif x.dtype in theano.tensor.complex_dtypes: raise NotImplementedError("No support for complex grad yet") else: if self.set_instead_of_inc: gx = set_subtensor( Subtensor(idx_list=self.idx_list)(g_output, *idx_list), theano.tensor.zeros_like(y)) else: gx = g_output gy = Subtensor(idx_list=self.idx_list)(g_output, *idx_list) gy = _sum_grad_over_bcasted_dims(y, gy) return [gx, gy] + [DisconnectedType()()] * len(idx_list) def _sum_grad_over_bcasted_dims(x, gx): """ Sum of gx over dimensions to reproduce x.broadcastable. This is useful to sum gradients over certain dimensions when x has been broadcasted, and we need to sum the gradient contributions over all duplications. """ if gx.broadcastable != x.broadcastable: x_dim_added = gx.ndim - x.ndim x_broad = (True,) * x_dim_added + x.broadcastable assert sum(gx.broadcastable) < sum(x_broad) axis_to_sum = [] for i in xrange(gx.ndim): if gx.broadcastable[i] is False and x_broad[i] is True: axis_to_sum.append(i) elif (gx.broadcastable[i] is True and x_broad[i] is False): # This means that Theano was able to infer that # gx.shape[i] is 1, so x.shape[i] is 1, but we # didn't know it. It is fine. pass else: assert gx.broadcastable[i] == x_broad[i] gx = gx.sum(axis=axis_to_sum, keepdims=True) if gx.ndim != x.ndim: assert gx.ndim > x.ndim for i in xrange(x_dim_added): assert gx.broadcastable[i] gx = gx.dimshuffle(*list(range(x_dim_added, gx.ndim))) assert gx.broadcastable == x.broadcastable return gx ######################### # Advanced indexing ######################### # # Should reproduce numpy's behaviour, see url: # docs.scipy.org/doc/numpy/reference/arrays.indexing.html#advanced-indexing class AdvancedSubtensor1(Op): """ Implement x[ilist] where ilist is a vector of integers. """ # sparse_grad doesn't go in here since it only affects the output # of the grad() method. __props__ = () _f16_ok = True check_input = False def __init__(self, sparse_grad=False): self.sparse_grad = sparse_grad def make_node(self, x, ilist): x_ = theano.tensor.as_tensor_variable(x) ilist_ = theano.tensor.as_tensor_variable(ilist) if ilist_.type.dtype not in theano.tensor.integer_dtypes: raise TypeError('index must be integers') if ilist_.type.ndim != 1: raise TypeError('index must be vector') if x_.type.ndim == 0: raise TypeError('cannot index into a scalar') bcast = (ilist_.broadcastable[0],) + x_.broadcastable[1:] return Apply(self, [x_, ilist_], [TensorType(dtype=x.dtype, broadcastable=bcast)()]) def perform(self, node, inp, out_): x, i = inp out, = out_ # Copy always implied by numpy advanced indexing semantic. if out[0] is not None and out[0].shape == (len(i),) + x.shape[1:]: o = out[0] else: o = None # If i.dtype is more precise than numpy.intp (int32 on 32-bit machines, # int64 on 64-bit machines), numpy may raise the following error: # TypeError: array cannot be safely cast to required type. # We need to check if values in i can fit in numpy.intp, because # if they don't, that should be an error (no array can have that # many elements on a 32-bit arch). if i.dtype != np.intp: i_ = theano._asarray(i, dtype=np.intp) if not np.can_cast(i.dtype, np.intp): # Check if there was actually an incorrect conversion if np.any(i != i_): raise IndexError( 'index contains values that are bigger ' 'than the maximum array size on this system.', i) i = i_ out[0] = x.take(i, axis=0, out=o) def connection_pattern(self, node): rval = [[True]] for ipt in node.inputs[1:]: rval.append([False]) return rval def grad(self, inputs, grads): global sparse_module_ref x, ilist = inputs gz, = grads assert len(inputs) == 2 if self.sparse_grad: if x.type.ndim != 2: raise TypeError( "AdvancedSubtensor1: you can't take the sparse grad" " from a tensor with ndim != 2. ndim is " + str(x.type.ndim)) if sparse_module_ref is None: import theano.sparse as sparse_module_ref rval1 = [sparse_module_ref.construct_sparse_from_list(x, gz, ilist)] else: rval1 = [advanced_inc_subtensor1(x.zeros_like(), gz, ilist)] return rval1 + [DisconnectedType()()] * (len(inputs) - 1) def R_op(self, inputs, eval_points): if eval_points[0] is None: return [None] return self.make_node(eval_points[0], *inputs[1:]).outputs def infer_shape(self, node, ishapes): x, ilist = ishapes return [ilist + x[1:]] def c_support_code(self): # In some versions of numpy, NPY_MIN_INTP is defined as MIN_LONG, # which is not defined. It should be NPY_MIN_LONG instead in that case. return dedent("""\ #ifndef MIN_LONG #define MIN_LONG NPY_MIN_LONG #endif""") def c_code(self, node, name, input_names, output_names, sub): if self.__class__ is not AdvancedSubtensor1: raise MethodNotDefined( "c_code defined for AdvancedSubtensor1," " not for child class", type(self)) a_name, i_name = input_names[0], input_names[1] output_name = output_names[0] fail = sub['fail'] return """ PyArrayObject *indices; int i_type = PyArray_TYPE(%(i_name)s); if (i_type != NPY_INTP) { // Cast %(i_name)s to NPY_INTP (expected by PyArray_TakeFrom), // if all values fit. if (!PyArray_CanCastSafely(i_type, NPY_INTP) && PyArray_SIZE(%(i_name)s) > 0) { npy_int64 min_val, max_val; PyObject* py_min_val = PyArray_Min(%(i_name)s, NPY_MAXDIMS, NULL); if (py_min_val == NULL) { %(fail)s; } min_val = PyLong_AsLongLong(py_min_val); Py_DECREF(py_min_val); if (min_val == -1 && PyErr_Occurred()) { %(fail)s; } PyObject* py_max_val = PyArray_Max(%(i_name)s, NPY_MAXDIMS, NULL); if (py_max_val == NULL) { %(fail)s; } max_val = PyLong_AsLongLong(py_max_val); Py_DECREF(py_max_val); if (max_val == -1 && PyErr_Occurred()) { %(fail)s; } if (min_val < NPY_MIN_INTP || max_val > NPY_MAX_INTP) { PyErr_SetString(PyExc_IndexError, "Index contains values " "that are bigger than the maximum array " "size on this system."); %(fail)s; } } indices = (PyArrayObject*) PyArray_Cast(%(i_name)s, NPY_INTP); if (indices == NULL) { %(fail)s; } } else { indices = %(i_name)s; Py_INCREF(indices); } if (%(output_name)s != NULL) { npy_intp nd, i, *shape; nd = PyArray_NDIM(%(a_name)s) + PyArray_NDIM(indices) - 1; if (PyArray_NDIM(%(output_name)s) != nd) { Py_CLEAR(%(output_name)s); } else { shape = PyArray_DIMS(%(output_name)s); for (i = 0; i < PyArray_NDIM(indices); i++) { if (shape[i] != PyArray_DIMS(indices)[i]) { Py_CLEAR(%(output_name)s); break; } } if (%(output_name)s != NULL) { for (; i < nd; i++) { if (shape[i] != PyArray_DIMS(%(a_name)s)[ i-PyArray_NDIM(indices)+1]) { Py_CLEAR(%(output_name)s); break; } } } } } %(output_name)s = (PyArrayObject*)PyArray_TakeFrom( %(a_name)s, (PyObject*)indices, 0, %(output_name)s, NPY_RAISE); Py_DECREF(indices); if (%(output_name)s == NULL) %(fail)s; """ % locals() def c_code_cache_version(self): return (0, 1, 2) advanced_subtensor1 = AdvancedSubtensor1() class AdvancedIncSubtensor1(Op): """ Increments a subtensor using advanced slicing (list of index). """ __props__ = ('inplace', 'set_instead_of_inc') check_input = False params_type = ParamsType(inplace=scal.bool, set_instead_of_inc=scal.bool) def __init__(self, inplace=False, set_instead_of_inc=False): self.inplace = bool(inplace) self.set_instead_of_inc = bool(set_instead_of_inc) if inplace: self.destroy_map = {0: [0]} def clone_inplace(self): return self.__class__( inplace=True, set_instead_of_inc=self.set_instead_of_inc) def __str__(self): if self.inplace: msg = "inplace" else: msg = "no_inplace" if self.set_instead_of_inc: msg += ",set" else: msg += ",inc" return self.__class__.__name__ + "{%s}" % msg def make_node(self, x, y, ilist): x_ = theano.tensor.as_tensor_variable(x) y_ = theano.tensor.as_tensor_variable(y) ilist_ = theano.tensor.as_tensor_variable(ilist) if ilist_.type.dtype not in theano.tensor.integer_dtypes: raise TypeError('index must be integers') if ilist_.type.ndim != 1: raise TypeError('index must be vector') if x_.type.ndim == 0: raise TypeError('cannot index into a scalar') if y_.type.ndim > x_.type.ndim: if self.set_instead_of_inc: opname = 'set' else: opname = 'increment' raise TypeError( 'cannot %s x subtensor with ndim=%s' ' by y with ndim=%s to x subtensor with ndim=%s ' % ( opname, x_.type.ndim, y_.type.ndim)) return Apply(self, [x_, y_, ilist_], [x_.type()]) def copy_of_x(self, x): """ Parameters ---------- x : string Gives the name of a C variable pointing to an array. Returns ------- object C code expression to make a copy of x. Base class uses PyArrayObject *, subclasses may override for different types of arrays. """ # Parameters of PyArrary_FromAny are: # array # dtype: we pass NULL to say any dtype is acceptable, so the existing # dtype will be copied # min_depth: we pass 0 to have this parameter ignored # max_depth: we pass 0 to have this parameter ignored # requirements: here we pass NPY_ARRAY_ENSURECOPY to force a copy # context: this is almost always NULL, I'm not sure what it's used for return """(PyArrayObject*)PyArray_FromAny(py_%(x)s, NULL, 0, 0, NPY_ARRAY_ENSURECOPY, NULL)""" % locals() def c_support_code(self): return inc_code() def c_code(self, node, name, input_names, output_names, sub): numpy_ver = [int(n) for n in np.__version__.split('.')[:2]] if bool(numpy_ver < [1, 8]): raise NotImplementedError x, y, idx = input_names out = output_names[0] copy_of_x = self.copy_of_x(x) return """ PyObject* rval = NULL; if (%(params)s->inplace) { if (%(x)s != %(out)s) { Py_XDECREF(%(out)s); Py_INCREF(%(x)s); %(out)s = %(x)s; } } else { Py_XDECREF(%(out)s); %(out)s = %(copy_of_x)s; if (!%(out)s) { // Exception already set %(fail)s } } if (inplace_increment(%(out)s, (PyObject *)%(idx)s, %(y)s, (1 - %(params)s->set_instead_of_inc))) { %(fail)s; } Py_XDECREF(rval); """ % dict(x=x, y=y, idx=idx, out=out, copy_of_x=copy_of_x, params=sub['params'], fail=sub['fail']) def c_code_cache_version(self): return (8,) def perform(self, node, inp, out_, params): # TODO opt to make this inplace x, y, idx = inp out, = out_ if not self.inplace: x = x.copy() # In Numpy, x[idx] += y doesn't work if the same index is present # many times: it does it only once. Is it a bug? In any case, for # this reason we implement our own 'inc' iteration. if self.set_instead_of_inc: x[idx] = y else: np.add.at(x, idx, y) out[0] = x def infer_shape(self, node, ishapes): x, y, ilist = ishapes return [x] def R_op(self, inputs, eval_points): if None in eval_points[:2]: return [None] return self.make_node(eval_points[0], eval_points[1], *inputs[2:]).outputs def connection_pattern(self, node): rval = [[True], [True], [False]] return rval def grad(self, inputs, grads): g_output, = grads x, y, idx_list = inputs if x.dtype in theano.tensor.discrete_dtypes: # The output dtype is the same as x gx = x.zeros_like(dtype=theano.config.floatX) if y.dtype in theano.tensor.discrete_dtypes: gy = y.zeros_like(dtype=theano.config.floatX) else: gy = y.zeros_like() elif x.dtype in theano.tensor.complex_dtypes: raise NotImplementedError("No support for complex grad yet") else: if self.set_instead_of_inc: gx = advanced_set_subtensor1( g_output, y.zeros_like(), idx_list) else: gx = g_output gy = advanced_subtensor1(g_output, idx_list) gy = _sum_grad_over_bcasted_dims(y, gy) return [gx, gy] + [DisconnectedType()()] advanced_inc_subtensor1 = AdvancedIncSubtensor1() advanced_set_subtensor1 = AdvancedIncSubtensor1(set_instead_of_inc=True) def as_index_variable(idx): if idx is None: return NoneConst.clone() if isinstance(idx, slice): return make_slice(idx) if isinstance(idx, gof.Variable) and isinstance(idx.type, SliceType): return idx if isinstance(idx, gof.Variable) and isinstance(idx.type, NoneTypeT): return idx idx = theano.tensor.as_tensor_variable(idx) if idx.type.dtype not in theano.tensor.discrete_dtypes: raise TypeError('index must be integers or a boolean mask') return idx def adv_index_broadcastable_pattern(a, idx): """ This function is only used to determine the broadcast pattern for AdvancedSubtensor output variable. For this, we make a fake ndarray and a fake idx and call use ask numpy the output. From this, we find the output broadcast pattern. """ def replace_slice(v): if isinstance(v, gof.Apply): if len(v.outputs) != 1: raise ValueError( "It is ambiguous which output of a multi-output Op has" " to be fetched.", v) else: v = v.outputs[0] if NoneConst.equals(v): return None if isinstance(v.type, SliceType): return slice(None, None) if v.dtype == 'bool': return np.ones((2,) * v.ndim, v.dtype) else: return np.zeros((2,) * v.ndim, int) newidx = tuple(map(replace_slice, idx)) # 2 - True = 1; 2 - False = 2 fakeshape = [2 - bc for bc in a.broadcastable] retshape = np.empty(fakeshape)[newidx].shape return tuple([dim == 1 for dim in retshape]) def check_advanced_indexing_dimensions(input, idx_list): """ This function checks if the index list in idx_list is correct. If there are any boolean masks, we check if the mask has the same shape as the input. This is enforced in NumPy 0.13.0 and newer, but not by earlier versions. If the size is not the same, this method raises an IndexError. """ dim_seen = 0 for index in idx_list: if index is np.newaxis: # skip, does not count as an input dimension pass elif isinstance(index, np.ndarray) and index.dtype == 'bool': for i in xrange(index.ndim): if index.shape[i] != input.shape[dim_seen + i]: raise IndexError('boolean index did not match indexed array ' 'along dimension %d; dimension is %d but ' 'corresponding boolean dimension is %d' % (dim_seen + i, input.shape[dim_seen + i], index.shape[i])) dim_seen += index.ndim else: dim_seen += 1 def check_and_reject_bool(args_el): try: if (isinstance(args_el, (np.bool_, bool)) or args_el.dtype == 'bool'): raise TypeError('AdvancedSubtensor does not support boolean ' 'masks for indexing. Use AdvancedBooleanSubtensor ' 'instead. ') except AttributeError: pass if (not isinstance(args_el, theano.tensor.Variable) and isinstance(args_el, collections.Iterable)): for el in args_el: check_and_reject_bool(el) class BaseAdvancedSubtensor(Op): """ Abstract base class for AdvancedSubtensor and AdvancedBooleanSubtensor. Implements advanced indexing with boolean masks. """ # Should be used by __getitem__ and __getslice__, as follows: # AdvancedSubtensor()(self, *args) or # AdvancedBooleanSubtensor()(self, *args), # if args contains and advanced indexing pattern __props__ = () def make_node(self, x, *index): x = theano.tensor.as_tensor_variable(x) index = tuple(map(as_index_variable, index)) bcast = adv_index_broadcastable_pattern(x, index) return gof.Apply(self, (x,) + index, [theano.tensor.tensor(dtype=x.type.dtype, broadcastable=bcast)]) def R_op(self, inputs, eval_points): if eval_points[0] is None: return [None] return self.make_node(eval_points[0], *inputs[1:]).outputs def infer_shape(self, node, ishapes): # Default case, we don't know raise theano.tensor.basic.ShapeError("case not implemented") def perform(self, node, inputs, out_): out, = out_ check_advanced_indexing_dimensions(inputs[0], inputs[1:]) rval = inputs[0].__getitem__(inputs[1:]) # When there are no arrays, we are not actually doing advanced # indexing, so __getitem__ will not return a copy. # Since no view_map is set, we need to copy the returned value if not any(isinstance(v.type, TensorType) and v.ndim > 0 for v in node.inputs[1:]): rval = rval.copy() out[0] = rval def connection_pattern(self, node): rval = [[True]] for ipt in node.inputs[1:]: rval.append([False]) return rval class AdvancedSubtensor(BaseAdvancedSubtensor): """ Return a subtensor copy, using advanced indexing. """ # Should be used by __getitem__ and __getslice__, as follows: # AdvancedSubtensor()(self, *args), # if args contains and advanced indexing pattern def make_node(self, x, *index): check_and_reject_bool(index) return super(AdvancedSubtensor, self).make_node(x, *index) def infer_shape(self, node, ishapes): # Really special case if len(ishapes) == 3: xshp, ind1shp, ind2shp = ishapes if (len(xshp) == 2 and ind1shp is not None and len(ind1shp) == 1 and ind2shp is not None and len(ind2shp) == 1): # if the graph is correct, we can assume ind1shp[0] and # ind2shp[0] will have the same value. # Try to return the one closest to the graph input. if node.inputs[2].owner is None: return [ind2shp] else: return [ind1shp] return super(AdvancedSubtensor, self).infer_shape(node, ishapes) def grad(self, inputs, grads): gz, = grads x = inputs[0] rest = inputs[1:] return [advanced_inc_subtensor(theano.tensor.zeros_like(x), gz, *rest)] + \ [DisconnectedType()()] * len(rest) advanced_subtensor = AdvancedSubtensor() class AdvancedBooleanSubtensor(BaseAdvancedSubtensor): """ Return a subtensor copy, using advanced indexing with boolean masks. """ # Should be used by __getitem__ and __getslice__, as follows: # AdvancedBooleanSubtensor()(self, *args), # if args contains and advanced indexing pattern with boolean masks def grad(self, inputs, grads): gz, = grads x = inputs[0] rest = inputs[1:] return [advanced_boolean_inc_subtensor(theano.tensor.zeros_like(x), gz, *rest)] + \ [DisconnectedType()()] * len(rest) advanced_boolean_subtensor = AdvancedBooleanSubtensor() class BaseAdvancedIncSubtensor(Op): """ Base class for AdvancedIncSubtensor and AdvancedBooleanIncSubtensor. Increments a subtensor using advanced indexing. """ __props__ = ("inplace", "set_instead_of_inc") def __init__(self, inplace=False, set_instead_of_inc=False): self.inplace = inplace self.set_instead_of_inc = set_instead_of_inc # The assert is needed as in the pass the first argument was # something else that was not used. assert isinstance(inplace, bool) if self.inplace: raise NotImplementedError('In place computation is not' ' implemented') def __str__(self): return "%s{%s, %s}" % (self.__class__.__name__, "inplace=" + str(self.inplace), " set_instead_of_inc=" + str(self. set_instead_of_inc)) def make_node(self, x, y, *inputs): x = theano.tensor.as_tensor_variable(x) y = theano.tensor.as_tensor_variable(y) new_inputs = [] for inp in inputs: if isinstance(inp, (list, tuple)): inp = theano.tensor.as_tensor_variable(inp) new_inputs.append(inp) return gof.Apply(self, (x, y) + tuple(new_inputs), [theano.tensor.tensor( dtype=x.type.dtype, broadcastable=x.type.broadcastable)]) def perform(self, node, inputs, out_): # TODO: 1. opt to make this in place 2. generalize as described in # AdvancedSubtensor's perform TODO check_advanced_indexing_dimensions(inputs[0], inputs[2:]) out, = out_ if not self.inplace: out[0] = inputs[0].copy() else: out[0] = inputs[0] if self.set_instead_of_inc: out[0][inputs[2:]] = inputs[1] else: np.add.at(out[0], tuple(inputs[2:]), inputs[1]) def infer_shape(self, node, ishapes): return [ishapes[0]] def connection_pattern(self, node): rval = [[True], [True]] for ipt in node.inputs[2:]: rval.append([False]) return rval def R_op(self, inputs, eval_points): if None in eval_points[:2]: return [None] return self.make_node(eval_points[0], eval_points[1], *inputs[2:]).outputs class AdvancedIncSubtensor(BaseAdvancedIncSubtensor): """ Increments a subtensor using advanced indexing. """ def make_node(self, x, y, *inputs): check_and_reject_bool(inputs) return super(AdvancedIncSubtensor, self).make_node(x, y, *inputs) def grad(self, inpt, output_gradients): x, y = inpt[:2] idxs = inpt[2:] outgrad, = output_gradients if x.dtype in theano.tensor.discrete_dtypes: # The output dtype is the same as x gx = x.zeros_like(dtype=theano.config.floatX) if y.dtype in theano.tensor.discrete_dtypes: gy = y.zeros_like(dtype=theano.config.floatX) else: gy = y.zeros_like() elif x.dtype in theano.tensor.complex_dtypes: raise NotImplementedError("No support for complex grad yet") else: if self.set_instead_of_inc: gx = advanced_set_subtensor( outgrad, y.zeros_like(), *idxs) else: gx = outgrad gy = advanced_subtensor(outgrad, *idxs) # Make sure to sum gy over the dimensions of y that have been # added or broadcasted gy = _sum_grad_over_bcasted_dims(y, gy) return [gx, gy] + \ [DisconnectedType()() for _ in idxs] advanced_inc_subtensor = AdvancedIncSubtensor() advanced_set_subtensor = AdvancedIncSubtensor(set_instead_of_inc=True) class AdvancedBooleanIncSubtensor(BaseAdvancedIncSubtensor): """ Increments a subtensor using advanced indexing with boolean masks. """ def grad(self, inpt, output_gradients): x, y = inpt[:2] idxs = inpt[2:] outgrad, = output_gradients if x.dtype in theano.tensor.discrete_dtypes: # The output dtype is the same as x gx = x.zeros_like(dtype=theano.config.floatX) if y.dtype in theano.tensor.discrete_dtypes: gy = y.zeros_like(dtype=theano.config.floatX) else: gy = y.zeros_like() elif x.dtype in theano.tensor.complex_dtypes: raise NotImplementedError("No support for complex grad yet") else: if self.set_instead_of_inc: gx = advanced_set_subtensor( outgrad, y.zeros_like(), *idxs) else: gx = outgrad gy = advanced_boolean_subtensor(outgrad, *idxs) # Make sure to sum gy over the dimensions of y that have been # added or broadcasted gy = _sum_grad_over_bcasted_dims(y, gy) return [gx, gy] + \ [DisconnectedType()() for _ in idxs] advanced_boolean_inc_subtensor = AdvancedBooleanIncSubtensor() advanced_boolean_set_subtensor = AdvancedBooleanIncSubtensor(set_instead_of_inc=True) def take(a, indices, axis=None, mode='raise'): a = theano.tensor.as_tensor_variable(a) indices = theano.tensor.as_tensor_variable(indices) # Reuse advanced_subtensor1 if indices is a vector if indices.ndim == 1: if mode == 'clip': indices = clip(indices, 0, a.shape[axis] - 1) elif mode == 'wrap': indices = indices % a.shape[axis] if axis is None: return advanced_subtensor1(a.flatten(), indices) elif axis == 0: return advanced_subtensor1(a, indices) else: if axis < 0: axis += a.ndim assert axis >= 0 shuffle = list(range(a.ndim)) shuffle[0] = axis shuffle[axis] = 0 return advanced_subtensor1( a.dimshuffle(shuffle), indices).dimshuffle(shuffle) if axis is None: shape = indices.shape ndim = indices.ndim else: # If axis is 0, don't generate a useless concatenation. if axis == 0: shape = theano.tensor.concatenate( [indices.shape, a.shape[axis + 1:]]) else: if axis < 0: axis += a.ndim shape = theano.tensor.concatenate( [a.shape[:axis], indices.shape, a.shape[axis + 1:]]) ndim = a.ndim + indices.ndim - 1 return take(a, indices.flatten(), axis, mode).reshape(shape, ndim) Theano-1.0.0/theano/tensor/var.py00006000163562€ÿÿÿþ0000010542413203335532020227 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import collections import copy import traceback as tb import warnings import numpy as np from six import integer_types from six.moves import xrange import theano from theano.compat import PY3 from theano.scalar import ComplexError, IntegerDivisionError from theano.gof import Constant, Variable from theano.gof.utils import hashtype from theano.tensor.utils import hash_from_ndarray from theano.tensor.type import TensorType from theano import config def equal_slices(s1, s2): return (s1.start == s2.start and s1.stop == s2.stop and s1.step == s2.step) class AsTensorError(TypeError): """ Raised when as_tensor_variable isn't able to create a TensorVariable. """ pass class _tensor_py_operators(object): # UNARY def __abs__(self): return theano.tensor.basic.abs_(self) def __neg__(self): return theano.tensor.basic.neg(self) # CASTS # REMOVED THESE BECAUSE PYTHON appears to require __int__ to return # an int. -JB 20081112 # def __int__(self): return convert_to_int32(self) # def __float__(self): return convert_to_float64(self) # def __complex__(self): return convert_to_complex128(self) # COMPARISONS _is_nonzero = True def __lt__(self, other): rval = theano.tensor.basic.lt(self, other) rval._is_nonzero = False return rval def __le__(self, other): rval = theano.tensor.basic.le(self, other) rval._is_nonzero = False return rval def __gt__(self, other): rval = theano.tensor.basic.gt(self, other) rval._is_nonzero = False return rval def __ge__(self, other): rval = theano.tensor.basic.ge(self, other) rval._is_nonzero = False return rval def __nonzero__(self): # Python 2.x return self.__bool__() def __bool__(self): # This is meant to prohibit stuff like a < b < c, which is internally # implemented as (a < b) and (b < c). The trouble with this is the # side-effect that checking for a non-NULL a by typing "if a: ..." # uses the same __nonzero__ method. We want these both to work, but # it seems impossible. Currently, all vars evaluate to nonzero except # the return values of comparison operators, which raise this # exception. If you can think of a better solution, go for it! # # __bool__ is Python 3.x data model. __nonzero__ is Python 2.x. if self._is_nonzero: return True else: raise TypeError( "Variables do not support boolean operations." ) # BITWISE def __invert__(self): return theano.tensor.basic.invert(self) def __and__(self, other): return theano.tensor.basic.and_(self, other) def __or__(self, other): return theano.tensor.basic.or_(self, other) def __xor__(self, other): return theano.tensor.basic.xor(self, other) def __rand__(self, other): return theano.tensor.basic.and_(other, self) def __ror__(self, other): return theano.tensor.basic.or_(other, self) def __rxor__(self, other): return theano.tensor.basic.xor(other, self) # def __iand__(self, other): # return _and_inplace(self, other) # # def __ior__(self, other): # return _or_inplace(self, other) # # def __ixor__(self, other): # return _xor_inplace(self, other) # ARITHMETIC - NORMAL def __add__(self, other): try: return theano.tensor.basic.add(self, other) # We should catch the minimum number of exception here. # Otherwise this will convert error when Theano flags # compute_test_value is used # Evidently, we need to catch NotImplementedError # TypeError from as_tensor_variable are caught in Elemwise.make_node # Oterwise TensorVariable * SparseVariable won't work! except (NotImplementedError, AsTensorError): # We must return NotImplemented and not an # NotImplementedError or raise an NotImplementedError. # That way python will give a good error message like this # `TypeError: unsupported operand type(s) for +: # 'TensorVariable' and 'TensorVariable'` return NotImplemented def __sub__(self, other): # See explanation in __add__ for the error catched # and the return value in that case try: return theano.tensor.basic.sub(self, other) except (NotImplementedError, AsTensorError): return NotImplemented def __mul__(self, other): # See explanation in __add__ for the error catched # and the return value in that case try: return theano.tensor.mul(self, other) except (NotImplementedError, AsTensorError): return NotImplemented def __div__(self, other): # See explanation in __add__ for the error catched # and the return value in that case try: return theano.tensor.basic.div_proxy(self, other) except IntegerDivisionError: # This is to raise the exception that occurs when trying to divide # two integer arrays (currently forbidden). raise except (NotImplementedError, AsTensorError): return NotImplemented if PY3: __truediv__ = __div__ def __pow__(self, other): # See explanation in __add__ for the error catched # adn the return value in that case try: return theano.tensor.basic.pow(self, other) except (NotImplementedError, AsTensorError): return NotImplemented def __mod__(self, other): # See explanation in __add__ for the error catched # adn the return value in that case try: return theano.tensor.basic.mod_check(self, other) except ComplexError: # This is to raise the exception that occurs when trying to compute # x % y with either x or y a complex number. raise except (NotImplementedError, AsTensorError): return NotImplemented def __divmod__(self, other): return theano.tensor.basic.divmod(self, other) def __truediv__(self, other): return theano.tensor.basic.true_div(self, other) def __floordiv__(self, other): return theano.tensor.basic.floor_div(self, other) def __rtruediv__(self, other): return theano.tensor.basic.true_div(other, self) def __rfloordiv__(self, other): return theano.tensor.basic.floor_div(other, self) # DO NOT USE THESE BECAUSE INPLACE OPS SHOULD BE INSERTED # BY OPTIMIZATIONS ONLY # ARITHMETIC - INPLACE # def __iadd__(self, other): # return _add_inplace(self, other) # def __isub__(self, other): # return _sub_inplace(self, other) # # def __imul__(self, other): # return _mul_inplace(self, other) # # def __idiv__(self, other): # return _div_inplace(self, other) # # def __ipow__(self, other): # return _pow_inplace(self, other) # ARITHMETIC - RIGHT-OPERAND def __radd__(self, other): return theano.tensor.basic.add(other, self) def __rsub__(self, other): return theano.tensor.basic.sub(other, self) def __rmul__(self, other): return theano.tensor.basic.mul(other, self) def __rdiv__(self, other): return theano.tensor.basic.div_proxy(other, self) def __rmod__(self, other): return theano.tensor.basic.mod(other, self) def __rdivmod__(self, other): return theano.tensor.basic.divmod(other, self) def __rpow__(self, other): return theano.tensor.basic.pow(other, self) # TRANSPOSE T = property(lambda self: theano.tensor.basic.transpose(self)) def transpose(self, *axes): """ Returns ------- object `tensor.transpose(self, axes)` or `tensor.transpose(self, axes[0])`. If only one `axes` argument is provided and it is iterable, then it is assumed to be the entire axes tuple, and passed intact to tensor.transpose. """ if len(axes) == 0: return theano.tensor.basic.transpose(self) try: iter(axes[0]) iterable = True except TypeError: iterable = False if len(axes) == 1 and iterable: return theano.tensor.basic.transpose(self, axes[0]) else: return theano.tensor.basic.transpose(self, axes) shape = property(lambda self: theano.tensor.basic.shape(self)) size = property(lambda self: self.shape[0] if self.ndim == 1 else theano.tensor.basic.prod(self.shape)) # We can't implement __len__ to provide a better error message. def any(self, axis=None, keepdims=False): return theano.tensor.basic.any(self, axis=axis, keepdims=keepdims) def all(self, axis=None, keepdims=False): return theano.tensor.basic.all(self, axis=axis, keepdims=keepdims) # Otherwise TensorVariable[:-1] does not work as Python 2.5.1 calls # __len__ before calling __getitem__. It also does not catch the raised # Exception! # def __len__(self): # # We can't implement __len__ as Python requests that this # # function returns an integer >=0 # raise Exception("Theano Variables can't work with len(Theano " # "Variable) due to Python restriction. You can use " # "TheanoVariable.shape[0] instead.") def reshape(self, shape, ndim=None): """Return a reshaped view/copy of this variable. Parameters ---------- shape Something that can be converted to a symbolic vector of integers. ndim The length of the shape. Passing None here means for Theano to try and guess the length of `shape`. .. warning:: This has a different signature than numpy's ndarray.reshape! In numpy you do not need to wrap the shape arguments in a tuple, in theano you do need to. """ if ndim is not None: if not isinstance(ndim, integer_types): raise ValueError("Expected ndim to be an integer, is " + str(type(ndim))) return theano.tensor.basic.reshape(self, shape, ndim=ndim) def dimshuffle(self, *pattern): """ Reorder the dimensions of this variable, optionally inserting broadcasted dimensions. Parameters ---------- pattern List/tuple of int mixed with 'x' for broadcastable dimensions. Examples -------- For example, to create a 3D view of a [2D] matrix, call ``dimshuffle([0,'x',1])``. This will create a 3D view such that the middle dimension is an implicit broadcasted dimension. To do the same thing on the transpose of that matrix, call ``dimshuffle([1, 'x', 0])``. Notes ----- This function supports the pattern passed as a tuple, or as a variable-length argument (e.g. ``a.dimshuffle(pattern)`` is equivalent to ``a.dimshuffle(*pattern)`` where ``pattern`` is a list/tuple of ints mixed with 'x' characters). See Also -------- DimShuffle """ if (len(pattern) == 1) and (isinstance(pattern[0], (list, tuple))): pattern = pattern[0] op = theano.tensor.basic.DimShuffle(list(self.type.broadcastable), pattern) return op(self) def flatten(self, ndim=1): return theano.tensor.basic.flatten(self, ndim) def ravel(self): return theano.tensor.basic.flatten(self) def diagonal(self, offset=0, axis1=0, axis2=1): return theano.tensor.basic.diagonal(self, offset, axis1, axis2) # Transfer the data to another device def transfer(self, target): """ If `target` is `'cpu'` this will transfer to a TensorType (if not already one). Other types may define additional targets. Parameters ---------- target : str The desired location of the output variable """ return theano.tensor.transfer(self, target) # Elemwise def arccos(self): return theano.tensor.arccos(self) def arccosh(self): return theano.tensor.arccosh(self) def arcsin(self): return theano.tensor.arcsin(self) def arcsinh(self): return theano.tensor.arcsinh(self) def arctan(self): return theano.tensor.arctan(self) def arctanh(self): return theano.tensor.arctanh(self) def ceil(self): return theano.tensor.ceil(self) def cos(self): return theano.tensor.cos(self) def cosh(self): return theano.tensor.cosh(self) def deg2rad(self): return theano.tensor.deg2rad(self) def exp(self): return theano.tensor.exp(self) def exp2(self): return theano.tensor.exp2(self) def expm1(self): return theano.tensor.expm1(self) def floor(self): return theano.tensor.floor(self) def log(self): return theano.tensor.log(self) def log10(self): return theano.tensor.log10(self) def log1p(self): return theano.tensor.log1p(self) def log2(self): return theano.tensor.log2(self) def rad2deg(self): return theano.tensor.rad2deg(self) def sin(self): return theano.tensor.sin(self) def sinh(self): return theano.tensor.sinh(self) def sqrt(self): return theano.tensor.sqrt(self) def tan(self): return theano.tensor.tan(self) def tanh(self): return theano.tensor.tanh(self) def trunc(self): return theano.tensor.trunc(self) # CASTING def astype(self, dtype): return theano.tensor.cast(self, dtype) # SLICING/INDEXING def __getitem__(self, args): def includes_bool(args_el): if (isinstance(args_el, (np.bool_, bool)) or (hasattr(args_el, 'dtype') and args_el.dtype == 'bool')): return True if (not isinstance(args_el, theano.tensor.Variable) and isinstance(args_el, collections.Iterable)): for el in args_el: if includes_bool(el): return True return False if (isinstance(args, list) and any([isinstance(a, slice) for a in args])): pass elif not isinstance(args, tuple): args = args, # Count the dimensions, check for bools and find ellipses. ellipses = [] index_dim_count = 0 for i, arg in enumerate(args): if arg is np.newaxis: # no increase in index_dim_count pass elif arg is Ellipsis: # no increase in index_dim_count ellipses.append(i) elif (isinstance(arg, (np.ndarray, theano.tensor.Variable)) and hasattr(arg, 'dtype') and arg.dtype == 'bool'): index_dim_count += arg.ndim else: # Python arrays can contain a mixture of bools and integers, # which requires complex rules to handle all special cases. # These rules differ slightly between NumPy versions. # Since earlier versions of Theano did not support any boolean # indexing, it is safe to throw an error if we encounter # any of these difficult cases. if includes_bool(arg): raise TypeError('TensorType does not support Python bools ' 'for indexing, such as tensor[[True, False]]. ' 'To use a boolean mask, convert the mask to ' 'a NumPy array first, e.g., ' 'tensor[numpy.array([True, False])].') index_dim_count += 1 # Check if the number of dimensions isn't too large. if self.ndim < index_dim_count: raise IndexError('too many indices for array') # Convert an Ellipsis if provided into an appropriate number of # slice(None). if len(ellipses) > 1: raise IndexError( "an index can only have a single Ellipsis (`...`)") elif len(ellipses) == 1: ellipsis_at = ellipses[0] args = list(args) args[ellipsis_at: ellipsis_at + 1] = ( [slice(None)] * (self.ndim - index_dim_count)) def is_empty_array(val): return ((isinstance(val, (tuple, list)) and len(val) == 0) or (isinstance(val, np.ndarray) and val.size == 0)) # Force input to be int64 datatype if input is an empty list or tuple # Else leave it as is if it is a real number args = tuple([np.array(inp, dtype=np.int64) if(is_empty_array(inp)) else inp for inp in args]) # Convert python literals to theano constants args = theano.tensor.subtensor.make_constant(args) # Determine if advanced indexing is needed or not # The logic is already in Subtensor.convert: if it succeeds, # standard indexing is used; if it fails with # AdvancedIndexingError, advanced indexing, or # AdvancedBooleanIndexingError, advanced indexing with boolean masks advanced = False advanced_boolean = False axis = None for i, arg in enumerate(args): try: if arg is not np.newaxis: theano.tensor.subtensor.Subtensor.convert(arg) except theano.tensor.subtensor.AdvancedIndexingError: if advanced: axis = None break else: advanced = True axis = i except theano.tensor.subtensor.AdvancedBooleanIndexingError: advanced = False advanced_boolean = True break if advanced_boolean: return theano.tensor.subtensor.advanced_boolean_subtensor(self, *args) elif advanced: if (axis is not None and all(isinstance(a, slice) and equal_slices(a, slice(None)) for a in args[:axis]) and all(isinstance(a, slice) and equal_slices(a, slice(None)) for a in args[axis + 1:]) and (not hasattr(args[axis], 'dtype') or args[axis].dtype != 'bool') and isinstance(args[axis], (np.ndarray, list, TensorVariable, TensorConstant, theano.tensor.sharedvar.TensorSharedVariable))): return self.take(args[axis], axis) else: return theano.tensor.subtensor.advanced_subtensor(self, *args) else: if np.newaxis in args: # None (aka np.newaxis) in numpy indexing means to add a # broadcastable dimension, which theano traditionally did with # the dimshuffle op. The following code converts numpy-style # indexing on self to traditional [read: implemented] theano # indexing on a dimshuffled view of self. counter = 0 pattern = [] new_args = [] for arg in args: if arg == np.newaxis: pattern.append('x') new_args.append(slice(None, None, None)) else: pattern.append(counter) counter += 1 new_args.append(arg) view = self.dimshuffle(pattern) full_slices = True for arg in new_args: # We can't do arg == slice(None, None, None) as in # Python 2.7, this call __lt__ if we have a slice # with some symbolic variable. if not (isinstance(arg, slice) and arg.start is None and arg.stop is None and arg.step is None): full_slices = False if full_slices: return view else: return view.__getitem__(tuple(new_args)) else: return theano.tensor.subtensor.Subtensor(args)( self, *theano.tensor.subtensor.Subtensor.collapse( args, lambda entry: isinstance(entry, Variable))) def take(self, indices, axis=None, mode='raise'): return theano.tensor.subtensor.take(self, indices, axis, mode) # COPYING def copy(self, name=None): """Return a symbolic copy and optionally assign a name. Does not copy the tags. """ copied_variable = theano.tensor.basic.tensor_copy(self) copied_variable.name = name return copied_variable def __iter__(self): try: for i in xrange(theano.tensor.basic.get_vector_length(self)): yield self[i] except TypeError: # This prevents accidental iteration via builtin.sum(self) raise TypeError(('TensorType does not support iteration. ' 'Maybe you are using builtin.sum instead of ' 'theano.tensor.sum? (Maybe .max?)')) # CONVENIENT ACCESS TO TYPE PROPERTIES ndim = property(lambda self: self.type.ndim) """The rank of this tensor.""" broadcastable = property(lambda self: self.type.broadcastable) """ The broadcastable signature of this tensor. See Also -------- broadcasting """ dtype = property(lambda self: self.type.dtype) """The dtype of this tensor.""" # extra pseudo-operator symbols def __dot__(left, right): return theano.tensor.basic.dot(left, right) def __rdot__(right, left): return theano.tensor.basic.dot(left, right) dot = __dot__ def sum(self, axis=None, dtype=None, keepdims=False, acc_dtype=None): """See `theano.tensor.sum`.""" return theano.tensor.basic.sum(self, axis=axis, dtype=dtype, keepdims=keepdims, acc_dtype=acc_dtype) def prod(self, axis=None, dtype=None, keepdims=False, acc_dtype=None): """See `theano.tensor.prod`.""" return theano.tensor.basic.prod(self, axis=axis, dtype=dtype, keepdims=keepdims, acc_dtype=acc_dtype) def norm(self, L, axis=None, keepdims=False): if L == 0: raise NotImplementedError() if np.isinf(L): raise NotImplementedError() # optimizations will/should catch cases like L=1, L=2 y = theano.tensor.basic.pow( theano.tensor.basic.pow( theano.tensor.basic.abs_(self), L).sum(axis=axis), 1.0 / L) if keepdims: return theano.tensor.basic.makeKeepDims(self, y, axis) else: return y def mean(self, axis=None, dtype=None, keepdims=False, acc_dtype=None): """See `theano.tensor.mean`.""" return theano.tensor.basic.mean(self, axis=axis, dtype=dtype, keepdims=keepdims, acc_dtype=acc_dtype) def var(self, axis=None, ddof=0, keepdims=False, corrected=False): """See `theano.tensor.var`.""" return theano.tensor.basic.var(self, axis=axis, ddof=ddof, keepdims=keepdims, corrected=corrected) def std(self, axis=None, ddof=0, keepdims=False, corrected=False): """See `theano.tensor.std`.""" return theano.tensor.basic.std(self, axis=axis, ddof=ddof, keepdims=keepdims, corrected=corrected) def min(self, axis=None, keepdims=False): """See `theano.tensor.min`.""" return theano.tensor.basic.min(self, axis, keepdims=keepdims) def max(self, axis=None, keepdims=False): """See `theano.tensor.max`.""" return theano.tensor.basic.max(self, axis, keepdims=keepdims) def argmin(self, axis=None, keepdims=False): """See `theano.tensor.argmin`.""" return theano.tensor.basic.argmin(self, axis, keepdims=keepdims) def argmax(self, axis=None, keepdims=False): """See `theano.tensor.argmax`.""" return theano.tensor.basic.argmax(self, axis, keepdims=keepdims) def nonzero(self, return_matrix=False): """See `theano.tensor.nonzero`.""" return theano.tensor.basic.nonzero(self, return_matrix=return_matrix) def nonzero_values(self): """See `theano.tensor.nonzero_values`.""" return theano.tensor.basic.nonzero_values(self) def sort(self, axis=-1, kind='quicksort', order=None): """See `theano.tensor.sort`.""" return theano.tensor.sort(self, axis, kind, order) def argsort(self, axis=-1, kind='quicksort', order=None): """See `theano.tensor.argsort`.""" return theano.tensor.argsort(self, axis, kind, order) def clip(self, a_min, a_max): "Clip (limit) the values in an array." return theano.tensor.basic.clip(self, a_min, a_max) def conj(self): """See `theano.tensor.conj`.""" return theano.tensor.basic.conj(self) conjugate = conj def repeat(self, repeats, axis=None): """See `theano.tensor.repeat`.""" return theano.tensor.extra_ops.repeat(self, repeats, axis) def round(self, mode=None): """See `theano.tensor.round`.""" return theano.tensor.basic.round(self, mode) def trace(self): return theano.tensor.nlinalg.trace(self) # TO TRUMP NUMPY OPERATORS __array_priority__ = 1000 def get_scalar_constant_value(self): return theano.tensor.basic.get_scalar_constant_value(self) def zeros_like(model, dtype=None): return theano.tensor.basic.zeros_like(model, dtype=dtype) def ones_like(model, dtype=None): return theano.tensor.basic.ones_like(model, dtype=dtype) def cumsum(self, axis=None): return theano.tensor.extra_ops.cumsum(self, axis) def cumprod(self, axis=None): return theano.tensor.extra_ops.cumprod(self, axis) def searchsorted(self, v, side='left', sorter=None): return theano.tensor.extra_ops.searchsorted(self, v, side, sorter) def ptp(self, axis=None): """See 'theano.tensor.ptp'.""" return theano.tensor.ptp(self, axis) def swapaxes(self, axis1, axis2): """ Return 'tensor.swapaxes(self, axis1, axis2). If a matrix is provided with the right axes, its transpose will be returned. """ return theano.tensor.basic.swapaxes(self, axis1, axis2) def fill(self, value): """Fill inputted tensor with the assigned value.""" return theano.tensor.basic.fill(self, value) def choose(self, choices, out=None, mode='raise'): """ Construct an array from an index array and a set of arrays to choose from. """ return theano.tensor.basic.choose(self, choices, out=None, mode='raise') def squeeze(self): """ Remove broadcastable dimensions from the shape of an array. It returns the input array, but with the broadcastable dimensions removed. This is always `x` itself or a view into `x`. """ return theano.tensor.extra_ops.squeeze(self) def compress(self, a, axis=None): """Return selected slices only.""" return theano.tensor.extra_ops.compress(self, a, axis=axis) class TensorVariable(_tensor_py_operators, Variable): """ Subclass to add the tensor operators to the basic `Variable` class. """ def __init__(self, type, owner=None, index=None, name=None): super(TensorVariable, self).__init__(type, owner=owner, index=index, name=name) if (config.warn_float64 != 'ignore' and type.dtype == 'float64'): msg = ('You are creating a TensorVariable ' 'with float64 dtype. You requested an action via ' 'the Theano flag warn_float64={ignore,warn,raise,pdb}.') if config.warn_float64 == "warn": # Get the user stack. We don't want function inside the # tensor and gof directory to be shown to the user. x = tb.extract_stack() nb_rm = 0 while x: file_path = x[-1][0] rm = False for p in ["theano/tensor/", "theano\\tensor\\", "theano/gof/", "theano\\tensor\\"]: if p in file_path: x = x[:-1] nb_rm += 1 rm = True break if not rm: break warnings.warn(msg, stacklevel=1 + nb_rm) elif config.warn_float64 == "raise": raise Exception(msg) elif config.warn_float64 == 'pdb': import pdb pdb.set_trace() TensorType.Variable = TensorVariable class TensorConstantSignature(tuple): """ A Signature object for comparing TensorConstant instances. An instance is a pair: (Type instance, ndarray). """ def __eq__(self, other): if type(self) != type(other): return False try: (t0, d0), (t1, d1) = self, other except Exception: return False # N.B. compare shape to ensure no broadcasting in == if t0 != t1 or d0.shape != d1.shape: return False self.no_nan # Ensure has_nan is computed. # Note that in the comparisons below, the elementwise comparisons # come last because they are the most expensive checks. if self.has_nan: other.no_nan # Ensure has_nan is computed. return (other.has_nan and self.sum == other.sum and (self.no_nan.mask == other.no_nan.mask).all() and # Note that the second test below (==) may crash e.g. for # a single scalar NaN value, so we do not run it when all # values are missing. (self.no_nan.mask.all() or (self.no_nan == other.no_nan).all())) else: # Simple case where we do not need to worry about NaN values. # (note that if there are NaN values in d1, this will return # False, which is why we do not bother with testing `other.has_nan` # here). return (self.sum == other.sum) and np.all(d0 == d1) def __ne__(self, other): return not self == other def __hash__(self): t, d = self return hashtype(self) ^ hash(t) ^ hash(d.shape) ^ hash(self.sum) def theano_hash(self): _, d = self return hash_from_ndarray(d) def _get_sum(self): """Compute sum of non NaN / Inf values in the array.""" try: return self._sum except AttributeError: self._sum = self.no_nan.sum() # The following 2 lines are needede as in Python 3.3 with NumPy # 1.7.1, numpy.ndarray and numpy.memmap aren't hashable. if type(self._sum) is np.memmap: self._sum = np.asarray(self._sum).item() if self.has_nan and self.no_nan.mask.all(): # In this case the sum is not properly computed by numpy. self._sum = 0 if np.isinf(self._sum) or np.isnan(self._sum): # NaN may happen when there are both -inf and +inf values. if self.has_nan: # Filter both NaN and Inf values. mask = self.no_nan.mask + np.isinf(self[1]) else: # Filter only Inf values. mask = np.isinf(self[1]) if mask.all(): self._sum = 0 else: self._sum = np.ma.masked_array(self[1], mask).sum() # At this point there should be no more NaN. assert not np.isnan(self._sum) return self._sum sum = property(_get_sum) def _get_no_nan(self): try: return self._no_nan except AttributeError: nan_mask = np.isnan(self[1]) if nan_mask.any(): self._no_nan = np.ma.masked_array(self[1], nan_mask) self.has_nan = True else: self._no_nan = self[1] self.has_nan = False return self._no_nan no_nan = property(_get_no_nan) class TensorConstant(_tensor_py_operators, Constant): """Subclass to add the tensor operators to the basic `Constant` class. To create a TensorConstant, use the `constant` function in this module. """ def __init__(self, type, data, name=None): Constant.__init__(self, type, data, name) self.tag.unique_value = None if isinstance(data, np.ndarray) and data.ndim > 0: flat_data = data.ravel() if flat_data.shape[0]: if (flat_data == flat_data[0]).all(): self.tag.unique_value = flat_data[0] def __str__(self): if self.tag.unique_value is not None: name = "%s of %s" % (str(self.data.shape), str(self.tag.unique_value)) else: name = "%s" % self.data if len(name) > 20: name = name[:10] + ".." + name[-10:] return "TensorConstant{%s}" % name def signature(self): return TensorConstantSignature((self.type, self.data)) def equals(self, other): # Override Contant.equals to allow to compare with # numpy.ndarray, and python type. if isinstance(other, (np.ndarray, int, float)): # Make a TensorConstant to be able to compare other = theano.tensor.basic.constant(other) return (isinstance(other, TensorConstant) and self.signature() == other.signature()) def __copy__(self): # We need to do this to remove the cached attribute return type(self)(self.type, self.data, self.name) def __deepcopy__(self, memo): # We need to do this to remove the cached attribute return type(self)(copy.deepcopy(self.type, memo), copy.deepcopy(self.data, memo), copy.deepcopy(self.name, memo)) TensorType.Constant = TensorConstant Theano-1.0.0/theano/tensor/tests/00007500163562€ÿÿÿþ0000000000013203336030020216 5ustar boccoset00000000000000Theano-1.0.0/theano/tensor/tests/test_io.py00006000163562€ÿÿÿþ0000000410013203335532022234 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import unittest import theano from theano import tensor, function, Variable, Generic import numpy as np import os class T_load_tensor(unittest.TestCase): def setUp(self): self.data = np.arange(5, dtype=np.int32) self.filename = os.path.join( theano.config.compiledir, "_test.npy") np.save(self.filename, self.data) def test0(self): path = Variable(Generic()) # Not specifying mmap_mode defaults to None, and the data is # copied into main memory x = tensor.load(path, 'int32', (False,)) y = x * 2 fn = function([path], y) assert (fn(self.filename) == (self.data * 2)).all() def test_invalid_modes(self): # Modes 'r+', 'r', and 'w+' cannot work with Theano, becausei # the output array may be modified inplace, and that should not # modify the original file. path = Variable(Generic()) for mmap_mode in ('r+', 'r', 'w+', 'toto'): self.assertRaises(ValueError, tensor.load, path, 'int32', (False,), mmap_mode) def test1(self): path = Variable(Generic()) # 'c' means "copy-on-write", which allow the array to be overwritten # by an inplace Op in the graph, without modifying the underlying # file. x = tensor.load(path, 'int32', (False,), 'c') # x ** 2 has been chosen because it will work inplace. y = (x ** 2).sum() fn = function([path], y) # Call fn() twice, to check that inplace ops do not cause trouble assert (fn(self.filename) == (self.data ** 2).sum()).all() assert (fn(self.filename) == (self.data ** 2).sum()).all() def test_memmap(self): path = Variable(Generic()) x = tensor.load(path, 'int32', (False,), mmap_mode='c') fn = function([path], x) assert type(fn(self.filename)) == np.core.memmap def tearDown(self): os.remove(os.path.join( theano.config.compiledir, "_test.npy")) Theano-1.0.0/theano/tensor/tests/test_raw_random.py00006000163562€ÿÿÿþ0000013415513203335532023774 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import numpy as np import pickle from theano.tests import unittest_tools as utt from theano.tensor.raw_random import * from theano.tensor import (raw_random, ivector, dvector, iscalar, dcol, dtensor3) from theano import tensor from theano import compile, config, gof __docformat__ = "restructuredtext en" class T_random_function(utt.InferShapeTester): def setUp(self): utt.seed_rng() def test_basic_usage(self): rf = RandomFunction(np.random.RandomState.uniform, tensor.dvector) assert not rf.inplace assert getattr(rf, 'destroy_map', {}) == {} rng_R = random_state_type() # If calling RandomFunction directly, all args have to be specified, # because shape will have to be moved to the end post_r, out = rf(rng_R, (4,), 0., 1.) assert out.type == tensor.dvector f = compile.function([rng_R], out) rng_state0 = np.random.RandomState(utt.fetch_seed()) f_0 = f(rng_state0) f_1 = f(rng_state0) assert np.all(f_0 == f_1) def test_inplace_norun(self): rf = RandomFunction(np.random.RandomState.uniform, tensor.dvector, inplace=True) assert rf.inplace assert getattr(rf, 'destroy_map', {}) != {} def test_args(self): # Test that arguments to RandomFunction are honored rf2 = RandomFunction(np.random.RandomState.uniform, tensor.dvector) rf4 = RandomFunction(np.random.RandomState.uniform, tensor.dvector, inplace=True) rng_R = random_state_type() # use make_node to override some of the self.args post_r2, out2 = rf2(rng_R, (4,), -2, 2) # NOT INPLACE post_r4, out4 = rf4(rng_R, (4,), -4, 4) # INPLACE post_r2_4, out2_4 = rf2(rng_R, (4, ), -4.0, 2) # NOT INPLACE post_r2_4_4, out2_4_4 = rf2(rng_R, (4, ), -4.0, 4.0) # NOT INPLACE # configure out4 to be computed inplace # The update expression means that the random state rng_R will # be maintained by post_r4 f = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=post_r4, mutable=True)], [out2, out4, out2_4, out2_4_4], accept_inplace=True) f2, f4, f2_4, f2_4_4 = f() f2b, f4b, f2_4b, f2_4_4b = f() # print f2 # print f4 # print f2_4 # print f2_4_4 # print f2b # print f4b # print f2_4b # print f2_4_4b # setting bounds is same as multiplying by 2 assert np.allclose(f2 * 2, f4), (f2, f4) # retrieving from non-inplace generator # is same as inplace one for first call assert np.allclose(f2_4_4, f4), (f2_4_4, f4) # f4 changes from call to call, that the update has worked assert not np.allclose(f4, f4b), (f4, f4b) def test_inplace_optimization(self): # Test that FAST_RUN includes the random_make_inplace optimization # inplace = False rf2 = RandomFunction(np.random.RandomState.uniform, tensor.dvector) rng_R = random_state_type() # If calling RandomFunction directly, all args have to be specified, # because shape will have to be moved to the end post_r2, out2 = rf2(rng_R, (4,), 0., 1.) f = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=post_r2, mutable=True)], out2, mode='FAST_RUN') # DEBUG_MODE can't pass the id-based # test below # test that the RandomState object stays the same from function call to # function call, but that the values returned change from call to call. id0 = id(f[rng_R]) val0 = f() assert id0 == id(f[rng_R]) val1 = f() assert id0 == id(f[rng_R]) assert not np.allclose(val0, val1) def test_no_inplace(self): # Test that when not running inplace, the RandomState is not updated rf = RandomFunction('uniform', tensor.dvector) rng_R = random_state_type() post_r, out = rf(rng_R, (3,), 0., 1.) f = compile.function([rng_R], [post_r, out]) rng = np.random.RandomState(utt.fetch_seed()) rng0, val0 = f(rng) rng_ = np.random.RandomState(utt.fetch_seed()) # rng should still be in a fresh state self.assertTrue(rng_R.type.values_eq(rng, rng_)) # rng0 should be in an updated state self.assertFalse(rng_R.type.values_eq(rng, rng0)) f2 = compile.function( [compile.In(rng_R, value=rng, update=post_r, mutable=False)], [post_r, out]) rng2, val2 = f2() # rng should be in a fresh state self.assertTrue(rng_R.type.values_eq(rng, rng_)) # rng2 should be in an updated state self.assertFalse(rng_R.type.values_eq(rng, rng2)) # The updated state should be the same for both functions self.assertTrue(rng_R.type.values_eq(rng2, rng0)) rng3, val3 = f2() # rng2 should not have changed self.assertTrue(rng_R.type.values_eq(rng2, rng0)) # rng3 should be an updated again version of rng2 self.assertFalse(rng_R.type.values_eq(rng3, rng2)) self.assertFalse(rng_R.type.values_eq(rng3, rng)) def test_random_function_ndim(self): # Test that random_function helper function accepts argument ndim rng_R = random_state_type() # ndim is an optional argument indicating the length of the 'shape' # ndim not specified, OK post_out4, out4 = uniform(rng_R, (4,)) # ndim specified, consistent with shape, OK post_out1_4, out1_4 = uniform(rng_R, (4, ), ndim=1) post_out2_4_4, out2_4_4 = uniform(rng_R, (4, 4), ndim=2) # ndim specified, but not compatible with shape self.assertRaises(ValueError, uniform, rng_R, (4,), ndim=2) f_ok = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=post_out2_4_4, mutable=True)], [out4, out1_4, out2_4_4], accept_inplace=True) # The correct cases should execute properly o4, o1_4, o2_4_4 = f_ok() # Check the sanity of the answers self.assertTrue(np.allclose(o4, o1_4)) self.assertTrue(np.allclose(o4, o2_4_4[0])) def test_random_function_noshape_args(self): # Test if random_function helper works with args but without shape rng_R = random_state_type() # No shape, default args -> OK post_out, out = uniform(rng_R, size=None, ndim=2) f = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=post_out, mutable=True)], [out], accept_inplace=True) o, = f() # No shape, args that have to be broadcasted -> OK low = tensor.TensorType(dtype='float64', broadcastable=(False, True, True))() high = tensor.TensorType(dtype='float64', broadcastable=(True, True, True, False))() post_out2, out2 = uniform(rng_R, size=None, ndim=2, low=low, high=high) self.assertEqual(out2.ndim, 4) self.assertEqual(out2.broadcastable, (True, False, True, False)) g = compile.function( [low, high, compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=post_out2, mutable=True)], [out2], accept_inplace=True) low_v = [[[3]], [[4]], [[-5]]] high_v = [[[[5, 8]]]] o2, = g(low_v, high_v) self.assertEqual(o2.shape, (1, 3, 1, 2)) def test_random_function_noshape_noargs(self): # Test if random_function helper works without args or shape rng_R = random_state_type() # No shape, no args -> TypeError self.assertRaises(TypeError, poisson, rng_R, size=None, ndim=2) def test_random_function_ndim_added(self): # Test that random_function helper function accepts ndim_added as # keyword argument # If using numpy's uniform distribution, ndim_added should be 0, # because the shape provided as argument is the output shape. # Specifying a different ndim_added will change the Op's output ndim, # so np.uniform will produce a result of incorrect shape, # and a ValueError should be raised. def ndim_added_deco(ndim_added): def randomfunction(random_state, size=(), low=0.0, high=0.0, ndim=None): ndim, size, bcast = raw_random._infer_ndim_bcast(ndim, size) if ndim_added < 0: bcast = bcast[:ndim_added] else: bcast = bcast + ((False,) * ndim_added) assert len(bcast) == ndim + ndim_added op = RandomFunction('uniform', tensor.TensorType(dtype='float64', broadcastable=bcast), ndim_added=ndim_added) return op(random_state, size, low, high) return randomfunction uni_1 = ndim_added_deco(1) uni_0 = ndim_added_deco(0) uni_m1 = ndim_added_deco(-1) rng_R = random_state_type() p_uni11, uni11 = uni_1(rng_R, size=(4,)) p_uni12, uni12 = uni_1(rng_R, size=(3, 4)) p_uni01, uni01 = uni_0(rng_R, size=(4,)) p_uni02, uni02 = uni_0(rng_R, size=(3, 4)) p_unim11, unim11 = uni_m1(rng_R, size=(4,)) p_unim12, unim12 = uni_m1(rng_R, size=(3, 4)) self.assertEqual(uni11.ndim, 2) self.assertEqual(uni12.ndim, 3) self.assertEqual(uni01.ndim, 1) self.assertEqual(uni02.ndim, 2) self.assertEqual(unim11.ndim, 0) self.assertEqual(unim12.ndim, 1) f11 = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=p_uni11, mutable=True)], [uni11], accept_inplace=True) f12 = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=p_uni12, mutable=True)], [uni12], accept_inplace=True) fm11 = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=p_unim11, mutable=True)], [unim11], accept_inplace=True) fm12 = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=p_unim12, mutable=True)], [unim12], accept_inplace=True) f0 = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=p_uni02, mutable=True)], [uni01, uni02], accept_inplace=True) self.assertRaises(ValueError, f11) self.assertRaises(ValueError, f12) self.assertRaises(ValueError, fm11) self.assertRaises(ValueError, fm12) u01, u02 = f0() self.assertTrue(np.allclose(u01, u02[0])) def test_uniform(self): # Test that raw_random.uniform generates the same results as numpy. # Check over two calls to see if the random state is correctly updated. rng_R = random_state_type() # Use non-default parameters post_r, out = uniform(rng_R, (4,), -2.0, 2.0) f = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=post_r, mutable=True)], [out], accept_inplace=True) numpy_rng = np.random.RandomState(utt.fetch_seed()) val0 = f() val1 = f() numpy_val0 = numpy_rng.uniform(-2.0, 2.0, size=(4,)) numpy_val1 = numpy_rng.uniform(-2.0, 2.0, size=(4,)) self.assertTrue(np.allclose(val0, numpy_val0)) self.assertTrue(np.allclose(val1, numpy_val1)) def test_binomial(self): # Test that raw_random.binomial generates the same results as numpy. # Check over two calls to see if the random state is correctly updated. rng_R = random_state_type() # Use non-default parameters, and larger dimensions because of # the integer nature of the result post_r, bin = binomial(rng_R, (7, 12), 5, 0.8) f = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=post_r, mutable=True)], [bin], accept_inplace=True) numpy_rng = np.random.RandomState(utt.fetch_seed()) val0 = f() val1 = f() numpy_val0 = numpy_rng.binomial(5, 0.8, size=(7, 12)) numpy_val1 = numpy_rng.binomial(5, 0.8, size=(7, 12)) self.assertTrue(np.all(val0 == numpy_val0)) self.assertTrue(np.all(val1 == numpy_val1)) def test_normal(self): # Test that raw_random.normal generates the same results as numpy. # Check over two calls to see if the random state is correctly updated. rng_R = random_state_type() # Use non-default parameters post_r, out = normal(rng_R, (2, 3), 4.0, 2.0) f = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=post_r, mutable=True)], [out], accept_inplace=True) numpy_rng = np.random.RandomState(utt.fetch_seed()) val0 = f() val1 = f() numpy_val0 = numpy_rng.normal(4.0, 2.0, size=(2, 3)) numpy_val1 = numpy_rng.normal(4.0, 2.0, size=(2, 3)) self.assertTrue(np.allclose(val0, numpy_val0)) self.assertTrue(np.allclose(val1, numpy_val1)) def test_random_integers(self): # Test that raw_random.random_integers generates the same # results as numpy. We use randint() for comparison since # random_integers() is deprecated. # Check over two calls to see if the random state is correctly updated. rng_R = random_state_type() # Use non-default parameters, and larger dimensions because of # the integer nature of the result post_r, out = random_integers(rng_R, (11, 8), -3, 16) f = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=post_r, mutable=True)], [out], accept_inplace=True) numpy_rng = np.random.RandomState(utt.fetch_seed()) val0 = f() val1 = f() numpy_val0 = numpy_rng.randint(-3, 17, size=(11, 8)) numpy_val1 = numpy_rng.randint(-3, 17, size=(11, 8)) self.assertTrue(np.allclose(val0, numpy_val0)) self.assertTrue(np.allclose(val1, numpy_val1)) def test_permutation_helper(self): # Test that raw_random.permutation_helper generates the same # results as numpy, # and that the 'ndim_added' keyword behaves correctly. # permutation_helper needs "ndim_added=1", because its output # is one dimension more than its "shape" argument (and there's # no way to determine that automatically). # Check the working case, over two calls to see if the random # state is correctly updated. rf = RandomFunction(permutation_helper, tensor.imatrix, 8, ndim_added=1) rng_R = random_state_type() post_r, out = rf(rng_R, (7,), 8) f = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=post_r, mutable=True)], [out], accept_inplace=True) numpy_rng = np.random.RandomState(utt.fetch_seed()) val0 = f() val1 = f() # numpy_rng.permutation outputs one vector at a time, # so we call it iteratively to generate all the samples. numpy_val0 = np.asarray([numpy_rng.permutation(8) for i in range(7)]) numpy_val1 = np.asarray([numpy_rng.permutation(8) for i in range(7)]) self.assertTrue(np.all(val0 == numpy_val0)) self.assertTrue(np.all(val1 == numpy_val1)) # This call lacks "ndim_added=1", so ndim_added defaults to 0. # A ValueError should be raised. rf0 = RandomFunction(permutation_helper, tensor.imatrix, 8) post_r0, out0 = rf0(rng_R, (7,), 8) f0 = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=post_r0, mutable=True)], [out0], accept_inplace=True) self.assertRaises(ValueError, f0) # Here, ndim_added is 2 instead of 1. A ValueError should be raised. rf2 = RandomFunction(permutation_helper, tensor.imatrix, 8, ndim_added=2) post_r2, out2 = rf2(rng_R, (7,), 8) f2 = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=post_r2, mutable=True)], [out2], accept_inplace=True) self.assertRaises(ValueError, f2) def test_choice(self): # Test that raw_random.choice generates the same results as numpy. # Check over two calls to see if the random state is correctly updated. rng_R = random_state_type() # Use non-default parameters, and larger dimensions because of # the integer nature of the result post_r, out = choice(rng_R, (11, 8), 10, 1, 0) f = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=post_r, mutable=True)], [out], accept_inplace=True) numpy_rng = np.random.RandomState(utt.fetch_seed()) val0 = f() val1 = f() numpy_val0 = numpy_rng.choice(10, (11, 8), True, None) numpy_val1 = numpy_rng.choice(10, (11, 8), True, None) self.assertTrue(np.allclose(val0, numpy_val0)) self.assertTrue(np.allclose(val1, numpy_val1)) def test_poisson(self): # Test that raw_random.poisson generates the same results as numpy. # Check over two calls to see if the random state is correctly updated. rng_R = random_state_type() # Use non-default parameters, and larger dimensions because of # the integer nature of the result post_r, out = poisson(rng_R, lam=5, size=(11, 8)) f = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=post_r, mutable=True)], [out], accept_inplace=True) numpy_rng = np.random.RandomState(utt.fetch_seed()) val0 = f() val1 = f() numpy_val0 = numpy_rng.poisson(5, size=(11, 8)) numpy_val1 = numpy_rng.poisson(5, size=(11, 8)) self.assertTrue(np.allclose(val0, numpy_val0)) self.assertTrue(np.allclose(val1, numpy_val1)) def test_permutation(self): # Test that raw_random.permutation generates the same results as numpy. rng_R = random_state_type() post_r, out = permutation(rng_R, size=(9,), n=6) f = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=post_r, mutable=True)], [out], accept_inplace=True) numpy_rng = np.random.RandomState(utt.fetch_seed()) # Check over two calls to see if the random state is correctly updated. # numpy_rng.permutation outputs one vector at a time, # so we call it iteratively to generate all the samples. val0 = f() val1 = f() numpy_val0 = np.asarray([numpy_rng.permutation(6) for i in range(9)]) numpy_val1 = np.asarray([numpy_rng.permutation(6) for i in range(9)]) self.assertTrue(np.all(val0 == numpy_val0)) self.assertTrue(np.all(val1 == numpy_val1)) # Test that we can generate a list: have size=None or (). for ndim in [1, None]: post_r, out = permutation(rng_R, n=10, size=None, ndim=ndim) inp = compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=post_r, mutable=True) f = theano.function([inp], out) o = f() assert o.shape == (10,) assert (np.sort(o) == np.arange(10)).all() # Wrong number of dimensions asked self.assertRaises(TypeError, permutation, rng_R, size=None, ndim=2) def test_multinomial(self): # Test that raw_random.multinomial generates the same results as numpy. # Check over two calls to see if the random state is correctly updated. rng_R = random_state_type() post_r, out = multinomial(rng_R, (7, 3), 6, [0.2] * 5) f = compile.function( [compile.In(rng_R, value=np.random.RandomState(utt.fetch_seed()), update=post_r, mutable=True)], [out], accept_inplace=True) numpy_rng = np.random.RandomState(utt.fetch_seed()) val0, = f() val1, = f() numpy_val0 = numpy_rng.multinomial(6, [0.2] * 5, (7, 3)) numpy_val1 = numpy_rng.multinomial(6, [0.2] * 5, (7, 3)) self.assertTrue(np.all(val0 == numpy_val0)) self.assertTrue(np.all(val1 == numpy_val1)) self.assertTrue(val0.shape == (7, 3, 5)) self.assertTrue(val1.shape == (7, 3, 5)) def test_symbolic_shape(self): rng_R = random_state_type() shape = tensor.lvector() post_r, out = uniform(rng_R, shape, ndim=2) f = compile.function([rng_R, shape], out) rng_state0 = np.random.RandomState(utt.fetch_seed()) assert f(rng_state0, [2, 3]).shape == (2, 3) assert f(rng_state0, [4, 8]).shape == (4, 8) self.assertRaises(ValueError, f, rng_state0, [4]) self.assertRaises(ValueError, f, rng_state0, [4, 3, 4, 5]) def test_mixed_shape(self): # Test when the provided shape is a tuple of ints and scalar vars rng_R = random_state_type() shape0 = tensor.lscalar() shape = (shape0, 3) post_r, u = uniform(rng_R, size=shape, ndim=2) f = compile.function([rng_R, shape0], u) rng_state0 = np.random.RandomState(utt.fetch_seed()) assert f(rng_state0, 2).shape == (2, 3) assert f(rng_state0, 8).shape == (8, 3) post_r, v = uniform(rng_R, size=shape) g = compile.function([rng_R, shape0], v) assert g(rng_state0, 2).shape == (2, 3) assert g(rng_state0, 8).shape == (8, 3) def test_mixed_shape_bcastable(self): # Test when the provided shape is a tuple of ints and scalar vars rng_R = random_state_type() shape0 = tensor.lscalar() shape = (shape0, 1) post_r, u = uniform(rng_R, size=shape, ndim=2) assert u.broadcastable == (False, True) f = compile.function([rng_R, shape0], u) rng_state0 = np.random.RandomState(utt.fetch_seed()) assert f(rng_state0, 2).shape == (2, 1) assert f(rng_state0, 8).shape == (8, 1) post_r, v = uniform(rng_R, size=shape) assert v.broadcastable == (False, True) g = compile.function([rng_R, shape0], v) assert g(rng_state0, 2).shape == (2, 1) assert g(rng_state0, 8).shape == (8, 1) def test_default_shape(self): rng_R = random_state_type() post_r, out = uniform(rng_R) f = compile.function([rng_R], [post_r, out], accept_inplace=True) rng_state0 = np.random.RandomState(utt.fetch_seed()) numpy_rng = np.random.RandomState(utt.fetch_seed()) post0, val0 = f(rng_state0) post1, val1 = f(post0) numpy_val0 = np.asarray(numpy_rng.uniform(), dtype=theano.config.floatX) numpy_val1 = np.asarray(numpy_rng.uniform(), dtype=theano.config.floatX) assert np.all(val0 == numpy_val0) assert np.all(val1 == numpy_val1) post_r, out = multinomial(rng_R) g = compile.function([rng_R], [post_r, out], accept_inplace=True) post2, val2 = g(post1) numpy_val2 = np.asarray(numpy_rng.multinomial(n=1, pvals=[.5, .5]), dtype=theano.config.floatX) assert np.all(val2 == numpy_val2) def test_vector_arguments(self): rng_R = random_state_type() low = tensor.vector() post_r, out = uniform(rng_R, low=low, high=1) assert out.ndim == 1 f = compile.function([rng_R, low], [post_r, out], accept_inplace=True) def as_floatX(thing): return np.asarray(thing, dtype=theano.config.floatX) rng_state0 = np.random.RandomState(utt.fetch_seed()) numpy_rng = np.random.RandomState(utt.fetch_seed()) post0, val0 = f(rng_state0, [-5, .5, 0, 1]) post1, val1 = f(post0, as_floatX([.9])) numpy_val0 = as_floatX(numpy_rng.uniform(low=[-5, .5, 0, 1], high=1)) numpy_val1 = as_floatX(numpy_rng.uniform(low=as_floatX([.9]), high=1)) assert np.all(val0 == numpy_val0) assert np.all(val1 == numpy_val1) high = tensor.vector() post_rb, outb = uniform(rng_R, low=low, high=high) assert outb.ndim == 1 fb = compile.function([rng_R, low, high], [post_rb, outb], accept_inplace=True) post0b, val0b = fb(post1, [-4., -2], [-1, 0]) post1b, val1b = fb(post0b, [-4.], [-1]) numpy_val0b = as_floatX(numpy_rng.uniform(low=[-4., -2], high=[-1, 0])) numpy_val1b = as_floatX(numpy_rng.uniform(low=[-4.], high=[-1])) assert np.all(val0b == numpy_val0b) assert np.all(val1b == numpy_val1b) self.assertRaises(ValueError, fb, post1b, [-4., -2], [-1, 0, 1]) # TODO: do we want that? #self.assertRaises(ValueError, fb, post1b, [-4., -2], [-1]) size = tensor.lvector() post_rc, outc = uniform(rng_R, low=low, high=high, size=size, ndim=1) fc = compile.function([rng_R, low, high, size], [post_rc, outc], accept_inplace=True) post0c, val0c = fc(post1b, [-4., -2], [-1, 0], [2]) post1c, val1c = fc(post0c, [-4.], [-1], [1]) numpy_val0c = as_floatX(numpy_rng.uniform(low=[-4., -2], high=[-1, 0])) numpy_val1c = as_floatX(numpy_rng.uniform(low=[-4.], high=[-1])) assert np.all(val0c == numpy_val0c) assert np.all(val1c == numpy_val1c) self.assertRaises(ValueError, fc, post1c, [-4., -2], [-1, 0], [1]) self.assertRaises(ValueError, fc, post1c, [-4., -2], [-1, 0], [1, 2]) self.assertRaises(ValueError, fc, post1c, [-4., -2], [-1, 0], [2, 1]) self.assertRaises(ValueError, fc, post1c, [-4., -2], [-1], [1]) # TODO: do we want that? #self.assertRaises(ValueError, fc, post1c, [-4., -2], [-1], [2]) def test_broadcast_arguments(self): rng_R = random_state_type() low = tensor.dvector() high = tensor.dcol() post_r, out = uniform(rng_R, low=low, high=high) assert out.ndim == 2 f = compile.function([rng_R, low, high], [post_r, out], accept_inplace=True) rng_state0 = np.random.RandomState(utt.fetch_seed()) numpy_rng = np.random.RandomState(utt.fetch_seed()) post0, val0 = f(rng_state0, [-5, .5, 0, 1], [[1.]]) post1, val1 = f(post0, [.9], [[1.], [1.1], [1.5]]) post2, val2 = f(post1, [-5, .5, 0, 1], [[1.], [1.1], [1.5]]) numpy_val0 = numpy_rng.uniform(low=[-5, .5, 0, 1], high=[1.]) numpy_val1 = numpy_rng.uniform(low=[.9], high=[[1.], [1.1], [1.5]]) numpy_val2 = numpy_rng.uniform(low=[-5, .5, 0, 1], high=[[1.], [1.1], [1.5]]) assert np.all(val0 == numpy_val0), (val0, numpy_val0) assert np.all(val1 == numpy_val1) assert np.all(val2 == numpy_val2) def test_uniform_vector(self): rng_R = random_state_type() low = tensor.vector() high = tensor.vector() post_r, out = uniform(rng_R, low=low, high=high) assert out.ndim == 1 f = compile.function([rng_R, low, high], [post_r, out], accept_inplace=True) def as_floatX(thing): return np.asarray(thing, dtype=theano.config.floatX) low_val = as_floatX([.1, .2, .3]) high_val = as_floatX([1.1, 2.2, 3.3]) rng = np.random.RandomState(utt.fetch_seed()) numpy_rng = np.random.RandomState(utt.fetch_seed()) # Arguments of size (3,) rng0, val0 = f(rng, low_val, high_val) numpy_val0 = as_floatX(numpy_rng.uniform(low=low_val, high=high_val)) assert np.all(val0 == numpy_val0) # arguments of size (2,) rng1, val1 = f(rng0, low_val[:-1], high_val[:-1]) numpy_val1 = as_floatX(numpy_rng.uniform(low=low_val[:-1], high=high_val[:-1])) assert np.all(val1 == numpy_val1) # Specifying the size explicitly g = compile.function([rng_R, low, high], uniform(rng_R, low=low, high=high, size=(3,)), accept_inplace=True) rng2, val2 = g(rng1, low_val, high_val) numpy_val2 = as_floatX(numpy_rng.uniform(low=low_val, high=high_val, size=(3,))) assert np.all(val2 == numpy_val2) self.assertRaises(ValueError, g, rng2, low_val[:-1], high_val[:-1]) def test_binomial_vector(self): rng_R = random_state_type() n = tensor.lvector() prob = tensor.vector() post_r, out = binomial(rng_R, n=n, p=prob) assert out.ndim == 1 f = compile.function([rng_R, n, prob], [post_r, out], accept_inplace=True) n_val = [1, 2, 3] prob_val = np.asarray([.1, .2, .3], dtype=config.floatX) rng = np.random.RandomState(utt.fetch_seed()) numpy_rng = np.random.RandomState(utt.fetch_seed()) # Arguments of size (3,) rng0, val0 = f(rng, n_val, prob_val) numpy_val0 = numpy_rng.binomial(n=n_val, p=prob_val) assert np.all(val0 == numpy_val0) # arguments of size (2,) rng1, val1 = f(rng0, n_val[:-1], prob_val[:-1]) numpy_val1 = numpy_rng.binomial(n=n_val[:-1], p=prob_val[:-1]) assert np.all(val1 == numpy_val1) # Specifying the size explicitly g = compile.function([rng_R, n, prob], binomial(rng_R, n=n, p=prob, size=(3,)), accept_inplace=True) rng2, val2 = g(rng1, n_val, prob_val) numpy_val2 = numpy_rng.binomial(n=n_val, p=prob_val, size=(3,)) assert np.all(val2 == numpy_val2) self.assertRaises(ValueError, g, rng2, n_val[:-1], prob_val[:-1]) def test_normal_vector(self): rng_R = random_state_type() avg = tensor.vector() std = tensor.vector() post_r, out = normal(rng_R, avg=avg, std=std) assert out.ndim == 1 f = compile.function([rng_R, avg, std], [post_r, out], accept_inplace=True) def as_floatX(thing): return np.asarray(thing, dtype=theano.config.floatX) avg_val = [1, 2, 3] std_val = as_floatX([.1, .2, .3]) rng = np.random.RandomState(utt.fetch_seed()) numpy_rng = np.random.RandomState(utt.fetch_seed()) # Arguments of size (3,) rng0, val0 = f(rng, avg_val, std_val) numpy_val0 = as_floatX(numpy_rng.normal(loc=as_floatX(avg_val), scale=as_floatX(std_val))) assert np.all(val0 == numpy_val0) # arguments of size (2,) rng1, val1 = f(rng0, avg_val[:-1], std_val[:-1]) numpy_val1 = np.asarray(numpy_rng.normal(loc=avg_val[:-1], scale=std_val[:-1]), dtype=theano.config.floatX) assert np.all(val1 == numpy_val1) # Specifying the size explicitly g = compile.function([rng_R, avg, std], normal(rng_R, avg=avg, std=std, size=(3,)), accept_inplace=True) rng2, val2 = g(rng1, avg_val, std_val) numpy_val2 = np.asarray(numpy_rng.normal(loc=avg_val, scale=std_val, size=(3,)), dtype=theano.config.floatX) assert np.all(val2 == numpy_val2) self.assertRaises(ValueError, g, rng2, avg_val[:-1], std_val[:-1]) def test_random_integers_vector(self): rng_R = random_state_type() low = tensor.lvector() high = tensor.lvector() post_r, out = random_integers(rng_R, low=low, high=high) assert out.ndim == 1 f = compile.function([rng_R, low, high], [post_r, out], accept_inplace=True) low_val = [100, 200, 300] high_val = [110, 220, 330] rng = np.random.RandomState(utt.fetch_seed()) numpy_rng = np.random.RandomState(utt.fetch_seed()) # Arguments of size (3,) rng0, val0 = f(rng, low_val, high_val) numpy_val0 = np.asarray([numpy_rng.randint(low=lv, high=hv+1) for lv, hv in zip(low_val, high_val)]) assert np.all(val0 == numpy_val0) # arguments of size (2,) rng1, val1 = f(rng0, low_val[:-1], high_val[:-1]) numpy_val1 = np.asarray([numpy_rng.randint(low=lv, high=hv+1) for lv, hv in zip(low_val[:-1], high_val[:-1])]) assert np.all(val1 == numpy_val1) # Specifying the size explicitly g = compile.function([rng_R, low, high], random_integers(rng_R, low=low, high=high, size=(3,)), accept_inplace=True) rng2, val2 = g(rng1, low_val, high_val) numpy_val2 = np.asarray([numpy_rng.randint(low=lv, high=hv+1) for lv, hv in zip(low_val, high_val)]) assert np.all(val2 == numpy_val2) self.assertRaises(ValueError, g, rng2, low_val[:-1], high_val[:-1]) # Vectorized permutation don't make sense: the only parameter, n, # controls one dimension of the returned tensor. def test_multinomial_vector(self): rng_R = random_state_type() n = tensor.lvector() pvals = tensor.matrix() post_r, out = multinomial(rng_R, n=n, pvals=pvals) assert out.ndim == 2 f = compile.function([rng_R, n, pvals], [post_r, out], accept_inplace=True) n_val = [1, 2, 3] pvals_val = [[.1, .9], [.2, .8], [.3, .7]] pvals_val = np.asarray(pvals_val, dtype=config.floatX) rng = np.random.RandomState(utt.fetch_seed()) numpy_rng = np.random.RandomState(utt.fetch_seed()) # Arguments of size (3,) rng0, val0 = f(rng, n_val, pvals_val) numpy_val0 = np.asarray([numpy_rng.multinomial(n=nv, pvals=pv) for nv, pv in zip(n_val, pvals_val)]) assert np.all(val0 == numpy_val0) # arguments of size (2,) rng1, val1 = f(rng0, n_val[:-1], pvals_val[:-1]) numpy_val1 = np.asarray([numpy_rng.multinomial(n=nv, pvals=pv) for nv, pv in zip(n_val[:-1], pvals_val[:-1])]) assert np.all(val1 == numpy_val1) # Specifying the size explicitly g = compile.function([rng_R, n, pvals], multinomial(rng_R, n=n, pvals=pvals, size=(3,)), accept_inplace=True) rng2, val2 = g(rng1, n_val, pvals_val) numpy_val2 = np.asarray([numpy_rng.multinomial(n=nv, pvals=pv) for nv, pv in zip(n_val, pvals_val)]) assert np.all(val2 == numpy_val2) self.assertRaises(ValueError, g, rng2, n_val[:-1], pvals_val[:-1]) def test_multinomial_tensor3_a(self): # Test the examples given in the multinomial documentation regarding # tensor3 objects rng_R = random_state_type() n = 9 pvals = tensor.dtensor3() post_r, out = multinomial(rng_R, n=n, pvals=pvals, size=(1, -1)) assert out.ndim == 3 assert out.broadcastable == (True, False, False) f = compile.function([rng_R, pvals], [post_r, out], accept_inplace=True) rng = np.random.RandomState(utt.fetch_seed()) numpy_rng = np.random.RandomState(utt.fetch_seed()) pvals_val = np.asarray([[[.1, .9], [.2, .8], [.3, .7]]]) assert pvals_val.shape == (1, 3, 2) new_rng, draw = f(rng, pvals_val) assert draw.shape == (1, 3, 2) assert np.allclose(draw.sum(axis=2), 9) def test_multinomial_tensor3_b(self): # Test the examples given in the multinomial documentation regarding # tensor3 objects rng_R = random_state_type() n = 9 pvals = tensor.dtensor3() post_r, out = multinomial(rng_R, n=n, pvals=pvals, size=(10, 1, -1)) assert out.ndim == 4 assert out.broadcastable == (False, True, False, False) f = compile.function([rng_R, pvals], [post_r, out], accept_inplace=True) rng = np.random.RandomState(utt.fetch_seed()) numpy_rng = np.random.RandomState(utt.fetch_seed()) pvals_val = np.asarray([[[.1, .9], [.2, .8], [.3, .7]]]) assert pvals_val.shape == (1, 3, 2) out_rng, draw = f(rng, pvals_val) assert draw.shape == (10, 1, 3, 2) assert np.allclose(draw.sum(axis=3), 9) def test_dtype(self): rng_R = random_state_type() low = tensor.lscalar() high = tensor.lscalar() post_r, out = random_integers(rng_R, low=low, high=high, size=(20, ), dtype='int8') assert out.dtype == 'int8' f = compile.function([rng_R, low, high], [post_r, out]) rng = np.random.RandomState(utt.fetch_seed()) rng0, val0 = f(rng, 0, 9) assert val0.dtype == 'int8' rng1, val1 = f(rng0, 255, 257) assert val1.dtype == 'int8' assert np.all(abs(val1) <= 1) def test_dtype_normal_uniform_687(self): # Regression test for #687. rng_R = random_state_type() assert uniform(rng_R, low=tensor.constant(0, dtype='float64'), dtype='float32')[1].dtype == 'float32' assert normal(rng_R, avg=tensor.constant(0, dtype='float64'), dtype='float32')[1].dtype == 'float32' def setUp(self): super(T_random_function, self).setUp() def test_infer_shape(self): rng_R = random_state_type() rng_R_val = np.random.RandomState(utt.fetch_seed()) # no shape specified, default args post_r, out = uniform(rng_R) self._compile_and_check([rng_R], [out], [rng_R_val], RandomFunction) post_r, out = uniform(rng_R, size=None, ndim=2) self._compile_and_check([rng_R], [out], [rng_R_val], RandomFunction) ''' #infer_shape don't work for multinomial. #The parameter ndim_added is set to 1 and in this case, the infer_shape #inplementation don't know how to infer the shape post_r, out = multinomial(rng_R) self._compile_and_check([rng_R], [out], [rng_R_val], RandomFunction) ''' # no shape specified, args have to be broadcasted low = tensor.TensorType(dtype='float64', broadcastable=(False, True, True))() high = tensor.TensorType(dtype='float64', broadcastable=(True, True, True, False))() post_r, out = uniform(rng_R, size=None, ndim=2, low=low, high=high) low_val = [[[3]], [[4]], [[-5]]] high_val = [[[[5, 8]]]] self._compile_and_check([rng_R, low, high], [out], [rng_R_val, low_val, high_val], RandomFunction) # multinomial, specified shape ''' #infer_shape don't work for multinomial n = iscalar() pvals = dvector() size_val = (7, 3) n_val = 6 pvals_val = [0.2] * 5 post_r, out = multinomial(rng_R, size=size_val, n=n, pvals=pvals, ndim=2) self._compile_and_check([rng_R, n, pvals], [out], [rng_R_val, n_val, pvals_val], RandomFunction) ''' # uniform vector low and high low = dvector() high = dvector() post_r, out = uniform(rng_R, low=low, high=1) low_val = [-5, .5, 0, 1] self._compile_and_check([rng_R, low], [out], [rng_R_val, low_val], RandomFunction) low_val = [.9] self._compile_and_check([rng_R, low], [out], [rng_R_val, low_val], RandomFunction) post_r, out = uniform(rng_R, low=low, high=high) low_val = [-4., -2] high_val = [-1, 0] self._compile_and_check([rng_R, low, high], [out], [rng_R_val, low_val, high_val], RandomFunction) low_val = [-4.] high_val = [-1] self._compile_and_check([rng_R, low, high], [out], [rng_R_val, low_val, high_val], RandomFunction) # uniform broadcasting low and high low = dvector() high = dcol() post_r, out = uniform(rng_R, low=low, high=high) low_val = [-5, .5, 0, 1] high_val = [[1.]] self._compile_and_check([rng_R, low, high], [out], [rng_R_val, low_val, high_val], RandomFunction) low_val = [.9] high_val = [[1.], [1.1], [1.5]] self._compile_and_check([rng_R, low, high], [out], [rng_R_val, low_val, high_val], RandomFunction) low_val = [-5, .5, 0, 1] high_val = [[1.], [1.1], [1.5]] self._compile_and_check([rng_R, low, high], [out], [rng_R_val, low_val, high_val], RandomFunction) # uniform with vector slice low = dvector() high = dvector() post_r, out = uniform(rng_R, low=low, high=high) low_val = [.1, .2, .3] high_val = [1.1, 2.2, 3.3] size_val = (3, ) self._compile_and_check([rng_R, low, high], [out], [rng_R_val, low_val[:-1], high_val[:-1]], RandomFunction) # uniform with explicit size and size implicit in parameters # NOTE 1: Would it be desirable that size could also be supplied # as a Theano variable? post_r, out = uniform(rng_R, size=size_val, low=low, high=high) self._compile_and_check([rng_R, low, high], [out], [rng_R_val, low_val, high_val], RandomFunction) # binomial with vector slice n = ivector() prob = dvector() post_r, out = binomial(rng_R, n=n, p=prob) n_val = [1, 2, 3] prob_val = [.1, .2, .3] size_val = (3, ) self._compile_and_check([rng_R, n, prob], [out], [rng_R_val, n_val[:-1], prob_val[:-1]], RandomFunction) # binomial with explicit size and size implicit in parameters # cf. NOTE 1 post_r, out = binomial(rng_R, n=n, p=prob, size=size_val) self._compile_and_check([rng_R, n, prob], [out], [rng_R_val, n_val, prob_val], RandomFunction) # normal with vector slice avg = dvector() std = dvector() post_r, out = normal(rng_R, avg=avg, std=std) avg_val = [1, 2, 3] std_val = [.1, .2, .3] size_val = (3, ) self._compile_and_check([rng_R, avg, std], [out], [rng_R_val, avg_val[:-1], std_val[:-1]], RandomFunction) # normal with explicit size and size implicit in parameters # cf. NOTE 1 post_r, out = normal(rng_R, avg=avg, std=std, size=size_val) self._compile_and_check([rng_R, avg, std], [out], [rng_R_val, avg_val, std_val], RandomFunction) # multinomial with tensor-3 probabilities ''' #multinomial infer_shape don't work. pvals = dtensor3() n = iscalar() post_r, out = multinomial(rng_R, n=n, pvals=pvals, size=(1, -1)) pvals_val = [[[.1, .9], [.2, .8], [.3, .7]]] n_val = 9 self._compile_and_check([rng_R, n, pvals], [out], [rng_R_val, n_val, pvals_val], RandomFunction) post_r, out = multinomial(rng_R, n=n, pvals=pvals, size=(10, 1, -1)) self._compile_and_check([rng_R, n, pvals], [out], [rng_R_val, n_val, pvals_val], RandomFunction) ''' def test_pkl(self): # Test pickling of RandomFunction. # binomial was created by calling RandomFunction on a string, # random_integers by calling it on a function. rng_r = random_state_type() mode = None if theano.config.mode in ["DEBUG_MODE", "DebugMode"]: mode = 'FAST_COMPILE' post_bin_r, bin_sample = binomial(rng_r, (3, 5), 1, .3) f = theano.function([rng_r], [post_bin_r, bin_sample], mode=mode) pkl_f = pickle.dumps(f) post_int_r, int_sample = random_integers(rng_r, (3, 5), -1, 8) g = theano.function([rng_r], [post_int_r, int_sample], mode=mode) pkl_g = pickle.dumps(g) pickle.loads(pkl_g) if __name__ == '__main__': from theano.tests import main main("test_raw_random") Theano-1.0.0/theano/tensor/tests/test_mpi.py00006000163562€ÿÿÿþ0000000621213203335532022420 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import os import subprocess from nose.plugins.skip import SkipTest import theano from theano import change_flags from theano.compat import PY3 from theano.gof.sched import sort_schedule_fn from theano.tensor.io import (send, recv, mpi_cmps, MPISend, MPISendWait, mpi_send_wait_cmp, mpi_tag_cmp, mpi_enabled) mpi_scheduler = sort_schedule_fn(*mpi_cmps) mpi_linker = theano.OpWiseCLinker(schedule=mpi_scheduler) mpi_mode = theano.Mode(linker=mpi_linker) @change_flags(compute_test_value='off') def test_recv(): x = recv((10, 10), 'float64', 0, 11) assert x.dtype == 'float64' assert x.broadcastable == (False, False) recvnode = x.owner.inputs[0].owner assert recvnode.op.source == 0 assert recvnode.op.tag == 11 def test_send(): x = theano.tensor.matrix('x') y = send(x, 1, 11) sendnode = y.owner.inputs[0].owner assert sendnode.op.dest == 1 assert sendnode.op.tag == 11 @change_flags(compute_test_value='off') def test_can_make_function(): x = recv((5, 5), 'float32', 0, 11) y = x+1 assert theano.function([], [y]) def test_mpi_roundtrip(): if not mpi_enabled: raise SkipTest('MPI not enabled') theano_root = theano.__file__.split('__init__')[0] env = os.environ.copy() flags = env.get('THEANO_FLAGS', '') keep_flags = ','.join((f for f in flags.split(',') if not f.startswith('init_gpu_device'))) env['THEANO_FLAGS'] = keep_flags p = subprocess.Popen("mpiexec -np 2 python " + theano_root + "tensor/tests/_test_mpi_roundtrip.py", stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, close_fds=True, env=env) (stdout, stderr) = p.communicate() result = theano.compat.decode(stdout) assert "True" in result, theano.compat.decode(stderr) def test_mpi_send_wait_cmp(): x = theano.tensor.matrix('x') y = send(x, 1, 11) z = x + x waitnode = y.owner sendnode = y.owner.inputs[0].owner addnode = z.owner assert mpi_send_wait_cmp(sendnode, addnode) < 0 # send happens first assert mpi_send_wait_cmp(waitnode, addnode) > 0 # wait happens last @change_flags(compute_test_value='off') def test_mpi_tag_ordering(): x = recv((2, 2), 'float32', 1, 12) y = recv((2, 2), 'float32', 1, 11) z = recv((2, 2), 'float32', 1, 13) f = theano.function([], [x, y, z], mode=mpi_mode) nodes = f.maker.linker.make_all()[-1] assert all(node.op.tag == tag for node, tag in zip(nodes, (11, 12, 13, 11, 12, 13))) def test_mpi_schedule(): x = theano.tensor.matrix('x') y = send(x, 1, 11) z = x + x waitnode = y.owner sendnode = y.owner.inputs[0].owner addnode = z.owner f = theano.function([x], [y, z], mode=mpi_mode) nodes = f.maker.linker.make_all()[-1] optypes = [MPISend, theano.tensor.Elemwise, MPISendWait] assert all(isinstance(node.op, optype) for node, optype in zip(nodes, optypes)) Theano-1.0.0/theano/tensor/tests/test_opt.py00006000163562€ÿÿÿþ0000105367013203335532022450 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import copy import logging import time import unittest import numpy as np from six.moves import xrange from nose.plugins.skip import SkipTest from nose.tools import assert_raises, assert_true import theano import theano.scalar as scal from six import StringIO from theano import compile from theano.compile import deep_copy_op, DeepCopyOp from theano.compile import get_mode from theano import config from theano import function from theano import gof from theano import pprint from theano import shared from theano.gof import FunctionGraph import theano.tensor.opt as opt from theano.tensor.opt import ( local_add_specialize, local_dimshuffle_lift, local_useless_dimshuffle_in_reshape, local_useless_alloc, local_merge_alloc, local_greedy_distributor, local_useless_reshape, local_reshape_to_dimshuffle, mul_canonizer, Shape_i, Assert, MakeVector, make_vector, local_canonicalize_alloc ) from theano import tensor from theano import tensor as T from theano.tensor import scalar, iscalar, lscalar, fscalar, dscalar from theano.tensor import vector, lvector, fvector, dvector from theano.tensor import matrix, fmatrix, dmatrix, tensor3 from theano.tensor import vectors, matrices, fmatrices, dmatrices from theano.tensor import ( AdvancedSubtensor, AdvancedSubtensor1, as_tensor_variable, IncSubtensor, AdvancedIncSubtensor, AdvancedIncSubtensor1, inplace, Join, join, Subtensor, TensorType, tile ) from theano.tensor.elemwise import DimShuffle from theano.tensor.type import values_eq_approx_remove_nan from theano.tests import unittest_tools as utt from theano.gof.opt import check_stack_trace, out2in from theano import change_flags from nose.plugins.attrib import attr mode_opt = theano.config.mode if mode_opt == 'FAST_COMPILE': mode_opt = 'FAST_RUN' mode_opt = theano.compile.mode.get_mode(mode_opt) dimshuffle_lift = out2in(local_dimshuffle_lift) _optimizer_stabilize = gof.Query(include=['fast_run']) _optimizer_stabilize.position_cutoff = 1.51 _optimizer_stabilize = compile.optdb.query(_optimizer_stabilize) _optimizer_specialize = gof.Query(include=['fast_run']) _optimizer_specialize.position_cutoff = 2.01 _optimizer_specialize = compile.optdb.query(_optimizer_specialize) _optimizer_fast_run = gof.Query(include=['fast_run']) _optimizer_fast_run = compile.optdb.query(_optimizer_fast_run) def ds(x, y): return DimShuffle(x.type.broadcastable, y)(x) def optimize(g, level='fast_run'): if level == 'fast_run': _optimizer_fast_run.optimize(g) elif level == 'specialize': _optimizer_specialize.optimize(g) elif level == 'stabilize': _optimizer_stabilize.optimize(g) else: raise ValueError(level) return g def inputs(xbc=(0, 0), ybc=(0, 0), zbc=(0, 0)): x = TensorType(broadcastable=xbc, dtype='float64')('x') y = TensorType(broadcastable=ybc, dtype='float64')('y') z = TensorType(broadcastable=zbc, dtype='float64')('z') return x, y, z class test_dimshuffle_lift(unittest.TestCase): def test_double_transpose(self): x, y, z = inputs() e = ds(ds(x, (1, 0)), (1, 0)) g = FunctionGraph([x], [e]) self.assertTrue(str(g) == "[InplaceDimShuffle{1,0}(InplaceDimShuffle{1,0}(x))]") dimshuffle_lift.optimize(g) self.assertTrue(str(g) == "[x]") # no need to check_stack_trace as graph is supposed to be empty def test_merge2(self): x, y, z = inputs() e = ds(ds(x, (1, 'x', 0)), (2, 0, 'x', 1)) g = FunctionGraph([x], [e]) self.assertTrue(str(g) == "[InplaceDimShuffle{2,0,x,1}(InplaceDimShuffle{1,x,0}(x))]", str(g)) dimshuffle_lift.optimize(g) self.assertTrue(str(g) == "[InplaceDimShuffle{0,1,x,x}(x)]", str(g)) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(g, ops_to_check='all')) def test_elim3(self): x, y, z = inputs() e = ds(ds(ds(x, (0, 'x', 1)), (2, 0, 'x', 1)), (1, 0)) g = FunctionGraph([x], [e]) self.assertTrue(str(g) == ("[InplaceDimShuffle{1,0}(InplaceDimShuffle{2,0,x,1}" "(InplaceDimShuffle{0,x,1}(x)))]"), str(g)) dimshuffle_lift.optimize(g) self.assertTrue(str(g) == "[x]", str(g)) # no need to check_stack_trace as graph is supposed to be empty def test_lift(self): x, y, z = inputs([False] * 1, [False] * 2, [False] * 3) e = x + y + z g = FunctionGraph([x, y, z], [e]) # It does not really matter if the DimShuffles are inplace # or not. init_str_g_inplace = ( "[Elemwise{add,no_inplace}(InplaceDimShuffle{x,0,1}" "(Elemwise{add,no_inplace}(InplaceDimShuffle{x,0}(x), y)), z)]") init_str_g_noinplace = ( "[Elemwise{add,no_inplace}(DimShuffle{x,0,1}" "(Elemwise{add,no_inplace}(DimShuffle{x,0}(x), y)), z)]") self.assertTrue(str(g) in (init_str_g_inplace, init_str_g_noinplace), str(g)) opt_str_g_inplace = ( "[Elemwise{add,no_inplace}(Elemwise{add,no_inplace}" "(InplaceDimShuffle{x,x,0}(x), InplaceDimShuffle{x,0,1}(y)), z)]") opt_str_g_noinplace = ( "[Elemwise{add,no_inplace}(Elemwise{add,no_inplace}" "(DimShuffle{x,x,0}(x), DimShuffle{x,0,1}(y)), z)]") dimshuffle_lift.optimize(g) self.assertTrue(str(g) in (opt_str_g_inplace, opt_str_g_noinplace), str(g)) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(g, ops_to_check='all')) def test_recursive_lift(self): v = T.vector(dtype="float64") m = T.matrix(dtype="float64") out = ((v + 42) * (m + 84)).T g = FunctionGraph([v, m], [out]) init_str_g = ("[InplaceDimShuffle{1,0}(Elemwise{mul,no_inplace}" "(InplaceDimShuffle{x,0}(Elemwise{add,no_inplace}" "(, " "InplaceDimShuffle{x}(TensorConstant{42}))), " "Elemwise{add,no_inplace}" "(, " "InplaceDimShuffle{x,x}(TensorConstant{84}))))]") self.assertTrue(str(g) == init_str_g) new_out = local_dimshuffle_lift.transform(g.outputs[0].owner)[0] new_g = FunctionGraph(g.inputs, [new_out]) opt_str_g = ("[Elemwise{mul,no_inplace}(Elemwise{add,no_inplace}" "(InplaceDimShuffle{0,x}(), " "InplaceDimShuffle{x,x}(TensorConstant{42})), " "Elemwise{add,no_inplace}(InplaceDimShuffle{1,0}" "(), " "InplaceDimShuffle{x,x}(TensorConstant{84})))]") self.assertTrue(str(new_g) == opt_str_g) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(new_g, ops_to_check='all')) def test_useless_dimshuffle(self): x, _, _ = inputs() e = ds(x, (0, 1)) g = FunctionGraph([x], [e]) self.assertTrue(str(g) == "[InplaceDimShuffle{0,1}(x)]") dimshuffle_lift.optimize(g) self.assertTrue(str(g) == "[x]") # Check stacktrace was copied over correctly after opt was applied self.assertTrue(hasattr(g.outputs[0].tag, 'trace')) def test_dimshuffle_on_broadcastable(self): x, y, z = inputs([False, True], [True, False, True], [False, False, True]) u = tensor.constant(1) ds_x = ds(x, (0, 'x')) # useless ds_y = ds(y, (2, 1, 0)) # useless ds_z = ds(z, (2, 1, 0)) # usefull ds_u = ds(u, ('x')) # usefull g = FunctionGraph([x, y, z, u], [ds_x, ds_y, ds_z, ds_u]) self.assertTrue(str(g) == "[InplaceDimShuffle{0,x}(x), InplaceDimShuffle{2,1,0}(y), InplaceDimShuffle{2,1,0}(z), InplaceDimShuffle{x}(TensorConstant{1})]") dimshuffle_lift.optimize(g) self.assertTrue(str(g) == "[x, y, InplaceDimShuffle{2,1,0}(z), InplaceDimShuffle{x}(TensorConstant{1})]") # Check stacktrace was copied over correctly after opt was applied self.assertTrue(hasattr(g.outputs[0].tag, 'trace')) def test_local_useless_dimshuffle_in_reshape(): vector = TensorType(broadcastable=(False,), dtype='float64')('vector') mat = TensorType(broadcastable=(False, False), dtype='float64')('mat') row = TensorType(broadcastable=(True, False), dtype='float64')('row') col = TensorType(broadcastable=(False, True), dtype='float64')('col') reshape_dimshuffle_vector = tensor.reshape(vector.dimshuffle('x', 0), vector.shape) reshape_dimshuffle_mat = tensor.reshape(mat.dimshuffle('x', 0, 'x', 1), mat.shape) reshape_dimshuffle_row = tensor.reshape(row.dimshuffle(1, 'x'), row.shape) reshape_dimshuffle_col = tensor.reshape(col.dimshuffle(0), col.shape) g = FunctionGraph([vector, mat, row, col], [reshape_dimshuffle_vector, reshape_dimshuffle_mat, reshape_dimshuffle_row, reshape_dimshuffle_col]) print(str(g)) assert_true(str(g) == "[Reshape{1}(InplaceDimShuffle{x,0}(vector), Shape(vector)), " "Reshape{2}(InplaceDimShuffle{x,0,x,1}(mat), Shape(mat)), " "Reshape{2}(InplaceDimShuffle{1,x}(row), Shape(row)), " "Reshape{2}(InplaceDimShuffle{0}(col), Shape(col))]") useless_dimshuffle_in_reshape = out2in(local_useless_dimshuffle_in_reshape) useless_dimshuffle_in_reshape.optimize(g) assert_true(str(g) == "[Reshape{1}(vector, Shape(vector)), " "Reshape{2}(mat, Shape(mat)), " "Reshape{2}(row, Shape(row)), " "Reshape{2}(col, Shape(col))]") # Check stacktrace was copied over correctly after opt was applied assert_true(check_stack_trace(g, ops_to_check='all')) # Check that the optimization does not get applied when the order # of dimensions has changed. reshape_dimshuffle_mat2 = tensor.reshape(mat.dimshuffle('x', 1, 'x', 0), mat.shape) h = FunctionGraph([mat], [reshape_dimshuffle_mat2]) str_h = str(h) useless_dimshuffle_in_reshape.optimize(h) assert_true(str(h) == str_h) def test_add_canonizer_problem0(): n_segments = 10 label = lscalar('label') segment_labels = label + theano._asarray([0] * n_segments, dtype='int64') r = segment_labels * 5 f = function([label], r) f(3) class test_greedy_distribute(unittest.TestCase): def test_main(self): a, b, c, d, x, y, z = matrices('abcdxyz') # 1. ((a/x + b/y) * x * y) --> a*y + b*x e = (a / z + b / x) * x * z g = FunctionGraph([a, b, c, d, x, y, z], [e]) # print pprint(g.outputs[0]) mul_canonizer.optimize(g) gof.TopoOptimizer(gof.LocalOptGroup(local_greedy_distributor), order='out_to_in').optimize(g) # print pprint(g.outputs[0]) assert str(pprint(g.outputs[0])) == "((a * x) + (b * z))" # 2. ((a/x + b) * x) --> a + b*x e = (a / x + b) * x g = FunctionGraph([a, b, x], [e]) # print pprint(g.outputs[0]) mul_canonizer.optimize(g) gof.TopoOptimizer(gof.LocalOptGroup(local_greedy_distributor), order='out_to_in').optimize(g) # print pprint(g.outputs[0]) assert str(pprint(g.outputs[0])) == "(a + (b * x))" def test_kording_bug(self): x, y = vectors('xy') eps = scalar('eps') s = scalar('s') # r = theano.tensor.mul(theano.tensor.fill(x, 2.*a), x/a , (y+z) , a) # r = theano.tensor.mul((x/a+y) , a, z) r = tensor.mul(s - 1, eps + x / s, eps + y / s, s) f = function([s, eps, x, y], r ** 2) s_val = np.asarray(4, dtype=config.floatX) eps_val = np.asarray(1.e-6, dtype=config.floatX) x_val = np.asarray([1.5, 2], dtype=config.floatX) y_val = np.asarray([2.3, 3.1], dtype=config.floatX) r0 = f(s_val, eps_val, x_val, y_val) r1 = f(s_val, eps_val, x_val, y_val) r2 = f(s_val, eps_val, x_val, y_val) assert np.all(r0 == r1) assert np.all(r0 == r2) class test_canonize(unittest.TestCase): def test_muldiv(self): x, y, z = matrices('xyz') a, b, c, d = matrices('abcd') # e = (2.0 * x) / (2.0 * y) # e = (2.0 * x) / (4.0 * y) # e = x / (y / z) # e = (x * y) / x # e = (x / y) * (y / z) * (z / x) # e = (a / b) * (b / c) * (c / d) # e = (a * b) / (b * c) / (c * d) # e = 2 * x / 2 # e = x / y / x # e = (x / x) * (y / y) e = (-1 * x) / y / (-2 * z) g = FunctionGraph([x, y, z, a, b, c, d], [e]) print(pprint(g.outputs[0])) mul_canonizer.optimize(g) print(pprint(g.outputs[0])) def test_elemwise_multiple_inputs_optimisation(self): # verify that the Canonizer merge sequential Elemwise({mul,add}) part 1 # # This part are that case that is done, but don't include case # that are not implemented but are suposed to be. # # Test with and without DimShuffle shp = (5, 5) fx, fy, fz = fmatrices('xyz') dx, dy, dz = dmatrices('xyz') # fv = fvector('r').dimshuffle('x', 0) # dv = dvector('s').dimshuffle('x', 0) fxv = theano._asarray(np.random.rand(*shp), dtype='float32') fyv = theano._asarray(np.random.rand(*shp), dtype='float32') fzv = theano._asarray(np.random.rand(*shp), dtype='float32') # fvv = theano._asarray(np.random.rand(shp[0]), dtype='float32').reshape(1, shp[0]) # dxv = theano._asarray(np.random.rand(*shp), dtype='float64') # dyv = theano._asarray(np.random.rand(*shp), dtype='float64') # dzv = theano._asarray(np.random.rand(*shp), dtype='float64') # dvv = theano._asarray(np.random.rand(shp[0]), dtype='float64').reshape(1, shp[0]) cases = [ (fx + fy, (fx, fy), (fxv, fyv), 1, 'float32'), (fx * fy, (fx, fy), (fxv, fyv), 1, 'float32'), # (fx+fy+fz,(fx,fy,fz),(fxv,fyv,fzv),1,'float32'), # (dx+dy+dz,(dx,dy,dz),(dxv,dyv,dzv),1,'float64'), # (fx*fy*fz,(fx,fy,fz),(fxv,fyv,fzv),1,'float32'), # (dx*dy*dz,(dx,dy,dz),(dxv,dyv,dzv),1,'float64'), # (fx*fy*(fx+fy+fz),(fx,fy,fz),(fxv,fyv,fzv),2,'float32'), # (dx*dy*(dx+dy+dz),(dx,dy,dz),(dxv,dyv,dzv),2,'float64'), # (fx*fy*(fx+fy+dz),(fx,fy,dz),(dxv,dyv,dzv),2,'float64'), # check mixed type add # (dz*fy*(fx+fy),(fx,fy,dz),(dxv,dyv,dzv),2,'float64'), # check mixed type mul # check with dimshuffle of constant (fx + fy + fz + 2, (fx, fy, fz), (fxv, fyv, fzv), 1, {'custom': 'float32', 'numpy+floatX': config.floatX, 'numpy': 'float64'}), (fx * fy * fz * 2, (fx, fy, fz), (fxv, fyv, fzv), 1, {'custom': 'float32', 'numpy+floatX': config.floatX, 'numpy': 'float64'}), # (2+fx+fy+fz,(fx,fy,fz),(fxv,fyv,fzv),1,'float32'), # (2*fx*fy*fz,(fx,fy,fz),(fxv,fyv,fzv),1,'float32'), (2 + fx + fy + fz + 2, (fx, fy, fz), (fxv, fyv, fzv), 1, {'custom': 'float32', 'numpy+floatX': config.floatX, 'numpy': 'float64'}), (2 * fx * fy * fz * 2, (fx, fy, fz), (fxv, fyv, fzv), 1, {'custom': 'float32', 'numpy+floatX': config.floatX, 'numpy': 'float64'}), # (fx*fy*2*(fx+fy+fz),(fx,fy,fz),(fxv,fyv,fzv),2,'float32'), # (fx*fy*(2+fx+fy+fz),(fx,fy,fz),(fxv,fyv,fzv),2,'float32'), (fx * fy * 2 * (fx + fy + fz + 2), (fx, fy, fz), (fxv, fyv, fzv), 2, {'custom': 'float32', 'numpy+floatX': config.floatX, 'numpy': 'float64'}), # check with broadcast of row # (fx+fy+fz+fv,(fx,fy,fz,fv),(fxv,fyv,fzv,fvv),1,'float32'), # (fx*fy*fz*fv,(fx,fy,fz,fv),(fxv,fyv,fzv,fvv),1,'float32'), # (fv+fx+fy+fz,(fx,fy,fz,fv),(fxv,fyv,fzv,fvv),1,'float32'), # (fv*fx*fy*fz,(fx,fy,fz,fv),(fxv,fyv,fzv,fvv),1,'float32'), # (fx*fy*fv*(fx+fy+fz),(fx,fy,fz,fv),(fxv,fyv,fzv,fvv),2,'float32'), # (fx*fy*(fv+fx+fy+fz),(fx,fy,fz,fv),(fxv,fyv,fzv,fvv),2,'float32'), # (fx*fy*fv*(fv+fx+fy+fz),(fx,fy,fz,fv),(fxv,fyv,fzv,fvv),2,'float32'), # (dx+dy+dz+dv,(dx,dy,dz,dv),(dxv,dyv,dzv,dvv),1,'float64'), # (dx*dy*dz*dv,(dx,dy,dz,dv),(dxv,dyv,dzv,dvv),1,'float64'), # (dv+dx+dy+dz,(dx,dy,dz,dv),(dxv,dyv,dzv,dvv),1,'float64'), # (dv*dx*dy*dz,(dx,dy,dz,dv),(dxv,dyv,dzv,dvv),1,'float64'), # (dx*dy*dv*(dx+dy+dz),(dx,dy,dz,dv),(dxv,dyv,dzv,dvv),2,'float64'), # (dx*dy*(dv+dx+dy+dz),(dx,dy,dz,dv),(dxv,dyv,dzv,dvv),2,'float64'), # (dx*dy*dv*(dv+dx+dy+dz),(dx,dy,dz,dv),(dxv,dyv,dzv,dvv),2,'float64'), ] # [10:11] # print cases # We must be sure that the Canonizer is working, but that we don't have other # optimisation that could hide bug in the Canonizer as local_elemwise_fusion mode = compile.mode.get_default_mode() opt = gof.Query(["canonicalize"]) opt = opt.excluding('local_elemwise_fusion') mode = mode.__class__(linker=mode.linker, optimizer=opt) for id, [g, sym_inputs, val_inputs, nb_elemwise, out_dtype] in enumerate(cases): if isinstance(out_dtype, dict): out_dtype = out_dtype[config.cast_policy] f = compile.function(list(sym_inputs), g, # we need the optimisation enabled, debug do this. mode=mode) out = f(*val_inputs) assert(len(f.maker.fgraph.toposort()) == nb_elemwise) assert(out_dtype == out.dtype) def test_elemwise_multiple_inputs_optimisation2(self): # verify that the Canonizer merge sequential Elemwise({mul,add}) part 2. # This part are that case that should have been done, but that are not implemented. # Test with and without DimShuffle raise SkipTest("Current implementation of Canonizer does not " "implement all cases. Skip the corresponding test.") shp = (5, 5) fx, fy, fz = fmatrices('xyz') dx, dy, dz = dmatrices('xyz') fv = fvector('r').dimshuffle('x', 0) dv = dvector('s').dimshuffle('x', 0) fxv = theano._asarray(np.random.rand(*shp), dtype='float32') fyv = theano._asarray(np.random.rand(*shp), dtype='float32') fzv = theano._asarray(np.random.rand(*shp), dtype='float32') fvv = theano._asarray(np.random.rand(shp[0]), dtype='float32').reshape(1, shp[0]) dxv = theano._asarray(np.random.rand(*shp), dtype='float64') dyv = theano._asarray(np.random.rand(*shp), dtype='float64') dzv = theano._asarray(np.random.rand(*shp), dtype='float64') dvv = theano._asarray(np.random.rand(shp[0]), dtype='float64').reshape(1, shp[0]) cases = [ (fx + fy, (fx, fy), (fxv, fyv), 1, 'float32'), (fx * fy, (fx, fy), (fxv, fyv), 1, 'float32'), (fx + fy + fz, (fx, fy, fz), (fxv, fyv, fzv), 1, 'float32'), (dx + dy + dz, (dx, dy, dz), (dxv, dyv, dzv), 1, 'float64'), (fx * fy * fz, (fx, fy, fz), (fxv, fyv, fzv), 1, 'float32'), (dx * dy * dz, (dx, dy, dz), (dxv, dyv, dzv), 1, 'float64'), (fx * fy * (fx + fy + fz), (fx, fy, fz), (fxv, fyv, fzv), 2, 'float32'), (dx * dy * (dx + dy + dz), (dx, dy, dz), (dxv, dyv, dzv), 2, 'float64'), (fx * fy * (fx + fy + dz), (fx, fy, dz), (dxv, dyv, dzv), 2, 'float64'), # check mixed type add (dz * fy * (fx + fy), (fx, fy, dz), (dxv, dyv, dzv), 2, 'float64'), # check mixed type mul # check with dimshuffle of constant (fx + fy + fz + 2, (fx, fy, fz), (fxv, fyv, fzv), 1, 'float32'), (fx * fy * fz * 2, (fx, fy, fz), (fxv, fyv, fzv), 1, 'float32'), (2 + fx + fy + fz, (fx, fy, fz), (fxv, fyv, fzv), 1, 'float32'), (2 * fx * fy * fz, (fx, fy, fz), (fxv, fyv, fzv), 1, 'float32'), (2 + fx + fy + fz + 2, (fx, fy, fz), (fxv, fyv, fzv), 1, 'float32'), (2 * fx * fy * fz * 2, (fx, fy, fz), (fxv, fyv, fzv), 1, 'float32'), (fx * fy * 2 * (fx + fy + fz), (fx, fy, fz), (fxv, fyv, fzv), 2, 'float32'), (fx * fy * (2 + fx + fy + fz), (fx, fy, fz), (fxv, fyv, fzv), 2, 'float32'), (fx * fy * 2 * (fx + fy + fz + 2), (fx, fy, fz), (fxv, fyv, fzv), 2, 'float32'), # check with broadcast of row (fx + fy + fz + fv, (fx, fy, fz, fv), (fxv, fyv, fzv, fvv), 1, 'float32'), (fx * fy * fz * fv, (fx, fy, fz, fv), (fxv, fyv, fzv, fvv), 1, 'float32'), (fv + fx + fy + fz, (fx, fy, fz, fv), (fxv, fyv, fzv, fvv), 1, 'float32'), (fv * fx * fy * fz, (fx, fy, fz, fv), (fxv, fyv, fzv, fvv), 1, 'float32'), (fx * fy * fv * (fx + fy + fz), (fx, fy, fz, fv), (fxv, fyv, fzv, fvv), 2, 'float32'), (fx * fy * (fv + fx + fy + fz), (fx, fy, fz, fv), (fxv, fyv, fzv, fvv), 2, 'float32'), (fx * fy * fv * (fv + fx + fy + fz), (fx, fy, fz, fv), (fxv, fyv, fzv, fvv), 2, 'float32'), (dx + dy + dz + dv, (dx, dy, dz, dv), (dxv, dyv, dzv, dvv), 1, 'float64'), (dx * dy * dz * dv, (dx, dy, dz, dv), (dxv, dyv, dzv, dvv), 1, 'float64'), (dv + dx + dy + dz, (dx, dy, dz, dv), (dxv, dyv, dzv, dvv), 1, 'float64'), (dv * dx * dy * dz, (dx, dy, dz, dv), (dxv, dyv, dzv, dvv), 1, 'float64'), (dx * dy * dv * (dx + dy + dz), (dx, dy, dz, dv), (dxv, dyv, dzv, dvv), 2, 'float64'), (dx * dy * (dv + dx + dy + dz), (dx, dy, dz, dv), (dxv, dyv, dzv, dvv), 2, 'float64'), (dx * dy * dv * (dv + dx + dy + dz), (dx, dy, dz, dv), (dxv, dyv, dzv, dvv), 2, 'float64'), ] # [10:11] # print cases # We must be sure that the Canonizer is working, but that we don't have other # optimisation that could hide bug in the Canonizer as local_elemwise_fusion mode = compile.mode.get_default_mode() mode._optimizer = gof.Query(["canonicalize"]) mode._optimizer = mode._optimizer.excluding('local_elemwise_fusion') for id, [g, sym_inputs, val_inputs, nb_elemwise, out_dtype] in enumerate(cases): f = compile.function(list(sym_inputs), g, # we need the optimisation enabled, debug do this. mode=mode) out = f(*val_inputs) assert(len(f.maker.fgraph.toposort()) == nb_elemwise) assert(out_dtype == out.dtype) @attr('slow') def test_multiple_case(self): # test those case take from the comment in Canonizer # x / x -> 1 # (x * y) / x -> y # x / y / x -> 1 / y # x / y / z -> x / (y * z) # x / (y / z) -> (x * z) / y # (a / b) * (b / c) * (c / d) -> a / d # (2.0 * x) / (4.0 * y) -> (0.5 * x) / y # 2 * x / 2 -> x # with and without DimShuffle # TODO: with DimShuffle shp = (3, 3) fx, fy, fz, fw = fmatrices('xyzw') dx, dy, dz, dw = dmatrices('xyzw') fv = fvector('r').dimshuffle('x', 0) dv = dvector('s').dimshuffle('x', 0) fxv = theano._asarray(np.random.rand(*shp), dtype='float32') fyv = theano._asarray(np.random.rand(*shp), dtype='float32') fzv = theano._asarray(np.random.rand(*shp), dtype='float32') fwv = theano._asarray(np.random.rand(*shp), dtype='float32') fvv = theano._asarray(np.random.rand(shp[0]), dtype='float32').reshape(1, shp[0]) dxv = theano._asarray(np.random.rand(*shp), dtype='float64') dyv = theano._asarray(np.random.rand(*shp), dtype='float64') dzv = theano._asarray(np.random.rand(*shp), dtype='float64') dwv = theano._asarray(np.random.rand(*shp), dtype='float64') dvv = theano._asarray(np.random.rand(shp[0]), dtype='float64').reshape(1, shp[0]) # We must be sure that the Canonizer is working, but that we don't have other # optimisation that could hide bug in the Canonizer as local_elemwise_fusion mode = compile.mode.get_default_mode() opt = gof.Query(["canonicalize"]) opt = opt.including('ShapeOpt', 'local_fill_to_alloc') opt = opt.excluding( 'local_elemwise_fusion') mode = mode.__class__(linker=mode.linker, optimizer=opt) # test x / x -> 1 for id, (g, sym_inputs, val_inputs, out_dtype) in enumerate([ (fx / fx, [fx], [fxv], 'float32'), (dx / dx, [dx], [dxv], 'float64'), (fv / fv, [fv], [fvv], 'float32'), (dv / dv, [dv], [dvv], 'float64')]): f = compile.function(list(sym_inputs), g, mode=mode) out = f(*val_inputs) assert (out == np.ones(shp, dtype=out_dtype)).all() topo = f.maker.fgraph.toposort() if sym_inputs[0].broadcastable[0]: assert len(topo) == 2 assert isinstance(topo[0].op, Shape_i) assert isinstance(topo[1].op, tensor.Alloc) else: assert len(topo) == 3 assert isinstance(topo[0].op, Shape_i) assert isinstance(topo[1].op, Shape_i) assert isinstance(topo[2].op, tensor.Alloc) assert(out_dtype == out.dtype) # test (x * y) / x -> y for id, (g, sym_inputs, val_inputs, nb_elemwise, out_dtype) in enumerate([ ((dx * dy) / dx, [dx, dy], [dxv, dyv], 0, 'float64'), ((fx * fy) / fx, [fx, fy], [fxv, fyv], 0, 'float32'), ((dv * dy) / dv, [dv, dy], [dvv, dyv], 0, 'float64'), ((fv * fy) / fv, [fv, fy], [fvv, fyv], 0, 'float32'), # must broadcast as there is a dimshuffle in the computation ((dx * dv) / dx, [dx, dv], [dxv, dvv], 1, 'float64'), # topo: [Elemwise{second,no_inplace}(x, )] ((fx * fv) / fx, [fx, fv], [fxv, fvv], 1, 'float32') # topo: [Elemwise{second,no_inplace}(x, )] ]): f = compile.function(list(sym_inputs), g, mode=mode) out = f(*val_inputs) assert(out_dtype == out.dtype) utt.assert_allclose(out, val_inputs[1]) topo = f.maker.fgraph.toposort() if topo and not(len(topo) == 1 and topo[0].op == deep_copy_op): for node in topo[:-1]: assert isinstance(node.op, Shape_i) assert isinstance(topo[-1].op, tensor.Alloc) # test x / y / x -> 1 / y for id, (g, sym_inputs, val_inputs, nb_elemwise, out_dtype) in enumerate([ ((dx / dy) / dx, [dx, dy], [dxv, dyv], 1, 'float64'), ((fx / fy) / fx, [fx, fy], [fxv, fyv], 1, 'float32'), ((dv / dy) / dv, [dv, dy], [dvv, dyv], 1, 'float64'), ((fv / fy) / fv, [fv, fy], [fvv, fyv], 1, 'float32'), # must broadcast as their is a dimshuffle in the computation ((dx / dv) / dx, [dx, dv], [dxv, dvv], 1, 'float64'), # topo: [Shape_i, Shape_i, Elemwise{inv,no_inplace}(), Alloc] ((fx / fv) / fx, [fx, fv], [fxv, fvv], 1, 'float32'), # topo: [Shape_i, Shape_i, Elemwise{inv,no_inplace}(), Alloc] ]): f = compile.function(list(sym_inputs), g, mode=mode) out = f(*val_inputs) utt.assert_allclose(out, (1 / val_inputs[1])) topo = f.maker.fgraph.toposort() elem = [t for t in topo if isinstance(t.op, T.Elemwise)] assert len(elem) == nb_elemwise assert isinstance(elem[0].op, (T.Elemwise, )) assert isinstance(elem[0].op.scalar_op, ( theano.scalar.basic.Inv, theano.scalar.basic.TrueDiv)) assert(out_dtype == out.dtype) # test (a / b) * (b / c) * (c / d) -> a / d for id, (g, sym_inputs, val_inputs, out_dtype) in enumerate([ ((dx / dy) * (dy / dz) * (dz / dw), [dx, dy, dz, dw], [dxv, dyv, dzv, dwv], 'float64'), ((fx / fy) * (fy / fz) * (fz / fw), [fx, fy, fz, fw], [fxv, fyv, fzv, fwv], 'float32'), ((dv / dy) * (dy / dz) * (dz / dw), [dv, dy, dz, dw], [dvv, dyv, dzv, dwv], 'float64'), ((fv / fy) * (fy / fz) * (fz / fw), [fv, fy, fz, fw], [fvv, fyv, fzv, fwv], 'float32'), ((dx / dv) * (dv / dz) * (dz / dw), [dx, dv, dz, dw], [dxv, dvv, dzv, dwv], 'float64'), ((fx / fv) * (fv / fz) * (fz / fw), [fx, fv, fz, fw], [fxv, fvv, fzv, fwv], 'float32'), ((dx / dy) * (dy / dv) * (dv / dw), [dx, dy, dv, dw], [dxv, dyv, dvv, dwv], 'float64'), ((fx / fy) * (fy / fv) * (fv / fw), [fx, fy, fv, fw], [fxv, fyv, fvv, fwv], 'float32'), ((dx / dy) * (dy / dz) * (dz / dv), [dx, dy, dz, dv], [dxv, dyv, dzv, dvv], 'float64'), ((fx / fy) * (fy / fz) * (fz / fv), [fx, fy, fz, fv], [fxv, fyv, fzv, fvv], 'float32'), ]): f = compile.function(list(sym_inputs), g, mode=mode) out = f(*val_inputs) utt.assert_allclose(out, (val_inputs[0] / val_inputs[3])) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, (T.Elemwise, )) assert isinstance(topo[0].op.scalar_op, theano.scalar.basic.TrueDiv) assert len(topo[0].inputs) == 2 assert(out_dtype == out.dtype) # test (2.0 * x) / (4.0 * y) -> (0.5 * x) / y for id, (g, sym_inputs, val_inputs, out_dtype) in enumerate([ (((2.0 * dx) / (4.0 * dy)), [dx, dy], [dxv, dyv], 'float64'), (((2.0 * fx) / (4.0 * fy)), [fx, fy], [fxv, fyv], {'custom': 'float32', 'numpy+floatX': config.floatX, 'numpy': 'float64'}), (((2.0 * dv) / (4.0 * dy)), [dv, dy], [dvv, dyv], 'float64'), (((2.0 * fv) / (4.0 * fy)), [fv, fy], [fvv, fyv], {'custom': 'float32', 'numpy+floatX': config.floatX, 'numpy': 'float64'}), (((2.0 * dx) / (4.0 * dv)), [dx, dv], [dxv, dvv], 'float64'), (((2.0 * fx) / (4.0 * fv)), [fx, fv], [fxv, fvv], {'custom': 'float32', 'numpy+floatX': config.floatX, 'numpy': 'float64'}), ]): if isinstance(out_dtype, dict): out_dtype = out_dtype[config.cast_policy] f = compile.function(list(sym_inputs), g, mode=mode) out = f(*val_inputs) utt.assert_allclose(out, (0.5 * val_inputs[0] / val_inputs[1])) topo = f.maker.fgraph.toposort() assert len(topo) == 2 assert isinstance(topo[0].op, (T.Elemwise, )) assert isinstance(topo[0].op.scalar_op, theano.scalar.basic.Mul) assert len(topo[0].inputs) == 2 assert isinstance(topo[1].op, (T.Elemwise, )) assert isinstance(topo[1].op.scalar_op, theano.scalar.basic.TrueDiv) assert len(topo[1].inputs) == 2 assert(out_dtype == out.dtype) # test 2 * x / 2 -> x for id, (g, sym_inputs, val_inputs, out_dtype) in enumerate([ ((2 * dx) / 2, [dx], [dxv], 'float64'), ((2 * fx) / 2, [fx], [fxv], {'custom': 'float32', 'numpy+floatX': config.floatX, 'numpy': 'float64'}), ((2 * dv) / 2, [dv], [dvv], 'float64'), ((2 * fv) / 2, [fv], [fvv], {'custom': 'float32', 'numpy+floatX': config.floatX, 'numpy': 'float64'}), ]): if isinstance(out_dtype, dict): out_dtype = out_dtype[config.cast_policy] f = compile.function(list(sym_inputs), g, mode=mode) out = f(*val_inputs) utt.assert_allclose(out, val_inputs[0]) topo = f.maker.fgraph.toposort() assert len(topo) == 1 topo[0].op == deep_copy_op assert(out_dtype == out.dtype) # test x / abs(x) -> sign(x) for id, (g, sym_inputs, val_inputs, out_dtype) in enumerate([ (dx / abs(dx), [dx], [0.5 - dxv], 'float64'), (fx / abs(fx), [fx], [0.5 - fxv], 'float32'), (dx / abs(dx), [dx], [0.1 * dxv], 'float64'), (fx / abs(fx), [fx], [0.1 * fxv], 'float32'), (dv / abs(dv), [dv], [0.5 - dvv], 'float64'), (fv / abs(fv), [fv], [0.5 - fvv], 'float32'), ]): f = compile.function(list(sym_inputs), g, mode=mode) out = f(*val_inputs) assert np.all(np.isfinite(out)) utt.assert_allclose(out, np.sign(val_inputs[0])) assert(out_dtype == out.dtype) assert len(f.maker.fgraph.toposort()) == 1 # test (2*x) / (3*abs(x)) -> sign(x) for id, (g, sym_inputs, val_inputs, out_dtype) in enumerate([ ((2 * dx) / (3 * abs(dx)), [dx], [0.5 - dxv], 'float64'), ((2 * fx) / (3 * abs(fx)), [fx], [0.5 - fxv], {'custom': 'float32', 'numpy+floatX': config.floatX, 'numpy': 'float64'}), ((2 * dx) / (3 * abs(dx)), [dx], [0.1 * dxv], 'float64'), ((2 * fx) / (3 * abs(fx)), [fx], [0.1 * fxv], {'custom': 'float32', 'numpy+floatX': config.floatX, 'numpy': 'float64'}), ((2 * dv) / (3 * abs(dv)), [dv], [0.5 - dvv], 'float64'), ((2 * fv) / (3 * abs(fv)), [fv], [0.5 - fvv], {'custom': 'float32', 'numpy+floatX': config.floatX, 'numpy': 'float64'}), ]): if isinstance(out_dtype, dict): out_dtype = out_dtype[config.cast_policy] f = compile.function(list(sym_inputs), g, mode=mode) topo = f.maker.fgraph.toposort() out = f(*val_inputs) assert np.all(np.isfinite(out)) utt.assert_allclose(out, np.sign(val_inputs[0]) * 2 / 3) assert(out_dtype == out.dtype) def test_abs_mul_div(self): # test that if we have # 4 * x / abs(2*x) it get simplifier during canonicalisation. x = T.dscalar() # a = T.abs_(x) if theano.config.mode == 'FAST_COMPILE': mode = theano.compile.mode.get_mode('FAST_RUN').excluding( "local_elemwise_fusion") else: mode = theano.compile.mode.get_default_mode().excluding( "local_elemwise_fusion") f = theano.function([x], [(4 * x) / abs(2 * x)], mode=mode) print(f.maker.fgraph.toposort()) print() f(.1) f(-1) # some stabilization optimization make the output be finite instead of nan # debug_mode will raise an error when he see nan if not isinstance(mode, theano.compile.debugmode.DebugMode): assert np.isfinite(f(0)) assert len(f.maker.fgraph.toposort()) == 2 assert f.maker.fgraph.toposort()[0].op == T.sgn f = theano.function([x], [(4 * x) / abs(x / 2)], mode=mode) print(f.maker.fgraph.toposort()) print() f(.1) f(-1) # some stabilization optimization make the output be finite instead of nan # debug_mode will raise an error when he see nan if not isinstance(mode, theano.compile.debugmode.DebugMode): assert np.isfinite(f(0)) assert len(f.maker.fgraph.toposort()) == 2 assert f.maker.fgraph.toposort()[0].op == T.sgn def test_multiple_case_that_fail(self): raise SkipTest("Current implementation of Canonizer does not " "implement all cases. Skip the corresponding test.") shp = (4, 4) fx, fy, fz = fmatrices('xyz') dx, dy, dz = dmatrices('xyz') fxv = theano._asarray(np.random.rand(*shp), dtype='float32') fyv = theano._asarray(np.random.rand(*shp), dtype='float32') fzv = theano._asarray(np.random.rand(*shp), dtype='float32') dxv = theano._asarray(np.random.rand(*shp), dtype='float32') dyv = theano._asarray(np.random.rand(*shp), dtype='float32') dzv = theano._asarray(np.random.rand(*shp), dtype='float32') # fvv = theano._asarray(np.random.rand(shp[0]), dtype='float32').reshape(1, shp[0]) # We must be sure that the Canonizer is working, but that we don't have other # optimisation that could hide bug in the Canonizer as local_elemwise_fusion mode = compile.mode.get_default_mode() opt = gof.Query(["canonicalize"]) opt = opt.excluding('local_elemwise_fusion') mode = mode.__class__(linker=mode.linker, optimizer=opt) # test fail! # test x / y / z -> x / (y * z) for (g, sym_inputs, val_inputs, out_dtype) in [ ((dx / dy) / dz, [dx, dy, dz], [dxv, dyv, dzv], 'float64'), ((fx / fy) / fz, [fx, fy, fz], [fxv, fyv, fzv], 'float32') ]: f = compile.function(list(sym_inputs), g, mode=mode) out = f(*val_inputs) utt.assert_allclose(out, val_inputs[0] / val_inputs[1] / val_inputs[2]) topo = f.maker.fgraph.toposort() assert len(topo) == 2 assert isinstance(topo[0].op, (T.Elemwise, )) assert isinstance(topo[0].op.scalar_op, theano.scalar.basic.Inv) assert len(topo[0].inputs) == 1 assert(out_dtype == out.dtype) # test x / (y / z) -> (x * z) / y for (g, sym_inputs, val_inputs, out_dtype) in [ (dx / (dy / dz), [dx, dy, dz], [dxv, dyv, dzv], 'float64'), (fx / (fy / fz), [fx, fy, fz], [fxv, fyv, fzv], 'float32') ]: f = compile.function(list(sym_inputs), g, mode=mode) out = f(*val_inputs) utt.assert_allclose(out, val_inputs[0] / (val_inputs[1] / val_inputs[2])) topo = f.maker.fgraph.toposort() assert len(topo) == 2 assert isinstance(topo[0].op, (T.Elemwise, )) assert isinstance(topo[0].op.scalar_op, theano.scalar.basic.Inv) assert len(topo[0].inputs) == 1 assert(out_dtype == out.dtype) def test_dont_merge_if_multiple_client(self): # test those case take from the comment in Canonizer raise SkipTest("Not implemented") def test_canonicalize_nan(self): # Regression test for bug in canonicalization of NaN values. # This bug caused an infinite loop which was caught by the equilibrium # optimizer, resulting in an error log message. sio = StringIO() handler = logging.StreamHandler(sio) handler.setLevel(logging.ERROR) logging.getLogger('theano.gof.opt').addHandler(handler) try: x = vector() theano.function([x], x + np.nan) finally: logging.getLogger('theano.gof.opt').removeHandler(handler) # Ideally this test would only catch the maxed out equilibrium # optimizer error message, but to be safe in case this message # is modified in the future, we assert that there is no error # at all. assert not sio.getvalue() def test_local_merge_abs(): x, y, z = T.matrices('xyz') x_val = np.random.rand(5, 5).astype(config.floatX) y_val = np.random.rand(5, 5).astype(config.floatX) z_val = np.random.rand(5, 5).astype(config.floatX) mode = theano.config.mode if mode == "FAST_COMPILE": mode = "FAST_RUN" mode = theano.compile.mode.get_mode(mode).excluding( "local_elemwise_fusion") f = theano.function([y, z], (abs(y * z * -2)), mode=mode) f(y_val, z_val) assert isinstance(f.maker.fgraph.toposort()[1].op.scalar_op, scal.Abs) assert len(f.maker.fgraph.toposort()) == 2 f = theano.function([x, y], abs(x / y), mode=mode) f(x_val, y_val) assert isinstance(f.maker.fgraph.toposort()[1].op.scalar_op, scal.Abs) assert len(f.maker.fgraph.toposort()) == 2 def test_merge_abs_bugfix(): # Test crash in optimization reported by Jeremiah Lowin at # https://groups.google.com/d/topic/theano-users/TaXfqXP2Mj0/discussion input = T.matrix() # normalize on cols step1 = input / input.sum(0) # normalize on rows step2 = step1 / step1.sum(1) # get l1 norm l1_norm = T.abs_(step2).sum() theano.function([input], T.grad(l1_norm, input)) def test_mixeddiv(): # Test that int division is preserved i = iscalar() d = dscalar() assert 0 == function([i, d], d * (i // (i + 1)))(3, 1.0) def test_const_type_in_mul_canonizer(): input = dmatrix() w = dmatrix() visb = dvector() hidb = dvector() betas = dvector() a = dvector() def sigm(x): return 1. / (1 + tensor.exp(-x)) hid = sigm((tensor.dot(w, input) + hidb) * betas) vis_gauss1 = (tensor.dot(w.T, hid) + visb) * betas / (2 * a * a) vis_gauss2 = (tensor.dot(w.T, hid) + visb) * betas / (2. * a * a) f1 = function([input, w, visb, hidb, betas, a], vis_gauss1) f2 = function([input, w, visb, hidb, betas, a], vis_gauss2) ival = np.random.rand(5, 5) wval = np.random.rand(5, 5) visbval = np.random.rand(5) hidbval = np.random.rand(5) betaval = np.random.rand(5) aval = np.random.rand(5) utt.assert_allclose( f2(ival, wval, visbval, hidbval, betaval, aval), f1(ival, wval, visbval, hidbval, betaval, aval)) def test_cast_in_mul_canonizer(): x, y = tensor.vectors('xy') m = tensor.minimum(x, y) o = m.sum() go = tensor.fill(o, 1) e = tensor.eq(go, x) o1 = (1 - e) * go o2 = e * go mode = theano.compile.get_default_mode().excluding('fusion').including('fast_run') f = theano.function([x, y], [o1, o2], mode=mode) theano.printing.debugprint(f, print_type=True) nodes = f.maker.fgraph.apply_nodes assert len([n for n in nodes if isinstance(getattr(n.op, 'scalar_op', None), theano.scalar.Identity)]) == 0 assert len([n for n in nodes if isinstance(getattr(n.op, 'scalar_op'), theano.scalar.Cast)]) == 1 f([1], [1]) class test_fusion(unittest.TestCase): mode = copy.copy(compile.mode.get_default_mode()) _shared = staticmethod(shared) topo_exclude = () def do(self, mode, shared_fn, shp, nb_repeat=1, assert_len_topo=True, slice=None): """ param shared_fn: if None, will use compile.function verify that the elemwise fusion work Test with and without DimShuffle """ # TODO: disable the canonizer? def my_init(shp, dtype='float64', num=0): ret = np.zeros(shp, dtype=dtype) + num return ret fw, fx, fy, fz = [theano.tensor.tensor(dtype='float32', broadcastable=[False] * len(shp), name=n) for n in 'wxyz'] dw, dx, dy, dz = [theano.tensor.tensor(dtype='float64', broadcastable=[False] * len(shp), name=n) for n in 'wxyz'] ix, iy, iz = [theano.tensor.tensor(dtype='int32', broadcastable=[False] * len(shp), name=n) for n in 'xyz'] fv = fvector('v') fs = fscalar('s') fwv = my_init(shp, 'float32', 1) fxv = my_init(shp, 'float32', 2) fyv = my_init(shp, 'float32', 3) fzv = my_init(shp, 'float32', 4) fvv = theano._asarray(np.random.rand(shp[0]), dtype='float32') fsv = np.asarray(np.random.rand(), dtype='float32') dwv = my_init(shp, 'float64', 5) ixv = theano._asarray(my_init(shp, num=60), dtype='int32') iyv = theano._asarray(my_init(shp, num=70), dtype='int32') izv = theano._asarray(my_init(shp, num=70), dtype='int32') fwx = fw + fx ftanx = theano.tensor.tan(fx) cases = [ (fx + fy + fz, (fx, fy, fz), (fxv, fyv, fzv), 1, fxv + fyv + fzv, 'float32'), # 0 (fx * fy * fz, (fx, fy, fz), (fxv, fyv, fzv), 1, fxv * fyv * fzv, 'float32'), # 1 (fx + fy * fz, (fx, fy, fz), (fxv, fyv, fzv), 1, fxv + fyv * fzv, 'float32'), # 2 (fx * fy + fz, (fx, fy, fz), (fxv, fyv, fzv), 1, fxv * fyv + fzv, 'float32'), # 3 (fw + fx + fy + fz, (fw, fx, fy, fz), (fwv, fxv, fyv, fzv), 1, fwv + fxv + fyv + fzv, 'float32'), ((fw + fx) + (fy + fz), (fw, fx, fy, fz), (fwv, fxv, fyv, fzv), 1, fwv + fxv + fyv + fzv, 'float32'), # 5 (((fw + fx) + fy) + fz, (fw, fx, fy, fz), (fwv, fxv, fyv, fzv), 1, fwv + fxv + fyv + fzv, 'float32'), ((fw + (fx + fy)) + fz, (fw, fx, fy, fz), (fwv, fxv, fyv, fzv), 1, fwv + fxv + fyv + fzv, 'float32'), ((fw + (fx + fy) + fz), (fw, fx, fy, fz), (fwv, fxv, fyv, fzv), 1, fwv + fxv + fyv + fzv, 'float32'), (fw + (fx + (fy + fz)), (fw, fx, fy, fz), (fwv, fxv, fyv, fzv), 1, fwv + fxv + fyv + fzv, 'float32'), ((fw + fx) + (fy + fz), (fw, fx, fy, fz), (fwv, fxv, fyv, fzv), 1, fwv + fxv + fyv + fzv, 'float32'), # 10 (fw * fx * fy * fz, (fw, fx, fy, fz), (fwv, fxv, fyv, fzv), 1, fwv * fxv * fyv * fzv, 'float32'), (fw + fx * fy * fz, (fw, fx, fy, fz), (fwv, fxv, fyv, fzv), 1, fwv + fxv * fyv * fzv, 'float32'), (fx + fy * fz * fx, (fx, fy, fz), (fxv, fyv, fzv), 1, fxv + fyv * fzv * fxv, 'float32'), (fx * fy + fz + fy, (fx, fy, fz), (fxv, fyv, fzv), 1, fxv * fyv + fzv + fyv, 'float32'), (fx * fy * fz * fw + fx + fy + fz + fw, (fw, fx, fy, fz), (fwv, fxv, fyv, fzv), 1, fxv * fyv * fzv * fwv + fxv + fyv + fzv + fwv, 'float32'), # 15 # test with constant ((fw + fx) + (fy + fz) + 2., (fw, fx, fy, fz), (fwv, fxv, fyv, fzv), 1, fwv + fxv + fyv + fzv + 2, 'float32'), (((fw + fx) + 2. + fy) + fz, (fw, fx, fy, fz), (fwv, fxv, fyv, fzv), 1, fwv + fxv + fyv + fzv + 2, 'float32'), ((fw + (fx + 2. + fy)) + fz, (fw, fx, fy, fz), (fwv, fxv, fyv, fzv), 1, fwv + fxv + fyv + fzv + 2, 'float32'), ((fw + (fx + fy) + 2 + fz), (fw, fx, fy, fz), (fwv, fxv, fyv, fzv), 1, fwv + fxv + fyv + fzv + 2, 'float32'), (fw + (fx + (fy + fz) + 2.), (fw, fx, fy, fz), (fwv, fxv, fyv, fzv), 1, fwv + fxv + fyv + fzv + 2, 'float32'), # 20 (2 + (fw + fx) + (fy + fz), (fw, fx, fy, fz), (fwv, fxv, fyv, fzv), 1, fwv + fxv + fyv + fzv + 2, 'float32'), # mix float32 and float64 (2 + (dw + fx) + (fy + fz), (dw, fx, fy, fz), (dwv, fxv, fyv, fzv), 1, dwv + fxv + fyv + fzv + 2, 'float64'), (2 + (fw + dw) + (fy + fz), (fw, dw, fy, fz), (fwv, dwv, fyv, fzv), 1, fwv + dwv + fyv + fzv + 2, 'float64'), (2 + (fw + fx) + (dw + fz), (fw, fx, dw, fz), (fwv, fxv, dwv, fzv), 1, fwv + fxv + dwv + fzv + 2, 'float64'), (2 + (fw + fx) + (fy + dw), (fw, fx, fy, dw), (fwv, fxv, fyv, dwv), 1, fwv + fxv + fyv + dwv + 2, 'float64'), # 25 # test when their is other op then elemwise. ((fwx.sum()) + (fwx) + (fy + fz), (fw, fx, fy, fz), (fwv, fxv, fyv, fzv), 4, (fwv + fxv).sum() + fwv + fxv + fyv + fzv, 'float32'), # test other elemwise op (fx + fy + tensor.cos(fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv + fyv + np.cos(fzv), 'float32'), (fx + fy + tensor.cosh(fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv + fyv + np.cosh(fzv), 'float32'), (fx + fy + abs(fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv + fyv + np.absolute(fzv), 'float32'), (ix + iy + abs(iz), (ix, iy, iz), (ixv, iyv, izv), 1, ixv + iyv + np.absolute(izv), 'int32'), # 30 (fx + fy + theano.tensor.log(fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv + fyv + np.log(fzv), 'float32'), (fx + fy + theano.tensor.log2(fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv + fyv + np.log2(fzv), 'float32'), (fx + fy + theano.tensor.log10(fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv + fyv + np.log10(fzv), 'float32'), (fx + fy ** fz, (fx, fy, fz), (fxv, fyv, fzv), 1, fxv + fyv ** fzv, 'float32'), # pow (fx + fy + theano.tensor.exp(fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv + fyv + np.exp(fzv), 'float32'), # 35 (fx - fy - fz, (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - fyv - fzv, 'float32'), (fx - (fy / fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - (fyv / fzv), 'float32'), (fx - theano.tensor.true_div(fy, 2), (fx, fy), (fxv, fyv), 1, fxv - (fyv / 2), 'float32'), (fx - theano.tensor.true_div(fy, fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - (fyv / fzv), 'float32'), (fx - theano.tensor.int_div(ix * 100, iy * 1000), (fx, ix, iy), (fxv, ixv, iyv), 1, fxv - ((ixv * 100) // (iyv * 1000)), {'custom': 'float64', 'numpy + floatX': config.floatX, 'numpy': 'float64'}), # 40 (fx - (fy / 2), (fx, fy), (fxv, fyv), 1, fxv - (fyv / 2), 'float32'), (fx - (fy % fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - (fyv % fzv), 'float32'), (fx - (fy > fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - (fyv > fzv), 'float32'), (fx - (fy >= fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - (fyv >= fzv), 'float32'), (fx - (fy < fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - (fyv < fzv), 'float32'), # 45 (fx - (fy <= fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - (fyv <= fzv), 'float32'), (fx - T.eq(fy, fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - (fyv == fzv), 'float32'), (fx - T.neq(fy, fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - ( fyv != fzv), 'float32'), (fx - fy + tensor.tan(fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - fyv + np.tan(fzv), 'float32'), (fx - fy + tensor.tanh(fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - fyv + np.tanh(fzv), 'float32'), # 50 (fx - fy + tensor.sin(fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - fyv + np.sin(fzv), 'float32'), (fx - fy + tensor.sinh(fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - fyv + np.sinh(fzv), 'float32'), (fx - fy + theano.tensor.sqr(fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - fyv + (fzv * fzv), 'float32'), (fx - fy + theano.tensor.sqrt(fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - fyv + np.sqrt(fzv), 'float32'), (fx - fy + theano.tensor.inv(fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - fyv + (1 / fzv), 'float32'), # 55 (fx - fy + theano.tensor.neg(fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - fyv + (-fzv), 'float32'), (fx - fy + theano.tensor.round(fz), (fx, fy, fz), (fxv, fyv, fzv), 1, fxv - fyv + np.round(fzv), 'float32'), (ix - iy + theano.tensor.iround(fz), (ix, iy, fz), (ixv, iyv, fzv), 1, ixv - iyv + np.round(fzv), 'int64'), # Bit op (fx - theano.tensor.or_(iy, iz), (fx, iy, iz), (fxv, iyv, izv), 1, fxv - (iyv | izv), {'custom': 'float64', 'numpy + floatX': config.floatX, 'numpy': 'float64'}), (fx - theano.tensor.xor(iy, iz), (fx, iy, iz), (fxv, iyv, izv), 1, fxv - (iyv ^ izv), {'custom': 'float64', 'numpy + floatX': config.floatX, 'numpy': 'float64'}), # 60 (fx - theano.tensor.and_(iy, iz), (fx, iy, iz), (fxv, iyv, izv), 1, fxv - (iyv & izv), {'custom': 'float64', 'numpy + floatX': config.floatX, 'numpy': 'float64'}), (fx - theano.tensor.invert(iy), (fx, iy), (fxv, iyv), 1, fxv - (~iyv), {'custom': 'float64', 'numpy + floatX': config.floatX, 'numpy': 'float64'}), (fx - theano.tensor.cast(fy, dtype='float64'), (fx, fy), (fxv, fyv), 1, fxv - np.asarray(fyv, 'float64'), 'float64'), (theano.tensor.pow(fx * fy + fz, fx * fy), (fx, fy, fz), (fxv, fyv, fzv), 1, np.power(fxv * fyv + fzv, fxv * fyv), 'float32'), (fv + fy ** fz, (fv, fy, fz), (fvv, fyv, fzv), 2, fvv + fyv ** fzv, 'float32'), # fused with a dimshuffle #65 (fv - fy + tensor.tanh(fz), (fv, fy, fz), (fvv, fyv, fzv), 2, fvv - fyv + np.tanh(fzv), 'float32'), # fused with a dimshuffle # Cases where the same input is reused many times. (theano.tensor.mul(fx, fx, fx, fx), (fx,), (fxv,), 1, fxv * fxv * fxv * fxv, 'float32'), (theano.tensor.mul(fx, ftanx, ftanx), (fx,), (fxv,), 1, fxv * np.tan(fxv) * np.tan(fxv), 'float32'), (theano.tensor.mul(fx, ftanx, ftanx, fx), (fx,), (fxv,), 1, fxv * np.tan(fxv) * np.tan(fxv) * fxv, 'float32'), (theano.tensor.mul(ftanx, ftanx, fx + fy), (fx, fy), (fxv, fyv), 1, np.tan(fxv) * np.tan(fxv) * (fxv + fyv), 'float32'), # 70 # Cases with different broadcast pattern. They should not # be merged as this would duplicate computation # The graph should have 2 elemwise and 1 dimshuffle (fx * theano.tensor.sin(fs), (fx, fs), (fxv, fsv), 3, fxv * np.sin(fsv), 'float32'), ] if slice: cases = cases[slice] times = np.zeros(len(cases)) fail1 = [] fail2 = [] fail3 = [] fail4 = [] for id, [g, sym_inputs, val_inputs, nb_elemwise, answer, out_dtype] in enumerate(cases): if isinstance(out_dtype, dict): out_dtype = out_dtype[config.cast_policy] print("new cases", id) if shared_fn is None: f = compile.function(list(sym_inputs), g, mode=mode) for x in xrange(nb_repeat): out = f(*val_inputs) t1 = time.time() else: out = shared_fn(np.zeros(shp, dtype=out_dtype), 'out') assert out.dtype == g.dtype f = function(sym_inputs, [], updates=[(out, g)], mode=mode) t0 = time.time() for x in xrange(nb_repeat): f(*val_inputs) t1 = time.time() out = out.get_value() times[id] = t1 - t0 atol = 1e-8 if out_dtype == 'float32': atol = 1e-6 if not np.allclose(out, answer * nb_repeat, atol=atol): fail1.append(id) print(val_inputs) print(out) print(answer * nb_repeat) topo = f.maker.fgraph.toposort() topo_ = [n for n in topo if not isinstance(n.op, self.topo_exclude)] if assert_len_topo: if not len(topo_) == nb_elemwise: fail3.append((id, topo_, nb_elemwise)) if nb_elemwise == 1: # if no variable appears multiple times in the # input of g, # check that the number of input to the Composite # Elemwise is ok if len(set(g.owner.inputs)) == len(g.owner.inputs): expected_len_sym_inputs = np.sum( [not isinstance(x, theano.gof.Constant) for x in topo_[0].inputs]) assert expected_len_sym_inputs == len(sym_inputs) if not out_dtype == out.dtype: fail4.append((id, out_dtype, out.dtype)) failed = len(fail1 + fail2 + fail3 + fail4) print("Executed", len(cases), "cases", "failed", failed) if failed > 0: raise Exception("Failed %d cases" % failed, fail1, fail2, fail3, fail4) return times def test_elemwise_fusion(self): shp = (5, 5) mode = copy.copy(self.mode) # we need the optimisation enabled and the canonicalize. # the canonicalize is needed to merge multiplication/addition by constant. mode._optimizer = mode._optimizer.including( 'local_elemwise_fusion', 'composite_elemwise_fusion', 'canonicalize') self.do(mode, self._shared, shp) @attr('slow') def test_elemwise_fusion_4d(self): shp = (3, 3, 3, 3) mode = copy.copy(self.mode) # we need the optimisation enabled and the canonicalize. # the canonicalize is needed to merge multiplication/addition by constant. mode._optimizer = mode._optimizer.including( 'local_elemwise_fusion', 'composite_elemwise_fusion', 'canonicalize') self.do(mode, self._shared, shp, slice=slice(0, 1)) def test_fusion_35inputs(self): # Make sure a fused graph with more than 35 inputs does not segfault # or error. inpts = vectors(['i%i' % i for i in xrange(35)]) # Make an elemwise graph looking like: # sin(i34 + sin(i33 + sin(... i1 + sin(i0) ...))) out = tensor.sin(inpts[0]) for idx in xrange(1, 35): out = tensor.sin(inpts[idx] + out) f = function(inpts, out, mode=self.mode) # Test it on some dummy values f(*[list(range(i, 4 + i)) for i in xrange(35)]) def test_pickle_big_fusion(self): # In the past, pickle of Composite generated in that case # crashed with max recusion limit. So we where not able to # generate C code in that case. if not theano.config.cxx: raise SkipTest("no c compiler, so can't use big elemwise!") factors = [] sd = tensor.dscalar() means = tensor.dvector() cst_05 = theano.tensor.constant(.5) cst_m05 = theano.tensor.constant(-.5) cst_2 = theano.tensor.constant(2) cst_m2 = theano.tensor.constant(-2) ones = theano.tensor.constant(np.ones(10)) n = 85 if theano.config.mode in ["DebugMode", "DEBUG_MODE"]: n = 10 for i in xrange(n): f = (cst_m05 * sd ** cst_m2 * (ones - means[i]) ** cst_2 + cst_05 * tensor.log(cst_05 * (sd ** cst_m2) / np.pi)) factors.append(tensor.sum(f)) logp = tensor.add(*factors) vars = [sd, means] dlogp = function(vars, [theano.grad(logp, v) for v in vars]) dlogp(2, np.random.rand(n)) def speed_fusion(self, s=None): """ param type s: a slice object param s: a slice to apply to the case to execute. If None, exec all case. """ shp = (3000, 3000) shp = (1000, 1000) nb_repeat = 50 # linker=gof.CLinker # linker=gof.OpWiseCLinker mode1 = copy.copy(self.mode) mode1._optimizer = mode1._optimizer.including('local_elemwise_fusion') # TODO:clinker is much faster... but use to much memory # Possible cause: as their is do deletion of intermediate value when we don't keep the fct. # More plausible cause: we keep a link to the output data? # Follow up. Clinker do the same... second cause? mode2 = copy.copy(self.mode) mode2._optimizer = mode2._optimizer.excluding('local_elemwise_fusion') print("test with linker", str(mode1.linker)) times1 = self.do(mode1, self._shared, shp, nb_repeat=nb_repeat, assert_len_topo=False, slice=s) times2 = self.do(mode2, self._shared, shp, nb_repeat=nb_repeat, assert_len_topo=False, slice=s) print("times1 with local_elemwise_fusion") print(times1, times1.min(), times1.max(), times1.sum()) print("times2 without local_elemwise_fusion") print(times2, times2.min(), times2.max(), times2.sum()) d = times2 / times1 print("times2/times1") print(d) print("min", d.min(), "argmin", d.argmin(), "max", d.max(), "mean", d.mean(), "std", d.std()) def test_fusion_inplace(self): mode = copy.copy(self.mode) # we need the optimisation enabled and the canonicalize. # the canonicalize is needed to merge multiplication/addition by constant. mode._optimizer = mode._optimizer.including( 'local_elemwise_fusion', 'composite_elemwise_fusion', 'canonicalize', 'inplace') x, y, z = dmatrices('xyz') f = theano.function([x, y, z], tensor.dot(x, y) + x + y + z, mode=mode) topo = [n for n in f.maker.fgraph.toposort() if not isinstance(n.op, self.topo_exclude)] assert len(topo) == 2 assert topo[-1].op.inplace_pattern f(np.random.random((5, 5)), np.random.random((5, 5)), np.random.random((5, 5))) def speed_log_exp(self): s = slice(31, 36) print("time", self.do(self.mode, self._shared, shp=(1000, 1000), assert_len_topo=False, slice=s, nb_repeat=100)) class TimesN(theano.scalar.basic.UnaryScalarOp): """ Used in test TestCompositeCodegen Must be outside of the class, otherwise, the c cache code can't pickle this class and this cause stuff printing during test. """ def __eq__(self, other): return super(TimesN, self).__eq__(other) and self.n == other.n def __hash__(self): return super(TimesN, self).__hash__() ^ hash(self.n) def __init__(self, n, *args, **kwargs): self.n = n theano.scalar.basic.UnaryScalarOp.__init__(self, *args, **kwargs) def impl(self, x): return x * self.n def c_support_code_apply(self, node, nodename): n = str(self.n) return """ float %(nodename)s_timesn(float x) { return x * %(n)s; } """ % locals() def c_code(self, node, name, inputs, outputs, sub): (x,) = inputs (z,) = outputs return "%(z)s = %(name)s_timesn(%(x)s);" % locals() class TestCompositeCodegen(unittest.TestCase): """ Test The Composite Ops code generation in a case where there is multiple scalar ops with support code. """ def setUp(self): upgrade_to_float = theano.scalar.basic.upgrade_to_float self.scal_times_2 = TimesN(2, upgrade_to_float, name='times_2') self.times_2 = theano.tensor.elemwise.Elemwise( self.scal_times_2, name='times_2') self.scal_times_3 = TimesN(3, upgrade_to_float, name='times_3') self.times_3 = theano.tensor.elemwise.Elemwise( self.scal_times_3, name='times_3') self.x = fvector() def test_nested_composite(self): y = self.times_2(self.x) z = self.times_3(y) f = function([self.x], z) if config.mode != "FAST_COMPILE": assert len(f.maker.fgraph.toposort()) == 1 fval = f([1, 2, 3]) assert np.all(fval == [6, 12, 18]) def test_local_useless_composite(self): x = theano.scalar.float32() c = theano.scalar.Composite([x], [x + 1, x - 1]) X = theano.tensor.matrix() o = theano.tensor.Elemwise(scalar_op=c)(X) mode = theano.compile.mode.get_default_mode().including( 'local_useless_composite') f = theano.function([X], o[0], mode=mode) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert len(topo[0].outputs) == 1 utt.assert_allclose(f([[1.]]), [[2.]]) f = theano.function([X], o[1], mode=mode) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert len(topo[0].outputs) == 1 utt.assert_allclose(f([[1.]]), [[0.]]) @utt.assertFailure_fast def test_log1p(): m = theano.config.mode if m == 'FAST_COMPILE': m = 'FAST_RUN' m = compile.mode.get_mode(m) m = m.excluding('fusion') # check some basic cases x = dvector() f = function([x], T.log(1 + (x)), mode=m) assert [node.op for node in f.maker.fgraph.toposort()] == [T.log1p] f = function([x], T.log(1 + (-x)), mode=m) assert [node.op for node in f.maker.fgraph.toposort()] == [ T.neg, inplace.log1p_inplace] f = function([x], -T.log(1 + (-x)), mode=m) assert [node.op for node in f.maker.fgraph.toposort()] == [ T.neg, inplace.log1p_inplace, inplace.neg_inplace] # check trickier cases (and use different dtype) y = fmatrix() f = function([x, y], T.log(tensor.fill(y, 1) + (x)), mode=m) # the first three ops are Shape_i, Shape_i, and Dimshuffle topo = f.maker.fgraph.toposort() assert topo[-1].op == tensor.alloc assert T.log1p in [node.op for node in topo] f = function([x, y], T.log(0 + (x) + tensor.fill(y, 1.0)), mode=m) topo = f.maker.fgraph.toposort() assert topo[-1].op == tensor.alloc assert T.log1p in [node.op for node in topo] f = function([x, y], T.log(2 + (x) - tensor.fill(y, 1.0)), mode=m) topo = f.maker.fgraph.toposort() assert topo[-1].op == tensor.alloc assert T.log1p in [node.op for node in topo] f([1e-7, 10], [[0, 0], [0, 0]]) # debugmode will verify values # should work for int z = tensor.imatrix() f = function([z], T.log(1 + (z)), mode=m) assert [node.op for node in f.maker.fgraph.toposort()] == [T.log1p] def test_log_add(): m = theano.config.mode if m == 'FAST_COMPILE': m = 'FAST_RUN' m = compile.mode.get_mode(m) m = m.excluding('fusion') m = copy.copy(m) # No need to put them back as we have a new object m.check_isfinite = False # check some basic cases x = dvector() y = dvector() f = function([x, y], T.log(T.exp(x) + T.exp(y)), mode=m) f([10000], [10000]) # causes overflow if handled incorrectly assert np.isfinite(f([10000], [10000])) utt.assert_allclose(f([10000], [10000]), 10000 + np.log1p(1)) # test that it give the same result when it don't overflow f([10], [10]) # don't causes overflow utt.assert_allclose(f([10], [10]), 10 + np.log1p(1)) # test that it also works with more than two args, (this currently fails) x = dvector() y = dvector() f = function([x, y], T.log(T.exp(x) + T.exp(y) + T.exp(x - y) + T.exp( x + y)), mode=m) try: f([10000], [10000]) # causes overflow if handled incorrectly utt.assert_allclose(f([10000], [10000]), 20000) except utt.WrongValue: raise SkipTest("log(add(exp)) is not stabilized when adding " "more than 2 elements, see #623") # TODO: test that the optimization works in the presence of broadcasting. # TODO: (write and) test that the optimization works with Sum in addition to working with Add. def test_local_useless_slice(): # test a simple matrix x = tensor.matrix('x') mode_unopt = compile.get_default_mode().excluding("local_useless_slice", "local_mul_canonizer") mode_opt = compile.get_default_mode().including("local_useless_slice").excluding("local_mul_canonizer") # test with and without the useless slice o = 2 * x[0, :] f_unopt = theano.function([x], o, mode=mode_unopt) f_opt = theano.function([x], o, mode=mode_opt) test_inp = np.random.randint(-10, 10, (4, 4)).astype('float32') assert all(f_opt(test_inp) == f_unopt(test_inp)),\ "The optimization caused a mismatch in the result" # test to see if the slice is truely gone apply_node = f_opt.maker.fgraph.toposort()[0] subtens = apply_node.op assert not any(isinstance(idx, slice) for idx in subtens.idx_list), "Slice should be gone" # Now test that the stack trace is copied over properly, # before before and after optimization. assert check_stack_trace(f_unopt, ops_to_check='all') assert check_stack_trace(f_opt, ops_to_check='all') # test a 4d tensor z = tensor.tensor4('z') o2 = z[1, :, :, 1] o3 = z[0, :, :, :] f_opt_check = theano.function([z], o2, mode=mode_opt) f_opt_check_apply = theano.function([z], o3, mode=mode_opt) # The optimization shouldn't apply here apply_node = f_opt_check.maker.fgraph.toposort()[0] subtens = apply_node.op assert [isinstance(idx, slice) for idx in subtens.idx_list].count(True) == 2 # But it should here apply_node = f_opt_check_apply.maker.fgraph.toposort()[0] subtens = apply_node.op assert not any(isinstance(idx, slice) for idx in subtens.idx_list) # Finally, test that the stack trace is copied over properly, # before before and after optimization. assert check_stack_trace(f_opt_check, ops_to_check=Subtensor) assert check_stack_trace(f_opt_check_apply, ops_to_check=Subtensor) def test_local_useless_inc_subtensor(): x = tensor.matrix('x') y = tensor.matrix('y') mode = compile.get_default_mode().including("local_useless_inc_subtensor") for sub in [slice(None), slice(None, None, -1)]: o = tensor.set_subtensor(x[::, sub], y) f = theano.function([x, y], o, mode=mode) o_shape = tensor.set_subtensor(x[::, sub], tensor.specify_shape(y, x.shape)) f_shape = theano.function([x, y], o_shape, mode=mode) # Test with shape info topo = f_shape.maker.fgraph.toposort() assert not any(isinstance(n.op, tensor.IncSubtensor) for n in topo) out = f_shape([[2, 3]], [[3, 4]]) assert (out == np.asarray([[3, 4]])[::, sub]).all() # Test that without shape info, we don't apply the opt. topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, tensor.IncSubtensor) out = f([[2, 3]], [[3, 4]]) assert (out == np.asarray([[3, 4]])[::, sub]).all() # Test that we don't remove shape error try: f([[2, 3]], [[3, 4], [4, 5]]) assert False except (ValueError, AssertionError): pass # Test that we don't remove broadcastability out = f([[2, 3], [3, 4]], [[5, 6]]) assert (out == np.asarray([[5, 6], [5, 6]])[::, sub]).all() # Test that we do not optimize others strides even when sub and y # have same shapes sub = x[::, ::2] o_shape = tensor.set_subtensor(sub, tensor.specify_shape(y, sub.shape)) f_shape = theano.function([x, y], o_shape) topo = f_shape.maker.fgraph.toposort() # theano.printing.debugprint(f_shape) assert any(isinstance(n.op, tensor.IncSubtensor) for n in topo) out = f_shape([[2, 3, 6, 7]], [[8, 9]]) assert (out == np.asarray([[8, 3, 9, 7]])).all() def test_local_useless_subtensor(): x = tensor.matrix('x') # Test default for dims in [(slice(0, None), ), (slice(0, None), slice(0, None)), ]: f = function([x], tensor.exp(x).__getitem__(dims), mode=mode_opt) # theano.printing.debugprint(f) prog = f.maker.fgraph.toposort() assert prog[0].op == tensor.exp assert len(prog) == 1 f([[0, 1, 2], [3, 4, 5]]) # let debugmode test something x_c = tensor.specify_shape(x, (2, 3)) # Test constant for dims, res in [((slice(0, 2), ), True), ((slice(0, 2), slice(0, None)), True), ((slice(0, 2), slice(0, 3)), True), ((slice(0, None), slice(0, 3)), True), ((slice(0, 3), slice(0, 13)), True), ((slice(0, 3), slice(0, 2)), False), ((slice(0, 1), slice(0, None)), False), ((slice(0, 1), 1), False)]: f = function([x], tensor.exp(x_c).__getitem__(dims), mode=mode_opt) # theano.printing.debugprint(f) prog = f.maker.fgraph.toposort() if res: assert isinstance(prog[0].op, theano.tensor.SpecifyShape), dims assert prog[1].op == tensor.exp, (dims, prog) assert len(prog) == 2, dims else: assert any([isinstance(node.op, Subtensor) for node in prog]) f([[0, 1, 2], [3, 4, 5]]) # let debugmode test something # Test Variable for idx, (dims, res) in enumerate([ ((slice(0, x.shape[0]), ), True), ((slice(0, x.shape[1]), ), False), ((slice(0, x.shape[0]), slice(0, x.shape[1]), ), True), ((slice(0, x.shape[0]), slice(0, x.shape[0]), ), False), ((slice(0, x.shape[1]), slice(0, x.shape[0]), ), False), ((slice(0, x.shape[1]), slice(0, x.shape[1]), ), False), ((slice(0, x.shape[1]), 2), False), ((slice(0, x.shape[1]), slice(x.shape[0] - x.shape[0], x.shape[1]),), False), ((slice(0, T.scalar_from_tensor(x.shape[0])), ), True), ]): f = function([x], tensor.exp(x).__getitem__(dims), mode=mode_opt) # theano.printing.debugprint(f) prog = f.maker.fgraph.toposort() if res: assert prog[0].op == tensor.exp, dims assert len(prog) == 1, dims else: assert any([isinstance(node.op, Subtensor) for node in prog]) f([[0, 1, 2], [3, 4, 5]]) # let debugmode test something # Test mix Variable and Constant # Currently not supported for idx, (dims, res) in enumerate([ ((slice(0, x.shape[0]), slice(0, 3)), False), ((slice(0, 3), slice(0, x.shape[1])), False), ]): f = function([x], tensor.exp(x_c).__getitem__(dims), mode=mode_opt) # theano.printing.debugprint(f) prog = f.maker.fgraph.toposort() if res: assert prog[0].op == tensor.exp, dims assert len(prog) == 1, dims else: assert any([isinstance(node.op, Subtensor) for node in prog]) f([[0, 1, 2], [3, 4, 5]]) # let debugmode test something # Test scalar variable s = scal.int32('s') for idx, (dims, res) in enumerate([ ((slice(0, s), ), False), ]): f = function([x, s], tensor.exp(x).__getitem__(dims), mode=mode_opt) # theano.printing.debugprint(f) prog = f.maker.fgraph.toposort() if res: assert prog[0].op == tensor.exp, dims assert len(prog) == 1, dims else: assert any([isinstance(node.op, Subtensor) for node in prog]) f([[1, 2, 3], [4, 5, 6]], 1) f([[1, 2, 3], [4, 5, 6]], 3) # Test AdvancedSubtensor1 case when all rows are selected by a list/vector # or ARange op for dims, res in (([0, 1], True), ([1, 0], False), ([0, 0], False), ([0, 0, 1], False), (T.arange(2), True), (T.arange(0, 2), True), (T.arange(0, 2, 2), False), (T.arange(0, 2, -1), False), (T.arange(1, 2), False)): f = function([x], tensor.exp(x_c).__getitem__(dims), mode=mode_opt) # theano.printing.debugprint(f) prog = f.maker.fgraph.toposort() if res: assert isinstance(prog[0].op, theano.tensor.SpecifyShape), dims assert prog[1].op == tensor.exp, dims assert len(prog) == 2, dims else: assert any([isinstance(node.op, AdvancedSubtensor1) for node in prog]) f([[0, 1, 2], [3, 4, 5]]) # let debugmode test something def test_local_subtensor_remove_broadcastable_index(): # testing local_subtensor_remove_broadcastable_index optimization # # tests removing broadcastable dimensions with index 0 or -1, # otherwise the optimzation should not be applied mode = theano.compile.mode.get_default_mode() mode = mode.including("local_subtensor_remove_broadcastable_index") x = T.dmatrix('x') y1 = x.dimshuffle(0, 'x', 1) y2 = x.dimshuffle('x', 1, 0, 'x') y3 = x.dimshuffle('x', 1, 'x', 0, 'x') # testing for cases that the optimzation should be applied z1 = y1[:, 0, :] z2 = y1[:, -1, :] z3 = y2[0, :, :, -1] z4 = y2[0, :, :, 0] z5 = y2[-1, :, :, -1] z6 = y3[-1, :, 0, :, -1] z7 = y3[-1, :, -1, :, -1] z8 = y3[0, :, 0, :, 0] f = theano.function([x], [z1, z2, z3, z4, z5, z6, z7, z8], mode=mode) for elem in f.maker.fgraph.toposort(): assert type(elem.op) not in [Subtensor, AdvancedSubtensor, AdvancedSubtensor1, IncSubtensor, AdvancedIncSubtensor, AdvancedIncSubtensor1] rng = np.random.RandomState(seed=utt.fetch_seed()) xn = rng.rand(5, 5) f(xn) # testing for cases that the optimzation should not be applied # to verify that other subtensor usage are passed without errors w1 = y1[3, 0, :] w2 = y1[2:4, -1, :] w3 = y2[0, :, 4:, -1] w4 = y2[:, :, 0, -1] w5 = y2[0, 2:4, :, 0] w6 = y2[0, -1, :, -1] w7 = y2[-1, 4:, :, -1] w8 = y2[-1, :, :3, -1] w9 = y2[-1, :, -1, -1] w10 = y3[-1, 2, 0, :, -1] w11 = y3[-1, 0, -1, :, -1] w12 = y3[-1, :, -1, -1, -1] w13 = y3[0, 0, 0, :, 0] w14 = y3[-1, 2:4, 0, 1:5, -1] w15 = y3[-1, 0, -1, 0, -1] w16 = y3[0, 2, 0, 4, 0] w17 = y3[:, 0, :, 1] w18 = y3[0, :, :, 2] w19 = y3[:, 2, 0] w20 = y3[:, 3] f2 = theano.function([x], [w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15, w16, w17, w18, w19, w20], mode=mode) f2(xn) class Test_subtensor_inc_subtensor(unittest.TestCase): @classmethod def setUpClass(cls): cls.mode = theano.compile.mode.get_default_mode().including('local_subtensor_inc_subtensor') def test_basic(self): # basic test x = tensor.matrix('x') i = tensor.iscalar('i') v = tensor.vector('v') y = tensor.set_subtensor(x[i], v) z = y[i] f = theano.function([x, i, v], z, mode=self.mode) prog = f.maker.fgraph.toposort() assert len(prog) == 1 assert isinstance(prog[0].op, DeepCopyOp) # basic test, numerical check x_ = np.random.uniform(size=[3, 4]).astype(config.floatX) v_ = np.random.uniform(size=[4, ]).astype(config.floatX) i_ = 1 assert np.array_equal(f(x_, i_, v_), v_) def test_multiple_idx(self): # complicated test x = tensor.tensor4('x') i1 = tensor.iscalar('i1') i2 = tensor.iscalar('i2') i3 = tensor.iscalar('i3') i4 = tensor.iscalar('i4') v = tensor.tensor3('v') y = tensor.set_subtensor(x[i1, :i2, i3:, ::i4], v) z = y[i1, :i2, i3:, ::i4] f = theano.function([x, i1, i2, i3, i4, v], z, mode=self.mode) prog = f.maker.fgraph.toposort() assert len(prog) == 1 assert isinstance(prog[0].op, DeepCopyOp) # complicated test, numerical check x_ = np.random.uniform(size=[3, 4, 5, 6]).astype(config.floatX) v_ = np.random.uniform(size=[2, 2, 2]).astype(config.floatX) i1_, i2_, i3_, i4_ = 1, 2, 3, 4 assert np.array_equal(f(x_, i1_, i2_, i3_, i4_, v_), v_) def test_not_applied(self): # case not use this optimization x = tensor.tensor4('x') i1 = tensor.iscalar('i1') i2 = tensor.iscalar('i2') i3 = tensor.iscalar('i3') i4 = tensor.iscalar('i4') v = tensor.tensor3('v') y = tensor.set_subtensor(x[i1, :i2, i3:, ::i4], v) z = y[i1, :i3, i2:, ::i4] f = theano.function([x, i1, i2, i3, i4, v], z, mode=self.mode) prog = f.maker.fgraph.toposort() assert len(prog) != 1 assert any(isinstance(x.op, tensor.IncSubtensor) for x in prog) assert any(isinstance(x.op, tensor.Subtensor) for x in prog) # case not use this optimization, numerical check x_ = np.random.uniform(size=[3, 4, 5, 6]).astype(config.floatX) v_ = np.random.uniform(size=[2, 2, 2]).astype(config.floatX) i1_, i2_, i3_, i4_ = 1, 2, 3, 4 x_[i1_, :i2_, i3_:, ::i4_] = v_ assert np.array_equal(f(x_, i1_, i2_, i3_, i4_, v_), x_[i1_, :i3_, i2_:, ::i4_]) def test_fewer_dims(self): # case when v has fewer dimensions x = tensor.matrix('x') i1 = tensor.iscalar('i') i2 = tensor.iscalar('i') v = tensor.vector('v') y = tensor.set_subtensor(x[:i1, :i2], v) z = y[:i1, :i2] f = theano.function([x, i1, i2, v], z, mode=self.mode) prog = f.maker.fgraph.toposort() assert any(isinstance(x.op, tensor.Alloc) for x in prog) # case when v is broadcastable, numerical check x_ = np.random.uniform(size=[3, 4]).astype(config.floatX) v_ = np.random.uniform(size=[2, ]).astype(config.floatX) i1_, i2_ = 2, 2 x_[:i1_, :i2_] = v_ assert np.array_equal(f(x_, i1_, i2_, v_), x_[:i1_, :i2_]) def test_broadcasted(self): # case when v has the same number of dimensions, some broadcastable x = tensor.matrix('x') i1 = tensor.iscalar('i') i2 = tensor.iscalar('i') v = tensor.col('v') y = tensor.set_subtensor(x[:i1, :i2], v) z = y[:i1, :i2] f = theano.function([x, i1, i2, v], z, mode=self.mode) prog = f.maker.fgraph.toposort() assert any(isinstance(x.op, tensor.Alloc) for x in prog) # case when v is broadcastable, numerical check x_ = np.random.uniform(size=[3, 4]).astype(config.floatX) v_ = np.random.uniform(size=[2, 1]).astype(config.floatX) i1_, i2_ = 2, 2 x_[:i1_, :i2_] = v_ assert np.array_equal(f(x_, i1_, i2_, v_), x_[:i1_, :i2_]) def test_different_dtypes(self): # Case when the dtype differs x = tensor.bmatrix('x') i = tensor.iscalar('i') v = tensor.vector('v') y = tensor.set_subtensor(x[i], v) z = y[i] f = theano.function([x, i, v], z, mode=self.mode) prog = f.maker.fgraph.toposort() assert len(prog) == 1 assert prog[0].op == tensor.basic._convert_to_int8 # basic test, numerical check x_ = np.random.randint(12, size=[3, 4]).astype('int8') v_ = np.random.uniform(12, size=[4, ]).astype(config.floatX) i_ = 1 assert np.array_equal(f(x_, i_, v_), v_.astype('int8')) class test_local_subtensor_make_vector(unittest.TestCase): def test_scalar_idx(self): x, y, z = tensor.lscalars('xyz') v = make_vector(x, y, z) f = function([x, y, z], v[0], mode=mode_opt) prog = f.maker.fgraph.toposort() assert len(prog) == 1 assert isinstance(prog[0].op, theano.compile.ops.DeepCopyOp) assert f(0, 1, 2) == 0 def test_slice_idx_stop(self): x, y, z = tensor.lscalars('xyz') v = make_vector(x, y, z) f = function([x, y, z], v[:2], mode=mode_opt) prog = f.maker.fgraph.toposort() assert len(prog) == 1 assert isinstance(prog[0].op, MakeVector) assert len(prog[0].inputs) == 2 r = f(0, 1, 2) assert r[0] == 0 and r[1] == 1 def test_slice_idx_step(self): x, y, z = tensor.lscalars('xyz') v = make_vector(x, y, z) f = function([x, y, z], v[::2], mode=mode_opt) prog = f.maker.fgraph.toposort() assert len(prog) == 1 assert isinstance(prog[0].op, MakeVector) assert len(prog[0].inputs) == 2 r = f(0, 1, 2) assert r[0] == 0 and r[1] == 2 def test_AdvancedSubtensor1_idx(self): x, y, z = tensor.lscalars('xyz') v = make_vector(x, y, z) f = function([x, y, z], v[[0, 2]], mode=mode_opt) prog = f.maker.fgraph.toposort() assert len(prog) == 1 assert isinstance(prog[0].op, MakeVector) assert len(prog[0].inputs) == 2 r = f(0, 1, 2) assert r[0] == 0 and r[1] == 2 def test_stack_trace(self): x, y, z = tensor.lscalars('xyz') v = make_vector(x, y, z) mode = theano.compile.mode.get_default_mode().including( "local_subtensor_make_vector") # list of subtensor cases, where local_subtensor_make_vector # inserts a new MakeVector node v_subtensors = [v[:2], v[::2], v[[0, 2]]] for v_subtensor in v_subtensors: f = function([x, y, z], v_subtensor, mode=mode) self.assertTrue(check_stack_trace(f, ops_to_check='all')) class test_local_subtensor_lift(unittest.TestCase): def test0(self): # basic test that the Op works x = tensor.matrix('x') f = function([x], tensor.exp(x)[0], mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f, ops_to_check='all')) prog = f.maker.fgraph.toposort() assert isinstance(prog[0].op, tensor.Subtensor) # first subtensor assert prog[1].op == tensor.exp assert len(prog) == 2 f([[0, 1], [2, 3]]) # let debugmode test something def test0b(self): # as test0, but we reuse the output of the elemwise # So we should not lift the subtensor x = tensor.matrix('x') f = function([x], [tensor.exp(x)[0], tensor.exp(x)], mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f, ops_to_check=[ Subtensor, tensor.Elemwise])) prog = f.maker.fgraph.toposort() assert prog[0].op == tensor.exp assert isinstance(prog[1].op, tensor.Subtensor) # first subtensor assert isinstance(prog[2].op, DeepCopyOp) assert len(prog) == 3 f([[0, 1], [2, 3]]) # let debugmode test something def test1(self): # basic test that the optimization work with scalar broadcasted x = tensor.matrix('x') y = tensor.scalar('y') z = tensor.matrix('z') f = function([x, y, z], tensor.exp(x + y + z)[0], mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f, ops_to_check=[ Subtensor, tensor.DimShuffle])) prog = f.maker.fgraph.toposort() assert isinstance(prog[0].op, tensor.Subtensor) assert isinstance(prog[1].op, tensor.DimShuffle) assert isinstance(prog[2].op, tensor.Subtensor) assert isinstance(prog[3].op.scalar_op, theano.scalar. Composite) # Composite{add,add} assert len(prog) == 4 # let debugmode test something f([[0, 1], [2, 3]], 4, [[4, 5], [6, 7]]) def test2(self): # as 1, but take a slice x = tensor.matrix('x') y = tensor.scalar('y') z = tensor.matrix('z') f = function([x, y, z], tensor.exp(x + y + z)[0:2], mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f, ops_to_check=[ Subtensor, tensor.DimShuffle])) prog = f.maker.fgraph.toposort() assert isinstance(prog[0].op, tensor.Subtensor) assert isinstance(prog[1].op, tensor.DimShuffle) assert isinstance(prog[2].op, tensor.Subtensor) assert isinstance(prog[3].op.scalar_op, theano.scalar. Composite) # Composite{add,add} assert len(prog) == 4 # let debugmode test something f([[0, 1], [2, 3]], 4, [[4, 5], [6, 7]]) def test3(self): # basic test that the optimization does work with broadcasting # for unary elemwise. y = tensor.vector('y') f = function([y], tensor.exp(y.dimshuffle(0, 'x'))[0], mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f, ops_to_check='all')) prog = f.maker.fgraph.toposort() assert isinstance(prog[0].op, tensor.DimShuffle) assert isinstance(prog[1].op, tensor.Subtensor) assert prog[2].op == tensor.exp assert len(prog) == 3 f([4, 5]) # let debugmode test something @utt.assertFailure_fast def test4(self): # basic test that the optimization doesn't work with broadcasting # ... It *could* be extended to, # ... but right now it doesn't, so it shouldn't try. x = tensor.matrix('x') y = tensor.vector('y') f = function([x, y], tensor.exp(x + y)[0], mode=mode_opt) # Opt doesn't apply, so no need for check_stack_trace # self.assertTrue(check_stack_trace(f, ops_to_check='all')) prog = f.maker.fgraph.toposort() assert isinstance(prog[0].op, tensor.DimShuffle) assert prog[1].op == tensor.add assert isinstance(prog[2].op, tensor.Subtensor) # first subtensor assert prog[3].op == inplace.exp_inplace assert len(prog) == 4 f([[0, 1], [2, 3]], [4, 5]) # let debugmode test something def test5(self): # test that we don't lift when we reuse the output of the # elemwise for other computation. x = tensor.matrix('x') y = tensor.vector('y') f = function([x, y], [tensor.exp(x + y)[0], tensor.exp(x + y) + x], mode=mode_opt) # Opt doesn't apply, so no need for check_stack_trace # self.assertTrue(check_stack_trace(f, ops_to_check=Subtensor)) prog = f.maker.fgraph.toposort() assert isinstance(prog[0].op, tensor.DimShuffle) assert isinstance(prog[1].op.scalar_op, theano.scalar. Composite) # Composite{add,exp} assert prog[2].op == tensor.add or prog[3].op == tensor.add # first subtensor assert isinstance(prog[2].op, tensor.Subtensor) or isinstance(prog[3].op, tensor.Subtensor) assert len(prog) == 4 f([[0, 1], [2, 3]], [4, 5]) # let debugmode test something def test6(self): # basic test that the optimization works with a scalar as input, # and a scalar as output (no broadcasting of the scalar needed). # The optimization used to fail and display an ERROR message. x = tensor.vector('x') y = tensor.scalar('y') f = function([x, y], tensor.exp(x + y)[0], mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f, ops_to_check=Subtensor)) prog = f.maker.fgraph.toposort() assert isinstance(prog[0].op, tensor.Subtensor) # Composite{add,exp} assert isinstance(prog[1].op.scalar_op, theano.scalar.Composite) assert len(prog) == 2 f([1, 2, 3], 4) # let debugmode test something def test7(self): # Test that Subtensor(Rebroadcast(x)) gets optimized into # Rebroadcast(Subtensor(x)). # test basic case x = tensor.matrix('x') xval = np.random.rand(1, 10).astype(config.floatX) assert x.broadcastable == (False, False) newx = tensor.Rebroadcast((0, True), (1, False))(x) assert newx.broadcastable == (True, False) f1 = function([x], newx[:2, :5], mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f1, ops_to_check=[ Subtensor, tensor.Rebroadcast])) prog = f1.maker.fgraph.toposort() assert isinstance(prog[0].op, tensor.Subtensor) assert isinstance(prog[1].op, tensor.Rebroadcast) assert (f1(xval) == xval[:2, :5]).all() # corner case 1: rebroadcast changes dims which are dropped through subtensor y = tensor.tensor4('x') yval = np.random.rand(1, 10, 1, 3).astype(config.floatX) assert y.broadcastable == (False, False, False, False) newy = tensor.Rebroadcast((0, True), (2, True))(y) assert newy.broadcastable == (True, False, True, False) f2 = function([y], newy[:, 3, 0, :], mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f2, ops_to_check=[ Subtensor, tensor.Rebroadcast])) prog = f2.maker.fgraph.toposort() assert isinstance(prog[0].op, tensor.Subtensor) assert isinstance(prog[1].op, tensor.Rebroadcast) assert (f2(yval) == yval[:, 3, 0, :]).all() # corner case 2: subtensor idx_list is shorter than resulting broadcast pattern f3 = function([y], newy[:, 3, 0], mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f3, ops_to_check=[ Subtensor, tensor.Rebroadcast])) prog = f3.maker.fgraph.toposort() assert isinstance(prog[0].op, tensor.Subtensor) assert isinstance(prog[1].op, tensor.Rebroadcast) assert (f3(yval) == yval[:, 3, 0]).all() # corner case 3: subtensor idx_list is shorter than rebroadcast.axis z = tensor.tensor4('x') zval = np.random.rand(4, 10, 3, 1).astype(config.floatX) assert z.broadcastable == (False, False, False, False) newz = tensor.Rebroadcast((3, True))(z) assert newz.broadcastable == (False, False, False, True) f4 = function([z], newz[:, 3, 0], mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f4, ops_to_check=[ Subtensor, tensor.Rebroadcast])) prog = f4.maker.fgraph.toposort() assert isinstance(prog[0].op, tensor.Subtensor) assert isinstance(prog[1].op, tensor.Rebroadcast) assert (f4(zval) == zval[:, 3, 0]).all() class test_local_subtensor_merge(unittest.TestCase): def setUp(self): utt.seed_rng() self.x_shapes = [(2, 2), (5, 3), (4, 1), (1, 2), (0, 2), (2, 0), (1, 0), (0, 0)] self.rng = np.random.RandomState(seed=utt.fetch_seed()) def test_const(self): # var[const::][-1] -> var[-1] x = tensor.matrix('x') for idx in xrange(-7, 6): f = function([x], x[idx::][-1], mode=mode_opt) g = function([x], x[idx::][-1], mode=mode_opt.excluding( 'local_subtensor_merge')) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f, ops_to_check=Subtensor)) topo = f.maker.fgraph.toposort() assert len([t for t in topo if isinstance(t.op, tensor.Subtensor)]) == 1 assert isinstance(topo[-1].op, DeepCopyOp) for x_s in self.x_shapes: x_val = self.rng.uniform(size=x_s).astype(config.floatX) if idx < x_s[0] and x_s[0] > 0: # The first subtensor is non-empty, so it makes sense f(x_val) # let debugmode test something else: # A non-empty subtensor of an empty one should be # an IndexError self.assertRaises(IndexError, f, x_val) self.assertRaises(IndexError, g, x_val) def test_scalar(self): # var[int::][-1] -> var[-1] x = tensor.matrix('x') y = tensor.iscalar('y') f = function([x, y], x[y::][-1], mode=mode_opt) g = function([x, y], x[y::][-1], mode=mode_opt.excluding('local_subtensor_merge')) # theano.printing.debugprint(f, print_type=True) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f, ops_to_check=Subtensor)) topo = f.maker.fgraph.toposort() # print [t for t in topo if isinstance(t.op, tensor.Subtensor)] assert len([t for t in topo if isinstance(t.op, tensor.Subtensor)]) == 1 # print topo[-1].op assert isinstance(topo[-1].op, DeepCopyOp) for x_s in self.x_shapes: x_val = self.rng.uniform(size=x_s).astype(config.floatX) for idx in xrange(-9, 8): if (idx < x_s[0]) and (x_s[0] > 0): # The first subtensor is non-empty f(x_val, idx) # let debugmode test something else: self.assertRaises(IndexError, f, x_val, idx) self.assertRaises(IndexError, g, x_val, idx) @attr('slow') def test_const2(self): # var[::-1][const] -> var[-1] x = tensor.matrix('x') for idx in xrange(-8, 7): f = function([x], x[::-1][idx], mode=mode_opt) g = function([x], x[::-1][idx], mode=mode_opt.excluding('local_subtensor_merge')) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f, ops_to_check=Subtensor)) # theano.printing.debugprint(f, print_type=True) topo = f.maker.fgraph.toposort() # print [t for t in topo if isinstance(t.op, tensor.Subtensor)] assert len([t for t in topo if isinstance(t.op, tensor.Subtensor)]) == 1 # print topo[-1].op assert isinstance(topo[-1].op, DeepCopyOp) for x_s in self.x_shapes: x_val = self.rng.uniform(size=x_s).astype(config.floatX) if (idx < x_s[0]) and (idx >= -x_s[0]): # The first subtensor is non-empty, so it makes sense f(x_val) # let debugmode test something else: # A non-empty subtensor of an empty one should be # an IndexError self.assertRaises(IndexError, f, x_val) self.assertRaises(IndexError, g, x_val) def test_scalar2(self): # var[::-1][int] -> var[-1] x = tensor.matrix('x') y = tensor.iscalar('y') f = function([x, y], x[::-1][y], mode=mode_opt) g = function([x, y], x[::-1][y], mode=mode_opt.excluding('local_subtensor_merge')) # theano.printing.debugprint(f, print_type=True) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f, ops_to_check=Subtensor)) topo = f.maker.fgraph.toposort() # print [t for t in topo if isinstance(t.op, tensor.Subtensor)] assert len([t for t in topo if isinstance(t.op, tensor.Subtensor)]) == 1 # print topo[-1].op assert isinstance(topo[-1].op, DeepCopyOp) for x_s in self.x_shapes: x_val = self.rng.uniform(size=x_s).astype(config.floatX) for idx in xrange(-x_s[0], x_s[0]): f(x_val, idx) # let debugmode test something for idx in (list(range(x_s[0], 9)) + list(range(-9, -x_s[0]))): self.assertRaises(IndexError, f, x_val, idx) self.assertRaises(IndexError, g, x_val, idx) def test_const3(self): # var[::-1][:const] -> var[-1] x = tensor.matrix('x') for idx in xrange(-9, 8): f = function([x], x[::-1][:idx], mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f, ops_to_check=Subtensor)) # theano.printing.debugprint(f, print_type=True) topo = f.maker.fgraph.toposort() # print [t for t in topo if isinstance(t.op, tensor.Subtensor)] assert len([t for t in topo if isinstance(t.op, tensor.Subtensor)]) == 1 # print topo[-1].op assert isinstance(topo[-1].op, DeepCopyOp) for x_s in self.x_shapes: x_val = self.rng.uniform(size=x_s).astype(config.floatX) f(x_val) # let debugmode test something def test_scalar3(self): # var[::-1][:int] -> var[-1] x = tensor.matrix('x') y = tensor.iscalar('y') f = function([x, y], x[::-1][:y], mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f, ops_to_check=Subtensor)) # theano.printing.debugprint(f, print_type=True) topo = f.maker.fgraph.toposort() # print [t for t in topo if isinstance(t.op, tensor.Subtensor)] assert len([t for t in topo if isinstance(t.op, tensor.Subtensor)]) == 1 # print topo[-1].op assert isinstance(topo[-1].op, DeepCopyOp) for x_s in self.x_shapes: x_val = self.rng.uniform(size=x_s).astype(config.floatX) for idx in xrange(-7, 7): f(x_val, idx) # let debugmode test something def test_const4(self): # var[const1::][:const2] x = tensor.matrix('x') for idx1 in xrange(-7, 7): for idx2 in xrange(-7, 7): f = function([x], x[idx1:][:idx2], mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f, ops_to_check=Subtensor)) # theano.printing.debugprint(f, print_type=True) topo = f.maker.fgraph.toposort() # print [t for t in topo if isinstance(t.op, tensor.Subtensor)] assert len([t for t in topo if isinstance(t.op, tensor.Subtensor)]) == 1 # print topo[-1].op assert isinstance(topo[-1].op, DeepCopyOp) for x_s in self.x_shapes: x_val = self.rng.uniform(size=x_s).astype(config.floatX) f(x_val) # let debugmode test something def test_scalar4(self): # var[int1:][:int2] x = tensor.matrix('x') y = tensor.iscalar('y') z = tensor.iscalar('y') f = function([x, y, z], x[y:][:z], mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f, ops_to_check=Subtensor)) # theano.printing.debugprint(f, print_type=True) topo = f.maker.fgraph.toposort() # print [t for t in topo if isinstance(t.op, tensor.Subtensor)] assert len([t for t in topo if isinstance(t.op, tensor.Subtensor)]) == 1 # print topo[-1].op assert isinstance(topo[-1].op, DeepCopyOp) for x_s in self.x_shapes: x_val = self.rng.uniform(size=x_s).astype(config.floatX) for idx1 in xrange(-11, 11): for idx2 in xrange(-11, 11): f(x_val, idx1, idx2) # let debugmode test something def test_const_general(self): # Some cases of merge: shape, (start, stop, step) of first, # (start, stop, step) of second subtensor cases = [ ((2, 3), (None, None, None), (None, None, -1)), ((12, 1), (None, None, -4), (None, None, 1)), ((5, 3), (1, 4, 2), (None, None, -1)), ] x = tensor.matrix('x') for shape, sl1, sl2 in cases: z = x[slice(*sl1)][slice(*sl2)] f = function([x], z, mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f, ops_to_check=Subtensor)) x_val = self.rng.uniform(size=shape).astype(config.floatX) f(x_val) def test_scalar5(self): # General case with two real slices # var[b1:e1:s1][b2:e2:s2] x = tensor.matrix('x') b1 = tensor.iscalar('b1') e1 = tensor.iscalar('e1') s1 = tensor.iscalar('s1') b2 = tensor.iscalar('b2') e2 = tensor.iscalar('e2') s2 = tensor.iscalar('s2') f = function([x, b1, e1, s1, b2, e2, s2], x[b1:e1:s1][b2:e2:s2], mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f, ops_to_check=Subtensor)) # theano.printing.debugprint(f, print_type=True) topo = f.maker.fgraph.toposort() # print [t for t in topo if isinstance(t.op, tensor.Subtensor)] assert len([t for t in topo if isinstance(t.op, tensor. Subtensor)]) == 1 # print topo[-1].op assert isinstance(topo[-1].op, DeepCopyOp) b1r = self.rng.permutation(list(range(-8, 8)))[:2] e1r = self.rng.permutation(list(range(-8, 8)))[:2] b2r = self.rng.permutation(list(range(-8, 8)))[:2] e2r = self.rng.permutation(list(range(-8, 8)))[:2] s1r = self.rng.permutation([-7, -6, -5, -4, -3, -2, -1, 1, 2, 3, 4, 5, 6, 7])[:2] s2r = self.rng.permutation([-7, -6, -5, -4, -3, -2, -1, 1, 2, 3, 4, 5, 6, 7])[:2] for x_s in self.x_shapes: x_val = self.rng.uniform(size=x_s).astype(config.floatX) for b1 in b1r: for e1 in e1r: for s1 in s1r: for b2 in b2r: for e2 in e2r: for s2 in s2r: f(x_val, b1, e1, s1, b2, e2, s2) def test_const5(self): # Bug reported by Razvan data = np.asarray(np.arange(8), dtype=theano.config.floatX) x = theano.tensor.vector('x') y = x[7:1:-1] t = theano.shared(np.int64(0)) fun = theano.function([x], y[t]) val = fun(data) assert val == data[7:1:-1][0] def test_const6(self): # Bug reported by Graham data = self.rng.uniform(size=(8, 8, 8)).astype(theano.config.floatX) x = theano.tensor.tensor3('x') nops = 1 if theano.config.mode == "FAST_COMPILE": nops = 2 # test 1) y = x[3:6, 2:6, 1:7][1] fun = theano.function([x], y) val = fun(data) assert np.all(val == data[3:6, 2:6, 1:7][1]) assert len([n for n in fun.maker.fgraph.toposort() if isinstance(n.op, Subtensor)]) == nops # test 2) y = x[2, 3][1] fun = theano.function([x], y) val = fun(data) assert np.all(val == data[2, 3][1]) assert len([n for n in fun.maker.fgraph.toposort() if isinstance(n.op, Subtensor)]) == nops # test 3) y = x[3:6, 2, 1:7][1] fun = theano.function([x], y) val = fun(data) assert np.all(val == data[3:6, 2, 1:7][1]) assert len([n for n in fun.maker.fgraph.toposort() if isinstance(n.op, Subtensor)]) == nops def test_scalar6(self): # General case with one slice and one index # var[b:e:s][i] x = tensor.matrix('x') b = tensor.iscalar('b') e = tensor.iscalar('e') s = tensor.iscalar('s') i = tensor.iscalar('i') f = function([x, b, e, s, i], x[b:e:s][i], mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f, ops_to_check=Subtensor)) # theano.printing.debugprint(f, print_type=True) topo = f.maker.fgraph.toposort() # print [t for t in topo if isinstance(t.op, tensor.Subtensor)] assert len([t for t in topo if isinstance(t.op, tensor. Subtensor)]) == 1 # print topo[-1].op assert isinstance(topo[-1].op, DeepCopyOp) b_r = self.rng.permutation(list(range(-4, 4)))[:3] e_r = self.rng.permutation(list(range(-4, 4)))[:3] i_r = self.rng.permutation(list(range(-4, 4)))[:3] s_r = self.rng.permutation([-3, -2, -1, 1, 2, 3])[:3] for x_s in self.x_shapes: n_index_err = 0 n_ok = 0 x_val = self.rng.uniform(size=x_s).astype(config.floatX) for b_v in b_r: for e_v in e_r: for s_v in s_r: for i_v in i_r: # The index could be out of bounds # In that case, an Exception should be raised, # otherwise, we let DebugMode check f try: x_val[b_v:e_v:s_v][i_v] except IndexError: n_index_err += 1 self.assertRaises(IndexError, f, x_val, b_v, e_v, s_v, i_v) else: # Executed if the "try" clause did not raise # any exception n_ok += 1 f(x_val, b_v, e_v, s_v, i_v) # print 'shape: %s' % (x_s,) # print '%% OK: %f' % (float(n_ok) * 100 / (n_ok + n_index_err)) @attr('slow') def test_none_slice(self): # Test case of two slices, var[b1:e1:s1][b2:e2:s2] # where any of the b, e, and s can be None x = tensor.matrix('x') b1 = tensor.iscalar('b1') e1 = tensor.iscalar('e1') s1 = tensor.iscalar('s1') b2 = tensor.iscalar('b2') e2 = tensor.iscalar('e2') s2 = tensor.iscalar('s2') # Generate all possible lists of positions for None in those 6 slots # A 1 indicates None is present, 0 that there is a Theano scalar. none_positions = np.ndindex(2, 2, 2, 2, 2, 2) # Ranges to be used when not None b1r = self.rng.permutation(list(range(-4, 4)))[:] e1r = self.rng.permutation(list(range(-4, 4)))[:] b2r = self.rng.permutation(list(range(-4, 4)))[:] e2r = self.rng.permutation(list(range(-4, 4)))[:] s1r = self.rng.permutation([-4, -3, -2, -1, 1, 2, 3, 4])[:] s2r = self.rng.permutation([-4, -3, -2, -1, 1, 2, 3, 4])[:] scalar_vars = [b1, e1, s1, b2, e2, s2] scalar_ranges = [b1r, e1r, s1r, b2r, e2r, s2r] # For each case, we will build a graph, function, and list of values # Then, we test it on each input shape. for none_pos in none_positions: slice_inputs = [] input_vars = [] values = [] if sum(none_pos) == 0: # Those case are already tested in test_scalar4 continue for i, none_i in enumerate(none_pos): if none_i: slice_inputs.append(None) else: slice_inputs.append(scalar_vars[i]) input_vars.append(scalar_vars[i]) values.append(scalar_ranges[i]) slice1 = slice(*slice_inputs[:3]) slice2 = slice(*slice_inputs[3:]) sub_x = x[slice1][slice2] f = theano.function([x] + input_vars, sub_x, mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied # for some cases, the optimization may remove all Subtensors, # which is why we pass "bug_print='ignore'". self.assertTrue(check_stack_trace(f, ops_to_check=Subtensor, bug_print='ignore')) topo = f.maker.fgraph.toposort() # print [t for t in topo if isinstance(t.op, tensor.Subtensor)] assert len([t for t in topo if isinstance(t.op, tensor.Subtensor)]) <= 1 assert isinstance(topo[-1].op, DeepCopyOp) for x_s in self.x_shapes: x_val = self.rng.uniform(size=x_s).astype(config.floatX) for i_val in zip(*values): f(x_val, *i_val) def test_none_index(self): # Test the general case of indexing into a subvector, # like x[b:e:s][i], where any of b, e, and s can be None x = tensor.matrix('x') b = tensor.iscalar('b') e = tensor.iscalar('e') s = tensor.iscalar('s') i = tensor.iscalar('i') # Generate all possible lists of positions for None in those 6 slots # A 1 indicates None is present, 0 that there is a Theano scalar. # The last index (i) is never None none_positions = np.ndindex(2, 2, 2, 1) # Ranges to be used when not None b_r = self.rng.permutation(list(range(-4, 4)))[:] e_r = self.rng.permutation(list(range(-4, 4)))[:] i_r = self.rng.permutation(list(range(-4, 4)))[:] s_r = self.rng.permutation([-4, -3, -2, -1, 1, 2, 3, 4])[:] scalar_vars = [b, e, s, i] scalar_ranges = [b_r, e_r, s_r, i_r] # For each case, we will build a graph, function, and list of values # Then, we test it on each input shape. for none_pos in none_positions: slice_inputs = [] input_vars = [] values = [] if sum(none_pos) == 0: # Those case are already tested in test_scalar6 continue for j, none_j in enumerate(none_pos): if none_j: slice_inputs.append(None) else: slice_inputs.append(scalar_vars[j]) input_vars.append(scalar_vars[j]) values.append(scalar_ranges[j]) symbol_slice = slice(*slice_inputs[:3]) sub_x = x[symbol_slice][i] f = theano.function([x] + input_vars, sub_x, mode=mode_opt) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f, ops_to_check=Subtensor)) topo = f.maker.fgraph.toposort() # print [t for t in topo if isinstance(t.op, tensor.Subtensor)] assert len([t for t in topo if isinstance(t.op, tensor.Subtensor)]) <= 1 assert isinstance(topo[-1].op, DeepCopyOp) for x_s in self.x_shapes: x_val = self.rng.uniform(size=x_s).astype(config.floatX) for i_val in zip(*values): # The index could be out of bounds # In that case, an Exception should be raised, # otherwise, we let DebugMode check f # For that, we need to create a numerical slice. i_val_idx = 0 num_slice_inputs = [] for none_j in none_pos: if none_j: num_slice_inputs.append(None) else: num_slice_inputs.append(i_val[i_val_idx]) i_val_idx += 1 num_slice = slice(*num_slice_inputs[:3]) num_i = num_slice_inputs[3] try: x_val[num_slice][num_i] except IndexError: self.assertRaises(IndexError, f, x_val, *i_val) else: # Executed if the "try" clause did not raise # any exception f(x_val, *i_val) class test_local_adv_sub1_adv_inc_sub1(unittest.TestCase): def setUp(self): utt.seed_rng() mode = theano.compile.mode.get_default_mode() self.mode = mode.including("local_adv_sub1_adv_inc_sub1").excluding("fusion") self.mode_no_assert = self.mode.including("local_remove_all_assert") def test0(self): for dtype1, dtype2 in [("float32", "float32"), ("float32", "float64"), ("float64", "float32"), ("float64", "float64")]: x = tensor.matrix(dtype=dtype1) y = tensor.matrix(dtype=dtype2) idx = tensor.ivector() dx = np.random.rand(4, 5).astype(dtype1) dy = np.random.rand(2, 5).astype(dtype2) # Duplicate the last row of dy dy = np.vstack([dy, dy[-1:]]) # Use the same index twice, with the same corresponding value. # That makes set_subtensor well-defined, and tests # duplication for inc_subtensor. didx = np.asarray([1, 3, 3], "int32") # set_subtensor inc = tensor.set_subtensor(x[idx], y) o = inc[idx] f = theano.function([x, y, idx], o, self.mode_no_assert) res = f(dx, dy, didx) utt.assert_allclose(dy, res) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, (compile.DeepCopyOp, T.Elemwise)) # inc_subtensor(data[idx], y) inc = tensor.inc_subtensor(x[idx], y) o = inc[idx] f = theano.function([x, y, idx], o, self.mode_no_assert) res = f(dx, dy, didx) _dx = dx.copy() np.add.at(_dx, didx, dy) utt.assert_allclose(_dx[didx], res) topo = f.maker.fgraph.toposort() len(topo) == 2 # inc_subtensor(0[idx], y) inc = tensor.inc_subtensor(x.zeros_like()[idx], y) o = inc[idx] f = theano.function([x, y, idx], o, self.mode_no_assert) res = f(dx, dy, didx) utt.assert_allclose(np.vstack([dy[0], 2 * dy[1], 2 * dy[2]]), res) def test_assert(self): x = tensor.matrix("x") y = tensor.matrix("y") idx = tensor.ivector() dx = np.random.rand(4, 5).astype(config.floatX) dy = np.random.rand(2, 5).astype(config.floatX) # set_subtensor inc = tensor.set_subtensor(x[idx], y) o = inc[idx] f = theano.function([x, y, idx], o, self.mode) # test wrong index for i in [dx.shape[0], -dx.shape[0] - 1]: self.assertRaises((AssertionError, IndexError), f, dx, dy, [i, i]) # test wrong shape self.assertRaises((AssertionError, ValueError), f, dx, dy, [1]) def test_stack_trace(self): x = tensor.matrix("x") # test cases with y.dtype # - equal to x.dtype # - different from x.dtype (to trigger the cast in # local_adv_sub1_adv_inc_sub1) ys = [tensor.matrix("y"), tensor.dmatrix("y")] idx = tensor.ivector() # set_subtensor and then subtensor with both ys incs = [tensor.set_subtensor(x[idx], y) for y in ys] outs = [inc[idx] for inc in incs] for y, out in zip(ys, outs): f = theano.function([x, y, idx], out, self.mode) self.assertTrue(check_stack_trace( f, ops_to_check=(Assert, scal.Cast))) class Test_alloc_zero(unittest.TestCase): def setUp(self): mode = theano.compile.mode.get_default_mode() self.mode = mode.including("local_incsubtensor_of_zeros", "local_setsubtensor_of_constants", "local_0_dot_x") def test_setsubtensor_allocs0(self): x = tensor.matrix() y = tensor.matrix() x0 = tensor.zeros_like(x) y0 = tensor.zeros_like(y) z = tensor.set_subtensor(x0[:4], y0) f = theano.function([x, y], z, mode=self.mode) assert np.all([not isinstance(n.op, tensor.IncSubtensor) for n in f.maker.fgraph.toposort()]) def test_setsubtensor_allocs1(self): y = tensor.matrix() x0 = tensor.constant(np.asarray(np.zeros((4, 4)), dtype=config.floatX)) y0 = tensor.zeros_like(y) z = tensor.set_subtensor(x0[:4], y0) f = theano.function([y], z, mode=self.mode) assert np.all([not isinstance(n.op, tensor.IncSubtensor) for n in f.maker.fgraph.toposort()]) def test_setsubtensor_allocs1t(self): y = tensor.matrix() x0 = tensor.constant(np.asarray(np.zeros((4, 4)), dtype=config.floatX)) y0 = tensor.zeros_like(y) z = tensor.set_subtensor(x0[:4], y0.T) f = theano.function([y], z, mode=mode_opt) assert np.all([not isinstance(n.op, tensor.IncSubtensor) for n in f.maker.fgraph.toposort()]) def test_setsubtensor_allocs2(self): x = tensor.matrix() y0 = tensor.constant(np.asarray(np.zeros_like((4, 4)), dtype=config.floatX)) x0 = tensor.zeros_like(x) z = tensor.set_subtensor(x0[:4], y0) f = theano.function([x], z, mode=self.mode) assert np.all([not isinstance(n.op, tensor.IncSubtensor) for n in f.maker.fgraph.toposort()]) def test_incsubtensor_allocs0(self): x = tensor.matrix() y = tensor.matrix() y0 = tensor.zeros_like(y) z = tensor.inc_subtensor(x[:4], y0) f = theano.function([x, y], z, mode=self.mode) assert np.all([not isinstance(n.op, tensor.IncSubtensor) for n in f.maker.fgraph.toposort()]) def test_incsubtensor_allocs0t(self): x = tensor.matrix() y = tensor.matrix() y0 = tensor.zeros_like(y) z = tensor.inc_subtensor(x[:4], y0.T) f = theano.function([x, y], z, mode=mode_opt) assert np.all([not isinstance(n.op, tensor.IncSubtensor) for n in f.maker.fgraph.toposort()]) def test_incsubtensor_allocs1(self): x = tensor.matrix() y0 = tensor.constant(np.asarray(np.zeros_like((4, 4)), dtype=config.floatX)) z = tensor.inc_subtensor(x[:4], y0) f = theano.function([x], z, mode=self.mode) assert np.all([not isinstance(n.op, tensor.IncSubtensor) for n in f.maker.fgraph.toposort()]) def test_incsubtensor_x_zeros(self): x = tensor.constant(np.asarray(np.zeros((4, 4)), dtype=config.floatX)) y = tensor.matrix() z = tensor.inc_subtensor(x[:4], y) f = theano.function([y], z) inc_nodes = [n for n in f.maker.fgraph.toposort() if isinstance(n.op, tensor.IncSubtensor)] assert(len(inc_nodes) == 1) node_is_set_instead_of_inc = inc_nodes[0].op.set_instead_of_inc mode = theano.config.mode assert((mode != "FAST_COMPILE" and node_is_set_instead_of_inc) or (mode == "FAST_COMPILE" and not node_is_set_instead_of_inc)) test_X = np.random.random((4, 4)).astype(config.floatX) utt.assert_allclose(f(test_X), test_X) # also check the flag doesn't get set if first input is not zeros: not_all_zeros = np.zeros((4, 4)) not_all_zeros[1, 0] = 0.001 x = tensor.constant(np.asarray(not_all_zeros, dtype=config.floatX)) y = tensor.matrix() z = tensor.inc_subtensor(x[:4], y) f = theano.function([y], z) inc_nodes = [n for n in f.maker.fgraph.toposort() if isinstance(n.op, tensor.IncSubtensor)] assert(len(inc_nodes) == 1) assert(inc_nodes[0].op.set_instead_of_inc is False) test_X = np.random.random((4, 4)).astype(config.floatX) utt.assert_allclose(f(test_X), test_X + not_all_zeros) def test_advancedincsubtensor1_allocs0(self): x = tensor.matrix() y = tensor.matrix() y0 = tensor.zeros_like(y) z = tensor.inc_subtensor(x[[0, 1, 2, 3]], y0) f = theano.function([x, y], z, mode=self.mode) assert np.all([not isinstance(n.op, tensor.AdvancedIncSubtensor1) for n in f.maker.fgraph.toposort()]) def test_advancedincsubtensor1_allocs0t(self): x = tensor.matrix() y = tensor.matrix() y0 = tensor.zeros_like(y) z = tensor.inc_subtensor(x[[0, 1, 2, 3]], y0.T) f = theano.function([x, y], z, mode=mode_opt) assert np.all([not isinstance(n.op, tensor.AdvancedIncSubtensor1) for n in f.maker.fgraph.toposort()]) def test_advancedincsubtensor1_allocs1(self): x = tensor.matrix() y0 = tensor.constant(np.asarray(np.zeros_like((4, 4)), dtype=config.floatX)) z = tensor.inc_subtensor(x[[0, 1, 2, 3]], y0) f = theano.function([x], z, mode=self.mode) assert np.all([not isinstance(n.op, tensor.AdvancedIncSubtensor1) for n in f.maker.fgraph.toposort()]) def test_advancedincsubtensor_allocs0(self): x = tensor.matrix() y = tensor.matrix() y0 = tensor.zeros_like(y) z = tensor.inc_subtensor(x[[[0, 0], [1, 1]], [[0, 1], [0, 1]]], y0) f = theano.function([x, y], z, mode=self.mode) assert np.all([not isinstance(n.op, tensor.AdvancedIncSubtensor) for n in f.maker.fgraph.toposort()]) def test_advancedincsubtensor_allocs0t(self): x = tensor.matrix() y = tensor.matrix() y0 = tensor.zeros_like(y) z = tensor.inc_subtensor(x[[[0, 0], [1, 1]], [[0, 1], [0, 1]]], y0.T) f = theano.function([x, y], z, mode=mode_opt) assert np.all([not isinstance(n.op, tensor.AdvancedIncSubtensor) for n in f.maker.fgraph.toposort()]) def test_advancedincsubtensor_allocs1(self): x = tensor.matrix() y0 = tensor.constant(np.asarray(np.zeros_like((2, 2)), dtype=config.floatX)) z = tensor.inc_subtensor(x[[[0, 0], [1, 1]], [[0, 1], [0, 1]]], y0) f = theano.function([x], z, mode=self.mode) assert np.all([not isinstance(n.op, tensor.AdvancedIncSubtensor) for n in f.maker.fgraph.toposort()]) def test_dot_allocs_0(self): v1 = tensor.vector('v1') v2 = tensor.vector('v2') m1 = tensor.matrix('m1') m2 = tensor.matrix('m2') vv2 = np.asarray([0, 1], dtype=theano.config.floatX) vm2 = np.asarray([[1, 2], [4, 5]], dtype=theano.config.floatX) vv3 = np.asarray([0, 1, 2], dtype=theano.config.floatX) vm3 = np.asarray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=theano.config.floatX) for _e1 in [(v1, vv2, vv3), (m1, vm2, vm3)]: for _e2 in [(v2, vv2, vv3), (m2, vm2, vm3)]: for p in [0, 1]: if p == 0: e1 = tensor.zeros_like(_e1[0]) e2 = _e2[0] else: e1 = _e1[0] e2 = tensor.zeros_like(_e2[0]) o = tensor.dot(e1, e2) f = theano.function([_e1[0], _e2[0]], o, mode=self.mode) f(_e1[1], _e2[1]) f(_e1[2], _e2[2]) assert np.all([not isinstance(n.op, tensor.Dot) for n in f.maker.fgraph.toposort()]) # test that we don't remove shape errors self.assertRaises((ValueError, AssertionError), f, _e1[1], _e2[2]) self.assertRaises((ValueError, AssertionError), f, _e1[2], _e2[1]) def test_local_IncSubtensor_serialize(): d = np.random.normal(0, 0.01, size=(100, 100)) d = d.astype(theano.config.floatX) W = theano.shared(d, name='W') i = T.vector('i', dtype='int64') j = T.vector('j', dtype='int64') t = T.scalar('t') y = (W[i] + W[j] + W[1] + W[i, j]).sum() cost = T.sqr(t - y) dW = theano.grad(cost, W) mode = theano.compile.mode.get_default_mode().excluding('fusion') mode = mode.including("local_IncSubtensor_serialize") f = theano.function([i, j, t], updates=[(W, W - 0.01 * dW)], mode=mode) topo = f.maker.fgraph.toposort() adds = [n for n in topo if isinstance(n.op, T.Elemwise) and isinstance(n.op.scalar_op, theano.scalar.Add)] for a in adds: assert not any([inp.owner and isinstance(inp.owner.op, (tensor.IncSubtensor, tensor.AdvancedIncSubtensor, tensor.AdvancedIncSubtensor1)) for inp in a.inputs]) # Now test that the stack trace is copied over properly, # if we return the gradients. We need to use same mode as before. f = theano.function([i, j, t], dW, mode=mode) assert check_stack_trace(f, ops_to_check=[ tensor.IncSubtensor, tensor.AdvancedIncSubtensor, tensor.AdvancedIncSubtensor1]) def test_local_set_to_inc_subtensor(): v = theano.tensor.fmatrix() s = v[[2, 1]] g = s + 3 r = theano.tensor.set_subtensor(s, g) moder = compile.get_default_mode().excluding('local_set_to_inc_subtensor') modet = compile.get_default_mode().including('local_set_to_inc_subtensor') f1 = theano.function([v], r, mode=moder) f2 = theano.function([v], r, mode=modet) advi1 = [n for n in f1.maker.fgraph.toposort() if isinstance(n.op, tensor.AdvancedIncSubtensor1)] advi2 = [n for n in f2.maker.fgraph.toposort() if isinstance(n.op, tensor.AdvancedIncSubtensor1)] # We only have SetSubtensor in f1 assert all(n.op.set_instead_of_inc for n in advi1) # We don't have any SetSubtensor in f2 assert all(not n.op.set_instead_of_inc for n in advi2) val = np.random.randn(3, 2).astype('float32') r1 = f1(val) r2 = f2(val) utt.assert_allclose(r1, r2) # Finally, test that the stack trace is copied over properly, # before and after optimization. assert check_stack_trace(f1, ops_to_check=tensor.AdvancedIncSubtensor1) assert check_stack_trace(f2, ops_to_check='all') def test_local_subtensor_of_dot(): m1 = theano.tensor.matrix() m2 = theano.tensor.matrix() d1 = np.arange(6).reshape((3, 2)).astype(config.floatX) d2 = np.arange(8).reshape((2, 4)).astype(config.floatX) + 10 mode = compile.get_default_mode().including("local_subtensor_of_dot") def test_equality(a, b): return a.shape == b.shape and np.allclose(a, b) # [cst] f = theano.function([m1, m2], theano.dot(m1, m2)[1], mode=mode) topo = f.maker.fgraph.toposort() assert test_equality(f(d1, d2), np.dot(d1, d2)[1]) # DimShuffle happen in FAST_COMPILE assert isinstance(topo[-1].op, (T.blas_c.CGemv, T.blas.Gemv, T.DimShuffle)) # slice f = theano.function([m1, m2], theano.dot(m1, m2)[1:2], mode=mode) topo = f.maker.fgraph.toposort() assert test_equality(f(d1, d2), np.dot(d1, d2)[1:2]) assert isinstance(topo[-1].op, (T.blas.Dot22)) m1 = theano.tensor.tensor3() m2 = theano.tensor.tensor3() idx = theano.tensor.iscalar() d1 = np.arange(30).reshape(2, 5, 3).astype(config.floatX) d2 = np.arange(72).reshape(4, 3, 6).astype(config.floatX) + 100 f = theano.function([m1, m2, idx], theano.dot(m1, m2)[idx, 1:4, :, idx:], mode=mode) assert test_equality(f(d1, d2, 1), np.dot(d1, d2)[1, 1:4, :, 1:]) # if we return the gradients. We need to use same mode as before. assert check_stack_trace(f, ops_to_check='last') f = theano.function([m1, m2, idx], theano.dot(m1, m2)[1:4, :, idx:, idx], mode=mode) assert test_equality(f(d1, d2, 1), np.dot(d1, d2)[1:4, :, 1:, 1]) # Now test that the stack trace is copied over properly, # if we return the gradients. We need to use same mode as before. assert check_stack_trace(f, ops_to_check='last') class Test_local_elemwise_alloc(unittest.TestCase): dtype = config.floatX def setUp(self): self.fast_compile_mode = get_mode('FAST_COMPILE') self.fast_run_mode = get_mode('FAST_RUN') self.vec = T.vector('vec', dtype=self.dtype) self.mat = T.matrix('mat', dtype=self.dtype) self.tens = T.tensor3('tens', dtype=self.dtype) self.alloc_wo_dep = T.alloc(self.vec, 2, 2) self.alloc_wo_dep_broad = T.alloc(self.vec, 1, 2) self.alloc_w_dep = T.alloc(self.vec, *self.mat.shape) self.alloc_w_dep_broad = T.alloc(self.vec, 1, *self.mat.shape) self.alloc_w_dep_broad2 = T.alloc(self.vec, self.mat.shape[0], self.mat.shape[1], 1) self.alloc_w_dep_tens = T.alloc( self.vec, self.tens.shape[0], self.tens.shape[1] ) self.tv_wo_dep = T.alloc(self.vec, 5, 5) self.tm_wo_dep = T.alloc(self.mat, 5, 5, 5) self.s = T.iscalar('s') self.tv_w_dep = T.alloc(self.vec, self.s, self.s) self.tm_w_dep = T.alloc(self.mat, 5, 5, 5) self.row = theano.tensor.row(dtype=self.dtype) self.o = T.alloc(self.row, 5, 5) def _verify_alloc_count(self, f, count): assert( sum([isinstance(elem.op, T.Alloc) for elem in f.maker.fgraph.toposort() if elem.op is not None]) == count ) def _verify_assert_count(self, f, count): assert( sum([isinstance(elem.op, T.opt.Assert) for elem in f.maker.fgraph.toposort() if elem.op is not None]) == count ) def test_remove_alloc_wo_dimshuffle(self): # Exclude local_useless_alloc, since it does not introduce # assert in all the same cases. self.fast_run_mode = self.fast_run_mode.excluding( 'local_useless_alloc', 'local_canonicalize_alloc') # No optimization on alloc func = function( [self.vec, self.mat], self.alloc_wo_dep + self.mat, mode=self.fast_compile_mode ) self._verify_alloc_count(func, 1) self._verify_assert_count(func, 0) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(func, ops_to_check='all')) # Optimization on alloc with assert func = function( [self.vec, self.mat], self.alloc_wo_dep + self.mat, mode=self.fast_run_mode ) self._verify_alloc_count(func, 0) self._verify_assert_count(func, 1) # Optimization on alloc with assert and broadcast func = function( [self.vec, self.mat], self.alloc_wo_dep_broad + self.mat, mode=self.fast_run_mode ) self._verify_alloc_count(func, 0) self._verify_assert_count(func, 1) # No optimization on alloc without assert func = function( [self.vec, self.mat], self.alloc_w_dep + self.mat, mode=self.fast_compile_mode ) self._verify_alloc_count(func, 1) self._verify_assert_count(func, 0) # Optimization on alloc without assert func = function( [self.vec, self.mat], self.alloc_w_dep + self. mat, mode=self.fast_run_mode ) self._verify_alloc_count(func, 0) self._verify_assert_count(func, 0) # Optimization on alloc without assert and with broadcast func = function( [self.vec, self.mat], self.alloc_w_dep_broad + self. mat, mode=self.fast_run_mode ) self._verify_alloc_count(func, 0) self._verify_assert_count(func, 0) # Not optimized case on alloc and with broadcast func = function( [self.vec, self.mat], self.alloc_w_dep_broad2 + self. mat, mode=self.fast_run_mode ) self._verify_alloc_count(func, 1) self._verify_assert_count(func, 0) def test_remove_alloc_w_dimshuffle(self): # No optimization on dimshuffle with assert func = function( [self.vec, self.tens], self.alloc_wo_dep.dimshuffle(0, 1, 'x') + self.tens, mode=self.fast_compile_mode ) self._verify_alloc_count(func, 1) self._verify_assert_count(func, 0) # Optimization on dimshuffle with assert func = function( [self.vec, self.tens], self.alloc_wo_dep.dimshuffle(0, 1, 'x') + self.tens, mode=self.fast_run_mode ) self._verify_alloc_count(func, 0) self._verify_assert_count(func, 1) # No optimization on dimshuffle without assert func = function( [self.vec, self.tens], self.alloc_w_dep_tens.dimshuffle(0, 1, 'x') + self.tens, mode=self.fast_compile_mode ) self._verify_alloc_count(func, 1) self._verify_assert_count(func, 0) # Optimization on dimshuffle without assert func = function( [self.vec, self.tens], self.alloc_w_dep_tens.dimshuffle(0, 1, 'x') + self.tens, mode=self.fast_run_mode ) self._verify_alloc_count(func, 0) self._verify_assert_count(func, 0) def test_multi_input_single_alloc(self): # No optimization on dimshuffle with assert func = function( [self.vec, self.mat], self.tv_wo_dep + self.tm_wo_dep, mode=self.fast_compile_mode ) self._verify_alloc_count(func, 2) self._verify_assert_count(func, 0) # Optimization on dimshuffle with assert func = function( [self.vec, self.mat], self.tv_wo_dep + self.tm_wo_dep, mode=self.fast_run_mode ) self._verify_alloc_count(func, 1) self._verify_assert_count(func, 0) # No optimization on dimshuffle without assert func = function( [self.vec, self.mat, self.s], self.tv_w_dep + self.tm_w_dep, mode=self.fast_compile_mode ) self._verify_alloc_count(func, 2) self._verify_assert_count(func, 0) # Optimization on dimshuffle without assert func = function( [self.vec, self.mat, self.s], self.tv_w_dep + self.tm_w_dep, mode=self.fast_run_mode ) self._verify_alloc_count(func, 1) self._verify_assert_count(func, 1) def test_error(self): t3fft = theano.tensor.tensor(dtype=self.dtype, broadcastable=(False, False, True)) o = self.o.dimshuffle(0, 1, 'x') + t3fft func = function( [t3fft, self.row], o, mode=self.fast_run_mode ) self._verify_alloc_count(func, 0) self._verify_assert_count(func, 1) d = np.random.rand(5, 5, 1).astype(self.dtype) r = np.random.rand(1, 5).astype(self.dtype) func(d, r) def test_local_subtensor_of_alloc(): # DebugMode should detect if something goes wrong. # test shape combination of odd and event shape. for shape in [(3, 5), (4, 6), (3, 8), (4, 7), (1, 5), (5, 1)]: x = tensor.tensor(dtype=theano.config.floatX, broadcastable=(shape[0] == 1, shape[1] == 1)) xval = np.zeros(shape, dtype=config.floatX) yval = np.arange(shape[1], dtype=config.floatX) for y in [theano.shared(yval), tensor.constant([1.])]: # The rows of yx are copies of y yx = tensor.alloc(y, x.shape[0], x.shape[1]) # Slice of each row z_mat = yx[:, 3:] assert z_mat.ndim == 2 # Only one column z_vec = yx[:, 3] assert z_vec.ndim == 1 # results are vector slicess = [] if shape[0] != 1: slicess.append((2, slice(None))) if shape[1] != 1: slicess.append((slice(None), 3)) # results are matrix slicess += [ (slice(None), slice(3, None)), (slice(3, None), ), (slice(3, None), slice(3, None)), (slice(1, 3), slice(None, -1)), (slice(None, None, 2)), (slice(1, None, 2)), ] for slices in slicess: z = yx.__getitem__(slices) f = theano.function([x], z) if theano.config.mode != 'FAST_COMPILE': # Subtensor can be in the input of Alloc assert not isinstance(f.maker.fgraph.toposort()[-1].op, Subtensor) val = f(xval) assert xval.__getitem__(slices).shape == val.shape def test_local_fill_useless(): # Test opt local_fill_useless x = dvector() y = dvector() z = lvector() m = dmatrix() x_ = np.random.rand(5,) y_ = np.random.rand(5,) z_ = (np.random.rand(5,) * 5).astype("int64") m_ = np.random.rand(5, 5) # basic case f = function([x], T.fill(x, x) * 2, mode=mode_opt) assert [node.op for node in f.maker.fgraph.toposort()] == [T.mul] f(x_) # basic case f = function([x, y], T.second(y, x) * 2, mode=mode_opt) assert [node.op for node in f.maker.fgraph.toposort()] == [T.mul] f(x_, y_) # basic case f = function([x, y], T.fill(x, y) * 2, mode=mode_opt) assert [node.op for node in f.maker.fgraph.toposort()] == [T.mul] f(x_, y_) # now with different type(cast) f = function([x, z], T.fill(z, x) * 2, mode=mode_opt) assert [node.op for node in f.maker.fgraph.toposort()] == [T.mul] f(x_, z_) # now with different type(cast) f = function([x, z], T.fill(x, z) * 2, mode=mode_opt) assert [node.op for node in f.maker.fgraph.toposort()] == [T.mul] f(x_, z_) # now cutting out the input ?? f = function([x, y], T.fill(x, y) * 2, mode=mode_opt) assert [node.op for node in f.maker.fgraph.toposort()] == [T.mul] f(x_, y_) # Test with different number of dimensions # The fill is not useless, so it should stay f = function([m, x], T.fill(m, x) * 2, mode=mode_opt) ops = [node.op.__class__ for node in f.maker.fgraph.toposort()] assert T.Alloc in ops f(m_, x_) def test_local_elemwise_sub_zeros(): # Test opt local_elemwise_sub_zeros # We test separately for scalars, vectors and matrices scalar = T.scalar() vect = T.vector() mat = T.matrix() rng = np.random.RandomState(seed=utt.fetch_seed()) scalar_val = rng.rand(1).astype(config.floatX)[0] vect_val = rng.rand(5).astype(config.floatX) mat_val = rng.rand(3, 2).astype(config.floatX) mode = theano.compile.get_default_mode()\ .excluding('canonicalize', 'uncanonicalize', 'ShapeOpt', 'local_fill_to_alloc', 'local_elemwise_alloc')\ .including('local_elemwise_sub_zeros') # Test scalar minus scalar f = function([scalar], scalar - scalar, mode=mode) # Check optimized graph is correct assert isinstance(f.maker.fgraph.toposort()[0].op, T.Elemwise) assert isinstance(f.maker.fgraph.toposort()[0].op.scalar_op, theano.scalar.Second) assert isinstance(f.maker.fgraph.toposort()[0].inputs[1], T.TensorConstant) or\ isinstance(f.maker.fgraph.toposort()[0].inputs[1], T.TensorConstant) utt.assert_allclose(f(scalar_val), 0.0) # Check stack trace is copied over assert check_stack_trace(f, ops_to_check='all') # Test vector minus vector f = function([vect], vect - vect, mode=mode) # Check optimized graph is correct assert isinstance(f.maker.fgraph.toposort()[0].op, T.Elemwise) assert isinstance(f.maker.fgraph.toposort()[0].op.scalar_op, theano.scalar.Second) assert isinstance(f.maker.fgraph.toposort()[0].inputs[1], T.TensorConstant) or\ isinstance(f.maker.fgraph.toposort()[0].inputs[1], T.TensorConstant) utt.assert_allclose(f(vect_val), np.zeros(vect_val.shape)) # Check stack trace is copied over assert check_stack_trace(f, ops_to_check='all') # Test vector minus vector f = function([mat], mat - mat, mode=mode) # Check optimized graph is correct assert isinstance(f.maker.fgraph.toposort()[0].op, T.Elemwise) assert isinstance(f.maker.fgraph.toposort()[0].op.scalar_op, theano.scalar.Second) assert isinstance(f.maker.fgraph.toposort()[0].inputs[1], T.TensorConstant) or\ isinstance(f.maker.fgraph.toposort()[0].inputs[1], T.TensorConstant) utt.assert_allclose(f(mat_val), np.zeros(mat_val.shape)) # Check stack trace is copied over assert check_stack_trace(f, ops_to_check='all') class Test_local_useless_elemwise_comparison(unittest.TestCase): def setUp(self): self.rng = np.random.RandomState(utt.fetch_seed()) def test_local_useless_elemwise_comparison(self): # TODO: test each case individually. # The following case is what made me discover those cases. X = T.matrix('X') Y = T.vector('Y') X_sum, updates = theano.scan(fn=lambda x: x.sum(), outputs_info=None, sequences=[X], non_sequences=None) Z = X_sum + Y # theano.printing.debugprint(Z) # here is the output for the debug print: """ Elemwise{add,no_inplace} [id A] '' |for{cpu,scan_fn} [id B] '' | |Subtensor{int64} [id C] '' | | |Shape [id D] '' | | | |Subtensor{int64::} [id E] 'X[0:]' | | | |X [id F] | | | |Constant{0} [id G] | | |Constant{0} [id H] | |Subtensor{:int64:} [id I] '' | | |Subtensor{int64::} [id E] 'X[0:]' | | |ScalarFromTensor [id J] '' | | |Subtensor{int64} [id C] '' | |Subtensor{int64} [id C] '' |Y [id K] Inner graphs of the scan ops: for{cpu,scan_fn} [id B] '' >Sum{acc_dtype=float64} [id L] '' > |X[t] [id M] -> [id I] """ mode = theano.compile.get_default_mode().excluding('fusion') f = theano.function([X, Y], Z, mode=mode) f(self.rng.rand(2, 3).astype(config.floatX), self.rng.rand(2).astype(config.floatX)) # theano.printing.debugprint(f, print_type=True) # here is the output for the debug print: """ Elemwise{Add}[(0, 0)] [id A] '' 7 |for{cpu,scan_fn} [id B] '' 6 | |Shape_i{0} [id C] '' 0 | | |X [id D] | |Subtensor{int64:int64:int8} [id E] '' 5 | | |X [id D] | | |ScalarFromTensor [id F] '' 4 | | | |Elemwise{switch,no_inplace} [id G] '' 3 | | | |Elemwise{le,no_inplace} [id H] '' 2 | | | | |Shape_i{0} [id C] '' 0 | | | | |TensorConstant{0} [id I] | | | |TensorConstant{0} [id I] | | | |TensorConstant{0} [id J] | | |ScalarFromTensor [id K] '' 1 | | | |Shape_i{0} [id C] '' 0 | | |Constant{1} [id L] | |Shape_i{0} [id C] '' 0 |Y [id M] Inner graphs of the scan ops: for{cpu,scan_fn} [id B] '' >Sum{acc_dtype=float64} [id N] '' > |X[t] [id O] -> [id E] """ def assert_eqs_const(self, f, val, op=deep_copy_op): topo = f.maker.fgraph.toposort() elem = topo[0] assert len(topo) == 1, topo assert elem.op == op, elem.op if op == deep_copy_op: assert len(elem.inputs) == 1, elem.inputs assert isinstance(elem.inputs[0], T.TensorConstant), elem assert T.extract_constant(elem.inputs[0]) == val, val else: assert len(elem.inputs) == 2, elem.inputs assert isinstance(elem.inputs[0], T.TensorConstant), elem assert T.extract_constant(elem.inputs[0]) == val, val def assert_identity(self, f): topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert topo[0].op == deep_copy_op if f.outputs[0].variable.dtype == 'bool': x_vals = [0, 1] else: x_vals = [0, 1, 10] for x_val in x_vals: assert f(x_val) == x_val def test_inequality_with_self(self): x = T.scalar('x', dtype=config.floatX) mode = theano.compile.get_default_mode().including('local_useless_elemwise_comparison') f = theano.function([x], T.lt(x, x), mode=mode) self.assert_eqs_const(f, 0) f = theano.function([x], T.le(x, x), mode=mode) self.assert_eqs_const(f, 1) f = theano.function([x], T.gt(x, x), mode=mode) self.assert_eqs_const(f, 0) f = theano.function([x], T.ge(x, x), mode=mode) self.assert_eqs_const(f, 1) f = theano.function([x], T.minimum(x, x), mode=mode) self.assert_identity(f) f = theano.function([x], T.maximum(x, x), mode=mode) self.assert_identity(f) def test_shape_inequality_with_self(self): x = T.vector('x', dtype=config.floatX) mode = theano.compile.get_default_mode().including( 'local_useless_elemwise_comparison', 'local_shape_to_shape_i', 'local_track_shape_i', 'local_subtensor_make_vector') f = theano.function([x], T.lt(x.shape[0], 0), mode=mode) self.assert_eqs_const(f, 0) f = theano.function([x], T.ge(x.shape[0], 0), mode=mode) self.assert_eqs_const(f, 1) f = theano.function([x], T.maximum(x.shape[0], 0), mode=mode) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, Shape_i), topo[0].op x_val = np.ones(100, dtype=config.floatX) assert f(x_val) == x_val.shape[0] f = theano.function([x], T.maximum(0, x.shape[0]), mode=mode) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, Shape_i), topo[0].op x_val = np.ones(100, dtype=config.floatX) assert f(x_val) == x_val.shape[0] f = theano.function([x], T.minimum(x.shape[0], 0), mode=mode) self.assert_eqs_const(f, 0) assert f(x_val) == 0 f = theano.function([x], T.minimum(0, x.shape[0]), mode=mode) self.assert_eqs_const(f, 0) assert f(x_val) == 0 f = theano.function([x], T.minimum([0, 0], x.shape[0]), mode=mode) # This case isn't optimized. # self.assert_eqs_const(f, 0) utt.assert_allclose(f(x_val), [0, 0]) def test_shape_add_inequality(self): x = T.vector('x', dtype=config.floatX) mode = theano.compile.get_default_mode().including( 'local_useless_elemwise_comparison', 'local_shape_to_shape_i', 'local_track_shape_i', 'local_subtensor_make_vector') y = T.vector('y', dtype=config.floatX) f = theano.function([x, y], T.lt(x.shape[0] + y.shape[0], 0), mode=mode) self.assert_eqs_const(f, 0) f = theano.function([x, y], T.ge(x.shape[0] + y.shape[0], 0), mode=mode) self.assert_eqs_const(f, 1) def test_equality_shapes(self): # Test equality where one sides contain only shapes related # stuff. if theano.config.mode == "FAST_COMPILE": raise SkipTest("Skip opt test as the opt is disabled") x = T.vector('x', dtype=config.floatX) for g in [x.shape[0], Shape_i(0)(x)]: f = theano.function([x], T.eq(g, 0)) assert f([3, 3]) == 0 assert f([]) == 1 f = theano.function([x], T.eq(g, -1)) self.assert_eqs_const(f, 0) assert f([3, 3]) == 0 g = join(0, x.shape[0:], # todo test reshape, dimshuffle x.shape[0:1]) f = theano.function([x], T.eq(g, 0)) assert (f([3, 3]) == 0).all() assert (f([]) == 1).all() f = theano.function([x], T.eq(g, -1)) self.assert_eqs_const(f, 0, op=T.alloc) assert (f([3, 3]) == 0).all() def test_and(self): # bitwise "and" with 0 should give 0 for both bool and int # bitwise "and" with 1 should only simplify for bool mode = theano.compile.get_default_mode().including('canonicalize') for dtype, zero, one in [('bool', np.array(False), np.array(True)), ('int8', np.int8(0), np.int8(1)), ('int8', 0, 1)]: x = T.scalar('x', dtype=dtype) f = theano.function([x], T.and_(x, zero), mode=mode) self.assert_eqs_const(f, 0) f = theano.function([x], T.and_(zero, x), mode=mode) self.assert_eqs_const(f, 0) f = theano.function([x], T.and_(x, one), mode=mode) if dtype == 'bool': self.assert_identity(f) f = theano.function([x], T.and_(one, x), mode=mode) if dtype == 'bool': self.assert_identity(f) def test_and_int(self): # Test that bitwise "and" is correctly computed on int constants. f = theano.function([], T.and_(5, 6)) assert f() == 4 def test_or(self): # bitwise "or" with 0 should simplify for both bool and int # bitwise "or" with 1 should only give 1 for bool mode = theano.compile.get_default_mode().including('canonicalize') for dtype, zero, one in [('bool', np.array(False), np.array(True)), ('int8', np.int8(0), np.int8(1)), ('int8', 0, 1)]: x = T.scalar('x', dtype=dtype) f = theano.function([x], T.or_(x, one), mode=mode) if dtype == 'bool': self.assert_eqs_const(f, 1) f = theano.function([x], T.or_(one, x), mode=mode) if dtype == 'bool': self.assert_eqs_const(f, 1) f = theano.function([x], T.or_(x, zero), mode=mode) self.assert_identity(f) f = theano.function([x], T.or_(zero, x), mode=mode) self.assert_identity(f) def test_or_int(self): # Test that bitwise "or" is correctly computed on int constants. f = theano.function([], T.or_(5, 6)) assert f() == 7 def test_xor(self): # bitwise "xor" with itself should always give 0 for both bool and int. mode = theano.compile.get_default_mode().including('canonicalize') for dtype in ('bool', 'int8'): x = T.scalar('x', dtype=dtype) f = theano.function([x], T.xor(x, x), mode=mode) self.assert_eqs_const(f, 0) def test_stacktrace(self): mode = theano.compile.get_default_mode().including( 'local_useless_elemwise_comparison') x = T.vector('x', dtype=config.floatX) f = theano.function([x], T.gt(x, x), mode=mode) self.assertTrue(check_stack_trace(f, ops_to_check='last')) f = theano.function([x], T.le(x, x), mode=mode) self.assertTrue(check_stack_trace(f, ops_to_check='last')) class Test_local_canonicalize_alloc(unittest.TestCase): def setUp(self): self.rng = np.random.RandomState(utt.fetch_seed()) @change_flags(compute_test_value='off') def test0(self): x = shared(self.rng.randn(3, 7)) a = tensor.alloc(x, 6, 7) # It is a bad idea to have tensor.alloc return x directly, # because the shape mismatch cannot be caught. assert a.owner and isinstance(a.owner.op, tensor.Alloc) f = function([], a, mode=mode_opt) # The optimization should then be applied, and remove Alloc assert ([node.op for node in f.maker.fgraph.toposort()] == [deep_copy_op]) # In DebugMode, the shape mismatch should be detected if isinstance(mode_opt, compile.DebugMode): self.assertRaises(ValueError, f) # No need to check_stack_trace as the optimization # local_canonicalize_alloc only removes nodes. def test1(self): # Test that alloc never gets instantiated during optimization mode = mode_opt.excluding('local_canonicalize_alloc') x = tensor.matrix('x') xx = tensor.fill(x, x) # The optimization 'locall_fill_to_alloc' should call tensor.alloc, # which should return x and not alloc(x, ...) f = function([x], [xx], mode=mode) op_classes = [node.op.__class__ for node in f.maker.fgraph.toposort()] assert tensor.Alloc not in op_classes # No need to check_stack_trace as the optimization # local_canonicalize_alloc only removes nodes. def test2(self): # Test that alloc never gets instantiated during optimization mode = mode_opt.excluding('local_canonicalize_alloc') x = tensor.matrix('x') y = tensor.tile(x, (1,) * 2) f = function([x], [y], mode=mode) op_classes = [node.op.__class__ for node in f.maker.fgraph.toposort()] print(op_classes) # We are supposed to test if tensr.Alloc is not in op_classes, # but since the proper proper optimization is not currently # implemented it will fail. Once the correct optimization is in place, # we have to change the following we should not see tensor.Alloc # in op_classes and we have to change the assert. assert tensor.Alloc in op_classes # The correct opt removes nodes, no need for check_stack_trace def test_useless_alloc_with_shape_one(self): alloc_lift = out2in(local_canonicalize_alloc) x = shared(self.rng.randn(2,)) y = shared(self.rng.randn()) z = shared(self.rng.randn(1, 1)) w = shared(self.rng.randn(1, 1)) alloc_x = tensor.alloc(x, 1, 3, 2) alloc_y = tensor.alloc(y, 1, 1) alloc_z = tensor.alloc(z, 1, 1, 2) alloc_w = tensor.alloc(w, 1, 2) g = FunctionGraph([x, y, z, w], [alloc_x, alloc_y, alloc_z, alloc_w]) self.assertTrue(str(g) == ("[Alloc(, " "TensorConstant{1}, " "TensorConstant{3}, " "TensorConstant{2}), " "Alloc(, " "TensorConstant{1}, " "TensorConstant{1}), " "Alloc(, " "TensorConstant{1}, " "TensorConstant{1}, " "TensorConstant{2}), " "Alloc(, " "TensorConstant{1}, " "TensorConstant{2})]")) alloc_lift.optimize(g) self.assertTrue(str(g) == "[InplaceDimShuffle{x,0,1}" "(Alloc(, " "TensorConstant{3}, " "TensorConstant{2})), " "InplaceDimShuffle{x,x}" "(), " "InplaceDimShuffle{x,0,1}" "(Alloc(, " "TensorConstant{1}, " "TensorConstant{2})), " "Alloc(, " "TensorConstant{1}, " "TensorConstant{2})]") # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(g, ops_to_check='all')) class Test_local_useless_inc_subtensor_alloc(unittest.TestCase): opt_name = 'local_useless_inc_subtensor_alloc' def setUp(self): # The optimization requires the shape feature so we need to compile in # FAST_RUN mode. mode = theano.config.mode if mode == 'FAST_COMPILE': mode = 'FAST_RUN' self.mode = compile.mode.get_mode(mode) def test_advanced_inc_subtensor(self): x = tensor.vector('x') y = tensor.scalar('y') i = tensor.matrix('i', dtype='int64') z = tensor.advanced_inc_subtensor(x, T.alloc(y, *i.shape), i) mode1 = self.mode.excluding(self.opt_name) mode2 = self.mode.including(self.opt_name) f1 = theano.function([x, i, y], z, mode=mode1) f2 = theano.function([x, i, y], z, mode=mode2) # the alloc op should still be there assert (len([n for n in f1.maker.fgraph.toposort() if isinstance(n.op, tensor.Alloc)]) == 1) # the alloc op should have been removed assert (len([n for n in f2.maker.fgraph.toposort() if isinstance(n.op, tensor.Alloc)]) == 0) x_value = np.random.randn(5).astype(config.floatX) y_value = np.random.randn() i_value = np.random.randint(0, 3, size=(2, 3)) r1 = f1(x_value, i_value, y_value) r2 = f2(x_value, i_value, y_value) utt.assert_allclose(r1, r2) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f1, ops_to_check=tensor.AdvancedIncSubtensor)) self.assertTrue(check_stack_trace(f2, ops_to_check=tensor.AdvancedIncSubtensor)) def test_advanced_inc_subtensor1(self): x = tensor.vector('x') y = tensor.scalar('y') i = tensor.vector('i', dtype='int64') z = tensor.advanced_inc_subtensor1(x, T.alloc(y, *i.shape), i) mode1 = self.mode.excluding(self.opt_name) mode2 = self.mode.including(self.opt_name) f1 = theano.function([x, i, y], z, mode=mode1) f2 = theano.function([x, i, y], z, mode=mode2) # the alloc op should still be there assert (len([n for n in f1.maker.fgraph.toposort() if isinstance(n.op, tensor.Alloc)]) == 1) # the alloc op should have been removed assert (len([n for n in f2.maker.fgraph.toposort() if isinstance(n.op, tensor.Alloc)]) == 0) x_value = np.random.randn(5).astype(config.floatX) y_value = np.random.randn() i_value = np.random.randint(0, 3, size=2) r1 = f1(x_value, i_value, y_value) r2 = f2(x_value, i_value, y_value) utt.assert_allclose(r1, r2) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace( f1, ops_to_check=tensor.AdvancedIncSubtensor1)) self.assertTrue(check_stack_trace(f2, ops_to_check='all')) def test_incsubtensor(self): x = tensor.vector('x') y = tensor.scalar('y') i = tensor.scalar('i', dtype='int64') z = tensor.inc_subtensor(x[:i], T.alloc(y, i)) mode1 = self.mode.excluding(self.opt_name) mode2 = self.mode.including(self.opt_name) f1 = theano.function([x, i, y], z, mode=mode1) f2 = theano.function([x, i, y], z, mode=mode2) # the alloc op should still be there assert (len([n for n in f1.maker.fgraph.toposort() if isinstance(n.op, tensor.Alloc)]) == 1) # the alloc op should have been removed assert (len([n for n in f2.maker.fgraph.toposort() if isinstance(n.op, tensor.Alloc)]) == 0) x_value = np.random.randn(5).astype(config.floatX) y_value = np.random.randn() i_value = 3 r1 = f1(x_value, i_value, y_value) r2 = f2(x_value, i_value, y_value) utt.assert_allclose(r1, r2) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(f1, ops_to_check='last')) self.assertTrue(check_stack_trace(f2, ops_to_check='last')) class test_shapeoptimizer(unittest.TestCase): def setUp(self): utt.seed_rng() def test0(self): mode = theano.config.mode if mode == 'FAST_COMPILE': mode = 'FAST_RUN' v = T.vector() m = T.matrix() f = function([v, m], (v + m).shape, mode=mode) for node in f.maker.fgraph.toposort(): assert node.op != T.add def test_constant(self): mode = theano.config.mode if mode == 'FAST_COMPILE': mode = 'FAST_RUN' v = T.vector() f = function([v], v.dimshuffle('x', 'x', 0).shape[1], mode=mode) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert topo[0].op == deep_copy_op @staticmethod def max_pool_c01b(c01b, pool_shp, pool_stride, img_shp): """ Like max_pool but with input using axes ('c', 0, 1, 'b') (Alex Krizhevsky format) pool_shp, pool_stride and img_shp are int that represent the same shp in x and y. """ mx = None # Compute index in pooled space of last needed pool # (needed = each input pixel must appear in at least one pool) def last_pool(im_shp, p_shp, p_strd): rval = int(np.ceil(float(im_shp - p_shp) / p_strd)) assert p_strd * rval + p_shp >= im_shp assert p_strd * (rval - 1) + p_shp < im_shp return rval # Compute starting row of the last pool last_pool_r = last_pool(img_shp, pool_shp, pool_stride) * pool_stride # Compute number of rows needed in img for all indexes to work out required_r = last_pool_r + pool_shp last_pool_c = last_pool(img_shp, pool_shp, pool_stride) * pool_stride required_c = last_pool_c + pool_shp wide_infinity = T.alloc(-np.inf, c01b.shape[0], required_r, required_c, c01b.shape[3]) c01b = T.set_subtensor(wide_infinity[:, 0:img_shp, 0:img_shp, :], c01b) for row_within_pool in xrange(pool_shp): row_stop = last_pool_r + row_within_pool + 1 for col_within_pool in xrange(pool_shp): col_stop = last_pool_c + col_within_pool + 1 cur = c01b[:, row_within_pool:row_stop:pool_stride, col_within_pool:col_stop:pool_stride, :] if mx is None: mx = cur else: mx = T.maximum(mx, cur) return mx def test_broadcasted_dims(self): # This test a case that caused a crash during optimization shp = (1, 1, 1, 1) rng = np.random.RandomState(utt.fetch_seed()) a = shared(rng.rand(*shp).astype(config.floatX)) out = self.max_pool_c01b(a, 1, 1, 1) # max_pool_c01b use -inf and this will trigger DebugMode error. mode = copy.copy(theano.compile.get_default_mode()) mode.check_isfinite = False f = theano.function([], out, mode=mode) f() def test_constant_merge(self): # This test the error in gh-1122 that is a caused by the # combination of merge optimizer and ShapeFeature. x = tensor.constant([0, 0]) y = x[1:] x1 = x - tensor.join(0, y, y) x1.eval() def test_local_track_shape_i(self): class IdentityNoShape(gof.Op): '''Op that does not infer the output shape from the input one''' def make_node(self, x): x = as_tensor_variable(x) return gof.Apply(self, [x], [x.type()]) def perform(self, node, inp, out_): x, = inp out, = out_ out[0] = x.copy() # def infer_shape(self, node, (xshp,)): # return [tuple([self.shape_i(i)(r) for i in xrange(r.ndim)])] identity_noshape = IdentityNoShape() class IdentityShape(gof.Op): '''Op that does infer the output shape from the input one''' def make_node(self, x): x = as_tensor_variable(x) return gof.Apply(self, [x], [x.type()]) def perform(self, node, inp, out_): x, = inp out, = out_ out[0] = x.copy() def infer_shape(self, node, xshp_): # Could also just return. xshp, = xshp_ return (xshp,) identity_shape = IdentityShape() @gof.local_optimizer([IdentityNoShape]) def local_identity_noshape_to_identity_shape(node): '''Optimization transforming the first Op into the second''' if isinstance(node.op, IdentityNoShape): return [identity_shape(node.inputs[0])] mode = theano.compile.get_default_mode().including( 'ShapeOpt', 'specialize') rng = np.random.RandomState(utt.fetch_seed()) x = T.tensor3('x') ins_x = identity_noshape(x) # Without the optimization f = theano.function([x], ins_x.shape, mode=mode) xval = rng.randn(3, 4, 7).astype(config.floatX) assert np.all(f(xval) == [3, 4, 7]) f_ops = [node.op for node in f.maker.fgraph.toposort()] assert len(f_ops) == 5 assert identity_noshape in f_ops assert identity_shape not in f_ops # Register the optimization opt.register_specialize(local_identity_noshape_to_identity_shape) mode = theano.compile.get_default_mode().including( 'ShapeOpt', 'specialize') # With the optimization # The identity_shape op should not be needed anymore to compute # the shape g = theano.function([x], ins_x.shape, mode=mode) xval = rng.randn(6, 1, 2).astype(config.floatX) assert np.all(g(xval) == [6, 1, 2]) g_ops = [node.op for node in g.maker.fgraph.toposort()] assert len(g_ops) == 4 assert identity_noshape not in g_ops assert identity_shape not in g_ops # test multiple level of op without infer_shape ins_x3 = identity_noshape(identity_noshape(identity_noshape(x))) h = theano.function([x], ins_x3.shape, mode=mode) xval = rng.randn(6, 1, 2).astype(config.floatX) assert np.all(h(xval) == [6, 1, 2]) h_ops = [node.op for node in h.maker.fgraph.toposort()] assert len(h_ops) == 4 assert identity_noshape not in h_ops assert identity_shape not in h_ops def test_no_shapeopt(self): # Test that a basic example works even when ShapeOpt is excluded X = T.matrix() expr = X.shape[0] mode = theano.compile.get_default_mode().excluding('ShapeOpt') f = theano.function([X], expr, mode=mode) print(f([[1, 2], [2, 3]])) class test_assert(utt.InferShapeTester): def setUp(self): super(test_assert, self).setUp() def test0(self): x = T.scalar() y = T.scalar() f = theano.function([x, y], theano.tensor.opt.assert_op(x, T.eq(x, y))) f(1, 1) self.assertRaises(AssertionError, f, 1, 0) def test_local_remove_useless_assert1(self): # remove assert that are always true mode = theano.config.mode if mode == 'FAST_COMPILE': mode = 'FAST_RUN' mode = compile.mode.get_mode(mode) x = T.scalar() f = theano.function([x], theano.tensor.opt.assert_op(x, 1), mode=mode) assert f(1) == 1 assert f(5) == 5 topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert topo[0].op == deep_copy_op def test_test_local_remove_useless_assert2(self): # remove assert condition that are always true mode = theano.config.mode if mode == 'FAST_COMPILE': mode = 'FAST_RUN' mode = compile.mode.get_mode(mode) x = T.scalar() y = T.scalar() f = theano.function([x, y], theano.tensor.opt.assert_op(x, y, 1), mode=mode) assert f(1, 1) == 1 assert f(5, 1) == 5 topo = f.maker.fgraph.toposort() assert len(topo) == 2 assert len(topo[0].inputs) == 2 assert topo[1].op == deep_copy_op def test_local_remove_useless_assert3(self): # don't remove assert condition that are always false mode = theano.config.mode if mode == 'FAST_COMPILE': mode = 'FAST_RUN' mode = compile.mode.get_mode(mode) x = T.scalar() y = T.scalar() f = theano.function([x, y], theano.tensor.opt.assert_op(x, y, 0), mode=mode) self.assertRaises(AssertionError, f, 1, 0) topo = f.maker.fgraph.toposort() assert len(topo) == 2 assert len(topo[0].inputs) == 3 assert topo[1].op == deep_copy_op def test_local_remove_all_assert1(self): # remove assert condition that are unknown mode = theano.config.mode if mode == 'FAST_COMPILE': mode = 'FAST_RUN' mode = compile.mode.get_mode(mode).including('local_remove_all_assert') x = T.scalar() y = T.scalar() f = theano.function([x, y], theano.tensor.opt.assert_op(x, y), mode=mode) if isinstance(mode, theano.compile.debugmode.DebugMode): # DebugMode will run the original version with the Assert self.assertRaises(AssertionError, f, 1, 0) else: f(1, 0) # Without opt, it should fail. topo = f.maker.fgraph.toposort() assert len(topo) == 1, topo assert topo[0].op == deep_copy_op, topo mode = compile.mode.get_default_mode() a = theano.tensor.opt.assert_op(x, T.eq(x, 0).any()) f = theano.function([x], a, mode=mode.excluding('unsafe')) topo = f.maker.fgraph.toposort() a_op = [n for n in topo if isinstance(n.op, T.opt.Assert)] assert len(a_op) == 1 def test_infer_shape(self): adscal = dscalar() bdscal = dscalar() adscal_val = np.random.rand() bdscal_val = np.random.rand() + 1 out = theano.tensor.opt.assert_op(adscal, bdscal) self._compile_and_check([adscal, bdscal], [out], [adscal_val, bdscal_val], Assert) admat = dmatrix() admat_val = np.random.rand(3, 4) adscal_val += 1 out = theano.tensor.opt.assert_op(admat, adscal, bdscal) self._compile_and_check([admat, adscal, bdscal], [out], [admat_val, adscal_val, bdscal_val], Assert) def test_local_mul_specialize(): mode = theano.config.mode if mode == 'FAST_COMPILE': mode = 'FAST_RUN' mode = compile.mode.get_mode(mode) mode = mode.excluding('fusion') v = T.vector() m = T.vector() f = function([v], v * 1, mode=mode) nodes = [node.op for node in f.maker.fgraph.toposort()] nodes == [deep_copy_op] f = function([v], v * 0, mode=mode) nodes = [node.op for node in f.maker.fgraph.toposort()] assert nodes == [Shape_i(0), T.alloc] f = function([v], v * (-1), mode=mode) nodes = [node.op for node in f.maker.fgraph.toposort()] assert nodes == [T.neg] f = function([v, m], v * 1 * (-m), mode=mode) nodes = [node.op for node in f.maker.fgraph.toposort()] assert nodes == [T.mul] f = function([v, m], v * 0 * (-m), mode=mode) nodes = [node.op for node in f.maker.fgraph.toposort()] assert nodes == [Shape_i(0), T.alloc] f = function([v, m], v * (-1) * (-m), mode=mode) nodes = [node.op for node in f.maker.fgraph.toposort()] assert nodes == [T.mul] f = function([v, m], v * (-1) * m, mode=mode) nodes = [node.op for node in f.maker.fgraph.toposort()] assert nodes == [T.mul] class T_Tile(unittest.TestCase): def test_local_useless_tile(self): v = T.vector() m = T.matrix() mode = None if theano.config.mode == "FAST_COMPILE": mode = "FAST_RUN" for var, data in [(v, [1, 2, 3]), (m, [[1, 2], [3, 4]])]: # When len(repeat pattern) <= var.ndim, everything is removed # for ndim in range(1, var.ndim): for ndim in range(var.ndim + 1): f = theano.function([var], tile(var, (1,) * ndim), mode=mode) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, compile.DeepCopyOp) f(data) # In this case the opt only removes nodes, # no need to check_stack_trace # When len(repeat pattern) > var.ndim, only a dimshuffle should be # left, but there can be a DeepCopy as well for ndim in range(var.ndim + 1, var.ndim + 3): f = theano.function([var], tile(var, (1,) * ndim), mode=mode) topo = f.maker.fgraph.toposort() assert len(topo) <= 2 assert isinstance(topo[0].op, DimShuffle) assert check_stack_trace(f, ops_to_check=[DimShuffle]) f(data) def speed_local_pow_specialize_range(): val = np.random.rand(1e7) v = T.vector() mode = compile.mode.get_default_mode() mode_without_pow_opt = mode.excluding('local_pow_specialize') for i in xrange(500, 513): f1 = function([v], v ** i, mode=mode) f2 = function([v], v ** i, mode=mode_without_pow_opt) assert len(f1.maker.fgraph.toposort()) == 1 t1 = time.time() f1(val) t2 = time.time() f2(val) t3 = time.time() print(i, t2 - t1, t3 - t2, t2 - t1 < t3 - t2) if not t2 - t1 < t3 - t2: print("WARNING WE ARE SLOWER") for i in xrange(-3, -1500, -1): f1 = function([v], v ** i, mode=mode) f2 = function([v], v ** i, mode=mode_without_pow_opt) assert len(f1.maker.fgraph.toposort()) == 1 t1 = time.time() f1(val) t2 = time.time() f2(val) t3 = time.time() print(i, t2 - t1, t3 - t2, t2 - t1 < t3 - t2) if not t2 - t1 < t3 - t2: print("WARNING WE ARE SLOWER") def test_local_pow_specialize(): mode = theano.config.mode if mode == 'FAST_COMPILE': mode = 'FAST_RUN' mode = compile.mode.get_mode(mode) mode = mode.excluding('fusion') v = T.vector() val = np.arange(10, dtype=theano.config.floatX) val_no0 = np.arange(1, 10, dtype=theano.config.floatX) f = function([v], v ** 0, mode=mode) nodes = [node.op for node in f.maker.fgraph.toposort()] assert nodes == [Shape_i(0), T.alloc] utt.assert_allclose(f(val), val ** 0) f = function([v], v ** 1, mode=mode) nodes = [node.op for node in f.maker.fgraph.toposort()] nodes == [deep_copy_op] utt.assert_allclose(f(val), val ** 1) f = function([v], v ** (-1), mode=mode) nodes = [node.op for node in f.maker.fgraph.toposort()] assert nodes == [T.inv] utt.assert_allclose(f(val_no0), val_no0 ** (-1)) f = function([v], v ** 2, mode=mode) nodes = [node.op for node in f.maker.fgraph.toposort()] assert nodes == [T.sqr] utt.assert_allclose(f(val), val ** 2) f = function([v], v ** (-2), mode=mode) nodes = [node.op for node in f.maker.fgraph.toposort()] assert len(nodes) == 2 assert nodes[0] == T.sqr assert isinstance(nodes[1].scalar_op, theano.scalar.basic.Inv) # assert nodes == [T.sqr,T.inv]#Why this don't work? utt.assert_allclose(f(val_no0), val_no0 ** (-2)) f = function([v], v ** (.5), mode=mode) nodes = [node.op for node in f.maker.fgraph.toposort()] assert nodes == [T.sqrt] utt.assert_allclose(f(val), val ** (.5)) f = function([v], v ** (-.5), mode=mode) nodes = [node.op for node in f.maker.fgraph.toposort()] assert len(nodes) == 2 assert nodes[0] == T.sqrt assert isinstance(nodes[1].scalar_op, theano.scalar.basic.Inv) # assert nodes == [T.sqrt,T.inv]#Why this don't work? utt.assert_allclose(f(val_no0), val_no0 ** (-.5)) def test_local_pow_specialize_device_more_aggressive_on_cpu(): mode = theano.config.mode if mode == 'FAST_COMPILE': mode = 'FAST_RUN' mode = compile.mode.get_mode(mode) mode = mode.excluding('fusion').excluding('gpu') v = T.vector() val = np.arange(10, dtype=theano.config.floatX) val_no0 = np.arange(1, 10, dtype=theano.config.floatX) f = function([v], v ** (15), mode=mode) nodes = [node.op for node in f.maker.fgraph.toposort()] assert len(nodes) == 1 assert len(f.maker.fgraph.toposort()[0].op.scalar_op.fgraph.apply_nodes) == 6 assert isinstance(nodes[0].scalar_op, theano.scalar.Composite) utt.assert_allclose(f(val), val ** 15) f = function([v], v ** (-15), mode=mode) nodes = [node.op for node in f.maker.fgraph.toposort()] assert len(nodes) == 2 assert len(f.maker.fgraph.toposort()[0].op.scalar_op.fgraph.apply_nodes) == 6 assert isinstance(nodes[0].scalar_op, theano.scalar.Composite) assert isinstance(nodes[-1].scalar_op, theano.scalar.basic.Inv) utt.assert_allclose(f(val_no0), val_no0 ** (-15)) f = function([v], v ** (16), mode=mode) nodes = [node.op for node in f.maker.fgraph.toposort()] assert len(nodes) == 1 assert len(f.maker.fgraph.toposort()[0].op.scalar_op.fgraph.apply_nodes) == 4 assert isinstance(nodes[0].scalar_op, theano.scalar.Composite) utt.assert_allclose(f(val), val ** 16) f = function([v], v ** (-16), mode=mode) nodes = [node.op for node in f.maker.fgraph.toposort()] assert len(nodes) == 2 assert len(f.maker.fgraph.toposort()[0].op.scalar_op.fgraph.apply_nodes) == 4 assert isinstance(nodes[0].scalar_op, theano.scalar.Composite) assert isinstance(nodes[-1].scalar_op, theano.scalar.basic.Inv) utt.assert_allclose(f(val_no0), val_no0 ** (-16)) class T_Rebroadcast(unittest.TestCase): def test_local_useless_rebroadcast(self): mode = theano.compile.get_default_mode().including('canonicalize') v1 = T.vector() v2 = T.vector() j = T.join(0, v1, v2) f = theano.function([v1, v2], j, mode=mode) f([1, 2], [3, 4, 5]) e = f.maker.fgraph.toposort() assert len([n for n in e if isinstance(n.op, T.Rebroadcast)]) == 0 assert check_stack_trace(f, ops_to_check='all') def test_rebroadcast_rebroadcast(self): mode = theano.compile.get_default_mode().including('canonicalize') m = T.matrix() s = T.addbroadcast(m, 0, 1) v = T.unbroadcast(s, 1) f = theano.function([m], v, mode=mode) f([[76]]) e = f.maker.fgraph.toposort() rebroadcast_nodes = [n for n in e if isinstance(n.op, T.Rebroadcast)] assert len(rebroadcast_nodes) == 1 assert rebroadcast_nodes[0].op.axis == {0: True} class T_useless_elemwise(unittest.TestCase): def setUp(self): self.mode = theano.compile.get_default_mode().including( 'canonicalize', 'local_fill_to_alloc') def test_eq(self): x = T.dmatrix() y = T.dmatrix() f = theano.function([x, y], T.eq(x, y), mode=self.mode) vx = np.random.rand(5, 4) vy = np.random.rand(5, 4) f(vx, vy) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, T.Elemwise) assert isinstance(topo[0].op.scalar_op, theano.scalar.EQ) f2 = theano.function([x], T.eq(x, x), mode=self.mode) assert np.all(f2(vx) == np.ones((5, 4))) topo2 = f2.maker.fgraph.toposort() # Shape_i{1}(), Shape_i{0}(), Alloc([[1]], Shape_i{0}.0, Shape_i{1}.0 assert len(topo2) == 3 assert isinstance(topo2[-1].op, T.Alloc) def test_neq(self): x = T.dmatrix() y = T.dmatrix() f = theano.function([x, y], T.neq(x, y), mode=self.mode) vx = np.random.rand(5, 4) vy = np.random.rand(5, 4) f(vx, vy) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, T.Elemwise) assert isinstance(topo[0].op.scalar_op, theano.scalar.NEQ) f2 = theano.function([x], T.neq(x, x), mode=self.mode) assert np.all(f2(vx) == np.zeros((5, 4))) topo2 = f2.maker.fgraph.toposort() assert len(topo2) == 3 assert isinstance(topo2[-1].op, T.Alloc) def test_mul(self): x = T.dmatrix() y = T.dmatrix() f = theano.function([x], T.mul(x), mode=self.mode) vx = np.random.rand(5, 4) vy = np.random.rand(5, 4) f(vx) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert topo[0].op == deep_copy_op f2 = theano.function([x, y], T.mul(x, y), mode=self.mode) assert np.all(f2(vx, vy) == vx * vy) topo2 = f2.maker.fgraph.toposort() assert len(topo2) == 1 assert isinstance(topo2[0].op, T.Elemwise) assert isinstance(topo2[0].op.scalar_op, theano.scalar.Mul) def test_add(self): x = T.dmatrix() y = T.dmatrix() f = theano.function([x], T.add(x), mode=self.mode) vx = np.random.rand(5, 4) vy = np.random.rand(5, 4) f(vx) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert topo[0].op == deep_copy_op f2 = theano.function([x, y], T.add(x, y), mode=self.mode) assert np.all(f2(vx, vy) == vx + vy) topo2 = f2.maker.fgraph.toposort() assert len(topo2) == 1 assert isinstance(topo2[0].op, T.Elemwise) assert isinstance(topo2[0].op.scalar_op, theano.scalar.Add) def test_identity(self): # scalar.identity is used in 2 Elemwise functions: # tensor_copy, and view x = T.matrix() f = theano.function([x], T.tensor_copy(x), mode=self.mode) vx = np.random.rand(5, 4).astype(config.floatX) f(vx) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert topo[0].op == deep_copy_op class T_cast_cast(unittest.TestCase): def setUp(self): mode = theano.compile.get_default_mode() self.mode = mode.including('local_cast_cast') def test_consecutive(self): x = T.fmatrix() o = T.Elemwise(scal.Cast(scal.Scalar("float64")))(x.astype("float64")) f = theano.function([x], o, mode=self.mode) dx = np.random.rand(5, 4).astype("float32") f(dx) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op.scalar_op, scal.basic.Cast) x = T.dmatrix() o = T.Elemwise(scal.Cast(scal.Scalar("float32")))(x.astype("float32")) f = theano.function([x], o, mode=self.mode) dx = np.random.rand(5, 4) f(dx) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op.scalar_op, scal.basic.Cast) def test_upcast(self): # Upcast followed by any other cast x = T.fmatrix() o = T.Elemwise(scal.Cast(scal.Scalar("complex128")))(x.astype("complex64")) f = theano.function([x], o, mode=self.mode) dx = np.random.rand(5, 4).astype("float32") f(dx) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op.scalar_op, scal.basic.Cast) # Upcast followed by a downcast back to the base type x = T.fmatrix() o = T.Elemwise(scal.Cast(scal.Scalar("float32")))(x.astype("float64")) f = theano.function([x], o, mode=self.mode) dx = np.random.rand(5, 4).astype('float32') f(dx) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, DeepCopyOp) # Downcast followed by an upcast back to the base type # Optimization shouldn't be applied x = T.dmatrix() o = T.Elemwise(scal.Cast(scal.Scalar("float64")))(x.astype("float32")) f = theano.function([x], o, mode=self.mode) dx = np.random.rand(5, 4) f(dx) topo = f.maker.fgraph.toposort() assert (len(topo) == 1 and isinstance(topo[0].op.scalar_op, scal.basic.Composite)) or (len(topo) > 1) class T_func_inverse(unittest.TestCase): def setUp(self): mode = theano.compile.get_default_mode() self.mode = mode.including('local_func_inv') def assert_func_pair_optimized(self, func1, func2, data, should_copy=True, is_complex=False): # Check that a pair of funcs is optimized properly x = T.cmatrix() if is_complex else T.fmatrix() o = func2(func1(x)) f = theano.function([x], o, mode=self.mode) delta = f(data) - data topo = f.maker.fgraph.toposort() if should_copy: acceptable_topo_lens = [1] else: # The 2 funcs can be split apart if they are not inverses acceptable_topo_lens = [1, 2] if should_copy: delta_condition = np.all(delta == 0) else: delta_condition = np.all(delta != 0) self.assertTrue(len(topo) in acceptable_topo_lens) self.assertTrue(delta_condition) self.assertEqual(isinstance(topo[0].op, DeepCopyOp), should_copy, "Inverse functions not removed!") def test(self): # test optimization for consecutive functional inverses dx = np.random.rand(5, 4).astype("float32") self.assert_func_pair_optimized(T.deg2rad, T.rad2deg, dx) dx = np.random.rand(5, 4).astype("float32") * 180 self.assert_func_pair_optimized(T.rad2deg, T.deg2rad, dx) # Test the other functional inverses dx = np.random.rand(5, 4).astype("float32") self.assert_func_pair_optimized(T.cosh, T.arccosh, dx) self.assert_func_pair_optimized(T.arcsinh, T.sinh, dx) self.assert_func_pair_optimized(T.arctanh, T.tanh, dx) self.assert_func_pair_optimized(T.inv, T.inv, dx) self.assert_func_pair_optimized(T.neg, T.neg, dx) cx = dx + complex(0, 1) * (dx + 0.01) self.assert_func_pair_optimized(T.conj, T.conj, cx, is_complex=True) # Test that non-inverse functions are ran normally self.assert_func_pair_optimized(T.conj, T.neg, cx, should_copy=False, is_complex=True) dx = np.random.rand(5, 4).astype("float32") + 0.01 self.assert_func_pair_optimized(T.rad2deg, T.rad2deg, dx, should_copy=False) self.assert_func_pair_optimized(T.rad2deg, T.cosh, dx, should_copy=False) def test_constant_folding(): # Test that constant folding get registered at fast_compile # An error removed that registration during the registration. x = tensor.dvector() mode = theano.compile.get_mode("FAST_COMPILE").excluding("fusion") f = theano.function([x], [x * 2, x + x], mode=mode) topo = f.maker.fgraph.toposort() assert len(topo) == 2 # Test that we do not crash when constant folding elemwise scalar # as they should not generate c code. x = tensor.constant(3) assert x.ndim == 0 mode = theano.compile.get_mode("FAST_COMPILE").excluding("fusion") f = theano.function([], [x * 2, x + x], mode=mode) topo = f.maker.fgraph.toposort() assert len(topo) == 2 assert all([isinstance(n.op, DeepCopyOp) for n in topo]) def test_constant_get_stabilized(): # Currently Theano enable the constant_folding optimization before stabilization optimization. # This cause some stabilization optimization not being implemented and thus cause inf value to appear # when it should not. # # .. note: we can't simply move the constant_folding optimization to specialize as this break other optimization! # We will need to partially duplicate some canonicalize optimzation to specialize to fix this issue. x2 = T.scalar() y2 = T.log(1 + T.exp(x2)) mode = theano.compile.get_default_mode() mode.check_isfinite = False f2 = theano.function([x2], y2, mode=mode) try: assert len(f2.maker.fgraph.toposort()) == 1 assert (f2.maker.fgraph.toposort()[0].op == theano.tensor.nnet.sigm.softplus) assert f2(800) == 800 x = T.as_tensor_variable(800) y = T.log(1 + T.exp(x)) f = theano.function([], y, mode=mode) assert len(f.maker.fgraph.toposort()) == 0 assert np.isinf(f()) # When this error is fixed, the following line should be ok. assert f() == 800, f() except AssertionError: raise SkipTest('Theano optimizes constant before stabilization. ' 'This breaks stabilization optimization in some ' 'cases. See #504.') class T_local_switch_sink(unittest.TestCase): def setUp(self): # condition values self.condm = np.asarray([[0.1, 0, 1, -1], [0., 0., 0., 0.], [1, 1, 1, 1]]) self.condv = np.asarray([0.1, 0, 1, -1]) self.conds = [0.1, 0, 1, -1] # x values self.xm = np.ones((3, 4)) self.xv = np.ones((4,)) self.xs = 1. # expected results self.resm = ( [np.asarray([[1, 0, 1, 0], [0, 0, 0, 0], [1, 1, 1, 1]])] * 3 + [np.asarray([[1, 0, 1, 0], [1, 0, 1, 0], [1, 0, 1, 0]])] + 2 * [np.asarray([[1, 0, 1, 0]])] + [[np.ones((3, 4)), np.zeros((3, 4)), np.ones((3, 4)), np.zeros((3, 4))]] + [[np.ones((4,)), np.zeros((4,)), np.ones((4,)), np.zeros((4,))]] + [[np.asarray(1.0), np.asarray(0.0), np.asarray(1.0), np.asarray(0.0)]]) self.mode = theano.compile.mode.get_default_mode().including( 'canonicalize', 'fast_run').excluding('gpu', 'fusion') self.mode = copy.copy(self.mode) self.mode.check_isfinite = False def function_remove_nan(self, *args, **kwargs): """ Wrapper around theano.function for this test. It disables checking for NaN removed by optimizations in DebugMode (it has false positives in that case). """ f = theano.function(*args, **kwargs) def wrapped_f(*args, **kwargs): # This is a bit ugly since it changes the global value of # TensorType.values_eq_approx. old_values_eq_approx = staticmethod(TensorType.values_eq_approx) TensorType.values_eq_approx = staticmethod(values_eq_approx_remove_nan) try: out = f(*args, **kwargs) finally: TensorType.values_eq_approx = old_values_eq_approx return out return wrapped_f def test_local_mul_switch_sink(self): c = T.dscalar() idx = 0 for condition in [(T.dmatrix('cond'), self.condm), (T.dvector('cond'), self.condv), (T.dscalar('cond'), self.conds)]: for x in [(T.dmatrix('x'), self.xm), (T.dvector('x'), self.xv), (T.dscalar('x'), self.xs)]: y = T.mul(T.switch(condition[0] > 0, 1. * x[0], 0. * x[0]), T.switch(condition[0] > 0, 1. * x[0], T.log(c) * x[0])) f = self.function_remove_nan([condition[0], x[0], c], [y], mode=self.mode) if type(condition[1]) is list: for i in xrange(len(condition[1])): res = f(condition[1][i], x[1], -1) assert (res == np.asarray( self.resm[idx][i])).sum() == self.resm[idx][i].size else: res = f(condition[1], x[1], -1) assert ((res == np.asarray(self.resm[idx])).sum() == self.resm[idx].size) idx += 1 # This case caused a missed optimization in the past. x = T.dscalar('x') y = T.switch(x < 7, x, T.sqrt(x - 7)) f = self.function_remove_nan([x], T.grad(y, x), self.mode) assert f(5) == 1, f(5) @attr('slow') def test_local_div_switch_sink(self): c = T.dscalar() idx = 0 for condition in [(T.dmatrix('cond'), self.condm), (T.dvector('cond'), self.condv), (T.dscalar('cond'), self.conds)]: for x in [(T.dmatrix('x'), self.xm), (T.dvector('x'), self.xv), (T.dscalar('x'), self.xs)]: y = T.true_div( T.switch(condition[0] > 0, 1. * x[0], 0. * x[0]), T.switch(condition[0] > 0, 1. * x[0], T.log(c) * x[0])) f = self.function_remove_nan([condition[0], x[0], c], [y], mode=self.mode) if type(condition[1]) is list: for i in xrange(len(condition[1])): res = f(condition[1][i], x[1], -1) assert ((res == np.asarray(self.resm[idx][i])).sum() == self.resm[idx][i].size) else: res = f(condition[1], x[1], -1) assert ((res == np.asarray(self.resm[idx])).sum() == self.resm[idx].size) idx += 1 class T_local_erf(unittest.TestCase): def setUp(self): self.mode = theano.compile.mode.get_default_mode().including( 'canonicalize', 'fast_run').excluding('gpu', 'fusion') self.mode._optimizer.position_cutoff = 1.50001 if theano.config.cxx == '' and not theano.scalar.basic_scipy.imported_scipy_special: raise SkipTest("erf need a c++ compiler or scipy") def test_local_one_plus_erf(self): val = np.asarray([-30, -3, -2, -1, 0, 1, 2, 3, 30], dtype=config.floatX) x = T.vector() f = theano.function([x], 1 + T.erf(x), mode=self.mode) assert [n.op for n in f.maker.fgraph.toposort()] == [ T.mul, T.erfc], f.maker.fgraph.toposort() f(val) f = theano.function([x], T.erf(x) + 1, mode=self.mode) assert [n.op for n in f.maker.fgraph.toposort()] == [ T.mul, T.erfc], f.maker.fgraph.toposort() f(val) f = theano.function([x], T.erf(x) + 2, mode=self.mode) topo = f.maker.fgraph.toposort() assert len(topo) == 2 assert topo[0].op == T.erf assert isinstance(topo[1].op, T.Elemwise) assert isinstance(topo[1].op.scalar_op, scal.Add) f(val) def test_local_one_minus_erf(self): val = np.asarray([-30, -3, -2, -1, 0, 1, 2, 3, 30], dtype=config.floatX) x = T.vector() f = theano.function([x], 1 - T.erf(x), mode=self.mode) assert [n.op for n in f.maker.fgraph.toposort()] == [T.erfc],\ f.maker.fgraph.toposort() print(f(val)) f = theano.function([x], 1 + (-T.erf(x)), mode=self.mode) assert [n.op for n in f.maker.fgraph.toposort()] == [T.erfc],\ f.maker.fgraph.toposort() print(f(val)) f = theano.function([x], (-T.erf(x)) + 1, mode=self.mode) assert [n.op for n in f.maker.fgraph.toposort()] == [T.erfc],\ f.maker.fgraph.toposort() print(f(val)) f = theano.function([x], 2 - T.erf(x), mode=self.mode) topo = f.maker.fgraph.toposort() assert len(topo) == 2, f.maker.fgraph.toposort() assert topo[0].op == T.erf, f.maker.fgraph.toposort() assert isinstance(topo[1].op, T.Elemwise), f.maker.fgraph.toposort() assert isinstance(topo[1].op.scalar_op, scal.Add)\ or isinstance(topo[1].op.scalar_op, scal.Sub), f.maker.fgraph.toposort() print(f(val)) def test_local_erf_minus_one(self): val = np.asarray([-30, -3, -2, -1, 0, 1, 2, 3, 30], dtype=config.floatX) x = T.vector() f = theano.function([x], T.erf(x) - 1, mode=self.mode) assert [n.op for n in f.maker.fgraph.toposort()] == [T.erfc, T.mul] print(f(val)) f = theano.function([x], T.erf(x) + (-1), mode=self.mode) assert [n.op for n in f.maker.fgraph.toposort()] == [T.erfc, T.mul] print(f(val)) f = theano.function([x], -1 + T.erf(x), mode=self.mode) assert [n.op for n in f.maker.fgraph.toposort()] == [T.erfc, T.mul] print(f(val)) f = theano.function([x], T.erf(x) - 2, mode=self.mode) topo = f.maker.fgraph.toposort() assert len(topo) == 2 assert topo[0].op == T.erf assert isinstance(topo[1].op, T.Elemwise) assert isinstance(topo[1].op.scalar_op, scal.Add)\ or isinstance(topo[1].op.scalar_op, scal.Sub) print(f(val)) class T_local_erfc(unittest.TestCase): def setUp(self): self.mode_fusion = theano.compile.mode.get_default_mode().including( 'canonicalize').including('fast_run').excluding('gpu') self.mode = self.mode_fusion.excluding('fusion') self.mode._optimizer.position_cutoff = 1.50001 if (theano.config.cxx == '' and not theano.scalar.basic_scipy.imported_scipy_special): raise SkipTest("erfc need a c++ compiler or scipy") def test_local_one_minus_erfc(self): # test opt: 1-erfc(x) => erf(x) and -erfc(x)+1 => erf(x) val = np.asarray([-30, -3, -2, -1, 0, 1, 2, 3, 30], dtype=config.floatX) x = T.vector('x') f = theano.function([x], 1 - T.erfc(x), mode=self.mode) assert [n.op for n in f.maker.fgraph.toposort()] == [T.erf],\ f.maker.fgraph.toposort() print(f(val)) f = theano.function([x], (-T.erfc(x)) + 1, mode=self.mode) assert [n.op for n in f.maker.fgraph.toposort()] == [T.erf],\ f.maker.fgraph.toposort() print(f(val)) f = theano.function([x], 2 - T.erfc(x), mode=self.mode) topo = f.maker.fgraph.toposort() assert len(topo) == 2, f.maker.fgraph.toposort() assert topo[0].op == T.erfc, f.maker.fgraph.toposort() assert isinstance(topo[1].op, T.Elemwise), f.maker.fgraph.toposort() assert isinstance(topo[1].op.scalar_op, scal.Sub),\ f.maker.fgraph.toposort() print(f(val)) def test_local_erf_neg_minus_one(self): # test opt: (-1)+erfc(-x)=>erf(x) val = np.asarray([-30, -3, -2, -1, 0, 1, 2, 3, 30], dtype=config.floatX) x = T.vector('x') f = theano.function([x], -1 + T.erfc(-x), mode=self.mode) assert [n.op for n in f.maker.fgraph.toposort()] == [T.erf],\ f.maker.fgraph.toposort() print(f(val)) f = theano.function([x], T.erfc(-x) - 1, mode=self.mode) assert [n.op for n in f.maker.fgraph.toposort()] == [T.erf],\ f.maker.fgraph.toposort() print(f(val)) f = theano.function([x], T.erfc(-x) + (-1), mode=self.mode) assert [n.op for n in f.maker.fgraph.toposort()] == [T.erf],\ f.maker.fgraph.toposort() print(f(val)) def test_local_log_erfc(self): val = [-30, -27, -26, -11, -10, -3, -2, -1, 0, 1, 2, 3, 10, 11, 26, 27, 28, 30] if theano.config.mode in ["DebugMode", "DEBUG_MODE", "FAST_COMPILE"]: # python mode don't like the inv(0) val.remove(0) val = np.asarray(val, dtype=config.floatX) x = T.vector('x') # their is some nan that will happear in the graph for the log of the negatives values mode = copy.copy(self.mode) mode.check_isfinite = False mode_fusion = copy.copy(self.mode_fusion) mode_fusion.check_isfinite = False f = theano.function([x], T.log(T.erfc(x)), mode=mode) assert len(f.maker.fgraph.apply_nodes) == 23, len(f.maker.fgraph.apply_nodes) assert f.maker.fgraph.outputs[0].dtype == theano.config.floatX assert all(np.isfinite(f(val))) f = theano.function([x], T.log(T.erfc(-x)), mode=mode) assert len(f.maker.fgraph.apply_nodes) == 24, len(f.maker.fgraph.apply_nodes) assert f.maker.fgraph.outputs[0].dtype == theano.config.floatX assert all(np.isfinite(f(-val))) f = theano.function([x], T.log(T.erfc(x)), mode=mode_fusion) assert len(f.maker.fgraph.apply_nodes) == 1, len(f.maker.fgraph.apply_nodes) assert f.maker.fgraph.outputs[0].dtype == theano.config.floatX assert len(f.maker.fgraph.toposort()[0].fgraph.toposort()[ 0].op.scalar_op.fgraph.apply_nodes) == 22, len(f.maker.fgraph.toposort()[0].fgraph.toposort()[0].op.scalar_op.fgraph.apply_nodes) # TODO: fix this problem if theano.config.floatX == "float32" and theano.config.mode in ["DebugMode", "DEBUG_MODE"]: raise SkipTest('The python code upcast somewhere internally ' 'some value of float32 to python float for ' 'part of its computation. That make that the ' 'c and python code dont generate the same value. ' 'You can ignore this error.') assert all(np.isfinite(f(val))) def test_local_grad_log_erfc_neg(self): val = [-100, -30, -27, -26.4, -26.2, -26, -11, -10, -9, -3, -2, -1, 0, 1, 2, 3, 9, 10, 11, 27, 26.4, 26.2, 26, 28, 30, 100] if theano.config.mode in ["DebugMode", "DEBUG_MODE", "FAST_COMPILE"]: # python mode don't like the inv(0) in computation, # but the switch don't select this value. # So it is computed for no good reason. val.remove(0) if theano.config.mode in ["DebugMode", "DEBUG_MODE"] and theano.config.floatX == 'float32': # In float32 their is a plage of values close to 10 that we stabilize as it give bigger error then the stabilized version. # The orig value in float32 -30.0, the stab value -20.1 the orig value in float64 -18.1. val.remove(10) val = np.asarray(val, dtype=config.floatX) x = T.vector('x') y = T.vector('y') # their is some nan that will happear in the graph for the log of the negatives values mode = copy.copy(self.mode) mode.check_isfinite = False mode_fusion = copy.copy(self.mode_fusion) mode_fusion.check_isfinite = False f = theano.function([x], T.grad(T.log(T.erfc(x)).sum(), x), mode=mode) assert len(f.maker.fgraph.apply_nodes) == 23, len(f.maker.fgraph.apply_nodes) assert all(np.isfinite(f(val))) assert f.maker.fgraph.outputs[0].dtype == theano.config.floatX # test with a different mul constant f = theano.function( [x], T.mul(T.exp(T.neg(T.sqr(x))), - 10.12837917) / T.erfc(x), mode=mode) assert len(f.maker.fgraph.apply_nodes) == 23, len(f.maker.fgraph.apply_nodes) assert f.maker.fgraph.outputs[0].dtype == theano.config.floatX assert all(np.isfinite(f(val))) # test that we work without the mul f = theano.function([x], T.exp(T.neg(T.sqr(x))) / T.erfc(x), mode=mode) assert len(f.maker.fgraph.apply_nodes) == 22, len(f.maker.fgraph.apply_nodes) assert f.maker.fgraph.outputs[0].dtype == theano.config.floatX assert all(np.isfinite(f(val))) # test that we don't work if x!=y f = theano.function([x, y], T.exp(T.neg(T.sqr(x))) / T.erfc( y), mode=mode) assert len(f.maker.fgraph.apply_nodes) == 5, len(f.maker.fgraph.apply_nodes) assert f.maker.fgraph.outputs[0].dtype == theano.config.floatX f(val, val - 3) # test that we work without the sqr and neg f = theano.function([x], T.exp(T.mul(-1, x, x)) / T.erfc(x), mode=mode) assert len(f.maker.fgraph.apply_nodes) == 21, len(f.maker.fgraph.apply_nodes) assert f.maker.fgraph.outputs[0].dtype == theano.config.floatX assert all(np.isfinite(f(val))) # test that it work correctly if x is x*2 in the graph. f = theano.function([x], T.grad(T.log(T.erfc(2 * x)).sum(), x), mode=mode) assert len(f.maker.fgraph.apply_nodes) == 23, len(f.maker.fgraph.apply_nodes) assert np.isfinite(f(val)).all() assert f.maker.fgraph.outputs[0].dtype == theano.config.floatX f = theano.function([x], T.grad(T.log(T.erfc(x)).sum(), x), mode=mode_fusion) assert len(f.maker.fgraph.apply_nodes) == 1, len(f.maker.fgraph.apply_nodes) assert f.maker.fgraph.outputs[0].dtype == theano.config.floatX # TODO: fix this problem if theano.config.floatX == "float32" and theano.config.mode in ["DebugMode", "DEBUG_MODE"]: # The python code upcast somewhere internally some value of float32 # to python float for part of its computation. That make that the c # and python code do not generate the same value. You can ignore # this error. This happen in an intermediate step that don't show # in the final result. # Showing this test error is a duplicate of the one in test_local_log_erfc. We hide it. pass else: assert all(np.isfinite(f(val))) def speed_local_log_erfc(self): val = np.random.rand(1e6) x = T.vector() mode = theano.compile.mode.get_mode("FAST_RUN") f1 = theano.function([x], T.log(T.erfc(x)), mode=mode.excluding("local_log_erfc")) f2 = theano.function([x], T.log(T.erfc(x)), mode=mode) print(f1.maker.fgraph.toposort()) print(f2.maker.fgraph.toposort()) t0 = time.time() f1(val) t1 = time.time() f2(val) t2 = time.time() print(t1 - t0, t2 - t1) class test_local_useless_switch(unittest.TestCase): def setUp(self): self.mode = mode_opt.excluding('constant_folding') def test_const0(self): for dtype1 in ['int32', 'int64']: for dtype2 in ['int32', 'int64']: x = theano.tensor.matrix('x', dtype=dtype1) y = theano.tensor.matrix('y', dtype=dtype2) z = theano.tensor.switch(0, x, y) f = theano.function([x, y], z, mode=self.mode) assert len([node.op for node in f.maker.fgraph.toposort() if (isinstance(node.op, theano.tensor.Elemwise) and isinstance(node.op.scalar_op, theano.scalar.basic.Switch))]) == 0 vx = np.array([[1, 2, 3], [4, 5, 6]], dtype=dtype1) vy = np.array([[7, 8, 9], [10, 11, 12]], dtype=dtype2) assert np.all(f(vx, vy) == vy) def test_const1(self): for dtype1 in ['int32', 'int64']: for dtype2 in ['int32', 'int64']: x = theano.tensor.matrix('x', dtype=dtype1) y = theano.tensor.matrix('y', dtype=dtype2) z = theano.tensor.switch(1, x, y) f = theano.function([x, y], z, mode=self.mode) assert len([node.op for node in f.maker.fgraph.toposort() if (isinstance(node.op, theano.tensor.Elemwise) and isinstance(node.op.scalar_op, theano.scalar.basic.Switch))]) == 0 vx = np.array([[1, 2, 3], [4, 5, 6]], dtype=dtype1) vy = np.array([[7, 8, 9], [10, 11, 12]], dtype=dtype2) assert np.all(f(vx, vy) == vx) def test_left_is_right(self): for dtype1 in ['int32', 'int64']: x = theano.tensor.matrix('x', dtype=dtype1) varc = theano.tensor.matrix('varc', dtype=dtype1) z1 = theano.tensor.switch(1, x, x) z0 = theano.tensor.switch(0, x, x) z2 = theano.tensor.switch(varc, x, x) f1 = theano.function([x], z1, mode=self.mode) f0 = theano.function([x], z0, mode=self.mode) f2 = theano.function([x, varc], z2, mode=self.mode) topo = f1.maker.fgraph.toposort() assert len(topo) == 1 assert topo[0].op == deep_copy_op topo = f0.maker.fgraph.toposort() assert len(topo) == 1 assert topo[0].op == deep_copy_op topo = f2.maker.fgraph.toposort() assert len(topo) == 1 assert topo[0].op == deep_copy_op vx = np.array([[1, 2, 3], [4, 5, 6]], dtype=dtype1) vc = np.array([[1, 2, 3], [4, 5, 6]], dtype=dtype1) assert np.all(f1(vx) == vx) assert np.all(f0(vx) == vx) assert np.all(f2(vx, vc) == vx) def test_shape_le_0(self): for dtype1 in ['float32', 'float64']: x = theano.tensor.matrix('x', dtype=dtype1) z0 = theano.tensor.switch(theano.tensor.le(x.shape[0], 0), 0, x.shape[0]) f0 = theano.function([x], z0, mode=self.mode) assert isinstance(f0.maker.fgraph.toposort()[0].op, Shape_i) z1 = theano.tensor.switch(theano.tensor.le(x.shape[1], 0), 0, x.shape[1]) f1 = theano.function([x], z1, mode=self.mode) assert isinstance(f1.maker.fgraph.toposort()[0].op, Shape_i) vx = np.random.randn(0, 5).astype(dtype1) assert f0(vx) == 0 assert f1(vx) == 5 def test_broadcast1(self): # test switch(cst, matrix, row) x = theano.tensor.matrix('x', dtype='int32') y = theano.tensor.vector('y', dtype='int64') z = theano.tensor.switch(1, x, y) f = theano.function([x, y], z, mode=self.mode) assert len([node.op for node in f.maker.fgraph.toposort() if isinstance(node.op, theano.tensor.Elemwise) and not isinstance(node.op.scalar_op, theano.scalar.basic.Cast)]) == 0 vx = np.array([[1, 2, 3], [4, 5, 6]], dtype='int32') vy = np.array([10, 11, 12], dtype='int64') assert np.all(f(vx, vy) == vx) z = theano.tensor.switch(0, x, y) f = theano.function([x, y], z, mode=self.mode) assert len([node.op for node in f.maker.fgraph.toposort() if isinstance(node.op, theano.tensor.Elemwise)]) == 0 vx = np.array([[1, 2, 3], [4, 5, 6]], dtype='int32') vy = np.array([10, 11, 12], dtype='int64') assert np.all(f(vx, vy) == vy) def test_broadcast2(self): # test switch(cst, vector, matrix) # This case is not optimized for now. x = theano.tensor.vector('x', dtype='int32') y = theano.tensor.matrix('y', dtype='int64') z = theano.tensor.switch(1, x, y) f = theano.function([x, y], z, mode=self.mode) assert len([node.op for node in f.maker.fgraph.toposort() if isinstance(node.op, theano.tensor.Elemwise) and not isinstance(node.op.scalar_op, theano.scalar.basic.Cast)]) == 0 vx = np.array([4, 5, 6], dtype='int32') vy = np.array([[7, 8, 9], [10, 11, 12]], dtype='int64') assert np.all(f(vx, vy) == vx) z = theano.tensor.switch(0, x, y) f = theano.function([x, y], z, mode=self.mode) assert len([node.op for node in f.maker.fgraph.toposort() if isinstance(node.op, theano.tensor.Elemwise)]) == 0 vx = np.array([4, 5, 6], dtype='int32') vy = np.array([[7, 8, 9], [10, 11, 12]], dtype='int64') assert np.all(f(vx, vy) == vy) def test_broadcast3(self): # test switch(matrix, same_vector, same_vector) x = theano.tensor.matrix('x', dtype='int32') y = theano.tensor.vector('y', dtype='int64') z = theano.tensor.switch(x, y, y) f = theano.function([x, y], z, mode=self.mode) vx = np.array([[0, 1], [1, 0]], dtype='int32') vy = np.array([7, 8], dtype='int64') utt.assert_allclose(f(vx, vy), np.where(vx, vy, vy)) assert len([node.op for node in f.maker.fgraph.toposort() if isinstance(node.op, theano.tensor.Elemwise)]) == 0 class test_local_merge_switch_same_cond(unittest.TestCase): def test_elemwise(self): # float Ops mats = theano.tensor.matrices('cabxy') c, a, b, x, y = mats s1 = T.switch(c, a, b) s2 = T.switch(c, x, y) for op in (T.add, T.sub, T.mul, T.true_div, T.int_div, T.floor_div, T.minimum, T.maximum, T.gt, T.lt, T.ge, T.le, T.eq, T.neq, T.pow): g = optimize(FunctionGraph(mats, [op(s1, s2)])) assert str(g).count('Switch') == 1 # integer Ops mats = theano.tensor.imatrices('cabxy') c, a, b, x, y = mats s1 = T.switch(c, a, b) s2 = T.switch(c, x, y) for op in (T.and_, T.or_, T.xor, T.bitwise_and, T.bitwise_or, T.bitwise_xor): g = optimize(FunctionGraph(mats, [op(s1, s2)])) assert str(g).count('Switch') == 1 # add/mul with more than two inputs u, v = theano.tensor.matrices('uv') s3 = T.switch(c, u, v) for op in (T.add, T.mul): g = optimize(FunctionGraph(mats + [u, v], [op(s1, s2, s3)])) assert str(g).count('Switch') == 1 class T_local_sum_prod(unittest.TestCase): """ Test sum/prod opts in opt.py """ def setUp(self): self.mode = theano.compile.get_default_mode().including('canonicalize', 'specialize') def test_local_sum_prod_mul_by_scalar(self): # Test the optimization local_sum_prod_mul_by_scalar for both Sum and # Prod ops in six cases each : # 1-the inputs to the mul contain a scalar and no non-scalar # 2-the inputs to the mul contain a scalar and one non-scalar # 3-the inputs to the mul contain a scalar and two non-scalars # 4-the inputs to the mul contain two scalars and no non-scalar # 5-the inputs to the mul contain two scalars and one non-scalar # 6-the inputs to the mul contain two scalars and two non-scalars vect = T.dvector() mat = T.dmatrix() scalar1 = T.dscalar() scalar2 = T.dscalar() v_val = np.random.rand(2) m_val = np.random.rand(2, 2) s1_val = np.random.rand() s2_val = np.random.rand() def test_reduction_opt(inputs, inputs_val, reduction_op, expected_output, nb_expected_sum_nodes): mul_out = T.mul(*inputs) f = theano.function(inputs, reduction_op()(mul_out), mode=self.mode) out = f(*inputs_val) utt.assert_allclose(out, expected_output) # Ensure that the optimization has been applied properly by # ensuring that the optimized graph contains the expected number # of apply nodes for the sum op prod_nodes = [n for n in f.maker.fgraph.toposort() if isinstance(n.op, reduction_op)] assert len(prod_nodes) == nb_expected_sum_nodes # Test sum # Case 1 test_reduction_opt([scalar1], [s1_val], T.Sum, s1_val, 0) # Case 2 test_reduction_opt([vect, scalar1], [v_val, s1_val], T.Sum, s1_val * v_val.sum(), 1) # Case 3 test_reduction_opt([vect, mat, scalar1], [v_val, m_val, s1_val], T.Sum, s1_val * (v_val * m_val).sum(), 1) # Case 4 test_reduction_opt([scalar1, scalar2], [s1_val, s2_val], T.Sum, s1_val * s2_val, 0) # Case 5 test_reduction_opt([vect, scalar1, scalar2], [v_val, s1_val, s2_val], T.Sum, s1_val * s2_val * v_val.sum(), 1) # Case 6 test_reduction_opt([vect, mat, scalar1, scalar2], [v_val, m_val, s1_val, s2_val], T.Sum, s1_val * s2_val * (v_val * m_val).sum(), 1) # Test prod # Case 1 test_reduction_opt([scalar1], [s1_val], T.elemwise.Prod, s1_val, 0) # Case 2 test_reduction_opt([vect, scalar1], [v_val, s1_val], T.elemwise.Prod, (s1_val * v_val).prod(), 1) # Case 3 test_reduction_opt([vect, mat, scalar1], [v_val, m_val, s1_val], T.elemwise.Prod, (s1_val * v_val * m_val).prod(), 2) # Case 4 test_reduction_opt([scalar1, scalar2], [s1_val, s2_val], T.elemwise.Prod, s1_val * s2_val, 0) # Case 5 test_reduction_opt([vect, scalar1, scalar2], [v_val, s1_val, s2_val], T.elemwise.Prod, (s1_val * s2_val * v_val).prod(), 1) # Case 6 test_reduction_opt([vect, mat, scalar1, scalar2], [v_val, m_val, s1_val, s2_val], T.elemwise.Prod, (s1_val * s2_val * v_val * m_val).prod(), 2) def test_local_sum_prod_all_to_none(self): a = T.tensor3() input = np.arange(3 * 4 * 5, dtype=config.floatX).reshape(3, 4, 5) # test sum f = theano.function([a], a.sum(), mode=self.mode) assert len(f.maker.fgraph.apply_nodes) == 1 utt.assert_allclose(f(input), input.sum()) # test prod f = theano.function([a], a.prod(), mode=self.mode) assert len(f.maker.fgraph.apply_nodes) == 1 utt.assert_allclose(f(input), input.prod()) # test sum f = theano.function([a], a.sum([0, 1, 2]), mode=self.mode) assert len(f.maker.fgraph.apply_nodes) == 1 utt.assert_allclose(f(input), input.sum()) # test prod f = theano.function([a], a.prod([0, 1, 2]), mode=self.mode) assert len(f.maker.fgraph.apply_nodes) == 1 utt.assert_allclose(f(input), input.prod()) backup = config.warn.sum_sum_bug config.warn.sum_sum_bug = False try: f = theano.function([a], a.sum(0).sum(0).sum(0), mode=self.mode) assert len(f.maker.fgraph.apply_nodes) == 1 utt.assert_allclose(f(input), input.sum()) finally: config.warn.sum_sum_bug = backup def test_local_sum_sum_prod_prod(self): a = T.tensor3() input = np.arange(3 * 4 * 5, dtype=config.floatX).reshape(3, 4, 5) dims = [(0, 0), (1, 0), (2, 0), (0, 1), (1, 1), (2, 1), ((0, 1), 0), ((1, 2), 0), (0, (0, 1)), (1, (0, 1)), (2, (0, 1))] backup = config.warn.sum_sum_bug config.warn.sum_sum_bug = False def my_prod(data, d, dd): # This prod when d or dd is a tuple of 2 dimensions. if not isinstance(d, tuple) and not isinstance(dd, tuple): return data.prod(d).prod(dd) if isinstance(d, tuple): d = sorted(d) return data.prod(d[1]).prod(d[0]).prod(dd) else: dd = sorted(dd) return data.prod(d).prod(dd[1]).prod(dd[0]) def my_sum(data, d, dd): # This sum when d or dd is a tuple of 2 dimensions. if not isinstance(d, tuple) and not isinstance(dd, tuple): return data.sum(d).sum(dd) if isinstance(d, tuple): d = sorted(d) return data.sum(d[1]).sum(d[0]).sum(dd) else: dd = sorted(dd) return data.sum(d).sum(dd[1]).sum(dd[0]) def my_sum_prod(data, d, dd): # This sum when d or dd is a tuple of 2 dimensions. if not isinstance(d, tuple) and not isinstance(dd, tuple): return data.sum(d).prod(dd) if isinstance(d, tuple): d = sorted(d) return data.sum(d[1]).sum(d[0]).prod(dd) else: dd = sorted(dd) return data.sum(d).prod(dd[1]).prod(dd[0]) try: for d, dd in dims: expected = my_sum(input, d, dd) f = theano.function([a], a.sum(d).sum(dd), mode=self.mode) utt.assert_allclose(f(input), expected) assert len(f.maker.fgraph.apply_nodes) == 1 for d, dd in dims[:6]: f = theano.function([a], a.sum(d).sum(dd). sum(0), mode=self.mode) utt.assert_allclose(f(input), input.sum(d).sum(dd).sum(0)) assert len(f.maker.fgraph.apply_nodes) == 1 for d in [0, 1, 2]: f = theano.function([a], a.sum(d).sum(None), mode=self.mode) utt.assert_allclose(f(input), input.sum(d).sum()) assert len(f.maker.fgraph.apply_nodes) == 1 f = theano.function([a], a.sum(None).sum(), mode=self.mode) utt.assert_allclose(f(input), input.sum()) assert len(f.maker.fgraph.apply_nodes) == 1 finally: config.warn.sum_sum_bug = backup # test prod for d, dd in dims: expected = my_prod(input, d, dd) f = theano.function([a], a.prod(d).prod(dd), mode=self.mode) utt.assert_allclose(f(input), expected) assert len(f.maker.fgraph.apply_nodes) == 1 for d, dd in dims[:6]: f = theano.function([a], a.prod(d).prod(dd). prod(0), mode=self.mode) utt.assert_allclose(f(input), input.prod(d).prod(dd).prod(0)) assert len(f.maker.fgraph.apply_nodes) == 1 for d in [0, 1, 2]: f = theano.function([a], a.prod(d).prod(None), mode=self.mode) utt.assert_allclose(f(input), input.prod(d).prod()) assert len(f.maker.fgraph.apply_nodes) == 1 f = theano.function([a], a.prod(None).prod(), mode=self.mode) utt.assert_allclose(f(input), input.prod()) assert len(f.maker.fgraph.apply_nodes) == 1 # test sum prod don't get opt. for d, dd in dims: expected = my_sum_prod(input, d, dd) f = theano.function([a], a.sum(d).prod(dd), mode=self.mode) utt.assert_allclose(f(input), expected) assert len(f.maker.fgraph.apply_nodes) == 2 for d, dd in dims[:6]: f = theano.function([a], a.sum(d).prod(dd). prod(0), mode=self.mode) utt.assert_allclose(f(input), input.sum(d).prod(dd).prod(0)) assert len(f.maker.fgraph.apply_nodes) == 2 for d in [0, 1, 2]: f = theano.function([a], a.sum(d).prod(None), mode=self.mode) utt.assert_allclose(f(input), input.sum(d).prod()) assert len(f.maker.fgraph.apply_nodes) == 2 f = theano.function([a], a.sum(None).prod(), mode=self.mode) utt.assert_allclose(f(input), input.sum()) assert len(f.maker.fgraph.apply_nodes) == 1 def test_local_sum_prod_alloc(self): # test local_opt_alloc a = T.dtensor3() input = np.asarray(np.arange(2 * 3 * 4).reshape(2, 3, 4), dtype='float64') mode = self.mode.including('specialize').excluding('fusion') for t_like, n_like, nb_nodes in [ (tensor.zeros_like, np.zeros_like, (1, 3, 3, 2)), (tensor.ones_like, np.ones_like, (5, 5, 5, 6))]: # test sum f = theano.function([a], t_like(a).sum(None), mode=mode) utt.assert_allclose(f(input), n_like(input).sum()) assert len(f.maker.fgraph.apply_nodes) == nb_nodes[0] f = theano.function([a], t_like(a).sum([0, 1, 2]), mode=mode) utt.assert_allclose(f(input), n_like(input).sum()) assert len(f.maker.fgraph.apply_nodes) == nb_nodes[0] for d in xrange(3): f = theano.function([a], t_like(a).sum(d), mode=mode) utt.assert_allclose(f(input), n_like(input).sum(d)) assert len(f.maker.fgraph.apply_nodes) == nb_nodes[1] topo = f.maker.fgraph.toposort() assert topo[-1].op == T.alloc assert not any([isinstance(node.op, T.Sum) for node in topo]) for i in xrange(3): f = theano.function([a], t_like(a).sum(i), mode=mode) utt.assert_allclose(f(input), n_like(input).sum(i)) assert len(f.maker.fgraph.apply_nodes) == nb_nodes[2] topo = f.maker.fgraph.toposort() assert topo[-1].op == T.alloc assert not any([isinstance(node.op, T.Sum) for node in topo]) # test prod f = theano.function([a], t_like(a).prod(None), mode=mode) utt.assert_allclose(f(input), n_like(input).prod()) # assert len(f.maker.fgraph.apply_nodes) == nb_nodes[0] f = theano.function([a], t_like(a).prod([0, 1, 2]), mode=mode) utt.assert_allclose(f(input), n_like(input).prod()) # assert len(f.maker.fgraph.apply_nodes) == nb_nodes[0] for d in range(3): f = theano.function([a], t_like(a).prod(d), mode=mode) utt.assert_allclose(f(input), n_like(input).prod(d)) # assert len(f.maker.fgraph.apply_nodes) == nb_nodes[1] topo = f.maker.fgraph.toposort() assert topo[-1].op == T.alloc assert not any([isinstance(node.op, T.elemwise.Prod) for node in topo]) for i in range(3): f = theano.function([a], t_like(a).prod(i), mode=mode) utt.assert_allclose(f(input), n_like(input).prod(i)) # assert len(f.maker.fgraph.apply_nodes) == nb_nodes[2] topo = f.maker.fgraph.toposort() assert topo[-1].op == T.alloc assert not any([isinstance(node.op, T.elemwise.Prod) for node in topo]) backup = config.warn.sum_sum_bug config.warn.sum_sum_bug = False try: for d, dd in [(0, 0), (1, 0), (2, 0), (0, 1), (1, 1), (2, 1)]: f = theano.function([a], t_like(a). sum(d).sum(dd), mode=mode) utt.assert_allclose(f(input), n_like(input).sum(d).sum(dd)) assert len(f.maker.fgraph.apply_nodes) == nb_nodes[3] topo = f.maker.fgraph.toposort() assert topo[-1].op == T.alloc assert not any([isinstance(node.op, T.Sum) for node in topo]) finally: config.warn.sum_sum_bug = backup def test_local_sum_sum_int8(self): # Test that local_sum_sum works when combining two sums on an int8 array. # This is a regression test for ticket gh-356. x = tensor.tensor3(dtype='int8') y = x.sum(axis=0).sum(axis=1) backup = config.on_opt_error config.on_opt_error = 'raise' try: # This compilation would fail prior to fix. theano.function([x], y) finally: config.on_opt_error = backup def test_local_sum_sum_dtype(self): # Test that local_sum_sum works when specifying dtypes manually. x = tensor.tensor3(dtype='int8') y = x.sum(axis=0, dtype='int32').sum(axis=1, dtype='int64') backup = config.on_opt_error config.on_opt_error = 'raise' try: # This compilation would fail prior to fix. theano.function([x], y) finally: config.on_opt_error = backup def test_local_sum_prod_mul_by_scalar_stack_trace(self): # Test that stack trace is copied over correctly for local_sum_prod_mul_by_scalar. m0 = theano.compile.get_default_mode()\ .excluding('inplace_elemwise_opt')\ .including('canonicalize', 'specialize') vect = T.dvector() mat = T.dmatrix() scalar = T.dscalar() f = theano.function([vect, scalar], T.sum(vect * scalar), mode=m0) assert check_stack_trace(f, ops_to_check='all') f = theano.function([vect], T.sum(-vect), mode=m0) assert check_stack_trace(f, ops_to_check=[T.Sum]) f = theano.function([vect, scalar], T.elemwise.Prod()(vect * scalar), mode=m0) assert check_stack_trace(f, ops_to_check=[T.elemwise.Prod]) f = theano.function([vect], T.elemwise.Prod()(-vect), mode=m0) assert check_stack_trace(f, ops_to_check=[T.elemwise.Prod]) f = theano.function([mat, scalar], T.sum(mat * scalar), mode=m0) assert check_stack_trace(f, ops_to_check='all') f = theano.function([mat], T.sum(-mat), mode=m0) assert check_stack_trace(f, ops_to_check=[T.Sum]) class T_local_opt_alloc(unittest.TestCase): dtype = 'float32' def test_sum_upcast(self): s = theano.tensor.lscalar() a = theano.tensor.alloc(np.asarray(5, dtype=self.dtype), s, s) orig = theano.config.warn_float64 theano.config.warn_float64 = "raise" try: f = theano.function([s], a.sum()) f(5) finally: theano.config.warn_float64 = orig def test_prod_upcast(self): s = theano.tensor.lscalar() a = theano.tensor.alloc(np.asarray(5, dtype=self.dtype), s, s) orig = theano.config.warn_float64 theano.config.warn_float64 = "raise" try: f = theano.function([s], a.prod()) f(5) finally: theano.config.warn_float64 = orig @change_flags(on_opt_error='raise') def test_sum_bool_upcast(self): s = theano.tensor.lscalar() a = theano.tensor.alloc(np.asarray(True, dtype='bool'), s, s) f = theano.function([s], a.sum()) f(5) # test with user specified dtype f = theano.function([s], a.sum(dtype=self.dtype)) f(5) # test only 1 axis summed f = theano.function([s], a.sum(axis=0, dtype=self.dtype)) f(5) print(self.dtype) class T_local_opt_alloc_f16(T_local_opt_alloc): dtype = 'float16' class T_local_reduce(unittest.TestCase): def setUp(self): self.mode = theano.compile.get_default_mode().including( 'canonicalize', 'specialize', 'uncanonicalize', 'local_max_and_argmax') def test_local_reduce_broadcast_all_0(self): for fct in [tensor.sum, tensor.all, tensor.any, tensor.prod, tensor.max, tensor.min]: x = T.TensorType('int64', (True, True, True))() f = theano.function([x], [fct(x)], mode=self.mode) assert not any([ isinstance(node.op, T.CAReduce) for node in f.maker.fgraph.toposort()]) def test_local_reduce_broadcast_all_1(self): for fct in [tensor.sum, tensor.all, tensor.any, tensor.prod, tensor.max, tensor.min]: x = T.TensorType('int64', (True, True))() f = theano.function([x], [fct(x, axis=[0, 1])], mode=self.mode) assert not any([ isinstance(node.op, T.CAReduce) for node in f.maker.fgraph.toposort()]) def test_local_reduce_broadcast_some_0(self): for fct in [tensor.sum, tensor.all, tensor.any, tensor.prod, tensor.max, tensor.min]: x = T.TensorType('int64', (True, False, True))() f = theano.function([x], [fct(x, axis=[0, 1])], mode=self.mode) order = f.maker.fgraph.toposort() assert 1 == sum([isinstance(node.op, T.CAReduce) for node in order]) node = [node for node in order if isinstance(node.op, tensor.CAReduce)][0] op = node.op assert isinstance(op, T.CAReduce) # -- the leading broadcastable dimension has been dropped # by the local_reduce_broadcastable optimization # now summation is over the original x's dimension 1. assert node.inputs[0].ndim == 2, node assert op.axis == (0,), op.axis def test_local_reduce_broadcast_some_1(self): for fct in [tensor.sum, tensor.all, tensor.any, tensor.prod, tensor.max, tensor.min]: x = T.TensorType('int64', (True, True, True))() f = theano.function([x], [fct(x, axis=[0, 2])], mode=self.mode) assert not any([ isinstance(node.op, T.CAReduce) for node in f.maker.fgraph.toposort()]) def test_local_reduce_join(self): vx = matrix() vy = matrix() vz = matrix() x = np.asarray([[1, 0], [3, 4]], dtype=config.floatX) y = np.asarray([[4, 0], [2, 1]], dtype=config.floatX) z = np.asarray([[5, 0], [1, 2]], dtype=config.floatX) # Test different reduction scalar operation for out, res in [ (T.max((vx, vy), 0), np.max((x, y), 0)), (T.min((vx, vy), 0), np.min((x, y), 0)), (T.sum((vx, vy, vz), 0), np.sum((x, y, z), 0)), (T.prod((vx, vy, vz), 0), np.prod((x, y, z), 0)), (T.prod((vx, vy.T, vz), 0), np.prod((x, y.T, z), 0)), ]: f = theano.function([vx, vy, vz], out, on_unused_input='ignore', mode=self.mode) assert (f(x, y, z) == res).all(), out topo = f.maker.fgraph.toposort() assert len(topo) <= 2, out assert isinstance(topo[-1].op, T.Elemwise), out # Test different axis for the join and the reduction # We must force the dtype, of otherwise, this tests will fail # on 32 bit systems A = theano.shared(np.array([1, 2, 3, 4, 5], dtype='int64')) f = theano.function([], T.sum(T.stack([A, A]), axis=0), mode=self.mode) utt.assert_allclose(f(), [2, 4, 6, 8, 10]) topo = f.maker.fgraph.toposort() assert isinstance(topo[-1].op, T.Elemwise) # Test a case that was bugged in a old Theano bug try: old = theano.config.warn.reduce_join theano.config.warn.reduce_join = False f = theano.function([], T.sum(T.stack([A, A]), axis=1), mode=self.mode) finally: theano.config.warn.reduce_join = old utt.assert_allclose(f(), [15, 15]) topo = f.maker.fgraph.toposort() assert not isinstance(topo[-1].op, T.Elemwise) # This case could be optimized A = theano.shared(np.array([1, 2, 3, 4, 5]).reshape(5, 1)) f = theano.function([], T.sum(T.concatenate((A, A), axis=1), axis=1), mode=self.mode) utt.assert_allclose(f(), [2, 4, 6, 8, 10]) topo = f.maker.fgraph.toposort() assert not isinstance(topo[-1].op, T.Elemwise) A = theano.shared(np.array([1, 2, 3, 4, 5]).reshape(5, 1)) f = theano.function([], T.sum(T.concatenate((A, A), axis=1), axis=0), mode=self.mode) utt.assert_allclose(f(), [15, 15]) topo = f.maker.fgraph.toposort() assert not isinstance(topo[-1].op, T.Elemwise) # Test that the optimization does not crash in one case where it # is not applied. Reported at # https://groups.google.com/d/topic/theano-users/EDgyCU00fFA/discussion old = theano.config.warn.reduce_join try: theano.config.warn.reduce_join = False out = tensor.sum([vx, vy, vz], axis=None) f = theano.function([vx, vy, vz], out) finally: theano.config.warn.reduce_join = old class T_local_sum_prod_dimshuffle(unittest.TestCase): def setUp(self): self.mode = theano.compile.get_default_mode().including('canonicalize') def test_local_sum_div_dimshuffle(self): a = T.matrix('a') b = T.vector('b') c = T.tensor3('c') d = T.scalar('d') sum = tensor.sum sums = [ sum(a / d), sum(a / d.dimshuffle('x', 'x')), sum(a / d.dimshuffle('x', 'x'), axis=0), sum(a / d.dimshuffle('x', 'x'), axis=1), sum(b / d), sum(b / d.dimshuffle('x')), sum(c / d), sum(c / d.dimshuffle('x', 'x', 'x')), sum(c / d.dimshuffle('x', 'x', 'x'), axis=0), sum(c / d.dimshuffle('x', 'x', 'x'), axis=1), sum(c / d.dimshuffle('x', 'x', 'x'), axis=2), sum(a / b, axis=0), sum(a / b.dimshuffle(0, 'x'), axis=1), sum(a.dimshuffle(0, 1) / b.dimshuffle(0, 'x'), axis=1), sum(a.dimshuffle(1, 0) / b.dimshuffle(0, 'x'), axis=1), sum(c / a, axis=0), sum(c / a.dimshuffle(1, 0), axis=0), sum(c / a.dimshuffle(0, 'x', 1), axis=1), sum(c / a.dimshuffle(1, 'x', 0), axis=1), sum(c / a.dimshuffle(0, 1, 'x'), axis=2), sum(c / a.dimshuffle(1, 0, 'x'), axis=2), sum(c / b, axis=0), sum(c / b, axis=1), sum(c / b, axis=(0, 1)), sum(c / b.dimshuffle(0, 'x'), axis=0), sum(c / b.dimshuffle(0, 'x'), axis=2), sum(c / b.dimshuffle(0, 'x'), axis=(0, 2)), sum(c / b.dimshuffle(0, 'x', 'x'), axis=1), sum(c / b.dimshuffle(0, 'x', 'x'), axis=2), sum(c / b.dimshuffle(0, 'x', 'x'), axis=(1, 2)), sum(sum(c, axis=0) / b, axis=0), sum(sum(c, axis=1) / b, axis=0), ] rng = np.random.RandomState(utt.fetch_seed()) a_val = rng.randn(2, 2).astype(config.floatX) b_val = rng.randn(2).astype(config.floatX) c_val = rng.randn(2, 2, 2).astype(config.floatX) d_val = np.asarray(rng.randn(), config.floatX) backup = config.warn.sum_sum_bug, config.warn.sum_div_dimshuffle_bug config.warn.sum_sum_bug = False config.warn.sum_div_dimshuffle_bug = False try: for i, s in enumerate(sums): print(i) f = theano.function([a, b, c, d], s, mode=self.mode, on_unused_input='ignore') g = f.maker.fgraph.toposort() assert isinstance(g[-1].op.scalar_op, theano.scalar.basic.TrueDiv) f(a_val, b_val, c_val, d_val) finally: config.warn.sum_sum_bug, config.warn.sum_div_dimshuffle_bug =\ backup def test_local_prod_div_dimshuffle(self): a = T.matrix('a') b = T.vector('b') c = T.tensor3('c') e = T.matrix('e') d = T.scalar('d') prod = T.prod prods = [ prod(a / d), prod(a / d.dimshuffle('x', 'x')), prod(a / d.dimshuffle('x', 'x'), axis=0), prod(a / d.dimshuffle('x', 'x'), axis=1), prod(b / d), prod(b / d.dimshuffle('x')), prod(c / d), prod(c / d.dimshuffle('x', 'x', 'x')), prod(c / d.dimshuffle('x', 'x', 'x'), axis=0), prod(c / d.dimshuffle('x', 'x', 'x'), axis=1), prod(c / d.dimshuffle('x', 'x', 'x'), axis=2), prod(a / b, axis=0), prod(a / b.dimshuffle(0, 'x'), axis=1), prod(a.dimshuffle(0, 1) / b.dimshuffle(0, 'x'), axis=1), prod(a.dimshuffle(1, 0) / b.dimshuffle(0, 'x'), axis=1), prod(c / a, axis=0), prod(c / a.dimshuffle(1, 0), axis=0), prod(c / a.dimshuffle(0, 'x', 1), axis=1), prod(c / a.dimshuffle(1, 'x', 0), axis=1), prod(c / a.dimshuffle(0, 1, 'x'), axis=2), prod(c / a.dimshuffle(1, 0, 'x'), axis=2), prod(c / b, axis=0), prod(c / b, axis=1), prod(c / b, axis=(0, 1)), prod(c / b.dimshuffle(0, 'x'), axis=0), prod(c / b.dimshuffle(0, 'x'), axis=2), prod(c / b.dimshuffle(0, 'x'), axis=(0, 2)), prod(c / b.dimshuffle(0, 'x', 'x'), axis=1), prod(c / b.dimshuffle(0, 'x', 'x'), axis=2), prod(c / b.dimshuffle(0, 'x', 'x'), axis=(1, 2)), prod(c / b.dimshuffle(0, 'x', 'x'), axis=(0, 1)), prod(c / b.dimshuffle(0, 'x', 'x'), axis=(1, 0)), prod(prod(c, axis=0) / b, axis=0), prod(prod(c, axis=1) / b, axis=0)] rng = np.random.RandomState(utt.fetch_seed()) a_val = rng.randn(2, 2).astype(config.floatX) b_val = rng.randn(2).astype(config.floatX) c_val = rng.randn(2, 2, 2).astype(config.floatX) d_val = np.asarray(rng.randn(), config.floatX) default_mode = theano.compile.mode.get_default_mode() # FusionOptimizer is included to make sure that expected_outer_operator # remains the same for all optimization modes. mode_with_opt = default_mode.including('local_sum_prod_div_dimshuffle', 'FusionOptimizer') mode_without_opt = default_mode.excluding('local_sum_prod_div_dimshuffle') # Numerical tests: tests whether the numerical values with and without # optimizer are equal or not. for i, s in enumerate(prods): f = theano.function([a, b, c, d], s, on_unused_input='ignore', mode=mode_without_opt) g = theano.function([a, b, c, d], s, on_unused_input='ignore', mode=mode_with_opt) utt.assert_allclose(f(a_val, b_val, c_val, d_val), g(a_val, b_val, c_val, d_val)) # Logical tests: tests whether the optimizer has been appplied or not # by checking graph structure. prods = [ prod(a / e), prod(a / d), prod(a / d.dimshuffle('x', 'x')), prod(c / d.dimshuffle('x', 'x', 'x'), axis=1), prod(a.dimshuffle(1, 0) / b.dimshuffle(0, 'x'), axis=1), prod(c / b.dimshuffle(0, 'x', 'x'), axis=(1, 0)), prod(prod(c, axis=1) / b, axis=0), prod(prod(c, axis=(1, 2)) / b, axis=0)] expected_outer_operator = [theano.scalar.basic.Mul, theano.scalar.basic.Composite, theano.scalar.basic.Composite, theano.scalar.basic.TrueDiv, theano.scalar.basic.Composite, theano.scalar.basic.Mul, theano.scalar.basic.Composite, theano.scalar.basic.Mul] for i, s in enumerate(prods): g = theano.function([a, b, c, d, e], s, on_unused_input='ignore', mode=mode_with_opt) assert isinstance(g.maker.fgraph.toposort()[-1].op.scalar_op, expected_outer_operator[i]) # TODO: # test_local_sum_prod_dimshuffle (a * b * c) # test_local_sum_divprod_dimshuffle ((a * b) / (c * d)) class TestMakeVector(utt.InferShapeTester): def setUp(self): super(TestMakeVector, self).setUp() def test_make_vector(self): b = T.bscalar() i = T.iscalar() d = T.dscalar() # TODO: draw random values instead. Not really important. val = {b: 2, i: -3, d: 0.7} # Should work for (dtype, inputs) in [("int8", (b, b)), ("int32", (i, b)), ("int32", (b, i)), ("float64", (b, i)), ("float64", (b, d)), ("float64", (d, i)), ("float64", ()), ("int64", ()), ]: mv = opt.MakeVector(dtype=dtype)(*inputs) assert mv.dtype == dtype f = theano.function([b, i, d], mv, on_unused_input='ignore') f(val[b], val[i], val[d]) s = mv.sum() gb = T.grad(s, b, disconnected_inputs='ignore') gi = T.grad(s, i, disconnected_inputs='ignore') gd = T.grad(s, d, disconnected_inputs='ignore') # print 'gb =', gb # print 'gi =', gi # print 'gd =', gd g = theano.function([b, i, d], [gb, gi, gd]) g_val = g(val[b], val[i], val[d]) # print 'g_val =', g_val if dtype in tensor.int_dtypes: # The gradient should be 0 utt.assert_allclose(g_val, 0) else: for var, grval in zip((b, i, d), g_val): float_inputs = [] if var.dtype in tensor.int_dtypes: pass # Currently we don't do any checks on these variables # verify_grad doesn't support integer inputs yet # however, the gradient on them is *not* defined to # be 0 elif var not in inputs: assert grval == 0 else: float_inputs.append(var) # Build a function that takes float_inputs, use fix values for the # other inputs, and returns the MakeVector. Use it for verify_grad. if float_inputs: def fun(*fl_inputs): f_inputs = [] for var in f_inputs: if var in fl_inputs: # use symbolic variable f_inputs.append(var) else: # use constant value f_inputs.append(val[var]) return opt.MakeVector(dtype=dtype)(*f_inputs) utt.verify_grad(fun, [val[ri] for ri in float_inputs]) # should fail for (dtype, inputs) in [("int8", (b, i)), ("int8", (i, b)), ("int8", (b, d)), ("int8", (i, i)), ("int32", (d, i)), ("int32", (i, d)), ("float32", (i, d)), ]: try: opt.MakeVector(dtype=dtype)(*inputs) raise Exception("Theano should have raised an error") except AssertionError: pass def test_infer_shape(self): adscal = dscalar() bdscal = dscalar() aiscal = iscalar() biscal = iscalar() ciscal = iscalar() discal = iscalar() adscal_val = np.random.rand() bdscal_val = np.random.rand() aiscal_val = np.random.randint(10) biscal_val = np.random.randint(10) ciscal_val = np.random.randint(10) discal_val = np.random.randint(10) self._compile_and_check([adscal, aiscal], [MakeVector('float64')(adscal, aiscal)], [adscal_val, aiscal_val], MakeVector) self._compile_and_check([adscal, bdscal, aiscal], [MakeVector('float64')(adscal, bdscal, aiscal)], [adscal_val, bdscal_val, aiscal_val], MakeVector) self._compile_and_check([aiscal, biscal, ciscal, discal], [MakeVector('int32')(aiscal, biscal, ciscal, discal)], [aiscal_val, biscal_val, ciscal_val, discal_val], MakeVector) def test_local_join_1(): # test for vector a = tensor.vector('a') s = tensor.stack([a]) f = function([a], s, mode=mode_opt) val = f([1]) assert np.all(val == [1]) e = f.maker.fgraph.toposort() assert len([n for n in e if isinstance(n.op, Join)]) == 0 assert f.maker.fgraph.outputs[0].dtype == config.floatX # test for matrix join(0,a) a = tensor.matrix('a') s = join(0, a) f = function([a], s, mode=mode_opt) val = f([[1]]) assert np.all(val == [[1]]) e = f.maker.fgraph.toposort() assert len([n for n in e if isinstance(n.op, Join)]) == 0 assert f.maker.fgraph.outputs[0].dtype == config.floatX # test for matrix join(1,a) s = join(1, a) f = function([a], s, mode=mode_opt) val = f([[1]]) assert np.all(val == [[1]]) e = f.maker.fgraph.toposort() assert len([n for n in e if isinstance(n.op, Join)]) == 0 assert f.maker.fgraph.outputs[0].dtype == config.floatX # test we don't apply when their is 2 inputs s = join(1, a, a) f = function([a], s, mode=mode_opt) val = f([[1]]) assert np.all(val == [[1]]) e = f.maker.fgraph.toposort() assert len([n for n in e if isinstance(n.op, Join)]) == 1 assert f.maker.fgraph.outputs[0].dtype == config.floatX def test_local_join_empty(): # test for vector, vector, empty to vector empty_vec = np.asarray([], dtype=config.floatX) a = tensor.vector('a') s = tensor.join(0, a, a, empty_vec) f = function([a], s, mode=mode_opt) val = f([1]) assert np.all(val == [1]) e = f.maker.fgraph.toposort() assert len([n for n in e if isinstance(n.op, Join)]) == 1 assert all([not isinstance(n.op, Join) or len(n.inputs) == 3 for n in e if isinstance(n.op, Join)]) assert f.maker.fgraph.outputs[0].dtype == config.floatX # test for matrix join(1,a) empty_mat = np.asarray([[]], dtype=config.floatX) m = tensor.matrix('m') s = join(1, empty_mat, m, m, m) f = function([m], s, mode=mode_opt) val = f([[1]]) assert np.all(val == [[1]]) e = f.maker.fgraph.toposort() assert len([n for n in e if isinstance(n.op, Join)]) == 1 assert all([not isinstance(n.op, Join) or len(n.inputs) == 4 for n in e if isinstance(n.op, Join)]) assert f.maker.fgraph.outputs[0].dtype == config.floatX # test for vector, vector, empty to matrix # We can't optimize this case. s = tensor.stack([a, a, empty_vec]) f = function([a], s, mode=mode_opt) val = f([]) assert np.all(val == [1]) e = f.maker.fgraph.toposort() assert len([n for n in e if isinstance(n.op, Join)]) == 1 assert all([not isinstance(n.op, Join) or len(n.inputs) == 4 for n in e if isinstance(n.op, Join)]) assert f.maker.fgraph.outputs[0].dtype == config.floatX # test for matrix join(0,a) # We can't optimize this case. s = join(0, m, np.asarray([[2.]], dtype=config.floatX), m) f = function([m], s, mode=mode_opt) val = f([[1]]) assert np.all(val == [[1], [2], [1]]) e = f.maker.fgraph.toposort() assert len([n for n in e if isinstance(n.op, Join)]) == 1 assert all([not isinstance(n.op, Join) or len(n.inputs) == 4 for n in e if isinstance(n.op, Join)]) assert f.maker.fgraph.outputs[0].dtype == config.floatX def test_local_join_make_vector(): a, b, c, d, e = tensor.scalars('abcde') v = tensor.vector('v') mv = MakeVector(config.floatX) s = tensor.join(0, mv(a), v, mv(b, c), mv(d, e)) f = function([a, b, c, d, e, v], s, mode=mode_opt) theano.printing.debugprint(f) val = f(1, 2, 3, 4, 6, [7, 8]) assert np.all(val == [1, 7, 8, 2, 3, 4, 6]) e = f.maker.fgraph.toposort() assert len([n for n in e if isinstance(n.op, Join)]) == 1 assert all([not isinstance(n.op, Join) or len(n.inputs) == 4 for n in e if isinstance(n.op, Join)]) assert f.maker.fgraph.outputs[0].dtype == config.floatX assert check_stack_trace(f, ops_to_check='all') def test_local_add_specialize(): # test of non-zero dimension a = tensor.vector() s = tensor.add(tensor.zeros_like(a)) assert local_add_specialize.transform(s.owner) # test of 0-d a = tensor.scalar() s = tensor.add(tensor.zeros_like(a)) assert local_add_specialize.transform(s.owner) # Test when the 0 input is forcing upcasting a = tensor.constant(0, dtype='int64') b = tensor.constant(1, dtype='int32') s = a + b transformed = local_add_specialize.transform(s.owner) assert transformed assert transformed[0].type == s.type def test_local_tensor_scalar_tensor(): dtypes = ['int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', 'float32', 'float64', 'complex64', 'complex128' ] for dtype in dtypes: t_type = TensorType(dtype=dtype, broadcastable=()) t = t_type() s = tensor.scalar_from_tensor(t) t2 = tensor.tensor_from_scalar(s) f = function([t], t2, mode=mode_opt) e = f.maker.fgraph.toposort() cast_nodes = [n for n in e if isinstance(n.op, (tensor.TensorFromScalar, tensor.ScalarFromTensor))] assert len(cast_nodes) == 0 f(0) def test_local_scalar_tensor_scalar(): dtypes = ['int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', 'float32', 'float64', 'complex64', 'complex128' ] for dtype in dtypes: s_type = theano.scalar.Scalar(dtype=dtype) s = s_type() t = tensor.tensor_from_scalar(s) s2 = tensor.scalar_from_tensor(t) f = function([s], s2, mode=mode_opt) e = f.maker.fgraph.toposort() cast_nodes = [n for n in e if isinstance(n.op, (tensor.TensorFromScalar, tensor.ScalarFromTensor))] assert len(cast_nodes) == 0 f(0) def test_local_div_to_inv(): num_len_s = tensor.lscalar('num_len') denom_s = tensor.scalar('denom') num_v = tensor.alloc(1, num_len_s) denom_m = denom_s.dimshuffle('x', 'x') out = num_v / denom_m assert np.all(out.broadcastable == (True, False)) f = theano.function([num_len_s, denom_s], out) out_val = f(3, 2.) assert out_val.shape == (1, 3) utt.assert_allclose(out_val, 0.5) def test_local_useless_split(): x = tensor.matrix('x') splits = tensor.ivector('splits') opt = tensor.split(x, splits, n_splits=1) nonopt = tensor.split(x, splits, n_splits=3) mode = compile.get_default_mode().including("local_useless_split") f_opt = theano.function([x, splits], opt, mode=mode) f_nonopt = theano.function([x, splits], nonopt, mode=mode) f_opt(np.random.rand(4, 4).astype(config.floatX), [4]) f_nonopt(np.random.rand(4, 4).astype(config.floatX), [1, 2, 1]) graph_opt = f_opt.maker.fgraph.toposort() graph_nonopt = f_nonopt.maker.fgraph.toposort() assert isinstance(graph_opt[-1].op, DeepCopyOp) assert len(graph_nonopt) == 1 assert isinstance(graph_nonopt[0].op, tensor.Split) assert check_stack_trace(f_opt, ops_to_check=[Assert]) assert check_stack_trace(f_nonopt, ops_to_check='all') def test_local_flatten_lift(): for i in xrange(1, 4): x = tensor.tensor4() out = tensor.flatten(T.exp(x), i) assert out.ndim == i mode = compile.mode.get_default_mode() mode = mode.including('local_reshape_lift') f = theano.function([x], out, mode=mode) x_np = np.random.rand(5, 4, 3, 2).astype(config.floatX) out_np = f(x_np) topo = f.maker.fgraph.toposort() shape_out_np = tuple(x_np.shape[:i - 1]) + (np.prod(x_np.shape[i - 1:]),) assert shape_out_np == out_np.shape reshape_nodes = [n for n in topo if isinstance(n.op, tensor.Reshape)] assert (len(reshape_nodes) == 1 and tensor.is_flat(reshape_nodes[0].outputs[0], ndim=i)) assert isinstance(topo[-1].op, tensor.Elemwise) class Test_Reshape(unittest.TestCase): def setUp(self): self.mode = mode_opt self.op = tensor.Reshape def test_local_reshape(self): a = tensor.fmatrix() b = self.op(3)(a, [2, 3, 4]) c = self.op(1)(b, [24]) f = theano.function([a], c, mode=self.mode) topo = f.maker.fgraph.toposort() assert sum(isinstance(node.op, self.op) for node in topo) == 1 # Check stack trace self.assertTrue(check_stack_trace(f, ops_to_check=[self.op])) class Test_local_useless_reshape(unittest.TestCase): def setUp(self): self.rng = np.random.RandomState(utt.fetch_seed()) def test_0(self): mode = theano.compile.get_default_mode().including( 'local_useless_reshape') i = T.iscalar('i') m = theano.tensor.mgrid[0:i, ] f = theano.function([i], m, mode=mode) topo = f.maker.fgraph.toposort() assert not any(isinstance(n.op, tensor.basic.Reshape) for n in topo) def test_1(self): x = theano.tensor.matrix('x') r = x.reshape(x.shape) m0 = theano.compile.get_default_mode() m1 = m0.including('local_useless_reshape') f1 = theano.function([x], r, mode=m1) topo = f1.maker.fgraph.toposort() assert not any(isinstance(n.op, tensor.basic.Reshape) for n in topo) m2 = m1.excluding('ShapeOpt') f2 = theano.function([x], r, mode=m2) topo = f2.maker.fgraph.toposort() assert not any(isinstance(n.op, tensor.basic.Reshape) for n in topo) # We do not need tests checking that stack traces are copied over, # because local_useless_reshape only removes nodes from the graph def test_2(self): x = theano.tensor.matrix('x') r = x.reshape([Shape_i(i)(x) for i in xrange(x.ndim)]) m0 = theano.compile.get_default_mode() m1 = m0.including('local_useless_reshape') f1 = theano.function([x], r, mode=m1) topo = f1.maker.fgraph.toposort() assert not any(isinstance(n.op, tensor.basic.Reshape) for n in topo) m2 = m1.excluding('ShapeOpt') f2 = theano.function([x], r, mode=m2) topo = f2.maker.fgraph.toposort() assert not any(isinstance(n.op, tensor.basic.Reshape) for n in topo) def test_m1(self): x = theano.tensor.matrix('x') r = x.reshape((x.shape[0], -1)) m0 = theano.compile.get_default_mode() m1 = m0.including('local_useless_reshape') f1 = theano.function([x], r, mode=m1) topo = f1.maker.fgraph.toposort() assert not any(isinstance(n.op, tensor.basic.Reshape) for n in topo) m2 = m1.excluding('ShapeOpt') f2 = theano.function([x], r, mode=m2) topo = f2.maker.fgraph.toposort() assert not any(isinstance(n.op, tensor.basic.Reshape) for n in topo) class Test_local_reshape_to_dimshuffle(unittest.TestCase): def setUp(self): self.rng = np.random.RandomState(utt.fetch_seed()) def test_1(self): reshape_lift = out2in(local_reshape_to_dimshuffle) useless_reshape = out2in(local_useless_reshape) x = shared(self.rng.randn(4,)) y = shared(self.rng.randn(5, 6)) reshape_x = tensor.reshape(x, (1, 4)) reshape_y = tensor.reshape(y, (1, 5, 1, 6, 1, 1)) g = FunctionGraph([x, y], [reshape_x, reshape_y]) self.assertTrue(str(g) == ("[Reshape{2}" "(, " "TensorConstant{[1 4]}), " "Reshape{6}" "(, " "TensorConstant{[1 5 1 6 1 1]})]")) reshape_lift.optimize(g) useless_reshape.optimize(g) self.assertTrue(str(g) == "[InplaceDimShuffle{x,0}" "(), " "InplaceDimShuffle{x,0,x,1,x,x}" "(Reshape{2}(, " "TensorConstant{[5 6]}))]") # Check stacktrace was copied over correctly after opt was applied assert check_stack_trace(g, ops_to_check=(T.DimShuffle, T.Reshape)) def test_local_reshape_lift(): x = tensor.tensor4() out = T.exp(x).reshape([x.size]) assert out.ndim == 1 mode = compile.mode.get_default_mode() mode = mode.including('local_reshape_lift') f = theano.function([x], out, mode=mode) f(np.random.rand(5, 4, 3, 2).astype(config.floatX)) topo = f.maker.fgraph.toposort() assert isinstance(topo[-2].op, tensor.Reshape) assert isinstance(topo[-1].op, tensor.Elemwise) # Check stacktrace was copied over correctly after opt was applied assert check_stack_trace(f, ops_to_check='last') class Test_lift_transpose_through_dot(unittest.TestCase): def simple_optimize(self, g): out2in(opt.local_useless_elemwise).optimize(g) out2in(opt.local_lift_transpose_through_dot).optimize(g) out2in(opt.local_useless_elemwise).optimize(g) return g def test_matrix_matrix(self): a, b = matrices('ab') g = self.simple_optimize(FunctionGraph([a, b], [tensor.dot(a, b).T])) sg = '[dot(InplaceDimShuffle{1,0}(b), InplaceDimShuffle{1,0}(a))]' assert str(g) == sg, (str(g), sg) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(g, ops_to_check='all')) def test_row_matrix(self): a = vector('a') b = matrix('b') g = optimize(FunctionGraph( [a, b], [tensor.dot(a.dimshuffle('x', 0), b).T]), level='stabilize') sg = '[dot(InplaceDimShuffle{1,0}(b), InplaceDimShuffle{0,x}(a))]' assert str(g) == sg, (str(g), sg) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(g, ops_to_check='all')) def test_matrix_col(self): a = vector('a') b = matrix('b') g = optimize(FunctionGraph( [a, b], [tensor.dot(b, a.dimshuffle(0, 'x')).T]), level='stabilize') sg = '[dot(InplaceDimShuffle{x,0}(a), InplaceDimShuffle{1,0}(b))]' assert str(g) == sg, (str(g), sg) # Check stacktrace was copied over correctly after opt was applied self.assertTrue(check_stack_trace(g, ops_to_check='all')) def test_local_upcast_elemwise_constant_inputs(): s = dvector("s") x = tensor.sum(tensor.log(10 ** s)) f = function([s], [tensor.grad(x, s)]) f([-42, -2.1, -1, -0.5, 0, 0.2, 1, 2, 12]) # This test a corner where the optimization should not be applied. old = theano.config.floatX theano.config.floatX = 'float32' try: v = lvector() function([v], theano.tensor.basic.true_div(v, 2)) finally: theano.config.floatX = old class TestShape_i(utt.InferShapeTester): def setUp(self): super(TestShape_i, self).setUp() def test_perform(self): advec = vector() advec_val = np.random.rand(3).astype(config.floatX) f = function([advec], Shape_i(0)(advec)) out = f(advec_val) utt.assert_allclose(out, advec_val.shape[0]) admat = matrix() admat_val = np.random.rand(4, 3).astype(config.floatX) for i in xrange(2): f = function([admat], Shape_i(i)(admat)) out = f(admat_val) utt.assert_allclose(out, admat_val.shape[i]) def test_infer_shape(self): admat = matrix() admat_val = np.random.rand(3, 4).astype(config.floatX) self._compile_and_check([admat], [Shape_i(0)(admat)], [admat_val], Shape_i) self._compile_and_check([admat], [Shape_i(1)(admat)], [admat_val], Shape_i) class TestShapeFeature(unittest.TestCase): def test_scalar(self): x = scalar() cst = T.constant(1).clone() o = x + cst fgraph = FunctionGraph([x], [o], clone=False) shape_feature = opt.ShapeFeature() fgraph.attach_feature(shape_feature) assert shape_feature.same_shape(x, o) def test_vector(self): x = vector() cst = T.constant(1).clone() o = x + cst fgraph = FunctionGraph([x], [o], clone=False) shape_feature = opt.ShapeFeature() fgraph.attach_feature(shape_feature) assert shape_feature.same_shape(x, o) def test_vector2(self): x = vector() y = vector() o = x + y fgraph = FunctionGraph([x, y], [o], clone=False) shape_feature = opt.ShapeFeature() fgraph.attach_feature(shape_feature) assert shape_feature.same_shape(x, o) # The following case isn't implemented assert not shape_feature.same_shape(y, o) def test_vector_dim(self): x = vector() y = vector() o = x + y fgraph = FunctionGraph([x, y], [o], clone=False) shape_feature = opt.ShapeFeature() fgraph.attach_feature(shape_feature) assert shape_feature.same_shape(x, o, 0, 0) # The following case isn't implemented assert not shape_feature.same_shape(y, o, 0, 0) def test_vector_dim_err(self): x = vector() y = vector() o = x + y fgraph = FunctionGraph([x, y], [o], clone=False) shape_feature = opt.ShapeFeature() fgraph.attach_feature(shape_feature) self.assertRaises(IndexError, shape_feature.same_shape, x, o, 1, 0) self.assertRaises(IndexError, shape_feature.same_shape, x, o, 0, 1) def test_assert_op_gradient(): x = T.vector('x') assert_op = Assert() cost = T.sum(assert_op(x, x.size < 2)) grad = T.grad(cost, x) func = theano.function([x], grad) x_val = np.ones(shape=(1,), dtype=theano.config.floatX) assert func(x_val) == 1 class TestIntDivByOne(unittest.TestCase): def setUp(self): self.mode = theano.compile.mode.get_default_mode() self.mode = self.mode.including('local_intdiv_by_one') def test1(self): # Tests removing the extra floor_div by 1 introduced by # local_subtensor_merge optimization y = T.tensor4('y') self.mode = self.mode.excluding('fusion') f = theano.function([y], y[::-1][::-1], mode=self.mode) graph = f.maker.fgraph.toposort() divs = [node for node in graph if isinstance(node.op, T.elemwise.Elemwise) and isinstance(node.op.scalar_op, theano.scalar.IntDiv)] assert len(divs) == 0 def test2(self): # Simple test case for removing dividing by 1 y = T.tensor4('y') z = y // 1 f = theano.function([y], z, mode=self.mode) graph = f.maker.fgraph.toposort() divs = [node for node in graph if isinstance(node.op, T.elemwise.Elemwise) and isinstance(node.op.scalar_op, theano.scalar.IntDiv)] assert len(divs) == 0 def test3(self): # Simple test case for removing dividing by a tensor of ones y = T.tensor4('y') z = y // np.ones((2, 2, 2, 2)) f = theano.function([y], z, mode=self.mode) graph = f.maker.fgraph.toposort() divs = [node for node in graph if isinstance(node.op, T.elemwise.Elemwise) and isinstance(node.op.scalar_op, theano.scalar.IntDiv)] assert len(divs) == 0 def test_local_zero_div(): # Tests 0/x -> 0 for t in (T.scalar, T.ivector, T.ftensor4): x = t('x') for op in (T.int_div, T.true_div): y = op(0, x) g = optimize(FunctionGraph([x], [y])) # the division should be gone divs = [node for node in g.toposort() if isinstance(node.op, T.elemwise.Elemwise) and isinstance(node.op.scalar_op, type(op.scalar_op))] assert len(divs) == 0 # the output type should match the unoptimized one output = g.outputs[0] assert output.ndim == y.ndim assert output.type == y.type # and the output should be zero assert theano.tensor.get_scalar_constant_value(output) == 0 def test_local_sumsqr2dot(): G = matrix('G') W = matrix('W') y = T.sqr(W.dimshuffle('x', 0, 1) * G.dimshuffle(0, 'x', 1)).sum(axis=(1, 2)) MODE = theano.compile.get_default_mode().including('local_sumsqr2dot') f = function([W, G], y, mode=MODE) w_val = np.random.rand(4, 3).astype(config.floatX) g_val = np.random.rand(5, 3).astype(config.floatX) f_val = f(w_val, g_val) f_test = np.dot(np.square(g_val), np.square(w_val).sum(axis=0)) utt.assert_allclose(f_val, f_test) assert any(isinstance(n.op, (tensor.basic.Dot, tensor.blas.Dot22, tensor.blas.Gemv, tensor.blas_c.CGemv)) for n in f.maker.fgraph.toposort()) def test_local_expm1(): x = matrix('x') u = T.scalar('u') y = T.exp(x) - 1. z = T.exp(x) - 2. t = T.exp(x) - x s = T.exp(u) - np.ones((4, 3)).astype(config.floatX) MODE = theano.compile.get_default_mode().including('local_expm1') f = function([x], y, mode=MODE) g = function([x], z, mode=MODE) h = function([x], t, mode=MODE) r = function([u], s, mode=MODE) x_val = np.random.rand(4, 3).astype(config.floatX) f_val = f(x_val) f_test = function([x], T.expm1(x), mode=MODE) utt.assert_allclose(f_val, f_test(x_val)) assert any(isinstance(n.op, T.Elemwise) and isinstance(n.op.scalar_op, theano.scalar.basic.Expm1) for n in f.maker.fgraph.toposort()) assert not any(isinstance(n.op, T.Elemwise) and isinstance(n.op.scalar_op, theano.scalar.basic.Expm1) for n in g.maker.fgraph.toposort()) assert not any(isinstance(n.op, T.Elemwise) and isinstance(n.op.scalar_op, theano.scalar.basic.Expm1) for n in h.maker.fgraph.toposort()) assert not any(isinstance(n.op, T.Elemwise) and isinstance(n.op.scalar_op, theano.scalar.basic.Expm1) for n in r.maker.fgraph.toposort()) def test_local_merge_alloc(): # Add this opt to the default mode, # otherwise, FAST_COMPILE fails. default_mode = theano.compile.mode.get_default_mode() opt_mode = default_mode.including("local_merge_alloc") x = T.iscalar('x') y = T.iscalar('y') y2 = T.iscalar('y2') z = T.iscalar('z') w = T.iscalar('w') m = T.fscalar('m') # case 1 # Alloc(Alloc(m, x, 1, 1, 1), x, y, z, w) -> Alloc(m, x, y, z, w) output = T.alloc(T.alloc(m, 1, y, 1, 1), x, y, z, w) f = theano.function([m, x, y, z, w], output, mode=opt_mode) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, T.Alloc) o = f(0., 1, 2, 3, 4) assert o.shape == (1, 2, 3, 4) # case 2 # Alloc(Alloc(m, y, 1, 1), x, y, z, w) -> Alloc(m, x, y, z, w) output = T.alloc(T.alloc(m, y, 1, 1), x, y, z, w) f = theano.function([m, x, y, z, w], output, mode=opt_mode) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, T.Alloc) o = f(0., 1, 2, 3, 4) assert o.shape == (1, 2, 3, 4) # case 3 # Alloc(Alloc(m, y1, 1, 1), x, y2, z, w) -> # Alloc(m, x, assert(y1, y1==y2), z, w) output = T.alloc(T.alloc(m, y, 1, 1), x, y2, z, w) f = theano.function([m, x, y, y2, z, w], output, mode=opt_mode) topo = f.maker.fgraph.toposort() assert len(topo) == 3 assert isinstance(topo[-2].op, T.opt.Assert) assert isinstance(topo[-1].op, T.Alloc) o = f(0., 1, 2, 2, 3, 4) assert o.shape == (1, 2, 3, 4) assert_raises((AssertionError, ValueError), f, 0., 1, 2, 5, 3, 4) def test_local_useless_alloc(): useless_alloc = out2in(local_useless_alloc) merge_alloc = out2in(local_merge_alloc) x = T.iscalar('x') y = T.iscalar('y') y2 = T.iscalar('y2') z = T.iscalar('z') w = T.iscalar('w') m = T.fscalar('m') # case 1 # Alloc(Alloc(m, x, 1, 1, 1), x, y, z, w) -> Alloc(m, x, y, z, w) output = T.alloc(T.alloc(m, 1, y, 1, 1), x, y, z, w) g = FunctionGraph([m, x, y, z, w], [output]) useless_alloc.optimize(g) merge_alloc.optimize(g) useless_alloc.optimize(g) topo = g.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, T.Alloc) # case 2 # Alloc(Alloc(m, y, 1, 1), x, y, z, w) -> Alloc(m, x, y, z, w) output = T.alloc(T.alloc(m, y, 1, 1), x, y, z, w) g = FunctionGraph([m, x, y, z, w], [output]) useless_alloc.optimize(g) merge_alloc.optimize(g) useless_alloc.optimize(g) topo = g.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, T.Alloc) # case 3 # Alloc(Alloc(m, y1, 1, 1), x, y2, z, w) -> # Alloc(m, x, assert(y1, y1==y2), z, w) output = T.alloc(T.alloc(m, y, 1, 1), x, y2, z, w) g = FunctionGraph([m, x, y, y2, z, w], [output]) useless_alloc.optimize(g) merge_alloc.optimize(g) useless_alloc.optimize(g) topo = g.toposort() assert len(topo) == 3 assert isinstance(topo[-2].op, T.opt.Assert) assert isinstance(topo[-1].op, T.Alloc) def compile_graph_log_sum_exp(x, axis, dimshuffle_op=None): sum_exp = T.sum(T.exp(x), axis=axis) if dimshuffle_op: sum_exp = dimshuffle_op(sum_exp) y = T.log(sum_exp) MODE = theano.compile.get_default_mode().including('local_log_sum_exp') return function([x], y, mode=MODE) def check_max_log_sum_exp(x, axis, dimshuffle_op=None): f = compile_graph_log_sum_exp(x, axis, dimshuffle_op) fgraph = f.maker.fgraph.toposort() for node in fgraph: if (hasattr(node.op, 'scalar_op') and node.op.scalar_op == theano.scalar.basic.maximum): return # in mode FAST_COMPILE, the optimisations don't replace the # MaxAndArgmax op. if isinstance(node.op, theano.tensor.MaxAndArgmax): return raise Exception('No maximum detected after log_sum_exp optimisation') def test_local_log_sum_exp1(): # Tests if optimization is applied by checking the presence of the maximum x = tensor3('x') check_max_log_sum_exp(x, axis=(0,), dimshuffle_op=None) check_max_log_sum_exp(x, axis=(1,), dimshuffle_op=None) check_max_log_sum_exp(x, axis=(2,), dimshuffle_op=None) check_max_log_sum_exp(x, axis=(0, 1), dimshuffle_op=None) check_max_log_sum_exp(x, axis=(0, 1, 2), dimshuffle_op=None) # If a transpose is applied to the sum transpose_op = DimShuffle((False, False), (1, 0)) check_max_log_sum_exp(x, axis=2, dimshuffle_op=transpose_op) # If the sum is performed with keepdims=True x = TensorType(dtype='floatX', broadcastable=(False, True, False))('x') sum_keepdims_op = x.sum(axis=(0, 1), keepdims=True).owner.op check_max_log_sum_exp(x, axis=(0, 1), dimshuffle_op=sum_keepdims_op) def test_local_log_sum_exp2(): # Tests if the optimization works (result is correct) around 1.0 x = tensor3('x') x_val = 1.0 + np.random.rand(4, 3, 2).astype(config.floatX) / 10.0 f = compile_graph_log_sum_exp(x, axis=(1,)) naive_ret = np.log(np.sum(np.exp(x_val), axis=1)) optimised_ret = f(x_val) assert np.allclose(naive_ret, optimised_ret) # If a transpose is applied transpose_op = DimShuffle((False, False), (1, 0)) f = compile_graph_log_sum_exp(x, axis=(1,), dimshuffle_op=transpose_op) naive_ret = np.log(np.sum(np.exp(x_val), axis=1).T) optimised_ret = f(x_val) assert np.allclose(naive_ret, optimised_ret) def test_local_log_sum_exp3(): # Tests if the optimization works (result is correct) for extreme value 100 x = vector('x') f = compile_graph_log_sum_exp(x, axis=0) x_val = np.array([-100., 100.]).astype(config.floatX) optimised_ret = f(x_val) assert np.allclose(optimised_ret, 100.) Theano-1.0.0/theano/tensor/tests/test_slinalg.py00006000163562€ÿÿÿþ0000003551513203335532023274 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import unittest import numpy as np import numpy.linalg from numpy.testing import assert_array_almost_equal from numpy.testing import dec, assert_array_equal, assert_allclose from numpy import inf import itertools import theano from theano import tensor, function, grad from theano.tensor.basic import _allclose from theano.tests.test_rop import break_op from theano.tests import unittest_tools as utt from theano import config from theano.tensor.slinalg import ( Cholesky, cholesky, CholeskyGrad, Solve, solve, Eigvalsh, EigvalshGrad, eigvalsh, expm, kron) from theano.tests.unittest_tools import attr from nose.plugins.skip import SkipTest from nose.tools import assert_raises try: import scipy.linalg imported_scipy = True except ImportError: # some ops (e.g. Cholesky, Solve, A_Xinv_b) won't work imported_scipy = False def check_lower_triangular(pd, ch_f): ch = ch_f(pd) assert ch[0, pd.shape[1] - 1] == 0 assert ch[pd.shape[0] - 1, 0] != 0 assert np.allclose(np.dot(ch, ch.T), pd) assert not np.allclose(np.dot(ch.T, ch), pd) def check_upper_triangular(pd, ch_f): ch = ch_f(pd) assert ch[4, 0] == 0 assert ch[0, 4] != 0 assert np.allclose(np.dot(ch.T, ch), pd) assert not np.allclose(np.dot(ch, ch.T), pd) def test_cholesky(): if not imported_scipy: raise SkipTest("Scipy needed for the Cholesky op.") rng = np.random.RandomState(utt.fetch_seed()) r = rng.randn(5, 5).astype(config.floatX) pd = np.dot(r, r.T) x = tensor.matrix() chol = cholesky(x) # Check the default. ch_f = function([x], chol) yield check_lower_triangular, pd, ch_f # Explicit lower-triangular. chol = Cholesky(lower=True)(x) ch_f = function([x], chol) yield check_lower_triangular, pd, ch_f # Explicit upper-triangular. chol = Cholesky(lower=False)(x) ch_f = function([x], chol) yield check_upper_triangular, pd, ch_f chol = Cholesky(lower=False, on_error='nan')(x) ch_f = function([x], chol) yield check_upper_triangular, pd, ch_f def test_cholesky_indef(): if not imported_scipy: raise SkipTest("Scipy needed for the Cholesky op.") x = tensor.matrix() matrix = np.array([[1, 0.2], [0.2, -2]]).astype(config.floatX) cholesky = Cholesky(lower=True, on_error='raise') chol_f = function([x], cholesky(x)) with assert_raises(scipy.linalg.LinAlgError): chol_f(matrix) cholesky = Cholesky(lower=True, on_error='nan') chol_f = function([x], cholesky(x)) assert np.all(np.isnan(chol_f(matrix))) def test_cholesky_grad(): if not imported_scipy: raise SkipTest("Scipy needed for the Cholesky op.") rng = np.random.RandomState(utt.fetch_seed()) r = rng.randn(5, 5).astype(config.floatX) # The dots are inside the graph since Cholesky needs separable matrices # Check the default. yield (lambda: utt.verify_grad(lambda r: cholesky(r.dot(r.T)), [r], 3, rng)) # Explicit lower-triangular. yield (lambda: utt.verify_grad(lambda r: Cholesky(lower=True)(r.dot(r.T)), [r], 3, rng)) # Explicit upper-triangular. yield (lambda: utt.verify_grad(lambda r: Cholesky(lower=False)(r.dot(r.T)), [r], 3, rng)) def test_cholesky_grad_indef(): if not imported_scipy: raise SkipTest("Scipy needed for the Cholesky op.") x = tensor.matrix() matrix = np.array([[1, 0.2], [0.2, -2]]).astype(config.floatX) cholesky = Cholesky(lower=True, on_error='raise') chol_f = function([x], grad(cholesky(x).sum(), [x])) with assert_raises(scipy.linalg.LinAlgError): chol_f(matrix) cholesky = Cholesky(lower=True, on_error='nan') chol_f = function([x], grad(cholesky(x).sum(), [x])) assert np.all(np.isnan(chol_f(matrix))) @attr('slow') def test_cholesky_and_cholesky_grad_shape(): if not imported_scipy: raise SkipTest("Scipy needed for the Cholesky op.") rng = np.random.RandomState(utt.fetch_seed()) x = tensor.matrix() for l in (cholesky(x), Cholesky(lower=True)(x), Cholesky(lower=False)(x)): f_chol = theano.function([x], l.shape) g = tensor.grad(l.sum(), x) f_cholgrad = theano.function([x], g.shape) topo_chol = f_chol.maker.fgraph.toposort() topo_cholgrad = f_cholgrad.maker.fgraph.toposort() if config.mode != 'FAST_COMPILE': assert sum([node.op.__class__ == Cholesky for node in topo_chol]) == 0 assert sum([node.op.__class__ == CholeskyGrad for node in topo_cholgrad]) == 0 for shp in [2, 3, 5]: m = np.cov(rng.randn(shp, shp + 10)).astype(config.floatX) yield np.testing.assert_equal, f_chol(m), (shp, shp) yield np.testing.assert_equal, f_cholgrad(m), (shp, shp) def test_eigvalsh(): if not imported_scipy: raise SkipTest("Scipy needed for the geigvalsh op.") import scipy.linalg A = theano.tensor.dmatrix('a') B = theano.tensor.dmatrix('b') f = function([A, B], eigvalsh(A, B)) rng = np.random.RandomState(utt.fetch_seed()) a = rng.randn(5, 5) a = a + a.T for b in [10 * np.eye(5, 5) + rng.randn(5, 5)]: w = f(a, b) refw = scipy.linalg.eigvalsh(a, b) np.testing.assert_array_almost_equal(w, refw) # We need to test None separatly, as otherwise DebugMode will # complain, as this isn't a valid ndarray. b = None B = theano.tensor.NoneConst f = function([A], eigvalsh(A, B)) w = f(a) refw = scipy.linalg.eigvalsh(a, b) np.testing.assert_array_almost_equal(w, refw) def test_eigvalsh_grad(): if not imported_scipy: raise SkipTest("Scipy needed for the geigvalsh op.") import scipy.linalg rng = np.random.RandomState(utt.fetch_seed()) a = rng.randn(5, 5) a = a + a.T b = 10 * np.eye(5, 5) + rng.randn(5, 5) tensor.verify_grad(lambda a, b: eigvalsh(a, b).dot([1, 2, 3, 4, 5]), [a, b], rng=np.random) class test_Solve(utt.InferShapeTester): def setUp(self): super(test_Solve, self).setUp() self.op_class = Solve self.op = Solve() def test_infer_shape(self): if not imported_scipy: raise SkipTest("Scipy needed for the Solve op.") rng = np.random.RandomState(utt.fetch_seed()) A = theano.tensor.matrix() b = theano.tensor.matrix() self._compile_and_check([A, b], # theano.function inputs [self.op(A, b)], # theano.function outputs # A must be square [np.asarray(rng.rand(5, 5), dtype=config.floatX), np.asarray(rng.rand(5, 1), dtype=config.floatX)], self.op_class, warn=False) rng = np.random.RandomState(utt.fetch_seed()) A = theano.tensor.matrix() b = theano.tensor.vector() self._compile_and_check([A, b], # theano.function inputs [self.op(A, b)], # theano.function outputs # A must be square [np.asarray(rng.rand(5, 5), dtype=config.floatX), np.asarray(rng.rand(5), dtype=config.floatX)], self.op_class, warn=False) def test_solve_correctness(self): if not imported_scipy: raise SkipTest("Scipy needed for the Cholesky and Solve ops.") rng = np.random.RandomState(utt.fetch_seed()) A = theano.tensor.matrix() b = theano.tensor.matrix() y = self.op(A, b) gen_solve_func = theano.function([A, b], y) cholesky_lower = Cholesky(lower=True) L = cholesky_lower(A) y_lower = self.op(L, b) lower_solve_func = theano.function([L, b], y_lower) cholesky_upper = Cholesky(lower=False) U = cholesky_upper(A) y_upper = self.op(U, b) upper_solve_func = theano.function([U, b], y_upper) b_val = np.asarray(rng.rand(5, 1), dtype=config.floatX) # 1-test general case A_val = np.asarray(rng.rand(5, 5), dtype=config.floatX) # positive definite matrix: A_val = np.dot(A_val.transpose(), A_val) assert np.allclose(scipy.linalg.solve(A_val, b_val), gen_solve_func(A_val, b_val)) # 2-test lower traingular case L_val = scipy.linalg.cholesky(A_val, lower=True) assert np.allclose(scipy.linalg.solve_triangular(L_val, b_val, lower=True), lower_solve_func(L_val, b_val)) # 3-test upper traingular case U_val = scipy.linalg.cholesky(A_val, lower=False) assert np.allclose(scipy.linalg.solve_triangular(U_val, b_val, lower=False), upper_solve_func(U_val, b_val)) def test_solve_dtype(self): if not imported_scipy: raise SkipTest("Scipy needed for the Solve op.") dtypes = ['uint8', 'uint16', 'uint32', 'uint64', 'int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64'] A_val = np.eye(2) b_val = np.ones((2, 1)) # try all dtype combinations for A_dtype, b_dtype in itertools.product(dtypes, dtypes): A = tensor.matrix(dtype=A_dtype) b = tensor.matrix(dtype=b_dtype) x = solve(A, b) fn = function([A, b], x) x_result = fn(A_val.astype(A_dtype), b_val.astype(b_dtype)) assert x.dtype == x_result.dtype def verify_solve_grad(self, m, n, A_structure, lower, rng): # ensure diagonal elements of A relatively large to avoid numerical # precision issues A_val = (rng.normal(size=(m, m)) * 0.5 + np.eye(m)).astype(config.floatX) if A_structure == 'lower_triangular': A_val = np.tril(A_val) elif A_structure == 'upper_triangular': A_val = np.triu(A_val) if n is None: b_val = rng.normal(size=m).astype(config.floatX) else: b_val = rng.normal(size=(m, n)).astype(config.floatX) eps = None if config.floatX == "float64": eps = 2e-8 solve_op = Solve(A_structure=A_structure, lower=lower) utt.verify_grad(solve_op, [A_val, b_val], 3, rng, eps=eps) def test_solve_grad(self): if not imported_scipy: raise SkipTest("Scipy needed for the Solve op.") rng = np.random.RandomState(utt.fetch_seed()) structures = ['general', 'lower_triangular', 'upper_triangular'] for A_structure in structures: lower = (A_structure == 'lower_triangular') self.verify_solve_grad(5, None, A_structure, lower, rng) self.verify_solve_grad(6, 1, A_structure, lower, rng) self.verify_solve_grad(4, 3, A_structure, lower, rng) # lower should have no effect for A_structure == 'general' so also # check lower=True case self.verify_solve_grad(4, 3, 'general', lower=True, rng=rng) def test_expm(): if not imported_scipy: raise SkipTest("Scipy needed for the expm op.") rng = np.random.RandomState(utt.fetch_seed()) A = rng.randn(5, 5).astype(config.floatX) ref = scipy.linalg.expm(A) x = tensor.matrix() m = expm(x) expm_f = function([x], m) val = expm_f(A) np.testing.assert_array_almost_equal(val, ref) def test_expm_grad_1(): # with symmetric matrix (real eigenvectors) if not imported_scipy: raise SkipTest("Scipy needed for the expm op.") rng = np.random.RandomState(utt.fetch_seed()) # Always test in float64 for better numerical stability. A = rng.randn(5, 5) A = A + A.T tensor.verify_grad(expm, [A], rng=rng) def test_expm_grad_2(): # with non-symmetric matrix with real eigenspecta if not imported_scipy: raise SkipTest("Scipy needed for the expm op.") rng = np.random.RandomState(utt.fetch_seed()) # Always test in float64 for better numerical stability. A = rng.randn(5, 5) w = rng.randn(5)**2 A = (np.diag(w**0.5)).dot(A + A.T).dot(np.diag(w**(-0.5))) assert not np.allclose(A, A.T) tensor.verify_grad(expm, [A], rng=rng) def test_expm_grad_3(): # with non-symmetric matrix (complex eigenvectors) if not imported_scipy: raise SkipTest("Scipy needed for the expm op.") rng = np.random.RandomState(utt.fetch_seed()) # Always test in float64 for better numerical stability. A = rng.randn(5, 5) tensor.verify_grad(expm, [A], rng=rng) class TestKron(utt.InferShapeTester): rng = np.random.RandomState(43) def setUp(self): super(TestKron, self).setUp() self.op = kron def test_perform(self): if not imported_scipy: raise SkipTest('kron tests need the scipy package to be installed') for shp0 in [(2,), (2, 3), (2, 3, 4), (2, 3, 4, 5)]: x = tensor.tensor(dtype='floatX', broadcastable=(False,) * len(shp0)) a = np.asarray(self.rng.rand(*shp0)).astype(config.floatX) for shp1 in [(6,), (6, 7), (6, 7, 8), (6, 7, 8, 9)]: if len(shp0) + len(shp1) == 2: continue y = tensor.tensor(dtype='floatX', broadcastable=(False,) * len(shp1)) f = function([x, y], kron(x, y)) b = self.rng.rand(*shp1).astype(config.floatX) out = f(a, b) # Newer versions of scipy want 4 dimensions at least, # so we have to add a dimension to a and flatten the result. if len(shp0) + len(shp1) == 3: scipy_val = scipy.linalg.kron( a[np.newaxis, :], b).flatten() else: scipy_val = scipy.linalg.kron(a, b) utt.assert_allclose(out, scipy_val) def test_numpy_2d(self): for shp0 in [(2, 3)]: x = tensor.tensor(dtype='floatX', broadcastable=(False,) * len(shp0)) a = np.asarray(self.rng.rand(*shp0)).astype(config.floatX) for shp1 in [(6, 7)]: if len(shp0) + len(shp1) == 2: continue y = tensor.tensor(dtype='floatX', broadcastable=(False,) * len(shp1)) f = function([x, y], kron(x, y)) b = self.rng.rand(*shp1).astype(config.floatX) out = f(a, b) assert np.allclose(out, np.kron(a, b)) Theano-1.0.0/theano/tensor/tests/test_subtensor.py00006000163562€ÿÿÿþ0000024346113203335532023670 0ustar boccoset00000000000000from __future__ import absolute_import, division, print_function import logging import sys import unittest import numpy as np from nose.tools import assert_equal from numpy.testing import assert_array_equal from six import StringIO from six.moves import xrange import theano import theano.scalar as scal import theano.tensor as tensor from theano import config, gof from theano.compat import PY3, izip from theano.compile import DeepCopyOp from theano.tensor import (_shared, cscalar, ctensor3, dmatrix, dscalar, dtensor4, dvector, fmatrix, fscalar, fvector, ftensor4, iscalar, lmatrix, lrow, lvector, matrix, vector) from theano.tensor.basic import DimShuffle from theano.tensor.subtensor import (AdvancedIncSubtensor, AdvancedIncSubtensor1, AdvancedSubtensor, IncSubtensor, Subtensor, advanced_inc_subtensor, advanced_inc_subtensor1, advanced_set_subtensor, advanced_set_subtensor1, get_canonical_form_slice, inc_subtensor, set_subtensor) from theano.tensor.tests.test_basic import inplace_func, rand, randint_ranged from theano.tests import unittest_tools as utt from theano.tests.unittest_tools import attr from theano import change_flags if PY3: def L(i): return i else: def L(i): return long(i) # noqa for Python 3 class T_subtensor(unittest.TestCase, utt.TestOptimizationMixin): """ This is build in a way that allow to reuse it to test the equivalent gpu op. """ def __init__(self, name, shared=tensor._shared, sub=tensor.Subtensor, inc_sub=tensor.IncSubtensor, adv_sub1=tensor.AdvancedSubtensor1, adv_incsub1=tensor.AdvancedIncSubtensor1, adv_sub=tensor.AdvancedSubtensor, adv_bool_sub=tensor.AdvancedBooleanSubtensor, adv_bool_inc_sub=tensor.AdvancedBooleanIncSubtensor, mode=None, dtype=theano.config.floatX, type=tensor.TensorType, ignore_topo=DeepCopyOp, dimshuffle=DimShuffle): self.shared = shared self.sub = sub self.inc_sub = inc_sub self.adv_sub1 = adv_sub1 self.adv_incsub1 = adv_incsub1 self.adv_sub = adv_sub self.adv_bool_sub = adv_bool_sub self.adv_bool_inc_sub = adv_bool_inc_sub self.dimshuffle = dimshuffle if mode is None: mode = theano.compile.mode.get_default_mode() mode = mode.including("local_useless_subtensor") self.mode = mode self.dtype = dtype self.type = type self.ignore_topo = ignore_topo self.fast_compile = theano.config.mode == 'FAST_COMPILE' self.ops = (sub, inc_sub, adv_sub1, adv_incsub1, adv_bool_sub, adv_bool_inc_sub) return super(T_subtensor, self).__init__(name) def function(self, inputs, outputs, accept_inplace=False, op=None, mode=None, N=1, N_fast=None): """ wrapper around theano.function that also check the output :param N: the number of op expected in the toposort if tuple of length 2, (expected if fast_compile, if not fast_compile) """ if self.fast_compile and N_fast is not None: N = N_fast if mode is None: mode = self.mode if op is None: op = self.sub f = theano.function(inputs, outputs, mode=mode, accept_inplace=accept_inplace) self.assertFunctionContainsClassN(f, op, N) return f def setUp(self): Subtensor.debug = False utt.seed_rng() def eval_output_and_check(self, t, op_type=None, mode=None, length=1): if op_type is None: op_type = self.sub if mode is None: mode = self.mode f = inplace_func([], t, mode=mode) topo = f.maker.fgraph.toposort() topo_ = [node for node in topo if not isinstance(node.op, self.ignore_topo)] assert_equal(len(topo_), length) if length == 1: assert isinstance(topo_[0].op, op_type) tval = f() return tval def test0_err_invalid(self): # it is impossible to retrieve a view of a 0-d tensor n = self.shared(np.ones((), dtype=self.dtype)) self.assertRaises(IndexError, n.__getitem__, 0) @change_flags(compute_test_value='off') def test1_err_bounds(self): n = self.shared(np.ones(3, dtype=self.dtype)) t = n[7] self.assertTrue(isinstance(t.owner.op, Subtensor)) # Silence expected error messages _logger = logging.getLogger('theano.gof.opt') oldlevel = _logger.level _logger.setLevel(logging.CRITICAL) try: try: self.eval_output_and_check(t) except IndexError: return self.fail() finally: _logger.setLevel(oldlevel) def test1_err_subslice(self): n = self.shared(np.ones(3, dtype=self.dtype)) try: n[slice(0, slice(1, 2, None), None)] except Exception: # Relax constraint on the type of Exception, # since this might be handled by AvancedSubtensor # if e[0] != Subtensor.e_indextype: # raise return self.fail() def test1_ok_range_finite(self): n = self.shared(np.arange(3, dtype=self.dtype)) t = n[0:2] self.assertTrue(isinstance(t.owner.op, Subtensor)) tval = self.eval_output_and_check(t) self.assertTrue(tval.shape == (2,)) self.assertTrue((tval == [0, 1]).all()) def test2_ok_range_finite(self): n = self.shared(np.arange(12, dtype=self.dtype).reshape((3, 4))) # Also check negative index for idx in [(slice(0, 2), 3), ((slice(0, 2), -1)), (slice(0, 2), -4)]: t = n[idx] # l]#0:2,3] self.assertTrue(isinstance(t.owner.op, Subtensor)) tval = self.eval_output_and_check(t) self.assertTrue(tval.shape == (2,)) self.assertTrue(np.allclose(tval, n.get_value()[idx])) def test1_0_dims(self): n = self.shared(np.ones((), dtype=self.dtype)) t = self.sub([])(n) self.assertTrue(isinstance(t.owner.op, Subtensor)) self.eval_output_and_check( t, mode=self.mode.excluding("local_useless_subtensor")) def test1_err_invalid(self): n = self.shared(np.ones(1, dtype=self.dtype)) self.assertRaises(IndexError, n.__getitem__, (0, 0)) def test1_ok_elem(self): n = self.shared(np.ones(1, dtype=self.dtype) * 5) t = n[0] self.assertTrue(isinstance(t.owner.op, Subtensor)) tval = self.eval_output_and_check(t) self.assertTrue(tval.shape == ()) self.assertTrue(tval == 5.0) def test1_ok_range_infinite(self): n = self.shared(np.arange(3, dtype=self.dtype)) t = n[1:] self.assertTrue(isinstance(t.owner.op, Subtensor)) tval = self.eval_output_and_check(t) self.assertTrue(tval.shape == (2,)) self.assertTrue((tval == [1.0, 2.0]).all()) def test1_ok_strided(self): n = self.shared(np.arange(5, dtype=self.dtype)) t = n[1::2] self.assertTrue(isinstance(t.owner.op, Subtensor)) tval = self.eval_output_and_check(t) self.assertTrue(tval.shape == (2,)) self.assertTrue((tval == [1.0, 3.0]).all()) t = n[0:-1:2] # 0 to 1 from the end stepping by 2 tval = self.eval_output_and_check(t) self.assertTrue(tval.shape == (2,)) self.assertTrue((tval == [0.0, 2.0]).all()) @change_flags(compute_test_value='off') def test2_err_bounds0(self): n = self.shared(np.ones((2, 3), dtype=self.dtype) * 5) for idx in [(0, 4), (0, -4)]: t = n[idx] self.assertTrue(isinstance(t.owner.op, Subtensor)) # Silence expected warnings _logger = logging.getLogger('theano.gof.opt') oldlevel = _logger.level _logger.setLevel(logging.CRITICAL) try: self.assertRaises(IndexError, self.eval_output_and_check, [t]) finally: _logger.setLevel(oldlevel) @change_flags(compute_test_value='off') def test2_err_bounds1(self): n = self.shared((np.ones((2, 3), dtype=self.dtype) * 5)) t = n[4:5, 3] self.assertTrue(isinstance(t.owner.op, Subtensor)) old_stderr = sys.stderr sys.stderr = StringIO() try: self.assertRaises(IndexError, self.eval_output_and_check, [t]) finally: sys.stderr = old_stderr def test2_ok_elem(self): n = self.shared(np.arange(6, dtype=self.dtype).reshape((2, 3))) t = n[0, 2] self.assertTrue(isinstance(t.owner.op, Subtensor)) tval = self.eval_output_and_check(t) self.assertTrue(tval.shape == ()) self.assertTrue(np.all(tval == 2)) def test2_ok_row(self): n = self.shared(np.arange(6, dtype=self.dtype).reshape((2, 3))) t = n[1] self.assertFalse(any(n.type.broadcastable)) self.assertTrue(isinstance(t.owner.op, Subtensor)) tval = self.eval_output_and_check(t) self.assertTrue(tval.shape == (3,)) self.assertTrue(np.all(tval == [3, 4, 5])) def test2_ok_col(self): n = self.shared(np.arange(6, dtype=self.dtype).reshape((2, 3))) t = n[:, 0] self.assertTrue(isinstance(t.owner.op, Subtensor)) self.assertFalse(any(n.type.broadcastable)) tval = self.eval_output_and_check(t) self.assertTrue(tval.shape == (2,)) self.assertTrue(np.all(tval == [0, 3])) def test2_ok_rows_finite(self): n = self.shared(np.arange(12, dtype=self.dtype).reshape((4, 3))) t = n[1:3, 0] self.assertTrue(isinstance(t.owner.op, Subtensor)) tval = self.eval_output_and_check(t) self.assertTrue(tval.shape == (2,)) self.assertTrue(np.all(tval == [3, 6])) def test2_ok_cols_infinite(self): n = self.shared(np.arange(12, dtype=self.dtype).reshape((4, 3))) t = n[1, 2:] self.assertTrue(isinstance(t.owner.op, Subtensor)) tval = self.eval_output_and_check(t) self.assertTrue(tval.shape == (1,)) self.assertTrue(np.all(tval == 5)) def test2_ok_strided(self): n = self.shared(np.arange(20, dtype=self.dtype).reshape((4, 5))) t = n[1:4:2, 1:5:2] self.assertTrue(isinstance(t.owner.op, Subtensor)) tval = self.eval_output_and_check(t) self.assertTrue(tval.shape == (2, 2)) self.assertTrue(np.all(tval == [[6, 8], [16, 18]])) def test3_ok_mat(self): n = self.shared(np.arange(24, dtype=self.dtype).reshape((2, 3, 4))) t = n[0, 0, 0] self.assertTrue(isinstance(t.owner.op, Subtensor)) tval = self.eval_output_and_check(t) self.assertTrue(tval.shape == ()) self.assertTrue(np.all(tval == 0)) def test_long(self): n = self.shared(np.arange(12, dtype=self.dtype).reshape((4, 3))) t = n[L(1):L(4):L(2), L(1)] self.assertTrue(isinstance(t.owner.op, Subtensor)) tval = self.eval_output_and_check(t) self.assertTrue(tval.shape == (2,)) self.assertTrue(np.all(tval == [4, 10])) def test_long_too_big(self): # Currently, we cast Python longs to int64 when used for indexing. # This test checks that using a long that does not fit raises an error. n = self.shared(np.arange(12, dtype=self.dtype).reshape((4, 3))) self.assertRaises(Exception, lambda: n[:L(2 ** 63)]) def test_list_slice(self): x = theano.tensor.arange(100).reshape((5, 5, 4)) res = x[[slice(1, -1)] * x.ndim].eval() x = np.arange(100).reshape((5, 5, 4)) np.allclose(res, x[[slice(1, -1)] * x.ndim]) def test_slice_symbol(self): x = self.shared(np.random.rand(5, 4).astype(self.dtype)) y = self.shared(np.random.rand(1, 2, 3).astype(self.dtype)) o = x[:y.shape[0], None, :] f = theano.function([], o, mode=self.mode) ret = f() assert ret.shape == (1, 1, 4) def test_ellipsis(self): numpy_n = np.arange(24, dtype=self.dtype).reshape((2, 3, 4)) n = self.shared(numpy_n) test_cases = [ (0, Subtensor, self.sub, np.index_exp[...]), (1, Subtensor, self.sub, np.index_exp[..., 1]), (1, Subtensor, self.sub, np.index_exp[1, ...]), (1, Subtensor, self.sub, np.index_exp[..., 1, 2, 3]), (1, Subtensor, self.sub, np.index_exp[1, ..., 2, 3]), (1, Subtensor, self.sub, np.index_exp[1, 2, 3, ...]), (3, DimShuffle, self.dimshuffle, np.index_exp[..., [0, 2, 3]]), (1, DimShuffle, self.dimshuffle, np.index_exp[np.newaxis, ...]), (1, AdvancedSubtensor, self.adv_sub, np.index_exp[..., np.newaxis, [1, 2]])] for length, op_type, op_type_opt, slice_ in test_cases: numpy_tval = numpy_n[slice_] t = n[slice_] self.assertTrue(isinstance(t.owner.op, op_type)) tval = self.eval_output_and_check(t, op_type=op_type_opt, length=length) assert_equal(tval.shape, numpy_tval.shape) assert_array_equal(tval, numpy_tval) def test_boolean(self): def numpy_inc_subtensor(x, idx, a): x = x.copy() x[idx] += a return x numpy_n = np.arange(6, dtype=self.dtype).reshape((2, 3)) n = self.shared(numpy_n) # indexing with a mask for some dimensions mask = np.array([True, False]) val = self.eval_output_and_check(n[mask], op_type=self.adv_bool_sub) assert_array_equal(numpy_n[mask], val) val = self.eval_output_and_check(inc_subtensor(n[mask], 1), op_type=self.adv_bool_inc_sub) assert_array_equal(numpy_inc_subtensor(numpy_n, mask, 1), val) assert_array_equal(numpy_inc_subtensor(numpy_n, mask, numpy_n[mask]), inc_subtensor(n[mask], n[mask]).eval()) # test gradient utt.verify_grad(lambda m: m[mask], [numpy_n]) utt.verify_grad(lambda m: inc_subtensor(m[mask], 1), [numpy_n]) # indexing with a comparison (should translate to a boolean mask) assert_array_equal(numpy_n[numpy_n > 2], n[n > 2].eval()) assert_array_equal(numpy_n[[0], numpy_n[0] > 2], n[[0], n[0] > 2].eval()) assert_array_equal(numpy_n[[1], numpy_n[0] > 2], n[[1], n[0] > 2].eval()) # indexing with a mask for the second dimension mask = np.array([True, False, True]) assert_array_equal(numpy_n[0, mask], n[0, mask].eval()) assert_array_equal(numpy_n[:, mask], n[:, mask].eval()) assert_array_equal(numpy_n[:, mask], n[:, self.shared(mask)].eval()) assert_array_equal(numpy_n[1:, mask], n[1:, mask].eval()) assert_array_equal(numpy_n[:1, mask], n[:1, mask].eval()) assert_array_equal(numpy_n[1:, mask, np.newaxis], n[1:, mask, np.newaxis].eval()) assert_array_equal(numpy_n[np.newaxis, 1:, mask], n[np.newaxis, 1:, mask].eval()) assert_array_equal(numpy_inc_subtensor(numpy_n, [0, mask], 1), inc_subtensor(n[(0,) + mask.nonzero()], 1).eval()) assert_array_equal(numpy_inc_subtensor(numpy_n, [0, mask], 1), inc_subtensor(n[0, mask], 1).eval()) assert_array_equal(numpy_inc_subtensor(numpy_n, [slice(None), mask], 1), inc_subtensor(n[:, mask], 1).eval()) # indexing with a boolean ndarray mask = np.array([[True, False, True], [False, False, True]]) assert_array_equal(numpy_n[mask], n[mask].eval()) assert_array_equal(numpy_n[mask], n[self.shared(mask)].eval()) assert_array_equal(numpy_inc_subtensor(numpy_n, mask, 1), inc_subtensor(n[mask], 1).eval()) # indexing with ellipsis numpy_n4 = np.arange(48, dtype=self.dtype).reshape((2, 3, 4, 2)) n4 = self.shared(numpy_n4) assert_array_equal(numpy_n4[numpy_n > 2, ...], n4[n > 2, ...].eval()) assert_array_equal(numpy_n4[numpy_n > 2, ..., 1], n4[n > 2, ..., 1].eval()) assert_array_equal(numpy_n4[numpy_n > 2, ..., 0, 1], n4[n > 2, ..., 0, 1].eval()) assert_array_equal(numpy_inc_subtensor(numpy_n4, [numpy_n > 2, Ellipsis], 1), inc_subtensor(n4[n > 2, ...], 1).eval()) assert_array_equal(numpy_inc_subtensor(numpy_n4, [numpy_n > 2, Ellipsis, 1], 1), inc_subtensor(n4[n > 2, ..., 1], 1).eval()) assert_array_equal(numpy_inc_subtensor(numpy_n4, [numpy_n > 2, Ellipsis, 0, 1], 1), inc_subtensor(n4[n > 2, ..., 0, 1], 1).eval()) with change_flags(compute_test_value='off'): # the boolean mask should have the correct shape # - too large, padded with True mask = np.array([True, False, True]) self.assertRaises(IndexError, n[mask].eval) self.assertRaises(IndexError, n[mask, ...].eval) self.assertRaises(IndexError, inc_subtensor(n[mask], 1).eval) self.assertRaises(IndexError, inc_subtensor(n[mask, ...], 1).eval) mask = np.array([[True, False, False, True], [False, True, False, True]]) self.assertRaises(IndexError, n[mask].eval) self.assertRaises(IndexError, inc_subtensor(n[mask], 1).eval) # - too large, padded with False (this works in NumPy < 0.13.0) mask = np.array([True, False, False]) self.assertRaises(IndexError, n[mask].eval) self.assertRaises(IndexError, n[mask, ...].eval) self.assertRaises(IndexError, inc_subtensor(n[mask], 1).eval) self.assertRaises(IndexError, inc_subtensor(n[mask, ...], 1).eval) mask = np.array([[True, False, False, False], [False, True, False, False]]) self.assertRaises(IndexError, n[mask].eval) self.assertRaises(IndexError, inc_subtensor(n[mask], 1).eval) # - mask too small (this works in NumPy < 0.13.0) mask = np.array([True]) self.assertRaises(IndexError, n[mask].eval) self.assertRaises(IndexError, n[mask, ...].eval) self.assertRaises(IndexError, inc_subtensor(n[mask], 1).eval) self.assertRaises(IndexError, inc_subtensor(n[mask, ...], 1).eval) mask = np.array([[True], [True]]) self.assertRaises(IndexError, n[mask].eval) self.assertRaises(IndexError, inc_subtensor(n[mask], 1).eval) # - too many dimensions mask = np.array([[[True, False, False], [False, True, False]]]) self.assertRaises(IndexError, n.__getitem__, mask) self.assertRaises(IndexError, n.__getitem__, mask) # special cases: Python bools and bools nested in Python arrays are not supported self.assertRaises(TypeError, n.__getitem__, (True,)) self.assertRaises(TypeError, n.__getitem__, (False,)) self.assertRaises(TypeError, n.__getitem__, (True, False)) self.assertRaises(TypeError, n.__getitem__, ([True, False])) self.assertRaises(TypeError, n.__getitem__, ([0, 1], [0, False])) self.assertRaises(TypeError, n.__getitem__, ([0, 1], [0, theano.shared(True)])) def test_newaxis(self): # newaxis support comes from logic in the __getitem__ of TensorType # Variables, which currently inserts dimshuffle to get the right number # of dimensions, and adjusts the slice tuple accordingly. # # So testing is done via square-bracket notation rather than direct # interaction with the Subtensor Op (which has no support of its own for # newaxis). newaxis = np.newaxis n = self.shared(np.arange(24, dtype=self.dtype).reshape((2, 3, 4))) assert n.ndim == 3 n4 = n[newaxis, :, :, :] assert n4.broadcastable == (True, False, False, False), n4 n4 = n[:, newaxis, :, :] assert n4.broadcastable == (False, True, False, False), n4 n4 = n[:, :, newaxis, :] assert n4.broadcastable == (False, False, True, False), n4 n4 = n[:, :, :, newaxis] assert n4.broadcastable == (False, False, False, True), n4 n3 = n.flatten()[newaxis, :, newaxis] assert n3.broadcastable == (True, False, True), n3 s = cscalar() s1 = s[newaxis] assert s1.broadcastable == (True,), s1 vs1, vn3, vn4 = theano.function([s], [s1, n3, n4], mode=self.mode)(-2.0) assert np.all(vs1 == [-2.0]) assert np.all(vn3 == np.arange(24)[newaxis, :, newaxis]) assert np.all(vn4 == np.arange(24).reshape((2, 3, 4))[:, :, :, newaxis]) def test_grad_1d(self): subi = 0 data = np.asarray(rand(2, 3), dtype=self.dtype) n = self.shared(data) z = scal.constant(subi).astype('int32') t = n[z:, z] gn = theano.tensor.grad(theano.tensor.sum(theano.tensor.exp(t)), n) f = inplace_func([], gn, mode=self.mode) topo = f.maker.fgraph.toposort() topo_ = [node for node in topo if not isinstance(node.op, self.ignore_topo)] if not self.fast_compile: assert len(topo_) == 6 assert np.sum([isinstance(node.op, self.inc_sub) for node in topo_]) == 1 assert np.sum([isinstance(node.op, self.sub) for node in topo_]) == 1 gval = f() good = np.zeros_like(data) good[subi:, subi] = np.exp(data[subi:, subi]) self.assertTrue(np.allclose(gval, good), (gval, good)) def test_grad_2d_inc_set_subtensor(self): for n_shape, m_shape in [ [(2, 3), (2, 2)], [(3, 2), (2, 2)], [(3, 2), (1, 2)], [(3, 2), (2,)], ]: for op in [inc_subtensor, set_subtensor]: subi = 2 data = np.asarray(rand(*n_shape), dtype=self.dtype) n = self.shared(data) z = scal.constant(subi) m = matrix('m', dtype=self.dtype) mv = np.asarray(rand(*m_shape), dtype=self.dtype) t = op(n[:z, :z], m) gn, gm = theano.tensor.grad(theano.tensor.sum(t), [n, m]) utt.verify_grad(lambda m: op(n[:z, :z], m), [mv], mode=self.mode) utt.verify_grad(lambda nn: op(nn[:z, :z], mv), [data], mode=self.mode) def test_grad_0d(self): data = np.asarray(rand(2, 3), dtype=self.dtype) n = self.shared(data) t = n[1, 0] gn = theano.tensor.grad(theano.tensor.sum(theano.tensor.exp(t)), n) f = self.function([], gn) topo = f.maker.fgraph.toposort() topo_ = [node for node in topo if not isinstance(node.op, self.ignore_topo)] if not self.fast_compile: assert_equal(len(topo_), 6) assert np.sum([isinstance(node.op, self.inc_sub) for node in topo_]) == 1 assert np.sum([isinstance(node.op, self.sub) for node in topo_]) == 1 gval = f() good = np.zeros_like(data) good[1, 0] = np.exp(data[1, 0]) self.assertTrue(np.allclose(gval, good), (gval, good)) def test_ok_list(self): for data, idx in [(rand(4), [1, 0]), (rand(4, 5), [2, 3, -1]), (rand(4, 2, 3), [0, 3]), (rand(4, 2, 3), [3, 3, 1, 1, 2, 2, 0, 0]), (rand(4, 2, 3), [3, 3, 1, 1, 2, 2, 0, 0, -1, -2, -3, -4]), # Test 4 dims as gpu code use another algo # in that case This new algo is not as much # optimized for that case. (rand(4, 4, 2, 3), [3, 3, 1, 1, 2, 2, 0, 0, -1, -2, -3, -4]), # Test with TensorConstant index. (rand(4, 2, 3), theano.tensor.constant([3, 3, 1, 1, 2, 2, 0, 0])), ]: data = np.asarray(data, dtype=self.dtype) n = self.shared(data) t = n[idx] # We test again AdvancedSubtensor1 as we transfer data to the cpu. self.assertTrue(isinstance(t.owner.op, tensor.AdvancedSubtensor1)) val = self.eval_output_and_check(t, op_type=self.adv_sub1) if isinstance(idx, list): good = data[idx] else: good = data[idx.data] self.assertTrue(val.ndim == data.ndim) self.assertTrue(np.allclose(val, good), (val, good)) # Test reuse of output memory if type(self.adv_sub1) == tensor.AdvancedSubtensor1: op = self.adv_sub1() # When idx is a TensorConstant. if hasattr(idx, "data"): idx = idx.data test_out = [[None]] op.perform(None, [data, idx], test_out) out1 = test_out[0][0] op.perform(None, [data, idx], test_out) out2 = test_out[0][0] assert out1 is out2 # test the grad gn = theano.grad(t.sum(), n) g = self.function([], gn, op=self.adv_incsub1) utt.verify_grad(lambda m: m[[1, 3]], [np.random.rand(5, 5).astype(self.dtype)], mode=self.mode) g() utt.verify_grad(lambda m: m[idx], [data], mode=self.mode) def test_noncontiguous_idx(self): data = rand(4, 2, 3) idx = [2, 2, 0, 0, 1, 1] n = self.shared(data) t = n[self.shared(np.asarray(idx).astype('int64'))[::2]] self.assertTrue(isinstance(t.owner.op, tensor.AdvancedSubtensor1)) val = self.eval_output_and_check(t, op_type=self.adv_sub1, length=2) utt.assert_allclose(data[idx[::2]], val) def test_err_invalid_list(self): n = self.shared(np.asarray(5, dtype=self.dtype)) self.assertRaises(IndexError, n.__getitem__, [0, 0]) def test_err_invalid_2list_dtype(self): n = self.shared(np.ones((3, 3), dtype=self.dtype) * 5) self.assertRaises(TypeError, n.__getitem__, ([0., 0], [1, 1])) def test_err_bound_list(self): n = self.shared(np.ones((2, 3), dtype=self.dtype) * 5) l = lvector() t = n[l] # We test again AdvancedSubtensor1 as we transfer data to the cpu. self.assertTrue(isinstance(t.owner.op, tensor.AdvancedSubtensor1)) f = self.function([l], t, op=self.adv_sub1) # the grad g = self.function([l], inc_subtensor(t, np.asarray([[1.]], self.dtype)), op=self.adv_incsub1) for shp in [[0, 4], [0, -3], [-10]]: self.assertRaises(IndexError, f, shp) self.assertRaises(IndexError, g, shp) def test_adv_sub1_broadcast(self): v = np.arange(3, dtype=self.dtype).reshape((1, 3)) n = self.shared(v * 5, broadcastable=(True, False)) idx = tensor.lvector() t = n[idx] self.assertTrue(isinstance(t.owner.op, tensor.AdvancedSubtensor1)) f = self.function([idx], t, op=self.adv_sub1) topo = f.maker.fgraph.toposort() topo_ = [node for node in topo if not isinstance(node.op, self.ignore_topo)] assert len(topo_) == 1 self.assertTrue(isinstance(topo_[0].op, self.adv_sub1)) f_0 = f([0]) self.assertTrue(f_0.shape == (1, 3)) self.assertTrue(np.allclose(f_0, v * 5)) f_00 = f([0, 0]) self.assertTrue(f_00.shape == (2, 3)) self.assertTrue(np.allclose(f_00, v * 5)) self.assertRaises(IndexError, f, [0, 1]) # Test the gradient c = t.sum() gn = theano.grad(c, n) g = self.function([idx], gn, op=self.adv_incsub1) g_0 = g([0]) self.assertTrue(g_0.shape == (1, 3)) self.assertTrue(np.allclose(g_0, 1)) g_00 = g([0, 0]) self.assertTrue(g_00.shape == (1, 3)) self.assertTrue(np.allclose(g_00, 2)) utt.verify_grad(lambda m: m[[1, 3]], [np.random.rand(5, 5).astype(self.dtype)], mode=self.mode) def fun(x, y): return advanced_inc_subtensor1(x, y, [1, 3]) utt.verify_grad(fun, [np.random.rand(5, 5).astype(self.dtype), np.random.rand(2, 5).astype(self.dtype)], mode=self.mode) def fun(x, y): return advanced_set_subtensor1(x, y, [1, 3]) utt.verify_grad(fun, [np.random.rand(5, 5).astype(self.dtype), np.random.rand(2, 5).astype(self.dtype)], mode=self.mode) # test set_subtensor broadcast self.dtype = 'float32' x = tensor.tensor4('x', dtype=self.dtype) indexes = theano.shared(np.int32([1, 2, 3, 4])) W = self.shared(np.random.random( (10, 10, 3, 3)).astype(self.dtype)) h = x + W h = tensor.set_subtensor(h[indexes], h[indexes]) g = tensor.grad(h.sum(), W) N = 2 if theano.config.mode == "FAST_COMPILE" and self.adv_incsub1 is tensor.AdvancedIncSubtensor1: N = 3 f = self.function([x], g, op=self.adv_incsub1, N=N) f(np.random.random((10, 10, 3, 3)).astype(self.dtype)) def test_adv_sub1_idx_broadcast(self): # The idx can be a broadcastable vector. ones = np.ones((4, 3), dtype=self.dtype) n = self.shared(ones * 5) idx = tensor.TensorType(dtype='int64', broadcastable=(True,))() assert idx.type.broadcastable == (True,) t = n[idx] self.assertTrue(isinstance(t.owner.op, tensor.AdvancedSubtensor1)) f = self.function([idx], t, op=self.adv_sub1) topo = f.maker.fgraph.toposort() topo_ = [node for node in topo if not isinstance(node.op, self.ignore_topo)] assert len(topo_) == 1 self.assertTrue(isinstance(topo_[0].op, self.adv_sub1)) f_0 = f([0]) self.assertTrue(f_0.shape == (1, 3)) self.assertTrue(np.allclose(f_0, 5)) # Test the gradient c = t.sum() gn = theano.grad(c, n) g = self.function([idx], gn, op=self.adv_incsub1) g_0 = g([0]) self.assertTrue(g_0.shape == (4, 3)) self.assertTrue(np.allclose(g_0[0], 1)) self.assertTrue(np.allclose(g_0[1:], 0)) @attr('slow') def test_shape_i_const(self): # Each axis is treated independently by shape_i/shape operators mode_opt = self.mode.including("fast_run") data = self.shared(np.array(np.arange(5), dtype=self.dtype)) for start in [None] + [-8, -5, -1, 0, 1, 5, 8]: outs = [] shapes = [] for stop in [None] + [-8, -5, -1, 0, 1, 5, 8]: for step in [None] + [-3, -1, 2]: outs += [data[start:stop:step].shape] shapes += [data.get_value( borrow=True)[start:stop:step].shape] f = self.function([], outs, mode=mode_opt, op=self.ops, N=0) t_shapes = f() for t_shape, shape in zip(t_shapes, shapes): assert np.all(t_shape == shape) assert tensor.Subtensor not in [x.op for x in f.maker.fgraph.toposort()] def test_shape_i_scalar(self): # Each axis is treated independently by shape_i/shape operators mode_opt = self.mode.including("fast_run") v_data = np.array(np.arange(5), dtype=self.dtype) t_data = self.shared(v_data) start = tensor.iscalar('b') stop = tensor.iscalar('e') step = tensor.iscalar('s') f = self.function([start, stop, step], t_data[start:stop:step].shape, mode=mode_opt, op=self.ops, N=0) assert tensor.Subtensor not in [x.op for x in f.maker.fgraph.toposort()] for start in [-8, -5, -4, -1, 0, 1, 4, 5, 8]: for stop in [-8, -5, -4, -1, 0, 1, 4, 5, 8]: for step in [-3, -1, 2, 5]: assert np.all(f(start, stop, step) == v_data[start:stop:step].shape) def test_slice_canonical_form_0(self): start = tensor.iscalar('b') stop = tensor.iscalar('e') step = tensor.iscalar('s') length = tensor.iscalar('l') cnf = get_canonical_form_slice(slice(start, stop, step), length) f = self.function([start, stop, step, length], [ tensor.as_tensor_variable(cnf[0].start), tensor.as_tensor_variable(cnf[0].stop), tensor.as_tensor_variable(cnf[0].step), tensor.as_tensor_variable(cnf[1])], N=0, op=self.ops) length = 5 a = np.arange(length) for start in [-8, -5, -4, -1, 0, 1, 4, 5, 8]: for stop in [-8, -5, -4, -1, 0, 1, 4, 5, 8]: for step in [-6, -3, -1, 2, 5]: out = f(start, stop, step, length) t_out = a[out[0]:out[1]:out[2]][::out[3]] v_out = a[start:stop:step] assert np.all(t_out == v_out) assert np.all(t_out.shape == v_out.shape) def test_slice_canonical_form_1(self): stop = tensor.iscalar('e') step = tensor.iscalar('s') length = tensor.iscalar('l') cnf = get_canonical_form_slice(slice(None, stop, step), length) f = self.function([stop, step, length], [ tensor.as_tensor_variable(cnf[0].start), tensor.as_tensor_variable(cnf[0].stop), tensor.as_tensor_variable(cnf[0].step), tensor.as_tensor_variable(cnf[1])], N=0, op=self.ops) length = 5 a = np.arange(length) for stop in [-8, -5, -4, -1, 0, 1, 4, 5, 8]: for step in [-6, -3, -1, 2, 5]: out = f(stop, step, length) t_out = a[out[0]:out[1]:out[2]][::out[3]] v_out = a[:stop:step] assert np.all(t_out == v_out) assert np.all(t_out.shape == v_out.shape) def test_slice_canonical_form_2(self): start = tensor.iscalar('b') step = tensor.iscalar('s') length = tensor.iscalar('l') cnf = get_canonical_form_slice(slice(start, None, step), length) f = self.function([start, step, length], [ tensor.as_tensor_variable(cnf[0].start), tensor.as_tensor_variable(cnf[0].stop), tensor.as_tensor_variable(cnf[0].step), tensor.as_tensor_variable(cnf[1])], N=0, op=self.ops) length = 5 a = np.arange(length) for start in [-8, -5, -4, -1, 0, 1, 4, 5, 8]: for step in [-6, -3, -1, 2, 5]: out = f(start, step, length) t_out = a[out[0]:out[1]:out[2]][::out[3]] v_out = a[start:None:step] assert np.all(t_out == v_out) assert np.all(t_out.shape == v_out.shape) def test_slice_canonical_form_3(self): start = tensor.iscalar('b') stop = tensor.iscalar('e') length = tensor.iscalar('l') cnf = get_canonical_form_slice(slice(start, stop, None), length) f = self.function([start, stop, length], [ tensor.as_tensor_variable(cnf[0].start), tensor.as_tensor_variable(cnf[0].stop), tensor.as_tensor_variable(cnf[0].step), tensor.as_tensor_variable(cnf[1])], N=0, op=self.ops) length = 5 a = np.arange(length) for start in [-8, -5, -4, -1, 0, 1, 4, 5, 8]: for stop in [-8, -5, -4, -1, 0, 1, 4, 5, 8]: out = f(start, stop, length) t_out = a[out[0]:out[1]:out[2]][::out[3]] v_out = a[start:stop:None] assert np.all(t_out == v_out) assert np.all(t_out.shape == v_out.shape) def test_slice_canonical_form_4(self): step = tensor.iscalar('s') length = tensor.iscalar('l') cnf = get_canonical_form_slice(slice(None, None, step), length) f = self.function([step, length], [ tensor.as_tensor_variable(cnf[0].start), tensor.as_tensor_variable(cnf[0].stop), tensor.as_tensor_variable(cnf[0].step), tensor.as_tensor_variable(cnf[1])], N=0, op=self.ops) length = 5 a = np.arange(length) for step in [-6, -3, -1, 2, 5]: out = f(step, length) t_out = a[out[0]:out[1]:out[2]][::out[3]] v_out = a[None:None:step] assert np.all(t_out == v_out) assert np.all(t_out.shape == v_out.shape) def test_slice_canonical_form_5(self): start = tensor.iscalar('b') length = tensor.iscalar('l') cnf = get_canonical_form_slice(slice(start, None, None), length) f = self.function([start, length], [ tensor.as_tensor_variable(cnf[0].start), tensor.as_tensor_variable(cnf[0].stop), tensor.as_tensor_variable(cnf[0].step), tensor.as_tensor_variable(cnf[1])], N=0, op=self.ops) length = 5 a = np.arange(length) for start in [-8, -5, -4, -1, 0, 1, 4, 5, 8]: out = f(start, length) t_out = a[out[0]:out[1]:out[2]][::out[3]] v_out = a[start:None:None] assert np.all(t_out == v_out) assert np.all(t_out.shape == v_out.shape) def test_slice_canonical_form_6(self): stop = tensor.iscalar('e') length = tensor.iscalar('l') cnf = get_canonical_form_slice(slice(None, stop, None), length) f = self.function([stop, length], [ tensor.as_tensor_variable(cnf[0].start), tensor.as_tensor_variable(cnf[0].stop), tensor.as_tensor_variable(cnf[0].step), tensor.as_tensor_variable(cnf[1])], N=0, op=self.ops) length = 5 a = np.arange(length) for stop in [-8, -5, -4, -1, 0, 1, 4, 5, 8]: out = f(stop, length) t_out = a[out[0]:out[1]:out[2]][::out[3]] v_out = a[None:stop:None] assert np.all(t_out == v_out) assert np.all(t_out.shape == v_out.shape) def grad_list_(self, idxs, data): n = self.shared(data) for idx in idxs: # Should stay on the cpu. idx_ = _shared(np.asarray(idx)) t = n[idx_] gn = theano.tensor.grad(theano.tensor.sum(theano.tensor.exp(t)), n) f = self.function([], [gn, gn.shape], op=self.adv_incsub1) topo = f.maker.fgraph.toposort() if not self.fast_compile: assert any([isinstance(node.op, self.adv_incsub1) and node.op.inplace for node in topo]) else: assert any([isinstance(node.op, self.adv_incsub1) for node in topo]) assert any([isinstance(node.op, self.adv_sub1) for node in topo]) gval, gshape = f() good = np.zeros_like(data) # don't work when the same index is used many time # good[idx] += np.exp(data[idx]) for i in idx: good[i] += np.exp(data[i]) self.assertTrue(gval.ndim == data.ndim) self.assertTrue(np.allclose(gval, good), (gval, good)) self.assertTrue(np.allclose(gshape, data.shape)) def fct(t): return theano.tensor.sum(t[idx_]) utt.verify_grad(fct, [data], mode=self.mode) # Test the grad of the grad (e.i. AdvancedIncSubtensor1.grad) def fct2(t): return theano.tensor.grad(theano.tensor.sum(t[idx_]), t) utt.verify_grad(fct2, [data], mode=self.mode) # Test shape of AdvancedIncSubtensor1 and AdvancedSubtensor1 if not self.fast_compile: ops = (self.adv_incsub1, self.adv_sub1) else: ops = self.ops if idx is idxs[0]: f = self.function([], [gn.shape, n[idx_].shape], op=ops, N=0, N_fast=2) f() def test_wrong_exception_regression(self): a = fscalar() b = fscalar() c = vector() try: c[a:b] except NotImplementedError: self.fail() except TypeError: pass try: c[a:] except NotImplementedError: self.fail() except TypeError: pass try: c[:b] except NotImplementedError: self.fail() except TypeError: pass @attr('slow') def test_grad_list(self): data = rand(4) data = np.asarray(data, dtype=self.dtype) idxs = [[i] for i in range(data.shape[0])] for i in range(data.shape[0]): for j in range(0, data.shape[0], 2): idxs.append([i, j, (i + 1) % data.shape[0]]) self.grad_list_(idxs, data) data = rand(4, 3) data = np.asarray(data, dtype=self.dtype) self.grad_list_(idxs, data) data = rand(4, 3, 2) data = np.asarray(data, dtype=self.dtype) self.grad_list_(idxs, data) def test_shape_list(self): # TODO for all type of subtensor shape for data, idx in [(rand(4), [1, 0]), (rand(4, 2), [2, 3]), (rand(4, 2, 3), [0, 3]), (rand(4, 2, 3), [3, 3, 1, 2, 2, ]), ]: data = np.asarray(data, dtype=self.dtype) n = self.shared(data) t = n[idx] f = self.function([], t.shape, op=self.ops, N=0, N_fast=1) val = f() self.assertTrue(np.allclose(val, data[idx].shape)) def test_grad_advanced_inc_subtensor(self): def inc_slice(*s): def just_numeric_args(a, b): cost = (a[s] + b).sum() cost_wrt_a = theano.tensor.grad(cost, a) cost_wrt_b = theano.tensor.grad(cost, b) grads = cost_wrt_a.sum() + cost_wrt_b.sum() return grads return just_numeric_args # vector utt.verify_grad( inc_slice(slice(2, 4, None)), (np.asarray([0, 1, 2, 3, 4, 5.]), np.asarray([9, 9.]),), mode=self.mode) # matrix utt.verify_grad( inc_slice(slice(1, 2, None), slice(None, None, None)), (np.asarray([[0, 1], [2, 3], [4, 5.]]), np.asarray([[9, 9.]]),), mode=self.mode) # single element utt.verify_grad( inc_slice(2, 1), (np.asarray([[0, 1], [2, 3], [4, 5.]]), np.asarray(9.),), mode=self.mode) def test_inc_and_set_subtensor(self): # Test increment and set with broadcast X = self.shared(np.ones((9, 9)).astype(self.dtype)) y = set_subtensor(X[1::, 1::], 0) f = self.function([], [y], op=self.inc_sub, N=1) out = f() res = np.ones((9, 9)) res[1::, 1::] = 0 assert np.allclose(out, res) def test_advanced1_inc_and_set(self): # Test advanced increment and set. rng = np.random.RandomState(seed=utt.fetch_seed()) all_inputs_var = [] all_inputs_num = [] all_outputs_var = [] all_outputs_num = [] all_params = [] for set_instead_of_inc in (False, True): for inplace in (False, True): for data_shape in ((10,), (4, 5), (1, 2, 3), (4, 5, 6, 7)): data_n_dims = len(data_shape) data_size = np.product(data_shape) # Corresponding numeric variable. data_num_init = np.arange(data_size, dtype=self.dtype) data_num_init = data_num_init.reshape(data_shape) inc_shapes = [data_shape[i:] for i in xrange(0, len(data_shape) + 1)] # Test broadcasting of y. inc_shapes += [(1,) + inc_shapes[-1][1:]] for inc_shape in inc_shapes: inc_n_dims = len(inc_shape) # We copy the numeric value to be 100% sure there is no # risk of accidentally sharing it. data_num = data_num_init.copy() # Symbolic variable to be incremented. # We create a new one every time in order not to # have duplicated variables in the function's inputs data_var = self.type( broadcastable=[False] * data_n_dims, dtype=self.dtype)() # Symbolic variable with rows to be incremented. idx_var = theano.tensor.vector(dtype='int64') n_to_inc = rng.randint(data_shape[0]) if (n_to_inc == 1 and len(inc_shape) > 0 and inc_shape[0] == 1 and data_shape[0] > 1): n_to_inc = 2 # Corresponding numeric variable. # If set_instead_of_inc, we want to avoid repeating # indices, as the order is not guaranteed. idx_num = rng.choice(np.arange(data_shape[0]), n_to_inc, replace=(not set_instead_of_inc)) idx_num = idx_num.astype('int64') # Symbolic variable with increment value. inc_var = self.type( broadcastable=[False] * inc_n_dims, dtype=self.dtype)() # Trick for the case where `inc_shape` is the same as # `data_shape`: what we actually want is the first # shape element to be equal to the number of rows to # increment. if len(inc_shape) == len(data_shape) and ( len(inc_shapes) == 0 or inc_shape[0] != 1): inc_shape = (n_to_inc,) + inc_shape[1:] # The param dtype is needed when inc_shape is empty. # By default, it would return a float and rng.uniform # with NumPy 1.10 will raise a Deprecation warning. inc_size = np.product(inc_shape, dtype='int') # Corresponding numeric variable. inc_num = rng.uniform(size=inc_size).astype(self.dtype) inc_num = inc_num.reshape(inc_shape) # Result of the incrementation. # (i) Theano if set_instead_of_inc: op = set_subtensor else: op = inc_subtensor output = op(data_var[idx_var], inc_var, inplace=inplace) # (ii) Numpy (note that Numpy increments only once # duplicated indices, so we cannot directly use +=). data_copy = data_num.copy() for j, idx in enumerate(idx_num): if len(inc_shape) == len(data_shape): if inc_shape[0] == 1: # Allow broadcasting of y[0] inc_num0 = inc_num[0] if set_instead_of_inc: data_copy[idx] = inc_num0 else: data_copy[idx] += inc_num0 else: # Special case where there is no broadcasting. if set_instead_of_inc: data_copy[idx] = inc_num[j] else: data_copy[idx] += inc_num[j] else: if set_instead_of_inc: data_copy[idx] = inc_num else: data_copy[idx] += inc_num data_var = theano.In(data_var, mutable=True) # Remember data for the Theano function (see below). all_inputs_var += [data_var, idx_var, inc_var] all_inputs_num += [data_num, idx_num, inc_num] all_outputs_var.append(output) all_outputs_num.append(data_copy) all_params.append((set_instead_of_inc, inplace, data_shape, inc_shape)) if False: # Enable for debugging purpose. f = self.function([data_var, idx_var, inc_var], output, accept_inplace=inplace, op=self.adv_incsub1) if inplace: # Ensure calling `f` will not alter `data_num`. data_num = data_num.copy() f_out = f(data_num.copy(), idx_num, inc_num) assert np.allclose(f_out, data_copy) if not inplace: # Sanity check: `data_num` should be intact. assert (data_num == data_num_init).all() # Actual test (we compile a single Theano function to make it faster). orig_warn = theano.config.warn.gpu_set_subtensor1 try: theano.config.warn.gpu_set_subtensor1 = False f = self.function(all_inputs_var, all_outputs_var, accept_inplace=True, op=self.adv_incsub1, N=len(all_outputs_var)) finally: theano.config.warn.gpu_set_subtensor1 = orig_warn f_outs = f(*all_inputs_num) assert len(f_outs) == len(all_outputs_num) for params, f_out, output_num in izip(all_params, f_outs, all_outputs_num): # NB: if this assert fails, it will probably be easier to debug if # you enable the debug code above. assert np.allclose(f_out, output_num), (params, f_out, output_num) def test_adv_constant_arg(self): # Test case provided (and bug detected, gh-607) by John Salvatier m = matrix('m') gv = np.array([0, 1, 3]) g = theano.tensor.constant(gv) i = theano.tensor.lvector('i') # s1 used to fail s1 = m[gv, i] s2 = m[g, i] assert gof.graph.is_same_graph(s1, s2) def test_adv1_inc_sub_notlastdim(self): # Test that taking 1-dimensional advanced indexing # over a dimension that's not the first (outer-most) works. m = matrix('m') i = lvector('i') m1 = set_subtensor(m[:, i], 0) m2 = inc_subtensor(m[:, i], 1) f = theano.function([m, i], [m1, m2], mode=self.mode) m_val = rand(3, 5) i_val = randint_ranged(min=0, max=4, shape=(4,)) m1_ref = m_val.copy() m2_ref = m_val.copy() m1_val, m2_val = f(m_val, i_val) for idx in i_val: m1_ref[:, idx] = 0 m2_ref[:, idx] += 1 assert np.allclose(m1_val, m1_ref), (m1_val, m1_ref) assert np.allclose(m2_val, m2_ref), (m2_val, m2_ref) def test_adv1_inc_sub_notlastdim_2didx(self): # Test that taking 1-dimensional advanced indexing # over a dimension that's not the first (outer-most) works, # if the index is a matrix. m = matrix('m') i = lmatrix('i') m1 = set_subtensor(m[:, i], 0) m2 = inc_subtensor(m[:, i], 1) f = theano.function([m, i], [m1, m2], mode=self.mode) m_val = rand(5, 7) i_val = randint_ranged(min=0, max=6, shape=(4, 2)) m1_ref = m_val.copy() m2_ref = m_val.copy() m1_val, m2_val = f(m_val, i_val) for idx in i_val.ravel(): m1_ref[:, idx] = 0 m2_ref[:, idx] += 1 assert np.allclose(m1_val, m1_ref), (m1_val, m1_ref) assert np.allclose(m2_val, m2_ref), (m2_val, m2_ref) def test_adv1_inc_sub_notlastdim_1_2dval_broadcast(self): # Test that taking 1-dimensional advanced indexing # over a dimension that's not the first (outer-most), # and incrementing/setting with broadcast m = matrix('m') # Test for both vector and matrix as index sym_i = (lvector('i'), lmatrix('i')) shape_i = ((4,), (4, 2)) shape_val = ((3, 1), (3, 1, 1)) # Disable the warning emitted for that case orig_warn = config.warn.inc_set_subtensor1 try: config.warn.inc_set_subtensor1 = False for i, shp_i, shp_v in zip(sym_i, shape_i, shape_val): sub_m = m[:, i] m1 = set_subtensor(sub_m, np.zeros(shp_v)) m2 = inc_subtensor(sub_m, np.ones(shp_v)) f = theano.function([m, i], [m1, m2], mode=self.mode) m_val = rand(3, 5) i_val = randint_ranged(min=0, max=4, shape=shp_i) m1_ref = m_val.copy() m2_ref = m_val.copy() m1_val, m2_val = f(m_val, i_val) for idx in i_val.ravel(): m1_ref[:, idx] = 0 m2_ref[:, idx] += 1 assert np.allclose(m1_val, m1_ref), (m1_val, m1_ref) assert np.allclose(m2_val, m2_ref), (m2_val, m2_ref) finally: config.warn.inc_set_subtensor1 = orig_warn def test_adv1_inc_sub_notlastdim_1_2dval_no_broadcast(self): # Test that taking 1-dimensional advanced indexing # over a dimension that's not the first (outer-most), # and incrementing/setting without broadcast m = matrix('m') # Test for both vector and matrix as index sym_i = (lvector('i'), lmatrix('i')) shape_i = ((4,), (4, 2)) shape_val = ((3, 4), (3, 4, 2)) # Disable the warning emitted for that case orig_warn = config.warn.inc_set_subtensor1 try: config.warn.inc_set_subtensor1 = False for i, shp_i, shp_v in zip(sym_i, shape_i, shape_val): sub_m = m[:, i] m1 = set_subtensor(sub_m, np.zeros(shp_v)) m2 = inc_subtensor(sub_m, np.ones(shp_v)) f = theano.function([m, i], [m1, m2], mode=self.mode) m_val = rand(3, 5) i_val = randint_ranged(min=0, max=4, shape=shp_i) m1_ref = m_val.copy() m2_ref = m_val.copy() m1_val, m2_val = f(m_val, i_val) # We have to explicitly loop over all individual indices, # not as a list or array, numpy only increments the indexed # elements once even if the indices are repeated. for idx in i_val.ravel(): m1_ref[:, idx] = 0 m2_ref[:, idx] += 1 assert np.allclose(m1_val, m1_ref), (m1_val, m1_ref) assert np.allclose(m2_val, m2_ref), (m2_val, m2_ref) finally: config.warn.inc_set_subtensor1 = orig_warn def test_take(self): a = tensor.matrix() f = theano.function( [a], a.take(0, axis=-1), allow_input_downcast=True, mode=self.mode) f(np.random.normal(0, 1, (30, 4))) class TestIncSubtensor1(unittest.TestCase): # test inc_subtensor # also tests set_subtensor def setUp(self): self.rng = np.random.RandomState(seed=utt.fetch_seed()) self.s = tensor.iscalar() self.v = tensor.fvector() self.m = tensor.dmatrix() self.t = tensor.ctensor3() self.adv1q = tensor.lvector() # advanced 1d query def test_cant_adv_idx_into_scalar(self): self.assertRaises(IndexError, lambda: self.s[self.adv1q]) def test_index_into_vec_w_vec(self): a = self.v[self.adv1q] assert a.type == self.v.type def test_1d_set_adv_selection(self): a = set_subtensor(self.v[self.adv1q], self.v[self.adv1q]) assert a.type == self.v.type # TODO: compile a function and verify that the subtensor is removed # completely, because the whole expression is redundant. f = theano.function([self.v, self.adv1q], a, allow_input_downcast=True) aval = f([.4, .9, .1], [1, 2]) assert np.allclose(aval, [.4, 0.9, 0.1]) def test_1d_inc_adv_selection(self): a = inc_subtensor(self.v[self.adv1q], self.v[self.adv1q]) assert a.type == self.v.type f = theano.function([self.v, self.adv1q], a, allow_input_downcast=True) aval = f([.4, .9, .1], [1, 2]) assert np.allclose(aval, [.4, 1.8, 0.2]) def test_1d_inc_adv_selection_w_broadcasting(self): a = inc_subtensor(self.v[self.adv1q], 3.0) assert a.type == self.v.type f = theano.function([self.v, self.adv1q], a, allow_input_downcast=True) aval = f([.4, .9, .1], [1, 2]) assert np.allclose(aval, [.4, 3.9, 3.1]) def test_assigning_matrix_to_vector_selection(self): self.assertRaises(TypeError, lambda: inc_subtensor(self.v[self.adv1q], fmatrix())) def test_matrix_idx(self): idx = tensor.lmatrix() a = self.m[idx] a2 = inc_subtensor(a, a) f = theano.function([self.m, idx], a2) mval = self.rng.random_sample((4, 10)) idxval = np.array([[1, 2], [3, 2]]) a2val = f(mval, idxval) utt.assert_allclose(a2val[0], mval[0]) utt.assert_allclose(a2val[1], mval[1] * 2) utt.assert_allclose(a2val[2], mval[2] * 3) utt.assert_allclose(a2val[3], mval[3] * 2) def test_inc_bcastableidx(self): idx = tensor.constant([0]) c_inc = tensor.col() m_inc = tensor.matrix() out1 = inc_subtensor(self.m[:, idx], c_inc) out2 = inc_subtensor(self.m[:, idx], m_inc) f = theano.function([self.m, c_inc, m_inc], [out1, out2]) mval = self.rng.random_sample((10, 5)) incval = self.rng.random_sample((10, 1)).astype(config.floatX) out1val, out2val = f(mval, incval, incval) utt.assert_allclose(out1val, out2val) class TestAdvancedSubtensor(unittest.TestCase): # test inc_subtensor # also tests set_subtensor def __init__(self, name, shared=tensor._shared, sub=tensor.AdvancedSubtensor, inc_sub=tensor.AdvancedIncSubtensor, mode=None, dtype=theano.config.floatX, ignore_topo=DeepCopyOp): self.shared = shared self.sub = sub self.inc_sub = inc_sub if mode is None: mode = theano.compile.mode.get_default_mode() self.mode = mode self.dtype = dtype self.ignore_topo = ignore_topo super(TestAdvancedSubtensor, self).__init__(name) def setUp(self): self.s = iscalar() self.v = fvector() self.m = dmatrix() self.t = ctensor3() self.ft4 = ftensor4() self.ix1 = lvector() # advanced 1d query self.ix12 = lvector() self.ix2 = lmatrix() self.ixr = lrow() def test_advinc_subtensor(self): x_shp = (20, 15, 10, 5) def check(idx, y_val, x_val, true): x = self.shared(x_val, name='x') y = tensor.tensor(dtype='float32', broadcastable=(False,) * len(y_val.shape), name='y') sym_idx = [tensor.as_tensor_variable(ix) for ix in idx] expr = tensor.advanced_inc_subtensor(x, y, *sym_idx) f = theano.function([y], expr, mode=self.mode) rval = f(y_val) assert np.allclose(rval, true) idxs_y_shp_pairs = [ ((0, [1, 3, 5], 1), (3, 5)), (([1, 2, 4, 8],), (4, 15, 10, 5)), (([0, 1, 2], 0, [0, 1, 2]), (3, 3, 5)), (([[0, 1], [2, 3]], [[0, 1], [2, 3]]), (2, 2, 10, 5)), ] for idx, y_shps in idxs_y_shp_pairs: for i in range(len(y_shps) - 1): y_shp = y_shps[i:] x_val = np.arange(np.prod(x_shp), dtype='float32').reshape(x_shp) + 1 y_val = np.arange(np.prod(y_shp), dtype='float32').reshape(y_shp) + 1 rep = x_val.copy() try: rep[idx] += y_val except ValueError: continue check(idx, y_val, x_val, rep) x_val = np.arange(np.prod(x_shp), dtype='float32').reshape(x_shp) + 1 y_val = np.array(1).astype(np.float32) rep = x_val.copy() rep[idx] += y_val check(idx, y_val, x_val, rep) def eval_output_and_check(self, t): f = inplace_func([], t, mode=self.mode) topo = f.maker.fgraph.toposort() topo_ = [node for node in topo if not isinstance(node.op, self.ignore_topo)] assert len(topo_) == 1 assert isinstance(topo_[0].op, self.sub) tval = f() return tval def test_cant_adv_idx_into_scalar(self): self.assertRaises(IndexError, lambda: self.s[self.ix1]) def test_index_into_vec_w_vec(self): a = self.v[self.ix1] assert a.type == self.v.type, (a.type, self.v.type) def test_index_into_vec_w_matrix(self): a = self.v[self.ix2] assert a.dtype == self.v.dtype, (a.dtype, self.v.dtype) assert a.broadcastable == self.ix2.broadcastable, ( a.broadcastable, self.ix2.broadcastable) def test_index_into_mat_w_row(self): a = self.m[self.ixr] assert a.dtype == self.m.dtype, (a.dtype, self.m.dtype) assert a.broadcastable == (True, False, False) def test_index_w_int_and_vec(self): # like test_ok_list, but with a single index on the first one # data has to have at least 2 dimensions for data, idx in [(rand(4, 5), [2, 3]), (rand(2, 4, 3), [0, 3]), (rand(2, 4, 3), [3, 3, 1, 1, 2, 2, 0, 0]), (rand(2, 4, 3), [3, 3, 1, 1, 2, 2, 0, 0, -1, -2, -3, -4]), # Test 4 dims as gpu code use another algo # in that case This new algo is not as much # optimized for that case. (rand(4, 4, 2, 3), [3, 3, 1, 1, 2, 2, 0, 0, -1, -2, -3, -4]), # Test with TensorConstant index. (rand(2, 4, 3), theano.tensor.constant([3, 3, 1, 1, 2, 2, 0, 0])), ]: data = np.asarray(data, dtype=self.dtype) n = self.shared(data) t = n[0, idx] self.assertTrue(isinstance(t.owner.op, tensor.AdvancedSubtensor)) val = self.eval_output_and_check(t) if isinstance(idx, list): good = data[0, idx] else: good = data[0, idx.data] self.assertTrue(val.ndim == data.ndim - 1) self.assertTrue(np.allclose(val, good), (val, good)) def test_inc_adv_subtensor_w_matrix(self): subt = self.v[self.ix2] a = inc_subtensor(subt, subt) assert a.type == self.v.type, (a.type, self.v.type) f = theano.function([self.v, self.ix2], a, allow_input_downcast=True, mode=self.mode) aval = f([.4, .9, .1], [[1, 2], [1, 2]]) assert np.allclose(aval, [.4, .9 * 3, .1 * 3]) def test_adv_subtensor_w_int_and_matrix(self): subt = self.ft4[0, :, self.ix2, :] f = theano.function([self.ft4, self.ix2], subt, mode=self.mode) ft4v = np.random.random((2, 3, 4, 5)).astype('float32') ix2v = np.asarray([[0, 1], [1, 0]]) aval = f(ft4v, ix2v) rval = ft4v[0, :, ix2v, :] utt.assert_allclose(rval, aval) def test_adv_subtensor_w_none_and_matrix(self): subt = self.ft4[:, None, :, self.ix2, :] f = theano.function([self.ft4, self.ix2], subt, mode=self.mode) ft4v = np.random.random((2, 3, 4, 5)).astype('float32') ix2v = np.asarray([[0, 1], [1, 0]]) aval = f(ft4v, ix2v) rval = ft4v[:, None, :, ix2v, :] utt.assert_allclose(rval, aval) def test_adv_subtensor_w_slice_and_matrix(self): subt = self.ft4[:, 0:1, self.ix2, :] f = theano.function([self.ft4, self.ix2], subt, mode=self.mode) ft4v = np.random.random((2, 3, 4, 5)).astype('float32') ix2v = np.asarray([[0, 1], [1, 0]]) aval = f(ft4v, ix2v) rval = ft4v[:, 0:1, ix2v, :] utt.assert_allclose(rval, aval) def test_adv_subtensor_w_matrix_and_int(self): subt = self.ft4[:, :, self.ix2, 0] f = theano.function([self.ft4, self.ix2], subt, mode=self.mode) ft4v = np.random.random((2, 3, 4, 5)).astype('float32') ix2v = np.asarray([[0, 1], [1, 0]]) aval = f(ft4v, ix2v) rval = ft4v[:, :, ix2v, 0] utt.assert_allclose(rval, aval) def test_adv_subtensor_w_matrix_and_none(self): subt = self.ft4[:, :, self.ix2, None, :] f = theano.function([self.ft4, self.ix2], subt, mode=self.mode) ft4v = np.random.random((2, 3, 4, 5)).astype('float32') ix2v = np.asarray([[0, 1], [1, 0]]) aval = f(ft4v, ix2v) rval = ft4v[:, :, ix2v, None, :] utt.assert_allclose(rval, aval) def test_inc_adv_subtensor_w_2vec(self): subt = self.m[self.ix1, self.ix12] a = inc_subtensor(subt, subt) typ = tensor.TensorType(self.m.type.dtype, self.ix2.type.broadcastable) assert a.type == typ, (a.type, typ) f = theano.function([self.m, self.ix1, self.ix12], a, allow_input_downcast=True, mode=self.mode) aval = f([[.4, .9, .1], [5, 6, 7], [.5, .3, .15]], [1, 2, 1], [0, 1, 0]) assert np.allclose(aval, [[.4, .9, .1], [5 * 3, 6, 7], [.5, .3 * 2, .15]]), aval def test_inc_adv_subtensor_with_broadcasting(self): inc = dscalar() a = inc_subtensor(self.m[self.ix1, self.ix12], inc) g_inc = tensor.grad(a.sum(), inc) assert a.type == self.m.type, (a.type, self.m.type) f = theano.function([self.m, self.ix1, self.ix12, inc], [a, g_inc], allow_input_downcast=True, mode=self.mode) aval, gval = f([[.4, .9, .1], [5, 6, 7], [.5, .3, .15]], [1, 2, 1], [0, 1, 0], 2.1) assert np.allclose(aval, [[.4, .9, .1], [5 + 2.1 * 2, 6, 7], [.5, .3 + 2.1, .15]]), aval assert np.allclose(gval, 3.0), gval def test_inc_adv_subtensor1_with_broadcasting(self): inc = dscalar() a = inc_subtensor(self.m[self.ix1], inc) g_inc = tensor.grad(a.sum(), inc) assert a.type == self.m.type, (a.type, self.m.type) f = theano.function([self.m, self.ix1, inc], [a, g_inc], allow_input_downcast=True, mode=self.mode) aval, gval = f([[.4, .9, .1], [5, 6, 7], [.5, .3, .15]], [0, 1, 0], 2.1) assert np.allclose(aval, [[.4 + 2.1 * 2, .9 + 2.1 * 2, .1 + 2.1 * 2], [5 + 2.1, 6 + 2.1, 7 + 2.1], [.5, .3, .15]]), aval assert np.allclose(gval, 9.0), gval def test_inc_adv_subtensor_with_index_broadcasting(self): a = inc_subtensor(self.m[self.ix1, self.ix2], 2.1) assert a.type == self.m.type, (a.type, self.m.type) f = theano.function([self.m, self.ix1, self.ix2], a, allow_input_downcast=True, mode=self.mode) aval = f([[.4, .9, .1], [5, 6, 7], [.5, .3, .15]], [0, 2, 0], [[0, 1, 0], [2, 2, 2]]) assert np.allclose(aval, [[.4 + 2 * 2.1, .9, .1 + 2 * 2.1], [5, 6, 7], [.5, .3 + 2.1, .15 + 2.1]]), aval def test_advanced_indexing(self): # tests advanced indexing in Theano for 2D and 3D tensors rng = np.random.RandomState(utt.fetch_seed()) a = rng.uniform(size=(3, 3)) b = theano.shared(a) i = tensor.iscalar() j = tensor.iscalar() z = b[[i, j], :] f1 = theano.function([i, j], z, mode=self.mode) cmd = f1(0, 1) == a[[0, 1], :] self.assertTrue(cmd.all()) aa = rng.uniform(size=(4, 2, 3)) bb = theano.shared(aa) k = tensor.iscalar() z = bb[[i, j, k], :, i:k] f2 = theano.function([i, j, k], z, mode=self.mode) cmd = f2(0, 1, 2) == aa[[0, 1, 2], :, 0:2] self.assertTrue(cmd.all()) def test_adv_sub_3d(self): # Reported in https://github.com/Theano/Theano/issues/5674 X = tensor.tensor3("X") xx = np.zeros((3, 2, 2), config.floatX) for i in range(3): for j in range(2): for k in range(2): xx[i, j, k] = 100 * i + 10 * j + k b_idx = np.zeros((2, 2), 'int32') b_idx[0, 1] = 1 b_idx[1, 1] = 2 r_idx = np.arange(xx.shape[1])[:, np.newaxis] c_idx = np.arange(xx.shape[2])[np.newaxis, :] f = theano.function([X], X[b_idx, r_idx, c_idx], mode=self.mode) out = f(xx) utt.assert_allclose(out, xx[b_idx, r_idx, c_idx]) def test_adv_sub_slice(self): # Reported in https://github.com/Theano/Theano/issues/5898 var = self.shared(np.zeros([3, 3], dtype=config.floatX)) slc = tensor.slicetype() f = theano.function([slc], var[slc], mode=self.mode) s = slice(1, 3) f(s) def test_adv_grouped(self): # Reported in https://github.com/Theano/Theano/issues/6152 rng = np.random.RandomState(utt.fetch_seed()) var_v = rng.rand(3, 63, 4).astype(config.floatX) var = self.shared(var_v) idx1_v = rng.randint(0, 61, size=(5, 4)).astype('int32') idx1 = self.shared(idx1_v) idx2 = tensor.arange(4) out = var[:, idx1, idx2] f = theano.function([], out, mode=self.mode) out_v = f() assert out_v.shape == (3, 5, 4) out_np = var_v[:, idx1_v, np.arange(4)] utt.assert_allclose(out_v, out_np) def test_grad(self): ones = np.ones((1, 3), dtype=self.dtype) n = self.shared(ones * 5, broadcastable=(True, False)) idx = tensor.lvector() idx2 = tensor.lvector() t = n[idx, idx2] self.assertTrue(isinstance(t.owner.op, tensor.AdvancedSubtensor)) utt.verify_grad(lambda m: m[[1, 3], [2, 4]], [np.random.rand(5, 5).astype(self.dtype)], mode=self.mode) def fun(x, y): return advanced_inc_subtensor(x, y, [1, 3], [2, 4]) utt.verify_grad(fun, [np.random.rand(5, 5).astype(self.dtype), np.random.rand(2).astype(self.dtype)], mode=self.mode) def fun(x, y): return advanced_set_subtensor(x, y, [1, 3], [2, 4]) utt.verify_grad(fun, [np.random.rand(5, 5).astype(self.dtype), np.random.rand(2).astype(self.dtype)], mode=self.mode) class TestInferShape(utt.InferShapeTester): @attr('slow') def test_infer_shape(self): # IncSubtensor admat = dmatrix() bdmat = dmatrix() advec = dvector() adscal = dscalar() admat_val = rand(5, 4) self._compile_and_check([admat, bdmat], [inc_subtensor(admat[2:4], bdmat)], [admat_val, [[1, 2, 3, 4]]], IncSubtensor) self._compile_and_check([admat, advec], [inc_subtensor(admat[2], advec)], [admat_val, [1, 2, 3, 4]], IncSubtensor) self._compile_and_check([admat, adscal], [inc_subtensor(admat[2, 3], adscal)], [admat_val, 1], IncSubtensor) self._compile_and_check([admat, adscal], [inc_subtensor(admat[1:3, 2], adscal)], [admat_val, 1], IncSubtensor) self._compile_and_check([admat, bdmat], [set_subtensor(admat[2:4], bdmat)], [admat_val, [[1, 2, 3, 4]]], IncSubtensor) self._compile_and_check([admat, advec], [set_subtensor(admat[2], advec)], [admat_val, [1, 2, 3, 4]], IncSubtensor) self._compile_and_check([admat, adscal], [set_subtensor(admat[2, 3], adscal)], [admat_val, 1], IncSubtensor) self._compile_and_check([admat, adscal], [set_subtensor(admat[1:3, 2], adscal)], [admat_val, 1], IncSubtensor) adtens4 = dtensor4() bdtens4 = dtensor4() adtens4_val = rand(3, 4, 2, 5) self._compile_and_check([adtens4, bdtens4], [inc_subtensor(adtens4[::, 2:4, ::, ::], bdtens4)], [adtens4_val, [[[[1, 2, 3, 4, 5]]]]], IncSubtensor, warn=False) self._compile_and_check([adtens4, bdmat], [inc_subtensor(adtens4[2, 2:4, 1, ::], bdmat)], [adtens4_val, [[1, 2, 3, 4, 5]]], IncSubtensor) self._compile_and_check([adtens4, advec], [inc_subtensor(adtens4[0, 1, ::, 4], advec)], [adtens4_val, [1, 2]], IncSubtensor) self._compile_and_check([adtens4, adscal], [inc_subtensor(adtens4[1:3, 1, ::, 2:4], adscal)], [adtens4_val, 1], IncSubtensor) self._compile_and_check([adtens4, bdtens4], [set_subtensor(adtens4[::, 2:4, ::, ::], bdtens4)], [adtens4_val, [[[[1, 2, 3, 4, 5]]]]], IncSubtensor, warn=False) self._compile_and_check([adtens4, bdmat], [set_subtensor(adtens4[2, 2:4, 1, ::], bdmat)], [adtens4_val, [[1, 2, 3, 4, 5]]], IncSubtensor) self._compile_and_check([adtens4, advec], [set_subtensor(adtens4[0, 1, ::, 4], advec)], [adtens4_val, [1, 2]], IncSubtensor) self._compile_and_check([adtens4, adscal], [set_subtensor(adtens4[1:3, 1, ::, 2:4], adscal)], [adtens4_val, 1], IncSubtensor) # AdvancedIncSubtensor1 admat = dmatrix() bdmat = dmatrix() advec = dvector() adscal = dscalar() admat_val = rand(5, 4) aivec_val = [2, 3] self._compile_and_check([admat, bdmat], [set_subtensor(admat[aivec_val], bdmat)], [admat_val, [[1, 2, 3, 4]]], AdvancedIncSubtensor1) aivec_val = [1, 3, 2] self._compile_and_check([admat, advec], [set_subtensor(admat[aivec_val], advec)], [admat_val, [1, 2, 3, 4]], AdvancedIncSubtensor1) aivec_val = [0, 3, 0] self._compile_and_check([admat, adscal], [set_subtensor(admat[aivec_val], adscal)], [admat_val, 1], AdvancedIncSubtensor1) bdtens4 = dtensor4() adtens4_val = rand(4, 3, 2, 5) aivec_val = [2, 3] self._compile_and_check([adtens4, bdtens4], [set_subtensor(adtens4[aivec_val], bdtens4)], [adtens4_val, [[[[1, 2, 3, 4, 5]]]]], AdvancedIncSubtensor1, warn=False) aivec_val = [1, 3, 2] self._compile_and_check([adtens4, advec], [set_subtensor(adtens4[aivec_val], advec)], [adtens4_val, [1, 2, 3, 4, 5]], AdvancedIncSubtensor1) aivec_val = [0, 3, 0] self._compile_and_check([adtens4, adscal], [set_subtensor(adtens4[aivec_val], adscal)], [adtens4_val, 1], AdvancedIncSubtensor1) aivec_val = [2, 3] self._compile_and_check([admat, bdmat], [inc_subtensor(admat[aivec_val], bdmat)], [admat_val, [[1, 2, 3, 4], [5, 6, 7, 8]]], AdvancedIncSubtensor1) aivec_val = [1, 3, 2] self._compile_and_check([admat, advec], [inc_subtensor(admat[aivec_val], advec)], [admat_val, [1, 2, 3, 4]], AdvancedIncSubtensor1) aivec_val = [0, 3, 0] self._compile_and_check([admat, adscal], [inc_subtensor(admat[aivec_val], adscal)], [admat_val, 1], AdvancedIncSubtensor1) bdtens4 = dtensor4() adtens4_val = rand(4, 3, 2, 5) aivec_val = [2, 3] self._compile_and_check([adtens4, bdtens4], [inc_subtensor(adtens4[aivec_val], bdtens4)], [adtens4_val, [[[[1, 2, 3, 4, 5]]], [[[6, 7, 8, 9, 10]]]]], AdvancedIncSubtensor1, warn=False) aivec_val = [1, 2, 1] self._compile_and_check([adtens4, advec], [inc_subtensor(adtens4[aivec_val], advec)], [adtens4_val, [1, 2, 3, 4, 5]], AdvancedIncSubtensor1) aivec_val = [0, 3, 0] self._compile_and_check([adtens4, adscal], [inc_subtensor(adtens4[aivec_val], adscal)], [adtens4_val, 2], AdvancedIncSubtensor1) # AdvancedIncSubtensor aivec_val = [1, 3, 2] bivec_val = [0, 3, 3] advec_val = [23, 24, 25] self._compile_and_check([admat, advec], [set_subtensor(admat[aivec_val, bivec_val], advec)], [admat_val, advec_val], AdvancedIncSubtensor) def test_adv_sub(self): admat = dmatrix() aivec = lvector() bivec = lvector() admat_val = rand(5, 4) aivec_val = [1, 3, 2] bivec_val = [0, 3, 3] self._compile_and_check([admat, aivec, bivec], [admat[aivec, bivec]], [admat_val, aivec_val, bivec_val], AdvancedSubtensor) # Test case that aren't implemented, but make sure they do not crash. self._compile_and_check([admat, aivec], [admat[aivec, 1:3]], [admat_val, aivec_val], AdvancedSubtensor, check_topo=False) self._compile_and_check([admat, aivec], [admat[1:3, aivec]], [admat_val, aivec_val], AdvancedSubtensor, check_topo=False) def test_boolean(self): n = dmatrix() n_val = np.arange(6).reshape((2, 3)) # infer_shape is not implemented, but it should not crash self._compile_and_check([n], [n[n[:, 0] > 2, n[0, :] > 2]], [n_val], tensor.AdvancedBooleanSubtensor, check_topo=False) self._compile_and_check([n], [n[n[:, 0] > 2]], [n_val], tensor.AdvancedBooleanSubtensor, check_topo=False) Theano-1.0.0/theano/tensor/tests/test_shared_randomstreams.py00006000163562€ÿÿÿþ0000007541113203335532026047 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division __docformat__ = "restructuredtext en" import sys import unittest import numpy as np from theano.tensor import raw_random from theano.tensor.shared_randomstreams import RandomStreams from theano import function, shared from theano import tensor from theano import compile, config, gof from theano.tests import unittest_tools as utt class T_SharedRandomStreams(unittest.TestCase): def setUp(self): utt.seed_rng() def test_tutorial(self): srng = RandomStreams(seed=234) rv_u = srng.uniform((2, 2)) rv_n = srng.normal((2, 2)) f = function([], rv_u) g = function([], rv_n, no_default_updates=True) #Not updating rv_n.rng nearly_zeros = function([], rv_u + rv_u - 2 * rv_u) assert np.all(f() != f()) assert np.all(g() == g()) assert np.all(abs(nearly_zeros()) < 1e-5) assert isinstance(rv_u.rng.get_value(borrow=True), np.random.RandomState) def test_basics(self): random = RandomStreams(utt.fetch_seed()) fn = function([], random.uniform((2, 2)), updates=random.updates()) gn = function([], random.normal((2, 2)), updates=random.updates()) fn_val0 = fn() fn_val1 = fn() gn_val0 = gn() rng_seed = np.random.RandomState(utt.fetch_seed()).randint(2**30) rng = np.random.RandomState(int(rng_seed)) # int() is for 32bit # print fn_val0 numpy_val0 = rng.uniform(size=(2, 2)) numpy_val1 = rng.uniform(size=(2, 2)) # print numpy_val0 assert np.allclose(fn_val0, numpy_val0) print(fn_val0) print(numpy_val0) print(fn_val1) print(numpy_val1) assert np.allclose(fn_val1, numpy_val1) def test_seed_fn(self): random = RandomStreams(234) fn = function([], random.uniform((2, 2)), updates=random.updates()) random.seed(utt.fetch_seed()) fn_val0 = fn() fn_val1 = fn() rng_seed = np.random.RandomState(utt.fetch_seed()).randint(2**30) rng = np.random.RandomState(int(rng_seed)) #int() is for 32bit # print fn_val0 numpy_val0 = rng.uniform(size=(2, 2)) numpy_val1 = rng.uniform(size=(2, 2)) # print numpy_val0 assert np.allclose(fn_val0, numpy_val0) assert np.allclose(fn_val1, numpy_val1) def test_getitem(self): random = RandomStreams(234) out = random.uniform((2, 2)) fn = function([], out, updates=random.updates()) random.seed(utt.fetch_seed()) rng = np.random.RandomState() rng.set_state(random[out.rng].get_state()) # tests getitem fn_val0 = fn() fn_val1 = fn() numpy_val0 = rng.uniform(size=(2, 2)) numpy_val1 = rng.uniform(size=(2, 2)) assert np.allclose(fn_val0, numpy_val0) assert np.allclose(fn_val1, numpy_val1) def test_setitem(self): random = RandomStreams(234) out = random.uniform((2, 2)) fn = function([], out, updates=random.updates()) random.seed(888) rng = np.random.RandomState(utt.fetch_seed()) random[out.rng] = np.random.RandomState(utt.fetch_seed()) fn_val0 = fn() fn_val1 = fn() numpy_val0 = rng.uniform(size=(2, 2)) numpy_val1 = rng.uniform(size=(2, 2)) assert np.allclose(fn_val0, numpy_val0) assert np.allclose(fn_val1, numpy_val1) def test_ndim(self): # Test that the behaviour of 'ndim' optional parameter # 'ndim' is an optional integer parameter, specifying the length # of the 'shape', passed as a keyword argument. # ndim not specified, OK random = RandomStreams(utt.fetch_seed()) fn = function([], random.uniform((2, 2))) # ndim specified, consistent with shape, OK random2 = RandomStreams(utt.fetch_seed()) fn2 = function([], random2.uniform((2, 2), ndim=2)) val1 = fn() val2 = fn2() assert np.all(val1 == val2) # ndim specified, inconsistent with shape, should raise ValueError random3 = RandomStreams(utt.fetch_seed()) self.assertRaises(ValueError, random3.uniform, (2, 2), ndim=1) def test_uniform(self): # Test that RandomStreams.uniform generates the same results as numpy # Check over two calls to see if the random state is correctly updated. random = RandomStreams(utt.fetch_seed()) fn = function([], random.uniform((2, 2), -1, 1)) fn_val0 = fn() fn_val1 = fn() rng_seed = np.random.RandomState(utt.fetch_seed()).randint(2**30) rng = np.random.RandomState(int(rng_seed)) # int() is for 32bit numpy_val0 = rng.uniform(-1, 1, size=(2, 2)) numpy_val1 = rng.uniform(-1, 1, size=(2, 2)) assert np.allclose(fn_val0, numpy_val0) assert np.allclose(fn_val1, numpy_val1) def test_normal(self): # Test that RandomStreams.normal generates the same results as numpy # Check over two calls to see if the random state is correctly updated. random = RandomStreams(utt.fetch_seed()) fn = function([], random.normal((2, 2), -1, 2)) fn_val0 = fn() fn_val1 = fn() rng_seed = np.random.RandomState(utt.fetch_seed()).randint(2**30) rng = np.random.RandomState(int(rng_seed)) # int() is for 32bit numpy_val0 = rng.normal(-1, 2, size=(2, 2)) numpy_val1 = rng.normal(-1, 2, size=(2, 2)) assert np.allclose(fn_val0, numpy_val0) assert np.allclose(fn_val1, numpy_val1) def test_random_integers(self): # Test that RandomStreams.random_integers generates the same # results as numpy. We use randint() for numpy since # random_integers() is deprecated. # Check over two calls to see if the random state is correctly updated. random = RandomStreams(utt.fetch_seed()) fn = function([], random.random_integers((20, 20), -5, 5)) fn_val0 = fn() fn_val1 = fn() rng_seed = np.random.RandomState(utt.fetch_seed()).randint(2**30) rng = np.random.RandomState(int(rng_seed)) # int() is for 32bit numpy_val0 = rng.randint(-5, 6, size=(20, 20)) numpy_val1 = rng.randint(-5, 6, size=(20, 20)) assert np.all(fn_val0 == numpy_val0) assert np.all(fn_val1 == numpy_val1) def test_choice(self): # Test that RandomStreams.choice generates the same results as numpy # Check over two calls to see if the random state is correctly updated. random = RandomStreams(utt.fetch_seed()) fn = function([], random.choice((11, 8), 10, 1, 0)) fn_val0 = fn() fn_val1 = fn() rng_seed = np.random.RandomState(utt.fetch_seed()).randint(2**30) rng = np.random.RandomState(int(rng_seed)) # int() is for 32bit numpy_val0 = rng.choice(10, (11, 8), True, None) numpy_val1 = rng.choice(10, (11, 8), True, None) assert np.all(fn_val0 == numpy_val0) assert np.all(fn_val1 == numpy_val1) def test_poisson(self): # Test that RandomStreams.poisson generates the same results as numpy # Check over two calls to see if the random state is correctly updated. random = RandomStreams(utt.fetch_seed()) fn = function([], random.poisson(lam=5, size=(11, 8))) fn_val0 = fn() fn_val1 = fn() rng_seed = np.random.RandomState(utt.fetch_seed()).randint(2**30) rng = np.random.RandomState(int(rng_seed)) # int() is for 32bit numpy_val0 = rng.poisson(lam=5, size=(11, 8)) numpy_val1 = rng.poisson(lam=5, size=(11, 8)) assert np.all(fn_val0 == numpy_val0) assert np.all(fn_val1 == numpy_val1) def test_permutation(self): # Test that RandomStreams.permutation generates the same results as numpy # Check over two calls to see if the random state is correctly updated. random = RandomStreams(utt.fetch_seed()) fn = function([], random.permutation((20,), 10), updates=random.updates()) fn_val0 = fn() fn_val1 = fn() rng_seed = np.random.RandomState(utt.fetch_seed()).randint(2**30) rng = np.random.RandomState(int(rng_seed)) # int() is for 32bit # rng.permutation outputs one vector at a time, so we iterate. numpy_val0 = np.asarray([rng.permutation(10) for i in range(20)]) numpy_val1 = np.asarray([rng.permutation(10) for i in range(20)]) assert np.all(fn_val0 == numpy_val0) assert np.all(fn_val1 == numpy_val1) def test_multinomial(self): # Test that RandomStreams.multinomial generates the same results as numpy # Check over two calls to see if the random state is correctly updated. random = RandomStreams(utt.fetch_seed()) fn = function([], random.multinomial((4, 4), 1, [0.1]*10), updates=random.updates()) fn_val0 = fn() fn_val1 = fn() rng_seed = np.random.RandomState(utt.fetch_seed()).randint(2**30) rng = np.random.RandomState(int(rng_seed)) # int() is for 32bit numpy_val0 = rng.multinomial(1, [0.1]*10, size=(4, 4)) numpy_val1 = rng.multinomial(1, [0.1]*10, size=(4, 4)) assert np.all(fn_val0 == numpy_val0) assert np.all(fn_val1 == numpy_val1) def test_shuffle_row_elements(self): # Test that RandomStreams.shuffle_row_elements generates the right results # Check over two calls to see if the random state is correctly updated. # On matrices, for each row, the elements of that row should be shuffled. # Note that this differs from np.random.shuffle, where all the elements # of the matrix are shuffled. random = RandomStreams(utt.fetch_seed()) m_input = tensor.dmatrix() f = function([m_input], random.shuffle_row_elements(m_input), updates=random.updates()) # Generate the elements to be shuffled val_rng = np.random.RandomState(utt.fetch_seed()+42) in_mval = val_rng.uniform(-2, 2, size=(20, 5)) fn_mval0 = f(in_mval) fn_mval1 = f(in_mval) print(in_mval[0]) print(fn_mval0[0]) print(fn_mval1[0]) assert not np.all(in_mval == fn_mval0) assert not np.all(in_mval == fn_mval1) assert not np.all(fn_mval0 == fn_mval1) rng_seed = np.random.RandomState(utt.fetch_seed()).randint(2**30) rng = np.random.RandomState(int(rng_seed)) numpy_mval0 = in_mval.copy() numpy_mval1 = in_mval.copy() for row in numpy_mval0: rng.shuffle(row) for row in numpy_mval1: rng.shuffle(row) assert np.all(numpy_mval0 == fn_mval0) assert np.all(numpy_mval1 == fn_mval1) # On vectors, the behaviour is the same as np.random.shuffle, # except that it does not work in place, but returns a shuffled vector. random1 = RandomStreams(utt.fetch_seed()) v_input = tensor.dvector() f1 = function([v_input], random1.shuffle_row_elements(v_input)) in_vval = val_rng.uniform(-3, 3, size=(12,)) fn_vval = f1(in_vval) numpy_vval = in_vval.copy() vrng = np.random.RandomState(int(rng_seed)) vrng.shuffle(numpy_vval) print(in_vval) print(fn_vval) print(numpy_vval) assert np.all(numpy_vval == fn_vval) # Trying to shuffle a vector with function that should shuffle # matrices, or vice versa, raises a TypeError self.assertRaises(TypeError, f1, in_mval) self.assertRaises(TypeError, f, in_vval) def test_default_updates(self): # Basic case: default_updates random_a = RandomStreams(utt.fetch_seed()) out_a = random_a.uniform((2, 2)) fn_a = function([], out_a) fn_a_val0 = fn_a() fn_a_val1 = fn_a() assert not np.all(fn_a_val0 == fn_a_val1) nearly_zeros = function([], out_a + out_a - 2 * out_a) assert np.all(abs(nearly_zeros()) < 1e-5) # Explicit updates #1 random_b = RandomStreams(utt.fetch_seed()) out_b = random_b.uniform((2, 2)) fn_b = function([], out_b, updates=random_b.updates()) fn_b_val0 = fn_b() fn_b_val1 = fn_b() assert np.all(fn_b_val0 == fn_a_val0) assert np.all(fn_b_val1 == fn_a_val1) # Explicit updates #2 random_c = RandomStreams(utt.fetch_seed()) out_c = random_c.uniform((2, 2)) fn_c = function([], out_c, updates=[out_c.update]) fn_c_val0 = fn_c() fn_c_val1 = fn_c() assert np.all(fn_c_val0 == fn_a_val0) assert np.all(fn_c_val1 == fn_a_val1) # No updates at all random_d = RandomStreams(utt.fetch_seed()) out_d = random_d.uniform((2, 2)) fn_d = function([], out_d, no_default_updates=True) fn_d_val0 = fn_d() fn_d_val1 = fn_d() assert np.all(fn_d_val0 == fn_a_val0) assert np.all(fn_d_val1 == fn_d_val0) # No updates for out random_e = RandomStreams(utt.fetch_seed()) out_e = random_e.uniform((2, 2)) fn_e = function([], out_e, no_default_updates=[out_e.rng]) fn_e_val0 = fn_e() fn_e_val1 = fn_e() assert np.all(fn_e_val0 == fn_a_val0) assert np.all(fn_e_val1 == fn_e_val0) def test_symbolic_shape(self): random = RandomStreams(utt.fetch_seed()) shape = tensor.lvector() f = function([shape], random.uniform(size=shape, ndim=2)) assert f([2, 3]).shape == (2, 3) assert f([4, 8]).shape == (4, 8) self.assertRaises(ValueError, f, [4]) self.assertRaises(ValueError, f, [4, 3, 4, 5]) def test_mixed_shape(self): # Test when the provided shape is a tuple of ints and scalar vars random = RandomStreams(utt.fetch_seed()) shape0 = tensor.lscalar() shape = (shape0, 3) f = function([shape0], random.uniform(size=shape, ndim=2)) assert f(2).shape == (2, 3) assert f(8).shape == (8, 3) g = function([shape0], random.uniform(size=shape)) assert g(2).shape == (2, 3) assert g(8).shape == (8, 3) def test_mixed_shape_bcastable(self): # Test when the provided shape is a tuple of ints and scalar vars random = RandomStreams(utt.fetch_seed()) shape0 = tensor.lscalar() shape = (shape0, 1) u = random.uniform(size=shape, ndim=2) assert u.broadcastable == (False, True) f = function([shape0], u) assert f(2).shape == (2, 1) assert f(8).shape == (8, 1) v = random.uniform(size=shape) assert v.broadcastable == (False, True) g = function([shape0], v) assert g(2).shape == (2, 1) assert g(8).shape == (8, 1) def test_default_shape(self): random = RandomStreams(utt.fetch_seed()) f = function([], random.uniform()) g = function([], random.multinomial()) # seed_rng is generator for generating *seeds* for RandomStates seed_rng = np.random.RandomState(utt.fetch_seed()) uniform_rng = np.random.RandomState(int(seed_rng.randint(2**30))) multinomial_rng = np.random.RandomState(int(seed_rng.randint(2**30))) val0 = f() val1 = f() numpy_val0 = uniform_rng.uniform() numpy_val1 = uniform_rng.uniform() assert np.allclose(val0, numpy_val0) assert np.allclose(val1, numpy_val1) for i in range(10): # every test has 50% chance of passing even with non-matching random states val2 = g() numpy_val2 = multinomial_rng.multinomial(n=1, pvals=[.5, .5]) assert np.all(val2 == numpy_val2) def test_vector_arguments(self): random = RandomStreams(utt.fetch_seed()) low = tensor.dvector() out = random.uniform(low=low, high=1) assert out.ndim == 1 f = function([low], out) seed_gen = np.random.RandomState(utt.fetch_seed()) numpy_rng = np.random.RandomState(int(seed_gen.randint(2**30))) val0 = f([-5, .5, 0, 1]) val1 = f([.9]) numpy_val0 = numpy_rng.uniform(low=[-5, .5, 0, 1], high=1) numpy_val1 = numpy_rng.uniform(low=[.9], high=1) assert np.all(val0 == numpy_val0) assert np.all(val1 == numpy_val1) high = tensor.vector() outb = random.uniform(low=low, high=high) assert outb.ndim == 1 fb = function([low, high], outb) numpy_rng = np.random.RandomState(int(seed_gen.randint(2**30))) val0b = fb([-4., -2], [-1, 0]) val1b = fb([-4.], [-1]) numpy_val0b = numpy_rng.uniform(low=[-4., -2], high=[-1, 0]) numpy_val1b = numpy_rng.uniform(low=[-4.], high=[-1]) assert np.all(val0b == numpy_val0b) assert np.all(val1b == numpy_val1b) self.assertRaises(ValueError, fb, [-4., -2], [-1, 0, 1]) # TODO: do we want that? #self.assertRaises(ValueError, fb, [-4., -2], [-1]) size = tensor.lvector() outc = random.uniform(low=low, high=high, size=size, ndim=1) fc = function([low, high, size], outc) numpy_rng = np.random.RandomState(int(seed_gen.randint(2**30))) val0c = fc([-4., -2], [-1, 0], [2]) val1c = fc([-4.], [-1], [1]) numpy_val0c = numpy_rng.uniform(low=[-4., -2], high=[-1, 0]) numpy_val1c = numpy_rng.uniform(low=[-4.], high=[-1]) assert np.all(val0c == numpy_val0c) assert np.all(val1c == numpy_val1c) self.assertRaises(ValueError, fc, [-4., -2], [-1, 0], [1]) self.assertRaises(ValueError, fc, [-4., -2], [-1, 0], [1, 2]) self.assertRaises(ValueError, fc, [-4., -2], [-1, 0], [2, 1]) self.assertRaises(ValueError, fc, [-4., -2], [-1], [1]) # TODO: do we want that? #self.assertRaises(ValueError, fc, [-4., -2], [-1], [2]) def test_broadcast_arguments(self): random = RandomStreams(utt.fetch_seed()) low = tensor.dvector() high = tensor.dcol() out = random.uniform(low=low, high=high) assert out.ndim == 2 f = function([low, high], out) rng_seed = np.random.RandomState(utt.fetch_seed()).randint(2**30) numpy_rng = np.random.RandomState(int(rng_seed)) val0 = f([-5, .5, 0, 1], [[1.]]) val1 = f([.9], [[1.], [1.1], [1.5]]) val2 = f([-5, .5, 0, 1], [[1.], [1.1], [1.5]]) numpy_val0 = numpy_rng.uniform(low=[-5, .5, 0, 1], high=[1.]) numpy_val1 = numpy_rng.uniform(low=[.9], high=[[1.], [1.1], [1.5]]) numpy_val2 = numpy_rng.uniform(low=[-5, .5, 0, 1], high=[[1.], [1.1], [1.5]]) assert np.all(val0 == numpy_val0) assert np.all(val1 == numpy_val1) assert np.all(val2 == numpy_val2) def test_uniform_vector(self): random = RandomStreams(utt.fetch_seed()) low = tensor.dvector() high = tensor.dvector() out = random.uniform(low=low, high=high) assert out.ndim == 1 f = function([low, high], out) low_val = [.1, .2, .3] high_val = [1.1, 2.2, 3.3] seed_gen = np.random.RandomState(utt.fetch_seed()) numpy_rng = np.random.RandomState(int(seed_gen.randint(2**30))) # Arguments of size (3,) val0 = f(low_val, high_val) numpy_val0 = numpy_rng.uniform(low=low_val, high=high_val) print('THEANO', val0) print('NUMPY', numpy_val0) assert np.all(val0 == numpy_val0) # arguments of size (2,) val1 = f(low_val[:-1], high_val[:-1]) numpy_val1 = numpy_rng.uniform(low=low_val[:-1], high=high_val[:-1]) print('THEANO', val1) print('NUMPY', numpy_val1) assert np.all(val1 == numpy_val1) # Specifying the size explicitly g = function([low, high], random.uniform(low=low, high=high, size=(3,))) val2 = g(low_val, high_val) numpy_rng = np.random.RandomState(int(seed_gen.randint(2**30))) numpy_val2 = numpy_rng.uniform(low=low_val, high=high_val, size=(3,)) assert np.all(val2 == numpy_val2) self.assertRaises(ValueError, g, low_val[:-1], high_val[:-1]) def test_binomial_vector(self): random = RandomStreams(utt.fetch_seed()) n = tensor.lvector() prob = tensor.vector() out = random.binomial(n=n, p=prob) assert out.ndim == 1 f = function([n, prob], out) n_val = [1, 2, 3] prob_val = np.asarray([.1, .2, .3], dtype=config.floatX) seed_gen = np.random.RandomState(utt.fetch_seed()) numpy_rng = np.random.RandomState(int(seed_gen.randint(2**30))) # Arguments of size (3,) val0 = f(n_val, prob_val) numpy_val0 = numpy_rng.binomial(n=n_val, p=prob_val) assert np.all(val0 == numpy_val0) # arguments of size (2,) val1 = f(n_val[:-1], prob_val[:-1]) numpy_val1 = numpy_rng.binomial(n=n_val[:-1], p=prob_val[:-1]) assert np.all(val1 == numpy_val1) # Specifying the size explicitly g = function([n, prob], random.binomial(n=n, p=prob, size=(3,))) val2 = g(n_val, prob_val) numpy_rng = np.random.RandomState(int(seed_gen.randint(2**30))) numpy_val2 = numpy_rng.binomial(n=n_val, p=prob_val, size=(3,)) assert np.all(val2 == numpy_val2) self.assertRaises(ValueError, g, n_val[:-1], prob_val[:-1]) def test_normal_vector(self): random = RandomStreams(utt.fetch_seed()) avg = tensor.dvector() std = tensor.dvector() out = random.normal(avg=avg, std=std) assert out.ndim == 1 f = function([avg, std], out) avg_val = [1, 2, 3] std_val = [.1, .2, .3] seed_gen = np.random.RandomState(utt.fetch_seed()) numpy_rng = np.random.RandomState(int(seed_gen.randint(2**30))) # Arguments of size (3,) val0 = f(avg_val, std_val) numpy_val0 = numpy_rng.normal(loc=avg_val, scale=std_val) assert np.allclose(val0, numpy_val0) # arguments of size (2,) val1 = f(avg_val[:-1], std_val[:-1]) numpy_val1 = numpy_rng.normal(loc=avg_val[:-1], scale=std_val[:-1]) assert np.allclose(val1, numpy_val1) # Specifying the size explicitly g = function([avg, std], random.normal(avg=avg, std=std, size=(3,))) val2 = g(avg_val, std_val) numpy_rng = np.random.RandomState(int(seed_gen.randint(2**30))) numpy_val2 = numpy_rng.normal(loc=avg_val, scale=std_val, size=(3,)) assert np.allclose(val2, numpy_val2) self.assertRaises(ValueError, g, avg_val[:-1], std_val[:-1]) def test_random_integers_vector(self): random = RandomStreams(utt.fetch_seed()) low = tensor.lvector() high = tensor.lvector() out = random.random_integers(low=low, high=high) assert out.ndim == 1 f = function([low, high], out) low_val = [100, 200, 300] high_val = [110, 220, 330] seed_gen = np.random.RandomState(utt.fetch_seed()) numpy_rng = np.random.RandomState(int(seed_gen.randint(2**30))) # Arguments of size (3,) val0 = f(low_val, high_val) numpy_val0 = np.asarray([numpy_rng.randint(low=lv, high=hv+1) for lv, hv in zip(low_val, high_val)]) assert np.all(val0 == numpy_val0) # arguments of size (2,) val1 = f(low_val[:-1], high_val[:-1]) numpy_val1 = np.asarray([numpy_rng.randint(low=lv, high=hv+1) for lv, hv in zip(low_val[:-1], high_val[:-1])]) assert np.all(val1 == numpy_val1) # Specifying the size explicitly g = function([low, high], random.random_integers(low=low, high=high, size=(3,))) val2 = g(low_val, high_val) numpy_rng = np.random.RandomState(int(seed_gen.randint(2**30))) numpy_val2 = np.asarray([numpy_rng.randint(low=lv, high=hv+1) for lv, hv in zip(low_val, high_val)]) assert np.all(val2 == numpy_val2) self.assertRaises(ValueError, g, low_val[:-1], high_val[:-1]) # Vectorized permutation don't make sense: the only parameter, n, # controls one dimension of the returned tensor. def test_multinomial_vector(self): random = RandomStreams(utt.fetch_seed()) n = tensor.lvector() pvals = tensor.matrix() out = random.multinomial(n=n, pvals=pvals) assert out.ndim == 2 f = function([n, pvals], out) n_val = [1, 2, 3] pvals_val = [[.1, .9], [.2, .8], [.3, .7]] pvals_val = np.asarray(pvals_val, dtype=config.floatX) seed_gen = np.random.RandomState(utt.fetch_seed()) numpy_rng = np.random.RandomState(int(seed_gen.randint(2**30))) # Arguments of size (3,) val0 = f(n_val, pvals_val) numpy_val0 = np.asarray([numpy_rng.multinomial(n=nv, pvals=pv) for nv, pv in zip(n_val, pvals_val)]) assert np.all(val0 == numpy_val0) # arguments of size (2,) val1 = f(n_val[:-1], pvals_val[:-1]) numpy_val1 = np.asarray([numpy_rng.multinomial(n=nv, pvals=pv) for nv, pv in zip(n_val[:-1], pvals_val[:-1])]) assert np.all(val1 == numpy_val1) # Specifying the size explicitly g = function([n, pvals], random.multinomial(n=n, pvals=pvals, size=(3,))) val2 = g(n_val, pvals_val) numpy_rng = np.random.RandomState(int(seed_gen.randint(2**30))) numpy_val2 = np.asarray([numpy_rng.multinomial(n=nv, pvals=pv) for nv, pv in zip(n_val, pvals_val)]) assert np.all(val2 == numpy_val2) self.assertRaises(ValueError, g, n_val[:-1], pvals_val[:-1]) def test_dtype(self): random = RandomStreams(utt.fetch_seed()) low = tensor.lscalar() high = tensor.lscalar() out = random.random_integers(low=low, high=high, size=(20,), dtype='int8') assert out.dtype == 'int8' f = function([low, high], out) val0 = f(0, 9) assert val0.dtype == 'int8' val1 = f(255, 257) assert val1.dtype == 'int8' assert np.all(abs(val1) <= 1) def test_default_dtype(self): random = RandomStreams(utt.fetch_seed()) low = tensor.dscalar() high = tensor.dscalar() # Should not silently downcast from low and high out0 = random.uniform(low=low, high=high, size=(42,)) assert out0.dtype == 'float64' f0 = function([low, high], out0) val0 = f0(-2.1, 3.1) assert val0.dtype == 'float64' # Should downcast, since asked explicitly out1 = random.uniform(low=low, high=high, size=(42,), dtype='float32') assert out1.dtype == 'float32' f1 = function([low, high], out1) val1 = f1(-1.1, 1.1) assert val1.dtype == 'float32' # Should use floatX lowf = tensor.fscalar() highf = tensor.fscalar() outf = random.uniform(low=lowf, high=highf, size=(42,)) assert outf.dtype == config.floatX ff = function([lowf, highf], outf) valf = ff(np.float32(-0.1), np.float32(0.3)) assert valf.dtype == config.floatX def test_shared_constructor_borrow(self): rng = np.random.RandomState(123) s_rng_default = shared(rng) s_rng_True = shared(rng, borrow=True) s_rng_False = shared(rng, borrow=False) # test borrow contract: that False means a copy must have been made assert s_rng_default.container.storage[0] is not rng assert s_rng_False.container.storage[0] is not rng # test current implementation: that True means a copy was not made assert s_rng_True.container.storage[0] is rng # ensure that all the random number generators are in the same state v = rng.randn() v0 = s_rng_default.container.storage[0].randn() v1 = s_rng_False.container.storage[0].randn() assert v == v0 == v1 def test_get_value_borrow(self): rng = np.random.RandomState(123) s_rng = shared(rng) r_ = s_rng.container.storage[0] r_T = s_rng.get_value(borrow=True) r_F = s_rng.get_value(borrow=False) # the contract requires that borrow=False returns a copy assert r_ is not r_F # the current implementation allows for True to return the real thing assert r_ is r_T # either way, the rngs should all be in the same state assert r_.rand() == r_F.rand() def test_get_value_internal_type(self): rng = np.random.RandomState(123) s_rng = shared(rng) # there is no special behaviour required of return_internal_type # this test just ensures that the flag doesn't screw anything up # by repeating the get_value_borrow test. r_ = s_rng.container.storage[0] r_T = s_rng.get_value(borrow=True, return_internal_type=True) r_F = s_rng.get_value(borrow=False, return_internal_type=True) # the contract requires that borrow=False returns a copy assert r_ is not r_F # the current implementation allows for True to return the real thing assert r_ is r_T # either way, the rngs should all be in the same state assert r_.rand() == r_F.rand() def test_set_value_borrow(self): rng = np.random.RandomState(123) s_rng = shared(rng) new_rng = np.random.RandomState(234234) # Test the borrow contract is respected: # assigning with borrow=False makes a copy s_rng.set_value(new_rng, borrow=False) assert new_rng is not s_rng.container.storage[0] assert new_rng.randn() == s_rng.container.storage[0].randn() # Test that the current implementation is actually borrowing when it can. rr = np.random.RandomState(33) s_rng.set_value(rr, borrow=True) assert rr is s_rng.container.storage[0] def test_multiple_rng_aliasing(self): # Test that when we have multiple random number generators, we do not alias # the state_updates member. `state_updates` can be useful when attempting to # copy the (random) state between two similar theano graphs. The test is # meant to detect a previous bug where state_updates was initialized as a # class-attribute, instead of the __init__ function. rng1 = RandomStreams(1234) rng2 = RandomStreams(2392) assert rng1.state_updates is not rng2.state_updates assert rng1.gen_seedgen is not rng2.gen_seedgen def test_random_state_transfer(self): # Test that random state can be transferred from one theano graph to another. class Graph: def __init__(self, seed=123): self.rng = RandomStreams(seed) self.y = self.rng.uniform(size=(1,)) g1 = Graph(seed=123) f1 = function([], g1.y) g2 = Graph(seed=987) f2 = function([], g2.y) for (su1, su2) in zip(g1.rng.state_updates, g2.rng.state_updates): su2[0].set_value(su1[0].get_value()) np.testing.assert_array_almost_equal(f1(), f2(), decimal=6) if __name__ == '__main__': from theano.tests import main main("test_shared_randomstreams") Theano-1.0.0/theano/tensor/tests/test_basic.py00006000163562€ÿÿÿþ0000120353313203335532022722 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division from copy import copy, deepcopy from functools import partial import itertools import logging from nose.plugins.skip import SkipTest from nose.tools import assert_raises import operator import os import sys from tempfile import mkstemp import unittest import warnings from six import iteritems from six.moves import StringIO, reduce from six.moves import xrange # Import builtin min to be able to use it after importing the tensor version. from six.moves.builtins import min as builtin_min import numpy as np from numpy.testing import dec, assert_array_equal, assert_allclose import theano from theano.compat import izip from theano.compat import PY3, exc_message, operator_div from theano import compile, config, function, gof, tensor, shared from theano.compile import DeepCopyOp from theano.compile.mode import get_default_mode from theano.scalar import autocast_float_as, autocast_float from theano.tensor import ( wvector, bvector, argmin, max_and_argmax, cscalar, join, horizontal_stack, vertical_stack, argmax, get_vector_length, fscalar, sum, tensor3, vector, add, addbroadcast, alloc, as_tensor_variable, tensor_from_scalar, ARange, clip, constant, default, diag, dot, batched_dot, dmatrix, dscalar, dvector, eq, eye, fill, flatten, inverse_permutation, tensor4, permute_row_elements, fmatrix, fscalars, grad, inplace, iscalar, matrix, minimum, matrices, maximum, mul, neq, Reshape, row, scalar, scalars, second, smallest, stack, sub, Tensor, tensor_copy, tensordot, TensorType, Tri, tri, tril, triu, unbroadcast, var, Argmax, Join, shape, MaxAndArgmax, lscalar, zvector, exp, get_scalar_constant_value, ivector, reshape, scalar_from_tensor, scal, iscalars, arange, dscalars, fvector, imatrix, numeric_grad, opt, lvector, true_div, max, min, Split, roll, tile, patternbroadcast, Eye, Shape, Dot, PermuteRowElements, ScalarFromTensor, TensorFromScalar, dtensor4, Rebroadcast, Alloc, dtensor3, SpecifyShape, Mean, itensor3, Tile, switch, ExtractDiag, AllocDiag, nonzero, flatnonzero, nonzero_values, stacklists, DimShuffle, hessian, ptp, power, swapaxes, choose, Choose, NoneConst, AllocEmpty, isclose, allclose, mgrid, ogrid, extract_constant, ) from theano.tests import unittest_tools as utt from theano.tests.unittest_tools import attr from theano import change_flags imported_scipy_special = False mode_no_scipy = get_default_mode() try: import scipy.special import scipy.stats imported_scipy_special = True except ImportError: if config.mode == "FAST_COMPILE": mode_no_scipy = "FAST_RUN" floatX = config.floatX if config.mode == "FAST_COMPILE": mode_opt = "FAST_RUN" else: mode_opt = get_default_mode() # Use a seeded random number generator so that unittests are deterministic utt.seed_rng() test_rng = np.random.RandomState(seed=utt.fetch_seed()) # In order to check random values close to the boundaries when designing # new tests, you can use utt.MockRandomState, for instance: # test_rng = MockRandomState(0) # test_rng = MockRandomState(0.99999982) # test_rng = MockRandomState(1) if PY3: def L(i): return i else: def L(i): return long(i) # noqa for Python 3 def inplace_func(inputs, outputs, mode=None, allow_input_downcast=False, on_unused_input='raise', name=None): if mode is None: mode = get_default_mode() return function(inputs, outputs, mode=mode, allow_input_downcast=allow_input_downcast, accept_inplace=True, on_unused_input=on_unused_input, name=name) def eval_outputs(outputs, ops=(), mode=None): f = inplace_func([], outputs, mode=mode) variables = f() if ops: assert any(isinstance(node.op, ops) for node in f.maker.fgraph.apply_nodes) if isinstance(variables, (tuple, list)) and len(variables) == 1: return variables[0] return variables def get_numeric_subclasses(cls=np.number, ignore=None): # Return subclasses of `cls` in the numpy scalar hierarchy. # # We only return subclasses that correspond to unique data types. # The hierarchy can be seen here: # http://docs.scipy.org/doc/numpy/reference/arrays.scalars.html if ignore is None: ignore = [] rval = [] dtype = np.dtype(cls) dtype_num = dtype.num if dtype_num not in ignore: # Safety check: we should be able to represent 0 with this data type. np.array(0, dtype=dtype) rval.append(cls) ignore.append(dtype_num) for sub_ in cls.__subclasses__(): rval += [c for c in get_numeric_subclasses(sub_, ignore=ignore)] return rval def get_numeric_types(with_int=True, with_float=True, with_complex=False, only_theano_types=True): # Return numpy numeric data types. # # :param with_int: Whether to include integer types. # # :param with_float: Whether to include floating point types. # # :param with_complex: Whether to include complex types. # # :param only_theano_types: If True, then numpy numeric data types that are # not supported by Theano are ignored (i.e. those that are not declared in # scalar/basic.py). # # :returns: A list of unique data type objects. Note that multiple data types # may share the same string representation, but can be differentiated through # their `num` attribute. # # Note that when `only_theano_types` is True we could simply return the list # of types defined in the `scalar` module. However with this function we can # test more unique dtype objects, and in the future we may use it to # automatically detect new data types introduced in numpy. if only_theano_types: theano_types = [d.dtype for d in theano.scalar.all_types] rval = [] def is_within(cls1, cls2): # Return True if scalars defined from `cls1` are within the hierarchy # starting from `cls2`. # The third test below is to catch for instance the fact that # one can use ``dtype=numpy.number`` and obtain a float64 scalar, even # though `numpy.number` is not under `numpy.floating` in the class # hierarchy. return (cls1 is cls2 or issubclass(cls1, cls2) or isinstance(np.array([0], dtype=cls1)[0], cls2)) for cls in get_numeric_subclasses(): dtype = np.dtype(cls) if ((not with_complex and is_within(cls, np.complexfloating)) or (not with_int and is_within(cls, np.integer)) or (not with_float and is_within(cls, np.floating)) or (only_theano_types and dtype not in theano_types)): # Ignore this class. continue rval.append([str(dtype), dtype, dtype.num]) # We sort it to be deterministic, then remove the string and num elements. return [x[1] for x in sorted(rval, key=str)] def _numpy_checker(x, y): # Checks if x.data and y.data have the same contents. # Used in DualLinker to compare C version with Python version. x, y = x[0], y[0] if (x.dtype != y.dtype or x.shape != y.shape or np.any(np.abs(x - y) > 1e-10)): raise Exception("Output mismatch.", {'performlinker': x, 'clinker': y}) def safe_make_node(op, *inputs): # Emulate the behaviour of make_node when op is a function. # # Normally op in an instead of the Op class. node = op(*inputs) if isinstance(node, list): return node[0].owner else: return node.owner def upcast_float16_ufunc(fn): # Decorator that enforces computation is not done in float16 by NumPy. # # Some ufuncs in NumPy will compute float values on int8 and uint8 # in half-precision (float16), which is not enough, and not compatible # with the C code. # # :param fn: numpy ufunc # :returns: function similar to fn.__call__, computing the same # value with a minimum floating-point precision of float32 def ret(*args, **kwargs): out_dtype = np.find_common_type( [a.dtype for a in args], [np.float16]) if out_dtype == 'float16': # Force everything to float32 sig = 'f' * fn.nin + '->' + 'f' * fn.nout kwargs.update(sig=sig) return fn(*args, **kwargs) return ret def upcast_int8_nfunc(fn): # Decorator that upcasts input of dtype int8 to float32. # # This is so that floating-point computation is not carried using # half-precision (float16), as some NumPy functions do. # # :param fn: function computing a floating-point value from inputs # :returns: function similar to fn, but upcasting its uint8 and int8 # inputs before carrying out the computation. def ret(*args, **kwargs): args = list(args) for i, a in enumerate(args): if getattr(a, 'dtype', None) in ('int8', 'uint8'): args[i] = a.astype('float32') return fn(*args, **kwargs) return ret def makeTester(name, op, expected, checks=None, good=None, bad_build=None, bad_runtime=None, grad=None, mode=None, grad_rtol=None, eps=1e-10, skip=False, test_memmap=True, check_name=True, grad_eps=None): # :param check_name: # Use only for tester that aren't in Theano. if checks is None: checks = {} if good is None: good = {} if bad_build is None: bad_build = {} if bad_runtime is None: bad_runtime = {} if grad is None: grad = {} if grad is True: grad = good _op, _expected, _checks, _good = op, expected, checks, good _bad_build, _bad_runtime, _grad = bad_build, bad_runtime, grad _mode, _grad_rtol, _eps, skip_ = mode, grad_rtol, eps, skip _test_memmap = test_memmap _check_name = check_name _grad_eps = grad_eps class Checker(unittest.TestCase): op = staticmethod(_op) expected = staticmethod(_expected) checks = _checks check_name = _check_name good = _good bad_build = _bad_build bad_runtime = _bad_runtime grad = _grad mode = _mode skip = skip_ test_memmap = _test_memmap def setUp(self): # Verify that the test's name is correctly set. # Some tests reuse it outside this module. if self.check_name: eval(self.__class__.__module__ + '.' + self.__class__.__name__) # We keep a list of temporary files created in add_memmap_values, # to remove them at the end of the test. self.tmp_files = [] def add_memmap_values(self, val_dict): # If test_memmap is True, we create a temporary file # containing a copy of the data passed in the "val_dict" dict, # then open it as a memmapped array, and we can use the result as a # new test value. if not self.test_memmap: return val_dict # Copy dict before modifying them val_dict = val_dict.copy() # Note that we sort items in the dictionary to ensure tests are # deterministic (since the loop below will break on the first valid # item that can be memmapped). for k, v in sorted(val_dict.items()): new_k = '_'.join((k, 'memmap')) if new_k in val_dict: # A corresponding key was already provided break new_v = [] for inp in v: if type(inp) is np.ndarray and inp.size > 0: f, fname = mkstemp() self.tmp_files.append((f, fname)) new_inp = np.memmap(fname, dtype=inp.dtype, mode='w+', shape=inp.shape) new_inp[...] = inp[...] new_v.append(new_inp) else: new_v.append(inp) val_dict[new_k] = new_v # We only need one value, no need to copy all of them break return val_dict def tearDown(self): # This is to avoid a problem with deleting memmap files on windows. import gc gc.collect() for f, fname in self.tmp_files: os.close(f) os.remove(fname) def test_good(self): if skip: raise SkipTest(skip) good = self.add_memmap_values(self.good) for testname, inputs in iteritems(good): inputs = [copy(input) for input in inputs] inputrs = [TensorType( dtype=input.dtype, broadcastable=[shape_elem == 1 for shape_elem in input.shape] )() for input in inputs] try: node = safe_make_node(self.op, *inputrs) except Exception as exc: err_msg = ("Test %s::%s: Error occurred while" " making a node with inputs %s") % ( self.op, testname, inputs) exc.args += (err_msg,) raise try: f = inplace_func(inputrs, node.outputs, mode=mode, name='test_good') except Exception as exc: err_msg = ("Test %s::%s: Error occurred while" " trying to make a Function") % (self.op, testname) exc.args += (err_msg,) raise if (isinstance(self.expected, dict) and testname in self.expected): expecteds = self.expected[testname] # with numpy version, when we print a number and read it # back, we don't get exactly the same result, so we accept # rounding error in that case. eps = 5e-9 else: expecteds = self.expected(*inputs) eps = 1e-10 if any([i.dtype in ('float32', 'int8', 'uint8', 'uint16') for i in inputs]): eps = 1e-6 eps = np.max([eps, _eps]) try: variables = f(*inputs) except Exception as exc: err_msg = ("Test %s::%s: Error occurred while calling" " the Function on the inputs %s") % ( self.op, testname, inputs) exc.args += (err_msg,) raise if not isinstance(expecteds, (list, tuple)): expecteds = (expecteds, ) for i, (variable, expected) in enumerate( izip(variables, expecteds)): if (variable.dtype != expected.dtype or variable.shape != expected.shape or not np.allclose(variable, expected, atol=eps, rtol=eps)): self.fail(("Test %s::%s: Output %s gave the wrong" " value. With inputs %s, expected %s (dtype %s)," " got %s (dtype %s). eps=%f" " np.allclose returns %s %s") % ( self.op, testname, i, inputs, expected, expected.dtype, variable, variable.dtype, eps, np.allclose(variable, expected, atol=eps, rtol=eps), np.allclose(variable, expected))) for description, check in iteritems(self.checks): if not check(inputs, variables): self.fail(("Test %s::%s: Failed check: %s (inputs" " were %s, outputs were %s)") % ( self.op, testname, description, inputs, variables)) def test_bad_build(self): if skip: raise SkipTest(skip) for testname, inputs in iteritems(self.bad_build): inputs = [copy(input) for input in inputs] inputrs = [shared(input) for input in inputs] self.assertRaises(Exception, safe_make_node, self.op, *inputrs) # The old error string was ("Test %s::%s: %s was successfully # instantiated on the following bad inputs: %s" # % (self.op, testname, node, inputs)) @change_flags(compute_test_value='off') def test_bad_runtime(self): if skip: raise SkipTest(skip) for testname, inputs in iteritems(self.bad_runtime): inputrs = [shared(input) for input in inputs] try: node = safe_make_node(self.op, *inputrs) except Exception as exc: err_msg = ("Test %s::%s: Error occurred while trying" " to make a node with inputs %s") % ( self.op, testname, inputs) exc.args += (err_msg,) raise try: f = inplace_func([], node.outputs, mode=mode, name="test_bad_runtime") except Exception as exc: err_msg = ("Test %s::%s: Error occurred while trying" " to make a Function") % (self.op, testname) exc.args += (err_msg,) raise # Add tester return a ValueError. Should we catch only this # one? # TODO: test that only this one is raised and catch only this # one or the subset that get raised. self.assertRaises(Exception, f, []) def test_grad(self): if skip: raise SkipTest(skip) # Disable old warning that may be triggered by this test. backup = config.warn.sum_div_dimshuffle_bug config.warn.sum_div_dimshuffle_bug = False try: for testname, inputs in iteritems(self.grad): inputs = [copy(input) for input in inputs] try: utt.verify_grad(self.op, inputs, mode=self.mode, rel_tol=_grad_rtol, eps=_grad_eps) except Exception as exc: err_msg = ("Test %s::%s: Error occurred while" " computing the gradient on the following" " inputs: %s") % (self.op, testname, inputs) exc.args += (err_msg,) raise finally: config.warn.sum_div_dimshuffle_bug = backup def test_grad_none(self): # Check that None is never returned as input gradient # when calling self.op.grad # We use all values in self.good because this has to be true # whether or not the values work for utt.verify_grad. if skip: raise SkipTest(skip) if not hasattr(self.op, 'grad'): # This is not actually an Op return for testname, inputs in iteritems(self.good): inputs = [copy(input) for input in inputs] inputrs = [TensorType( dtype=input.dtype, broadcastable=[shape_elem == 1 for shape_elem in input.shape] )() for input in inputs] if (isinstance(self.expected, dict) and testname in self.expected): expecteds = self.expected[testname] # with numpy version, when we print a number and read it # back, we don't get exactly the same result, so we accept # rounding error in that case. else: expecteds = self.expected(*inputs) if not isinstance(expecteds, (list, tuple)): expecteds = (expecteds, ) out_grad_vars = [] for out in expecteds: if str(out.dtype) in tensor.discrete_dtypes: dtype = floatX else: dtype = str(out.dtype) bcast = [shape_elem == 1 for shape_elem in out.shape] var = TensorType(dtype=dtype, broadcastable=bcast)() out_grad_vars.append(var) try: in_grad_vars = self.op.grad(inputrs, out_grad_vars) except (gof.utils.MethodNotDefined, NotImplementedError): pass else: assert None not in in_grad_vars Checker.__name__ = name if hasattr(Checker, '__qualname__'): Checker.__qualname__ = name return Checker def rand(*shape): r = test_rng.rand(*shape) * 2 - 1 return np.asarray(r, dtype=config.floatX) def rand_nonzero(shape, eps=3e-4): # Like rand, but the absolute value has to be at least eps # covers [0, 1) r = np.asarray(test_rng.rand(*shape), dtype=config.floatX) # covers [0, (1 - eps) / 2) U [(1 + eps) / 2, 1) r = r * (1 - eps) + eps * (r >= 0.5) # covers [-1, -eps) U [eps, 1) r = r * 2 - 1 return r def randint(*shape): return test_rng.randint(-5, 6, shape) def randuint32(*shape): return np.array(test_rng.randint(5, size=shape), dtype=np.uint32) def randuint16(*shape): return np.array(test_rng.randint(5, size=shape), dtype=np.uint16) # XXX: this so-called complex random array as all-zero imaginary parts def randcomplex(*shape): r = np.asarray(test_rng.rand(*shape), dtype=config.floatX) return np.complex128(2 * r - 1) def randcomplex_nonzero(shape, eps=1e-4): return np.complex128(rand_nonzero(shape, eps)) def randint_nonzero(*shape): r = test_rng.randint(-5, 5, shape) return r + (r == 0) * 5 def rand_ranged(min, max, shape): return np.asarray(test_rng.rand(*shape) * (max - min) + min, dtype=config.floatX) def randint_ranged(min, max, shape): return test_rng.randint(min, max + 1, shape) def randc128_ranged(min, max, shape): return np.asarray(test_rng.rand(*shape) * (max - min) + min, dtype='complex128') def rand_of_dtype(shape, dtype): if dtype in tensor.discrete_dtypes: return randint(*shape).astype(dtype) elif dtype in tensor.float_dtypes: return rand(*shape).astype(dtype) elif dtype in tensor.complex_dtypes: return randcomplex(*shape).astype(dtype) else: raise TypeError() # Used to exclude random numbers too close to certain values _eps = 1e-2 def makeBroadcastTester(op, expected, checks=None, name=None, **kwargs): if checks is None: checks = {} if name is None: name = str(op) # Here we ensure the test name matches the name of the variable defined in # this script. This is needed to properly identify the test e.g. with the # --with-id option of nosetests, or simply to rerun a specific test that # failed. capitalize = False if name.startswith('Elemwise{') and name.endswith(',no_inplace}'): # For instance: Elemwise{add,no_inplace} -> Add name = name[9:-12] capitalize = True elif name.endswith('_inplace'): # For instance: sub_inplace -> SubInplace capitalize = True if capitalize: name = ''.join([x.capitalize() for x in name.split('_')]) # Some tests specify a name that already ends with 'Tester', while in other # cases we need to add it manually. if not name.endswith('Tester'): name += "Tester" if 'inplace' in kwargs: if kwargs['inplace']: _expected = expected if not isinstance(_expected, dict): def expected(*inputs): return np.array(_expected(*inputs), dtype=inputs[0].dtype) def inplace_check(inputs, outputs): # this used to be inputs[0] is output[0] # I changed it so that it was easier to satisfy by the # DebugMode return np.all(inputs[0] == outputs[0]) checks = dict(checks, inplace_check=inplace_check) del kwargs['inplace'] return makeTester(name, op, expected, checks, **kwargs) _good_broadcast_binary_normal = dict( same_shapes=(rand(2, 3), rand(2, 3)), not_same_dimensions=(rand(2, 2), rand(2)), scalar=(rand(2, 3), rand(1, 1)), row=(rand(2, 3), rand(1, 3)), column=(rand(2, 3), rand(2, 1)), integers=(randint(2, 3), randint(2, 3)), uint32=(randuint32(2, 3), randuint32(2, 3)), uint16=(randuint16(2, 3), randuint16(2, 3)), dtype_mixup_1=(rand(2, 3), randint(2, 3)), dtype_mixup_2=(randint(2, 3), rand(2, 3)), complex1=(randcomplex(2, 3), randcomplex(2, 3)), complex2=(randcomplex(2, 3), rand(2, 3)), # Disabled as we test the case where we reuse the same output as the # first inputs. # complex3=(rand(2,3),randcomplex(2,3)), empty=(np.asarray([], dtype=config.floatX), np.asarray([1], dtype=config.floatX)), ) _bad_build_broadcast_binary_normal = dict() _bad_runtime_broadcast_binary_normal = dict( bad_shapes=(rand(2, 3), rand(3, 2)), bad_row=(rand(2, 3), rand(1, 2))) _grad_broadcast_binary_normal = dict( same_shapes=(rand(2, 3), rand(2, 3)), scalar=(rand(2, 3), rand(1, 1)), row=(rand(2, 3), rand(1, 3)), column=(rand(2, 3), rand(2, 1)), # This don't work as verify grad don't support that # empty=(np.asarray([]), np.asarray([1])) # complex1=(randcomplex(2,3),randcomplex(2,3)), # complex2=(randcomplex(2,3),rand(2,3)), # Disabled as we test the case where we reuse the same output as the # first inputs. # complex3=(rand(2,3),randcomplex(2,3)), ) def check_floatX(inputs, rval): # :param inputs: Inputs to a function that returned `rval` with these inputs. # # :param rval: Value returned by a function with inputs set to `inputs`. # # :returns: Either `rval` unchanged, or `rval` cast in float32. The idea is # that when a numpy function would have returned a float64, Theano may prefer # to return a float32 instead when `config.cast_policy` is set to # 'numpy+floatX' and config.floatX to 'float32', and there was no float64 # input. if (isinstance(rval, np.ndarray) and rval.dtype == 'float64' and config.cast_policy == 'numpy+floatX' and config.floatX == 'float32' and all(x.dtype != 'float64' for x in inputs)): # Then we expect float32 instead of float64. return rval.astype('float32') else: return rval AddTester = makeBroadcastTester( op=add, expected=lambda *inputs: check_floatX( inputs, reduce(lambda x, y: x + y, inputs)), good=dict( three_inputs_same_shapes=(rand(2, 3), rand(2, 3), rand(2, 3)), three_inputs_same_shapes_uint=(randuint32(2, 3), randuint32(2, 3), randuint32(2, 3)), four_inputs_broadcast=(rand(2, 3), rand(1, 3), rand(2, 1), rand(1, 1)), **_good_broadcast_binary_normal), bad_build=_bad_build_broadcast_binary_normal, bad_runtime=_bad_runtime_broadcast_binary_normal) AddInplaceTester = makeBroadcastTester( op=inplace.add_inplace, expected=lambda x, y: x + y, good=_good_broadcast_binary_normal, bad_build=_bad_build_broadcast_binary_normal, bad_runtime=_bad_runtime_broadcast_binary_normal, inplace=True) SubTester = makeBroadcastTester( op=sub, expected=lambda x, y: check_floatX((x, y), x - y), good=_good_broadcast_binary_normal, bad_build=_bad_build_broadcast_binary_normal, bad_runtime=_bad_runtime_broadcast_binary_normal, grad=_grad_broadcast_binary_normal) SubInplaceTester = makeBroadcastTester(op=inplace.sub_inplace, expected=lambda x, y: x - y, good=_good_broadcast_binary_normal, bad_build=_bad_build_broadcast_binary_normal, bad_runtime=_bad_runtime_broadcast_binary_normal, inplace=True) SwitchTester = makeBroadcastTester( op=switch, expected=np.where, good=dict(all_true=(np.asarray(1, dtype=config.floatX), rand(4, 5), rand(4, 5)), false_true=(np.asarray(0, dtype=config.floatX), rand(4, 5), rand(4, 5)), mixed=(randint_ranged(0, 1, (4, 5)), rand(4, 5), rand(4, 5)) ), bad_build=dict(all_true=(np.asarray(1, dtype=config.floatX), rand(4, 5))), bad_runtime=dict(all_true=(np.asarray(1, dtype=config.floatX), rand(3, 5), rand(4, 5)), false_true=(np.asarray(0, dtype=config.floatX), rand(4, 6), rand(4, 5)), ), # We suppose that cond+eps do not switch branch in switch.grad() # So we can't call verify_grad with cond 0. grad=dict(all_true=(np.asarray(1, dtype=config.floatX), rand(4, 5), rand(4, 5)), # false_true=(np.asarray(0, dtype=config.floatX), # rand(4, 5), rand(4, 5)), # mixed=(randint_ranged(0, 1, (4, 5)).astype(config.floatX), # rand(4, 5), rand(4, 5)) ), ) MaximumTester = makeBroadcastTester( op=maximum, expected=lambda *inputs: check_floatX(inputs, np.maximum(*inputs)), good=_good_broadcast_binary_normal, bad_build=_bad_build_broadcast_binary_normal, bad_runtime=_bad_runtime_broadcast_binary_normal, grad=_grad_broadcast_binary_normal) MaximumInplaceTester = makeBroadcastTester( op=inplace.maximum_inplace, expected=np.maximum, good=_good_broadcast_binary_normal, bad_build=_bad_build_broadcast_binary_normal, bad_runtime=_bad_runtime_broadcast_binary_normal, inplace=True) def test_maximum_minimum_grad(): # Test the discontinuity point. # We decided that we only pass the gradient to the first input in that case. x, y = tensor.vectors('xy') for op in [tensor.maximum, tensor.minimum]: o = op(x, y) g = theano.grad(o.sum(), [x, y]) theano.printing.debugprint(g) f = theano.function([x, y], g) assert np.allclose(f([1], [1]), [[1], [0]]) MinimumTester = makeBroadcastTester( op=minimum, expected=lambda *inputs: check_floatX(inputs, np.minimum(*inputs)), good=_good_broadcast_binary_normal, bad_build=_bad_build_broadcast_binary_normal, bad_runtime=_bad_runtime_broadcast_binary_normal, grad=_grad_broadcast_binary_normal) MinimumInplaceTester = makeBroadcastTester( op=inplace.minimum_inplace, expected=np.minimum, good=_good_broadcast_binary_normal, bad_build=_bad_build_broadcast_binary_normal, bad_runtime=_bad_runtime_broadcast_binary_normal, inplace=True) MulTester = makeBroadcastTester( op=mul, expected=lambda *inputs: check_floatX(inputs, reduce(lambda x, y: x * y, inputs)), good=dict(three_inputs_same_shapes=(rand(2, 3), rand(2, 3), rand(2, 3)), four_inputs_broadcast=(rand(2, 3), rand(1, 3), rand(2, 1), rand(1, 1)), **_good_broadcast_binary_normal), bad_build=_bad_build_broadcast_binary_normal, bad_runtime=_bad_runtime_broadcast_binary_normal, grad=dict(three_inputs_same_shapes=(rand(2, 3), rand(2, 3), rand(2, 3)), four_inputs_broadcast=(rand(2, 3), rand(1, 3), rand(2, 1), rand(1, 1)), **_grad_broadcast_binary_normal)) MulInplaceTester = makeBroadcastTester( op=inplace.mul_inplace, expected=lambda x, y: x * y, good=_good_broadcast_binary_normal, bad_build=_bad_build_broadcast_binary_normal, bad_runtime=_bad_runtime_broadcast_binary_normal, inplace=True) def copymod(dct, without=None, **kwargs): # Return dct but with the keys named by args removed, and with # kwargs added. if without is None: without = [] rval = copy(dct) for a in without: if a in rval: del rval[a] for kw, val in iteritems(kwargs): rval[kw] = val return rval _good_broadcast_div_mod_normal_float_no_complex = dict( same_shapes=(rand(2, 3), rand_nonzero((2, 3))), scalar=(rand(2, 3), rand_nonzero((1, 1))), row=(rand(2, 3), rand_nonzero((1, 3))), column=(rand(2, 3), rand_nonzero((2, 1))), dtype_mixup_1=(rand(2, 3), randint_nonzero(2, 3)), dtype_mixup_2=(randint_nonzero(2, 3), rand_nonzero((2, 3))), integer=(randint(2, 3), randint_nonzero(2, 3)), uint8=(randint(2, 3).astype("uint8"), randint_nonzero(2, 3).astype("uint8")), uint16=(randint(2, 3).astype("uint16"), randint_nonzero(2, 3).astype("uint16")), int8=[np.tile(np.arange(-127, 128, dtype='int8'), [254, 1]).T, np.tile(np.array(list(range(-127, 0)) + list(range(1, 128)), dtype='int8'), [255, 1])], # This empty2 doesn't work for some tests. I don't remember why # empty2=(np.asarray([0]), np.asarray([])), ) if PY3: _good_broadcast_div_mod_normal_float_inplace = copymod( _good_broadcast_div_mod_normal_float_no_complex, empty1=(np.asarray([]), np.asarray([1])), # No complex floor division in python 3.x ) else: _good_broadcast_div_mod_normal_float_inplace = copymod( _good_broadcast_div_mod_normal_float_no_complex, empty1=(np.asarray([], dtype=config.floatX), np.asarray([1], dtype=config.floatX)), complex1=(randcomplex(2, 3), randcomplex_nonzero((2, 3))), complex2=(randcomplex(2, 3), rand_nonzero((2, 3))), # Inplace on the first element. Must have the same type. # complex3=(rand(2, 3) ,randcomplex(2, 3)), ) _good_broadcast_div_mod_normal_float = copymod( _good_broadcast_div_mod_normal_float_inplace, empty2=(np.asarray([0], dtype=config.floatX), np.asarray([], dtype=config.floatX)) ) _grad_broadcast_div_mod_normal = dict( same_shapes=(rand(2, 3), rand_nonzero((2, 3))), scalar=(rand(2, 3), rand_nonzero((1, 1))), row=(rand(2, 3), rand_nonzero((1, 3))), column=(rand(2, 3), rand_nonzero((2, 1))), # complex1=(randcomplex(2, 3), randcomplex_nonzero((2, 3))), # complex2=(randcomplex(2, 3), rand_nonzero((2, 3))), # complex3=(rand(2, 3), randcomplex_nonzero((2, 3))), # dtype_mixup_1=(rand(2, 3), randint_nonzero(2, 3)), # dtype_mixup_2=(randint_nonzero(2, 3), rand_nonzero((2, 3))), # empty1=(np.asarray([]), np.asarray([1.])), # empty2=(np.asarray([0]), np.asarray([])), ) div_grad_rtol = None if config.floatX == 'float32': # We raise the relative tolerance for the grad as there can be errors in # float32. # This is probably caused by our way of computing the gradient error. div_grad_rtol = 0.025 def _numpy_true_div(x, y): # Performs true division, and cast the result in the type we expect. # # We define that function so we can use it in TrueDivTester.expected, # because simply calling np.true_divide could cause a dtype mismatch. out = np.true_divide(x, y) # Use floatX as the result of int / int if x.dtype in tensor.discrete_dtypes and y.dtype in tensor.discrete_dtypes: out = theano._asarray(out, dtype=config.floatX) return out TrueDivTester = makeBroadcastTester( op=tensor.true_div, expected=_numpy_true_div, good=_good_broadcast_div_mod_normal_float_no_complex, grad=_grad_broadcast_div_mod_normal, grad_rtol=div_grad_rtol, ) TrueDivInplaceTester = makeBroadcastTester( op=inplace.true_div_inplace, expected=_numpy_true_div, good=copymod( _good_broadcast_div_mod_normal_float_inplace, # The output is now in float, we cannot work inplace on an int. without=['integer', 'uint8', 'uint16', 'int8']), grad_rtol=div_grad_rtol, inplace=True) _good_inv = dict( normal=[5 * rand_nonzero((2, 3))], integers=[randint_nonzero(2, 3)], int8=[np.array(list(range(-127, 0)) + list(range(1, 127)), dtype='int8')], uint8=[np.array(list(range(0, 255)), dtype='uint8')], uint16=[np.array(list(range(0, 65535)), dtype='uint16')], complex=[randcomplex_nonzero((2, 3))], empty=[np.asarray([], dtype=config.floatX)]) _good_inv_inplace = copymod(_good_inv, without=['integers', 'int8', 'uint8', 'uint16', 'complex']) _grad_inv = copymod(_good_inv, without=['integers', 'int8', 'uint8', 'uint16', 'complex', 'empty']) _bad_runtime_inv = dict( float=[np.zeros((2, 3))], integers=[np.zeros((2, 3), dtype='int64')], int8=[np.zeros((2, 3), dtype='int8')], complex=[np.zeros((2, 3), dtype='complex128')]) InvTester = makeBroadcastTester( op=tensor.inv, expected=lambda x: upcast_int8_nfunc(np.true_divide)(np.int8(1), x), good=_good_inv, bad_runtime=_bad_runtime_inv, grad=_grad_inv, grad_rtol=div_grad_rtol) InvInplaceTester = makeBroadcastTester( op=inplace.inv_inplace, expected=lambda x: _numpy_true_div(np.int8(1), x), good=_good_inv_inplace, bad_runtime=_bad_runtime_inv, grad_rtol=div_grad_rtol, inplace=True) CeilIntDivTester = makeBroadcastTester( op=tensor.ceil_intdiv, expected=lambda x, y: check_floatX((x, y), (x // y) + ((x % y) != 0)), good=_good_broadcast_div_mod_normal_float_no_complex, name='CeilIntDiv', # As we implement this function with neq, the gradient returned is always 0. # grad=_grad_broadcast_div_mod_normal, # grad_rtol=div_grad_rtol, ) ModTester = makeBroadcastTester( op=tensor.mod, expected=lambda x, y: np.asarray( x % y, dtype=theano.scalar.basic.upcast(x.dtype, y.dtype)), good=copymod(_good_broadcast_div_mod_normal_float, ['complex1', 'complex2']), grad=_grad_broadcast_div_mod_normal, grad_eps=1e-5, ) ModInplaceTester = makeBroadcastTester( op=inplace.mod_inplace, expected=lambda x, y: np.asarray( x % y, dtype=theano.scalar.basic.upcast(x.dtype, y.dtype)), good=copymod(_good_broadcast_div_mod_normal_float_inplace, ["complex1", "complex2"]), grad_eps=1e-5, inplace=True) _good_broadcast_pow_normal_float = dict( same_shapes=(rand_ranged(1, 5, (2, 3)), rand_ranged(-3, 3, (2, 3))), scalar=(rand_ranged(1, 5, (2, 3)), rand_ranged(-3, 3, (1, 1))), row=(rand_ranged(1, 5, (2, 3)), rand_ranged(-3, 3, (1, 3))), column=(rand_ranged(1, 5, (2, 3)), rand_ranged(-3, 3, (2, 1))), dtype_mixup=(rand_ranged(-3, 3, (2, 3)), randint_ranged(-3, 3, (2, 3))), complex1=(randcomplex(2, 3), randcomplex(2, 3)), complex2=(randcomplex(2, 3), rand(2, 3)), # complex3 = (rand(2,3),randcomplex(2,3)), # Inplace on the first element. empty1=(np.asarray([], dtype=config.floatX), np.asarray([1], dtype=config.floatX)), empty2=(np.asarray([0], dtype=config.floatX), np.asarray([], dtype=config.floatX)), empty3=(np.asarray([], dtype=config.floatX), np.asarray([], dtype=config.floatX)), ) _grad_broadcast_pow_normal = dict( same_shapes=(rand_ranged(1, 5, (2, 3)), rand_ranged(-3, 3, (2, 3))), scalar=(rand_ranged(1, 5, (2, 3)), rand_ranged(-3, 3, (1, 1))), row=(rand_ranged(1, 5, (2, 3)), rand_ranged(-3, 3, (1, 3))), column=(rand_ranged(1, 5, (2, 3)), rand_ranged(-3, 3, (2, 1))), # complex1 = (randcomplex(2,3),randcomplex(2,3)), # complex2 = (randcomplex(2,3),rand(2,3)), # complex3 = (rand(2,3),randcomplex(2,3)), # empty1 = (np.asarray([]), np.asarray([1])), # empty2 = (np.asarray([0]), np.asarray([])), x_eq_zero=( np.asarray([0.], dtype=config.floatX), np.asarray([2.], dtype=config.floatX) ), # Test for issue 1780 ) # empty2 case is not supported by numpy. _good_broadcast_pow_normal_float_pow = copy(_good_broadcast_pow_normal_float) del _good_broadcast_pow_normal_float_pow["empty2"] # Disable NAN checking for pow operator per issue #1780 m = copy(theano.compile.get_default_mode()) m.check_isfinite = False PowTester = makeBroadcastTester( op=pow, expected=lambda x, y: check_floatX((x, y), x ** y), good=_good_broadcast_pow_normal_float, grad=_grad_broadcast_pow_normal, name='Pow', mode=m ) PowInplaceTester = makeBroadcastTester( op=inplace.pow_inplace, expected=lambda x, y: x ** y, good=_good_broadcast_pow_normal_float_pow, inplace=True, mode=m ) # Those are corner case when rounding. Their is many rounding algo. # c round() fct and numpy round are not the same! corner_case = np.asarray( [-2.5, -2., -1.5, -1., -0.5, -.51, -.49, 0, 0.49, 0.5, 0.9, 1, 1.5, 2, 2.5], dtype=floatX) # we remove 0 here as the grad is not always computable numerically. corner_case_grad = np.asarray( [-2.5, -2., -1.5, -1., -0.5, -.51, -.49, 0.49, 0.5, 0.9, 1, 1.5, 2, 2.5], dtype=floatX) _good_broadcast_unary_normal_float = dict( normal=[rand_ranged(-5, 5, (2, 3))], corner_case=[corner_case], complex=[randcomplex(2, 3)], empty=[np.asarray([], dtype=config.floatX)]) _good_broadcast_unary_normal_float_no_empty = copymod( _good_broadcast_unary_normal_float, without=['empty']) _good_broadcast_unary_normal_float_no_empty_no_complex = copymod( _good_broadcast_unary_normal_float_no_empty, without=['complex']) _good_broadcast_unary_normal_float_no_complex = copymod( _good_broadcast_unary_normal_float, without=['complex']) _good_broadcast_unary_normal_float_no_complex_small_neg_range = dict( normal=[rand_ranged(-2, 5, (2, 3))], corner_case=[corner_case], empty=[np.asarray([], dtype=config.floatX)]) _good_broadcast_unary_normal = dict( normal=[np.asarray(rand_ranged(-5, 5, (2, 3)), dtype=config.floatX)], integers=[randint_ranged(-5, 5, (2, 3))], # not using -128 because np.allclose would return False int8=[np.arange(-127, 128, dtype='int8')], uint8=[np.arange(0, 255, dtype='uint8')], uint16=[np.arange(0, 65535, dtype='uint16')], corner_case=[corner_case], complex=[randcomplex(2, 3)], empty=[np.asarray([], dtype=config.floatX)], ) _good_broadcast_unary_normal_no_complex = dict( normal=[np.asarray(rand_ranged(-5, 5, (2, 3)), dtype=floatX)], integers=[randint_ranged(-5, 5, (2, 3))], int8=[np.arange(-127, 128, dtype='int8')], uint8=[np.arange(0, 89, dtype='uint8')], uint16=[np.arange(0, 89, dtype='uint16')], corner_case=[corner_case], empty=[np.asarray([], dtype=config.floatX)], ) _grad_broadcast_unary_normal_no_complex = dict( normal=[np.asarray(rand_ranged(-5, 5, (2, 3)), dtype=floatX)], corner_case=[corner_case_grad]) _grad_broadcast_unary_normal = dict( normal=[np.asarray(rand_ranged(-5, 5, (2, 3)), dtype=floatX)], corner_case=[corner_case_grad], # empty = [np.asarray([])] # XXX: should this be included? ) # Avoid epsilon around integer values _grad_broadcast_unary_normal_noint = dict( normal=[(rand_ranged(_eps, 1 - _eps, (2, 3)) + randint(2, 3)) .astype(floatX)]) _grad_broadcast_unary_normal_small_neg_range = dict( normal=[np.asarray(rand_ranged(-2, 5, (2, 3)), dtype=floatX)], corner_case=[corner_case_grad]) _grad_broadcast_unary_normal_no_complex_no_corner_case = copymod( _grad_broadcast_unary_normal_no_complex, without=['corner_case']) _grad_broadcast_unary_abs1_no_complex = dict( normal=[np.asarray(rand_ranged(-1 + _eps, 1 - _eps, (2, 3)), dtype=floatX)], ) _grad_broadcast_unary_0_2_no_complex = dict( # Don't go too close to 0 or 2 for tests in float32 normal=[np.asarray(rand_ranged(_eps, 1 - _eps, (2, 3)), dtype=floatX)], ) # inplace ops when the input is integer and the output is float* # don't have a well defined behavior. We don't test that case. AbsTester = makeBroadcastTester( op=tensor.abs_, expected=lambda x: abs(x), good=_good_broadcast_unary_normal, grad=_grad_broadcast_unary_normal) _good_broadcast_unary_normal_abs = copy(_good_broadcast_unary_normal) # Can't do inplace on Abs as the input/output are not of the same type! del _good_broadcast_unary_normal_abs['complex'] AbsInplaceTester = makeBroadcastTester( op=inplace.abs__inplace, expected=lambda x: np.abs(x), good=_good_broadcast_unary_normal_abs, inplace=True) NegTester = makeBroadcastTester( op=tensor.neg, expected=lambda x: -x, good=_good_broadcast_unary_normal, grad=_grad_broadcast_unary_normal) NegInplaceTester = makeBroadcastTester( op=inplace.neg_inplace, expected=lambda x: -x, good=_good_broadcast_unary_normal, inplace=True) SgnTester = makeBroadcastTester( op=tensor.sgn, expected=np.sign, good=_good_broadcast_unary_normal_no_complex, grad=_grad_broadcast_unary_normal,) SgnInplaceTester = makeBroadcastTester( op=inplace.sgn_inplace, expected=np.sign, good=_good_broadcast_unary_normal_no_complex, inplace=True) IntDivTester = makeBroadcastTester( op=tensor.int_div, expected=lambda x, y: check_floatX((x, y), x // y), good=_good_broadcast_div_mod_normal_float, # I don't test the grad as the output is always an integer # (this is not a continuous output). # grad=_grad_broadcast_div_mod_normal, ) IntDivInplaceTester = makeBroadcastTester( op=inplace.int_div_inplace, expected=lambda x, y: check_floatX((x, y), x // y), good=_good_broadcast_div_mod_normal_float_inplace, # I don't test the grad as the output is always an integer # (this is not a continuous output). # grad=_grad_broadcast_div_mod_normal, inplace=True ) CeilTester = makeBroadcastTester( op=tensor.ceil, expected=upcast_float16_ufunc(np.ceil), good=_good_broadcast_unary_normal_no_complex, grad=copymod(_grad_broadcast_unary_normal_noint, extra=[np.asarray([-2.5, -1.5, -1.51, 0.49, .98, 1.02], dtype=floatX)])) CeilInplaceTester = makeBroadcastTester( op=inplace.ceil_inplace, expected=upcast_float16_ufunc(np.ceil), good=copymod(_good_broadcast_unary_normal_no_complex, without=['integers', 'int8', 'uint8', 'uint16']), # corner cases includes a lot of integers: points where Ceil is not # continuous (not differentiable) inplace=True) FloorTester = makeBroadcastTester( op=tensor.floor, expected=upcast_float16_ufunc(np.floor), good=_good_broadcast_unary_normal_no_complex, grad=_grad_broadcast_unary_normal_noint) FloorInplaceTester = makeBroadcastTester( op=inplace.floor_inplace, expected=upcast_float16_ufunc(np.floor), good=copymod(_good_broadcast_unary_normal_no_complex, without=["integers", "int8", "uint8", "uint16"]), inplace=True) TruncInplaceTester = makeBroadcastTester( op=inplace.trunc_inplace, expected=upcast_float16_ufunc(np.trunc), good=_good_broadcast_unary_normal_no_complex, inplace=True) TruncTester = makeBroadcastTester( op=tensor.trunc, expected=upcast_float16_ufunc(np.trunc), good=_good_broadcast_unary_normal_no_complex) RoundHalfToEvenTester = makeBroadcastTester( op=tensor.round_half_to_even, expected=np.round, good=_good_broadcast_unary_normal_float_no_complex, grad=_grad_broadcast_unary_normal_no_complex_no_corner_case) RoundHalfToEvenInplaceTester = makeBroadcastTester( op=inplace.round_half_to_even_inplace, expected=np.round, good=_good_broadcast_unary_normal_float_no_complex, inplace=True) # np.vectorize don't handle correctly empty ndarray. # see in their file numpy/lib/function_base.py in class vectorize.__call__ # This happen in float32 mode. RoundHalfAwayFromZeroTester = makeBroadcastTester( op=tensor.round_half_away_from_zero, expected=lambda a: theano.scalar.basic.round_half_away_from_zero_vec(a), good=_good_broadcast_unary_normal_float_no_empty_no_complex, grad=_grad_broadcast_unary_normal_no_complex_no_corner_case) RoundHalfAwayFromZeroInplaceTester = makeBroadcastTester( op=inplace.round_half_away_from_zero_inplace, expected=lambda a: theano.scalar.basic.round_half_away_from_zero_vec(a), good=_good_broadcast_unary_normal_float_no_empty_no_complex, inplace=True) SqrTester = makeBroadcastTester( op=tensor.sqr, expected=np.square, good=_good_broadcast_unary_normal, grad=_grad_broadcast_unary_normal) SqrInplaceTester = makeBroadcastTester( op=inplace.sqr_inplace, expected=np.square, good=_good_broadcast_unary_normal, inplace=True) ExpTester = makeBroadcastTester( op=tensor.exp, expected=upcast_float16_ufunc(np.exp), good=dict(_good_broadcast_unary_normal, int8=[np.arange(-127, 89, dtype='int8')], uint8=[np.arange(0, 89, dtype='uint8')], uint16=[np.arange(0, 89, dtype='uint16')]), grad=_grad_broadcast_unary_normal) ExpInplaceTester = makeBroadcastTester( op=inplace.exp_inplace, expected=np.exp, good=_good_broadcast_unary_normal_float, inplace=True) Exp2Tester = makeBroadcastTester( op=tensor.exp2, expected=upcast_float16_ufunc(np.exp2), good=_good_broadcast_unary_normal, grad=_grad_broadcast_unary_normal) Exp2InplaceTester = makeBroadcastTester( op=inplace.exp2_inplace, expected=np.exp2, good=_good_broadcast_unary_normal_float, inplace=True) Expm1Tester = makeBroadcastTester( op=tensor.expm1, expected=upcast_float16_ufunc(np.expm1), good=dict(_good_broadcast_unary_normal, int8=[np.arange(-127, 89, dtype='int8')], uint8=[np.arange(0, 89, dtype='uint8')], uint16=[np.arange(0, 89, dtype='uint16')]), grad=_grad_broadcast_unary_normal) Expm1InplaceTester = makeBroadcastTester( op=inplace.expm1_inplace, expected=np.expm1, good=_good_broadcast_unary_normal_float, inplace=True) _good_broadcast_unary_positive = dict( normal=(rand_ranged(0.001, 5, (2, 3)),), integers=(randint_ranged(1, 5, (2, 3)),), uint8=[np.arange(1, 256, dtype='uint8')], complex=(randc128_ranged(1, 5, (2, 3)),), empty=(np.asarray([], dtype=config.floatX),), ) _good_broadcast_unary_positive_float = copymod( _good_broadcast_unary_positive, without=['integers', 'uint8']) _grad_broadcast_unary_positive = dict(normal=(rand_ranged(_eps, 5, (2, 3)),),) LogTester = makeBroadcastTester( op=tensor.log, expected=upcast_float16_ufunc(np.log), good=_good_broadcast_unary_positive, grad=_grad_broadcast_unary_positive) LogInplaceTester = makeBroadcastTester( op=inplace.log_inplace, expected=np.log, good=_good_broadcast_unary_positive_float, inplace=True) Log2Tester = makeBroadcastTester( op=tensor.log2, expected=upcast_float16_ufunc(np.log2), good=_good_broadcast_unary_positive, grad=_grad_broadcast_unary_positive) Log2InplaceTester = makeBroadcastTester( op=inplace.log2_inplace, expected=np.log2, good=_good_broadcast_unary_positive_float, inplace=True) Log10Tester = makeBroadcastTester( op=tensor.log10, expected=upcast_float16_ufunc(np.log10), good=_good_broadcast_unary_positive, grad=_grad_broadcast_unary_positive) Log10InplaceTester = makeBroadcastTester( op=inplace.log10_inplace, expected=np.log10, good=_good_broadcast_unary_positive_float, inplace=True) Log1pTester = makeBroadcastTester( op=tensor.log1p, expected=upcast_float16_ufunc(np.log1p), good=_good_broadcast_unary_positive, grad=_grad_broadcast_unary_positive) Log1pInplaceTester = makeBroadcastTester( op=inplace.log1p_inplace, expected=np.log1p, good=_good_broadcast_unary_positive_float, inplace=True) SqrtTester = makeBroadcastTester( op=tensor.sqrt, expected=upcast_float16_ufunc(np.sqrt), good=_good_broadcast_unary_positive, grad=_grad_broadcast_unary_positive) SqrtInplaceTester = makeBroadcastTester( op=inplace.sqrt_inplace, expected=np.sqrt, good=_good_broadcast_unary_positive_float, inplace=True) _good_broadcast_unary_wide = dict( normal=(rand_ranged(-1000, 1000, (2, 3)),), integers=(randint_ranged(-1000, 1000, (2, 3)),), int8=[np.arange(-127, 128, dtype='int8')], uint8=[np.arange(0, 255, dtype='uint8')], uint16=[np.arange(0, 65535, dtype='uint16')], complex=(randc128_ranged(-1000, 1000, (2, 3)),), empty=(np.asarray([], dtype=config.floatX),),) _good_broadcast_unary_wide_float = copymod( _good_broadcast_unary_wide, without=['integers', 'int8', 'uint8', 'uint16']) _grad_broadcast_unary_wide = dict(normal=(rand_ranged(-1000, 1000, (2, 3)),),) if theano.config.floatX == 'float32': angle_eps = 1e-4 else: angle_eps = 1e-10 Deg2radTester = makeBroadcastTester( op=tensor.deg2rad, expected=upcast_float16_ufunc(np.deg2rad), good=_good_broadcast_unary_normal_no_complex, grad=_grad_broadcast_unary_normal_no_complex, eps=angle_eps) Deg2radInplaceTester = makeBroadcastTester( op=inplace.deg2rad_inplace, expected=np.deg2rad, good=_good_broadcast_unary_normal_float_no_complex, inplace=True, eps=angle_eps) Rad2degTester = makeBroadcastTester( op=tensor.rad2deg, expected=upcast_float16_ufunc(np.rad2deg), good=_good_broadcast_unary_normal_no_complex, grad=_grad_broadcast_unary_normal_no_complex, eps=angle_eps) Rad2degInplaceTester = makeBroadcastTester( op=inplace.rad2deg_inplace, expected=np.rad2deg, good=_good_broadcast_unary_normal_float_no_complex, inplace=True, eps=angle_eps) SinTester = makeBroadcastTester( op=tensor.sin, expected=upcast_float16_ufunc(np.sin), good=_good_broadcast_unary_wide, grad=_grad_broadcast_unary_wide) SinInplaceTester = makeBroadcastTester( op=inplace.sin_inplace, expected=np.sin, good=_good_broadcast_unary_wide_float, inplace=True) _good_broadcast_unary_arcsin = dict( normal=(rand_ranged(-1, 1, (2, 3)),), integers=(randint_ranged(-1, 1, (2, 3)),), int8=[np.arange(-1, 2, dtype='int8')], uint8=[np.arange(0, 2, dtype='uint8')], uint16=[np.arange(0, 2, dtype='uint16')], complex=(randc128_ranged(-1, 1, (2, 3)),), empty=(np.asarray([], dtype=config.floatX),),) _good_broadcast_unary_arcsin_float = copymod( _good_broadcast_unary_arcsin, without=['integers', 'int8', 'uint8', 'uint16']) # The actual range is [-1, 1] but the numerical gradient is too # unstable near those values _grad_broadcast_unary_arcsin = dict(normal=(rand_ranged(-0.9, 0.9, (2, 3)),),) ArcsinTester = makeBroadcastTester( op=tensor.arcsin, expected=upcast_float16_ufunc(np.arcsin), good=_good_broadcast_unary_arcsin, grad=_grad_broadcast_unary_arcsin) ArcsinInplaceTester = makeBroadcastTester( op=inplace.arcsin_inplace, expected=np.arcsin, good=_good_broadcast_unary_arcsin_float, inplace=True) CosTester = makeBroadcastTester( op=tensor.cos, expected=upcast_float16_ufunc(np.cos), good=_good_broadcast_unary_wide, grad=_grad_broadcast_unary_wide) CosInplaceTester = makeBroadcastTester( op=inplace.cos_inplace, expected=np.cos, good=_good_broadcast_unary_wide_float, inplace=True) def test_py_c_match(): a = tensor.TensorType(dtype='int8', broadcastable=(False,))() f = theano.function([a], tensor.arccos(a), mode='DebugMode') # This can fail in DebugMode f(np.asarray([1, 0, -1], dtype='int8')) ArccosTester = makeBroadcastTester( op=tensor.arccos, expected=upcast_float16_ufunc(np.arccos), good=_good_broadcast_unary_arcsin, grad=_grad_broadcast_unary_arcsin) ArccosInplaceTester = makeBroadcastTester( op=inplace.arccos_inplace, expected=np.arccos, good=_good_broadcast_unary_arcsin_float, inplace=True) _good_broadcast_unary_tan = dict( normal=(rand_ranged(-3.14, 3.14, (2, 3)),), shifted=(rand_ranged(3.15, 6.28, (2, 3)),), integers=(randint_ranged(-3, 3, (2, 3)),), int8=[np.arange(-3, 4, dtype='int8')], uint8=[np.arange(0, 4, dtype='uint8')], uint16=[np.arange(0, 4, dtype='uint16')], complex=(randc128_ranged(-3.14, 3.14, (2, 3)),), empty=(np.asarray([], dtype=config.floatX),),) # We do not want to test around the discontinuity. _grad_broadcast_unary_tan = dict(normal=(rand_ranged(-1.5, 1.5, (2, 3)),), shifted=(rand_ranged(1.6, 4.6, (2, 3)),)) TanTester = makeBroadcastTester( op=tensor.tan, expected=upcast_float16_ufunc(np.tan), good=_good_broadcast_unary_tan, grad=_grad_broadcast_unary_tan) TanInplaceTester = makeBroadcastTester( op=inplace.tan_inplace, expected=np.tan, good=copymod(_good_broadcast_unary_tan, without=['integers', 'int8', 'uint8', 'uint16']), inplace=True) ArctanTester = makeBroadcastTester( op=tensor.arctan, expected=upcast_float16_ufunc(np.arctan), good=_good_broadcast_unary_wide, grad=_grad_broadcast_unary_wide) ArctanInplaceTester = makeBroadcastTester( op=inplace.arctan_inplace, expected=np.arctan, good=_good_broadcast_unary_wide_float, inplace=True) _good_broadcast_binary_arctan2 = dict( same_shapes=(rand(2, 3), rand(2, 3)), not_same_dimensions=(rand(2, 2), rand(2)), scalar=(rand(2, 3), rand(1, 1)), row=(rand(2, 3), rand(1, 3)), column=(rand(2, 3), rand(2, 1)), integers=(randint(2, 3), randint(2, 3)), int8=[np.arange(-127, 128, dtype='int8'), np.arange(-127, 128, dtype='int8')[:, np.newaxis]], uint8=[np.arange(0, 128, dtype='uint8'), np.arange(0, 128, dtype='uint8')[:, np.newaxis]], uint16=[np.arange(0, 128, dtype='uint16'), np.arange(0, 128, dtype='uint16')[:, np.newaxis]], dtype_mixup_1=(rand(2, 3), randint(2, 3)), dtype_mixup_2=(randint(2, 3), rand(2, 3)), empty=(np.asarray([], dtype=config.floatX), np.asarray([1], dtype=config.floatX)), ) _grad_broadcast_binary_arctan2 = dict( same_shapes=(rand(2, 3), rand(2, 3)), scalar=(rand(2, 3), rand(1, 1)), row=(rand(2, 3), rand(1, 3)), column=(rand(2, 3), rand(2, 1)), ) Arctan2Tester = makeBroadcastTester( op=tensor.arctan2, expected=upcast_float16_ufunc(np.arctan2), good=_good_broadcast_binary_arctan2, grad=_grad_broadcast_binary_arctan2) Arctan2InplaceTester = makeBroadcastTester( op=inplace.arctan2_inplace, expected=np.arctan2, good=copymod(_good_broadcast_binary_arctan2, without=['integers', 'int8', 'uint8', 'uint16', 'dtype_mixup_2']), inplace=True) CoshTester = makeBroadcastTester( op=tensor.cosh, expected=upcast_float16_ufunc(np.cosh), good=dict(_good_broadcast_unary_normal, int8=[np.arange(-89, 90, dtype='int8')], uint8=[np.arange(0, 90, dtype='uint8')], uint16=[np.arange(0, 90, dtype='uint16')]), grad=_grad_broadcast_unary_normal) CoshInplaceTester = makeBroadcastTester( op=inplace.cosh_inplace, expected=np.cosh, good=_good_broadcast_unary_normal_float, inplace=True) _good_broadcast_unary_arccosh = dict( normal=(rand_ranged(1, 1000, (2, 3)),), integers=(randint_ranged(1, 1000, (2, 3)),), uint8=[np.arange(1, 256, dtype='uint8')], complex=(randc128_ranged(1, 1000, (2, 3)),), empty=(np.asarray([], dtype=config.floatX),),) _grad_broadcast_unary_arccosh = dict(normal=(rand_ranged(1 + _eps, 1000, (2, 3)),),) ArccoshTester = makeBroadcastTester( op=tensor.arccosh, expected=upcast_float16_ufunc(np.arccosh), good=_good_broadcast_unary_arccosh, grad=_grad_broadcast_unary_arccosh) ArccoshInplaceTester = makeBroadcastTester( op=inplace.arccosh_inplace, expected=np.arccosh, good=copymod(_good_broadcast_unary_arccosh, without=['integers', 'uint8']), inplace=True) SinhTester = makeBroadcastTester( op=tensor.sinh, expected=upcast_float16_ufunc(np.sinh), good=dict(_good_broadcast_unary_normal, int8=[np.arange(-89, 90, dtype='int8')], uint8=[np.arange(0, 90, dtype='uint8')], uint16=[np.arange(0, 90, dtype='uint16')]), grad=_grad_broadcast_unary_normal) SinhInplaceTester = makeBroadcastTester( op=inplace.sinh_inplace, expected=np.sinh, good=_good_broadcast_unary_normal_float, inplace=True) ArcsinhTester = makeBroadcastTester( op=tensor.arcsinh, expected=upcast_float16_ufunc(np.arcsinh), good=_good_broadcast_unary_normal, grad=_grad_broadcast_unary_normal) ArcsinhInplaceTester = makeBroadcastTester( op=inplace.arcsinh_inplace, expected=np.arcsinh, good=_good_broadcast_unary_normal_float, inplace=True) TanhTester = makeBroadcastTester( op=tensor.tanh, expected=upcast_float16_ufunc(np.tanh), good=_good_broadcast_unary_normal, grad=_grad_broadcast_unary_normal) TanhInplaceTester = makeBroadcastTester( op=inplace.tanh_inplace, expected=np.tanh, good=_good_broadcast_unary_normal_float, inplace=True) _good_broadcast_unary_arctanh = dict( normal=(rand_ranged(-1 + _eps, 1 - _eps, (2, 3)),), integers=(randint_ranged(-1 + _eps, 1 - _eps, (2, 3)),), int8=[np.arange(0, 1, dtype='int8')], uint8=[np.arange(0, 1, dtype='uint8')], uint16=[np.arange(0, 1, dtype='uint16')], complex=(randc128_ranged(-1 + _eps, 1 - _eps, (2, 3)),), empty=(np.asarray([], dtype=config.floatX),),) _grad_broadcast_unary_arctanh = dict( normal=(rand_ranged(-1 + _eps, 1 - _eps, (2, 3)),),) ArctanhTester = makeBroadcastTester( op=tensor.arctanh, expected=upcast_float16_ufunc(np.arctanh), good=_good_broadcast_unary_arctanh, grad=_grad_broadcast_unary_arctanh) ArctanhInplaceTester = makeBroadcastTester( op=inplace.arctanh_inplace, expected=np.arctanh, good=copymod(_good_broadcast_unary_arctanh, without=['integers', 'int8', 'uint8', 'uint16']), inplace=True) # We can't test it if scipy is not installed! # Precomputing the result is brittle(it have been broken!) # As if we do any modification to random number here, # The input random number will change and the output! if imported_scipy_special: expected_erf = scipy.special.erf expected_erfc = scipy.special.erfc expected_erfinv = scipy.special.erfinv expected_erfcinv = scipy.special.erfcinv expected_gamma = scipy.special.gamma expected_gammaln = scipy.special.gammaln expected_psi = scipy.special.psi expected_tri_gamma = partial(scipy.special.polygamma, 1) expected_chi2sf = scipy.stats.chi2.sf expected_j0 = scipy.special.j0 expected_j1 = scipy.special.j1 expected_jv = scipy.special.jv expected_i0 = scipy.special.i0 expected_i1 = scipy.special.i1 expected_iv = scipy.special.iv skip_scipy = False expected_erfcx = scipy.special.erfcx else: expected_erf = [] expected_erfc = [] expected_erfcx = [] expected_erfinv = [] expected_erfcinv = [] expected_gamma = [] expected_gammaln = [] expected_psi = [] expected_tri_gamma = [] expected_chi2sf = [] expected_j0 = [] expected_j1 = [] expected_jv = [] expected_i0 = [] expected_i1 = [] expected_iv = [] skip_scipy = "scipy is not present" ErfTester = makeBroadcastTester( op=tensor.erf, expected=expected_erf, good=_good_broadcast_unary_normal, grad=_grad_broadcast_unary_normal, eps=2e-10, mode=mode_no_scipy, skip=skip_scipy) ErfInplaceTester = makeBroadcastTester( op=inplace.erf_inplace, expected=expected_erf, good=_good_broadcast_unary_normal_float, mode=mode_no_scipy, eps=2e-10, inplace=True, skip=skip_scipy) ErfcTester = makeBroadcastTester( op=tensor.erfc, expected=expected_erfc, good=_good_broadcast_unary_normal_float_no_complex, grad=_grad_broadcast_unary_normal, eps=2e-10, mode=mode_no_scipy, skip=skip_scipy) ErfcInplaceTester = makeBroadcastTester( op=inplace.erfc_inplace, expected=expected_erfc, good=_good_broadcast_unary_normal_float_no_complex, eps=2e-10, mode=mode_no_scipy, inplace=True, skip=skip_scipy) ErfcxTester = makeBroadcastTester( op=tensor.erfcx, expected=expected_erfcx, good=_good_broadcast_unary_normal_float_no_complex_small_neg_range, grad=_grad_broadcast_unary_normal_small_neg_range, eps=2e-10, mode=mode_no_scipy, skip=skip_scipy) ErfcxInplaceTester = makeBroadcastTester( op=inplace.erfcx_inplace, expected=expected_erfcx, good=_good_broadcast_unary_normal_float_no_complex_small_neg_range, eps=2e-10, mode=mode_no_scipy, inplace=True, skip=skip_scipy) ErfinvTester = makeBroadcastTester( op=tensor.erfinv, expected=expected_erfinv, good={'normal': [rand_ranged(-.9, .9, (2, 3))], 'empty': [np.asarray([], dtype=config.floatX)]}, grad=_grad_broadcast_unary_abs1_no_complex, eps=2e-10, mode=mode_no_scipy, skip=skip_scipy) ErfcinvTester = makeBroadcastTester( op=tensor.erfcinv, expected=expected_erfcinv, good={'normal': [rand_ranged(0.001, 1.9, (2, 3))], 'empty': [np.asarray([], dtype=config.floatX)]}, grad=_grad_broadcast_unary_0_2_no_complex, eps=2e-10, mode=mode_no_scipy, skip=skip_scipy) _good_broadcast_unary_gammaln = dict( normal=(rand_ranged(-1 + 1e-2, 10, (2, 3)),), empty=(np.asarray([], dtype=config.floatX),), int=(randint_ranged(1, 10, (2, 3)),), uint8=(randint_ranged(1, 6, (2, 3)).astype('uint8'),), uint16=(randint_ranged(1, 10, (2, 3)).astype('uint16'),), uint64=(randint_ranged(1, 10, (2, 3)).astype('uint64'),)) _grad_broadcast_unary_gammaln = dict( # smaller range as our grad method does not estimate it well enough. normal=(rand_ranged(1e-1, 8, (2, 3)),),) GammaTester = makeBroadcastTester( op=tensor.gamma, expected=expected_gamma, good=_good_broadcast_unary_gammaln, grad=_grad_broadcast_unary_gammaln, mode=mode_no_scipy, eps=1e-5, skip=skip_scipy) GammaInplaceTester = makeBroadcastTester( op=inplace.gamma_inplace, expected=expected_gamma, good=_good_broadcast_unary_gammaln, mode=mode_no_scipy, eps=1e-5, inplace=True, skip=skip_scipy) GammalnTester = makeBroadcastTester( op=tensor.gammaln, expected=expected_gammaln, good=_good_broadcast_unary_gammaln, grad=_grad_broadcast_unary_gammaln, eps=2e-10, mode=mode_no_scipy, skip=skip_scipy) GammalnInplaceTester = makeBroadcastTester( op=inplace.gammaln_inplace, expected=expected_gammaln, good=_good_broadcast_unary_gammaln, eps=2e-10, mode=mode_no_scipy, inplace=True, skip=skip_scipy) _good_broadcast_unary_psi = dict( normal=(rand_ranged(1, 10, (2, 3)),), empty=(np.asarray([], dtype=config.floatX),), int=(randint_ranged(1, 10, (2, 3)),), uint8=(randint_ranged(1, 10, (2, 3)).astype('uint8'),), uint16=(randint_ranged(1, 10, (2, 3)).astype('uint16'),)) PsiTester = makeBroadcastTester( op=tensor.psi, expected=expected_psi, good=_good_broadcast_unary_psi, eps=2e-10, mode=mode_no_scipy, skip=skip_scipy) PsiInplaceTester = makeBroadcastTester( op=inplace.psi_inplace, expected=expected_psi, good=_good_broadcast_unary_psi, eps=2e-10, mode=mode_no_scipy, inplace=True, skip=skip_scipy) _good_broadcast_unary_tri_gamma = _good_broadcast_unary_psi TriGammaTester = makeBroadcastTester( op=tensor.tri_gamma, expected=expected_tri_gamma, good=_good_broadcast_unary_psi, eps=2e-8, mode=mode_no_scipy, skip=skip_scipy) TriGammaInplaceTester = makeBroadcastTester( op=inplace.tri_gamma_inplace, expected=expected_tri_gamma, good=_good_broadcast_unary_tri_gamma, eps=2e-8, mode=mode_no_scipy, inplace=True, skip=skip_scipy) # chi2sf takes two inputs, a value (x) and a degrees of freedom (k). # not sure how to deal with that here... _good_broadcast_unary_chi2sf = dict( normal=(rand_ranged(1, 10, (2, 3)), np.asarray(1, dtype=config.floatX)), empty=(np.asarray([], dtype=config.floatX), np.asarray(1, dtype=config.floatX)), integers=(randint_ranged(1, 10, (2, 3)), np.asarray(1, dtype=config.floatX)), uint8=(randint_ranged(1, 10, (2, 3)).astype('uint8'), np.asarray(1, dtype=config.floatX)), uint16=(randint_ranged(1, 10, (2, 3)).astype('uint16'), np.asarray(1, dtype=config.floatX))) Chi2SFTester = makeBroadcastTester( op=tensor.chi2sf, expected=expected_chi2sf, good=_good_broadcast_unary_chi2sf, eps=2e-10, mode=mode_no_scipy, skip=skip_scipy, name='Chi2SF') Chi2SFInplaceTester = makeBroadcastTester( op=inplace.chi2sf_inplace, expected=expected_chi2sf, good=_good_broadcast_unary_chi2sf, eps=2e-10, mode=mode_no_scipy, inplace=True, skip=skip_scipy, name='Chi2SF') _good_broadcast_unary_bessel = dict( normal=(rand_ranged(-10, 10, (2, 3)),), empty=(np.asarray([], dtype=config.floatX),), int=(randint_ranged(-10, 10, (2, 3)),), uint8=(randint_ranged(0, 10, (2, 3)).astype('uint8'),), uint16=(randint_ranged(0, 10, (2, 3)).astype('uint16'),)) _grad_broadcast_unary_bessel = dict( normal=(rand_ranged(-10., 10., (2, 3)),),) _good_broadcast_binary_bessel = dict( normal=(rand_ranged(-5, 5, (2, 3)), rand_ranged(0, 10, (2, 3))), empty=(np.asarray([], dtype=config.floatX), np.asarray([], dtype=config.floatX)), integers=(randint_ranged(-5, 5, (2, 3)), randint_ranged(-10, 10, (2, 3))), uint8=(randint_ranged(0, 5, (2, 3)).astype('uint8'), randint_ranged(0, 10, (2, 3)).astype('uint8')), uint16=(randint_ranged(0, 5, (2, 3)).astype('uint16'), randint_ranged(0, 10, (2, 3)).astype('uint16'))) _grad_broadcast_binary_bessel = dict( normal=(rand_ranged(1, 5, (2, 3)), rand_ranged(0, 10, (2, 3)))) J0Tester = makeBroadcastTester( op=tensor.j0, expected=expected_j0, good=_good_broadcast_unary_bessel, grad=_grad_broadcast_unary_bessel, eps=2e-10, mode=mode_no_scipy, skip=skip_scipy) J0InplaceTester = makeBroadcastTester( op=inplace.j0_inplace, expected=expected_j0, good=_good_broadcast_unary_bessel, eps=2e-10, mode=mode_no_scipy, inplace=True, skip=skip_scipy) J1Tester = makeBroadcastTester( op=tensor.j1, expected=expected_j1, good=_good_broadcast_unary_bessel, grad=_grad_broadcast_unary_bessel, eps=2e-10, mode=mode_no_scipy, skip=skip_scipy) J1InplaceTester = makeBroadcastTester( op=inplace.j1_inplace, expected=expected_j1, good=_good_broadcast_unary_bessel, eps=2e-10, mode=mode_no_scipy, inplace=True, skip=skip_scipy) JvTester = makeBroadcastTester( op=tensor.jv, expected=expected_jv, good=_good_broadcast_binary_bessel, eps=2e-10, mode=mode_no_scipy, skip=skip_scipy) JvInplaceTester = makeBroadcastTester( op=inplace.jv_inplace, expected=expected_jv, good=_good_broadcast_binary_bessel, eps=2e-10, mode=mode_no_scipy, inplace=True, skip=skip_scipy) def test_verify_jv_grad(): # Verify Jv gradient. # Implemented separately due to need to fix first input for which grad is # not defined. if skip_scipy: raise SkipTest("SciPy needed") v_val, x_val = _grad_broadcast_binary_bessel['normal'] def fixed_first_input_jv(x): return tensor.jv(v_val, x) utt.verify_grad(fixed_first_input_jv, [x_val]) I0Tester = makeBroadcastTester( op=tensor.i0, expected=expected_i0, good=_good_broadcast_unary_bessel, grad=_grad_broadcast_unary_bessel, eps=2e-10, mode=mode_no_scipy, skip=skip_scipy) I0InplaceTester = makeBroadcastTester( op=inplace.i0_inplace, expected=expected_i0, good=_good_broadcast_unary_bessel, eps=2e-10, mode=mode_no_scipy, inplace=True, skip=skip_scipy) I1Tester = makeBroadcastTester( op=tensor.i1, expected=expected_i1, good=_good_broadcast_unary_bessel, grad=_grad_broadcast_unary_bessel, eps=2e-10, mode=mode_no_scipy, skip=skip_scipy) I1InplaceTester = makeBroadcastTester( op=inplace.i1_inplace, expected=expected_i1, good=_good_broadcast_unary_bessel, eps=2e-10, mode=mode_no_scipy, inplace=True, skip=skip_scipy) IvTester = makeBroadcastTester( op=tensor.iv, expected=expected_iv, good=_good_broadcast_binary_bessel, eps=2e-10, mode=mode_no_scipy, skip=skip_scipy) IvInplaceTester = makeBroadcastTester( op=inplace.iv_inplace, expected=expected_iv, good=_good_broadcast_binary_bessel, eps=2e-10, mode=mode_no_scipy, inplace=True, skip=skip_scipy) def test_verify_iv_grad(): # Verify Iv gradient. # Implemented separately due to need to fix first input for which grad is # not defined. if skip_scipy: raise SkipTest("SciPy needed") v_val, x_val = _grad_broadcast_binary_bessel['normal'] def fixed_first_input_iv(x): return tensor.iv(v_val, x) utt.verify_grad(fixed_first_input_iv, [x_val]) ZerosLikeTester = makeBroadcastTester( op=tensor.zeros_like, expected=np.zeros_like, good=_good_broadcast_unary_normal, grad=_grad_broadcast_unary_normal, name='ZerosLike') OnesLikeTester = makeBroadcastTester( op=tensor.ones_like, expected=np.ones_like, good=_good_broadcast_unary_normal, grad=_grad_broadcast_unary_normal, name='OnesLike') # Complex operations _good_complex_from_polar = dict( same_shapes=(abs(rand(2, 3)), rand(2, 3)), not_same_dimensions=(abs(rand(2, 2)), rand(2)), scalar=(abs(rand(2, 3)), rand(1, 1)), row=(abs(rand(2, 3)), rand(1, 3)), column=(abs(rand(2, 3)), rand(2, 1)), integers=(abs(randint(2, 3)), randint(2, 3)), empty=(np.asarray([], dtype=config.floatX), np.asarray([1], dtype=config.floatX)),) _grad_complex_from_polar = dict( same_shapes=(abs(rand(2, 3)), rand(2, 3)), scalar=(abs(rand(2, 3)), rand(1, 1)), row=(abs(rand(2, 3)), rand(1, 3)), column=(abs(rand(2, 3)), rand(2, 1))) ComplexFromPolarTester = makeBroadcastTester( op=tensor.complex_from_polar, expected=lambda r, theta: r * np.cos(theta) + 1j * r * np.sin(theta), good=_good_complex_from_polar) ConjTester = makeBroadcastTester( op=tensor.conj, expected=np.conj, good=_good_broadcast_unary_normal) ConjInplaceTester = makeBroadcastTester( op=inplace.conj_inplace, expected=np.conj, good=_good_broadcast_unary_normal, inplace=True) DotTester = makeTester( name='DotTester', op=dot, expected=lambda x, y: np.dot(x, y), checks={}, good=dict(correct1=(rand(5, 7), rand(7, 5)), correct2=(rand(5, 7), rand(7, 9)), correct3=(rand(5, 7), rand(7)), correct4=(rand(5), rand(5, 7)), mixed1=(rand(5).astype('float32'), rand(5, 7)), mixed2=(rand(5).astype('float64'), rand(5, 7)), complex1=(randcomplex(5, 7), randcomplex(7)), complex2=(rand(5, 7), randcomplex(7)), complex3=(randcomplex(5, 7), rand(7)), empty1=(np.asarray([], dtype=config.floatX), np.asarray([], dtype=config.floatX)), empty2=(rand(5, 0), rand(0, 2)), empty3=(rand(0, 5), rand(5, 0)), ), bad_build=dict(), bad_runtime=dict(bad1=(rand(5, 7), rand(5, 7)), bad2=(rand(5, 7), rand(8, 3)))) BatchedDotTester = makeTester( name='BatchedDotTester', op=batched_dot, expected=(lambda xs, ys: np.asarray( list(x * y if x.ndim == 0 or y.ndim == 0 else np.dot(x, y) for x, y in zip(xs, ys)), dtype=theano.scalar.upcast(xs.dtype, ys.dtype))), checks={}, grad=dict(correct1=(rand(3, 5, 7), rand(3, 7, 5)), correct2=(rand(3, 5, 7), rand(3, 7, 9)), correct3=(rand(3, 5, 7), rand(3, 7)), correct4=(rand(3, 5), rand(3, 5, 7)), correct5=(rand(3), rand(3, 5, 7)), correct6=(rand(3, 5), rand(3)), correct7=(rand(3, 5), rand(3, 5)), correct8=(rand(3), rand(3)), correct9=(rand(3, 5, 7, 11), rand(3)), correct10=(rand(3, 2, 6, 5), rand(3, 5)), correct11=(rand(3, 2, 6, 5), rand(3, 5, 7)), correct12=(rand(3, 2, 6, 5), rand(3, 7, 5, 8)), mixed1=(rand(3, 5).astype('float32'), rand(3, 5, 7)), mixed2=(rand(3, 5).astype('float64'), rand(3, 5, 7))), good=dict(correct1=(rand(3, 5, 7), rand(3, 7, 5)), correct2=(rand(3, 5, 7), rand(3, 7, 9)), correct3=(rand(3, 5, 7), rand(3, 7)), correct4=(rand(3, 5), rand(3, 5, 7)), correct5=(rand(3), rand(3, 5, 7)), correct6=(rand(3, 5), rand(3)), correct7=(rand(3, 5), rand(3, 5)), correct8=(rand(3), rand(3)), correct9=(rand(3, 5, 7, 11), rand(3)), correct10=(rand(3, 7, 11, 5), rand(3, 5)), correct11=(rand(3, 7, 11, 5), rand(3, 5, 13)), correct12=(rand(3, 7, 11, 5), rand(3, 13, 5, 17)), mixed1=(rand(3, 5).astype('float32'), rand(3, 5, 7)), mixed2=(rand(3, 5).astype('float64'), rand(3, 5, 7))), bad_build=dict(no_batch_axis2=(rand(), rand(3, 5)), no_batch_axis3=(rand(3, 5), rand())), bad_runtime=dict(batch_dim_mismatch1=(rand(2, 5, 7), rand(3, 7, 9)), batch_dim_mismatch2=(rand(3, 5, 7), rand(2, 7, 9)), batch_dim_mismatch3=(rand(3), rand(5)), bad_dim1=(rand(3, 5, 7), rand(3, 5, 7)), bad_dim2=(rand(3, 5, 7), rand(3, 8, 3)), bad_dim3=(rand(3, 5), rand(3, 7)), bad_dim4=(rand(3, 5, 7, 11), rand(3, 5)), bad_dim5=(rand(3, 5, 7, 11), rand(3, 5, 13)), bad_dim6=(rand(3, 5, 7, 11), rand(3, 13, 5, 17)))) def _numpy_second(x, y): return np.broadcast_arrays(x, y)[1] # Don't forget to modify the two lines after! ALL_DTYPES = ('int8', 'int16', 'int32', 'int64', 'float32', 'float64', 'uint8', 'uint16', 'complex64', 'complex128') REAL_DTYPES = ALL_DTYPES[:6] COMPLEX_DTYPES = ALL_DTYPES[-2:] def multi_dtype_checks(shape1, shape2, dtypes=ALL_DTYPES, nameprefix=''): for dtype1, dtype2 in itertools.combinations(dtypes, 2): name1 = '%s_%s_%s' % (nameprefix, dtype1, dtype2) name2 = '%s_%s_%s' % (nameprefix, dtype2, dtype1) obj1 = rand_of_dtype(shape1, dtype1) obj2 = rand_of_dtype(shape2, dtype2) yield (name1, (obj1, obj2)) yield (name2, (obj2, obj1)) def multi_dtype_cast_checks(shape, dtypes=ALL_DTYPES, nameprefix=''): for dtype1, dtype2 in itertools.combinations(dtypes, 2): name1 = '%s_%s_%s' % (nameprefix, dtype1, dtype2) name2 = '%s_%s_%s' % (nameprefix, dtype2, dtype1) obj1 = rand_of_dtype(shape, dtype1) obj2 = rand_of_dtype(shape, dtype2) yield (name1, (obj1, dtype2)) yield (name2, (obj2, dtype1)) SecondBroadcastTester = makeTester( name='SecondBroadcastTester', op=second, expected=_numpy_second, good=dict(itertools.chain( multi_dtype_checks((4, 5), (5,)), multi_dtype_checks((2, 3, 2), (3, 2)), multi_dtype_checks((2, 3, 2), (2,)), )), # I can't think of any way to make this fail at build time # Just some simple smoke tests bad_runtime=dict( fail1=(rand(5, 4), rand(5)), fail2=(rand(3, 2, 3), rand(6, 9)), fail3=(randint(6, 2, 9), rand(3, 2)), ) ) # We exclude local_fill_to_alloc because it optimizes the "second" node # away from the graph. SecondSameRankTester = makeTester( name='SecondSameRankTester', op=second, expected=_numpy_second, good=dict(itertools.chain( multi_dtype_checks((4, 5), (4, 5)), multi_dtype_checks((1, 2), (3, 2)), multi_dtype_checks((3, 2), (1, 2)), )), # These sizes are not broadcastable to one another # and SHOULD raise an error, but currently don't. bad_runtime=dict(itertools.chain( multi_dtype_checks((4, 5), (5, 4)), multi_dtype_checks((1, 5), (5, 4)), )), mode=get_default_mode().excluding( 'local_fill_to_alloc', 'local_useless_fill') ) # Alloc AllocTester = makeBroadcastTester( name='AllocTester', op=alloc, expected=(lambda x, *shp: np.zeros(shp, dtype=x.dtype) + x), good=dict( correct01=(rand(), np.int32(7)), correct01_bcast=(rand(1), np.int32(7)), correct02=(rand(), np.int32(4), np.int32(7)), correct12=(rand(7), np.int32(4), np.int32(7)), correct13=(rand(7), np.int32(2), np.int32(4), np.int32(7)), correct23=(rand(4, 7), np.int32(2), np.int32(4), np.int32(7)), correctb1=(rand(1, 7), np.int32(4), np.int32(7)), correctb2=(rand(1, 7), np.int32(2), np.int32(4), np.int32(7)), correctb3=(rand(7, 1), np.int32(7), np.int32(4)), correctb4=(rand(7, 1), np.int32(2), np.int32(7), np.int32(4)), ), bad_runtime=dict( bad_shape12=(rand(7), np.int32(7), np.int32(5)), ), bad_build=dict( vec=(rand(1), [np.int32(2)]), too_big32=(rand(6, 2, 4), np.int32(6), np.int32(2)), too_big32b=(rand(6, 2, 4), np.int32(6), np.int32(4)), too_big32c=(rand(6, 2, 4), np.int32(2), np.int32(4)), too_big32d=(rand(6, 2, 4), np.int32(2), np.int32(6)), too_big32e=(rand(6, 2, 4), np.int32(4), np.int32(6)), too_big32f=(rand(6, 2, 4), np.int32(4), np.int32(2)), ), ) # Since not all inputs of Alloc are differentiable, we need different testers s1, s2, s3 = randint_ranged(1, 13, (3,)) # alloc a scalar into a vector Alloc01GradTester = makeBroadcastTester( name='Alloc01GradTester', op=(lambda x: alloc(x, s1)), expected=(lambda x: np.zeros((s1,), dtype=x.dtype) + x), grad=dict( x1=(rand(),), x2=(rand(),), x3=(rand(),), ), ) # alloc a vector into a tensor3 Alloc13GradTester = makeBroadcastTester( name='Alloc13GradTester', op=(lambda x: alloc(x, s1, s2, s3)), expected=(lambda x: np.zeros((s1, s2, s3), dtype=x.dtype) + x), grad=dict( x1=(rand(s3),), x2=(rand(s3),), x3=(rand(s3),), ), ) # unbroadcast a row to a matrix Allocb1GradTester = makeBroadcastTester( name='Allocb1GradTester', op=lambda x: alloc(x, s1, s2), expected=(lambda x: np.zeros((s1, s2), dtype=x.dtype) + x), grad=dict( x1=(rand(1, s2),), x2=(rand(1, s2),), x3=(rand(1, s2),), ), ) # unbroadcast a row to a tensor3 Allocb2GradTester = makeBroadcastTester( name='Allocb2GradTester', op=lambda x: alloc(x, s1, s2, s3), expected=(lambda x: np.zeros((s1, s2, s3), dtype=x.dtype) + x), grad=dict( x1=(rand(1, s3),), x2=(rand(1, s3),), x3=(rand(1, s3),), ), ) # unbroadcast a col to a matrix Allocb3GradTester = makeBroadcastTester( name='Allocb3GradTester', op=lambda x: alloc(x, s1, s2), expected=(lambda x: np.zeros((s1, s2), dtype=x.dtype) + x), grad=dict( x1=(rand(s1, 1),), x2=(rand(s1, 1),), x3=(rand(s1, 1),), ), ) # unbroadcast a col to a tensor3 Allocb4GradTester = makeBroadcastTester( name='Allocb4GradTester', op=lambda x: alloc(x, s1, s2, s3), expected=(lambda x: np.zeros((s1, s2, s3), dtype=x.dtype) + x), grad=dict( x1=(rand(s2, 1),), x2=(rand(s2, 1),), x3=(rand(s2, 1),), ), ) # Partial un broadcast of a dimshuffled input AllocDimshuffleGradTester = makeBroadcastTester( name='Allocb4GradTester', op=lambda x: alloc(x.dimshuffle('x', 'x', 0), 1, s2, s3), expected=(lambda x: np.zeros((1, s2, s3), dtype=x.dtype) + x), grad=dict( x1=(rand(s3),), x2=(rand(s3),), x3=(rand(s3),), ), ) AllocDimshuffleGradTester2 = makeBroadcastTester( name='Allocb4GradTester', op=lambda x: alloc(x.dimshuffle('x', 0), 1, s2, s3), expected=(lambda x: np.zeros((1, s2, s3), dtype=x.dtype) + x), grad=dict( x1=(rand(s3),), x2=(rand(s3),), x3=(rand(s3),), ), ) class ApplyDefaultTestOp(theano.Op): def __init__(self, id): self.default_output = id def make_node(self, x): x = theano.tensor.as_tensor_variable(x) return theano.Apply(self, [x], [x.type()]) class TestAsTensorVariable(unittest.TestCase): # Unit test for ensuring that as_tensor_variable handles Apply objects # correctly and removes leading broadcastable dimensions when possible. def setUp(self): self.x = tensor.scalar('x') def test_one_output(self): good_apply_var = ApplyDefaultTestOp(0).make_node(self.x) as_tensor_variable(good_apply_var) def test_below_zero_output(self): bad_apply_var = ApplyDefaultTestOp(-1).make_node(self.x) self.assertRaises(AttributeError, as_tensor_variable, bad_apply_var) def test_above_output_len(self): bad_apply_var = ApplyDefaultTestOp(2).make_node(self.x) self.assertRaises(AttributeError, as_tensor_variable, bad_apply_var) def test_list(self): bad_apply_var = ApplyDefaultTestOp([0, 1]).make_node(self.x) self.assertRaises(AttributeError, as_tensor_variable, bad_apply_var) def test_strip_leading_broadcastable(self): x = tensor.TensorType(config.floatX, (True, False))('x') x = as_tensor_variable(x, ndim=1) assert(x.ndim == 1) x = tensor.matrix('x', dtype=config.floatX) self.assertRaises(ValueError, as_tensor_variable, x, ndim=1) class TestAlloc(unittest.TestCase): dtype = config.floatX mode = mode_opt shared = staticmethod(theano.shared) allocs = [tensor.Alloc()] * 3 def setUp(self): self.rng = np.random.RandomState(seed=utt.fetch_seed()) def test_alloc_constant_folding(self): test_params = np.asarray(self.rng.randn(50 * 60), self.dtype) some_vector = vector('some_vector', dtype=self.dtype) some_matrix = some_vector.reshape((60, 50)) variables = self.shared(np.ones((50,), dtype=self.dtype)) idx = tensor.constant(np.arange(50)) for alloc_, (subtensor, n_alloc) in zip(self.allocs, [ # IncSubtensor1 (some_matrix[:60], 2), # AdvancedIncSubtensor1 (some_matrix[arange(60)], 2), # AdvancedIncSubtensor (some_matrix[idx, idx], 1) ]): derp = sum(dot(subtensor, variables)) fobj = theano.function([some_vector], derp, mode=self.mode) grad_derp = theano.grad(derp, some_vector) fgrad = theano.function([some_vector], grad_derp, mode=self.mode) topo_obj = fobj.maker.fgraph.toposort() assert np.sum([isinstance(node.op, type(alloc_)) for node in topo_obj]) == 0 topo_grad = fgrad.maker.fgraph.toposort() assert np.sum([isinstance(node.op, type(alloc_)) for node in topo_grad]) == n_alloc, ( alloc_, subtensor, n_alloc, topo_grad) fobj(test_params) fgrad(test_params) def test_alloc_output(self): val = tensor.constant(self.rng.randn(1, 1), dtype=self.dtype) for alloc_ in self.allocs: # The output is the result of the alloc operation, # we do not want it to be constant-folded out = alloc_(val, 50, 60) f = theano.function([], out, mode=self.mode) topo = f.maker.fgraph.toposort() assert np.sum([isinstance(node.op, type(alloc_)) for node in topo]) == 1 assert not isinstance(topo[0].op, DeepCopyOp) def test_ones(self): for shp in [[], 1, [1], [1, 2], [1, 2, 3]]: ones = theano.function([], [tensor.ones(shp)], mode=self.mode) assert np.allclose(ones(), np.ones(shp)) # scalar doesn't have to be provided as input x = scalar() shp = [] ones_scalar = theano.function([], [tensor.ones(x.shape)], mode=self.mode) assert np.allclose(ones_scalar(), np.ones(shp)) for (typ, shp) in [(vector, [3]), (matrix, [3, 4])]: x = typ() ones_tensor = theano.function([x], [tensor.ones(x.shape)], mode=self.mode) inp = np.zeros(shp, dtype=config.floatX) assert np.allclose(ones_tensor(inp), np.ones(shp)) def test_zeros(self): for shp in [[], 1, [1], [1, 2], [1, 2, 3]]: zeros = theano.function([], [tensor.zeros(shp)], mode=self.mode) assert np.allclose(zeros(), np.zeros(shp)) # scalar doesn't have to be provided as input x = scalar() shp = [] zeros_scalar = theano.function([], [tensor.zeros(x.shape)], mode=self.mode) assert np.allclose(zeros_scalar(), np.zeros(shp)) for (typ, shp) in [(vector, [3]), (matrix, [3, 4])]: x = typ() zeros_tensor = theano.function([x], [tensor.zeros(x.shape)], mode=self.mode) inp = np.zeros(shp, dtype=config.floatX) assert np.allclose(zeros_tensor(inp), np.zeros(shp)) # This is slow for the ('int8', 3) version. def test_eye(): def check(dtype, N, M_=None, k=0): # Theano does not accept None as a tensor. # So we must use a real value. M = M_ # Currently DebugMode does not support None as inputs even if this is # allowed. if M is None and theano.config.mode in ['DebugMode', 'DEBUG_MODE']: M = N N_symb = tensor.iscalar() M_symb = tensor.iscalar() k_symb = tensor.iscalar() f = function([N_symb, M_symb, k_symb], eye(N_symb, M_symb, k_symb, dtype=dtype)) result = f(N, M, k) assert np.allclose(result, np.eye(N, M_, k, dtype=dtype)) assert result.dtype == np.dtype(dtype) for dtype in ALL_DTYPES: yield check, dtype, 3 # M != N, k = 0 yield check, dtype, 3, 5 yield check, dtype, 5, 3 # N == M, k != 0 yield check, dtype, 3, 3, 1 yield check, dtype, 3, 3, -1 # N < M, k != 0 yield check, dtype, 3, 5, 1 yield check, dtype, 3, 5, -1 # N > M, k != 0 yield check, dtype, 5, 3, 1 yield check, dtype, 5, 3, -1 class test_triangle(unittest.TestCase): def test_tri(self): def check(dtype, N, M_=None, k=0): # Theano does not accept None as a tensor. # So we must use a real value. M = M_ # Currently DebugMode does not support None as inputs even if this is # allowed. if M is None and theano.config.mode in ['DebugMode', 'DEBUG_MODE']: M = N N_symb = tensor.iscalar() M_symb = tensor.iscalar() k_symb = tensor.iscalar() f = function([N_symb, M_symb, k_symb], tri(N_symb, M_symb, k_symb, dtype=dtype)) result = f(N, M, k) self.assertTrue( np.allclose(result, np.tri(N, M_, k, dtype=dtype))) self.assertTrue(result.dtype == np.dtype(dtype)) for dtype in ALL_DTYPES: yield check, dtype, 3 # M != N, k = 0 yield check, dtype, 3, 5 yield check, dtype, 5, 3 # N == M, k != 0 yield check, dtype, 3, 3, 1 yield check, dtype, 3, 3, -1 # N < M, k != 0 yield check, dtype, 3, 5, 1 yield check, dtype, 3, 5, -1 # N > M, k != 0 yield check, dtype, 5, 3, 1 yield check, dtype, 5, 3, -1 def test_tril_triu(self): def check_l(m, k=0): m_symb = matrix(dtype=m.dtype) k_symb = iscalar() f = function([m_symb, k_symb], tril(m_symb, k_symb)) result = f(m, k) self.assertTrue(np.allclose(result, np.tril(m, k))) self.assertTrue(result.dtype == np.dtype(dtype)) def check_u(m, k=0): m_symb = matrix(dtype=m.dtype) k_symb = iscalar() f = function([m_symb, k_symb], triu(m_symb, k_symb)) result = f(m, k) self.assertTrue(np.allclose(result, np.triu(m, k))) self.assertTrue(result.dtype == np.dtype(dtype)) for dtype in ALL_DTYPES: m = rand_of_dtype((10, 10), dtype) yield check_l, m, 0 yield check_l, m, 1 yield check_l, m, -1 yield check_u, m, 0 yield check_u, m, 1 yield check_u, m, -1 m = rand_of_dtype((10, 5), dtype) yield check_l, m, 0 yield check_l, m, 1 yield check_l, m, -1 yield check_u, m, 0 yield check_u, m, 1 yield check_u, m, -1 class test_nonzero(unittest.TestCase): def test_nonzero(self): def check(m): m_symb = theano.tensor.tensor(dtype=m.dtype, broadcastable=(False,) * m.ndim) f_tuple = function([m_symb], nonzero(m_symb, return_matrix=False)) f_matrix = function([m_symb], nonzero(m_symb, return_matrix=True)) self.assertTrue(np.allclose(f_matrix(m), np.vstack(np.nonzero(m)))) for i, j in zip(f_tuple(m), np.nonzero(m)): self.assertTrue(np.allclose(i, j)) rand0d = np.array(rand()) self.assertRaises(ValueError, check, rand0d) rand1d = rand(8) rand1d[:4] = 0 check(rand1d) rand2d = rand(8, 9) rand2d[:4] = 0 check(rand2d) rand3d = rand(8, 9, 10) rand3d[:4] = 0 check(rand3d) rand4d = rand(8, 9, 10, 11) rand4d[:4] = 0 check(rand4d) def test_flatnonzero(self): def check(m): m_symb = theano.tensor.tensor(dtype=m.dtype, broadcastable=(False,) * m.ndim) f = function([m_symb], flatnonzero(m_symb)) result = f(m) assert np.allclose(result, np.flatnonzero(m)) rand0d = np.array(rand()) self.assertRaises(ValueError, check, rand0d) rand1d = rand(8) rand1d[:4] = 0 check(rand1d) rand2d = rand(8, 9) rand2d[:4] = 0 check(rand2d) rand3d = rand(8, 9, 10) rand3d[:4] = 0 check(rand3d) rand4d = rand(8, 9, 10, 11) rand4d[:4] = 0 check(rand4d) def test_nonzero_values(self): def check(m): m_symb = theano.tensor.tensor(dtype=m.dtype, broadcastable=(False,) * m.ndim) f = function([m_symb], nonzero_values(m_symb)) result = f(m) assert np.allclose(result, m[np.nonzero(m)]) rand0d = rand() self.assertRaises(ValueError, check, rand0d) rand1d = rand(8) rand1d[:4] = 0 check(rand1d) rand2d = rand(8, 9) rand2d[:4] = 0 check(rand2d) rand3d = rand(8, 9, 10) rand3d[:4] = 0 check(rand3d) rand4d = rand(8, 9, 10, 11) rand4d[:4] = 0 check(rand4d) def test_identity(): def check(dtype): obj = rand_of_dtype((2,), dtype) sym = tensor.vector(dtype=dtype) f = function([sym], tensor_copy(sym)) assert np.all(obj == f(obj)) assert obj.dtype == f(obj).dtype topo = f.maker.fgraph.toposort() assert len(topo) == 1 if theano.config.mode != 'FAST_COMPILE': assert isinstance(topo[0].op, DeepCopyOp) for dtype in ALL_DTYPES: yield check, dtype class CastTester(unittest.TestCase): def test_good_between_real_types(self): good = itertools.chain( multi_dtype_cast_checks((2,), dtypes=REAL_DTYPES), # Casts from foo to foo [('%s_%s' % (rand_of_dtype((2,), dtype), dtype), (rand_of_dtype((2,), dtype), dtype)) for dtype in ALL_DTYPES]) for testname, (obj, dtype) in good: inp = tensor.vector(dtype=obj.dtype) out = tensor.cast(inp, dtype=dtype) f = function([inp], out) assert f(obj).dtype == np.dtype(dtype) # Test astype too out2 = inp.astype(dtype=dtype) assert out2.type == out.type def test_cast_from_real_to_complex(self): for real_dtype in REAL_DTYPES: for complex_dtype in COMPLEX_DTYPES: inp = tensor.vector(dtype=real_dtype) out = tensor.cast(inp, dtype=complex_dtype) f = function([inp], out) obj = rand_of_dtype((2, ), real_dtype) assert f(obj).dtype == np.dtype(complex_dtype) def test_cast_from_complex_to_real_raises_error(self): for real_dtype in REAL_DTYPES: for complex_dtype in COMPLEX_DTYPES: inp = tensor.vector(dtype=real_dtype) self.assertRaises(TypeError, tensor.cast( inp, dtype=complex_dtype)) ClipTester = makeTester( name='ClipTester', op=clip, expected=lambda x, y, z: np.clip(x, y, z), good=dict(correct1=((5 * rand(5, 5)).astype('float32'), np.array(-1, dtype='float32'), np.array(1, dtype='float32')), correct2=((5 * rand(5, 5)).astype('float64'), np.array(-1, dtype='float64'), np.array(1, dtype='float64')), correct3=(randint(5, 5).astype('int8'), np.array(-1, dtype='int8'), np.array(1, dtype='int8')), correct4=(randint(5, 5).astype('int16'), np.array(-1, dtype='int16'), np.array(1, dtype='int16')), correct5=(randint(5, 5).astype('int32'), np.array(-1, dtype='int32'), np.array(1, dtype='int32')), correct6=(randint(5, 5).astype('int64'), np.array(-1, dtype='int64'), np.array(1, dtype='int64')), # min > max. messed up behaviour, but # should be same as NumPy's correct7=((5 * rand(5, 5)).astype('float64'), np.array(1, dtype='float64'), np.array(-1, dtype='float64')), correct8=(randint(0, 5).astype('uint8'), np.array(2, dtype='uint8'), np.array(4, dtype='uint8')), correct9=(randint(0, 5).astype('uint16'), np.array(2, dtype='uint16'), np.array(4, dtype='uint16')),) # I can't think of any way to make this fail at runtime ) class T_Clip(unittest.TestCase): def test_complex_value(self): for dtype in ['complex64', 'complex128']: a = tensor.vector(dtype=dtype) b = tensor.scalar() c = tensor.scalar() self.assertRaises(TypeError, clip, a, b, c) def test_clip_repeat_grad(self): # This is testing for the issue #633 x, y = tensor.vectors('xy') a = clip(x, y, x) g = theano.gradient.grad(a.sum(), x) fn = theano.function([x, y], [g]) # Test the other way around as well a2 = clip(x, x, y) g2 = theano.gradient.grad(a2.sum(), x) fn2 = theano.function([x, y], [g2]) # Test for the equal case too a3 = theano.tensor.clip(x, x, x) g3 = theano.gradient.grad(a3.sum(), x) fn3 = theano.function([x], [g3]) rng = np.random.RandomState(utt.fetch_seed()) nvals = 50 xval = rng.rand(nvals).astype(config.floatX) # To ensure that the min < x yval_mn = rng.rand(nvals).astype(config.floatX) - 1.0 # To ensure that the max > x yval_mx = rng.rand(nvals).astype(config.floatX) + 1.0 aval, = fn(xval, yval_mn) aval2, = fn2(xval, yval_mx) aval3, = fn3(xval) self.assertTrue(np.all(aval == 1.)) self.assertTrue(np.all(aval2 == 1.)) self.assertTrue(np.all(aval3 == 1.)) def test_clip_repeat_verify_grad(self): # Additional tests for issue gh-633 utt.verify_grad( op=lambda x: clip(x, 0, x), pt=[rand_nonzero((3, 7))]) utt.verify_grad( op=lambda x: clip(x, x, 0), pt=[rand_nonzero((3, 7))]) utt.verify_grad( op=lambda x: clip(0, x, x), pt=[rand_nonzero((3, 7))]) utt.verify_grad( op=lambda x: clip(x, x, x), pt=[rand_nonzero((3, 7))]) # TODO: consider moving this function / functionality to gradient.py # rationale: it's tricky, and necessary everytime you want to verify # gradient numerically def test_batched_dot(): first = theano.tensor.tensor3("first") second = theano.tensor.tensor3("second") output = theano.tensor.basic.batched_dot(first, second) first_val = np.random.rand(10, 10, 20).astype(config.floatX) second_val = np.random.rand(10, 20, 5).astype(config.floatX) result_fn = theano.function([first, second], output) result = result_fn(first_val, second_val) assert result.shape[0] == first_val.shape[0] assert result.shape[1] == first_val.shape[1] assert result.shape[2] == second_val.shape[2] first_mat = theano.tensor.dmatrix("first") second_mat = theano.tensor.dmatrix("second") output = theano.tensor.basic.batched_dot(first_mat, second_mat) first_mat_val = np.random.rand(10, 10).astype(config.floatX) second_mat_val = np.random.rand(10, 10).astype(config.floatX) result_fn = theano.function([first_mat, second_mat], output) result = result_fn(first_mat_val, second_mat_val) assert result.shape[0] == first_mat_val.shape[0] def test_batched_dot_not_contiguous(): def np_genarray(*_shape): size = 1 for dimsize in _shape: size *= dimsize return np.arange(size, dtype=floatX).reshape(_shape) X = tensor3() W = tensor3() Z = batched_dot(X, W) f = function([X, W], Z) w = np_genarray(30, 10, 5) reversed_x_container = np_genarray(20, 40, 30) x_container = reversed_x_container.T def check_first_dim(inverted): direction = -1 if inverted else 1 x = x_container[::direction, ::2, ::2] assert x.shape == (30, 20, 10) assert x.strides[0] == direction * np.dtype(floatX).itemsize assert not (x.flags['C_CONTIGUOUS'] or x.flags['F_CONTIGUOUS']) result = f(x, w) ref_result = np.asarray(list(np.dot(u, v) for u, v in zip(x, w))) utt.assert_allclose(ref_result, result) for inverted in (0, 1): yield (check_first_dim, inverted) def test_batched_tensordot(): first = theano.tensor.tensor4("first") second = theano.tensor.tensor4("second") axes = [[1, 2], [3, 1]] output = theano.tensor.basic.batched_tensordot(first, second, axes) first_val = np.random.rand(8, 10, 20, 3).astype(config.floatX) second_val = np.random.rand(8, 20, 5, 10).astype(config.floatX) result_fn = theano.function([first, second], output) result = result_fn(first_val, second_val) assert result.shape[0] == first_val.shape[0] assert result.shape[1] == first_val.shape[3] assert result.shape[2] == second_val.shape[2] first_mat = theano.tensor.dmatrix("first") second_mat = theano.tensor.dmatrix("second") axes = 1 output = theano.tensor.basic.batched_tensordot(first_mat, second_mat, axes) first_mat_val = np.random.rand(10, 4).astype(config.floatX) second_mat_val = np.random.rand(10, 4).astype(config.floatX) result_fn = theano.function([first_mat, second_mat], output) result = result_fn(first_mat_val, second_mat_val) assert result.shape[0] == first_mat_val.shape[0] assert len(result.shape) == 1 def test_tensor_values_eq_approx(): # test, inf, -inf and nan equal themself a = np.asarray([-np.inf, -1, 0, 1, np.inf, np.nan]) assert TensorType.values_eq_approx(a, a) # test inf, -inf don't equal themself b = np.asarray([np.inf, -1, 0, 1, np.inf, np.nan]) assert not TensorType.values_eq_approx(a, b) b = np.asarray([-np.inf, -1, 0, 1, -np.inf, np.nan]) assert not TensorType.values_eq_approx(a, b) # test allow_remove_inf b = np.asarray([np.inf, -1, 0, 1, 5, np.nan]) assert TensorType.values_eq_approx(a, b, allow_remove_inf=True) b = np.asarray([np.inf, -1, 0, 1, 5, 6]) assert not TensorType.values_eq_approx(a, b, allow_remove_inf=True) # test allow_remove_nan b = np.asarray([np.inf, -1, 0, 1, 5, np.nan]) assert not TensorType.values_eq_approx(a, b, allow_remove_nan=False) b = np.asarray([-np.inf, -1, 0, 1, np.inf, 6]) assert not TensorType.values_eq_approx(a, b, allow_remove_nan=False) def test_nan_inf_constant_signature(): # Test that the signature of a constant tensor containing NaN and Inf # values is correct. test_constants = [ [np.nan, np.inf, 0, 1], [np.nan, np.inf, -np.inf, 1], [0, np.inf, -np.inf, 1], [0, 3, -np.inf, 1], [0, 3, np.inf, 1], [np.nan, 3, 4, 1], [0, 3, 4, 1], np.nan, np.inf, -np.inf, 0, 1, ] n = len(test_constants) # We verify that signatures of two rows i, j in the matrix above are # equal if and only if i == j. for i in xrange(n): for j in xrange(n): x = constant(test_constants[i]) y = constant(test_constants[j]) assert (x.signature() == y.signature()) == (i == j) # Also test that nan !=0 and nan != nan. x = tensor.scalar() mode = get_default_mode() if isinstance(mode, theano.compile.debugmode.DebugMode): # Disable the check preventing usage of NaN / Inf values. # We first do a copy of the mode to avoid side effects on other tests. mode = copy(mode) mode.check_isfinite = False f = theano.function([x], eq(x, np.nan), mode=mode) assert f(0) == 0 assert f(np.nan) == 0 def test_isnan(): for x in [tensor.matrix(), tensor.imatrix(), tensor.matrix(dtype='bool')]: y = tensor.isnan(x) assert isinstance(y.owner.op, tensor.Elemwise) == ( x.dtype not in tensor.discrete_dtypes) assert y.dtype == 'bool' # Test c code generator even for int type. y = tensor.isnan_(x) assert isinstance(y.owner.op, tensor.Elemwise) assert y.dtype == 'bool' f = theano.function([x], y, allow_input_downcast=True) f([[0, 1, 2]]) class T_Shape(unittest.TestCase): def test_basic0(self): s = shape(np.ones((5, 3))) self.assertTrue((eval_outputs([s]) == [5, 3]).all()) def test_basic1(self): s = shape(np.ones((2))) self.assertTrue((eval_outputs([s]) == [2]).all()) def test_basic2(self): s = shape(np.ones((5, 3, 10))) self.assertTrue((eval_outputs([s]) == [5, 3, 10]).all()) class T_max_and_argmax(unittest.TestCase): def setUp(self): utt.seed_rng() MaxAndArgmax.debug = 0 def test0(self): n = as_tensor_variable(5.0) v, i = eval_outputs(max_and_argmax(n)) self.assertTrue(v == 5.0) self.assertTrue(i == 0) assert i.dtype == 'int64' v = eval_outputs(max_and_argmax(n)[0].shape) assert len(v) == 0 v = eval_outputs(max_and_argmax(n)[1].shape) assert len(v) == 0 def test1(self): n = as_tensor_variable([1, 2, 3, 2, -6]) v, i = eval_outputs(max_and_argmax(n)) self.assertTrue(v == 3) self.assertTrue(i == 2) assert i.dtype == 'int64' v = eval_outputs(max_and_argmax(n)[0].shape) assert len(v) == 0 def test2(self): data = rand(2, 3) n = as_tensor_variable(data) for (axis, np_axis) in [(-1, -1), (0, 0), (1, 1), (None, None), ([0, 1], None), ([1, 0], None), (NoneConst.clone(), None), (constant(0), 0)]: v, i = eval_outputs(max_and_argmax(n, axis)) assert i.dtype == 'int64' self.assertTrue(np.all(v == np.max(data, np_axis))) self.assertTrue(np.all(i == np.argmax(data, np_axis))) v_shape = eval_outputs(max_and_argmax(n, axis)[0].shape) assert tuple(v_shape) == np.max(data, np_axis).shape def test2_float16(self): # Test negative values and bigger range to make sure numpy don't do the argmax as on uint16 data = (rand(20, 30).astype("float16") - 0.5) * 20 n = shared(data) for (axis, np_axis) in [(-1, -1), (0, 0), (1, 1), (None, None), ([0, 1], None), ([1, 0], None), (NoneConst.clone(), None), (constant(0), 0)]: v, i = eval_outputs(max_and_argmax(n, axis), (MaxAndArgmax,)) assert i.dtype == 'int64' self.assertTrue(np.all(v == np.max(data, np_axis))) self.assertTrue(np.all(i == np.argmax(data, np_axis))) v_shape = eval_outputs(max_and_argmax(n, axis)[0].shape) assert tuple(v_shape) == np.max(data, np_axis).shape def test2_invalid(self): n = as_tensor_variable(rand(2, 3)) # Silence expected error messages _logger = logging.getLogger('theano.gof.opt') oldlevel = _logger.level _logger.setLevel(logging.CRITICAL) try: try: eval_outputs(max_and_argmax(n, 3)) assert False except ValueError: pass finally: _logger.setLevel(oldlevel) def test2_invalid_neg(self): n = as_tensor_variable(rand(2, 3)) old_stderr = sys.stderr sys.stderr = StringIO() try: try: eval_outputs(max_and_argmax(n, -3)) assert False except ValueError: pass finally: sys.stderr = old_stderr def test2_valid_neg(self): n = as_tensor_variable(rand(2, 3)) v, i = eval_outputs(max_and_argmax(n, -1)) assert i.dtype == 'int64' self.assertTrue(v.shape == (2,)) self.assertTrue(i.shape == (2,)) self.assertTrue(np.all(v == np.max(n.value, -1))) self.assertTrue(np.all(i == np.argmax(n.value, -1))) v, i = eval_outputs(max_and_argmax(n, -2)) assert i.dtype == 'int64' self.assertTrue(v.shape == (3,)) self.assertTrue(i.shape == (3,)) self.assertTrue(np.all(v == np.max(n.value, -2))) self.assertTrue(np.all(i == np.argmax(n.value, -2))) v = eval_outputs(max_and_argmax(n, -1)[0].shape) assert v == (2) v = eval_outputs(max_and_argmax(n, -2)[0].shape) assert v == (3) def test3(self): data = rand(2, 3, 4) n = as_tensor_variable(data) for (axis, np_axis) in [(-1, -1), (0, 0), (1, 1), (None, None), ([0, 1, 2], None), ([1, 2, 0], None)]: v, i = eval_outputs(max_and_argmax(n, axis)) assert i.dtype == 'int64' self.assertTrue(np.all(v == np.max(data, np_axis))) self.assertTrue(np.all(i == np.argmax(data, np_axis))) v = eval_outputs(max_and_argmax(n, axis)[0].shape) assert tuple(v) == np.max(data, np_axis).shape def test_arg_grad(self): # The test checks that the gradient of argmax(x).sum() is 0 x = matrix() cost = argmax(x, axis=0).sum() gx = grad(cost, x) val = tensor.get_scalar_constant_value(gx) assert val == 0.0 def test_grad(self): data = rand(2, 3) n = as_tensor_variable(data) def safe_verify_grad(func, data): # Wrapper around 'verify_grad' that picks a proper value for epsilon. # # This is needed because 'verify_grad' may fail when its epsilon is # too large, due to the fact the argmax is not continuous. # We make sure epsilon is less than the minimum absolute value found # in the matrix of pairwise differences between all elements in the # data. This way, the argmax will not change when adding epsilon. # 'data' is a one-element list. data_tensor, = data # Flatten it into a 1D vector. data_vector = data_tensor.flatten() # Compute pairwise absolute differences. diff = np.abs(data_vector.reshape((-1, 1)) - data_vector) # Alter the diagonal to avoid a zero minimum. for i in xrange(len(diff)): diff[i, i] = 1 # Find an appropriate epsilon. eps = builtin_min(numeric_grad.type_eps[config.floatX], diff.min() / 2) # Run gradient verification. utt.verify_grad(func, data, eps=eps) def check_grad_max(data, max_grad_data, axis=None): # Why this is needed? verify_grad is not enough? # This works only for axis in [0, None]. assert axis in [0, None] z = np.zeros_like(data) z = z.flatten() argmax = np.argmax(data, axis=axis) if argmax.ndim == 0: z[argmax] += 1 else: for id, v in enumerate(argmax): z[v * np.prod(data.shape[data.ndim - 1:axis:-1]) + id] += 1 z = z.reshape(data.shape) assert np.all(max_grad_data == z) for axis in (-1, 0, 1, None): for j in xrange(2): safe_verify_grad(lambda v: max_and_argmax(v, axis=axis)[j], [data]) if axis != 1: safe_verify_grad(lambda v: max_and_argmax(v.flatten(), axis=axis)[j], [data]) if axis in (0, None): check_grad_max(data, eval_outputs(grad( max_and_argmax(n, axis=axis)[0].sum(), n)), axis=axis) check_grad_max(data, eval_outputs(grad( max_and_argmax(n.flatten())[0], n))) # Test 3d inner dimensions data = rand(3, 4, 5) for i in [0, 1, 2]: safe_verify_grad(lambda v: max_and_argmax(v, axis=[i])[0], [data]) safe_verify_grad(lambda v: max_and_argmax(v, axis=[i])[1], [data]) # Test 4d inner dimensions data = rand(2, 3, 4, 5) for i in [0, 1, 2, 3]: safe_verify_grad(lambda v: max_and_argmax(v, axis=[i])[0], [data]) safe_verify_grad(lambda v: max_and_argmax(v, axis=[i])[1], [data]) # Test grad with multiple axes for i in [[0, 1], [0, 0]]: safe_verify_grad(lambda v: max_and_argmax(v, axis=i)[0], [data]) safe_verify_grad(lambda v: max_and_argmax(v, axis=i)[1], [data]) def test_preserve_broadcastable(self): # Ensure the original broadcastable flags are preserved by Max/Argmax. x = tensor.matrix().dimshuffle('x', 0, 'x', 1, 'x') y = x.max(axis=1) assert y.type.broadcastable == (True, True, False, True) def test_multiple_axes(self): data = np.arange(24).reshape(3, 2, 4) x = as_tensor_variable(data) v, i = eval_outputs(max_and_argmax(x, [1, -1])) assert np.all(v == np.array([7, 15, 23])) assert np.all(i == np.array([7, 7, 7])) v = eval_outputs(max_and_argmax(x, [1, -1])[0].shape) assert tuple(v) == np.max(data, (1, -1)).shape def test_zero_shape(self): x = tensor.matrix() m, i = max_and_argmax(x, axis=1) f = theano.function([x], [m, i]) xv = np.zeros((0, 4), dtype=floatX) mv, iv = f(xv) assert mv.shape == (0,) assert iv.shape == (0,) def test_numpy_input(self): ar = np.array([1, 2, 3]) max, argmax = max_and_argmax(ar, axis=None) self.assertEqual(max.eval(), 3) self.assertEqual(argmax.eval(), 2) class T_argmin_argmax(unittest.TestCase): def setUp(self): utt.seed_rng() MaxAndArgmax.debug = 0 def test_scalar(self): for fct in [argmin, argmax]: n = as_tensor_variable(5.0) i = eval_outputs(fct(n)) self.assertTrue(i == 0) v = eval_outputs(fct(n).shape) assert len(v) == 0 def test_list(self): n = as_tensor_variable([1, 2, 3, 2, -6]) i = eval_outputs(argmin(n)) self.assertTrue(i == 4) v = eval_outputs(argmin(n).shape) assert len(v) == 0 n = as_tensor_variable([1, 2, 3, 2, -6]) i = eval_outputs(argmax(n)) self.assertTrue(i == 2) v = eval_outputs(argmax(n).shape) assert len(v) == 0 def test2(self): data = rand(2, 3) n = as_tensor_variable(data) for fct, nfct in [(argmax, np.argmax), (argmin, np.argmin)]: for (axis, np_axis) in [(-1, -1), (0, 0), (1, 1), (None, None), ([0, 1], None), ([1, 0], None)]: v = eval_outputs(fct(n, axis)) self.assertTrue(np.all(v == nfct(data, np_axis))) v_shape = eval_outputs(fct(n, axis).shape) assert tuple(v_shape) == nfct(data, np_axis).shape def test2_float16(self): # Test negative values and bigger range to make sure numpy don't do the argmax as on uint16 data = (rand(20, 30).astype("float16") - 0.5) * 20 n = shared(data) mode = get_default_mode().including("local_max_and_argmax", "uncanonicalize") for fct, nfct in [(argmax, np.argmax), (argmin, np.argmin)]: for (axis, np_axis) in [(-1, -1), (0, 0), (1, 1), (None, None), ([0, 1], None), ([1, 0], None)]: v = eval_outputs(fct(n, axis), (Argmax,), mode=mode) self.assertTrue(np.all(v == nfct(data, np_axis))) v_shape = eval_outputs(fct(n, axis).shape, mode=mode) assert tuple(v_shape) == nfct(data, np_axis).shape def test2_invalid(self): for fct, nfct in [(argmax, np.argmax), (argmin, np.argmin)]: n = as_tensor_variable(rand(2, 3)) # Silence expected error messages _logger = logging.getLogger('theano.gof.opt') oldlevel = _logger.level _logger.setLevel(logging.CRITICAL) try: try: eval_outputs(fct(n, 3)) assert False except ValueError: pass finally: _logger.setLevel(oldlevel) def test2_invalid_neg(self): for fct, nfct in [(argmax, np.argmax), (argmin, np.argmin)]: n = as_tensor_variable(rand(2, 3)) old_stderr = sys.stderr sys.stderr = StringIO() try: try: eval_outputs(fct(n, -3)) assert False except ValueError: pass finally: sys.stderr = old_stderr def test2_valid_neg(self): for fct, nfct in [(argmax, np.argmax), (argmin, np.argmin)]: n = as_tensor_variable(rand(2, 3)) i = eval_outputs(fct(n, -1)) self.assertTrue(i.shape == (2,)) self.assertTrue(np.all(i == nfct(n.value, -1))) i = eval_outputs(fct(n, -2)) self.assertTrue(i.shape == (3,)) self.assertTrue(np.all(i == nfct(n.value, -2))) v = eval_outputs(fct(n, -1).shape) assert v == (2) v = eval_outputs(fct(n, -2).shape) assert v == (3) def test3(self): data = rand(2, 3, 4) n = as_tensor_variable(data) for fct, nfct in [(argmax, np.argmax), (argmin, np.argmin)]: for (axis, np_axis) in [(-1, -1), (0, 0), (1, 1), (2, 2), (None, None), ([0, 1, 2], None), ([1, 0, 2], None)]: v = eval_outputs(fct(n, axis)) self.assertTrue(np.all(v == nfct(data, np_axis))) v_shape = eval_outputs(fct(n, axis).shape) assert tuple(v_shape) == nfct(data, np_axis).shape def test_grad_argmin(self): data = rand(2, 3) n = as_tensor_variable(data) n.name = 'n' # test grad of argmin utt.verify_grad(lambda v: argmin(v, axis=-1), [data]) utt.verify_grad(lambda v: argmin(v, axis=[0]), [data]) utt.verify_grad(lambda v: argmin(v, axis=[1]), [data]) utt.verify_grad(lambda v: argmin(v.flatten()), [data]) try: cost = argmin(n, axis=-1) cost.name = None grad(cost, n) raise Exception('Expected an error') except TypeError: pass def test_grad_argmax(self): data = rand(2, 3) n = as_tensor_variable(data) # test grad of argmax utt.verify_grad(lambda v: argmax(v, axis=-1), [data]) utt.verify_grad(lambda v: argmax(v, axis=[0]), [data]) utt.verify_grad(lambda v: argmax(v, axis=[1]), [data]) utt.verify_grad(lambda v: argmax(v.flatten()), [data]) try: grad(argmax(n, axis=-1), n) raise Exception('Expected an error') except TypeError: pass def test_uint(self): for dtype in ('uint8', 'uint16', 'uint32', 'uint64'): itype = np.iinfo(dtype) data = np.array([itype.min + 3, itype.min, itype.max - 5, itype.max], dtype) n = as_tensor_variable(data) i = eval_outputs(argmin(n)) self.assertEqual(i, 1) i = eval_outputs(argmax(n)) self.assertEqual(i, 3) def test_bool(self): data = np.array([True, False], 'bool') n = as_tensor_variable(data) i = eval_outputs(argmin(n)) self.assertEqual(i, 1) i = eval_outputs(argmax(n)) self.assertEqual(i, 0) class T_min_max(unittest.TestCase): def setUp(self): utt.seed_rng() MaxAndArgmax.debug = 0 def test_scalar(self): for fct in [max, min]: n = as_tensor_variable(5.0) v = eval_outputs(fct(n)) self.assertTrue(v == 5.0) v = eval_outputs(fct(n).shape) assert len(v) == 0 def test_list(self): for fct, nfct in [(max, np.max), (min, np.min)]: n = as_tensor_variable([1, 2, 3, 2, -6]) v = eval_outputs([fct(n)]) self.assertTrue(v == nfct(n.value)) v = eval_outputs(fct(n).shape) assert len(v) == 0 def test2(self): data = rand(2, 3) n = as_tensor_variable(data) for fct, nfct in [(max, np.max), (min, np.min)]: for (axis, np_axis) in [(-1, -1), (0, 0), (1, 1), (None, None), ([0, 1], None), ([1, 0], None)]: v = eval_outputs(fct(n, axis)) self.assertTrue(np.all(v == nfct(data, np_axis))) v_shape = eval_outputs(fct(n, axis).shape) assert tuple(v_shape) == nfct(data, np_axis).shape def test2_invalid(self): for fct in [max, min]: n = as_tensor_variable(rand(2, 3)) # Silence expected error messages _logger = logging.getLogger('theano.gof.opt') oldlevel = _logger.level _logger.setLevel(logging.CRITICAL) try: try: eval_outputs(fct(n, 3)) assert False except ValueError: pass finally: _logger.setLevel(oldlevel) def test2_invalid_neg(self): for fct in [max, min]: n = as_tensor_variable(rand(2, 3)) old_stderr = sys.stderr sys.stderr = StringIO() try: try: eval_outputs(fct(n, -3)) assert False except ValueError: pass finally: sys.stderr = old_stderr def test2_valid_neg(self): for fct, nfct in [(max, np.max), (min, np.min)]: n = as_tensor_variable(rand(2, 3)) v = eval_outputs(fct(n, -1)) self.assertTrue(v.shape == (2,)) self.assertTrue(np.all(v == nfct(n.value, -1))) v = eval_outputs(fct(n, -2)) self.assertTrue(v.shape == (3,)) self.assertTrue(np.all(v == nfct(n.value, -2))) v = eval_outputs(fct(n, -1).shape) assert v == (2) v = eval_outputs(fct(n, -2).shape) assert v == (3) def test3(self): # Test with 1 axis or all axis out of 3 dims data = rand(2, 3, 4) n = as_tensor_variable(data) for fct, nfct in [(max, np.max), (min, np.min)]: for (axis, np_axis) in [(-1, -1), (0, 0), (1, 1), (2, 2), (None, None), ([0, 1, 2], None), ([1, 0, 2], None)]: v = eval_outputs(fct(n, axis)) self.assertTrue(np.all(v == nfct(data, np_axis))) v_shape = eval_outputs(fct(n, axis).shape) assert tuple(v_shape) == nfct(data, np_axis).shape def test3b(self): # Test with 2 axis out of 3 dims data = rand(2, 3, 4) n = as_tensor_variable(data) for fct, nfct in [(max, np.max), (min, np.min)]: for axis in [[0, 1], [1, 2], [0, 2]]: v = eval_outputs(fct(n, axis)) np_v = nfct(nfct(data, axis[1]), axis[0]) self.assertTrue(np.all(v == np_v)) v_shape = eval_outputs(fct(n, axis).shape) assert tuple(v_shape) == np_v.shape def test_grad_max(self): data = rand(2, 3) n = as_tensor_variable(data) def check_grad_max(data, max_grad_data, axis=None): # This work only for axis in [0,None] assert axis in [0, None] z = np.zeros_like(data) z = z.flatten() argmax = np.argmax(data, axis=axis) if argmax.ndim == 0: z[np.argmax(data, axis=axis)] += 1 else: for id, v in enumerate(argmax): z[v * np.prod(data.shape[data.ndim - 1:axis:-1]) + id] += 1 z = z.reshape(data.shape) assert np.all(max_grad_data == z) # test grad of max # axis is the last one utt.verify_grad(lambda v: max(v, axis=-1), [data]) utt.verify_grad(lambda v: max(v, axis=[0]), [data]) check_grad_max(data, eval_outputs(grad(max(n, axis=0).sum(), n)), axis=0) utt.verify_grad(lambda v: max(v, axis=[1]), [data]) # check_grad_max(data,eval_outputs(grad(max(n,axis=1),n)),axis=1) utt.verify_grad(lambda v: max(v.flatten()), [data]) check_grad_max(data, eval_outputs(grad(max(n.flatten()), n))) def test_grad_min(self): data = rand(2, 3) n = as_tensor_variable(data) def check_grad_min(data, min_grad_data, axis=None): # This work only for axis in [0, None] assert axis in [0, None] z = np.zeros_like(data) z = z.flatten() argmin = np.argmin(data, axis=axis) if argmin.ndim == 0: z[np.argmin(data, axis=axis)] += 1 else: for id, v in enumerate(argmin): z[v * np.prod(data.shape[data.ndim - 1:axis:-1]) + id] += 1 z = z.reshape(data.shape) assert np.all(min_grad_data == z) # test grad of min # axis is the last one utt.verify_grad(lambda v: min(v, axis=-1), [data]) utt.verify_grad(lambda v: min(v, axis=[0]), [data]) check_grad_min(data, eval_outputs(grad(min(n, axis=0).sum(), n)), axis=0) utt.verify_grad(lambda v: min(v, axis=[1]), [data]) # check_grad_min(data,eval_outputs(grad(min(n,axis=1),n)),axis=1) utt.verify_grad(lambda v: min(v.flatten()), [data]) check_grad_min(data, eval_outputs(grad(min(n.flatten()), n))) def _grad_list(self): # Test the gradient when we have multiple axis at the same time. # # This not implemented, so we disable the test. See ticket: # http://www.assembla.com/spaces/theano/tickets/511 data = rand(2, 3) for fct in [max_and_argmax, max, min]: utt.verify_grad(lambda v: fct(v, axis=[0, 1]), [data]) # n = as_tensor_variable(data) # check_grad_max(data, eval_outputs(grad(max_and_argmax(n, # axis=1)[0], n)),axis=1) def test_uint(self): for dtype in ('uint8', 'uint16', 'uint32', 'uint64'): itype = np.iinfo(dtype) data = np.array([itype.min + 3, itype.min, itype.max - 5, itype.max], dtype) n = as_tensor_variable(data) self.assertEqual(min(n).dtype, dtype) i = eval_outputs(min(n)) self.assertEqual(i, itype.min) self.assertEqual(max(n).dtype, dtype) i = eval_outputs(max(n)) self.assertEqual(i, itype.max) def test_bool(self): data = np.array([True, False], 'bool') n = as_tensor_variable(data) self.assertEqual(min(n).dtype, 'bool') i = eval_outputs(min(n)) self.assertEqual(i, False) self.assertEqual(max(n).dtype, 'bool') i = eval_outputs(max(n)) self.assertEqual(i, True) def test_basic_allclose(): # This was raised by a user in https://github.com/Theano/Theano/issues/2975 assert tensor.basic._allclose(-0.311023883434, -0.311022856884) class T_outer(unittest.TestCase): def test_outer(self): for m in range(4): for n in range(4): x = tensor.tensor(dtype='floatX', broadcastable=(False,) * m) y = tensor.tensor(dtype='floatX', broadcastable=(False,) * n) s1 = np.random.randint(1, 10, m) s2 = np.random.randint(1, 10, n) v1 = np.asarray(np.random.rand(*s1)).astype(floatX) v2 = np.asarray(np.random.rand(*s2)).astype(floatX) o = tensor.outer(x, y).eval({x: v1, y: v2}) assert_allclose(o, np.outer(v1, v2)) def test_grad(self): # Test the combined graph of the graph of outer # with broadcastable dimensions, just in case. for shp0, shp1 in [((1,), (2,)), ((3,), (1,)), ((1,), (1,)), ((3,), (2,)), ((3, 2), (1, 1)), ((3, 2), (1, 4)), ((3, 2), (4, 1)), ((3, 2), (4, 5)), ((1, 2), (4, 5)), ((3, 1), (4, 5)), ((1, 1), (4, 5)), ((1, 1), (1, 1)), ]: data0 = np.random.rand(*shp0).astype(floatX) data1 = np.random.rand(*shp1).astype(floatX) utt.verify_grad(tensor.outer, [data0, data1]) class T_GetVectorLength(unittest.TestCase): def test_get_vector_length(self): x = theano.shared(np.zeros((2, 3, 4, 5))) assert len(list(x.shape)) == 4 assert len(list(x.shape[2:4])) == 2 assert len(list(x.shape[2:])) == 2 assert len(list(x.shape[1:4])) == 3 assert len(list(x.shape[2:2])) == 0 assert len(list(x.shape[1:5])) == 3 assert len(list(x.shape[1:10])) == 3 # Test step assert len(list(x.shape[1:10:2])) == 2 # Test neg start assert len(list(x.shape[-1:4])) == 1 assert len(list(x.shape[-6:4])) == 4 # test neg stop assert len(list(x.shape[1:-2])) == 1 assert len(list(x.shape[1:-1])) == 2 class T_Join_and_Split(unittest.TestCase): # Split is tested by each verify_grad method. def setUp(self): Join.debug = False utt.seed_rng() self.mode = theano.compile.get_default_mode().excluding( 'constant_folding') self.join_op = Join() self.split_op_class = Split self.make_vector_op = opt.MakeVector() self.floatX = config.floatX self.hide_error = theano.config.mode not in [ 'DebugMode', 'DEBUG_MODE', 'FAST_COMPILE'] self.shared = shared def eval_outputs_and_check_join(self, outputs): f = theano.function([], outputs, self.mode) topo = f.maker.fgraph.toposort() assert [True for node in topo if isinstance(node.op, type(self.join_op))] variables = f() if isinstance(variables, (tuple, list)) and len(variables) == 1: return variables[0] return variables def eval_outputs_and_check_vector(self, outputs, make_vector_op=None): if make_vector_op is None: make_vector_op = self.make_vector_op f = theano.function([], outputs, self.mode) topo = f.maker.fgraph.toposort() assert [True for node in topo if isinstance(node.op, type(make_vector_op))] variables = f() if isinstance(variables, (tuple, list)) and len(variables) == 1: return variables[0] return variables def test_join_scalar(self): a = as_tensor_variable(1) b = as_tensor_variable(2) self.assertRaises(TypeError, join, 0, a, b) def test_stack_mixed_type_constants(self): # tested only on cpu as gpu support only float32 a = as_tensor_variable(1) b = as_tensor_variable(2.0) c = tensor._shared(np.asarray(3.0, dtype=self.floatX)) s = stack([a, b, c]) want = np.array([1, 2, 3]) out = self.eval_outputs_and_check_vector([s], opt.MakeVector()) self.assertTrue((out == want).all()) def test_stack_scalar(self): a = self.shared(np.asarray(1., dtype=self.floatX)) b = as_tensor_variable(2.) c = as_tensor_variable(3.) s = stack([a, b, c]) want = np.array([1, 2, 3]) out = self.eval_outputs_and_check_vector([s]) self.assertTrue((out == want).all()) def test_stack_scalar_make_vector(self): # Test that calling stack() on scalars instantiates MakeVector, # not Join. Test that the floatX dtype stay floatX, not downcasted # to int64 a = tensor.scalar('a', dtype=self.floatX) b = tensor.scalar('b', dtype=self.floatX) s = stack([a, b, a, b]) f = function([a, b], s, mode=self.mode) val = f(1, 2) # print val self.assertTrue(np.all(val == [1, 2, 1, 2])) topo = f.maker.fgraph.toposort() assert len([n for n in topo if isinstance(n.op, opt.MakeVector)]) > 0 assert len([n for n in topo if isinstance(n, type(self.join_op))]) == 0 assert f.maker.fgraph.outputs[0].dtype == self.floatX def test_stack_scalar_make_vector_dtype(self): # Test that calling stack() on scalars instantiates MakeVector, # event when the scalar don't have the same dtype. a = tensor.iscalar('a') b = tensor.lscalar('b') s = stack([a, b, a, b]) f = function([a, b], s, mode=self.mode) val = f(1, 2) self.assertTrue(np.all(val == [1, 2, 1, 2])) topo = f.maker.fgraph.toposort() assert len([n for n in topo if isinstance(n.op, opt.MakeVector)]) > 0 assert len([n for n in topo if isinstance(n, type(self.join_op))]) == 0 assert f.maker.fgraph.outputs[0].dtype == 'int64' def test_stack_scalar_make_vector_constant(self): # Test that calling stack() on scalars instantiates MakeVector, # event when the scalar are simple int type. a = tensor.iscalar('a') b = tensor.lscalar('b') # test when the constant is the first element. # The first element is used in a special way s = stack([10, a, b, np.int8(3)]) f = function([a, b], s, mode=self.mode) val = f(1, 2) self.assertTrue(np.all(val == [10, 1, 2, 3])) topo = f.maker.fgraph.toposort() assert len([n for n in topo if isinstance(n.op, opt.MakeVector)]) > 0 assert len([n for n in topo if isinstance(n, type(self.join_op))]) == 0 assert f.maker.fgraph.outputs[0].dtype == 'int64' def test_stack_new_interface(self): # Test the new numpy-like interface: stack(tensors, axis=0). # Testing against old interface warnings.simplefilter('always', DeprecationWarning) a = tensor.imatrix('a') b = tensor.imatrix('b') s1 = stack(a, b) s2 = stack([a, b]) f = function([a, b], [s1, s2], mode=self.mode) v1, v2 = f([[1, 2]], [[3, 4]]) self.assertTrue(v1.shape == v2.shape) self.assertTrue(np.all(v1 == v2)) # Testing axis parameter s3 = stack([a, b], 1) f = function([a, b], s3, mode=self.mode) v3 = f([[1, 2]], [[3, 4]]) v4 = np.array([[[1, 2], [3, 4]]]) self.assertTrue(v3.shape == v4.shape) self.assertTrue(np.all(v3 == v4)) # Testing negative axis v1 = [[1, 2, 3], [4, 5, 6]] v2 = [[7, 8, 9], [10, 11, 12]] s = stack([a, b], axis=-1) f = function([a, b], s, mode=self.mode) v = np.zeros((2, 3, 2)) v[:, :, 0] = v1 v[:, :, 1] = v2 out = f(v1, v2) self.assertTrue(v.shape == out.shape) self.assertTrue(np.all(v == out)) s = stack([a, b], axis=-2) f = function([a, b], s, mode=self.mode) v = np.zeros((2, 2, 3)) v[:, 0, :] = v1 v[:, 1, :] = v2 out = f(v1, v2) self.assertTrue(v.shape == out.shape) self.assertTrue(np.all(v == out)) # Testing out-of-bounds axis self.assertRaises(IndexError, stack, [a, b], 4) self.assertRaises(IndexError, stack, [a, b], -4) # Testing depreciation warning with warnings.catch_warnings(record=True) as w: s = stack(a, b) assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) with warnings.catch_warnings(record=True) as w: s = stack([a, b]) s = stack([a, b], 1) s = stack([a, b], axis=1) s = stack(tensors=[a, b]) s = stack(tensors=[a, b], axis=1) assert not w def test_stack_hessian(self): # Test the gradient of stack when used in hessian, see gh-1589 a = tensor.dvector('a') b = tensor.dvector('b') A = stack([a, b]) B = A.T.dot(A) Ha, Hb = hessian(B.sum(), [a, b]) # Try some values a_v = np.random.rand(4) b_v = np.random.rand(4) f = theano.function([a, b], [Ha, Hb]) Ha_v, Hb_v = f(a_v, b_v) # The Hessian is always a matrix full of 2 assert Ha_v.shape == (4, 4) assert Hb_v.shape == (4, 4) assert np.allclose(Ha_v, 2.) assert np.allclose(Hb_v, 2.) def test_stack_hessian2(self): # Test the hessian macro when the gradient itself does not depend # on the input (but the cost does) a = tensor.dvector('a') b = tensor.dvector('b') A = stack([a, b]) Ha, Hb = hessian(A.sum(), [a, b]) # Try some values a_v = np.random.rand(4) b_v = np.random.rand(4) f = theano.function([a, b], [Ha, Hb]) Ha_v, Hb_v = f(a_v, b_v) # The Hessian is always a matrix full of 0 assert Ha_v.shape == (4, 4) assert Hb_v.shape == (4, 4) assert np.allclose(Ha_v, 0.) assert np.allclose(Hb_v, 0.) def test_join_concatenate_one_element(self): # Fast test of concatenate as this is an alias for join. # also test that we remove the Join op if there is only 1 input m = tensor.fmatrix() c = tensor.concatenate([m]) f = theano.function(inputs=[m], outputs=[c], mode=self.mode.including('local_join_1')) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, DeepCopyOp) def test_join_vector(self): a = self.shared(np.array([1, 2, 3], dtype=self.floatX)) b = as_tensor_variable(np.array([7, 8, 9], dtype=self.floatX)) s = join(0, a, b) want = np.array([1, 2, 3, 7, 8, 9]) out = self.eval_outputs_and_check_join([s]) self.assertTrue((out == want).all()) def test_roll(self): for get_shift in [lambda a:a, lambda x:theano.shared(x)]: # Test simple 1D example a = self.shared(np.array([1, 2, 3, 4, 5, 6], dtype=self.floatX)) b = roll(a, get_shift(2)) want = np.array([5, 6, 1, 2, 3, 4]) out = theano.function([], b)() assert (out == want).all() # Test simple 1D example with explicit 0 axis b = roll(a, get_shift(-1), 0) want = np.array([2, 3, 4, 5, 6, 1]) out = theano.function([], b)() assert (out == want).all() # Test 2D example - ensure that behavior matches np.roll behavior a = self.shared(np.arange(21).reshape((3, 7)).astype(self.floatX)) b = roll(a, get_shift(-2), 1) want = np.roll(a.get_value(borrow=True), -2, 1) out = theano.function([], b)() assert (out == want).all() # Test example when axis < 0 - ensure that behavior matches np.roll behavior a = self.shared(np.arange(24).reshape((3, 2, 4)).astype(self.floatX)) b = roll(a, get_shift(-2), -2) want = np.roll(a.get_value(borrow=True), -2, -2) out = theano.function([], b)() assert (out == want).all() # Test rolling on axis 0 want = np.roll(a.get_value(borrow=True), -2, 0) b = roll(a, get_shift(-2), 0) out = theano.function([], b)() assert (out == want).all() # Test rolling on default axis with ndim > 1 want = np.roll(a.get_value(borrow=True), 2) b = roll(a, get_shift(2)) out = theano.function([], b)() assert (out == want).all() # Test rolling on axis 0 with a positive shift that is # larger than axis size want = np.roll(a.get_value(borrow=True), 4, 0) b = roll(a, get_shift(4), 0) out = theano.function([], b)() assert (out == want).all() # Test rolling on axis 0 with a negative shift that is # larger than axis size want = np.roll(a.get_value(borrow=True), -4, 0) b = roll(a, get_shift(-4), 0) out = theano.function([], b)() assert (out == want).all() def test_stack_vector(self): a = self.shared(np.array([1, 2, 3], dtype=self.floatX)) b = as_tensor_variable(np.array([7, 8, 9], dtype=self.floatX)) s = stack([a, b]) want = np.array([[1, 2, 3], [7, 8, 9]]) out = self.eval_outputs_and_check_join([s]) self.assertTrue((out == want).all()) def test_join_matrix0(self): a = self.shared(np.array([[1, 2, 3], [4, 5, 6]], dtype=self.floatX)) b = as_tensor_variable(np.array([[7, 8, 9]], dtype=self.floatX)) s = join(0, a, b) want = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) out = self.eval_outputs_and_check_join([s]) self.assertTrue((out == want).all()) def test_join_matrix1(self): av = np.array([[.1, .2, .3], [.4, .5, .6]], dtype='float32') bv = np.array([[.7], [.8]], dtype='float32') a = self.shared(av) b = as_tensor_variable(bv) s = join(1, a, b) want = np.array([[.1, .2, .3, .7], [.4, .5, .6, .8]], dtype='float32') out = self.eval_outputs_and_check_join([s]) self.assertTrue((out == want).all()) utt.verify_grad(lambda a, b: join(1, a, b), [av, bv], mode=self.mode) def test_join_matrix_dtypes(self): if "float32" in self.shared.__name__: raise SkipTest( "The shared variable constructor" " need to support other dtype then float32") # Test mixed dtype. There was a bug that caused crash in the past. av = np.array([[1, 2, 3], [4, 5, 6]], dtype='int8') bv = np.array([[7], [8]], dtype='float32') a = self.shared(av) b = as_tensor_variable(bv) s = join(1, a, b) want = np.array([[1, 2, 3, 7], [4, 5, 6, 8]], dtype='float32') out = self.eval_outputs_and_check_join([s]) self.assertTrue((out == want).all()) grad(s.sum(), b) grad(s.sum(), a) utt.verify_grad(lambda b: join(1, a, b), [bv], eps=1.0e-2, mode=self.mode) def test_join_matrix_ints(self): if "float32" in self.shared.__name__: raise SkipTest( "The shared variable constructor" " need to support other dtype then float32") # Test mixed dtype. There was a bug that caused crash in the past. av = np.array([[1, 2, 3], [4, 5, 6]], dtype='int8') bv = np.array([[7], [8]], dtype='int32') a = self.shared(av) b = as_tensor_variable(bv) s = join(1, a, b) want = np.array([[1, 2, 3, 7], [4, 5, 6, 8]], dtype='float32') out = self.eval_outputs_and_check_join([s]) self.assertTrue((out == want).all()) assert (np.asarray(grad(s.sum(), b).eval()) == 0).all() assert (np.asarray(grad(s.sum(), a).eval()) == 0).all() def test_join_matrix1_using_vertical_stack(self): a = self.shared(np.array([[1, 2, 3], [4, 5, 6]], dtype=self.floatX)) b = as_tensor_variable(np.array([[7, 8, 9]], dtype=self.floatX)) c = as_tensor_variable(np.array([[9, 8, 7]], dtype=self.floatX)) s = vertical_stack(a, b, c) want = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9], [9, 8, 7]]) out = self.eval_outputs_and_check_join([s]) self.assertTrue((out == want).all()) def test_join_matrix1_using_horizontal_stack(self): av = np.array([[.1, .2, .3], [.4, .5, .6]], dtype='float32') bv = np.array([[.7], [.8]], dtype='float32') cv = np.array([[.3, .2, .1], [.6, .5, .4]], dtype='float32') a = self.shared(av) b = as_tensor_variable(bv) c = as_tensor_variable(cv) s = horizontal_stack(a, b, c) want = np.array([[.1, .2, .3, .7, .3, .2, .1], [.4, .5, .6, .8, .6, .5, .4]], dtype='float32') out = self.eval_outputs_and_check_join([s]) self.assertTrue((out == want).all()) utt.verify_grad(lambda a, b: join(1, a, b), [av, bv], mode=self.mode) def test_join_matrixV(self): # variable join axis v = np.array([[.1, .2, .3], [.4, .5, .6]], dtype=self.floatX) a = self.shared(v) b = as_tensor_variable(v) ax = lscalar() s = join(ax, a, b) f = inplace_func([ax], [s], mode=self.mode) topo = f.maker.fgraph.toposort() assert [True for node in topo if isinstance(node.op, type(self.join_op))] want = np.array([[.1, .2, .3], [.4, .5, .6], [.1, .2, .3], [.4, .5, .6]]) got = f(0) assert np.allclose(got, want) want = np.array([[.1, .2, .3, .1, .2, .3], [.4, .5, .6, .4, .5, .6]]) got = f(1) assert np.allclose(got, want) utt.verify_grad(lambda a, b: join(0, a, b), [v, 2 * v], mode=self.mode) utt.verify_grad(lambda a, b: join(1, a, b), [v, 2 * v], mode=self.mode) def test_join_matrixV_negative_axis(self): # variable join negative axis v = np.array([[.1, .2, .3], [.4, .5, .6]], dtype=self.floatX) a = self.shared(v) b = as_tensor_variable(v) ax = lscalar() s = join(ax, a, b) f = inplace_func([ax], [s], mode=self.mode) topo = f.maker.fgraph.toposort() assert [True for node in topo if isinstance(node.op, type(self.join_op))] want = np.array([[.1, .2, .3, .1, .2, .3], [.4, .5, .6, .4, .5, .6]]) got = f(-1) assert np.allclose(got, want) want = np.array([[.1, .2, .3], [.4, .5, .6], [.1, .2, .3], [.4, .5, .6]]) got = f(-2) assert np.allclose(got, want) self.assertRaises(IndexError, f, -3) def test_join_matrixC_negative_axis(self): # constant join negative axis v = np.array([[.1, .2, .3], [.4, .5, .6]], dtype=self.floatX) a = self.shared(v) b = as_tensor_variable(v) s = join(-1, a, b) f = theano.function([], [s], mode=self.mode) topo = f.maker.fgraph.toposort() assert [True for node in topo if isinstance(node.op, type(self.join_op))] want = np.array([[.1, .2, .3, .1, .2, .3], [.4, .5, .6, .4, .5, .6]]) got = f() assert np.allclose(got, want) s = join(-2, a, b) f = theano.function([], [s], mode=self.mode) topo = f.maker.fgraph.toposort() assert [True for node in topo if isinstance(node.op, type(self.join_op))] want = np.array([[.1, .2, .3], [.4, .5, .6], [.1, .2, .3], [.4, .5, .6]]) got = f() assert np.allclose(got, want) self.assertRaises(IndexError, join, -3, a, b) utt.verify_grad(lambda a, b: join(-1, a, b), [v, 2 * v], mode=self.mode) def test_vector_len(self): x = lscalar('x') y = dscalar('y') triple = as_tensor_variable((x, y, 9.0)) assert 3 == get_vector_length(triple) a, b, c = triple f = function([x, y], [b, c, a], mode=self.mode) topo = f.maker.fgraph.toposort() assert [True for node in topo if isinstance(node.op, opt.MakeVector)] assert np.allclose(f(4, 5), [5, 9, 4]) def test_broadcastable_flag_assignment_mixed_otheraxes(self): # Test that the broadcastable flags for the output of # a join operation on non-join axes are True if one or # more inputs is broadcastable on that dimension. rng = np.random.RandomState(seed=utt.fetch_seed()) a_val = rng.rand(1, 4, 1).astype(self.floatX) b_val = rng.rand(1, 3, 1).astype(self.floatX) a = self.shared(a_val, broadcastable=(False, False, True)) b = self.shared(b_val, broadcastable=(True, False, True)) c = self.join_op(1, a, b) assert c.type.broadcastable[0] and c.type.broadcastable[2] assert not c.type.broadcastable[1] # Opt can remplace the int by a Theano constant c = self.join_op(theano.tensor.constant(1), a, b) assert c.type.broadcastable[0] and c.type.broadcastable[2] assert not c.type.broadcastable[1] # In case futur opt insert other useless stuff c = self.join_op(theano.tensor.cast(theano.tensor.constant(1), dtype="int32"), a, b) assert c.type.broadcastable[0] and c.type.broadcastable[2] assert not c.type.broadcastable[1] f = function([], c, mode=self.mode) topo = f.maker.fgraph.toposort() assert [True for node in topo if isinstance(node.op, type(self.join_op))] f() utt.verify_grad((lambda a, b: join(1, a, b)), [a_val, b_val], rng=rng, mode=self.mode) # Should raise an error if dimension 0 does not match a.set_value(rng.rand(2, 4, 1).astype(self.floatX)) self.assertRaises(ValueError, f) def test_broadcastable_flag_assignment_mixed_thisaxes(self): # Test that the broadcastable flag of the join axis # is False when some inputs are broadcastable on that # dimension. rng = np.random.RandomState(seed=utt.fetch_seed()) a_val = rng.rand(2, 4, 1).astype(self.floatX) b_val = rng.rand(1, 4, 1).astype(self.floatX) a = self.shared(a_val, broadcastable=(False, False, True)) b = self.shared(b_val, broadcastable=(True, False, True)) c = self.join_op(0, a, b) assert not c.type.broadcastable[0] f = function([], c, mode=self.mode) topo = f.maker.fgraph.toposort() assert [True for node in topo if isinstance(node.op, type(self.join_op))] f() utt.verify_grad((lambda a, b: join(0, a, b)), [a_val, b_val], rng=rng, mode=self.mode) # Should raise an error if b_val.shape[0] is not 1 # We can't set the value| self.assertRaises(TypeError, b.set_value, rng.rand(3, 4, 1).astype(self.floatX)) a = TensorType(dtype=self.floatX, broadcastable=[0, 0, 1])() b = TensorType(dtype=self.floatX, broadcastable=[1, 0, 1])() c = self.join_op(0, a, b) f = function([a, b], c, mode=self.mode) bad_b_val = rng.rand(3, 4, 1).astype(self.floatX) self.assertRaises(TypeError, f, a_val, bad_b_val) def test_broadcastable_flags_all_broadcastable_on_joinaxis(self): # Test that joining together several inputs which are all # broadcastable on the join dimension results in the output # being non-broadcastable on the join dimension. rng = np.random.RandomState(seed=utt.fetch_seed()) a_val = rng.rand(1, 4, 1).astype(self.floatX) b_val = rng.rand(1, 4, 1).astype(self.floatX) a = self.shared(a_val, broadcastable=(True, False, True)) b = self.shared(b_val, broadcastable=(True, False, True)) c = self.join_op(0, a, b) assert not c.type.broadcastable[0] f = function([], c, mode=self.mode) topo = f.maker.fgraph.toposort() assert [True for node in topo if isinstance(node.op, type(self.join_op))] f() utt.verify_grad((lambda a, b: join(0, a, b)), [a_val, b_val], rng=rng, mode=self.mode) def test_broadcastable_single_input_broadcastable_dimension(self): # Test that all broadcastable flags are preserved by a # single-input join. rng = np.random.RandomState(seed=utt.fetch_seed()) a_val = rng.rand(1, 4, 1).astype(self.floatX) a = self.shared(a_val, broadcastable=(True, False, True)) b = self.join_op(0, a) assert b.type.broadcastable[0] assert b.type.broadcastable[2] assert not b.type.broadcastable[1] f = function([], b, mode=self.mode) topo = f.maker.fgraph.toposort() if theano.config.mode != 'FAST_COMPILE': assert not [True for node in topo if isinstance(node.op, type(self.join_op))] f() utt.verify_grad((lambda a: join(0, a)), [a_val], rng=rng, mode=self.mode) # Should raise an error if length of dimension 0 is not 1 self.assertRaises(TypeError, a.set_value, rng.rand(2, 4, 1).astype(self.floatX)) # self.assertRaises(TypeError, f, bad_a_val) def test_broadcastable_flags_many_dims_and_inputs(self): # Test that the right broadcastable flags get set for a join # with many inputs and many input dimensions. a = TensorType(dtype=self.floatX, broadcastable=[1, 0, 1, 0, 0, 0])() b = TensorType(dtype=self.floatX, broadcastable=[1, 1, 1, 0, 0, 0])() c = TensorType(dtype=self.floatX, broadcastable=[1, 0, 0, 0, 0, 0])() d = TensorType(dtype=self.floatX, broadcastable=[1, 0, 1, 1, 0, 1])() e = TensorType(dtype=self.floatX, broadcastable=[1, 0, 1, 0, 0, 1])() f = self.join_op(0, a, b, c, d, e) fb = f.type.broadcastable assert not fb[0] and fb[1] and fb[2] and fb[3] and not fb[4] and fb[5] g = self.join_op(1, a, b, c, d, e) gb = g.type.broadcastable assert gb[0] and not gb[1] and gb[2] and gb[3] and not gb[4] and gb[5] h = self.join_op(4, a, b, c, d, e) hb = h.type.broadcastable assert hb[0] and hb[1] and hb[2] and hb[3] and not hb[4] and hb[5] f = function([a, b, c, d, e], f, mode=self.mode) topo = f.maker.fgraph.toposort() assert [True for node in topo if isinstance(node.op, type(self.join_op))] rng = np.random.RandomState(seed=utt.fetch_seed()) a_val = rng.rand(1, 1, 1, 1, 2, 1).astype(self.floatX) b_val = rng.rand(1, 1, 1, 1, 2, 1).astype(self.floatX) c_val = rng.rand(1, 1, 1, 1, 2, 1).astype(self.floatX) d_val = rng.rand(1, 1, 1, 1, 2, 1).astype(self.floatX) e_val = rng.rand(1, 1, 1, 1, 2, 1).astype(self.floatX) f(a_val, b_val, c_val, d_val, e_val) utt.verify_grad((lambda a, b, c, d, e: join(0, a, b, c, d, e)), [a_val, b_val, c_val, d_val, e_val], rng=rng, mode=self.mode) # Should raise an error if length of dimension 0 is not 1 bad_val = rng.rand(2, 1, 1, 1, 2, 1).astype(self.floatX) self.assertRaises(TypeError, f, bad_val, b_val, c_val, d_val, e_val) self.assertRaises(TypeError, f, a_val, bad_val, c_val, d_val, e_val) self.assertRaises(TypeError, f, a_val, b_val, bad_val, d_val, e_val) self.assertRaises(TypeError, f, a_val, b_val, c_val, bad_val, e_val) self.assertRaises(TypeError, f, a_val, b_val, c_val, d_val, bad_val) # Should raise an error if any dimension other than 4 has length != 1 bad_a_val = rng.rand(1, 2, 1, 1, 2, 1).astype(self.floatX) bad_b_val = rng.rand(1, 1, 1, 1, 2, 2).astype(self.floatX) bad_c_val = rng.rand(1, 1, 2, 1, 2, 1).astype(self.floatX) bad_d_val = rng.rand(1, 2, 1, 1, 2, 1).astype(self.floatX) bad_e_val = rng.rand(1, 1, 1, 2, 2, 1).astype(self.floatX) self.assertRaises(ValueError, f, bad_a_val, b_val, c_val, d_val, e_val) self.assertRaises(ValueError, f, a_val, bad_b_val, c_val, d_val, e_val) self.assertRaises(ValueError, f, a_val, b_val, bad_c_val, d_val, e_val) self.assertRaises(ValueError, f, a_val, b_val, c_val, bad_d_val, e_val) self.assertRaises(ValueError, f, a_val, b_val, c_val, d_val, bad_e_val) def test_infer_shape_join(self): def get_mat(s1, s2): return np.asarray(np.random.uniform(size=(s1, s2)), dtype=self.floatX) x1 = self.shared(get_mat(3, 4)) x2 = self.shared(get_mat(2, 4)) x3 = self.shared(get_mat(1, 4)) # Test dim 0 z = self.join_op(0, x1, x2, x3) f = theano.function([], z.shape, mode=self.mode) topo = f.maker.fgraph.toposort() out = f() assert (out == [6, 4]).all() if theano.config.mode != 'FAST_COMPILE': for node in f.maker.fgraph.toposort(): assert not isinstance(node.op, type(self.join_op)) # Test dim 1 x1.set_value(get_mat(3, 4)) x2.set_value(get_mat(3, 4)) x3.set_value(get_mat(3, 5)) z = self.join_op(1, x1, x2, x3) f = theano.function([], z.shape, mode=self.mode) topo = f.maker.fgraph.toposort() out = f() assert (out == [3, 13]).all() if theano.config.mode != 'FAST_COMPILE': for node in topo: assert not isinstance(node.op, type(self.join_op)) with change_flags(compute_test_value='off'): # Test hide error x1.set_value(get_mat(3, 4)) x2.set_value(get_mat(3, 4)) x3.set_value(get_mat(2, 5)) if not self.hide_error: self.assertRaises(ValueError, f) else: f() def test_rebroadcast(self): # Regression test for a crash that used to happen when rebroadcasting. x = tensor.TensorType(self.floatX, [False, False, True])() u = tensor.TensorType(self.floatX, [False, False, True])() # This line used to crash. tensor.concatenate([x, -u], axis=2) def test_concatenate_same(self): # Test that we can concatenate the same tensor multiple time. # In the past it was broken on the GPU. rng = np.random.RandomState(seed=utt.fetch_seed()) T_shared = self.shared(rng.rand(3, 4).astype(self.floatX)) Tout = tensor.concatenate([T_shared, T_shared]) f = function([], Tout, mode=self.mode) out = f() if theano.config.mode != 'FAST_COMPILE': assert [True for node in f.maker.fgraph.toposort() if isinstance(node.op, type(self.join_op))] assert np.allclose(out, np.concatenate([T_shared.get_value(), T_shared.get_value()])) def test_mixed_ndim_error(self): rng = np.random.RandomState(seed=utt.fetch_seed()) v = self.shared(rng.rand(4).astype(self.floatX)) m = self.shared(rng.rand(4, 4).astype(self.floatX)) self.assertRaises(TypeError, self.join_op, 0, v, m) def test_split_0elem(self): rng = np.random.RandomState(seed=utt.fetch_seed()) m = self.shared(rng.rand(4, 6).astype(self.floatX)) o = self.split_op_class(2)(m, 0, [4, 0]) f = function([], o, mode=self.mode) assert any([isinstance(node.op, self.split_op_class) for node in f.maker.fgraph.toposort()]) o1, o2 = f() assert np.allclose(o1, m.get_value(borrow=True)) assert np.allclose(o2, m.get_value(borrow=True)[4:]) @change_flags(compute_test_value='off') def test_split_neg(self): rng = np.random.RandomState(seed=utt.fetch_seed()) m = self.shared(rng.rand(4, 6).astype(self.floatX)) o = self.split_op_class(2)(m, 0, [5, -1]) f = function([], o, mode=self.mode) assert any([isinstance(node.op, self.split_op_class) for node in f.maker.fgraph.toposort()]) self.assertRaises(ValueError, f) def test_join_inplace(): # Test join to work inplace. # # This function tests the case when several elements are passed to the # join function but all except one of them are empty. In this case join # should work inplace and the output should be the view of the non-empty # element. s = tensor.lscalar() x = tensor.vector('x') z = tensor.zeros((s,)) join = Join(view=0) c = join(0, x, z, z) f = theano.function([theano.In(x, borrow=True), s], theano.Out(c, borrow=True)) data = np.array([3, 4, 5], dtype=theano.config.floatX) print(f(data, 0)) if theano.config.mode not in ["DebugMode", "DEBUG_MODE"]: assert f(data, 0) is data assert np.allclose(f(data, 0), [3, 4, 5]) def test_join_oneInput(): # Test join when only 1 input is given. # # This functions tests the case when concatenate is called # on an array of tensors but the array has only one element. # In this case, we would like to avoid the computational # overhead of concatenation of one element. x_0 = theano.tensor.fmatrix() x_1 = theano.tensor.fmatrix() x_2 = theano.tensor.fvector() join_0 = theano.tensor.concatenate([x_0], axis=1) join_1 = theano.tensor.concatenate([x_0, x_1, theano.tensor.shape_padright(x_2)], axis=1) assert join_0 is x_0 assert join_1 is not x_0 class test_comparison(unittest.TestCase): # Test <, >, <=, >=, == and != # # Test that we can do the comparison with different # combination of tensor(shared and constant variable) with # ndarray. ndarray cmp tensor was crashing. In a NumPy PR (should # be in the release 1.8 of NumPy), it will work. So we assert it # work(futur behavior) or raise an error(current NumPy release). def setUp(self): utt.seed_rng() self.mode = None self.shared = shared self.dtypes = ['float64', 'float32', 'complex64', 'complex128'] def inplace_func(self, inputs, outputs, check_isfinite=None): mode = self.mode if check_isfinite is False: if mode is None: mode = get_default_mode() mode.check_isfinite = False f = inplace_func(inputs, outputs, mode=mode) return f def test_gt(self): for dtype in self.dtypes: l = np.asarray([0., -1., 1.], dtype=dtype) r = np.asarray([0., 1., -1.], dtype=dtype) for x, y, err in [ (self.shared(l.astype(dtype)), self.shared(r.astype(dtype)), False), (l, self.shared(r.astype(dtype)), True), (tensor.constant(l), self.shared(r.astype(dtype)), False), (self.shared(l.astype(dtype)), r, False), (self.shared(l.astype(dtype)), tensor.constant(r), False), ]: try: fn = self.inplace_func([], x > y) v = fn() self.assertTrue(np.all(v == (l > r)), (v, (l > r))) except TypeError: assert err def test_lt(self): for dtype in self.dtypes: l = np.asarray([0., -1., 1.], dtype=dtype) r = np.asarray([0., 1., -1.], dtype=dtype) for x, y, err in [ (self.shared(l.astype(dtype)), self.shared(r.astype(dtype)), False), (l, self.shared(r.astype(dtype)), True), (tensor.constant(l), self.shared(r.astype(dtype)), False), (self.shared(l.astype(dtype)), r, False), (self.shared(l.astype(dtype)), tensor.constant(r), False), ]: try: fn = self.inplace_func([], x < y) v = fn() self.assertTrue(np.all(v == (l < r)), (v, (l < r))) except TypeError: assert err def test_le(self): for dtype in self.dtypes: l = np.asarray([0., -1., 1.], dtype=dtype) r = np.asarray([0., 1., -1.], dtype=dtype) for x, y, err in [ (self.shared(l.astype(dtype)), self.shared(r.astype(dtype)), False), (l, self.shared(r.astype(dtype)), True), (tensor.constant(l), self.shared(r.astype(dtype)), False), (self.shared(l.astype(dtype)), r, False), (self.shared(l.astype(dtype)), tensor.constant(r), False), ]: try: fn = self.inplace_func([], x <= y) v = fn() self.assertTrue(np.all(v == (l <= r)), (v, (l <= r))) except TypeError: assert err def test_ge(self): for dtype in self.dtypes: l = np.asarray([0., -1., 1.], dtype=dtype) r = np.asarray([0., 1., -1.], dtype=dtype) for x, y, err in [ (self.shared(l.astype(dtype)), self.shared(r.astype(dtype)), False), (l, self.shared(r.astype(dtype)), True), (tensor.constant(l), self.shared(r.astype(dtype)), False), (self.shared(l.astype(dtype)), r, False), (self.shared(l.astype(dtype)), tensor.constant(r), False), ]: try: fn = self.inplace_func([], x >= y) v = fn() self.assertTrue(np.all(v == (l >= r)), (v, (l >= r))) except TypeError: assert err def test_eq(self): for dtype in self.dtypes: l = np.asarray([0., -1., 1.], dtype=dtype) r = np.asarray([0., 1., -1.], dtype=dtype) for x, y, err in [ (self.shared(l.astype(dtype)), self.shared(r.astype(dtype)), False), (l, self.shared(r.astype(dtype)), True), (tensor.constant(l), self.shared(r.astype(dtype)), False), (self.shared(l.astype(dtype)), r, False), (self.shared(l.astype(dtype)), tensor.constant(r), False), ]: try: fn = self.inplace_func([], eq(x, y)) v = fn() self.assertTrue(np.all(v == (l == r)), (v, (l == r))) except TypeError: assert err def test_neq(self): for dtype in self.dtypes: l = np.asarray([0., -1., 1.], dtype=dtype) r = np.asarray([0., 1., -1.], dtype=dtype) for x, y, err in [ (self.shared(l.astype(dtype)), self.shared(r.astype(dtype)), False), (l, self.shared(r.astype(dtype)), True), (tensor.constant(l), self.shared(r.astype(dtype)), False), (self.shared(l.astype(dtype)), r, False), (self.shared(l.astype(dtype)), tensor.constant(r), False), ]: try: fn = self.inplace_func([], neq(x, y)) v = fn() self.assertTrue(np.all(v == (l != r)), (v, (l != r))) except TypeError: assert err def test_isclose(self): for dtype in self.dtypes: l = np.asarray( [0., 1., -1., 0., np.nan, np.inf, -np.inf, np.inf], dtype=dtype) r = np.asarray( [0., 1.0001, -1.000000000001, np.nan, np.nan, np.inf, np.inf, 0.], dtype=dtype) for x, y, err in [ (self.shared(l.astype(dtype)), self.shared(r.astype(dtype)), False), (l, self.shared(r.astype(dtype)), True), (constant(l), self.shared(r.astype(dtype)), False), (self.shared(l.astype(dtype)), r, False), (self.shared(l.astype(dtype)), constant(r), False), ]: try: o1 = isclose(x, y, equal_nan=False) fn1 = self.inplace_func([], o1, check_isfinite=False) o2 = isclose(x, y, equal_nan=True) fn2 = self.inplace_func([], o2, check_isfinite=False) v1 = fn1() v2 = fn2() self.assertTrue( np.all( v1 == np.asarray( [True, False, True, False, False, True, False, False], dtype="bool" ) ), np.all( v2 == np.asarray( [True, False, True, False, True, True, False, False], dtype="bool" ) ) ) except TypeError: if not dtype.startswith('complex'): raise assert err def test_allclose(self): # equal_nan argument not in current version of numpy allclose, # force it to False. for dtype in self.dtypes: l = np.asarray( [0., 1., -1., 0., np.nan, np.inf, -np.inf, np.inf], dtype=dtype) r = np.asarray( [0., 1.0001, -1.000000000001, np.nan, np.nan, np.inf, np.inf, 0.], dtype=dtype) for x, y, err in [ (self.shared(l.astype(dtype)), self.shared(r.astype(dtype)), False), (l, self.shared(r.astype(dtype)), True), (constant(l), self.shared(r.astype(dtype)), False), (self.shared(l.astype(dtype)), r, False), (self.shared(l.astype(dtype)), constant(r), False), ]: try: fn = self.inplace_func([], allclose(x, y, equal_nan=False), check_isfinite=False) v = fn() self.assertTrue(np.all(v == np.allclose(l, r))) except TypeError: if not dtype.startswith('complex'): assert err class test_bitwise(unittest.TestCase): dtype = ['int8', 'int16', 'int32', 'int64', ] def test_or(self): for dtype in self.dtype: x, y = vector(dtype=dtype), vector(dtype=dtype) fn = inplace_func([x, y], x | y) l = theano._asarray([0, 0, 1, 1], dtype=dtype) r = theano._asarray([0, 1, 0, 1], dtype=dtype) v = fn(l, r) self.assertTrue(np.all(v == (operator.or_(l, r))), (l, r, v)) def test_xor(self): for dtype in self.dtype: x, y = vector(dtype=dtype), vector(dtype=dtype) fn = inplace_func([x, y], x ^ y) ix = x ix = inplace.xor_inplace(ix, y) gn = inplace_func([x, y], ix) l = theano._asarray([0, 0, 1, 1], dtype=dtype) r = theano._asarray([0, 1, 0, 1], dtype=dtype) v = fn(l, r) self.assertTrue(np.all(v == (operator.xor(l, r))), (l, r, v)) v = gn(l, r) # test the in-place stuff self.assertTrue(np.all(l == np.asarray([0, 1, 1, 0])), l) def test_and(self): for dtype in self.dtype: x, y = vector(dtype=dtype), vector(dtype=dtype) fn = inplace_func([x, y], x & y) l = theano._asarray([0, 0, 1, 1], dtype=dtype) r = theano._asarray([0, 1, 0, 1], dtype=dtype) v = fn(l, r) self.assertTrue(np.all(v == (operator.and_(l, r))), (l, r, v)) def test_inv(self): for dtype in self.dtype: x = vector(dtype=dtype) fn = inplace_func([x], ~x) for l in [[0, 0, 1, 1], [0, 1, 0, 1], [0, 0, 1, 1], [0, 1, 0, 1], [-1, 2 ** 16, 2 ** 16 - 1] ]: l = theano._asarray([0, 0, 1, 1], dtype=dtype) v = fn(l) self.assertTrue(np.all(v == (~l)), (l, v)) def test_eye(self): n = iscalar() m = iscalar() k = iscalar() fn = theano.function([m, n, k], eye(m, n, k)) self.assertTrue(np.all(fn(5, 6, 1) == np.eye(5, 6, 1))) class T_add(unittest.TestCase): def setUp(self): utt.seed_rng() def test_complex_all_ops(self): for nbits in (64, 128): a = shared(np.ones(3, dtype='complex%i' % nbits) + 0.5j) b = shared(np.ones(3, dtype='complex%i' % nbits) + 1.5j) tests = (("+", lambda x, y: x + y), ("-", lambda x, y: x - y), ("*", lambda x, y: x * y), ("/", lambda x, y: x / y)) for s, fn in tests: f = inplace_func([], fn(a, b)) # print 'valid output:', fn(a.data, b.data) # print 'theano output:', f(a.data, b.data) self.assertTrue(a.type.values_eq_approx(fn( a.get_value(), b.get_value()), f())) def test_grad_scalar_l(self): utt.verify_grad(add, [np.asarray([3.0]), rand(3)]) def test_grad_scalar_r(self): utt.verify_grad(add, [rand(3), np.asarray([3.0])]) def test_grad_row(self): utt.verify_grad(add, [rand(3, 5), rand(1, 5)]) def test_grad_col(self): utt.verify_grad(add, [rand(3, 5), rand(3, 1)]) class T_ceil(unittest.TestCase): def test_complex(self): self.assertRaises(TypeError, tensor.ceil, tensor.zvector()) class T_exp(unittest.TestCase): def test_grad_0(self): utt.verify_grad(exp, [ np.asarray([[1.5089518, 1.48439076, -4.7820262], [2.04832468, 0.50791564, -1.58892269]])]) def test_grad_1(self): utt.verify_grad(inplace.exp_inplace, [ np.asarray([[1.5089518, 1.48439076, -4.7820262], [2.04832468, 0.50791564, -1.58892269]])]) if theano.config.cycle_detection == 'fast' and theano.config.mode != 'FAST_COMPILE': test_grad_1 = unittest.expectedFailure(test_grad_1) def test_int(self): x = ivector() f = function([x], exp(x)) exp_3 = f([3]) assert exp_3.dtype == 'float64' def test_complex(self): x = zvector() assert exp(x).dtype == 'complex128' f = function([x], exp(x)) exp_3 = f([3 + 2j]) assert np.allclose(exp_3, np.exp(3 + 2j)) class T_divimpl(unittest.TestCase): def test_impls(self): i = iscalar() ii = lscalar() d = dscalar() f = fscalar() c = cscalar() assert np.allclose(function([i, d], i / d)(5, 7.0), (5.0 / 7.0)) assert np.allclose(function([i, d], d / i)(5, 7.0), (7.0 / 5.0)) assert np.allclose(function([i, f], i / f)(5, 11.0), (5.0 / 11.0)) assert np.allclose(function([i, f], f / i)(5, 11.0), (11.0 / 5.0)) assert np.allclose(function([i, ii], i // ii)(5, 3), (5 // 3)) assert np.allclose(function([i, ii], ii // i)(5, 3), (3 // 5)) assert np.allclose(function([i, ii], true_div(i, ii))(5, 3), (5. / 3.)) assert np.allclose(function([i, ii], true_div(ii, i))(5, 3), (3. / 5.)) assert np.allclose(function([i, c], i / c)(5, np.complex(5, 3)), (5. / (5 + 3j))) assert np.allclose(function([i, c], c / i)(5, np.complex(5, 3)), ((5 + 3j) / 5.)) class T_mean(unittest.TestCase): def test_regression_mean_of_ndarray_failure(self): try: tensor.mean(np.zeros(1)) except AttributeError: self.fail() def test_mean_f16(self): x = tensor.vector(dtype='float16') y = x.mean() f = theano.function([x], y) utt.assert_allclose(f(np.ones((100000,), dtype='float16')), 1.0) def test0(self): # Simple test... x = tensor.vector() f = theano.function([x], tensor.mean(x)) data = rand(50) assert np.allclose(f(data), np.mean(data)) def test_list(self): ll = [theano.shared(0.), theano.shared(2.)] tensor.mean(ll).eval() == 1 class test_matinv(unittest.TestCase): def mat_reciprocal(self, dim): # symbolic program # broadcastable=[False,False] means that the shape of matrix is two dimensional, # and none of the dimensions are constrained to have length 1. # Note that TensorType's constructor does not actually allocate any memory. # TODO: Make TensorType syntax more explicit, and maybe give shape or number of dimensions. rng = np.random.RandomState(seed=utt.fetch_seed()) a, b = matrices('ab') ab = a * b # Here, as_tensor_variable actually uses the data allocated by np. diff = ab - as_tensor_variable(np.ones((dim, dim), dtype=config.floatX)) # Sum of squared errors ssdiff = sum((diff ** 2.0)) g_b = grad(ssdiff, b) # compilation to function # [a,b] are the inputs, [ssdiff,g_b] are the outputs fn = inplace_func([a, b], [ssdiff, g_b]) # use the function x = rng.rand(dim, dim) + 0.1 # Initialized s.t. x is not too tiny w = rng.rand(dim, dim) x = np.asarray(x, dtype=config.floatX) w = np.asarray(w, dtype=config.floatX) for i in xrange(100): ssd, gw = fn(x, w) # print ssd, x*w, x, w if i == 0: ssd0 = ssd w -= 0.4 * gw return ssd0, ssd def test_reciprocal(self): # Matrix reciprocal by gradient descent ssd0, ssd = self.mat_reciprocal(3) rng = np.random.RandomState(seed=utt.fetch_seed()) # hand-coded numpy implementation for verification x = rng.rand(3, 3) + 0.1 w = rng.rand(3, 3) x = np.asarray(x, dtype=config.floatX) w = np.asarray(w, dtype=config.floatX) ones = np.ones((3, 3), dtype=config.floatX) myssd0 = np.sum((x * w - ones) ** 2.0) # we want at least a test that is not too fast. So we make one here. for i in xrange(100): gw = 2 * (x * w - ones) * x # derivative of dMSE/dw myssd = np.sum((x * w - ones) ** 2) w -= 0.4 * gw self.assertAlmostEqual(ssd0, myssd0) self.assertAlmostEqual(ssd, myssd) class t_dot(unittest.TestCase): def setUp(self): utt.seed_rng() def cmp_dot(self, x, y): # x, y are matrices or numbers def spec(x): x = np.asarray(x) return type(x), x.dtype, x.shape nz = np.dot(x, y) tz = eval_outputs([dot(as_tensor_variable(x), as_tensor_variable(y))]) self.assertTrue(tz.dtype == nz.dtype, (tz.dtype, tz.dtype.num, nz.dtype, nz.dtype.num)) self.assertTrue(tz.shape == nz.shape, (tz.shape, nz.shape)) utt.assert_allclose(nz, tz, rtol=1e-4, atol=1e-4) def test_Op_dims(self): # _dot is a Dot op instance _dot = theano.tensor.basic._dot d0 = scalar() d1 = vector() d2 = matrix() d3 = tensor3() self.assertRaises(TypeError, _dot, d0, d0) self.assertRaises(TypeError, _dot, d0, d1) self.assertRaises(TypeError, _dot, d0, d2) self.assertRaises(TypeError, _dot, d0, d3) self.assertRaises(TypeError, _dot, d1, d0) _dot(d1, d1) _dot(d1, d2) self.assertRaises(TypeError, _dot, d1, d3) self.assertRaises(TypeError, _dot, d2, d0) _dot(d2, d1) _dot(d2, d2) self.assertRaises(TypeError, _dot, d2, d3) self.assertRaises(TypeError, _dot, d3, d0) self.assertRaises(TypeError, _dot, d3, d1) self.assertRaises(TypeError, _dot, d3, d2) self.assertRaises(TypeError, _dot, d3, d3) def test_dot_0d_0d(self): self.cmp_dot(rand(), rand()) def test_dot_0d_1d(self): self.cmp_dot(rand(), rand(5)) def test_dot_0d_2d(self): self.cmp_dot(rand(), rand(6, 7)) def test_dot_0d_3d(self): self.cmp_dot(rand(), rand(8, 6, 7)) def test_dot_1d_0d(self): self.cmp_dot(rand(5), rand()) def test_dot_1d_1d(self): self.cmp_dot(rand(5), rand(5)) def test_dot_1d0_1d0(self): self.cmp_dot(rand(0), rand(0)) # numpy return matrix not aligned... def test_dot_1d_1d0(self): self.assertRaises(ValueError, self.cmp_dot, rand(5), rand(0)) # numpy return matrix not aligned... def test_dot_1d0_1d(self): self.assertRaises(ValueError, self.cmp_dot, rand(0), rand(5)) def test_dot_1d_2d(self): self.cmp_dot(rand(6), rand(6, 7)) def test_dot_1d0_2d(self): self.cmp_dot(rand(0), rand(0, 7)) def test_dot_1d_2d0(self): self.cmp_dot(rand(6), rand(6, 0)) def test_dot_1d0_2d0(self): self.cmp_dot(rand(0), rand(0, 0)) def test_dot_1d_3d(self): self.cmp_dot(rand(6), rand(8, 6, 7)) def test_dot_2d_0d(self): self.cmp_dot(rand(5, 6), rand()) def test_dot_2d_1d(self): self.cmp_dot(rand(5, 6), rand(6)) def test_dot_2d0_1d(self): self.cmp_dot(rand(0, 6), rand(6)) def test_dot_2d_1d0(self): self.cmp_dot(rand(5, 0), rand(0)) def test_dot_2d0_1d0(self): self.cmp_dot(rand(0, 0), rand(0)) def test_dot_2d_2d(self): self.cmp_dot(rand(5, 6), rand(6, 7)) def test_dot_2d0_2d(self): self.cmp_dot(rand(0, 6), rand(6, 7)) def test_dot_2d_2d0(self): self.cmp_dot(rand(5, 6), rand(6, 0)) def test_dot_2d0_2d0(self): self.cmp_dot(rand(0, 6), rand(6, 0)) def test_dot_2d_0_2d(self): self.cmp_dot(rand(5, 0), rand(0, 7)) def test_dot_2d0_0_2d0(self): self.cmp_dot(rand(0, 6), rand(6, 0)) def test_dot_2d_3d(self): self.cmp_dot(rand(5, 6), rand(8, 6, 7)) def test_dot_3d_0d(self): self.cmp_dot(rand(4, 5, 6), rand()) def test_dot_3d_1d(self): self.cmp_dot(rand(4, 5, 6), rand(6)) def test_dot_3d_2d(self): self.cmp_dot(rand(4, 5, 6), rand(6, 7)) def test_dot_3d_3d(self): self.cmp_dot(rand(4, 5, 6), rand(8, 6, 7)) def not_aligned(self, x, y): ctv_backup = config.compute_test_value config.compute_test_value = 'off' try: z = dot(x, y) finally: config.compute_test_value = ctv_backup # constant folding will complain to _logger that things are not aligned # this is normal, testers are not interested in seeing that output. _logger = logging.getLogger('theano.gof.opt') oldlevel = _logger.level _logger.setLevel(logging.CRITICAL) try: try: eval_outputs([z]) assert False # should have raised exception except ValueError as e: e0 = exc_message(e) self.assertTrue( # Reported by numpy. e0.split()[1:4] == ['are', 'not', 'aligned'] or # Reported by blas or Theano. e0.split()[0:2] == ['Shape', 'mismatch:'] or # Reported by Theano perform (e0.split()[0:4] == ['Incompatible', 'shapes', 'for', 'gemv']) or e) finally: _logger.setLevel(oldlevel) def test_align_1_1(self): self.not_aligned(rand(5), rand(6)) def test_align_1_2(self): self.not_aligned(rand(5), rand(6, 4)) def test_align_1_3(self): self.not_aligned(rand(5), rand(6, 4, 7)) def test_align_2_1(self): self.not_aligned(rand(5, 4), rand(6)) def test_align_2_2(self): self.not_aligned(rand(5, 4), rand(6, 7)) def test_align_2_3(self): self.not_aligned(rand(5, 4), rand(6, 7, 8)) def test_align_3_1(self): self.not_aligned(rand(5, 4, 3), rand(6)) def test_align_3_2(self): self.not_aligned(rand(5, 4, 3), rand(6, 7)) def test_align_3_3(self): self.not_aligned(rand(5, 4, 3), rand(6, 7, 8)) def test_grad(self): utt.verify_grad(dot, [rand(2, 3), rand(3, 2)]) utt.verify_grad(dot, [rand(2), rand(2, 3)]) utt.verify_grad(dot, [rand(3, 2), rand(2)]) utt.verify_grad(dot, [rand(2), rand(2)]) utt.verify_grad(dot, [rand(), rand(2)]) utt.verify_grad(dot, [rand(), rand(2, 5)]) utt.verify_grad(dot, [rand(2), rand()]) utt.verify_grad(dot, [rand(2, 5), rand()]) utt.verify_grad(dot, [rand(2, 3, 4), rand(4)]) utt.verify_grad(dot, [rand(3), rand(2, 3, 4)]) utt.verify_grad(dot, [rand(4, 3), rand(2, 3, 4)]) utt.verify_grad(dot, [rand(2, 3, 4), rand(4, 5)]) utt.verify_grad(dot, [rand(2, 3, 4), rand(3, 4, 5)]) @attr('slow') def test_broadcastable_patterns(self): # # These examples should all work because we broadcastable or # no, all dimensions of all results have size 1. # def val_for(r): if r.dtype.startswith('complex'): # We want to test complex at the same time, so we give a value # To the imaginary component. # This strange way of doing things is the only way that worked # on numpy 1.4.1 if r.ndim == 0: return np.asarray(np.complex(1.1, 2.1), dtype=r.dtype) if r.ndim == 1: if r.dtype == 'complex64': return np.complex64([np.complex(1.2, 2.2)]) elif r.dtype == 'complex128': return np.complex128([np.complex(1.2, 2.2)]) elif r.ndim == 2: if r.dtype == 'complex64': return np.complex64([[np.complex(1.3, 2.3)]]) elif r.dtype == 'complex128': return np.complex128([[np.complex(1.3, 2.3)]]) if r.ndim == 0: return np.asarray(1.1, dtype=r.dtype) if r.ndim == 1: return np.asarray([1.2], dtype=r.dtype) elif r.ndim == 2: return np.asarray([[1.3]], dtype=r.dtype) raise ValueError() for dtype0 in ('float32', 'float64', 'complex64'): for dtype1 in ('float32', 'complex64', 'complex128'): for bc0 in ((True,), (False,), (True, True), (True, False), (False, True), (False, False)): x = TensorType(dtype=dtype0, broadcastable=bc0)() for bc1 in ((True,), (False,), (True, True), (True, False), (False, True), (False, False)): y = TensorType(dtype=dtype1, broadcastable=bc1)() z = dot(x, y) t = TensorType(dtype=dtype0, broadcastable=z.broadcastable)() rval = z * 3 + 2 * t f = function([x, y, t], rval) xval = val_for(x) yval = val_for(y) tval = val_for(t) f(xval, yval, tval) # debugmode checks result if (dtype0.startswith('float') and dtype1.startswith('float')): g = grad(z.sum(), x) assert g.broadcastable == x.broadcastable g = grad(z.sum(), y) assert g.broadcastable == y.broadcastable class T_tensorfromscalar(unittest.TestCase): def test0(self): s = scal.constant(56) t = tensor_from_scalar(s) self.assertTrue(t.owner.op is tensor_from_scalar) self.assertTrue(t.type.broadcastable == (), t.type.broadcastable) self.assertTrue(t.type.ndim == 0, t.type.ndim) self.assertTrue(t.type.dtype == s.type.dtype) v = eval_outputs([t]) self.assertTrue(v == 56, v) self.assertTrue(isinstance(v, np.ndarray)) self.assertTrue(v.shape == (), v.shape) def test1(self): s = scal.constant(56) t = as_tensor_variable(s) self.assertTrue(t.owner.op is tensor_from_scalar) self.assertTrue(t.type.broadcastable == (), t.type.broadcastable) self.assertTrue(t.type.ndim == 0, t.type.ndim) self.assertTrue(t.type.dtype == s.type.dtype) v = eval_outputs([t]) self.assertTrue(v == 56, v) self.assertTrue(isinstance(v, np.ndarray)) self.assertTrue(v.shape == (), v.shape) g = grad(t, s) self.assertTrue(eval_outputs([g]) == 0.) def test2(self): s = scal.constant(56.) t = as_tensor_variable(s) self.assertTrue(t.owner.op is tensor_from_scalar) self.assertTrue(t.type.broadcastable == (), t.type.broadcastable) self.assertTrue(t.type.ndim == 0, t.type.ndim) self.assertTrue(t.type.dtype == s.type.dtype) v = eval_outputs([t]) self.assertTrue(v == 56., v) self.assertTrue(isinstance(v, np.ndarray)) self.assertTrue(v.shape == (), v.shape) g = grad(t, s) self.assertTrue(eval_outputs([g]) == 1.) class T_scalarfromtensor(unittest.TestCase): def test0(self): tt = constant(56) # scal.constant(56) ss = scalar_from_tensor(tt) self.assertTrue(ss.owner.op is scalar_from_tensor) self.assertTrue(ss.type.dtype == tt.type.dtype) v = eval_outputs([ss]) self.assertTrue(v == 56, v) if config.cast_policy == 'custom': self.assertTrue(isinstance(v, np.int8)) elif config.cast_policy in ('numpy', 'numpy+floatX'): self.assertTrue(isinstance( v, getattr(np, str(np.asarray(56).dtype)))) else: raise NotImplementedError(config.cast_policy) self.assertTrue(v.shape == (), v.shape) tt = lscalar() ss = scalar_from_tensor(tt) ss.owner.op.grad([tt], [ss]) fff = function([tt], ss) v = fff(np.asarray(5)) self.assertTrue(v == 5, v) self.assertTrue(isinstance(v, np.int64)) self.assertTrue(v.shape == (), v.shape) class test_grad(unittest.TestCase): class O(gof.op.Op): def __init__(self): self.gval0 = scalar('e') self.gval1 = scalar('f') def make_node(self): inputs = [scalar('a'), scalar('c')] outputs = [scalar('b'), scalar('d')] return gof.Apply(self, inputs, outputs) def grad(self, inp, grads): x0, x1 = inp gz0, gz1 = grads return self.gval0, self.gval1 def test_1param(self): # grad: Test passing a single variable param o = test_grad.O() a1 = o.make_node() self.assertTrue(o.gval0 is tensor.grad(a1.outputs[0], a1.inputs[0])) def test_Nparam(self): # grad: Test passing multiple variable params o = test_grad.O() a1 = o.make_node() g0, g1 = grad(a1.outputs[0], a1.inputs) g0.name = None self.assertTrue(o.gval0 is g0) self.assertTrue(o.gval1 is g1) def test_grad_keep_type(self): # Tests that the theano grad method returns a list if it is passed a list # and a single variable if it is passed a single variable. # pylearn2 depends on theano behaving this way. This functionality has been # added three times and erroneously removed twice. If you do anything that # requires changing this test or making it fail you are almost certainly # making a common mistake, NOT fixing something. X = tensor.matrix() y = X.sum() G = tensor.grad(y, [X]) assert isinstance(G, list) G = tensor.grad(y, X) assert not isinstance(G, list) def test_1None_rval(self): # grad: Test returning a single zero value from grad o = test_grad.O() a1 = o.make_node() g = grad(a1.outputs[0], a1.outputs[1], disconnected_inputs='ignore') self.assertTrue(g.owner.op == fill) self.assertTrue(g.owner.inputs[1].data == 0) self.assertRaises(TypeError, grad, a1.outputs[0], 'wtf') def test_NNone_rval(self): # grad: Test returning some zero value from grad o = test_grad.O() a1 = o.make_node() g0, g1, g2 = grad(a1.outputs[0], a1.inputs + [scalar('z')], disconnected_inputs='ignore') self.assertTrue(o.gval0 is g0) self.assertTrue(o.gval1 is g1) self.assertTrue(g2.owner.op == fill) self.assertTrue(g2.owner.inputs[1].data == 0) def test_zero_gradient_shape(self): # Ensure that a zero gradient has the proper shape. x = dmatrix() f = theano.function([x], grad(dscalar(), x, disconnected_inputs='ignore')) a = np.ones((3, 7)) self.assertTrue((f(a) == 0).all()) # Zero gradient. self.assertTrue(a.shape == f(a).shape) # With proper shape. def test_cost_is_scalar(self): # grad: Test that a non-scalar cost raises a TypeError v = vector() m = matrix() # grad(v,...) and grad(m,...) should fail self.assertRaises(TypeError, grad, v, v) self.assertRaises(TypeError, grad, m, m) class T_op_cache(unittest.TestCase): def setUp(self): utt.seed_rng() def test0(self): # trigger bug in ticket #162 v = matrix() v.name = 'v' gv = fill(v / v, 1.0) / v - (fill(v / v, 1.0) * v) / (v * v) fn_py = inplace_func([v], gv) fn_c_or_py = inplace_func([v], gv) a = rand(5, 2).astype(config.floatX) self.assertTrue(np.all(fn_py(a) == fn_c_or_py(a))) class T_reshape(utt.InferShapeTester, utt.TestOptimizationMixin): def __init__(self, name, shared=tensor._shared, op=Reshape, mode=None, ignore_topo=(DeepCopyOp, opt.MakeVector, opt.Shape_i, DimShuffle, theano.tensor.Elemwise)): self.shared = shared self.op = op # The tag canonicalize is needed for the shape test in FAST_COMPILE self.mode = mode self.ignore_topo = ignore_topo super(T_reshape, self).__init__(name) def function(self, inputs, outputs, ignore_empty=False): f = function(inputs, outputs, mode=self.mode) if self.mode is not None or theano.config.mode != "FAST_COMPILE": topo = f.maker.fgraph.toposort() topo_ = [node for node in topo if not isinstance(node.op, self.ignore_topo)] if ignore_empty: assert len(topo_) <= 1, topo_ else: assert len(topo_) == 1, topo_ if len(topo_) > 0: assert type(topo_[0].op) is self.op return f def test_reshape(self): a = dvector() b = dmatrix() d = dmatrix() # basic to 1 dim(without list) c = reshape(b, as_tensor_variable(6), ndim=1) f = self.function([b], c) b_val1 = np.asarray([[0, 1, 2], [3, 4, 5]]) c_val1 = np.asarray([0, 1, 2, 3, 4, 5]) b_val2 = b_val1.T c_val2 = np.asarray([0, 3, 1, 4, 2, 5]) f_out1 = f(b_val1) f_out2 = f(b_val2) assert np.all(f_out1 == c_val1), (f_out1, c_val1) assert np.all(f_out2 == c_val2), (f_out2, c_val2) # print f.maker.fgraph.toposort() # check that we remove the useless reshape # basic to 1 dim(with list) c = reshape(b, (as_tensor_variable(6),), ndim=1) f = self.function([b], c) assert np.all(f(np.asarray([[0, 1, 2], [3, 4, 5]])) == np.asarray([0, 1, 2, 3, 4, 5])) # print f.maker.fgraph.toposort() # check that we remove the useless reshape # basic to shape object of same ndim c = reshape(b, d.shape) f = self.function([b, d], c) assert np.all(f(np.asarray([[0, 1, 2], [3, 4, 5]]), [[0, 1], [2, 3], [4, 5]]) == np.asarray([[0, 1], [2, 3], [4, 5]])) # basic to 2 dims c = reshape(a, [2, 3]) f = self.function([a], c) assert np.all(f(np.asarray([0, 1, 2, 3, 4, 5])) == np.asarray([[0, 1, 2], [3, 4, 5]])) # test that it works without inplace operations a_val = np.asarray([0, 1, 2, 3, 4, 5]) a_val_copy = np.asarray([0, 1, 2, 3, 4, 5]) b_val = np.asarray([[0, 1, 2], [3, 4, 5]]) f_sub = self.function([a, b], c - b) assert np.all(f_sub(a_val, b_val) == 0.0) assert np.all(a_val == a_val_copy) # test that it works with inplace operations a_val = theano._asarray([0, 1, 2, 3, 4, 5], dtype='float64') a_val_copy = theano._asarray([0, 1, 2, 3, 4, 5], dtype='float64') b_val = theano._asarray([[0, 1, 2], [3, 4, 5]], dtype='float64') f_sub = self.function([a, b], c - b) assert np.all(f_sub(a_val, b_val) == 0.0) assert np.all(a_val == a_val_copy) # verify gradient def just_vals(v): return Reshape(2)(v, theano._asarray([2, 3], dtype='int32')) utt.verify_grad(just_vals, [a_val], mode=self.mode) # test infer_shape self._compile_and_check([a], [c], (a_val,), self.op) # test broadcast flag for constant value of 1 c = reshape(b, (b.shape[0], b.shape[1], 1)) # That reshape may get replaced with a dimshuffle, with is ignored, # so we pass "ignore_empty=True" f = self.function([b], c, ignore_empty=True) assert np.all(f(np.asarray([[0, 1, 2], [3, 4, 5]])) == np.asarray([[[0], [1], [2]], [[3], [4], [5]]])) assert (f.maker.fgraph.toposort()[-1].outputs[0].type.broadcastable == (False, False, True)) # test broadcast flag for constant value of 1 if it cannot be # replaced with dimshuffle c = reshape(b, (b.shape[1], b.shape[0], 1)) f = self.function([b], c, ignore_empty=True) assert np.all(f(np.asarray([[0, 1, 2], [3, 4, 5]])) == np.asarray([[[0], [1]], [[2], [3]], [[4], [5]]])) assert (f.maker.fgraph.toposort()[-1].outputs[0].type.broadcastable == (False, False, True)) def test_m1(self): t = tensor3() rng = np.random.RandomState(seed=utt.fetch_seed()) val = rng.uniform(size=(3, 4, 5)).astype(config.floatX) for out in [t.reshape([-1]), t.reshape([-1, 5]), t.reshape([5, -1]), t.reshape([5, -1, 3])]: self._compile_and_check([t], [out], [val], self.op) def test_reshape_long_in_shape(self): v = dvector('v') r = v.reshape((v.shape[0], L(1))) print(r.eval({v: np.arange(5.)})) assert np.allclose(r.eval({v: np.arange(5.)}).T, np.arange(5.)) def test_bad_shape(self): a = matrix('a') shapes = ivector('shapes') rng = np.random.RandomState(seed=utt.fetch_seed()) a_val = rng.uniform(size=(3, 4)).astype(config.floatX) # Test reshape to 1 dim r = a.reshape(shapes, ndim=1) f = self.function([a, shapes], r) self.assertRaises(ValueError, f, a_val, [13]) # Test reshape to 2 dim r = a.reshape(shapes, ndim=2) f = self.function([a, shapes], r) self.assertRaises(ValueError, f, a_val, [-1, 5]) self.assertRaises(ValueError, f, a_val, [7, -1]) self.assertRaises(ValueError, f, a_val, [7, 5]) self.assertRaises(ValueError, f, a_val, [-1, -1]) def test_0(self): x = fvector('x') f = self.function([x], x.reshape((0, 100))) assert f(np.ndarray((0,), dtype='float32')).shape == (0, 100) def test_empty_shp(self): const = theano.tensor.constant([1]).reshape(()) f = function([], const) assert f().shape == () def test_make_column_matrix_broadcastable(): # The goal of the operation made by `b` is to ensure the second dimension # of the column matrix is broadcastable. a = tensor.dmatrix() b = a.reshape((a.shape[0], )).dimshuffle(0, 'x') f = function([a], b) assert (f(np.zeros((3, 1))) + np.ones(2) == np.ones((3, 2))).all() def test_flatten_outdimNone(): a = dmatrix() c = flatten(a) f = inplace_func([a], c) a_val = theano._asarray([[0, 1, 2], [3, 4, 5]], dtype='float64') c_val = theano._asarray([0, 1, 2, 3, 4, 5], dtype='float64') assert np.all(f(a_val) == c_val) f = inplace_func([a], c) assert np.all(f(a_val) == c_val) utt.verify_grad(flatten, [a_val]) def test_flatten_scalar(): a = dscalar() c = flatten(a) f = inplace_func([a], c) a_val = theano._asarray(3.0, dtype='float64') c_val = theano._asarray([3.0], dtype='float64') assert np.all(f(a_val) == c_val) f = inplace_func([a], c) assert np.all(f(a_val) == c_val) # utt.verify_grad(flatten, [a_val]) #TODO: fix verify_grd to work on scalars def test_flatten_ndim1(): a = dmatrix() c = flatten(a, 1) f = inplace_func([a], c) a_val = theano._asarray([[0, 1, 2], [3, 4, 5]], dtype='float64') c_val = theano._asarray([0, 1, 2, 3, 4, 5], dtype='float64') assert np.all(f(a_val) == c_val) f = inplace_func([a], c) assert np.all(f(a_val) == c_val) utt.verify_grad(flatten, [a_val]) def test_flatten_ndim2(): a = dmatrix() c = flatten(a, 2) f = inplace_func([a], c) a_val = theano._asarray([[0, 1, 2], [3, 4, 5]], dtype='float64') assert np.all(f(a_val) == a_val) f = inplace_func([a], c) assert np.all(f(a_val) == a_val) flatten_2 = partial(flatten, ndim=2) utt.verify_grad(flatten_2, [a_val]) def test_flatten_ndim2_of_3(): a = TensorType('float64', (False, False, False))() c = flatten(a, 2) f = inplace_func([a], c) a_val = theano._asarray([[[0, 1], [2, 3]], [[4, 5], [6, 7]]], dtype='float64') c_val = theano._asarray([[0, 1, 2, 3], [4, 5, 6, 7]], dtype='float64') assert np.all(f(a_val) == c_val) f = inplace_func([a], c) assert np.all(f(a_val) == c_val) flatten_2 = partial(flatten, ndim=2) utt.verify_grad(flatten_2, [a_val]) # test outdim parameter name flatten_2 = partial(flatten, outdim=2) utt.verify_grad(flatten_2, [a_val]) def test_flatten_broadcastable(): # Ensure that the broadcastable pattern of the output is coherent with # that of the input inp = TensorType('float64', (False, False, False, False))() out = flatten(inp, ndim=2) assert out.broadcastable == (False, False) inp = TensorType('float64', (False, False, False, True))() out = flatten(inp, ndim=2) assert out.broadcastable == (False, False) inp = TensorType('float64', (False, True, False, True))() out = flatten(inp, ndim=2) assert out.broadcastable == (False, False) inp = TensorType('float64', (False, True, True, True))() out = flatten(inp, ndim=2) assert out.broadcastable == (False, True) inp = TensorType('float64', (True, False, True, True))() out = flatten(inp, ndim=3) assert out.broadcastable == (True, False, True) def test_flatten_ndim_invalid(): a = dmatrix() assert_raises(ValueError, flatten, a, 3) assert_raises(ValueError, flatten, a, 0) def test_is_flat(): # tests is_flat method for constant and symbolic variables, # as well as reshaped constant and symbolic variables on the # given outdim # Constant variable assert tensor.is_flat(tensor.as_tensor_variable(np.zeros((10)))) assert tensor.is_flat(tensor.as_tensor_variable(np.zeros((10, 10, 10))), ndim=3) assert not tensor.is_flat( tensor.as_tensor_variable(np.zeros((10, 10, 10)))) # Symbolic variable assert tensor.is_flat(tensor.vector()) assert tensor.is_flat(tensor.tensor3(), ndim=3) assert not tensor.is_flat(tensor.tensor3()) # Reshape with constant shape X = tensor.tensor4() assert tensor.is_flat(X.reshape((-1, ))) assert tensor.is_flat(X.reshape((10, 10, -1)), ndim=3) assert not tensor.is_flat(X.reshape((10, 10, -1))) # Reshape with symbolic shape X = tensor.tensor4() assert tensor.is_flat(X.reshape((tensor.iscalar(), ))) assert tensor.is_flat(X.reshape((tensor.iscalar(), ) * 3), ndim=3) assert not tensor.is_flat(X.reshape((tensor.iscalar(), ) * 3)) def test_tile(): def run_tile(x, x_, reps, use_symbolic_reps): if use_symbolic_reps: rep_symbols = [iscalar() for _ in range(len(reps))] f = function([x] + rep_symbols, tile(x, rep_symbols)) return f(*([x_] + list(reps))) else: f = function([x], tile(x, reps)) return f(x_) rng = np.random.RandomState(utt.fetch_seed()) for use_symbolic_reps in [False, True]: # Test the one-dimensional case. x = vector() x_ = rng.randn(5).astype(config.floatX) assert np.all(run_tile(x, x_, (2,), use_symbolic_reps) == np.tile(x_, (2,))) # Test the two-dimensional case. x = matrix() x_ = rng.randn(2, 4).astype(config.floatX) assert np.all(run_tile(x, x_, (2, 3), use_symbolic_reps) == np.tile(x_, (2, 3))) # Test the three-dimensional case. x = tensor3() x_ = rng.randn(2, 4, 3).astype(config.floatX) assert np.all(run_tile(x, x_, (2, 3, 4), use_symbolic_reps) == np.tile(x_, (2, 3, 4))) # Test the four-dimensional case. x = tensor4() x_ = rng.randn(2, 4, 3, 5).astype(config.floatX) assert np.all(run_tile(x, x_, (2, 3, 4, 6), use_symbolic_reps) == np.tile(x_, (2, 3, 4, 6))) # Test when reps is integer, tensor.scalar or tensor.vector. # Test 1,2,3,4-dimensional cases. # Test input x has the shape [2], [2, 4], [2, 4, 3], [2, 4, 3, 5]. test_shape = [2, 4, 3, 5] k = 0 for xtype in [vector(), matrix(), tensor3(), tensor4()]: x = xtype k = k + 1 x_ = rng.randn(*test_shape[0:k]).astype(config.floatX) # integer: reps_ = 2 f = function([x], tile(x, reps_)) assert np.all(f(x_) == np.tile(x_, reps_)) # tensor.scalar: reps = iscalar() reps_ = 2 f = function([x, reps], tile(x, reps)) assert np.all(f(x_, reps_) == np.tile(x_, reps_)) # tensor.vector: reps = ivector() reps_ = [2] if k == 1 or k == 2 else [2, 3] ndim_ = k f = function([x, reps], tile(x, reps, ndim_)) assert np.all(f(x_, reps_) == np.tile(x_, reps_)) # list of integers: reps_ = [2, 3, 4] f = function([x], tile(x, reps_)) assert np.all(f(x_) == np.tile(x_, reps_)) # list of integers and tensor.scalars: d = iscalar() reps = [2, d, 4] f = function([x, d], tile(x, reps)) reps_ = [2, 3, 4] assert np.all(f(x_, 3) == np.tile(x_, reps_)) # reps is list, len(reps) > x.ndim, 3 cases below: r = [2, 3, 4, 5, 6] reps_ = r[:k + 1] # len(reps_) = x.ndim+1 # (1) ndim = None. f = function([x], tile(x, reps_)) assert np.all(f(x_) == np.tile(x_, reps_)) # (2) ndim = len(reps). ndim_ = len(reps_) f = function([x], tile(x, reps_, ndim_)) assert np.all(f(x_) == np.tile(x_, reps_)) # (3) ndim > len(reps) ndim_ = len(reps_) + 1 f = function([x], tile(x, reps_, ndim_)) assert np.all(f(x_) == np.tile(x_, [1] + reps_)) # reps is list, ndim > x.ndim > len(reps): r = [2, 3, 4, 5] if k > 1: ndim_ = k + 1 reps_ = r[:k - 1] f = function([x], tile(x, reps_, ndim_)) assert np.all(f(x_) == np.tile(x_, [1, 1] + reps_)) # error raising test: ndim not specified when reps is vector reps = ivector() np.testing.assert_raises(ValueError, tile, x, reps) # error raising test: not a integer for reps in [2.5, fscalar(), fvector()]: np.testing.assert_raises(ValueError, tile, x, reps) # error raising test: the dimension of reps exceeds 1 reps = imatrix() np.testing.assert_raises(ValueError, tile, x, reps) # error raising test: ndim is not None, ndim < x.ndim # 3 cases below (reps is list/tensor.scalar/tensor.vector): for reps in [[2, 3, 4], iscalar(), ivector()]: if k > 1: ndim = k - 1 np.testing.assert_raises(ValueError, tile, x, reps, ndim) # error raising test: reps is list, len(reps) > ndim r = [2, 3, 4, 5, 6] reps = r[:k + 1] ndim = k np.testing.assert_raises(ValueError, tile, x, reps, ndim) # error raising test: # reps is tensor.vector and len(reps_value) > ndim, # reps_value is the real value when excuting the function. reps = ivector() r = [2, 3, 4, 5, 6, 7] reps_ = r[:k + 2] ndim_ = k + 1 f = function([x, reps], tile(x, reps, ndim_)) np.testing.assert_raises(AssertionError, f, x_, reps_) def test_tile_grad(): def grad_tile(x, reps, np_x): y = tile(x, reps) z = y.sum() g = theano.function([x], grad(z, x)) grad_res = g(np_x) # The gradient should be the product of the tiling dimensions # (since the gradients are additive through the tiling operation) assert np.all(grad_res == np.prod(reps)) rng = np.random.RandomState(utt.fetch_seed()) # test vector grad_tile(vector('x'), [3], rng.randn(5).astype(config.floatX)) # test matrix grad_tile(matrix('x'), [3, 4], rng.randn(2, 3).astype(config.floatX)) # test tensor3 grad_tile(tensor3('x'), [3, 4, 5], rng.randn(2, 4, 3).astype(config.floatX)) # test tensor4 grad_tile(tensor4('x'), [3, 4, 5, 6], rng.randn(2, 4, 3, 5).astype(config.floatX)) class TestARange(unittest.TestCase): def setUp(self): utt.seed_rng() def test_Op_integers(self): # Test behaviour of ARange Op on integer inputs start, stop, step = iscalars('start', 'stop', 'step') out = ARange(start.type.dtype)(start, stop, step) f = function([start, stop, step], out) assert np.all(f(0, 5, 1) == np.arange(0, 5, 1)) assert np.all(f(2, 11, 4) == np.arange(2, 11, 4)) assert np.all(f(-5, 1, 1) == np.arange(-5, 1, 1)) assert np.all(f(10, 2, -2) == np.arange(10, 2, -2)) assert np.all(f(10, 2, 2) == np.arange(10, 2, 2)) assert np.all(f(0, 0, 1) == np.arange(0, 0, 1)) def test_grads(self): def f(start, stop, step): return ARange(start.type.dtype)(start, stop, step) rng = np.random.RandomState(utt.fetch_seed()) # Due to the random projection, we should not use the exact # point that change the shape of the output. for start, stop, step in [(0, 4.9, 1), (5.1, 0, -0.5), (1, 5.1, 0.5)]: utt.verify_grad(f, [np.asarray(start).astype(config.floatX), np.asarray(stop).astype(config.floatX), np.asarray(step).astype(config.floatX)], rng=rng) def test_integers(self): # Test arange constructor, on integer outputs start, stop, step = iscalars('start', 'stop', 'step') out = arange(start, stop, step) f = function([start, stop, step], out) if config.cast_policy == 'custom': assert out.dtype == 'int64' elif config.cast_policy in ('numpy', 'numpy+floatX'): numpy_dtype = np.arange(np.array(1, dtype='int32')).dtype assert out.dtype == numpy_dtype else: raise NotImplementedError(config.cast_policy) assert np.all(f(0, 5, 1) == np.arange(0, 5, 1)) assert np.all(f(2, 11, 4) == np.arange(2, 11, 4)) assert np.all(f(-5, 1, 1) == np.arange(-5, 1, 1)) assert np.all(f(10, 2, -2) == np.arange(10, 2, -2)) assert np.all(f(10, 2, 2) == np.arange(10, 2, 2)) assert np.all(f(0, 0, 1) == np.arange(0, 0, 1)) def test_float32(self): # Test arange constructor, on float32 outputs start, stop, step = fscalars('start', 'stop', 'step') out = arange(start, stop, step) f = function([start, stop, step], out) if config.cast_policy == 'custom': assert out.dtype == start.type.dtype elif config.cast_policy == 'numpy': numpy_dtype = np.arange(np.array(0, dtype=start.dtype), np.array(1, dtype=stop.dtype), np.array(1, dtype=step.dtype)).dtype assert out.dtype == numpy_dtype elif config.cast_policy == 'numpy+floatX': assert out.dtype == config.floatX else: raise NotImplementedError(config.cast_policy) arg_vals = [(0, 5, 1), (2, 11, 4), (-5, 1.1, 1.2), (1.3, 2, -2.1), (10, 2, 2)] for arg_v in arg_vals: start_v, stop_v, step_v = arg_v start_v_, stop_v_, step_v_ = np.asarray(arg_v, dtype=start.type.dtype) f_val = f(start_v_, stop_v_, step_v_) if config.cast_policy == 'custom': expected_val = np.arange(start_v, stop_v, step_v, dtype=start.type.dtype) elif config.cast_policy in ('numpy', 'numpy+floatX'): expected_val = np.arange(start_v_, stop_v_, step_v_, dtype=out.dtype) else: raise NotImplementedError(config.cast_policy) assert np.all(f_val == expected_val) def test_float64(self): # Test arange constructor, on float64 outputs start, stop, step = dscalars('start', 'stop', 'step') out = arange(start, stop, step) f = function([start, stop, step], out) assert out.dtype == start.type.dtype arg_vals = [(0, 5, 1), (2, 11, 4), (-5, 1.1, 1.2), (1.3, 2, -2.1), (10, 2, 2)] for arg_v in arg_vals: start_v, stop_v, step_v = arg_v start_v_, stop_v_, step_v_ = np.asarray(arg_v, dtype=start.type.dtype) f_val = f(start_v_, stop_v_, step_v_) if config.cast_policy == 'custom': expected_val = np.arange(start_v, stop_v, step_v, dtype=start.type.dtype) elif config.cast_policy in ('numpy', 'numpy+floatX'): expected_val = np.arange(start_v_, stop_v_, step_v_) else: raise NotImplementedError(config.cast_policy) assert np.all(f_val == expected_val) def test_default_step(self): # Test that arange constructor uses the correct default step start, stop = iscalars('start', 'stop') out = arange(start, stop) f = function([start, stop], out) if config.cast_policy == 'custom': assert out.dtype == 'int64' elif config.cast_policy in ('numpy', 'numpy+floatX'): assert out.dtype == np.arange(np.int32(0), np.int32(1)).dtype else: raise NotImplementedError(config.cast_policy) assert np.all(f(0, 5) == np.arange(0, 5)) assert np.all(f(-5, 1) == np.arange(-5, 1)) assert np.all(f(0, 0) == np.arange(0, 0)) dstart, dstop = dscalars('start', 'stop') dout = arange(dstart, dstop) df = function([dstart, dstop], dout) assert dout.dtype == dstart.type.dtype # print df(0.2, 5.3) # print np.arange(0.2, 5.3) assert np.all(df(0.2, 5.3) == np.arange(0.2, 5.3)) assert np.all(df(0.8, 5.3) == np.arange(0.8, 5.3)) assert np.all(df(-0.7, 5.3) == np.arange(-0.7, 5.3)) def test_default_start(self): # Test that arange constructor uses the correct default start stop = iscalar('stop') out = arange(stop) f = function([stop], out) if config.cast_policy == 'custom': assert out.dtype == 'int64' elif config.cast_policy in ('numpy', 'numpy+floatX'): assert out.dtype == np.arange(np.int32(1)).dtype else: raise NotImplementedError(config.cast_policy) assert np.all(f(8) == np.arange(8)) assert np.all(f(-2) == np.arange(-2)) fstop = fscalar('stop') fout = arange(fstop) ff = function([fstop], fout) if config.cast_policy == 'custom': assert fout.dtype == fstop.type.dtype elif config.cast_policy == 'numpy': assert fout.dtype == np.arange(np.float32(1)).dtype elif config.cast_policy == 'numpy+floatX': if config.floatX == 'float32': assert fout.dtype == 'float32' else: assert fout.dtype == np.arange(np.float32(1)).dtype else: raise NotImplementedError(config.cast_policy) fstop_values = [0.2, -0.7, 8.5] for fstop_v in fstop_values: fstop_v32 = np.float32(fstop_v) assert np.all(ff(fstop_v32) == np.arange(fstop_v)) def test_upcast(self): # Test that arange computes output type adequately if config.cast_policy == 'custom': assert arange(iscalar()).dtype == 'int64' assert arange(fscalar()).dtype == fscalar().dtype assert arange(dscalar()).dtype == dscalar().dtype # int32 + float32 -> float64 assert arange(iscalar(), fscalar()).dtype == dscalar().dtype assert arange(iscalar(), dscalar()).dtype == dscalar().dtype assert arange(fscalar(), dscalar()).dtype == dscalar().dtype assert arange(iscalar(), fscalar(), dscalar()).dtype == \ dscalar().dtype elif config.cast_policy in ('numpy', 'numpy+floatX'): for dtype in get_numeric_types(): # Test with a single argument. arange_dtype = arange(scalar(dtype=str(dtype))).dtype numpy_dtype = np.arange(np.array(1, dtype=dtype)).dtype if (dtype != 'float64' and numpy_dtype == 'float64' and config.cast_policy == 'numpy+floatX' and config.floatX == 'float32'): # We want a float32 arange. assert arange_dtype == 'float32' else: # Follow numpy. assert arange_dtype == numpy_dtype # Test with two arguments. for stop_dtype in get_numeric_types(): arange_dtype = arange( start=scalar(dtype=str(dtype)), stop=scalar(dtype=str(stop_dtype))).dtype numpy_dtype = np.arange( start=np.array(0, dtype=dtype), stop=np.array(1, dtype=stop_dtype)).dtype if (dtype != 'float64' and stop_dtype != 'float64' and numpy_dtype == 'float64' and config.cast_policy == 'numpy+floatX' and config.floatX == 'float32'): # We want a float32 arange. assert arange_dtype == 'float32' else: # Follow numpy. assert arange_dtype == numpy_dtype # Test with three arguments. for step_dtype in get_numeric_types(): arange_dtype = arange( start=scalar(dtype=str(dtype)), stop=scalar(dtype=str(stop_dtype)), step=scalar(dtype=str(step_dtype))).dtype numpy_dtype = np.arange( start=np.array(0, dtype=dtype), stop=np.array(1, dtype=stop_dtype), step=np.array(1, dtype=step_dtype)).dtype if (dtype != 'float64' and stop_dtype != 'float64' and step_dtype != 'float64' and numpy_dtype == 'float64' and config.cast_policy == 'numpy+floatX' and config.floatX == 'float32'): # We want a float32 arange. assert arange_dtype == 'float32' else: # Follow numpy. assert arange_dtype == numpy_dtype else: raise NotImplementedError(config.cast_policy) def test_dtype_cache(self): # Checks that the same Op is returned on repeated calls to arange # using the same dtype, but not for different dtypes. start, stop, step = iscalars('start', 'stop', 'step') out1 = arange(start, stop, step) out2 = arange(start, stop, step, dtype=out1.dtype) out3 = arange(start, stop, 2., dtype=out1.dtype) out4 = arange(start, stop, 2.) assert out1.owner.op is out2.owner.op assert out2.owner.op is out3.owner.op assert out3.owner.op is not out4.owner.op def test_infer_shape(self): start, stop, step = iscalars('start', 'stop', 'step') out = arange(start, stop, step) mode = theano.config.mode if mode == 'FAST_COMPILE': mode = 'FAST_RUN' mode = compile.mode.get_mode(mode).excluding('fusion') f = function([start, stop, step], out.shape, mode=mode) assert len(f.maker.fgraph.toposort()) == 9 if config.cast_policy == 'custom': assert out.dtype == 'int64' elif config.cast_policy in ('numpy', 'numpy+floatX'): numpy_dtype = np.arange(np.array(0, dtype=start.dtype), np.array(1, dtype=stop.dtype), np.array(1, dtype=step.dtype)).dtype assert out.dtype == numpy_dtype else: raise NotImplementedError(config.cast_policy) assert np.all(f(0, 5, 1) == len(np.arange(0, 5, 1))) assert np.all(f(2, 11, 4) == len(np.arange(2, 11, 4))) assert np.all(f(-5, 1, 1) == len(np.arange(-5, 1, 1))) assert np.all(f(10, 2, -2) == len(np.arange(10, 2, -2))) assert np.all(f(10, 2, 2) == len(np.arange(10, 2, 2))) assert np.all(f(0, 0, 1) == len(np.arange(0, 0, 1))) out = arange(start, stop, 1) f = function([start, stop], out.shape, mode=mode) assert len(f.maker.fgraph.toposort()) == 5 # 4 [Elemwise{sub,no_inplace}(stop, start), Elemwise{Cast{int64}}(Elemwise{sub,no_inplace}.0), Elemwise{Maximum{output_types_preference=transfer_type{0}}}[(0, 0)](Elemwise{Cast{int64}}.0, 0), MakeVector(Elemwise{Maximum{output_types_preference=transfer_type{0}}}[(0, 0)].0)] if config.cast_policy == 'custom': assert out.dtype == 'int64' elif config.cast_policy in ('numpy', 'numpy+floatX'): assert out.dtype == np.arange( np.int32(0), np.int32(1), np.int32(1)).dtype else: raise NotImplementedError(config.cast_policy) assert np.all(f(0, 5) == len(np.arange(0, 5))) assert np.all(f(2, 11) == len(np.arange(2, 11))) assert np.all(f(-5, 1) == len(np.arange(-5, 1))) assert np.all(f(10, 2) == len(np.arange(10, 2))) assert np.all(f(10, 2) == len(np.arange(10, 2))) assert np.all(f(0, 0) == len(np.arange(0, 0))) assert np.all(f(-64, 64) == len(np.arange(-64, 64))) assert arange(-64, 64).shape.eval() == [128] assert arange(-64, 64, 2).shape.eval() == [64] out = arange(0, stop, 1) f = function([stop], out.shape, mode=mode) assert len(f.maker.fgraph.toposort()) == 2 # [Elemwise{Cast{int64}}(stop), MakeVector(Elemwise{Cast{int64}}.0)] if config.cast_policy == 'custom': assert out.dtype == 'int64' elif config.cast_policy in ('numpy', 'numpy+floatX'): numpy_dtype = np.arange(0, np.array(1, dtype=stop.dtype), 1).dtype assert out.dtype == numpy_dtype else: raise NotImplementedError(config.cast_policy) assert np.all(f(5) == len(np.arange(0, 5))) assert np.all(f(11) == len(np.arange(0, 11))) assert np.all(f(1) == len(np.arange(0, 1))) assert np.all(f(2) == len(np.arange(0, 2))) assert np.all(f(2) == len(np.arange(0, 2))) assert np.all(f(0) == len(np.arange(0, 0))) class TestNdGrid(unittest.TestCase): def setUp(self): pass def test_mgrid_numpy_equiv(self): nmgrid = (np.mgrid[0:1:.1, 1:10:1., 10:100:10.], np.mgrid[0:2:1, 1:10:1, 10:100:10]) tmgrid = (mgrid[0:1:.1, 1:10:1., 10:100:10.], mgrid[0:2:1, 1:10:1, 10:100:10]) for n, t in zip(nmgrid, tmgrid): for ng, tg in zip(n, t): utt.assert_allclose(ng, tg.eval()) def test_ogrid_numpy_equiv(self): nogrid = (np.ogrid[0:1:.1, 1:10:1., 10:100:10.], np.ogrid[0:2:1, 1:10:1, 10:100:10]) togrid = (ogrid[0:1:.1, 1:10:1., 10:100:10.], ogrid[0:2:1, 1:10:1, 10:100:10]) for n, t in zip(nogrid, togrid): for ng, tg in zip(n, t): utt.assert_allclose(ng, tg.eval()) def test_mgrid_theano_variable_numpy_equiv(self): nfmgrid = np.mgrid[0:1:.1, 1:10:1., 10:100:10.] nimgrid = np.mgrid[0:2:1, 1:10:1, 10:100:10] i, j, k = dscalars('i', 'j', 'k') l, m, n = iscalars('l', 'm', 'n') tfmgrid = mgrid[i:1:.1, 1:j:1., 10:100:k] timgrid = mgrid[l:2:1, 1:m:1, 10:100:n] ff = theano.function([i, j, k], tfmgrid) fi = theano.function([l, m, n], timgrid) for n, t in zip((nfmgrid, nimgrid), (ff(0, 10, 10.), fi(0, 10, 10))): for ng, tg in zip(n, t): utt.assert_allclose(ng, tg) def test_ogrid_theano_variable_numpy_equiv(self): nfogrid = np.ogrid[0:1:.1, 1:10:1., 10:100:10.] niogrid = np.ogrid[0:2:1, 1:10:1, 10:100:10] i, j, k = dscalars('i', 'j', 'k') l, m, n = iscalars('l', 'm', 'n') tfogrid = ogrid[i:1:.1, 1:j:1., 10:100:k] tiogrid = ogrid[l:2:1, 1:m:1, 10:100:n] ff = theano.function([i, j, k], tfogrid) fi = theano.function([l, m, n], tiogrid) for n, t in zip((nfogrid, niogrid), (ff(0, 10, 10.), fi(0, 10, 10))): for ng, tg in zip(n, t): utt.assert_allclose(ng, tg) class TestInversePermutation(unittest.TestCase): def setUp(self): utt.seed_rng() def test_dim1(self): # Test the inversion of one permutation (int vector) p = ivector() inv = inverse_permutation(p) assert inv.dtype == p.dtype f_inverse = function([p], inv) # Generate a random permutation rng = np.random.RandomState(utt.fetch_seed()) p_val = rng.permutation(10).astype('int32') inv_val = f_inverse(p_val) # Check that the inverse of the inverse is the original permutation assert np.all(f_inverse(inv_val) == p_val) # Check that permutation(inverse) == inverse(permutation) = identity assert np.all(p_val[inv_val] == np.arange(10)) assert np.all(inv_val[p_val] == np.arange(10)) def test_dim2(self): # Test the inversion of several permutations at a time # Each row of p is a different permutation to inverse p = imatrix() inv = inverse_permutation(p) f_inverse = function([p], inv) rng = np.random.RandomState(utt.fetch_seed()) # Generate 10 random permutations p_val = np.asarray([rng.permutation(10) for i in range(7)], dtype='int32') inv_val = f_inverse(p_val) # Check that the inverse of the inverse is the original permutation list assert np.all(f_inverse(inv_val) == p_val) # Check that, for each permutation, # permutation(inverse) == inverse(permutation) = identity for p_row, i_row in zip(p_val, inv_val): assert np.all(p_row[i_row] == np.arange(10)) assert np.all(i_row[p_row] == np.arange(10)) class TestPermuteRowElements(unittest.TestCase): def setUp(self): utt.seed_rng() def test_1_1(self): # Test PermuteRowElements(vector, vector) input = dvector() p = ivector() out = permute_row_elements(input, p) permute = function([input, p], out) rng = np.random.RandomState(utt.fetch_seed()) input_val = rng.uniform(size=(5,)) p_val = rng.permutation(5).astype('int32') out_val = permute(input_val, p_val) # Should be equivalent to advanced indexing out_bis = input_val[p_val] assert np.all(out_val == out_bis) # Verify gradient def permute_fixed(s_input): # Auxiliary op defined to get rid of gradient wrt p_val return permute_row_elements(s_input, p_val) utt.verify_grad(permute_fixed, [input_val]) def test_2_1(self): # Test broadcasting in PermuteRowElements(matrix, vector) input = matrix() p = ivector() out = permute_row_elements(input, p) permute = function([input, p], out) rng = np.random.RandomState(utt.fetch_seed()) input_val = rng.uniform(size=(3, 5)).astype(config.floatX) p_val = rng.permutation(5).astype('int32') out_val = permute(input_val, p_val) # The same permutation should be applied to every row of the input matrix. out_bis = np.asarray([r[p_val] for r in input_val]) assert np.all(out_val == out_bis) # Verify gradient def permute_fixed(s_input): # Auxiliary op defined to get rid of gradient wrt p_val return permute_row_elements(s_input, p_val) utt.verify_grad(permute_fixed, [input_val]) def test_2_2(self): # Test PermuteRowElements(matrix, matrix) input = matrix() p = imatrix() out = permute_row_elements(input, p) permute = function([input, p], out) rng = np.random.RandomState(utt.fetch_seed()) input_val = rng.uniform(size=(3, 5)).astype(config.floatX) p_val = np.asarray([rng.permutation(5) for i in range(3)], dtype='int32') out_val = permute(input_val, p_val) # Each row of p contains a permutation to apply to the corresponding # row of input out_bis = np.asarray([i_row[p_row] for i_row, p_row in zip(input_val, p_val)]) assert np.all(out_val == out_bis) # Verify gradient def permute_fixed(s_input): # Auxiliary op defined to get rid of gradient wrt p_val return permute_row_elements(s_input, p_val) utt.verify_grad(permute_fixed, [input_val]) def test_1_2(self): # Test PermuteRowElements(vector, matrix) # Different permutations will be applied to the same input vector input = vector() p = imatrix() out = permute_row_elements(input, p) permute = function([input, p], out) rng = np.random.RandomState(utt.fetch_seed()) input_val = rng.uniform(size=(5,)).astype(config.floatX) p_val = np.asarray([rng.permutation(5) for i in range(3)], dtype='int32') out_val = permute(input_val, p_val) # Each row of p contains a permutation to apply to the input vector out_bis = np.asarray([input_val[p_row] for p_row in p_val]) assert np.all(out_val == out_bis) # Verify gradient def permute_fixed(s_input): # Auxiliary op defined to get rid of gradient wrt p_val return permute_row_elements(s_input, p_val) utt.verify_grad(permute_fixed, [input_val]) def test_3b_2(self): # Test permute_row_elements on a more complex broadcasting pattern: # input.type.broadcastable = (False, True, False), # p.type.broadcastable = (False, False). input = TensorType('floatX', (False, True, False))() p = imatrix() out = permute_row_elements(input, p) permute = function([input, p], out) rng = np.random.RandomState(utt.fetch_seed()) input_val = rng.uniform(size=(4, 1, 5)).astype(config.floatX) p_val = np.asarray([rng.permutation(5) for i in range(3)], dtype='int32') out_val = permute(input_val, p_val) # Each row of p contains a permutation to apply to each row # of the input tensor out_bis = np.asarray([[in_mat[0, p_row] for p_row in p_val] for in_mat in input_val]) assert np.all(out_val == out_bis) # Verify gradient def permute_fixed(s_input): # Auxiliary op defined to get rid of gradient wrt p_val return permute_row_elements(s_input, p_val) utt.verify_grad(permute_fixed, [input_val]) class test_tensordot(unittest.TestCase): def TensorDot(self, axes): # Since tensordot is no longer an op, mimic the old op signature # to allow easy use of verify_grad. return lambda a, b: tensordot(a, b, axes) def setUp(self): utt.seed_rng() def test0(self): # Test vector-vector avec = vector() bvec = vector() axes = ((0, ), (0, )) c = tensordot(avec, bvec, axes) f1 = inplace_func([avec, bvec], c) aval = rand(5) bval = rand(5) out0 = np.tensordot(aval, bval, axes) out1 = f1(aval, bval) utt.assert_allclose(out0, out1) utt.verify_grad(self.TensorDot(axes), [aval, bval]) # Test matrix-vector bmat = matrix() axes = ((0, ), (1, )) c = tensordot(avec, bmat, axes) f2 = inplace_func([avec, bmat], c) aval = rand(5) bval = rand(8, 5) utt.assert_allclose(np.tensordot(aval, bval, axes), f2(aval, bval)) utt.verify_grad(self.TensorDot(axes), [aval, bval]) # Test matrix-matrix amat = matrix() for axes, shps in [[((0,), (0,)), [(4, 7), (4, 9)]], [((0,), (1,)), [(4, 7), (9, 4)]], [((1,), (0,)), [(4, 7), (7, 9)]], [((1,), (1,)), [(4, 7), (9, 7)]], [((0, 1), (0, 1)), [(4, 7), (4, 7)]], # [((0, 1), (1, 0)), [(4, 7), (7, 4)]], # [((1, 0), (1, 0)), [(4, 7), (4, 7)]], # [((1, 0), (0, 1)), [(4, 7), (7, 4)]], ]: c = tensordot(amat, bmat, axes) f3 = inplace_func([amat, bmat], c) aval = rand(*shps[0]) bval = rand(*shps[1]) utt.assert_allclose(np.tensordot(aval, bval, axes), f3(aval, bval)) utt.verify_grad(self.TensorDot(axes), [aval, bval]) # Test ndarray-matrix, sum over one dim of matrix for axes, shps in [[((2,), (1,)), [(1, 2, 3, 4), (2, 3)]], [((0,), (1,)), [(1, 2, 3, 4), (3, 1)]], [((0,), (0,)), [(1, 2, 3, 4), (1, 3)]], [((3,), (0,)), [(1, 2, 3, 4), (4, 1)]], # [((3, 1), (0, 1)), [(1, 2, 3, 4), (4, 2)]], # [((0, 1), (1, 0)), [(1, 2, 3, 4), (2, 1)]], # [((3, 1), (1, 0)), [(1, 2, 3, 4), (2, 4)]], ]: atens = tensor4() c = tensordot(atens, bmat, axes) f4 = inplace_func([atens, bmat], c) aval = rand(*shps[0]) bval = rand(*shps[1]) utt.assert_allclose(np.tensordot(aval, bval, axes), f4(aval, bval)) utt.verify_grad(self.TensorDot(axes), [aval, bval]) # Test ndarray-ndarray atens = tensor4() btens = tensor3() axes = ((1, 3), (0, 2)) c = tensordot(atens, btens, axes) f5 = inplace_func([atens, btens], c) aval = rand(4, 3, 5, 2) bval = rand(3, 4, 2) utt.assert_allclose(np.tensordot(aval, bval, axes), f5(aval, bval)) utt.verify_grad(self.TensorDot(axes), [aval, bval]) axes = (axes[1], axes[0]) c = tensordot(btens, atens, axes) f6 = inplace_func([btens, atens], c) utt.assert_allclose(np.tensordot(bval, aval, axes), f6(bval, aval)) utt.verify_grad(self.TensorDot(axes), [bval, aval]) def test_raise_error(self): amat = matrix() bmat = matrix() bvec = vector() # Test invalid length for axes self.assertRaises(ValueError, tensordot, amat, bmat, (0, 1, 2)) # Test axes of uneven length self.assertRaises(ValueError, tensordot, amat, bmat, ((0, 1), (0))) # Test invalid len(axes) given inputs are matrices self.assertRaises(ValueError, tensordot, amat, bmat, ((0, 1, 2), (0, 1, 2))) # Test invalid axes[1] given that y is a vector self.assertRaises(ValueError, tensordot, amat, bvec, (0, 1)) # Test invalid scalar axes given inputs are matrices self.assertRaises(ValueError, tensordot, amat, bvec, 2) def test_weird_valid_axes(self): # Test matrix-matrix amat = matrix() bmat = matrix() for axes in [0, (1, 0), [1, 0], (1, (0, )), ((1, ), 0), ([1], [0]), ([], [])]: c = tensordot(amat, bmat, axes) f3 = inplace_func([amat, bmat], c) aval = rand(4, 7) bval = rand(7, 9) self.assertTrue(np.allclose(np.tensordot(aval, bval, axes), f3(aval, bval))) utt.verify_grad(self.TensorDot(axes), [aval, bval]) def test_scalar_axes(self): # Test matrix-matrix amat = fmatrix() bmat = dmatrix() # We let at float64 to test mix of float32 and float64. axes = 1 aval = rand(4, 5).astype('float32') bval = rand(5, 3) c = tensordot(amat, bmat, axes) f3 = inplace_func([amat, bmat], c) self.assertTrue(np.allclose(np.tensordot(aval, bval, axes), f3(aval, bval))) utt.verify_grad(self.TensorDot(axes), [aval, bval]) # Test tensor-tensor amat = tensor3() bmat = tensor3() axes = 2 aval = rand(3, 4, 5) bval = rand(4, 5, 3) c = tensordot(amat, bmat, axes) f3 = inplace_func([amat, bmat], c) self.assertTrue(np.allclose(np.tensordot(aval, bval, axes), f3(aval, bval))) utt.verify_grad(self.TensorDot(axes), [aval, bval]) def test_scalar0(self): # Test tensor-tensor amat = matrix() bmat = matrix() axes = 0 aval = rand(4, 5) bval = rand(5, 4) c = tensordot(amat, bmat, axes) f3 = inplace_func([amat, bmat], c) self.assertTrue(np.allclose(np.tensordot(aval, bval, axes), f3(aval, bval))) utt.verify_grad(self.TensorDot(axes), [aval, bval]) def test_broadcastable1(self): x = TensorType(dtype=floatX, broadcastable=(True, False, False))('x') y = tensor3('y') z = tensordot(x, y) assert z.broadcastable == (True, False) f = inplace_func([x, y], z) xv = rand(1, 3, 4) yv = rand(3, 4, 5) zv = f(xv, yv) self.assertTrue(np.allclose(np.tensordot(xv, yv), zv)) def test_broadcastable2(self): x = TensorType(dtype=floatX, broadcastable=(True, False, False))('x') y = tensor3('y') axes = [[2, 1], [0, 1]] z = tensordot(x, y, axes=axes) assert z.broadcastable == (True, False) f = inplace_func([x, y], z) xv = rand(1, 3, 4) yv = rand(4, 3, 5) zv = f(xv, yv) self.assertTrue(np.allclose(np.tensordot(xv, yv, axes=axes), zv)) def test_smallest_stack(): sx, sy = dscalar(), dscalar() rval = inplace_func([sx, sy], stack([sx, sy]))(-4.0, -2.0) assert type(rval) == np.ndarray assert [-4, -2] == list(rval) def test_smallest(): x = dvector() y = dvector() z = dvector() f1 = inplace_func([x], smallest(x)) assert np.all([1, 2, 3] == f1([1, 2, 3])) f3 = inplace_func([x, y, z], smallest(x, y, z)) assert np.all([1, 2, 3] == f3([1, 3, 9], [7, 7, 7], [8, 2, 3])) sx, sy = dscalar(), dscalar() assert -4 == inplace_func([sx, sy], smallest(sx, sy))(-4.0, -2.0) def test_reshape_member_fn(): x = dmatrix() y = x.reshape((4, 5, 6)) assert y.owner.op == Reshape(3) def test_var(): a = Tensor(dtype='float64', broadcastable=[False, False, False])() f = function([a], var(a)) a_val = np.arange(60).reshape(3, 4, 5) assert np.allclose(np.var(a_val), f(a_val)) f = function([a], var(a, axis=0)) assert np.allclose(np.var(a_val, axis=0), f(a_val)) f = function([a], var(a, axis=1)) assert np.allclose(np.var(a_val, axis=1), f(a_val)) f = function([a], var(a, axis=2)) assert np.allclose(np.var(a_val, axis=2), f(a_val)) f = function([a], var(a, axis=0, ddof=0)) assert np.allclose(np.var(a_val, axis=0, ddof=0), f(a_val)) f = function([a], var(a, axis=1, ddof=1)) assert np.allclose(np.var(a_val, axis=1, ddof=1), f(a_val)) f = function([a], var(a, axis=2, ddof=1)) assert np.allclose(np.var(a_val, axis=2, ddof=1), f(a_val)) f = function([a], var(a, ddof=0, corrected=True)) mean_a = np.mean(a_val) centered_a = a_val - mean_a v = np.mean(centered_a ** 2) error = (np.mean(centered_a)) ** 2 v = v - error assert np.allclose(v, f(a_val)) f = function([a], var(a, axis=2, ddof=1, corrected=True)) mean_a = np.mean(a_val, axis=2, keepdims=True) centered_a = a_val - mean_a v = np.var(a_val, axis=2, ddof=1) shp_inp = np.shape(a_val) shp = shp_inp - np.array(1) error = (np.sum(centered_a, axis=2)) ** 2 error = np.true_divide(error, shp[1] * shp_inp[1]) v = v - error assert np.allclose(v, f(a_val)) # Test that we don't upcast float16 computation assert theano.tensor.vector(dtype='float16').var().dtype == 'float16' class T_sum(unittest.TestCase): def test_sum_overflow(self): # Ensure that overflow errors are a little bit harder to get a = Tensor(dtype='int8', broadcastable=[False])() f = function([a], sum(a)) assert f([1] * 300) == 300 def test_list(self): ll = [theano.shared(0.), theano.shared(2.)] tensor.sum(ll).eval() == 2 @dec.skipif( isinstance(get_default_mode(), theano.compile.debugmode.DebugMode), ("This test fails in DEBUG_MODE, but the generated code is OK. " "It is actually a problem of DEBUG_MODE, see #626.")) def test_default(): x, y = scalars('xy') z = default(x, y) f = function([x, y], z) assert f(1, 2) == 1 assert f(None, 2) == 2 assert f(1, None) == 1 @dec.skipif( isinstance(get_default_mode(), theano.compile.debugmode.DebugMode), ("This test fails in DEBUG_MODE, but the generated code is OK. " "It is actually a problem of DEBUG_MODE, see #626.")) def test_default_state(): x, y = scalars('xy') # print config.floatX # print x.type # print y.type z = default(x, 3.8) new_x = y + z f = function([y, compile.In(x, update=new_x, value=12.0)], new_x) assert f(3) == 15 f['x'] = None assert np.allclose(f(1), 4.8) assert np.allclose(f(np.asarray(2.2, dtype=config.floatX)), 7) def test_autocast(): backup_config = config.cast_policy # Call test functions for all possible values of `config.cast_policy`. for autocast_cfg in ( 'custom', # 'numpy', # Commented out until it is implemented properly. 'numpy+floatX', ): config.cast_policy = autocast_cfg try: eval('_test_autocast_' + autocast_cfg.replace('+', '_'))() finally: config.cast_policy = backup_config def _test_autocast_custom(): # Called from `test_autocast`. assert config.cast_policy == 'custom' orig_autocast = autocast_float.dtypes # Test that autocast_float_as sets the autocast dtype correctly with autocast_float_as('float32'): assert autocast_float.dtypes == ('float32',) assert autocast_float.dtypes == orig_autocast with autocast_float_as('float64'): assert autocast_float.dtypes == ('float64',) assert autocast_float.dtypes == orig_autocast # Test that we can set it back to something, and nest it with autocast_float_as('float32'): assert autocast_float.dtypes == ('float32',) with autocast_float_as('float64'): assert autocast_float.dtypes == ('float64',) assert autocast_float.dtypes == ('float32',) assert autocast_float.dtypes == orig_autocast # Test that the autocasting dtype is used correctly in expression-building with autocast_float_as('float32'): assert (dvector() + 1.1).dtype == 'float64' assert (fvector() + 1.1).dtype == 'float32' assert ((fvector() + theano._asarray(1.1, dtype='float64')).dtype == 'float64') assert ((fvector() + theano._asarray(1.1, dtype='float32')).dtype == 'float32') assert (dvector() + 1).dtype == 'float64' assert (fvector() + 1).dtype == 'float32' # Test that the autocasting dtype is used correctly in expression-building with autocast_float_as('float64'): assert (dvector() + 1.1).dtype == 'float64' assert (fvector() + 1.1).dtype == 'float64' assert (fvector() + 1.0).dtype == 'float64' assert ((fvector() + theano._asarray(1.1, dtype='float64')).dtype == 'float64') assert ((fvector() + theano._asarray(1.1, dtype='float32')).dtype == 'float32') assert (dvector() + 1).dtype == 'float64' assert (fvector() + 1).dtype == 'float32' # Test that the autocasting dtype is used correctly in expression-building with autocast_float_as('float32', 'float64'): assert (dvector() + 1.1).dtype == 'float64' assert (fvector() + 1.1).dtype == theano.config.floatX assert (fvector() + 1.0).dtype == 'float32' assert (dvector() + np.float32(1.1)).dtype == 'float64' assert (dvector() + np.float64(1.1)).dtype == 'float64' assert (dvector() + np.float(1.1)).dtype == 'float64' assert (fvector() + np.float32(1.1)).dtype == 'float32' assert (fvector() + np.float64(1.1)).dtype == 'float64' assert (fvector() + np.float(1.1)).dtype == theano.config.floatX assert (lvector() + np.int64(1)).dtype == 'int64' assert (lvector() + np.int32(1)).dtype == 'int64' assert (lvector() + np.int16(1)).dtype == 'int64' assert (lvector() + np.int8(1)).dtype == 'int64' assert (ivector() + np.int8(1)).dtype == 'int32' assert (wvector() + np.int8(1)).dtype == 'int16' assert (bvector() + np.int8(1)).dtype == 'int8' with autocast_float_as('float64'): assert (fvector() + 1.0).dtype == 'float64' def _test_autocast_numpy(): # Called from `test_autocast`. assert config.cast_policy == 'numpy' # Go through some typical scalar values. def ok(z): assert tensor.constant(z).dtype == np.asarray(z).dtype for x in ([2 ** i for i in xrange(63)] + [0, L(0), L(1), L(2 ** 63 - 1)] + [0., 1., 1.1, 1.5]): n_x = np.asarray(x) # Make sure the data type is the same as the one found by numpy. ok(x) ok(-x) ok(x - 1) ok(-x + 1) ok(n_x) def _test_autocast_numpy_floatX(): # Called from `test_autocast`. assert config.cast_policy == 'numpy+floatX' backup_floatX = config.floatX def ok(z, floatX): if (isinstance(z, float) and floatX == 'float32' and not hasattr(z, 'dtype')): # Special case where we use 'float32' instead of 'float64'. assert tensor.constant(z).dtype == 'float32' else: assert tensor.constant(z).dtype == np.asarray(z).dtype try: # Test with various values of `config.floatX`. for floatX in ('float32', 'float64'): config.floatX = floatX # Go through some typical scalar values. # We only consider 'int' and 'long' Python values that can fit # into int64, as that is the maximal integer type that Theano # supports, and that is the maximal type in Python indexing. for x in ([2 ** i - 1 for i in xrange(64)] + [0, L(0), L(1), L(2 ** 63 - 1)] + [0., 1., 1.1, 1.5]): ok(x, floatX) ok(-x, floatX) ok(x - 1, floatX) ok(-x + 1, floatX) ok(np.asarray(x), floatX) ok(np.float64(x), floatX) finally: config.floatX = backup_floatX class test_arithmetic_cast(unittest.TestCase): # Test output types of basic arithmeric operations (* / + - //). # # We only test the behavior for `config.cast_policy` set to either 'numpy' or # 'numpy+floatX': the 'custom' behavior is (at least partially) tested in # `_test_autocast_custom`. def test_arithmetic_cast(self): backup_config = config.cast_policy dtypes = get_numeric_types(with_complex=True) # Here: # scalar == scalar stored as a 0d array # array == 1d array # i_scalar == scalar type used internally by Theano def theano_scalar(dtype): return tensor.scalar(dtype=str(dtype)) def numpy_scalar(dtype): return np.array(1, dtype=dtype) def theano_array(dtype): return tensor.vector(dtype=str(dtype)) def numpy_array(dtype): return np.array([1], dtype=dtype) def theano_i_scalar(dtype): return theano.scalar.Scalar(str(dtype))() def numpy_i_scalar(dtype): return numpy_scalar(dtype) if config.int_division == 'int': # Avoid deprecation warning during tests. warnings.filterwarnings('ignore', message='Division of two integer', category=DeprecationWarning) try: for cfg in ('numpy+floatX', ): # Used to test 'numpy' as well. config.cast_policy = cfg for op in (operator.add, operator.sub, operator.mul, operator_div, operator.floordiv): for a_type in dtypes: for b_type in dtypes: # Note that we do not test division between # integers if it is forbidden. # Theano deals with integer division in its own # special way (depending on `config.int_division`). is_int_division = ( op is operator_div and a_type in tensor.discrete_dtypes and b_type in tensor.discrete_dtypes) # We will test all meaningful combinations of # scalar and array operations. for combo in ( ('scalar', 'scalar'), ('array', 'array'), ('scalar', 'array'), ('array', 'scalar'), ('i_scalar', 'i_scalar'), ): theano_args = list( map(eval, ['theano_%s' % c for c in combo])) numpy_args = list( map(eval, ['numpy_%s' % c for c in combo])) try: theano_dtype = op( theano_args[0](a_type), theano_args[1](b_type)).type.dtype # Should have crashed if it is an integer # division and `config.int_division` does # not allow it. assert not (is_int_division and config.int_division == 'raise') except theano.scalar.IntegerDivisionError: assert (is_int_division and config.int_division == 'raise') # This is the expected behavior. continue # For numpy we have a problem: # http://projects.scipy.org/numpy/ticket/1827 # As a result we only consider the highest data # type that numpy may return. numpy_dtypes = [ op(numpy_args[0](a_type), numpy_args[1](b_type)).dtype, op(numpy_args[1](b_type), numpy_args[0](a_type)).dtype] numpy_dtype = theano.scalar.upcast( *list(map(str, numpy_dtypes))) if numpy_dtype == theano_dtype: # Same data type found, all is good! continue if (cfg == 'numpy+floatX' and config.floatX == 'float32' and a_type != 'float64' and b_type != 'float64' and numpy_dtype == 'float64'): # We should keep float32. assert theano_dtype == 'float32' continue if 'array' in combo and 'scalar' in combo: # For mixed scalar / array operations, # Theano may differ from numpy as it does # not try to prevent the scalar from # upcasting the array. array_type, scalar_type = ( (a_type, b_type)[list(combo).index(arg)] for arg in ('array', 'scalar')) up_type = theano.scalar.upcast(array_type, scalar_type) if ( # The two data types are different. scalar_type != array_type and # The array type is not enough to hold # the scalar type as well. array_type != up_type and # Theano upcasted the result array. theano_dtype == up_type and # But Numpy kept its original type. array_type == numpy_dtype): # Then we accept this difference in # behavior. continue if (is_int_division and config.int_division == 'floatX'): assert theano_dtype == config.floatX continue if (cfg == 'numpy+floatX' and a_type == 'complex128' and (b_type == 'float32' or b_type == 'float16') and combo == ('scalar', 'array') and theano_dtype == 'complex128' and numpy_dtype == 'complex64'): # In numpy 1.6.x adding a complex128 with # a float32 may result in a complex64. As # of 1.9.2. this is still the case so it is # probably by design raise SkipTest("Known issue with" "numpy see #761") # In any other situation: something wrong is # going on! assert False finally: config.cast_policy = backup_config if config.int_division == 'int': # Restore default deprecation warning behavior. warnings.filterwarnings( 'default', message='Division of two integer', category=DeprecationWarning) class T_long_tensor(unittest.TestCase): def test_fit_int64(self): bitwidth = theano.configdefaults.python_int_bitwidth() for exponent in xrange(bitwidth): val = L(2 ** exponent - 1) scalar_ct = constant(val) assert scalar_ct.dtype in tensor.int_dtypes, (exponent, val, scalar_ct.dtype) assert scalar_ct.value == val vector_ct = constant([val, val]) # On Python 2, np.array() on a "long" returns int64, # but on Python 3, all integers are long, and np.asarray # will not force the upcasting, and return the native int width. if PY3 and bitwidth == 32: assert vector_ct.dtype == 'int32' else: assert vector_ct.dtype == 'int64' assert np.all(vector_ct.value == val) matrix_ct = constant([[val, val]]) # On Python 2, np.array() on a "long" returns int64, # but on Python 3, all integers are long, and np.asarray # will not force the upcasting, and return the native int width. if PY3 and bitwidth == 32: assert matrix_ct.dtype == 'int32' else: assert matrix_ct.dtype == 'int64' assert np.all(matrix_ct.value == val) def test_too_big(self): val = L(2 ** 64) # This fail for all NumPy version. self.assertRaises(Exception, constant, val) self.assertRaises(Exception, constant, [val, val]) self.assertRaises(Exception, constant, [[val, val]]) class test_broadcast(unittest.TestCase): def test_broadcast_bigdim(self): def f(): x = matrix() addbroadcast(x, 2) self.assertRaises(ValueError, f) def test_unbroadcast_addbroadcast(self): # test that the unbroadcast fct don't insert not needed broadcast # and fuse consecutive Rebroadcast op x = matrix() assert unbroadcast(x, 0) is x assert unbroadcast(x, 1) is x assert unbroadcast(x, 1, 0) is x assert unbroadcast(x, 0, 1) is x assert addbroadcast(x, 0) is not x assert addbroadcast(x, 1) is not x assert addbroadcast(x, 1, 0).owner.inputs[0] is x assert unbroadcast(addbroadcast(x, 0), 0) is x assert addbroadcast(unbroadcast(x, 0), 0) is not x x = row() assert unbroadcast(x, 0) is not x assert unbroadcast(x, 1) is x assert unbroadcast(x, 1, 0) is not x assert unbroadcast(x, 0, 1) is not x assert addbroadcast(x, 0) is x assert addbroadcast(x, 1).owner.inputs[0] is x assert addbroadcast(x, 1, 0).owner.inputs[0] is x assert addbroadcast(x, 0, 1).owner.inputs[0] is x assert unbroadcast(addbroadcast(x, 1), 1) is x assert addbroadcast(unbroadcast(x, 1), 1) is not x # The first broadcast is remove the broadcast, so the second # should not make one assert unbroadcast(unbroadcast(x, 0), 0).owner.inputs[0] is x # Test that consecutive Rebroadcast op are fused x = TensorType(dtype='float64', broadcastable=(True, True))() assert unbroadcast(unbroadcast(x, 1), 0).owner.inputs[0] is x assert addbroadcast(unbroadcast(x, 1), 0).owner.inputs[0] is x assert addbroadcast(unbroadcast(x, 0), 0) is x def test_patternbroadcast(self): # Test that patternbroadcast with an empty broadcasting pattern works x = scalar('x') m = tensor.matrix('m') s = patternbroadcast(m, x.broadcastable) assert s is m x2 = patternbroadcast(x, x.broadcastable) assert x2 is x def test_infer_shape(self): x = matrix() y = addbroadcast(x, 0) f = theano.function([x], y.shape) assert (f(np.zeros((1, 5), dtype=config.floatX)) == [1, 5]).all() topo = f.maker.fgraph.toposort() if theano.config.mode != 'FAST_COMPILE': assert len(topo) == 2 assert isinstance(topo[0].op, opt.Shape_i) assert isinstance(topo[1].op, opt.MakeVector) x = matrix() y = unbroadcast(x, 0) f = theano.function([x], y.shape) assert (f(np.zeros((2, 5), dtype=config.floatX)) == [2, 5]).all() topo = f.maker.fgraph.toposort() if theano.config.mode != 'FAST_COMPILE': assert len(topo) == 3 assert isinstance(topo[0].op, opt.Shape_i) assert isinstance(topo[1].op, opt.Shape_i) assert isinstance(topo[2].op, opt.MakeVector) x = row() y = unbroadcast(x, 0) f = theano.function([x], y.shape) assert (f(np.zeros((1, 5), dtype=config.floatX)) == [1, 5]).all() topo = f.maker.fgraph.toposort() if theano.config.mode != 'FAST_COMPILE': assert len(topo) == 2 assert isinstance(topo[0].op, opt.Shape_i) assert isinstance(topo[1].op, opt.MakeVector) def test_len(): for shape_ in [(5,), (3, 4), (7, 4, 6)]: x = tensor.tensor(dtype='floatX', broadcastable=(False,) * len(shape_)) assert_raises(TypeError, len, x) def test_mod(): # We add this test as not all language and C implementation give the same # sign to the result. This check that the c_code of `Mod` is implemented # as Python. That is what we want. x, y = fscalars('xy') fn = gof.DualLinker().accept( gof.FunctionGraph([x, y], [x % y])).make_function() for a, b in ((0, 1), (1, 1), (0, -1), (1, -1), (-1, -1), (1, 2), (-1, 2), (1, -2), (-1, -2), (5, 3), (-5, 3), (5, -3), (-5, -3) ): assert fn(a, b) == a % b, (a,) def test_divmod(): # Confirm that divmod is equivalent to the python version. x, y = fscalars('xy') d, r = divmod(x, y) fn = gof.DualLinker().accept( gof.FunctionGraph([x, y], [d, r])).make_function() for a, b in ((0, 1), (1, 1), (0, -1), (1, -1), (-1, -1), (1, 2), (-1, 2), (1, -2), (-1, -2), (5, 3), (-5, 3), (5, -3), (-5, -3) ): d_v, r_v = fn(a, b) d_vp, r_vp = divmod(a, b) assert d_v == d_vp and r_v == r_vp, (a,) def test_mod_compile(): # This test generate an Elemwise of Composite as: # Elemwise{ # Composite{ # Composite{ # Composite{ # Composite{mod,EQ}, # Switch}, # mul}, # add}} # # The c_code generated is not compiling as of 30 June 2010. I fix the # compilation in the same commit. x = tensor.vector() y = tensor.vector() out = tensor.switch(tensor.eq(3 % x.shape[0], 0), y, y[:-1]) theano.function([x, y], out) def test_unalign(): if config.floatX == 'float64': dtype = "b1,f8" else: dtype = "b1,f4" a = np.empty(10000, dtype=dtype)['f1'] b = np.empty(10000, dtype=dtype)['f1'] assert not a.flags.aligned assert not b.flags.aligned a[:] = rand(len(a)) b[:] = rand(len(b)) out_numpy = 2 * a + 3 * b av, bv = tensor.vectors('ab') f = theano.function([av, bv], 2 * av + 3 * bv) f.maker.fgraph.toposort() try: out_theano = f(a, b) assert not a.flags.aligned assert not b.flags.aligned assert np.allclose(out_numpy, out_theano) assert False except TypeError: pass a = np.empty((), dtype=dtype)['f1'] b = np.empty((), dtype=dtype)['f1'] assert not a.flags.aligned assert not b.flags.aligned out_numpy = 2 * a + 3 * b av, bv = tensor.scalars('ab') f = theano.function([av, bv], 2 * av + 3 * bv) f.maker.fgraph.toposort() try: out_theano = f(a, b) assert not a.flags.aligned assert not b.flags.aligned assert np.allclose(out_numpy, out_theano) assert False except TypeError: pass def test_dimshuffle_duplicate(): x = tensor.vector() success = False try: tensor.DimShuffle((False, ), (0, 0))(x) except ValueError as e: assert str(e).find("may not appear twice") != -1 success = True assert success class T_get_scalar_constant_value(unittest.TestCase): def test_get_scalar_constant_value(self): a = tensor.stack([1, 2, 3]) assert get_scalar_constant_value(a[0]) == 1 assert get_scalar_constant_value(a[1]) == 2 assert get_scalar_constant_value(a[2]) == 3 b = tensor.iscalar() a = tensor.stack([b, 2, 3]) self.assertRaises(tensor.basic.NotScalarConstantError, get_scalar_constant_value, a[0]) assert get_scalar_constant_value(a[1]) == 2 assert get_scalar_constant_value(a[2]) == 3 # For now get_scalar_constant_value goes through only MakeVector and Join of # scalars. v = tensor.ivector() a = tensor.stack([v, [2], [3]]) self.assertRaises(tensor.NotScalarConstantError, get_scalar_constant_value, a[0]) self.assertRaises(tensor.NotScalarConstantError, get_scalar_constant_value, a[1]) self.assertRaises(tensor.NotScalarConstantError, get_scalar_constant_value, a[2]) # Test the case SubTensor(Shape(v)) when the dimensions # is broadcastable. v = tensor.row() assert get_scalar_constant_value(v.shape[0]) == 1 def test_subtensor_of_constant(self): c = constant(rand(5)) for i in range(c.value.shape[0]): assert get_scalar_constant_value(c[i]) == c.value[i] c = constant(rand(5, 5)) for i in range(c.value.shape[0]): for j in range(c.value.shape[1]): assert get_scalar_constant_value(c[i, j]) == c.value[i, j] def test_numpy_array(self): # Regression test for crash when called on a numpy array. assert get_scalar_constant_value(np.array(3)) == 3 self.assertRaises( tensor.NotScalarConstantError, get_scalar_constant_value, np.array([0, 1])) self.assertRaises( tensor.EmptyConstantError, get_scalar_constant_value, np.array([])) def test_make_vector(self): mv = opt.make_vector(1, 2, 3) self.assertRaises( tensor.NotScalarConstantError, get_scalar_constant_value, mv) assert get_scalar_constant_value(mv[0]) == 1 assert get_scalar_constant_value(mv[1]) == 2 assert get_scalar_constant_value(mv[2]) == 3 assert get_scalar_constant_value(mv[np.int32(0)]) == 1 assert get_scalar_constant_value(mv[np.int64(1)]) == 2 assert get_scalar_constant_value(mv[np.uint(2)]) == 3 t = theano.scalar.Scalar('int64') self.assertRaises( tensor.NotScalarConstantError, get_scalar_constant_value, mv[t()]) def test_shape_i(self): c = theano.tensor.constant(np.random.rand(3, 4)) s = opt.Shape_i(0)(c) assert get_scalar_constant_value(s) == 3 s = opt.Shape_i(1)(c) assert get_scalar_constant_value(s) == 4 d = theano.shared(np.random.randn(1, 1), broadcastable=(True, True)) f = theano.tensor.basic.ScalarFromTensor()(opt.Shape_i(0)(d)) assert get_scalar_constant_value(f) == 1 def test_elemwise(self): # We test only for a few elemwise, the list of all supported # elemwise are in the fct. c = theano.tensor.constant(np.random.rand()) s = c + 1 assert np.allclose(get_scalar_constant_value(s), c.data + 1) s = c - 1 assert np.allclose(get_scalar_constant_value(s), c.data - 1) s = c * 1.2 assert np.allclose(get_scalar_constant_value(s), c.data * 1.2) s = c < 0.5 assert np.allclose(get_scalar_constant_value(s), int(c.data < 0.5)) s = tensor.second(c, .4) assert np.allclose(get_scalar_constant_value(s), .4) def test_assert(self): # Make sure we still get the constant value if it is wrapped in # an Assert. c = theano.tensor.constant(2) x = theano.tensor.scalar() # condition is always True a = opt.Assert()(c, c > 1) assert get_scalar_constant_value(a) == 2 with change_flags(compute_test_value='off'): # condition is always False a = opt.Assert()(c, c > 2) self.assertRaises( tensor.NotScalarConstantError, get_scalar_constant_value, a) # condition is not constant a = opt.Assert()(c, c > x) self.assertRaises( tensor.NotScalarConstantError, get_scalar_constant_value, a) def test_second(self): # Second should apply when the value is constant but not the shape c = theano.tensor.constant(np.random.rand()) shp = theano.tensor.vector() s = theano.tensor.second(shp, c) assert get_scalar_constant_value(s) == c.data def test_copy(self): # Make sure we do not return the internal storage of a constant, # so we cannot change the value of a constant by mistake. c = theano.tensor.constant(3) d = extract_constant(c) d += 1 e = extract_constant(c) self.assertTrue(e == 3, (c, d, e)) class T_as_tensor_variable(unittest.TestCase): # We test that ticket #649 stay fixed. # We should not allow as_tensor_variable to accept True or False # But it should upcast an ndarray of bool to uint8 def test_bool(self): self.assertRaises(TypeError, as_tensor_variable, True) self.assertRaises(TypeError, as_tensor_variable, False) def test_ndarray_bool(self): ten = as_tensor_variable(np.array([True, False, False, True, True])) assert ten.type.dtype == 'bool' def test_memmap(self): inp = np.random.rand(4, 3) f, fname = mkstemp() new_inp = np.memmap(fname, dtype=inp.dtype, mode='w+', shape=inp.shape) new_inp[...] = inp as_tensor_variable(new_inp) def test_empty_dtype(self): old = theano.config.floatX for dtype in ['float16', 'float32', 'float64']: try: theano.config.floatX = dtype assert theano.tensor.as_tensor_variable(()).dtype == dtype assert theano.tensor.as_tensor_variable([]).dtype == dtype finally: theano.config.floatX = old class test_complex_mod(unittest.TestCase): # Make sure % fails on complex numbers. def test_fail(self): x = vector(dtype='complex64') try: x % 5 assert False except theano.scalar.ComplexError: pass class test_size(unittest.TestCase): # Ensure the `size` attribute of tensors behaves as in numpy. def test_matrix(self): x = tensor.matrix() y = np.zeros((5, 7), dtype=config.floatX) assert y.size == function([x], x.size)(y) def test_vector(self): x = tensor.vector() y = np.zeros(7, dtype=config.floatX) assert y.size == function([x], x.size)(y) def test_scalar(self): x = tensor.scalar() y = np.array(7, dtype=config.floatX) assert y.size == function([x], x.size)(y) def test_shared(self): # NB: we also test higher order tensors at the same time. y = np.zeros((1, 2, 3, 4), dtype=config.floatX) x = theano.shared(y) assert y.size == function([], x.size)() class test_diag(unittest.TestCase): # Test that tensor.diag has the same behavior as np.diag. # np.diag has two behaviors: # # (1) when given a vector, it returns a matrix with that vector as the # diagonal. # (2) when given a matrix, returns a vector which is the diagonal of the # matrix. # # (1) and (2) are tested by test_alloc_diag and test_extract_diag # respectively. # # test_diag test makes sure that linalg.diag instantiates # the right op based on the dimension of the input. def __init__(self, name, mode=None, shared=tensor._shared, floatX=None, type=tensor.TensorType): self.mode = mode self.shared = shared if floatX is None: floatX = config.floatX self.floatX = floatX self.type = type super(test_diag, self).__init__(name) def test_diag(self): rng = np.random.RandomState(utt.fetch_seed()) # test vector input x = theano.tensor.vector() g = diag(x) assert isinstance(g.owner.op, AllocDiag) f = theano.function([x], g) for shp in [5, 0, 1]: m = rng.rand(shp).astype(self.floatX) v = np.diag(m) r = f(m) # The right matrix is created assert (r == v).all() # Test matrix input xx = self.shared(rng.rand(3, 5)) g = diag(xx) assert isinstance(g.owner.op, ExtractDiag) f = theano.function([], g) for shp in [(5, 3), (3, 5), (5, 1), (1, 5), (5, 0), (0, 5), (1, 0), (0, 1)]: m = rng.rand(*shp).astype(self.floatX) xx.set_value(m) v = np.diag(m) r = f() # The right matrix is created assert (r == v).all() # Test scalar input xx = theano.tensor.scalar() np.testing.assert_raises(ValueError, diag, xx) def test_infer_shape(self): rng = np.random.RandomState(utt.fetch_seed()) x = theano.tensor.vector() g = diag(x) f = theano.function([x], g.shape) topo = f.maker.fgraph.toposort() if config.mode != 'FAST_COMPILE': assert np.sum( [isinstance(node.op, AllocDiag) for node in topo]) == 0 for shp in [5, 0, 1]: m = rng.rand(shp).astype(self.floatX) assert (f(m) == np.diag(m).shape).all() x = theano.tensor.matrix() g = diag(x) f = theano.function([x], g.shape) topo = f.maker.fgraph.toposort() if config.mode != 'FAST_COMPILE': assert np.sum( [isinstance(node.op, ExtractDiag) for node in topo]) == 0 for shp in [(5, 3), (3, 5), (5, 1), (1, 5), (5, 0), (0, 5), (1, 0), (0, 1)]: m = rng.rand(*shp).astype(self.floatX) assert (f(m) == np.diag(m).shape).all() def test_diag_grad(self): rng = np.random.RandomState(utt.fetch_seed()) x = rng.rand(5) tensor.verify_grad(diag, [x], rng=rng) x = rng.rand(5, 3) tensor.verify_grad(diag, [x], rng=rng) class TestAllocDiag(unittest.TestCase): def __init__(self, name, alloc_diag=AllocDiag, mode=None): self.alloc_diag = alloc_diag if mode is None: mode = theano.compile.mode.get_default_mode() self.mode = mode return super(TestAllocDiag, self).__init__(name) def _generator(self): dims = 4 shape = (5,) * dims xv = np.random.randn(*shape).astype(config.floatX) for d in xrange(1, dims + 1): # Create a TensorType of the same dimensions as # as the data we want to test. x = TensorType(dtype=config.floatX, broadcastable=(False,) * d)('x') # Make a slice of the test data that has the # dimensions we need by doing xv[0,...,0] # For example, for an array of shape (5,), we # need to do xv[0, 0, 0, 0]. test_val = xv[((0,) * (dims - d))] yield x, test_val def test_alloc_diag_values(self): for x, test_val in self._generator(): for offset, axis1, axis2 in [(0, 0, 1), (0, 1, 2), (1, 0, 1), (0, 1, 3), (0, 2, 3), (1, 2, 3), (-1, 0, 1), (-2, 0, 1), (-1, 1, 2)]: # Test AllocDiag values if np.maximum(axis1, axis2) > len(test_val.shape): continue adiag_op = self.alloc_diag(offset=offset, axis1=axis1, axis2=axis2) f = theano.function([x], adiag_op(x)) # AllocDiag and extract the diagonal again # to check diag_arr = f(test_val) rediag = np.diagonal( diag_arr, offset=offset, axis1=axis1, axis2=axis2 ) assert np.all(rediag == test_val) # Test infer_shape f_shape = theano.function([x], adiag_op(x).shape, mode='FAST_RUN') theano.printing.debugprint(f_shape.maker.fgraph.outputs[0]) output_shape = f_shape(test_val) assert not any(isinstance(node.op, self.alloc_diag) for node in f_shape.maker.fgraph.toposort()) rediag_shape = np.diagonal( np.ones(output_shape), offset=offset, axis1=axis1, axis2=axis2 ).shape assert np.all(rediag_shape == test_val.shape) diag_x = adiag_op(x) sum_diag_x = tensor.sum(diag_x) grad_x = tensor.grad(sum_diag_x, x) grad_diag_x = tensor.grad(sum_diag_x, diag_x) f_grad_x = theano.function([x], grad_x, mode=self.mode) f_grad_diag_x = theano.function([x], grad_diag_x, mode=self.mode) grad_input = f_grad_x(test_val) grad_diag_input = f_grad_diag_x(test_val) true_grad_input = np.diagonal( grad_diag_input, offset=offset, axis1=axis1, axis2=axis2 ) assert np.all(true_grad_input == grad_input) class test_numpy_assumptions(unittest.TestCase): # Verify that some assumptions Theano makes on Numpy's behavior still hold. def test_ndarray_copy(self): # A copy or deepcopy of the ndarray type should not create a new object. # # This is because Theano makes some comparisons of the form: # if type(x) is np.ndarray assert copy(np.ndarray) is np.ndarray assert deepcopy(np.ndarray) is np.ndarray def test_dtype_equality(self): # Ensure dtype string comparisons are consistent. # # Theano often uses string representations of dtypes (e.g. 'float32'). We # need to make sure that comparing the string representations is the same # as comparing the dtype objects themselves. dtypes = get_numeric_types(with_complex=True) # Perform all pairwise comparisons of dtypes, making sure comparing # their string representation yields the same result. for dtype1_idx, dtype1 in enumerate(dtypes): for dtype2 in dtypes[dtype1_idx + 1:]: assert (dtype1 == dtype2) == (str(dtype1) == str(dtype2)) def test_transpose(): x1 = tensor.dvector('x1') x2 = tensor.dmatrix('x2') x3 = tensor.dtensor3('x3') x1v = np.arange(24) x2v = np.arange(24).reshape(2, 12) x3v = np.arange(24).reshape(2, 3, 4) f = theano.function([x1, x2, x3], [ tensor.transpose(x1), tensor.transpose(x2), tensor.transpose(x3), x1.transpose(), x2.transpose(), x3.transpose(), x2.transpose(0, 1), x3.transpose((0, 2, 1)), tensor.transpose(x2, [0, 1]), tensor.transpose(x3, [0, 2, 1]), ]) t1, t2, t3, t1b, t2b, t3b, t2c, t3c, t2d, t3d = f(x1v, x2v, x3v) assert t1.shape == np.transpose(x1v).shape assert t2.shape == np.transpose(x2v).shape assert t3.shape == np.transpose(x3v).shape assert np.all(t1 == np.transpose(x1v)) assert np.all(t2 == np.transpose(x2v)) assert np.all(t3 == np.transpose(x3v)) assert np.all(t1b == x1v.transpose()) assert np.all(t2b == x2v.transpose()) assert np.all(t3b == x3v.transpose()) assert t2c.shape == (2, 12) assert t3c.shape == (2, 4, 3) assert np.all(t2c == x2v.transpose([0, 1])) assert np.all(t3c == x3v.transpose([0, 2, 1])) assert t2d.shape == (2, 12) assert t3d.shape == (2, 4, 3) assert np.all(t2d == np.transpose(x2v, [0, 1])) assert np.all(t3d == np.transpose(x3v, [0, 2, 1])) # Check that we create a name. assert tensor.transpose(x1).name == 'x1.T' assert tensor.transpose(x2).name == 'x2.T' assert tensor.transpose(x3).name == 'x3.T' assert tensor.transpose(tensor.dmatrix()).name is None def test_stacklists(): a, b, c, d = map(scalar, 'abcd') X = stacklists([[a, b], [c, d]]) f = function([a, b, c, d], X) result = f(1, 2, 3, 4) assert result.shape == (2, 2) assert np.allclose(f(1, 2, 3, 4), np.asarray([[1, 2], [3, 4]])) X = stacklists([a, b, c, d]) f = function([a, b, c, d], X) result = f(1, 2, 3, 4) assert result.shape == (4,) assert np.allclose(f(1, 2, 3, 4), np.asarray([[1, 2, 3, 4]])) X = stacklists([[[a], [b]], [[c], [d]]]) f = function([a, b, c, d], X) result = f(1, 2, 3, 4) assert result.shape == (2, 2, 1) a, b, c, d = [matrix(x) for x in 'abcd'] X = stacklists([[a, b], [c, d]]) f = function([a, b, c, d], X) x = np.ones((4, 4), 'float32') assert f(x, x, x, x).shape == (2, 2, 4, 4) class TestSpecifyShape(unittest.TestCase): mode = None input_type = TensorType def shortDescription(self): return None def test_bad_shape(self): # Test that at run time we raise an exception when the shape # is not the one specified specify_shape = SpecifyShape() x = vector() xval = np.random.rand(2).astype(floatX) f = theano.function([x], specify_shape(x, [2]), mode=self.mode) f(xval) xval = np.random.rand(3).astype(floatX) self.assertRaises(AssertionError, f, xval) theano.printing.debugprint(f) assert isinstance([n for n in f.maker.fgraph.toposort() if isinstance(n.op, SpecifyShape)][0].inputs[0].type, self.input_type) x = matrix() xval = np.random.rand(2, 3).astype(floatX) f = theano.function([x], specify_shape(x, [2, 3]), mode=self.mode) assert isinstance([n for n in f.maker.fgraph.toposort() if isinstance(n.op, SpecifyShape)][0].inputs[0].type, self.input_type) f(xval) for shape_ in [(1, 3), (2, 2), (5, 5)]: xval = np.random.rand(*shape_).astype(floatX) self.assertRaises(AssertionError, f, xval) def test_bad_number_of_shape(self): # Test that the number of dimensions provided is good specify_shape = SpecifyShape() x = vector() shape_vec = ivector() xval = np.random.rand(2).astype(floatX) self.assertRaises(AssertionError, specify_shape, x, []) self.assertRaises(AssertionError, specify_shape, x, [2, 2]) f = theano.function([x, shape_vec], specify_shape(x, shape_vec), mode=self.mode) assert isinstance([n for n in f.maker.fgraph.toposort() if isinstance(n.op, SpecifyShape)][0].inputs[0].type, self.input_type) self.assertRaises(AssertionError, f, xval, []) self.assertRaises(AssertionError, f, xval, [2, 2]) x = matrix() xval = np.random.rand(2, 3).astype(floatX) for shape_ in [(), (1,), (2, 3, 4)]: self.assertRaises(AssertionError, specify_shape, x, shape_) f = theano.function([x, shape_vec], specify_shape(x, shape_vec), mode=self.mode) assert isinstance([n for n in f.maker.fgraph.toposort() if isinstance(n.op, SpecifyShape)][0].inputs[0].type, self.input_type) self.assertRaises(AssertionError, f, xval, shape_) class TestInferShape(utt.InferShapeTester): def test_infer_shape(self): # Flatten atens3 = tensor3() atens3_val = rand(4, 5, 3) for outdim in (3, 2, 1): self._compile_and_check([atens3], [flatten(atens3, outdim)], [atens3_val], Reshape, excluding=['local_useless_reshape']) amat = matrix() amat_val = rand(4, 5) for outdim in (2, 1): self._compile_and_check([amat], [flatten(amat, outdim)], [amat_val], Reshape, excluding=['local_useless_reshape']) avec = vector() avec_val = rand(4) outdim = 1 self._compile_and_check([avec], [flatten(avec, outdim)], [avec_val], Reshape, excluding=['local_useless_reshape']) # Eye aiscal = iscalar() biscal = iscalar() ciscal = iscalar() self._compile_and_check([aiscal, biscal, ciscal], [Eye()(aiscal, biscal, ciscal)], [4, 4, 0], Eye) self._compile_and_check([aiscal, biscal, ciscal], [Eye()(aiscal, biscal, ciscal)], [4, 5, 0], Eye) self._compile_and_check([aiscal, biscal, ciscal], [Eye()(aiscal, biscal, ciscal)], [3, 5, 0], Eye) # Tri aiscal = iscalar() biscal = iscalar() ciscal = iscalar() self._compile_and_check([aiscal, biscal, ciscal], [Tri()(aiscal, biscal, ciscal)], [4, 4, 0], Tri) self._compile_and_check([aiscal, biscal, ciscal], [Tri()(aiscal, biscal, ciscal)], [4, 5, 0], Tri) self._compile_and_check([aiscal, biscal, ciscal], [Tri()(aiscal, biscal, ciscal)], [3, 5, 0], Tri) # ExtractDiag atens3 = tensor3() atens3_val = rand(4, 5, 3) atens3_diag = ExtractDiag()(atens3) self._compile_and_check([atens3], [atens3_diag], [atens3_val], ExtractDiag) atens3_diag = ExtractDiag(1)(atens3) self._compile_and_check([atens3], [atens3_diag], [atens3_val], ExtractDiag) atens3_diag = ExtractDiag(-1)(atens3) self._compile_and_check([atens3], [atens3_diag], [atens3_val], ExtractDiag) atens3_diag = ExtractDiag(1, 0, 2)(atens3) self._compile_and_check([atens3], [atens3_diag], [atens3_val], ExtractDiag) atens3_diag = ExtractDiag(1, 1, 2)(atens3) self._compile_and_check([atens3], [atens3_diag], [atens3_val], ExtractDiag) atens3_diag = ExtractDiag(1, 2, 0)(atens3) self._compile_and_check([atens3], [atens3_diag], [atens3_val], ExtractDiag) # AllocDiag advec = dvector() advec_val = rand(4) self._compile_and_check([advec], [AllocDiag()(advec)], [advec_val], AllocDiag) # Shape # 'opt.Makevector' precludes optimizer from disentangling # elements of shape adtens = tensor3() adtens_val = rand(4, 5, 3) self._compile_and_check([adtens], [Shape()(adtens)], [adtens_val], (opt.MakeVector, Shape)) # Dot # vec/vec advec = dvector() bdvec = dvector() advec_val = rand(4) bdvec_val = rand(4) self._compile_and_check([advec, bdvec], [Dot()(advec, bdvec)], [advec_val, bdvec_val], (Dot, tensor.blas.Dot22, tensor.blas.Gemv, tensor.blas_c.CGemv)) # mat/mat admat = dmatrix() bdmat = dmatrix() admat_val = rand(4, 5) bdmat_val = rand(5, 3) self._compile_and_check([admat, bdmat], [Dot()(admat, bdmat)], [admat_val, bdmat_val], (Dot, tensor.blas.Dot22)) # vec/mat bdmat_val = rand(4, 5) self._compile_and_check([advec, bdmat], [Dot()(advec, bdmat)], [advec_val, bdmat_val], (Dot, tensor.blas.Dot22, tensor.blas.Gemv, tensor.blas_c.CGemv)) # mat/vec admat_val = rand(5, 4) self._compile_and_check([admat, bdvec], [Dot()(admat, bdvec)], [admat_val, bdvec_val], (Dot, tensor.blas.Dot22, tensor.blas.Gemv, tensor.blas_c.CGemv)) # Split aivec = ivector() adtens_val = rand(4, 10, 3) aivec_val = [2, 5, 3] for aiscal_val in [1, -2]: self._compile_and_check( [adtens, aiscal, aivec], [Split(3)(adtens, aiscal, aivec)[0]], [adtens_val, aiscal_val, aivec_val], (Split)) # Join cdmat = dmatrix() admat_val = rand(1, 3) bdmat_val = rand(2, 3) cdmat_val = rand(4, 3) for aiscal_val in [0, -2]: self._compile_and_check( [aiscal, admat, bdmat, cdmat], [Join()(aiscal, admat, bdmat, cdmat)], [aiscal_val, admat_val, bdmat_val, cdmat_val], Join) admat_val = rand(4, 1) bdmat_val = rand(4, 3) cdmat_val = rand(4, 2) for aiscal_val in [-1, 1]: self._compile_and_check( [aiscal, admat, bdmat, cdmat], [Join()(aiscal, admat, bdmat, cdmat)], [aiscal_val, admat_val, bdmat_val, cdmat_val], Join) # PermuteRowElements abool = True rng = np.random.RandomState(utt.fetch_seed()) advec_val = rand(5) aivec_val = rng.permutation(5).astype('int32') self._compile_and_check([advec, aivec], [PermuteRowElements()(advec, aivec, abool)], [advec_val, aivec_val], PermuteRowElements) admat_val = rand(3, 5) self._compile_and_check([admat, aivec], [PermuteRowElements()(admat, aivec, abool)], [admat_val, aivec_val], PermuteRowElements) adtens3 = dtensor3() adtens3_val = rand(3, 2, 5) self._compile_and_check([adtens3, aivec], [PermuteRowElements()(adtens3, aivec, abool)], [adtens3_val, aivec_val], PermuteRowElements) aimat = imatrix() perma = rng.permutation(5).astype('int32') permb = rng.permutation(5).astype('int32') permc = rng.permutation(5).astype('int32') aimat_val = np.vstack((perma, permb, permc)) admat_val = rand(3, 5) self._compile_and_check([admat, aimat], [PermuteRowElements()(admat, aimat, abool)], [admat_val, aimat_val], PermuteRowElements) aitens3 = itensor3() perma = rng.permutation(5).astype('int32') permb = rng.permutation(5).astype('int32') permc = rng.permutation(5).astype('int32') bimat_val = np.vstack((perma, permb, permc)) aitens3_val = np.empty((2, 3, 5), 'int32') aitens3_val[0, ::, ::] = aimat_val aitens3_val[1, ::, ::] = bimat_val self._compile_and_check([admat, aitens3], [PermuteRowElements()(admat, aitens3, abool)], [admat_val, aitens3_val], PermuteRowElements) # ScalarFromTensor aiscal = iscalar() self._compile_and_check([aiscal], [TensorFromScalar()(ScalarFromTensor()(aiscal))], [45], ScalarFromTensor, excluding=["local_tensor_scalar_tensor"]) # TensorFromScalar aiscal = scal.float64() self._compile_and_check([aiscal], [TensorFromScalar()(aiscal)], [4.], TensorFromScalar) # Rebroadcast adtens4 = dtensor4() adict = [(0, False), (1, True), (2, False), (3, True)] adtens4_val = rand(2, 1, 3, 1) self._compile_and_check([adtens4], [Rebroadcast(*adict)(adtens4)], [adtens4_val], Rebroadcast, warn=False) adtens4_bro = TensorType('float64', (True, True, True, False))() bdict = [(0, True), (1, False), (2, False), (3, False)] adtens4_bro_val = rand(1, 1, 1, 3) self._compile_and_check([adtens4_bro], [Rebroadcast(*bdict)(adtens4_bro)], [adtens4_bro_val], Rebroadcast) # Alloc randint = np.random.randint adscal = dscalar() aiscal = lscalar() biscal = lscalar() ciscal = lscalar() discal = lscalar() adscal_val = rand() aiscal_val = randint(3, 6, size=()) biscal_val = randint(3, 6, size=()) ciscal_val = randint(3, 6, size=()) discal_val = randint(3, 6, size=()) self._compile_and_check( [adscal, aiscal, biscal, ciscal, discal], [Alloc()(adscal, aiscal, biscal, ciscal, discal)], [adscal_val, aiscal_val, biscal_val, ciscal_val, discal_val], Alloc) # MaxAndArgmax, adtens3_val = rand(4, 5, 3) self._compile_and_check([adtens3], max_and_argmax(adtens3, None), [adtens3_val], MaxAndArgmax) self._compile_and_check([adtens3], max_and_argmax(adtens3, 0), [adtens3_val], MaxAndArgmax) self._compile_and_check([adtens3], max_and_argmax(adtens3, 1), [adtens3_val], MaxAndArgmax) self._compile_and_check([adtens3], max_and_argmax(adtens3, 2), [adtens3_val], MaxAndArgmax) self._compile_and_check([adtens3], max_and_argmax(adtens3, [0, 1, 2]), [adtens3_val], MaxAndArgmax) # ARange self._compile_and_check([aiscal, biscal, ciscal], [ARange('int64')(aiscal, biscal, ciscal)], [0, 5, 1], ARange) self._compile_and_check([aiscal, biscal, ciscal], [ARange('int64')(aiscal, biscal, ciscal)], [2, 11, 4], ARange) self._compile_and_check([aiscal, biscal, ciscal], [ARange('int64')(aiscal, biscal, ciscal)], [-5, 1, 1], ARange) self._compile_and_check([aiscal, biscal, ciscal], [ARange('int64')(aiscal, biscal, ciscal)], [10, 2, -2], ARange) self._compile_and_check([aiscal, biscal, ciscal], [ARange('int64')(aiscal, biscal, ciscal)], [10, 2, 2], ARange) self._compile_and_check([aiscal, biscal, ciscal], [ARange('int64')(aiscal, biscal, ciscal)], [0, 0, 1], ARange) # SpecifyShape aivec_val = [3, 4, 2, 5] adtens4_val = rand(*aivec_val) self._compile_and_check([adtens4, aivec], [SpecifyShape()(adtens4, aivec)], [adtens4_val, aivec_val], SpecifyShape) # Mean adtens3_val = rand(3, 4, 5) aiscal_val = 2 self._compile_and_check([adtens3], [Mean(None)(adtens3)], [adtens3_val], Mean) self._compile_and_check([adtens3], [Mean(aiscal_val)(adtens3)], [adtens3_val], Mean) # Reshape # TODO: generalize infer_shape to account for tensor variable # (non-constant) input shape admat = dmatrix() aivec = ivector() ndim = 1 admat_val = rand(3, 4) self._compile_and_check([admat], [Reshape(ndim)(admat, [12])], [admat_val], Reshape) self._compile_and_check([admat], [Reshape(ndim)(admat, [-1])], [admat_val], Reshape) ndim = 2 self._compile_and_check([admat], [Reshape(ndim)(admat, [4, 3])], [admat_val], Reshape) self._compile_and_check([admat], [Reshape(ndim)(admat, [4, -1])], [admat_val], Reshape) self._compile_and_check([admat], [Reshape(ndim)(admat, [3, -1])], [admat_val], Reshape) self._compile_and_check([admat], [Reshape(ndim)(admat, [-1, 3])], [admat_val], Reshape) self._compile_and_check([admat], [Reshape(ndim)(admat, [-1, 4])], [admat_val], Reshape) # enable when infer_shape is generalized: # self._compile_and_check([admat, aivec], # [Reshape(ndim)(admat, aivec)], # [admat_val, [4, 3]], Reshape) # # self._compile_and_check([admat, aivec], # [Reshape(ndim)(admat, aivec)], # [admat_val, [4, -1]], Reshape) adtens4 = dtensor4() ndim = 4 adtens4_val = rand(2, 4, 3, 5) self._compile_and_check([adtens4], [Reshape(ndim)(adtens4, [1, -1, 10, 4])], [adtens4_val], Reshape) self._compile_and_check([adtens4], [Reshape(ndim)(adtens4, [1, 3, 10, 4])], [adtens4_val], Reshape) # enable when infer_shape is generalized: # self._compile_and_check([adtens4, aivec], # [Reshape(ndim)(adtens4, aivec)], # [adtens4_val, [1, -1, 10, 4]], Reshape) # # self._compile_and_check([adtens4, aivec], # [Reshape(ndim)(adtens4, aivec)], # [adtens4_val, [1, 3, 10, 4]], Reshape) # Tile op is deprecated so the tile function doesn't use it # anymore, we'll test here the op directly advec = dvector() advec_val = rand(5) aivec_val = [3] ndim = 1 self._compile_and_check([advec], [Tile(ndim)(advec, aivec_val)], [advec_val], Tile) admat = dmatrix() admat_val = rand(2, 4) aivec_val = [2, 3] ndim = 2 self._compile_and_check([admat], [Tile(ndim)(admat, aivec_val)], [admat_val], Tile) adtens4 = dtensor4() adtens4_val = rand(2, 4, 3, 5) aivec_val = [2, 3, 1, 4] ndim = 4 self._compile_and_check([adtens4], [Tile(ndim)(adtens4, aivec_val)], [adtens4_val], Tile) class TestTensorInstanceMethods(unittest.TestCase): def setUp(self): self.vars = matrices('X', 'Y') self.vals = [m.astype(floatX) for m in [rand(2, 2), rand(2, 2)]] def test_argmin(self): X, _ = self.vars x, _ = self.vals assert_array_equal(X.argmin().eval({X: x}), x.argmin()) def test_argmax(self): X, _ = self.vars x, _ = self.vals assert_array_equal(X.argmax().eval({X: x}), x.argmax()) def test_argsort(self): X, _ = self.vars x, _ = self.vals assert_array_equal(X.argsort().eval({X: x}), x.argsort()) assert_array_equal(X.argsort(1).eval({X: x}), x.argsort(1)) def test_clip(self): X, Y = self.vars x, y = self.vals # np.clip gives unexpected values when min > max, # so we have to make sure that min <= max in that test, # otherwise it randomly fails. Z = X.clip(Y - 0.5, Y + 0.5) z = x.clip(y - 0.5, y + 0.5) assert_array_equal(Z.eval({X: x, Y: y}), z) def test_dot(self): X, Y = self.vars x, y = self.vals # Use allclose comparison as a user reported on the mailing # list failure otherwise with array that print exactly the same. assert_allclose(x.dot(y), X.dot(Y).eval({X: x, Y: y})) Z = X.dot(Y) z = x.dot(y) assert_allclose(x.dot(z), X.dot(Z).eval({X: x, Z: z})) def test_real_imag(self): X, Y = self.vars x, y = self.vals Z = X + Y * 1j z = x + y * 1j assert_array_equal(Z.real.eval({Z: z}), x) assert_array_equal(Z.imag.eval({Z: z}), y) def test_conj(self): X, Y = self.vars x, y = self.vals Z = X + Y * 1j z = x + y * 1j assert_array_equal(Z.conj().eval({Z: z}), z.conj()) assert_array_equal(Z.conjugate().eval({Z: z}), z.conj()) def test_round(self): X, _ = self.vars x, _ = self.vals assert_array_equal(X.round().eval({X: x}), x.round()) def test_std(self): X, _ = self.vars x, _ = self.vals # std() is implemented as theano tree and does not pass its # args directly to numpy. This sometimes results in small # difference, so we use allclose test. assert_allclose(X.std().eval({X: x}), x.std()) def test_repeat(self): X, _ = self.vars x, _ = self.vals assert_array_equal(X.repeat(2).eval({X: x}), x.repeat(2)) def test_trace(self): X, _ = self.vars x, _ = self.vals assert_array_equal(X.trace().eval({X: x}), x.trace()) def test_ravel(self): X, _ = self.vars x, _ = self.vals assert_array_equal(X.ravel().eval({X: x}), x.ravel()) def test_diagonal(self): X, _ = self.vars x, _ = self.vals assert_array_equal(X.diagonal().eval({X: x}), x.diagonal()) assert_array_equal(X.diagonal(1).eval({X: x}), x.diagonal(1)) assert_array_equal(X.diagonal(-1).eval({X: x}), x.diagonal(-1)) for offset, axis1, axis2 in [(1, 0, 1), (-1, 0, 1), (0, 1, 0), (-2, 1, 0)]: assert_array_equal(X.diagonal(offset, axis1, axis2).eval({X: x}), x.diagonal(offset, axis1, axis2)) def test_take(self): X, _ = self.vars x, _ = self.vals indices = [1, 0, 3] assert_array_equal(X.take(indices).eval({X: x}), x.take(indices)) indices = [1, 0, 1] assert_array_equal(X.take(indices, 1).eval({X: x}), x.take(indices, 1)) indices = np.array([-10, 5, 12], dtype='int32') assert_array_equal(X.take(indices, 1, mode='wrap').eval({X: x}), x.take(indices, 1, mode='wrap')) assert_array_equal(X.take(indices, -1, mode='wrap').eval({X: x}), x.take(indices, -1, mode='wrap')) assert_array_equal(X.take(indices, 1, mode='clip').eval({X: x}), x.take(indices, 1, mode='clip')) assert_array_equal(X.take(indices, -1, mode='clip').eval({X: x}), x.take(indices, -1, mode='clip')) # Test error handling self.assertRaises(IndexError, X.take(indices).eval, {X: x}) self.assertRaises(IndexError, (2 * X.take(indices)).eval, {X: x}) self.assertRaises(TypeError, X.take, [0.0]) indices = [[1, 0, 1], [0, 1, 1]] assert_array_equal(X.take(indices, 1).eval({X: x}), x.take(indices, 1)) # Test equivalent advanced indexing assert_array_equal(X[:, indices].eval({X: x}), x[:, indices]) def test_cumsum(self): X, _ = self.vars x, _ = self.vals assert_array_equal(X.cumsum().eval({X: x}), x.cumsum()) def test_cumprod(self): X, _ = self.vars x, _ = self.vals assert_array_equal(X.cumprod().eval({X: x}), x.cumprod()) def test_norm(): x = theano.tensor.vector('x') n = x.norm(2) f = theano.function([x], n) assert np.allclose(f([1, 1]), np.sqrt(2)) class test_cov(unittest.TestCase): def test_core(self): x = theano.tensor.matrix('x') c = theano.tensor.cov(x) f = theano.function([x], c) # basic cov function data = np.asarray(np.random.rand(3, 5), dtype=config.floatX) assert np.allclose(f(data), np.cov(data)) data = np.asarray(np.random.rand(5, 3), dtype=config.floatX) assert np.allclose(f(data), np.cov(data)) data = np.asarray(np.random.rand(10, 10), dtype=config.floatX) assert np.allclose(f(data), np.cov(data)) data = np.asarray(np.random.rand(2, 2), dtype=config.floatX) assert np.allclose(f(data), np.cov(data)) data = np.asarray(np.random.rand(1, 2), dtype=config.floatX) assert np.allclose(f(data), np.cov(data)) def test_rowvar(self): for rowvar in [True, False]: x = theano.tensor.matrix('x') c = theano.tensor.cov(x, rowvar=rowvar) f = theano.function([x], c) data = np.asarray(np.random.rand(3, 5), dtype=config.floatX) assert np.allclose(f(data), np.cov(data, rowvar=rowvar)) data = np.asarray(np.random.rand(5, 3), dtype=config.floatX) assert np.allclose(f(data), np.cov(data, rowvar=rowvar)) data = np.asarray(np.random.rand(10, 10), dtype=config.floatX) assert np.allclose(f(data), np.cov(data, rowvar=rowvar)) data = np.asarray(np.random.rand(2, 2), dtype=config.floatX) assert np.allclose(f(data), np.cov(data, rowvar=rowvar)) # check when variables are along the first axis x = theano.tensor.matrix('x') c = theano.tensor.cov(x, rowvar=False) f = theano.function([x], c) data = np.asarray(np.random.rand(2, 1), dtype=config.floatX) assert np.allclose(f(data), np.cov(data, rowvar=False)) def test_y(self): # test y x = theano.tensor.matrix('x') y = theano.tensor.matrix('y') c = theano.tensor.cov(x, y=y) f = theano.function([x, y], c) data = np.asarray(np.random.rand(3, 5), dtype=config.floatX) y = np.asarray(np.random.rand(3, 5), dtype=config.floatX) assert np.allclose(f(data, y), np.cov(data, y=y)) data = np.asarray(np.random.rand(5, 3), dtype=config.floatX) y = np.asarray(np.random.rand(5, 3), dtype=config.floatX) assert np.allclose(f(data, y), np.cov(data, y=y)) data = np.asarray(np.random.rand(10, 10), dtype=config.floatX) y = np.asarray(np.random.rand(10, 10), dtype=config.floatX) assert np.allclose(f(data, y), np.cov(data, y=y)) data = np.asarray(np.random.rand(2, 2), dtype=config.floatX) y = np.asarray(np.random.rand(2, 2), dtype=config.floatX) assert np.allclose(f(data, y), np.cov(data, y=y)) def test_ddof(self): for ddof in range(0, 5): x = theano.tensor.matrix('x') c = theano.tensor.cov(x, ddof=ddof) f = theano.function([x], c) data = np.asarray(np.random.rand(3, 5), dtype=config.floatX) assert np.allclose(f(data), np.cov(data, ddof=ddof)) def test_bias(self): for bias in [True, False]: x = theano.tensor.matrix('x') c = theano.tensor.cov(x, bias=bias) f = theano.function([x], c) data = np.asarray(np.random.rand(3, 5), dtype=config.floatX) assert np.allclose(f(data), np.cov(data, bias=bias)) for ddof in range(0, 5): for bias in [True, False]: x = theano.tensor.matrix('x') c = theano.tensor.cov(x, ddof=ddof, bias=bias) f = theano.function([x], c) data = np.asarray(np.random.rand(3, 5), dtype=config.floatX) assert np.allclose(f(data), np.cov(data, ddof=ddof, bias=bias)) class test_ptp(unittest.TestCase): def test_scalar(self): # Should return 0 for all scalar x = scalar('x') p = ptp(x) f = theano.function([x], p) y = np.asarray(rand() * 2000 - 1000, dtype=config.floatX) result = f(y) numpyResult = np.ptp(y) self.assertTrue(np.array_equal(result, numpyResult)) def test_vector(self): x = vector('x') p = ptp(x, 0) f = theano.function([x], p) y = rand_ranged(-1000, 1000, [100]) result = f(y) numpyResult = np.ptp(y, 0) self.assertTrue(np.array_equal(result, numpyResult)) def test_matrix_first_axis(self): x = matrix('x') p = ptp(x, 1) f = theano.function([x], p) y = rand_ranged(-1000, 1000, [100, 100]) result = f(y) numpyResult = np.ptp(y, 1) self.assertTrue(np.array_equal(result, numpyResult)) def test_matrix_second_axis(self): x = matrix('x') p = ptp(x, 0) f = theano.function([x], p) y = rand_ranged(-1000, 1000, [100, 100]) result = f(y) numpyResult = np.ptp(y, 0) self.assertTrue(np.array_equal(result, numpyResult)) def test_matrix_neg_axis(self): x = matrix('x') p = ptp(x, -1) f = theano.function([x], p) y = rand_ranged(-1000, 1000, [100, 100]) result = f(y) numpyResult = np.ptp(y, -1) self.assertTrue(np.array_equal(result, numpyResult)) def test_matrix_no_axis(self): x = matrix('x') p = ptp(x) f = theano.function([x], p) y = rand_ranged(-1000, 1000, [100, 100]) result = f(y) numpyResult = np.ptp(y) self.assertTrue(np.array_equal(result, numpyResult)) def test_interface(self): x = matrix('x') p = x.ptp(1) f = theano.function([x], p) y = rand_ranged(-1000, 1000, [100, 100]) result = f(y) numpyResult = np.ptp(y, 1) self.assertTrue(np.array_equal(result, numpyResult)) if __name__ == '__main__': t = TestInferShape('setUp') t.setUp() t.test_infer_shape() class T_swapaxes(unittest.TestCase): def test_no_dimensional_input(self): self.assertRaises(IndexError, swapaxes, 2, 0, 1) def test_unidimensional_input(self): self.assertRaises(IndexError, swapaxes, [2, 1], 0, 1) def test_not_enough_dimension(self): self.assertRaises(IndexError, swapaxes, [[2, 1], [3, 4]], 3, 4) def test_doubleswap(self): y = matrix() n = swapaxes(y, 0, 1) f = function([y], n) testMatrix = [[2, 1], [3, 4]] self.assertTrue(np.array_equal(testMatrix, f(f(testMatrix)))) def test_interface(self): x = theano.tensor.matrix() x.swapaxes(0, 1) def test_numpy_compare(self): rng = np.random.RandomState(utt.fetch_seed()) A = tensor.matrix("A", dtype=theano.config.floatX) Q = swapaxes(A, 0, 1) fn = function([A], [Q]) a = rng.rand(4, 4).astype(theano.config.floatX) n_s = np.swapaxes(a, 0, 1) t_s = fn(a) assert np.allclose(n_s, t_s) class T_Power(unittest.TestCase): def test_numpy_compare(self): rng = np.random.RandomState(utt.fetch_seed()) A = tensor.matrix("A", dtype=theano.config.floatX) Q = power(A, 3) fn = function([A], [Q]) a = rng.rand(4, 4).astype(theano.config.floatX) n_p = np.power(a, 3) t_p = fn(a) assert np.allclose(n_p, t_p) def test_multiple_power(self): x = tensor.vector() y = [1, 2, 3] z = power(x, y) f = function([x], z) assert np.allclose(f([1, 2, 3]), [1, 4, 27]) def test_wrong_shape(self): x = tensor.vector() y = [1, 2, 3] z = power(x, y) f = function([x], z) self.assertRaises(ValueError, f, [1, 2, 3, 4]) class T_Choose(utt.InferShapeTester): op = staticmethod(choose) op_class = Choose modes = ['raise', 'wrap', 'clip'] def test_numpy_compare(self): a = tensor.vector(dtype='int32') b = tensor.matrix(dtype='float32') A = np.random.randint(0, 4, 4).astype('int32') B = np.asarray(np.random.rand(4, 4), dtype='float32') for m in self.modes: f = function([a, b], choose(a, b, mode=m)) t_c = f(A, B) n_c = np.choose(A, B, mode=m) assert np.allclose(t_c, n_c) def test_method(self): a = tensor.vector(dtype='int32') b = tensor.matrix(dtype='float32') A = np.random.randint(0, 4, 4).astype('int32') B = np.asarray(np.random.rand(4, 4), dtype='float32') for m in self.modes: f = function([a, b], a.choose(b, mode=m)) t_c = f(A, B) n_c = A.choose(B, mode=m) assert np.allclose(t_c, n_c) def test_broadcasted(self): a = tensor.scalar(dtype='int32') b = tensor.matrix(dtype='float32') # Test when a is broadcastable A = 3 B = np.asarray(np.random.rand(4, 4), dtype='float32') for m in self.modes: f = function([a, b], choose(a, b, mode=m)) t_c = f(A, B) n_c = np.choose(A, B, mode=m) assert np.allclose(t_c, n_c) # Test when the result should be broadcastable b = theano.tensor.col(dtype='float32') B = np.asarray(np.random.rand(4, 1), dtype='float32') for m in self.modes: f = function([a, b], choose(a, b, mode=m)) assert choose(a, b, mode=m).broadcastable[0] t_c = f(A, B) n_c = np.choose(A, B, mode=m) assert np.allclose(t_c, n_c) def test_dtype_error(self): a = tensor.scalar(dtype='float32') b = tensor.matrix(dtype='float32') self.assertRaises(TypeError, choose, a, b) def test_numpy_compare_tuple(self): a = tensor.tensor3(dtype='int32') b = tensor.tensor3(dtype='float32') c = tensor.tensor3(dtype='float32') A = np.random.randint(0, 2, (2, 1, 1)).astype('int32') B = np.asarray(np.random.rand(1, 6, 1), dtype='float32') C = np.asarray(np.random.rand(1, 1, 5), dtype='float32') for m in self.modes: f = function([a, b, c], choose(a, (b, c), mode=m)) t_c = f(A, B, C) n_c = np.choose(A, (B, C), mode=m) assert np.allclose(t_c, n_c) def test_infer_shape(self): for shp1, shp2 in [ ((5, 4), (7, 4)), ((1, 4), (7, 4)), ((5, 1), (7, 4)), ((5, 4), (1, 4)), ((5, 4), (7, 1)), ((5, 4), (4,)), ((1, 4), (4,)), ((5, 1), (4,)), ((5, 4), (1,)), ((4,), (5, 4)), ((1,), (5, 4)), ((4,), (1, 4)), ((4,), (3, 1)), ((4,), (4,)), ((1,), (4,)), ((4,), (1,)), ((1,), (1,)), ]: a = tensor.tensor(dtype='int32', broadcastable=[n == 1 for n in shp1]) c = tensor.tensor(dtype='float32', broadcastable=[n == 1 for n in shp2]) A = np.asarray(np.random.rand(*shp1) * shp2[0], dtype='int32') C = np.asarray(np.random.rand(*shp2) * shp2[0], dtype='float32') self._compile_and_check([a, c], # theano.function inputs [self.op(a, c)], # theano.function outputs # Always use not square matrix! # inputs data [A, C], # Op that should be removed from the graph. self.op_class) # Disabled as it isn't implemented. def ___test_infer_shape_tuple(self): a = tensor.tensor3(dtype='int32') b = tensor.tensor3(dtype='int32') c = tensor.tensor3(dtype='int32') A = np.asarray([1, 0], dtype='int32').reshape((2, 1, 1)) B = np.asarray(np.random.rand(1, 4, 1), dtype='int32') C = np.asarray(np.random.rand(1, 1, 7), dtype='int32') f = function([a, b, c], choose(a, (b, c))) shape = (2, 4, 7) assert np.allclose(f(A, B, C).shape, shape) self._compile_and_check([a, b, c], # theano.function inputs [self.op(a, (b, c))], # theano.function outputs # Always use not square matrix! # inputs data [A, B, C], # Op that should be removed from the graph. self.op_class) def test_allocempty(): # Test that we allocated correctly f = theano.function([], AllocEmpty("float32")(2, 3)) assert len(f.maker.fgraph.apply_nodes) == 1 out = f() assert out.shape == (2, 3) assert out.dtype == 'float32' def test_symbolic_slice(): x = theano.tensor.tensor4('x') a, b = x.shape[:2] output = a.eval({x: np.zeros((5, 4, 3, 2), dtype=theano.config.floatX)}) assert output == np.array(5) Theano-1.0.0/theano/tensor/tests/test_elemwise.py00006000163562€ÿÿÿþ0000015326513203335532023460 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division from copy import copy import unittest import math import numpy as np from nose.plugins.skip import SkipTest from nose.tools import raises from six.moves import xrange import six.moves.cPickle as pickle import theano from theano.compat import imap from theano import gof, scalar, config from theano import tensor from theano.tensor import TensorType, as_tensor_variable from theano.compile.mode import get_default_mode, Mode from theano.tensor.elemwise import (CAReduce, Elemwise, DimShuffle, Prod, ProdWithoutZeros) from theano.tests import unittest_tools from theano.tests.unittest_tools import attr import theano.tests.unittest_tools as utt def FunctionGraph(i, o): e = gof.FunctionGraph(i, o) return e class test_DimShuffle(unittest_tools.InferShapeTester): op = DimShuffle type = TensorType dtype = theano.config.floatX def with_linker(self, linker): for xsh, shuffle, zsh in [((2, 3), (1, 'x', 0), (3, 1, 2)), ((1, 2, 3), (1, 2), (2, 3)), ((1, 2, 1, 3), (1, 3), (2, 3)), ((2, 3, 4), (2, 1, 0), (4, 3, 2)), ((2, 3, 4), ('x', 2, 1, 0, 'x'), (1, 4, 3, 2, 1)), ((1, 4, 3, 2, 1), (3, 2, 1), (2, 3, 4)), ((1, 1, 4), (1, 2), (1, 4)), ((1, 1, 1), (), ()), ((1,), ('x', 'x'), (1, 1))]: ib = [(entry == 1) for entry in xsh] x = self.type(self.dtype, ib)('x') e = self.op(ib, shuffle)(x) f = copy(linker).accept(FunctionGraph([x], [e])).make_function() assert f(np.ones(xsh, dtype=self.dtype)).shape == zsh # test that DimShuffle.infer_shape work correctly x = self.type(self.dtype, ib)('x') e = self.op(ib, shuffle)(x) f = copy(linker).accept(FunctionGraph([x], [e.shape])).make_function() assert all(f(np.ones(xsh, dtype=self.dtype))) == all(zsh) # Test when we drop a axis that is not broadcastable ib = [False, True, False] x = self.type(self.dtype, ib)('x') self.assertRaises(ValueError, self.op, ib, shuffle) # Test when we drop a axis that don't have shape 1 ib = [True, True, False] x = self.type(self.dtype, ib)('x') e = self.op(ib, (1, 2))(x) f = copy(linker).accept(FunctionGraph([x], [e.shape])).make_function() self.assertRaises(TypeError, f, np.ones((2, 1, 4))) # Test that we can't take a dimensions multiple time xsh, shuffle, zsh = ((1, 1, 4), (0, 1, 2, 0), (1, 4)) ib = [False, True, False] x = self.type(self.dtype, ib)('x') self.assertRaises(ValueError, DimShuffle, ib, shuffle) def test_perform(self): self.with_linker(gof.PerformLinker()) def test_c_or_py(self): # Shape op don't have C code. # But This will test DimShuffle c code self.with_linker(gof.OpWiseCLinker()) def test_infer_shape(self): for xsh, shuffle in [((2, 3), (1, 'x', 0)), ((1, 2, 3), (1, 2)), ((1, 2, 1, 3), (1, 3)), ((2, 3, 4), (2, 1, 0)), ((2, 3, 4), ('x', 2, 1, 0, 'x')), ((1, 4, 3, 2, 1), (3, 2, 1)), ((1, 1, 4), (1, 2)), ((1, 1, 1), ()), ((1,), ('x', 'x'))]: ib = [(entry == 1) for entry in xsh] adtens = self.type(self.dtype, ib)('x') adtens_val = np.ones(xsh, dtype=self.dtype) self._compile_and_check([adtens], [self.op(ib, shuffle)(adtens)], [adtens_val], self.op, warn=False) def test_too_big_rank(self): x = self.type(self.dtype, broadcastable=())() y = x.dimshuffle(('x',) * (np.MAXDIMS + 1)) self.assertRaises(ValueError, y.eval, {x: 0}) class test_reduce_axes(unittest.TestCase): def test_sum_axes(self): axes = [None, 0, 1, [0, 1], np.array(1), [np.array(0), np.array(1)]] for a in axes: x = tensor.matrix() x.sum(a) def test_mean_axes(self): axes = [None, 0, 1, [0, 1], np.array(1), [np.array(0), np.array(1)]] for a in axes: x = tensor.matrix() x.mean(a) def test_max_axes(self): axes = [None, 0, 1, [0, 1], np.array(1), [np.array(0), np.array(1)]] for a in axes: x = tensor.matrix() x.max(a) def test_min_axes(self): axes = [None, 0, 1, [0, 1], np.array(1), [np.array(0), np.array(1)]] for a in axes: x = tensor.matrix() x.min(a) def test_argmax_axes(self): axes = [None, 0, 1, [0, 1], np.array(1), [np.array(0), np.array(1)]] for a in axes: x = tensor.matrix() x.argmax(a) def test_var_axes(self): axes = [None, 0, 1, [0, 1], np.array(1), [np.array(0), np.array(1)]] for a in axes: x = tensor.matrix() x.var(a) class test_Broadcast(unittest.TestCase): # this is to allow other types to reuse this class to test their ops type = TensorType op = Elemwise ctype = TensorType cop = Elemwise openmp_minsize = 2 * config.openmp_elemwise_minsize openmp_minsize_sqrt = int(math.ceil(math.sqrt(openmp_minsize))) # The order is important if you change them. linkers = [gof.PerformLinker, gof.CLinker] def rand_val(self, shp): return np.asarray(np.random.rand(*shp), dtype=theano.config.floatX) def rand_cval(self, shp): return np.asarray(np.random.rand(*shp), dtype=theano.config.floatX) def setUp(self): unittest_tools.seed_rng() def with_linker(self, linker, op, type, rand_val): for xsh, ysh in [((3, 5), (3, 5)), ((3, 5), (1, 5)), ((3, 5), (3, 1)), ((1, 5), (5, 1)), ((1, 1), (1, 1)), ((self.openmp_minsize,), (self.openmp_minsize,)), ((self.openmp_minsize_sqrt, self.openmp_minsize_sqrt), (self.openmp_minsize_sqrt, self.openmp_minsize_sqrt)), ((2, 3, 4, 5), (2, 3, 4, 5)), ((2, 3, 4, 5), (1, 3, 1, 5)), ((2, 3, 4, 5), (1, 1, 1, 1)), ((), ())]: x = type(theano.config.floatX, [(entry == 1) for entry in xsh])('x') y = type(theano.config.floatX, [(entry == 1) for entry in ysh])('y') e = op(scalar.add)(x, y) f = copy(linker).accept(FunctionGraph([x, y], [e])).make_function() xv = rand_val(xsh) yv = rand_val(ysh) zv = xv + yv unittest_tools.assert_allclose(f(xv, yv), zv) # test Elemwise.infer_shape # the Shape op don't implement c_code! if isinstance(linker, gof.PerformLinker): x = type(theano.config.floatX, [(entry == 1) for entry in xsh])('x') y = type(theano.config.floatX, [(entry == 1) for entry in ysh])('y') e = op(scalar.add)(x, y) f = copy(linker).accept(FunctionGraph( [x, y], [e.shape])).make_function() assert tuple(f(xv, yv)) == tuple(zv.shape) def with_linker_inplace(self, linker, op, type, rand_val): for xsh, ysh in [((5, 5), (5, 5)), ((5, 5), (1, 5)), ((5, 5), (5, 1)), ((1, 1), (1, 1)), ((2, 3, 4, 5), (2, 3, 4, 5)), ((2, 3, 4, 5), (1, 3, 1, 5)), ((2, 3, 4, 5), (1, 1, 1, 1)), ((), ())]: x = type(theano.config.floatX, [(entry == 1) for entry in xsh])('x') y = type(theano.config.floatX, [(entry == 1) for entry in ysh])('y') e = op(scalar.Add(scalar.transfer_type(0)), {0: 0})(x, y) f = copy(linker).accept(FunctionGraph([x, y], [e])).make_function() xv = rand_val(xsh) yv = rand_val(ysh) zv = xv + yv f(xv, yv) self.assertTrue((xv == zv).all()) # test Elemwise.infer_shape # the Shape op don't implement c_code! if isinstance(linker, gof.PerformLinker): x = type(theano.config.floatX, [(entry == 1) for entry in xsh])('x') y = type(theano.config.floatX, [(entry == 1) for entry in ysh])('y') e = op(scalar.Add(scalar.transfer_type(0)), {0: 0})(x, y) f = copy(linker).accept(FunctionGraph( [x, y], [e.shape])).make_function() xv = rand_val(xsh) yv = rand_val(ysh) zv = xv + yv f(xv, yv) assert xv.shape == zv.shape def test_perform(self): self.with_linker(gof.PerformLinker(), self.op, self.type, self.rand_val) def test_c(self): if not theano.config.cxx: raise SkipTest("G++ not available, so we need to skip this test.") self.with_linker(gof.CLinker(), self.cop, self.ctype, self.rand_cval) def test_perform_inplace(self): self.with_linker_inplace(gof.PerformLinker(), self.op, self.type, self.rand_val) def test_c_inplace(self): if not theano.config.cxx: raise SkipTest("G++ not available, so we need to skip this test.") self.with_linker_inplace(gof.CLinker(), self.cop, self.ctype, self.rand_cval) def test_fill(self): if not theano.config.cxx: raise SkipTest("G++ not available, so we need to skip this test.") for linker, op, t, rval in zip(self.linkers, [self.op, self.cop], [self.type, self.ctype], [self.rand_val, self.rand_cval]): x = t(theano.config.floatX, [0, 0])('x') y = t(theano.config.floatX, [1, 1])('y') e = op(scalar.Second(scalar.transfer_type(0)), {0: 0})(x, y) f = linker().accept(FunctionGraph([x, y], [e])).make_function() xv = rval((5, 5)) yv = rval((1, 1)) f(xv, yv) assert (xv == yv).all() def test_fill_var(self): x = tensor.matrix() x.fill(3) def test_fill_grad(self): # Fix bug reported at # https://groups.google.com/d/topic/theano-users/nQshB8gUA6k/discussion x = TensorType(config.floatX, [0, 1, 0])('x') y = TensorType(config.floatX, [0, 1, 0])('y') e = tensor.second(x, y) theano.grad(e.sum(), y) def test_weird_strides(self): if not theano.config.cxx: raise SkipTest("G++ not available, so we need to skip this test.") for linker, op, t, rval in zip(self.linkers, [self.op, self.cop], [self.type, self.ctype], [self.rand_val, self.rand_cval]): x = t(theano.config.floatX, [0, 0, 0, 0, 0])('x') y = t(theano.config.floatX, [0, 0, 0, 0, 0])('y') e = op(scalar.add)(x, y) f = linker().accept(FunctionGraph([x, y], [e])).make_function() xv = rval((2, 2, 2, 2, 2)) yv = rval((2, 2, 2, 2, 2)).transpose(4, 0, 3, 1, 2) zv = xv + yv assert (f(xv, yv) == zv).all() def test_same_inputs(self): if not theano.config.cxx: raise SkipTest("G++ not available, so we need to skip this test.") for linker, op, t, rval in zip(self.linkers, [self.op, self.cop], [self.type, self.ctype], [self.rand_val, self.rand_cval]): x = t(theano.config.floatX, [0, 0])('x') e = op(scalar.add)(x, x) f = linker().accept(FunctionGraph([x], [e])).make_function() xv = rval((2, 2)) zv = xv + xv assert (f(xv) == zv).all() def reduce_bitwise_and(x, axis=-1, dtype='int8'): identity = np.array((-1,), dtype=dtype)[0] shape_without_axis = tuple([s for i, s in enumerate(x.shape) if i != axis]) if 0 in shape_without_axis: return np.empty(shape=shape_without_axis, dtype=x.dtype) def custom_reduce(a): out = identity for i in range(a.size): out = np.bitwise_and(a[i], out) return out return np.apply_along_axis(custom_reduce, axis, x) class test_CAReduce(unittest_tools.InferShapeTester): op = CAReduce cases = [((5, 6), None), ((5, 6), (0, 1)), ((5, 6), (0, )), ((5, 6), (1, )), ((5, 6), (-1, )), ((5, 6), (-2, )), ((5, 6), ()), ((2, 3, 4, 5), (0, 1, 3)), ((2, 3, 4, 5), (-2, -3)), ((5, 0), None), ((5, 0), (0, )), ((5, 0), (1, )), ((5, 0), ()), ((), None), ((), ())] type = TensorType def with_mode(self, mode, scalar_op=scalar.add, dtype="floatX", pre_scalar_op=None, test_nan=False, tensor_op=None): for xsh, tosum in self.cases: if dtype == "floatX": dtype = theano.config.floatX x = self.type(dtype, [(entry == 1) for entry in xsh])('x') d = {} if pre_scalar_op is not None: d = {"pre_scalar_op": pre_scalar_op} if tensor_op is None: e = as_tensor_variable(self.op(scalar_op, axis=tosum, **d)(x)) else: e = as_tensor_variable(tensor_op(x, axis=tosum, **d)) if tosum is None: tosum = list(range(len(xsh))) f = theano.function([x], e, mode=mode) xv = np.asarray(np.random.rand(*xsh)) if dtype not in tensor.discrete_dtypes: xv = np.asarray(xv, dtype=dtype) else: xv = np.asarray(xv < 0.5, dtype=dtype) if test_nan and xv.size > 0: if len(xsh) > 0: xv = xv.flatten() xv[0] = np.nan xv = xv.reshape(*xsh) else: xv = np.asarray(np.nan, dtype=dtype) zv = xv if pre_scalar_op is not None: zv = Elemwise(scalar_op=pre_scalar_op)(x).eval({x: xv}) numpy_raised = False if len(tosum) > 1 and any([a < 0 for a in tosum]): # In that case, we need to use the good order of axis # in the reduction. axis2 = [] for a in tosum: if a < 0: axis2.append(a + len(xsh)) else: axis2.append(a) assert len(axis2) == len(tosum) tosum = tuple(axis2) if tensor_op == tensor.all: for axis in reversed(sorted(tosum)): zv = np.all(zv, axis) if len(tosum) == 0: zv = zv != 0 elif tensor_op == tensor.any: for axis in reversed(sorted(tosum)): zv = np.any(zv, axis) if len(tosum) == 0: zv = zv != 0 elif scalar_op == scalar.add: for axis in reversed(sorted(tosum)): zv = np.add.reduce(zv, axis) if dtype == 'bool': # np.add of a bool upcast, while CAReduce don't zv = zv.astype(dtype) elif scalar_op == scalar.mul: for axis in reversed(sorted(tosum)): zv = np.multiply.reduce(zv, axis) elif scalar_op == scalar.maximum: try: for axis in reversed(sorted(tosum)): zv = np.maximum.reduce(zv, axis) except ValueError: numpy_raised = True elif scalar_op == scalar.minimum: try: for axis in reversed(sorted(tosum)): zv = np.minimum.reduce(zv, axis) except ValueError: numpy_raised = True elif scalar_op == scalar.or_: for axis in reversed(sorted(tosum)): zv = np.bitwise_or.reduce(zv, axis) elif scalar_op == scalar.and_: for axis in reversed(sorted(tosum)): zv = reduce_bitwise_and(zv, axis, dtype=dtype) elif scalar_op == scalar.xor: # There is no identity value for the xor function # So we can't support shape of dimensions 0. if np.prod(zv.shape) == 0: continue for axis in reversed(sorted(tosum)): zv = np.bitwise_xor.reduce(zv, axis) else: raise Exception( "Test for CAReduce with scalar_op %s not implemented" % str(scalar_op)) if scalar_op in [scalar.maximum, scalar.minimum] and numpy_raised: try: out = f(xv) assert out.dtype == dtype except ValueError: pass else: self.fail() else: if test_nan: try: self.assertTrue( self.type.values_eq(f(xv), zv), (f(xv), zv)) except NotImplementedError: # GpuCAReduce don't implement all cases when size is 0 assert xv.size == 0 else: try: f_xv = f(xv) self.assertTrue((f_xv.shape == zv.shape), (f_xv, zv)) utt.assert_allclose(zv, f_xv) except NotImplementedError: # GpuCAReduce don't implement all cases when size is 0 assert xv.size == 0 x = self.type(dtype, [(entry == 1) for entry in xsh])('x') if tensor_op is None: e = self.op(scalar_op, axis=tosum)(x) else: e = tensor_op(x, axis=tosum) if tosum is None: tosum = list(range(len(xsh))) f = theano.function([x], e.shape, mode=mode) if not(scalar_op in [scalar.maximum, scalar.minimum] and ((xsh == () or np.prod(xsh) == 0))): try: assert all(f(xv) == zv.shape) except NotImplementedError: # GpuCAReduce don't implement all cases when size is 0 assert xv.size == 0 def test_perform_noopt(self): self.with_mode(Mode(linker='py', optimizer=None), scalar.add, dtype='floatX') def test_perform(self): for dtype in ["bool", "floatX", "complex64", "complex128", "int8", "uint8"]: self.with_mode(Mode(linker='py'), scalar.add, dtype=dtype) self.with_mode(Mode(linker='py'), scalar.mul, dtype=dtype) self.with_mode(Mode(linker='py'), scalar.maximum, dtype=dtype) self.with_mode(Mode(linker='py'), scalar.minimum, dtype=dtype) self.with_mode(Mode(linker='py'), scalar.and_, dtype=dtype, tensor_op=tensor.all) self.with_mode(Mode(linker='py'), scalar.or_, dtype=dtype, tensor_op=tensor.any) for dtype in ["int8", "uint8"]: self.with_mode(Mode(linker='py'), scalar.or_, dtype=dtype) self.with_mode(Mode(linker='py'), scalar.and_, dtype=dtype) self.with_mode(Mode(linker='py'), scalar.xor, dtype=dtype) def test_perform_nan(self): for dtype in ["floatX", "complex64", "complex128"]: self.with_mode(Mode(linker='py'), scalar.add, dtype=dtype, test_nan=True) self.with_mode(Mode(linker='py'), scalar.mul, dtype=dtype, test_nan=True) self.with_mode(Mode(linker='py'), scalar.maximum, dtype=dtype, test_nan=True) self.with_mode(Mode(linker='py'), scalar.minimum, dtype=dtype, test_nan=True) self.with_mode(Mode(linker='py'), scalar.or_, dtype=dtype, test_nan=True, tensor_op=tensor.any) self.with_mode(Mode(linker='py'), scalar.and_, dtype=dtype, test_nan=True, tensor_op=tensor.all) def test_c_noopt(self): # We need to make sure that we cover the corner cases that # optimizations normally cover if not theano.config.cxx: raise SkipTest("G++ not available, so we need to skip this test.") self.with_mode(Mode(linker='c', optimizer=None), scalar.add, dtype='floatX') @attr('slow') def test_c(self): if not theano.config.cxx: raise SkipTest("G++ not available, so we need to skip this test.") for dtype in ["bool", "floatX", "complex64", "complex128", "int8", "uint8"]: self.with_mode(Mode(linker='c'), scalar.add, dtype=dtype) self.with_mode(Mode(linker='c'), scalar.mul, dtype=dtype) for dtype in ["bool", "floatX", "int8", "uint8"]: self.with_mode(Mode(linker='c'), scalar.minimum, dtype=dtype) self.with_mode(Mode(linker='c'), scalar.maximum, dtype=dtype) self.with_mode(Mode(linker='c'), scalar.and_, dtype=dtype, tensor_op=tensor.all) self.with_mode(Mode(linker='c'), scalar.or_, dtype=dtype, tensor_op=tensor.any) for dtype in ["bool", "int8", "uint8"]: self.with_mode(Mode(linker='c'), scalar.or_, dtype=dtype) self.with_mode(Mode(linker='c'), scalar.and_, dtype=dtype) self.with_mode(Mode(linker='c'), scalar.xor, dtype=dtype) @attr('slow') def test_c_nan(self): if not theano.config.cxx: raise SkipTest("G++ not available, so we need to skip this test.") for dtype in ["floatX", "complex64", "complex128"]: self.with_mode(Mode(linker='c'), scalar.add, dtype=dtype, test_nan=True) self.with_mode(Mode(linker='c'), scalar.mul, dtype=dtype, test_nan=True) for dtype in ["floatX"]: self.with_mode(Mode(linker='c'), scalar.minimum, dtype=dtype, test_nan=True) self.with_mode(Mode(linker='c'), scalar.maximum, dtype=dtype, test_nan=True) def test_infer_shape(self, dtype=None, pre_scalar_op=None): if dtype is None: dtype = theano.config.floatX for xsh, tosum in self.cases: x = self.type(dtype, [(entry == 1) for entry in xsh])('x') if pre_scalar_op is not None: x = pre_scalar_op(x) if tosum is None: tosum = list(range(len(xsh))) xv = np.asarray(np.random.rand(*xsh), dtype=dtype) d = {} if pre_scalar_op is not None: xv = x.eval({x.owner.inputs[0]: xv}) d = {pre_scalar_op: pre_scalar_op} self._compile_and_check([x], [self.op(scalar.add, axis=tosum, *d)(x)], [xv], self.op, ["local_cut_useless_reduce"], warn=0 not in xsh) class test_Prod(unittest.TestCase): def setUp(self): unittest_tools.seed_rng() # we want to allow nans in the matrices, so we disable this # DEBUG_MODE check mode = theano.compile.mode.get_default_mode() mode = copy(mode) mode.check_isfinite = False self.mode = mode @attr('slow') def test_verify_grad(self): # including zeros, as the case with zeros is important # (and special cases: 1 zero in the row, more than 1 zero in the row) x_val = np.asarray([[.1, .2, .3], [.4, .5, .6], [.7, .8, .9]], dtype='float32') # now with verify_grad unittest_tools.verify_grad(Prod(axis=1), [x_val], mode=self.mode) # second time, with some added complexity # verify_grad takes the sum of the matrices anyway def fn(x2): return theano.tensor.sqr(Prod(axis=1)(x2)) unittest_tools.verify_grad(fn, [x_val], mode=self.mode) def test_verify_grad_with_zeros(self): # including zeros, as the case with zeros is important # (and special cases: 1 zero in the row, more than 1 zero in the row) x_val = np.asarray([[1., 2., 3.], [0., 5., 6.], [0., 0., 9.]], dtype='float32') x = theano.tensor.dmatrix() # sanity check p = Prod(axis=1)(x) # Uncomment this for debugging if needed # x2 = theano.tensor.dmatrix() # p2 = Prod(axis=1)(x2) # fn = theano.function([x, x2], [p - p2], mode=self.mode) # print("hand computed diff for each row") # x2_val = np.asarray([[1., 2., 3.003], [0.003, 5., 6], [ # 0., 0., 9.01]]) # print(fn(x_val, x2_val)) # fn2 = theano.function([x], [theano.tensor.grad(p.sum(), x)], # mode=self.mode) # print("real grad") # print(fn2(x_val)) fn3 = theano.function([x], [p], mode=self.mode) assert np.allclose(fn3(x_val), [6., 0., 0.]) # now with verify_grad unittest_tools.verify_grad(Prod(axis=1), [x_val], mode=self.mode) # second time, with some added complexity # verify_grad takes the sum of the matrices anyway # def fn5(x5): # return theano.tensor.sqr(Prod(axis=1)(x5)) # x4 = theano.tensor.dmatrix() # p4 = theano.tensor.sqr(Prod(axis=1)(x4)) # fn4 = theano.function([x4], p4) # print("with sqr") # print(fn4(x_val)) # print(fn4(x2_val)) # unittest_tools.verify_grad(fn5, [x_val]) @attr('slow') def test_prod_no_zeros_in_input(self): x = theano.tensor.dmatrix() x_val = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype='float32') pwz = Prod(axis=1, no_zeros_in_input=True)(x) fn = theano.function([x], pwz, mode=self.mode) assert np.allclose(fn(x_val), [6, 120, 504]) pwz = Prod(no_zeros_in_input=True)(x) g = theano.grad(pwz, x) gg = theano.grad(g.sum(), x) fn = theano.function([x], g, mode=self.mode) assert np.allclose(fn(x_val), [[362880., 181440., 120960.], [90720., 72576., 60480.], [51840., 45360., 40320.]]) fn = theano.function([x], gg, mode=self.mode) assert np.allclose(fn(x_val), [[663696., 422568., 301872.], [233964., 190800., 161016.], [139248., 122652., 109584.]]) unittest_tools.verify_grad(Prod(axis=1, no_zeros_in_input=True), [x_val], mode=self.mode) unittest_tools.verify_grad(Prod(no_zeros_in_input=True), [x_val], mode=self.mode) def second_deriv(x): return theano.grad(Prod(no_zeros_in_input=True)(x), x) unittest_tools.verify_grad(second_deriv, [x_val], mode=self.mode) def test_prod_without_zeros(self): x = theano.tensor.dmatrix() x_val = np.array([[1, 2, 3], [0, 5, 6], [0, 0, 9]], dtype='float32') pwz = ProdWithoutZeros(axis=1)(x) fn = theano.function([x], pwz, mode=self.mode) assert np.allclose(fn(x_val), [6, 30, 9]) pwz_a0 = ProdWithoutZeros(axis=0)(x) fn_a0 = theano.function([x], pwz_a0, mode=self.mode) assert np.allclose(fn_a0(x_val), [1, 10, 162]) @raises(theano.gradient.NullTypeGradError) def test_prod_without_zeros_grad(self): x = theano.tensor.dmatrix() pwz_a1 = ProdWithoutZeros(axis=0)(x) pwz_grad = theano.grad(theano.tensor.sum(pwz_a1), x) theano.function([x], pwz_grad, mode=self.mode) @attr('slow') def test_other_grad_tests(self): x = theano.tensor.dmatrix() x_val1 = np.array([[1, 2, 3], [0, 5, 6], [0, 0, 9]], dtype='float32') x_val2 = np.array([[1, 2, 0], [0, 5, 6], [7, 8, 9], [9, 10, 0]], dtype='float32') rng = rng = np.random.RandomState(43) p = Prod(axis=1) grad_p = theano.tensor.grad(p(x).sum(), x) grad_fn = theano.function([x], grad_p, mode=self.mode) assert np.allclose( grad_fn(x_val1), [[6., 3., 2.], [30., 0., 0.], [0., 0., 0.]]) assert np.allclose( grad_fn(x_val2), [[0., 0., 2.], [30., 0., 0.], [72., 63., 56.], [0., 0., 90.]]) p_axis0 = Prod(axis=0) grad_p_axis0 = theano.tensor.grad(p_axis0(x).sum(), x) grad_fn_axis0 = theano.function([x], grad_p_axis0, mode=self.mode) assert np.allclose( grad_fn_axis0(x_val2), [[0., 400., 0.], [63., 160., 0.], [0., 100., 0.], [0., 80., 0.]]) tensor.verify_grad(p, [x_val1], rng=rng, mode=self.mode) def test_mul_without_zeros_zeros(self): a = np.zeros((3, 3)) x = theano.tensor.dmatrix() mul1 = ProdWithoutZeros(axis=0)(x) fn_debug = theano.function([x], mul1, mode=self.mode) fn_debug(a) def test_pickle_bug(self): # Regression test for bug fixed in 24d4fd291054. o = Prod() s = pickle.dumps(o, protocol=-1) o = pickle.loads(s) pickle.dumps(o) class test_IsInf_IsNan(unittest.TestCase): def setUp(self): self.test_vals = [np.array(x, dtype=config.floatX) for x in [ 0, 1, np.nan, np.inf, -np.inf, [np.nan, np.inf, -np.inf, 0, 1, -1], ]] self.scalar = tensor.scalar() self.vector = tensor.vector() self.mode = get_default_mode() if isinstance(self.mode, theano.compile.debugmode.DebugMode): # Disable the check preventing usage of NaN / Inf values. self.mode = copy(self.mode) self.mode.check_isfinite = False def run_isfunc(self, isfunc): for input in (self.scalar, self.vector): theano_isfunc = theano.function([input], getattr(tensor, isfunc)(input), mode=self.mode) numpy_isfunc = getattr(np, isfunc) for x in self.test_vals: if ((x.ndim == 0 and input is not self.scalar) or (x.ndim == 1 and input is not self.vector)): # We only test with the appropriate input type. continue t_out = theano_isfunc(x) n_out = numpy_isfunc(x) assert (t_out == n_out).all(), (t_out, n_out) def test_isinf(self): return self.run_isfunc('isinf') def test_isnan(self): return self.run_isfunc('isnan') class T_reduce_dtype(unittest.TestCase): mode = theano.compile.get_default_mode().excluding( 'local_cut_useless_reduce') op = CAReduce axes = [None, 0, 1, [], [0], [1], [0, 1]] methods = ['sum', 'prod'] dtypes = list(imap(str, theano.scalar.all_types)) # Test the default dtype of a method(). def test_reduce_default_dtype(self): # We try multiple axis combinations even though axis should not matter. for method in self.methods: for idx, dtype in enumerate(self.dtypes): axis = self.axes[idx % len(self.axes)] x = tensor.matrix(dtype=dtype) s = getattr(x, method)(axis=axis) assert s.dtype == dict( bool='int64', int8='int64', int16='int64', int32='int64', uint8='uint64', uint16='uint64', uint32='uint64', ).get(dtype, dtype) f = theano.function([x], s, mode=self.mode) topo = f.maker.fgraph.toposort() assert [n for n in topo if isinstance(n.op, self.op)], (topo, dtype) data = np.random.rand(3, 4) * 10 data = data.astype(dtype) f(data) def test_reduce_default_acc_dtype(self): # Test the default acc_dtype of a reduce(). # We try multiple axis combinations even though axis should not matter. for method in self.methods: for idx, dtype in enumerate(self.dtypes): axis = self.axes[idx % len(self.axes)] x = tensor.matrix(dtype=dtype) s = getattr(x, method)(axis=axis) assert s.owner.op.acc_dtype == dict( bool='int64', int8='int64', int16='int64', int32='int64', uint8='uint64', uint16='uint64', uint32='uint64', float16='float32', float32='float64', complex64='complex128', ).get(dtype, dtype) f = theano.function([x], s, mode=self.mode) topo = f.maker.fgraph.toposort() assert [n for n in topo if isinstance(n.op, self.op)], (topo, dtype) data = np.random.rand(3, 4) * 10 data = data.astype(dtype) f(data) @attr('slow') def test_reduce_custom_dtype(self): # Test the ability to provide your own output dtype for a reduce. # We try multiple axis combinations even though axis should not matter. idx = 0 for method in self.methods: for input_dtype in self.dtypes: x = tensor.matrix(dtype=input_dtype) for output_dtype in self.dtypes: # Only tests case where both input and output are complex. icomplex = input_dtype.startswith('complex') ocomplex = output_dtype.startswith('complex') if icomplex != ocomplex: continue axis = self.axes[idx % len(self.axes)] var = getattr(x, method)(dtype=output_dtype, axis=axis) assert var.dtype == output_dtype f = theano.function([x], var, mode=self.mode) topo = f.maker.fgraph.toposort() assert [n for n in topo if isinstance(n.op, self.op)], \ (topo, output_dtype) data = np.random.rand(3, 4) * 10 data = data.astype(input_dtype) if (method == 'prod' and output_dtype in ['float16', 'int8', 'uint8', 'int16', 'uint16']): # We will likely get something infinite, # or the overflow will be different between CPU and GPU, # and DebugMode will complain. data = data[0:1] f(data) if "complex" in input_dtype: continue # Check that we can take the gradient tensor.grad(var.sum(), x, disconnected_inputs='ignore') idx += 1 def test_reduce_custom_acc_dtype(self): # Test the ability to provide your own accumulator dtype for a reduce. # We try multiple axis combinations even though axis should not matter. idx = 0 for method in self.methods: for input_dtype in self.dtypes: x = tensor.matrix(dtype=input_dtype) for acc_dtype in self.dtypes: # If the accumulator is a complex, the gradient of the reduce will # cast the complex to the input dtype. We can't call the normal # cast on a complex to a not complex as this is ambiguous. if (not input_dtype.startswith('complex') and acc_dtype.startswith('complex')): continue axis = self.axes[idx % len(self.axes)] # If output_dtype would force a downcast, we expect a TypeError # We always allow int/uint inputs with float/complex outputs. upcasted_dtype = scalar.upcast(input_dtype, acc_dtype) if (acc_dtype == upcasted_dtype or (input_dtype in tensor.discrete_dtypes and acc_dtype in tensor.continuous_dtypes)): var = getattr(x, method)(acc_dtype=acc_dtype, axis=axis) assert var.owner.op.acc_dtype == acc_dtype if "complex" in input_dtype: continue # Check that we can take the gradient tensor.grad(var.sum(), x, disconnected_inputs='ignore') else: self.assertRaises(TypeError, getattr(x, method), acc_dtype=acc_dtype, axis=axis) idx += 1 def test_reduce_precision(self): # Check that the default accumulator precision is sufficient for method in self.methods: x = theano.shared(np.asarray([1e8, 1, -1e8], dtype='float32')) s = getattr(x, method)() f = theano.function([], s, mode=self.mode) topo = f.maker.fgraph.toposort() assert [n for n in topo if isinstance(n.op, self.op)], topo s_val = f() # Use extra precision in NumPy to compute the good answer. ret = getattr(np.asarray([1e8, 1, -1e8], dtype='float64'), method)() assert np.allclose(s_val, ret), (s_val, ret) class T_mean_dtype(unittest.TestCase): def test_mean_default_dtype(self): # Test the default dtype of a mean(). # We try multiple axis combinations even though axis should not matter. axes = [None, 0, 1, [], [0], [1], [0, 1]] for idx, dtype in enumerate(imap(str, theano.scalar.all_types)): axis = axes[idx % len(axes)] x = tensor.matrix(dtype=dtype) m = x.mean(axis=axis) if dtype in tensor.discrete_dtypes: assert m.dtype == 'float64' else: assert m.dtype == dtype, (m, m.dtype, dtype) f = theano.function([x], m) data = np.random.rand(3, 4) * 10 data = data.astype(dtype) f(data) @attr('slow') def test_mean_custom_dtype(self): # Test the ability to provide your own output dtype for a mean. # We try multiple axis combinations even though axis should not matter. axes = [None, 0, 1, [], [0], [1], [0, 1]] idx = 0 for input_dtype in imap(str, theano.scalar.all_types): x = tensor.matrix(dtype=input_dtype) for sum_dtype in imap(str, theano.scalar.all_types): axis = axes[idx % len(axes)] # If the inner sum cannot be created, it will raise a # TypeError. try: mean_var = x.mean(dtype=sum_dtype, axis=axis) except TypeError: pass else: # Executed if no TypeError was raised if sum_dtype in tensor.discrete_dtypes: assert mean_var.dtype == 'float64', ( (mean_var.dtype, sum_dtype)) else: assert mean_var.dtype == sum_dtype, ( (mean_var.dtype, sum_dtype)) if (('complex' in input_dtype or 'complex' in sum_dtype) and input_dtype != sum_dtype): continue f = theano.function([x], mean_var) data = np.random.rand(3, 4) * 10 data = data.astype(input_dtype) f(data) # Check that we can take the gradient, when implemented if "complex" in mean_var.dtype: continue try: tensor.grad(mean_var.sum(), x, disconnected_inputs='ignore') except NotImplementedError: # TrueDiv does not seem to have a gradient when # the numerator is complex. if mean_var.dtype in tensor.complex_dtypes: pass else: raise idx += 1 def test_mean_precision(self): # Check that the default accumulator precision is sufficient x = theano.shared(np.asarray([1e8, 1, -1e8], dtype='float32')) m = x.mean() f = theano.function([], m) m_val = f() assert np.allclose(m_val, 1. / 3) class T_prod_without_zeros_dtype(unittest.TestCase): def test_prod_without_zeros_default_dtype(self): # Test the default dtype of a ProdWithoutZeros(). # We try multiple axis combinations even though axis should not matter. axes = [None, 0, 1, [], [0], [1], [0, 1]] for idx, dtype in enumerate(imap(str, theano.scalar.all_types)): axis = axes[idx % len(axes)] x = ProdWithoutZeros(axis=axis)(tensor.matrix(dtype=dtype)) assert x.dtype == dict( bool='int64', int8='int64', int16='int64', int32='int64', uint8='uint64', uint16='uint64', uint32='uint64', ).get(dtype, dtype) def test_prod_without_zeros_default_acc_dtype(self): # Test the default dtype of a ProdWithoutZeros(). # We try multiple axis combinations even though axis should not matter. axes = [None, 0, 1, [], [0], [1], [0, 1]] for idx, dtype in enumerate(imap(str, theano.scalar.all_types)): axis = axes[idx % len(axes)] x = tensor.matrix(dtype=dtype) p = ProdWithoutZeros(axis=axis)(x) assert p.owner.op.acc_dtype == dict( bool='int64', int8='int64', int16='int64', int32='int64', uint8='uint64', uint16='uint64', uint32='uint64', float16='float32', float32='float64', complex64='complex128' ).get(dtype, dtype) if 'complex' in dtype: continue f = theano.function([x], p) data = np.random.rand(2, 3) * 3 data = data.astype(dtype) f(data) @attr('slow') def test_prod_without_zeros_custom_dtype(self): # Test ability to provide your own output dtype for a ProdWithoutZeros(). # We try multiple axis combinations even though axis should not matter. axes = [None, 0, 1, [], [0], [1], [0, 1]] idx = 0 for input_dtype in imap(str, theano.scalar.all_types): x = tensor.matrix(dtype=input_dtype) for output_dtype in imap(str, theano.scalar.all_types): axis = axes[idx % len(axes)] prod_woz_var = ProdWithoutZeros( axis=axis, dtype=output_dtype)(x) assert prod_woz_var.dtype == output_dtype idx += 1 if ('complex' in output_dtype or 'complex' in input_dtype): continue f = theano.function([x], prod_woz_var) data = np.random.rand(2, 3) * 3 data = data.astype(input_dtype) f(data) @attr('slow') def test_prod_without_zeros_custom_acc_dtype(self): # Test ability to provide your own acc_dtype for a ProdWithoutZeros(). # We try multiple axis combinations even though axis should not matter. axes = [None, 0, 1, [], [0], [1], [0, 1]] idx = 0 for input_dtype in imap(str, theano.scalar.all_types): x = tensor.matrix(dtype=input_dtype) for acc_dtype in imap(str, theano.scalar.all_types): axis = axes[idx % len(axes)] # If acc_dtype would force a downcast, we expect a TypeError # We always allow int/uint inputs with float/complex outputs. upcasted_dtype = scalar.upcast(input_dtype, acc_dtype) if (acc_dtype == upcasted_dtype or (input_dtype in tensor.discrete_dtypes and acc_dtype in tensor.continuous_dtypes)): prod_woz_var = ProdWithoutZeros( axis=axis, acc_dtype=acc_dtype)(x) assert prod_woz_var.owner.op.acc_dtype == acc_dtype if (acc_dtype.startswith('complex') and input_dtype != acc_dtype): continue f = theano.function([x], prod_woz_var) data = np.random.rand(2, 3) * 3 data = data.astype(input_dtype) f(data) else: self.assertRaises( TypeError, ProdWithoutZeros(axis=axis, acc_dtype=acc_dtype), x) idx += 1 class TestBitOpReduceGrad(unittest.TestCase): def setUp(self): self.rng = np.random.RandomState(unittest_tools.fetch_seed()) def test_all_grad(self): x = tensor.bmatrix('x') x_all = x.all() gx = theano.grad(x_all, x) f = theano.function([x], gx) x_random = self.rng.binomial(n=1, p=0.5, size=(5, 7)).astype('int8') for x_val in (x_random, np.zeros_like(x_random), np.ones_like(x_random)): gx_val = f(x_val) assert gx_val.shape == x_val.shape assert np.all(gx_val == 0) def test_any_grad(self): x = tensor.bmatrix('x') x_all = x.any() gx = theano.grad(x_all, x) f = theano.function([x], gx) x_random = self.rng.binomial(n=1, p=0.5, size=(5, 7)).astype('int8') for x_val in (x_random, np.zeros_like(x_random), np.ones_like(x_random)): gx_val = f(x_val) assert gx_val.shape == x_val.shape assert np.all(gx_val == 0) class TestElemwise(unittest_tools.InferShapeTester): def test_elemwise_grad_bool(self): x = theano.tensor.scalar(dtype='bool') y = theano.tensor.bscalar() z = x * y dx, dy = theano.grad(z, [x, y]) def test_infer_shape(self): for s_left, s_right in [ ((5, 6), (5, 6)), ((5, 6), (5, 1)), ((5, 6), (1, 6)), ((5, 1), (5, 6)), ((1, 6), (5, 6)), ((2, 3, 4, 5), (2, 3, 4, 5)), ((2, 3, 4, 5), (2, 3, 1, 5)), ((2, 3, 4, 5), (1, 3, 4, 5)), ((2, 1, 4, 5), (2, 3, 4, 5)), ((2, 3, 4, 1), (2, 3, 4, 5))]: dtype = theano.config.floatX t_left = TensorType(dtype, [(entry == 1) for entry in s_left])() t_right = TensorType(dtype, [(entry == 1) for entry in s_right])() t_left_val = np.zeros(s_left, dtype=dtype) t_right_val = np.zeros(s_right, dtype=dtype) self._compile_and_check( [t_left, t_right], [Elemwise(scalar.add)(t_left, t_right)], [t_left_val, t_right_val], Elemwise) def test_input_dimensions_overflow(self): # Elemwise.perform used to compute the product # of input shapes to check if there was a zero in them, # it overflowed in this case. a, b, c, d, e, f = tensor.vectors('abcdef') s = a + b + c + d + e + f g = theano.function([a, b, c, d, e, f], s, mode=theano.compile.Mode(linker='py')) g(*[np.zeros(2 ** 11, config.floatX) for i in xrange(6)]) def test_gt_grad(): # A user test that failed. # Something about it made Elemwise.grad return something that was # too complicated for get_scalar_constant_value to recognize as being 0, so # gradient.grad reported that it was not a valid gradient of an # integer. floatX = config.floatX T = theano.tensor input_ = T.vector(dtype=floatX) random_values = np.random.RandomState(1234).uniform( low=-1, high=1, size=(2, 2)) W_values = np.asarray(random_values, dtype=floatX) W = theano.shared(value=W_values, name='weights') correct_score = T.dot(input_, W) wrong_input = T.vector(dtype=floatX) wrong_score = theano.clone(correct_score, {input_: wrong_input}) # Hinge loss scores = T.ones_like(correct_score) - correct_score + wrong_score cost = (scores * (scores > 0)).sum() T.grad(cost, input_) def test_clip_grad(): # test the gradient of clip def func(x, y, z): return theano.tensor.clip(x, y, z) # use an x value less than y, an x value between y and z, and an x value # greater than z unittest_tools.verify_grad(func, [np.asarray([-1., 0.5, 2.]), 0., 1.]) def test_grad_useless_sum(): # Test absence of useless sum. # # When an operation (such as T.mul) is done on a broadcastable vector and # a matrix, the gradient in backward path is computed for the broadcasted # vector. So a sum reverts the broadcasted vector to a vector. In the case # of operations on two broadcastable vectors, the sum should not be generated. # # This test checks whether there is a useless sum in the gradient # computations. mode = theano.compile.get_default_mode().including('canonicalize') mode.check_isfinite = False x = TensorType(theano.config.floatX, (True,))('x') l = tensor.log(1.0 - tensor.nnet.sigmoid(x))[0] g = tensor.grad(l, x) nodes = theano.gof.graph.ops([x], [g]) f = theano.function([x], g, mode=mode) test_values = [-100, -1, 0, 1, 100] outputs = [] old_values_eq_approx = staticmethod(TensorType.values_eq_approx) TensorType.values_eq_approx = staticmethod( tensor.type.values_eq_approx_remove_nan) try: for test_value in test_values: outputs.append(f(np.array([test_value]).astype('float32'))) finally: TensorType.values_eq_approx = old_values_eq_approx assert not any([isinstance(node.op, theano.tensor.elemwise.Sum) for node in nodes]) assert np.allclose(outputs, [[-3.72007598e-44], [-0.26894142], [-0.5], [-0.73105858], [-1.]]) def test_elemwise_grad_broadcast(): # This crashed in the past. x = tensor.tensor(dtype='float32', broadcastable=(True, False, False, False)) y = tensor.tensor(dtype='float32', broadcastable=(True, True, False, False)) theano.grad(theano.tensor.tanh(x).sum(), x) theano.grad(theano.tensor.tanh(x + y).sum(), y) theano.grad(theano.tensor.tanh(x + y).sum(), [x, y]) def test_clip_grad_int(): # test that integers don't crash clip gradient x = tensor.iscalar() y = tensor.iscalar() z = tensor.iscalar() c = tensor.clip(x, y, z) tensor.grad(c, [x, y, z]) def test_not_implemented_elemwise_grad(): # Regression test for unimplemented gradient in an Elemwise Op. class TestOp(scalar.ScalarOp): def __init__(self): self.output_types_preference = scalar.upgrade_to_float def impl(self, n, x): return x * n def grad(self, inputs, gout): (n, x) = inputs (gz,) = gout dy_dx = n return [theano.gradient.grad_not_implemented(self, 0, n), gz * dy_dx] test_op = tensor.Elemwise(TestOp()) x = tensor.scalar() # The call to `grad` used to crash. tensor.grad(test_op(2, x), x) # Verify that trying to use the not implemented gradient fails. try: tensor.grad(test_op(x, 2), x) assert False except theano.gradient.NullTypeGradError: pass if __name__ == '__main__': t = TestElemwise('setUp') t.setUp() t.test_infer_shape() Theano-1.0.0/theano/tensor/tests/test_utils.py00006000163562€ÿÿÿþ0000000472413203335532023001 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import unittest import numpy as np import theano from theano.tensor.utils import (hash_from_ndarray, shape_of_variables) def test_hash_from_ndarray(): hashs = [] rng = np.random.rand(5, 5) for data in [-2, -1, 0, 1, 2, np.zeros((1, 5)), np.zeros((1, 6)), # Data buffer empty but different shapes np.zeros((1, 0)), np.zeros((2, 0)), # Same data buffer and shapes but different strides np.arange(25).reshape(5, 5), np.arange(25).reshape(5, 5).T, # Same data buffer, shapes and strides but different dtypes np.zeros((5, 5), dtype="uint32"), np.zeros((5, 5), dtype="int32"), # Test slice rng, rng[1:], rng[:4], rng[1:3], rng[::2], rng[::-1] ]: data = np.asarray(data) hashs.append(hash_from_ndarray(data)) assert len(set(hashs)) == len(hashs) # test that different type of views and their copy give the same hash assert hash_from_ndarray(rng[1:]) == hash_from_ndarray(rng[1:].copy()) assert hash_from_ndarray(rng[1:3]) == hash_from_ndarray(rng[1:3].copy()) assert hash_from_ndarray(rng[:4]) == hash_from_ndarray(rng[:4].copy()) assert hash_from_ndarray(rng[::2]) == hash_from_ndarray(rng[::2].copy()) assert hash_from_ndarray(rng[::-1]) == hash_from_ndarray(rng[::-1].copy()) class Tshape_of_variables(unittest.TestCase): def test_simple(self): x = theano.tensor.matrix('x') y = x+x fgraph = theano.FunctionGraph([x], [y], clone=False) shapes = shape_of_variables(fgraph, {x: (5, 5)}) assert shapes == {x: (5, 5), y: (5, 5)} x = theano.tensor.matrix('x') y = theano.tensor.dot(x, x.T) fgraph = theano.FunctionGraph([x], [y], clone=False) shapes = shape_of_variables(fgraph, {x: (5, 1)}) assert shapes[x] == (5, 1) assert shapes[y] == (5, 5) def test_subtensor(self): x = theano.tensor.matrix('x') subx = x[1:] fgraph = theano.FunctionGraph([x], [subx], clone=False) shapes = shape_of_variables(fgraph, {x: (10, 10)}) assert shapes[subx] == (9, 10) def test_err(self): x = theano.tensor.matrix('x') subx = x[1:] fgraph = theano.FunctionGraph([x], [subx]) self.assertRaises(ValueError, shape_of_variables, fgraph, {x: (10, 10)}) Theano-1.0.0/theano/tensor/tests/test_sharedvar.py00006000163562€ÿÿÿþ0000006546113203335532023625 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import six import numpy as np import unittest from functools import update_wrapper import theano from theano import tensor from theano.tests import unittest_tools as utt from theano.misc.may_share_memory import may_share_memory import theano.sparse utt.seed_rng() def makeSharedTester(shared_constructor_, dtype_, get_value_borrow_true_alias_, shared_borrow_true_alias_, set_value_borrow_true_alias_, set_value_inplace_, set_cast_value_inplace_, shared_constructor_accept_ndarray_, internal_type_, test_internal_type_, theano_fct_, ref_fct_, cast_value_=np.asarray, expect_fail_fast_shape_inplace=True, ): """ This is a generic fct to allow reusing the same test function for many shared variable of many types. :param shared_constructor_: The shared variable constructor to use :param dtype_: The dtype of the data to test :param get_value_borrow_true_alias_: Should a get_value(borrow=True) return the internal object :param shared_borrow_true_alias_: Should shared(val,borrow=True) reuse the val memory space :param set_value_borrow_true_alias_: Should set_value(val,borrow=True) reuse the val memory space :param set_value_inplace_: Should this shared variable overwrite the current memory when the new value is an ndarray :param set_cast_value_inplace_: Should this shared variable overwrite the current memory when the new value is of the same type as the internal type. :param shared_constructor_accept_ndarray_: Do the shared_constructor accept an ndarray as input? :param internal_type_: The internal type used. :param test_internal_type_: A function that tell if its input is of the same type as this shared variable internal type. :param theano_fct_: A theano op that will be used to do some computation on the shared variable :param ref_fct_: A reference function that should return the same value as the theano_fct_ :param cast_value_: A callable that cast an ndarray into the internal shared variable representation :param name: This string is used to set the returned class' __name__ attribute. This is needed for nosetests to properly tag the test with its correct name, rather than use the generic SharedTester name. This parameter is mandatory (keeping the default None value will raise an error), and must be set to the name of the variable that will hold the returned class. :note: We must use /= as sparse type don't support other inplace operation. """ class m(type): pass class SharedTester(unittest.TestCase): # This is to allow setting __doc__ with python 2 if not six.PY3: __metaclass__ = m shared_constructor = staticmethod(shared_constructor_) dtype = dtype_ get_value_borrow_true_alias = get_value_borrow_true_alias_ shared_borrow_true_alias = shared_borrow_true_alias_ internal_type = internal_type_ test_internal_type = staticmethod(test_internal_type_) theano_fct = staticmethod(theano_fct_) ref_fct = staticmethod(ref_fct_) set_value_borrow_true_alias = set_value_borrow_true_alias_ set_value_inplace = set_value_inplace_ set_cast_value_inplace = set_cast_value_inplace_ shared_constructor_accept_ndarray = shared_constructor_accept_ndarray_ cast_value = staticmethod(cast_value_) def test_shared_dont_alias(self): dtype = self.dtype if dtype is None: dtype = theano.config.floatX rng = np.random.RandomState(utt.fetch_seed()) x = np.asarray(rng.uniform(0, 1, [2, 4]), dtype=dtype) x = self.cast_value(x) x_ref = self.ref_fct(x) x_shared = self.shared_constructor(x, borrow=False) total = self.theano_fct(x_shared) total_func = theano.function([], total) total_val = total_func() assert np.allclose(self.ref_fct(x), total_val) x /= .5 total_val_2 = total_func() # value used to construct should not alias with internal assert np.allclose(total_val, total_val_2) x = x_shared.get_value(borrow=False) x /= .5 total_val_3 = total_func() # value returned by access should not alias with internal assert np.allclose(total_val, total_val_3) # in this case we can alias x = x_shared.get_value(borrow=True) x /= .5 # this is not required by the contract but it is a feature we've # implemented for some type of SharedVariable. if self.get_value_borrow_true_alias: assert np.allclose(self.ref_fct(x), total_func()) else: assert np.allclose(x_ref, total_func()) def test_shape(self): dtype = self.dtype if dtype is None: dtype = theano.config.floatX rng = np.random.RandomState(utt.fetch_seed()) x = np.asarray(rng.uniform(0, 1, [2, 4]), dtype=dtype) x = self.cast_value(x) x_ref = self.ref_fct(x) x_shared = self.shared_constructor(x, borrow=False) total = self.theano_fct(x_shared) f = theano.function([], x_shared.shape) topo = f.maker.fgraph.toposort() assert np.all(f() == (2, 4)) if theano.config.mode != 'FAST_COMPILE': assert len(topo) == 3 assert isinstance(topo[0].op, tensor.opt.Shape_i) assert isinstance(topo[1].op, tensor.opt.Shape_i) assert isinstance(topo[2].op, tensor.opt.MakeVector) def test_shape_i(self): dtype = self.dtype if dtype is None: dtype = theano.config.floatX rng = np.random.RandomState(utt.fetch_seed()) x = np.asarray(rng.uniform(0, 1, [2, 4]), dtype=dtype) x = self.cast_value(x) x_ref = self.ref_fct(x) x_shared = self.shared_constructor(x, borrow=False) total = self.theano_fct(x_shared) f = theano.function([], x_shared.shape[1]) topo = f.maker.fgraph.toposort() assert np.all(f() == (4)) if theano.config.mode != 'FAST_COMPILE': assert len(topo) == 1 assert isinstance(topo[0].op, tensor.opt.Shape_i) def test_return_internal_type(self): dtype = self.dtype if dtype is None: dtype = theano.config.floatX rng = np.random.RandomState(utt.fetch_seed()) x = np.asarray(rng.uniform(0, 1, [2, 4]), dtype=dtype) x = self.cast_value(x) x_ref = self.ref_fct(x) x_shared = self.shared_constructor(x, borrow=False) total = self.theano_fct(x_shared) total_func = theano.function([], total) # in this case we can alias with the internal value x = x_shared.get_value(borrow=True, return_internal_type=True) assert self.test_internal_type(x) x /= .5 # this is not required by the contract but it is a feature we can # implement for some type of SharedVariable. assert np.allclose(self.ref_fct(x), total_func()) x = x_shared.get_value(borrow=False, return_internal_type=True) assert self.test_internal_type(x) assert x is not x_shared.container.value x /= .5 # this is required by the contract assert not np.allclose(self.ref_fct(x), total_func()) def test_get_value(self): # Test that get_value returns a ndarray dtype = self.dtype if dtype is None: dtype = theano.config.floatX rng = np.random.RandomState(utt.fetch_seed()) x_orig = np.asarray(rng.uniform(0, 1, [2, 4]), dtype=dtype) x_cast = self.cast_value(x_orig) if self.shared_constructor_accept_ndarray: x_shared = self.shared_constructor(x_orig, borrow=False) assert isinstance(x_shared.get_value(), x_orig.__class__) x_shared = self.shared_constructor(x_cast, borrow=False) assert isinstance(x_shared.get_value(), x_cast.__class__) def test_set_value(self): dtype = self.dtype if dtype is None: dtype = theano.config.floatX rng = np.random.RandomState(utt.fetch_seed()) x = np.asarray(rng.uniform(0, 1, [2, 4]), dtype=dtype) x = self.cast_value(x) x_orig = x x_orig_copy = x.copy() x_ref = self.ref_fct(x) x_shared = self.shared_constructor(x, borrow=False) total = self.theano_fct(x_shared) total_func = theano.function([], total) total_func() # test if that theano shared variable optimize set_value(borrow=True) get_x = x_shared.get_value(borrow=True) assert get_x is not x_orig # borrow=False to shared_constructor get_x /= .5 x_shared.set_value(get_x, borrow=True) x = x_shared.get_value(borrow=True) if self.set_value_borrow_true_alias: assert x is get_x else: assert x is not get_x assert np.allclose(self.ref_fct(np.asarray(x_orig)/.5), self.ref_fct(x)) # test optimized get set value on the gpu(don't pass data to the cpu) get_x = x_shared.get_value(borrow=True, return_internal_type=True) assert get_x is not x_orig # borrow=False to shared_constructor assert self.test_internal_type(get_x) get_x /= .5 assert self.test_internal_type(get_x) x_shared.set_value(get_x, borrow=True) x = x_shared.get_value(borrow=True, return_internal_type=True) assert self.test_internal_type(x) assert x is get_x # TODO test Out. def test_shared_do_alias(self): dtype = self.dtype if dtype is None: dtype = theano.config.floatX rng = np.random.RandomState(utt.fetch_seed()) x = np.asarray(rng.uniform(1, 2, [4, 2]), dtype=dtype) x = self.cast_value(x) x_ref = self.ref_fct(x) x_shared = self.shared_constructor(x, borrow=True) total = self.theano_fct(x_shared) total_func = theano.function([], total) total_val = total_func() assert np.allclose(self.ref_fct(x), total_val) x /= .5 # not required by the contract but it is a feature we've implemented if self.shared_borrow_true_alias: assert np.allclose(self.ref_fct(x), total_func()) else: assert np.allclose(x_ref, total_func()) def test_inplace_set_value(self): # We test that if the SharedVariable implement it we do inplace set_value # We also test this for partial inplace modification when accessing the internal of theano. dtype = self.dtype if dtype is None: dtype = theano.config.floatX shp = (100//4, 1024) # 100KB x = np.zeros(shp, dtype=dtype) x = self.cast_value(x) x_shared = self.shared_constructor(x, borrow=True) old_data = x_shared.container.storage[0] nd = np.ones(shp, dtype=dtype) if x.__class__.__name__ != 'csr_matrix': # sparse matrix don't support inplace affectation x_shared.container.value[:] = nd assert (np.asarray(x_shared.get_value(borrow=True)) == nd).all() # This should always share value! assert may_share_memory(old_data, x_shared.container.storage[0]) assert may_share_memory(old_data, x_shared.get_value(borrow=True, return_internal_type=True)) nd[0] += 1 x_shared.container.value[0] = nd[0] assert (np.asarray(x_shared.get_value(borrow=True)[0]) == nd[0]).all() assert (np.asarray(x_shared.get_value(borrow=True)[1:]) == nd[1:]).all() # This should always share value! assert may_share_memory(old_data, x_shared.container.storage[0]) assert may_share_memory(old_data, x_shared.get_value(borrow=True, return_internal_type=True)) if x.__class__.__name__ != 'csr_matrix': # sparse matrix don't support inplace affectation nd += 1 # THIS DOENS'T DO WHAT WE EXPECT the content of a is # not updated for GpuArray, but it is for ndarray x_shared.get_value(borrow=True)[:] = nd assert may_share_memory(old_data, x_shared.container.storage[0]) x_shared.get_value(borrow=True) # Test by set_value with borrow=False nd += 1 old_data = x_shared.container.storage[0] x_shared.set_value(nd, borrow=False) assert np.allclose(self.ref_fct(x_shared.get_value(borrow=True)), self.ref_fct(self.cast_value(nd))) assert may_share_memory(old_data, x_shared.container.storage[0]) == self.set_value_inplace # Test by set_value with borrow=False when new data cast. # specificaly useful for gpu data nd += 1 old_data = x_shared.container.storage[0] x_shared.set_value(self.cast_value(nd), borrow=False) assert np.allclose(self.ref_fct(x_shared.get_value(borrow=True)), self.ref_fct(self.cast_value(nd))) assert may_share_memory(old_data, x_shared.container.storage[0]) == self.set_cast_value_inplace # Test by set_value with borrow=True nd += 1 old_data = x_shared.container.storage[0] x_shared.set_value(nd.copy(), borrow=True) assert np.allclose(self.ref_fct(x_shared.get_value(borrow=True)), self.ref_fct(self.cast_value(nd))) assert may_share_memory(old_data, x_shared.container.storage[0]) == self.set_value_inplace # Test by set_value with borrow=True when new data cast. nd += 1 old_data = x_shared.container.storage[0] x_shared.set_value(self.cast_value(nd.copy()), borrow=True) assert np.allclose(self.ref_fct(x_shared.get_value(borrow=True)), self.ref_fct(self.cast_value(nd))) assert may_share_memory(old_data, x_shared.container.storage[0]) == self.set_cast_value_inplace def test_specify_shape(self): dtype = self.dtype if dtype is None: dtype = theano.config.floatX rng = np.random.RandomState(utt.fetch_seed()) x1_1 = np.asarray(rng.uniform(1, 2, [4, 2]), dtype=dtype) x1_1 = self.cast_value(x1_1) x1_2 = np.asarray(rng.uniform(1, 2, [4, 2]), dtype=dtype) x1_2 = self.cast_value(x1_2) x2 = np.asarray(rng.uniform(1, 2, [4, 3]), dtype=dtype) x2 = self.cast_value(x2) # Test that we can replace with values of the same shape x1_shared = self.shared_constructor(x1_1) x1_specify_shape = tensor.specify_shape(x1_shared, x1_1.shape) x1_shared.set_value(x1_2) assert np.allclose(self.ref_fct(x1_shared.get_value(borrow=True)), self.ref_fct( x1_2)) shape_op_fct = theano.function([], x1_shared.shape) topo = shape_op_fct.maker.fgraph.toposort() if theano.config.mode != 'FAST_COMPILE': assert len(topo) == 3 assert isinstance(topo[0].op, tensor.opt.Shape_i) assert isinstance(topo[1].op, tensor.opt.Shape_i) assert isinstance(topo[2].op, tensor.opt.MakeVector) # Test that we forward the input specify_shape_fct = theano.function([], x1_specify_shape) assert np.all(self.ref_fct(specify_shape_fct()) == self.ref_fct(x1_2)) topo_specify = specify_shape_fct.maker.fgraph.toposort() assert len(topo_specify) == 2 # Test that we put the shape info into the graph shape_constant_fct = theano.function([], x1_specify_shape.shape) assert np.all(shape_constant_fct() == shape_op_fct()) topo_cst = shape_constant_fct.maker.fgraph.toposort() if theano.config.mode != 'FAST_COMPILE': assert len(topo_cst) == 1 topo_cst[0].op == theano.compile.function_module.deep_copy_op # Test that we can take the grad. if (theano.sparse.enable_sparse and isinstance(x1_specify_shape.type, theano.sparse.SparseType)): # SparseVariable don't support sum for now. assert not hasattr(x1_specify_shape, 'sum') else: shape_grad = tensor.grad(x1_specify_shape.sum(), x1_shared) shape_constant_fct_grad = theano.function([], shape_grad) # theano.printing.debugprint(shape_constant_fct_grad) shape_constant_fct_grad() # Test that we can replace with values of the different shape # but that will raise an error in some case, but not all specify_shape_fct() x1_shared.set_value(x2) self.assertRaises(AssertionError, specify_shape_fct) # No assertion will be raised as the Op is removed from the graph # when their is optimization if theano.config.mode not in ['FAST_COMPILE', 'DebugMode', 'DEBUG_MODE']: shape_constant_fct() else: self.assertRaises(AssertionError, shape_constant_fct) def test_specify_shape_partial(self): dtype = self.dtype if dtype is None: dtype = theano.config.floatX rng = np.random.RandomState(utt.fetch_seed()) x1_1 = np.asarray(rng.uniform(1, 2, [4, 2]), dtype=dtype) x1_1 = self.cast_value(x1_1) x1_2 = np.asarray(rng.uniform(1, 2, [4, 2]), dtype=dtype) x1_2 = self.cast_value(x1_2) x2 = np.asarray(rng.uniform(1, 2, [5, 2]), dtype=dtype) x2 = self.cast_value(x2) # Test that we can replace with values of the same shape x1_shared = self.shared_constructor(x1_1) x1_specify_shape = tensor.specify_shape(x1_shared, (tensor.as_tensor_variable(x1_1.shape[0]), x1_shared.shape[1])) x1_shared.set_value(x1_2) assert np.allclose( self.ref_fct(x1_shared.get_value(borrow=True)), self.ref_fct( x1_2)) shape_op_fct = theano.function([], x1_shared.shape) topo = shape_op_fct.maker.fgraph.toposort() shape_op_fct() if theano.config.mode != 'FAST_COMPILE': assert len(topo) == 3 assert isinstance(topo[0].op, tensor.opt.Shape_i) assert isinstance(topo[1].op, tensor.opt.Shape_i) assert isinstance(topo[2].op, tensor.opt.MakeVector) # Test that we forward the input specify_shape_fct = theano.function([], x1_specify_shape) specify_shape_fct() # theano.printing.debugprint(specify_shape_fct) assert np.all(self.ref_fct(specify_shape_fct()) == self.ref_fct(x1_2)) topo_specify = specify_shape_fct.maker.fgraph.toposort() if theano.config.mode != 'FAST_COMPILE': assert len(topo_specify) == 4 # Test that we put the shape info into the graph shape_constant_fct = theano.function([], x1_specify_shape.shape) # theano.printing.debugprint(shape_constant_fct) assert np.all(shape_constant_fct() == shape_op_fct()) topo_cst = shape_constant_fct.maker.fgraph.toposort() if theano.config.mode != 'FAST_COMPILE': assert len(topo_cst) == 2 # Test that we can replace with values of the different shape # but that will raise an error in some case, but not all x1_shared.set_value(x2) self.assertRaises(AssertionError, specify_shape_fct) # No assertion will be raised as the Op is removed from the graph if theano.config.mode not in ['FAST_COMPILE', 'DebugMode', 'DEBUG_MODE']: shape_constant_fct() else: self.assertRaises(AssertionError, shape_constant_fct) def test_specify_shape_inplace(self): # test that specify_shape don't break inserting inplace op dtype = self.dtype if dtype is None: dtype = theano.config.floatX rng = np.random.RandomState(utt.fetch_seed()) a = np.asarray(rng.uniform(1, 2, [40, 40]), dtype=dtype) a = self.cast_value(a) a_shared = self.shared_constructor(a) b = np.asarray(rng.uniform(1, 2, [40, 40]), dtype=dtype) b = self.cast_value(b) b_shared = self.shared_constructor(b) s = np.zeros((40, 40), dtype=dtype) s = self.cast_value(s) s_shared = self.shared_constructor(s) f = theano.function([], updates=[(s_shared, theano.dot(a_shared, b_shared) + s_shared)]) topo = f.maker.fgraph.toposort() f() # [Gemm{inplace}(, 0.01, , , 2e-06)] if theano.config.mode != 'FAST_COMPILE': assert sum([node.op.__class__.__name__ in ["Gemm", "GpuGemm", "StructuredDot"] for node in topo]) == 1 assert all(node.op == tensor.blas.gemm_inplace for node in topo if isinstance(node.op, tensor.blas.Gemm)) assert all(node.op.inplace for node in topo if node.op.__class__.__name__ == "GpuGemm") # Their is no inplace gemm for sparse # assert all(node.op.inplace for node in topo if node.op.__class__.__name__ == "StructuredDot") s_shared_specify = tensor.specify_shape(s_shared, s_shared.get_value(borrow=True).shape) # now test with the specify shape op in the output f = theano.function([], s_shared.shape, updates=[(s_shared, theano.dot(a_shared, b_shared) + s_shared_specify)]) topo = f.maker.fgraph.toposort() shp = f() assert np.all(shp == (40, 40)) if theano.config.mode != 'FAST_COMPILE': assert sum([node.op.__class__.__name__ in ["Gemm", "GpuGemm", "StructuredDot"] for node in topo]) == 1 assert all(node.op == tensor.blas.gemm_inplace for node in topo if isinstance(node.op, tensor.blas.Gemm)) assert all(node.op.inplace for node in topo if node.op.__class__.__name__ == "GpuGemm") # now test with the specify shape op in the inputs and outputs a_shared = tensor.specify_shape(a_shared, a_shared.get_value(borrow=True).shape) b_shared = tensor.specify_shape(b_shared, b_shared.get_value(borrow=True).shape) f = theano.function([], s_shared.shape, updates=[(s_shared, theano.dot(a_shared, b_shared) + s_shared_specify)]) topo = f.maker.fgraph.toposort() shp = f() assert np.all(shp == (40, 40)) if theano.config.mode != 'FAST_COMPILE': assert sum([node.op.__class__.__name__ in ["Gemm", "GpuGemm", "StructuredDot"] for node in topo]) == 1 assert all(node.op == tensor.blas.gemm_inplace for node in topo if isinstance(node.op, tensor.blas.Gemm)) assert all(node.op.inplace for node in topo if node.op.__class__.__name__ == "GpuGemm") if theano.config.cycle_detection == 'fast' and expect_fail_fast_shape_inplace and theano.config.mode != 'FAST_COMPILE': test_specify_shape_inplace = unittest.expectedFailure(test_specify_shape_inplace) def test_values_eq(self): # Test the type.values_eq[_approx] function dtype = self.dtype if dtype is None: dtype = theano.config.floatX # We need big shape as in the past there have been a bug in the # sparse values_eq_approx. shp = (1024, 1024) # Test the case with all zeros element rng = np.random.RandomState(utt.fetch_seed()) for x in [np.asarray(rng.rand(*shp), dtype=dtype), np.zeros(shp, dtype=dtype)]: zeros = (x == 0).all() x = self.cast_value(x) x_shared = self.shared_constructor(x, borrow=True) y = x.copy() y[0, 0], y[1, 0] = y[1, 0], y[0, 0] y = self.cast_value(y) assert x_shared.type.values_eq(x, x) assert x_shared.type.values_eq_approx(x, x) if not zeros: assert not np.allclose(self.ref_fct(x), self.ref_fct(y)) assert not x_shared.type.values_eq(x, y) assert not x_shared.type.values_eq_approx(x, y) def f(cls): return update_wrapper(SharedTester, cls, updated=()) return f @makeSharedTester( shared_constructor_=tensor._shared, dtype_=theano.config.floatX, get_value_borrow_true_alias_=True, shared_borrow_true_alias_=True, set_value_borrow_true_alias_=True, set_value_inplace_=False, set_cast_value_inplace_=False, shared_constructor_accept_ndarray_=True, internal_type_=np.ndarray, test_internal_type_=lambda a: isinstance(a, np.ndarray), theano_fct_=lambda a: a*2, ref_fct_=lambda a: np.asarray((a*2)), cast_value_=np.asarray) class test_shared_options(object): pass def test_scalar_shared_options(): # Simple test to make sure we do not loose that fonctionality. theano.shared(value=0., name='lk', borrow=True) theano.shared(value=np.float32(0.), name='lk', borrow=True) Theano-1.0.0/theano/tensor/tests/test_extra_ops.py00006000163562€ÿÿÿþ0000010660113203335532023642 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import numpy as np import theano from theano.tests import unittest_tools as utt from theano.tensor.extra_ops import (SearchsortedOp, searchsorted, CumOp, cumsum, cumprod, CpuContiguous, cpu_contiguous, bincount, DiffOp, diff, squeeze, compress, RepeatOp, repeat, Bartlett, bartlett, FillDiagonal, fill_diagonal, FillDiagonalOffset, fill_diagonal_offset, to_one_hot, Unique, unravel_index, UnravelIndex, ravel_multi_index, RavelMultiIndex) from theano import tensor as T from theano import config, tensor, function from theano.tests.unittest_tools import attr def test_cpu_contiguous(): a = T.fmatrix('a') i = T.iscalar('i') a_val = np.asarray(np.random.rand(4, 5), dtype='float32') f = theano.function([a, i], cpu_contiguous(a.reshape((5, 4))[::i])) topo = f.maker.fgraph.toposort() assert any([isinstance(node.op, CpuContiguous) for node in topo]) assert f(a_val, 1).flags['C_CONTIGUOUS'] assert f(a_val, 2).flags['C_CONTIGUOUS'] assert f(a_val, 3).flags['C_CONTIGUOUS'] # Test the grad: theano.tests.unittest_tools.verify_grad(cpu_contiguous, [np.random.rand(5, 7, 2)]) class TestSearchsortedOp(utt.InferShapeTester): def setUp(self): super(TestSearchsortedOp, self).setUp() self.op_class = SearchsortedOp self.op = SearchsortedOp() self.x = T.vector('x') self.v = T.tensor3('v') self.a = 30 * np.random.random(50).astype(config.floatX) self.b = 30 * np.random.random((8, 10, 5)).astype(config.floatX) self.idx_sorted = np.argsort(self.a).astype('int32') def test_searchsortedOp_on_sorted_input(self): f = theano.function([self.x, self.v], searchsorted(self.x, self.v)) assert np.allclose(np.searchsorted(self.a[self.idx_sorted], self.b), f(self.a[self.idx_sorted], self.b)) sorter = T.vector('sorter', dtype='int32') f = theano.function([self.x, self.v, sorter], self.x.searchsorted(self.v, sorter=sorter, side='right')) assert np.allclose(self.a.searchsorted(self.b, sorter=self.idx_sorted, side='right'), f(self.a, self.b, self.idx_sorted)) sa = self.a[self.idx_sorted] f = theano.function([self.x, self.v], self.x.searchsorted(self.v, side='right')) assert np.allclose(sa.searchsorted(self.b, side='right'), f(sa, self.b)) def test_searchsortedOp_wrong_side_kwd(self): self.assertRaises(ValueError, searchsorted, self.x, self.v, side='asdfa') def test_searchsortedOp_on_no_1d_inp(self): no_1d = T.dmatrix('no_1d') self.assertRaises(ValueError, searchsorted, no_1d, self.v) self.assertRaises(ValueError, searchsorted, self.x, self.v, sorter=no_1d) def test_searchsortedOp_on_float_sorter(self): sorter = T.vector('sorter', dtype="float32") self.assertRaises(TypeError, searchsorted, self.x, self.v, sorter=sorter) def test_searchsortedOp_on_int_sorter(self): compatible_types = ('int8', 'int16', 'int32') if theano.configdefaults.python_int_bitwidth() == 64: compatible_types += ('int64',) # 'uint8', 'uint16', 'uint32', 'uint64') for dtype in compatible_types: sorter = T.vector('sorter', dtype=dtype) f = theano.function([self.x, self.v, sorter], searchsorted(self.x, self.v, sorter=sorter), allow_input_downcast=True) assert np.allclose(np.searchsorted(self.a, self.b, sorter=self.idx_sorted), f(self.a, self.b, self.idx_sorted)) def test_searchsortedOp_on_right_side(self): f = theano.function([self.x, self.v], searchsorted(self.x, self.v, side='right')) assert np.allclose(np.searchsorted(self.a, self.b, side='right'), f(self.a, self.b)) def test_infer_shape(self): # Test using default parameters' value self._compile_and_check([self.x, self.v], [searchsorted(self.x, self.v)], [self.a[self.idx_sorted], self.b], self.op_class) # Test parameter ``sorter`` sorter = T.vector('sorter', dtype="int32") self._compile_and_check([self.x, self.v, sorter], [searchsorted(self.x, self.v, sorter=sorter)], [self.a, self.b, self.idx_sorted], self.op_class) # Test parameter ``side`` la = np.ones(10).astype(config.floatX) lb = np.ones(shape=(1, 2, 3)).astype(config.floatX) self._compile_and_check([self.x, self.v], [searchsorted(self.x, self.v, side='right')], [la, lb], self.op_class) def test_grad(self): utt.verify_grad(self.op, [self.a[self.idx_sorted], self.b]) class TestCumOp(utt.InferShapeTester): def setUp(self): super(TestCumOp, self).setUp() self.op_class = CumOp self.op = CumOp() def test_cum_op(self): x = T.tensor3('x') a = np.random.random((3, 5, 2)).astype(config.floatX) # Test axis out of bounds self.assertRaises(ValueError, cumsum, x, axis=3) self.assertRaises(ValueError, cumsum, x, axis=-4) self.assertRaises(ValueError, cumprod, x, axis=3) self.assertRaises(ValueError, cumprod, x, axis=-4) f = theano.function([x], [cumsum(x), cumprod(x)]) s, p = f(a) assert np.allclose(np.cumsum(a), s) # Test axis=None assert np.allclose(np.cumprod(a), p) # Test axis=None for axis in range(-len(a.shape), len(a.shape)): f = theano.function([x], [cumsum(x, axis=axis), cumprod(x, axis=axis)]) s, p = f(a) assert np.allclose(np.cumsum(a, axis=axis), s) assert np.allclose(np.cumprod(a, axis=axis), p) def test_infer_shape(self): x = T.tensor3('x') a = np.random.random((3, 5, 2)).astype(config.floatX) # Test axis=None self._compile_and_check([x], [self.op(x)], [a], self.op_class) for axis in range(-len(a.shape), len(a.shape)): self._compile_and_check([x], [cumsum(x, axis=axis)], [a], self.op_class) def test_grad(self): a = np.random.random((3, 5, 2)).astype(config.floatX) utt.verify_grad(self.op_class(mode='add'), [a]) # Test axis=None utt.verify_grad(self.op_class(mode='mul'), [a]) # Test axis=None for axis in range(-len(a.shape), len(a.shape)): utt.verify_grad(self.op_class(axis=axis, mode='add'), [a], eps=4e-4) utt.verify_grad(self.op_class(axis=axis, mode='mul'), [a], eps=4e-4) class TestBinCount(utt.InferShapeTester): def test_bincountFn(self): w = T.vector('w') def ref(data, w=None, minlength=None): size = int(data.max() + 1) if minlength: size = max(size, minlength) if w is not None: out = np.zeros(size, dtype=w.dtype) for i in range(data.shape[0]): out[data[i]] += w[i] else: out = np.zeros(size, dtype=a.dtype) for i in range(data.shape[0]): out[data[i]] += 1 return out for dtype in ('int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64'): x = T.vector('x', dtype=dtype) a = np.random.randint(1, 51, size=(25)).astype(dtype) weights = np.random.random((25,)).astype(config.floatX) f1 = theano.function([x], bincount(x)) f2 = theano.function([x, w], bincount(x, weights=w)) assert (ref(a) == f1(a)).all() assert np.allclose(ref(a, weights), f2(a, weights)) f3 = theano.function([x], bincount(x, minlength=55)) f4 = theano.function([x], bincount(x, minlength=5)) assert (ref(a, minlength=55) == f3(a)).all() assert (ref(a, minlength=5) == f4(a)).all() # skip the following test when using unsigned ints if not dtype.startswith('u'): a[0] = -1 f5 = theano.function([x], bincount(x, assert_nonneg=True)) self.assertRaises(AssertionError, f5, a) class TestDiffOp(utt.InferShapeTester): nb = 10 # Number of time iterating for n def setUp(self): super(TestDiffOp, self).setUp() self.op_class = DiffOp self.op = DiffOp() def test_diffOp(self): x = T.matrix('x') a = np.random.random((30, 50)).astype(config.floatX) f = theano.function([x], diff(x)) assert np.allclose(np.diff(a), f(a)) for axis in range(len(a.shape)): for k in range(TestDiffOp.nb): g = theano.function([x], diff(x, n=k, axis=axis)) assert np.allclose(np.diff(a, n=k, axis=axis), g(a)) def test_infer_shape(self): x = T.matrix('x') a = np.random.random((30, 50)).astype(config.floatX) self._compile_and_check([x], [self.op(x)], [a], self.op_class) for axis in range(len(a.shape)): for k in range(TestDiffOp.nb): self._compile_and_check([x], [diff(x, n=k, axis=axis)], [a], self.op_class) def test_grad(self): x = T.vector('x') a = np.random.random(50).astype(config.floatX) theano.function([x], T.grad(T.sum(diff(x)), x)) utt.verify_grad(self.op, [a]) for k in range(TestDiffOp.nb): theano.function([x], T.grad(T.sum(diff(x, n=k)), x)) utt.verify_grad(DiffOp(n=k), [a], eps=7e-3) class SqueezeTester(utt.InferShapeTester): shape_list = [(1, 3), (1, 2, 3), (1, 5, 1, 1, 6)] broadcast_list = [[True, False], [True, False, False], [True, False, True, True, False]] def setUp(self): super(SqueezeTester, self).setUp() self.op = squeeze def test_op(self): for shape, broadcast in zip(self.shape_list, self.broadcast_list): data = np.random.random(size=shape).astype(theano.config.floatX) variable = tensor.TensorType(theano.config.floatX, broadcast)() f = theano.function([variable], self.op(variable)) expected = np.squeeze(data) tested = f(data) assert tested.shape == expected.shape assert np.allclose(tested, expected) def test_infer_shape(self): for shape, broadcast in zip(self.shape_list, self.broadcast_list): data = np.random.random(size=shape).astype(theano.config.floatX) variable = tensor.TensorType(theano.config.floatX, broadcast)() self._compile_and_check([variable], [self.op(variable)], [data], tensor.DimShuffle, warn=False) def test_grad(self): for shape, broadcast in zip(self.shape_list, self.broadcast_list): data = np.random.random(size=shape).astype(theano.config.floatX) utt.verify_grad(self.op, [data]) def test_var_interface(self): # same as test_op, but use a_theano_var.squeeze. for shape, broadcast in zip(self.shape_list, self.broadcast_list): data = np.random.random(size=shape).astype(theano.config.floatX) variable = tensor.TensorType(theano.config.floatX, broadcast)() f = theano.function([variable], variable.squeeze()) expected = np.squeeze(data) tested = f(data) assert tested.shape == expected.shape assert np.allclose(tested, expected) class CompressTester(utt.InferShapeTester): axis_list = [None, -1, 0, 0, 0, 1] cond_list = [[1, 0, 1, 0, 0, 1], [0, 1, 1, 0], [0, 1, 1, 0], [], [0, 0, 0, 0], [1, 1, 0, 1, 0]] shape_list = [(2, 3), (4, 3), (4, 3), (4, 3), (4, 3), (3, 5)] def setUp(self): super(CompressTester, self).setUp() self.op = compress def test_op(self): for axis, cond, shape in zip(self.axis_list, self.cond_list, self.shape_list): cond_var = theano.tensor.ivector() data = np.random.random(size=shape).astype(theano.config.floatX) data_var = theano.tensor.matrix() f = theano.function([cond_var, data_var], self.op(cond_var, data_var, axis=axis)) expected = np.compress(cond, data, axis=axis) tested = f(cond, data) assert tested.shape == expected.shape assert np.allclose(tested, expected) class TestRepeatOp(utt.InferShapeTester): def _possible_axis(self, ndim): return [None] + list(range(ndim)) + [-i for i in range(ndim)] def setUp(self): super(TestRepeatOp, self).setUp() self.op_class = RepeatOp self.op = RepeatOp() # uint64 always fails # int64 and uint32 also fail if python int are 32-bit ptr_bitwidth = theano.configdefaults.local_bitwidth() if ptr_bitwidth == 64: self.numpy_unsupported_dtypes = ('uint64',) if ptr_bitwidth == 32: self.numpy_unsupported_dtypes = ('uint32', 'int64', 'uint64') def test_repeatOp(self): for ndim in [1, 3]: x = T.TensorType(config.floatX, [False] * ndim)() a = np.random.random((10, ) * ndim).astype(config.floatX) for axis in self._possible_axis(ndim): for dtype in tensor.integer_dtypes: r_var = T.scalar(dtype=dtype) r = np.asarray(3, dtype=dtype) if (dtype == 'uint64' or (dtype in self.numpy_unsupported_dtypes and r_var.ndim == 1)): self.assertRaises(TypeError, repeat, x, r_var, axis=axis) else: f = theano.function([x, r_var], repeat(x, r_var, axis=axis)) assert np.allclose(np.repeat(a, r, axis=axis), f(a, r)) r_var = T.vector(dtype=dtype) if axis is None: r = np.random.randint( 1, 6, size=a.size).astype(dtype) else: r = np.random.randint( 1, 6, size=(10,)).astype(dtype) if dtype in self.numpy_unsupported_dtypes and r_var.ndim == 1: self.assertRaises(TypeError, repeat, x, r_var, axis=axis) else: f = theano.function([x, r_var], repeat(x, r_var, axis=axis)) assert np.allclose(np.repeat(a, r, axis=axis), f(a, r)) # check when r is a list of single integer, e.g. [3]. r = np.random.randint( 1, 11, size=()).astype(dtype) + 2 f = theano.function([x], repeat(x, [r], axis=axis)) assert np.allclose(np.repeat(a, r, axis=axis), f(a)) assert not np.any([isinstance(n.op, RepeatOp) for n in f.maker.fgraph.toposort()]) # check when r is theano tensortype that broadcastable is (True,) r_var = theano.tensor.TensorType(broadcastable=(True,), dtype=dtype)() r = np.random.randint(1, 6, size=(1,)).astype(dtype) f = theano.function([x, r_var], repeat(x, r_var, axis=axis)) assert np.allclose(np.repeat(a, r[0], axis=axis), f(a, r)) assert not np.any([isinstance(n.op, RepeatOp) for n in f.maker.fgraph.toposort()]) @attr('slow') def test_infer_shape(self): for ndim in [1, 3]: x = T.TensorType(config.floatX, [False] * ndim)() shp = (np.arange(ndim) + 1) * 3 a = np.random.random(shp).astype(config.floatX) for axis in self._possible_axis(ndim): for dtype in ["int8", "uint8", "uint64"]: r_var = T.scalar(dtype=dtype) r = np.asarray(3, dtype=dtype) if dtype in self.numpy_unsupported_dtypes: r_var = T.vector(dtype=dtype) self.assertRaises(TypeError, repeat, x, r_var) else: self._compile_and_check([x, r_var], [RepeatOp(axis=axis)(x, r_var)], [a, r], self.op_class) r_var = T.vector(dtype=dtype) if axis is None: r = np.random.randint( 1, 6, size=a.size).astype(dtype) elif a.size > 0: r = np.random.randint( 1, 6, size=a.shape[axis]).astype(dtype) else: r = np.random.randint( 1, 6, size=(10,)).astype(dtype) self._compile_and_check( [x, r_var], [RepeatOp(axis=axis)(x, r_var)], [a, r], self.op_class) def test_grad(self): for ndim in range(3): a = np.random.random((10, ) * ndim).astype(config.floatX) for axis in self._possible_axis(ndim): utt.verify_grad(lambda x: RepeatOp(axis=axis)(x, 3), [a]) def test_broadcastable(self): x = T.TensorType(config.floatX, [False, True, False])() r = RepeatOp(axis=1)(x, 2) self.assertEqual(r.broadcastable, (False, False, False)) r = RepeatOp(axis=1)(x, 1) self.assertEqual(r.broadcastable, (False, True, False)) r = RepeatOp(axis=0)(x, 2) self.assertEqual(r.broadcastable, (False, True, False)) class TestBartlett(utt.InferShapeTester): def setUp(self): super(TestBartlett, self).setUp() self.op_class = Bartlett self.op = bartlett def test_perform(self): x = tensor.lscalar() f = function([x], self.op(x)) M = np.random.randint(3, 51, size=()) assert np.allclose(f(M), np.bartlett(M)) assert np.allclose(f(0), np.bartlett(0)) assert np.allclose(f(-1), np.bartlett(-1)) b = np.array([17], dtype='uint8') assert np.allclose(f(b[0]), np.bartlett(b[0])) def test_infer_shape(self): x = tensor.lscalar() self._compile_and_check([x], [self.op(x)], [np.random.randint(3, 51, size=())], self.op_class) self._compile_and_check([x], [self.op(x)], [0], self.op_class) self._compile_and_check([x], [self.op(x)], [1], self.op_class) class TestFillDiagonal(utt.InferShapeTester): rng = np.random.RandomState(43) def setUp(self): super(TestFillDiagonal, self).setUp() self.op_class = FillDiagonal self.op = fill_diagonal def test_perform(self): x = tensor.matrix() y = tensor.scalar() f = function([x, y], fill_diagonal(x, y)) for shp in [(8, 8), (5, 8), (8, 5)]: a = np.random.rand(*shp).astype(config.floatX) val = np.cast[config.floatX](np.random.rand()) out = f(a, val) # We can't use np.fill_diagonal as it is bugged. assert np.allclose(np.diag(out), val) assert (out == val).sum() == min(a.shape) # test for 3d tensor a = np.random.rand(3, 3, 3).astype(config.floatX) x = tensor.tensor3() y = tensor.scalar() f = function([x, y], fill_diagonal(x, y)) val = np.cast[config.floatX](np.random.rand() + 10) out = f(a, val) # We can't use np.fill_diagonal as it is bugged. assert out[0, 0, 0] == val assert out[1, 1, 1] == val assert out[2, 2, 2] == val assert (out == val).sum() == min(a.shape) @attr('slow') def test_gradient(self): utt.verify_grad(fill_diagonal, [np.random.rand(5, 8), np.random.rand()], n_tests=1, rng=TestFillDiagonal.rng) utt.verify_grad(fill_diagonal, [np.random.rand(8, 5), np.random.rand()], n_tests=1, rng=TestFillDiagonal.rng) def test_infer_shape(self): z = tensor.dtensor3() x = tensor.dmatrix() y = tensor.dscalar() self._compile_and_check([x, y], [self.op(x, y)], [np.random.rand(8, 5), np.random.rand()], self.op_class) self._compile_and_check([z, y], [self.op(z, y)], # must be square when nd>2 [np.random.rand(8, 8, 8), np.random.rand()], self.op_class, warn=False) class TestFillDiagonalOffset(utt.InferShapeTester): rng = np.random.RandomState(43) def setUp(self): super(TestFillDiagonalOffset, self).setUp() self.op_class = FillDiagonalOffset self.op = fill_diagonal_offset def test_perform(self): x = tensor.matrix() y = tensor.scalar() z = tensor.iscalar() f = function([x, y, z], fill_diagonal_offset(x, y, z)) for test_offset in (-5, -4, -1, 0, 1, 4, 5): for shp in [(8, 8), (5, 8), (8, 5), (5, 5)]: a = np.random.rand(*shp).astype(config.floatX) val = np.cast[config.floatX](np.random.rand()) out = f(a, val, test_offset) # We can't use np.fill_diagonal as it is bugged. assert np.allclose(np.diag(out, test_offset), val) if test_offset >= 0: assert (out == val).sum() == min(min(a.shape), a.shape[1] - test_offset) else: assert (out == val).sum() == min(min(a.shape), a.shape[0] + test_offset) def test_gradient(self): for test_offset in (-5, -4, -1, 0, 1, 4, 5): # input 'offset' will not be tested def fill_diagonal_with_fix_offset(a, val): return fill_diagonal_offset(a, val, test_offset) utt.verify_grad(fill_diagonal_with_fix_offset, [np.random.rand(5, 8), np.random.rand()], n_tests=1, rng=TestFillDiagonalOffset.rng) utt.verify_grad(fill_diagonal_with_fix_offset, [np.random.rand(8, 5), np.random.rand()], n_tests=1, rng=TestFillDiagonalOffset.rng) utt.verify_grad(fill_diagonal_with_fix_offset, [np.random.rand(5, 5), np.random.rand()], n_tests=1, rng=TestFillDiagonalOffset.rng) def test_infer_shape(self): x = tensor.dmatrix() y = tensor.dscalar() z = tensor.iscalar() for test_offset in (-5, -4, -1, 0, 1, 4, 5): self._compile_and_check([x, y, z], [self.op(x, y, z)], [np.random.rand(8, 5), np.random.rand(), test_offset], self.op_class) self._compile_and_check([x, y, z], [self.op(x, y, z)], [np.random.rand(5, 8), np.random.rand(), test_offset], self.op_class) def test_to_one_hot(): v = theano.tensor.ivector() o = to_one_hot(v, 10) f = theano.function([v], o) out = f([1, 2, 3, 5, 6]) assert out.dtype == theano.config.floatX assert np.allclose( out, [[0., 1., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 1., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 1., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 1., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 1., 0., 0., 0.]]) v = theano.tensor.ivector() o = to_one_hot(v, 10, dtype="int32") f = theano.function([v], o) out = f([1, 2, 3, 5, 6]) assert out.dtype == "int32" assert np.allclose( out, [[0., 1., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 1., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 1., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 1., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 1., 0., 0., 0.]]) class test_Unique(utt.InferShapeTester): def setUp(self): super(test_Unique, self).setUp() self.op_class = Unique self.ops = [Unique(), Unique(True), Unique(False, True), Unique(True, True), Unique(False, False, True), Unique(True, False, True), Unique(False, True, True), Unique(True, True, True)] def test_basic_vector(self): # Basic test for a vector. # Done by using the op and checking that it returns the right answer. x = theano.tensor.vector() inp = np.asarray([2, 1, 3, 2], dtype=config.floatX) list_outs_expected = [[np.unique(inp)], np.unique(inp, True), np.unique(inp, False, True), np.unique(inp, True, True), np.unique(inp, False, False, True), np.unique(inp, True, False, True), np.unique(inp, False, True, True), np.unique(inp, True, True, True)] for op, outs_expected in zip(self.ops, list_outs_expected): f = theano.function(inputs=[x], outputs=op(x, return_list=True)) outs = f(inp) # Compare the result computed to the expected value. for out, out_exp in zip(outs, outs_expected): utt.assert_allclose(out, out_exp) def test_basic_matrix(self): # Basic test for a matrix. # Done by using the op and checking that it returns the right answer. x = theano.tensor.matrix() inp = np.asarray([[2, 1], [3, 2], [2, 3]], dtype=config.floatX) list_outs_expected = [[np.unique(inp)], np.unique(inp, True), np.unique(inp, False, True), np.unique(inp, True, True), np.unique(inp, False, False, True), np.unique(inp, True, False, True), np.unique(inp, False, True, True), np.unique(inp, True, True, True)] for op, outs_expected in zip(self.ops, list_outs_expected): f = theano.function(inputs=[x], outputs=op(x, return_list=True)) outs = f(inp) # Compare the result computed to the expected value. for out, out_exp in zip(outs, outs_expected): utt.assert_allclose(out, out_exp) def test_infer_shape_vector(self): # Testing the infer_shape with a vector. x = theano.tensor.vector() for op in self.ops: if not op.return_inverse: continue if op.return_index: f = op(x)[2] else: f = op(x)[1] self._compile_and_check([x], [f], [np.asarray(np.array([2, 1, 3, 2]), dtype=config.floatX)], self.op_class) def test_infer_shape_matrix(self): # Testing the infer_shape with a matrix. x = theano.tensor.matrix() for op in self.ops: if not op.return_inverse: continue if op.return_index: f = op(x)[2] else: f = op(x)[1] self._compile_and_check([x], [f], [np.asarray(np.array([[2, 1], [3, 2], [2, 3]]), dtype=config.floatX)], self.op_class) class test_unravel_index(utt.InferShapeTester): def test_unravel_index(self): def check(shape, index_ndim, order): indices = np.arange(np.product(shape)) # test with scalars and higher-dimensional indices if index_ndim == 0: indices = indices[-1] elif index_ndim == 2: indices = indices[:, np.newaxis] indices_symb = theano.shared(indices) # reference result ref = np.unravel_index(indices, shape) def fn(i, d, nd=None): if nd is None: return function([], unravel_index(i, d, order=order)) else: return function([], unravel_index(i, d, order=order, ndim=nd)) # shape given as a tuple f_array_tuple = fn(indices, shape) f_symb_tuple = fn(indices_symb, shape) np.testing.assert_equal(ref, f_array_tuple()) np.testing.assert_equal(ref, f_symb_tuple()) # shape given as an array shape_array = np.array(shape) f_array_array = fn(indices, shape_array) np.testing.assert_equal(ref, f_array_array()) # shape given as a theano variable shape_symb = theano.shared(shape_array) f_array_symb = fn(indices, shape_symb, len(shape)) np.testing.assert_equal(ref, f_array_symb()) # shape given as a Shape op (unravel_index will use get_vector_length # to infer the number of dimensions) indexed_array = theano.shared(np.random.uniform(size=shape_array)) f_array_shape = fn(indices, indexed_array.shape) np.testing.assert_equal(ref, f_array_shape()) # shape testing self._compile_and_check([], unravel_index(indices, shape_symb, order=order, ndim=len(shape)), [], UnravelIndex) for order in ('C', 'F'): for index_ndim in (0, 1, 2): check((3,), index_ndim, order) check((3, 4), index_ndim, order) check((3, 4, 5), index_ndim, order) # must specify ndim if length of dims is not fixed self.assertRaises(ValueError, unravel_index, theano.tensor.ivector(), theano.tensor.ivector()) # must provide integers self.assertRaises(TypeError, unravel_index, theano.tensor.fvector(), (3, 4)) self.assertRaises(TypeError, unravel_index, (3, 4), (3.4, 3.2)) self.assertRaises(ValueError, unravel_index, (3, 4), (3, 3), ndim=5.4) # dims must be a 1D sequence self.assertRaises(TypeError, unravel_index, (3, 4), 3) self.assertRaises(TypeError, unravel_index, (3, 4), ((3, 4),)) class test_ravel_multi_index(utt.InferShapeTester): def test_ravel_multi_index(self): def check(shape, index_ndim, mode, order): multi_index = np.unravel_index(np.arange(np.product(shape)), shape, order=order) # create some invalid indices to test the mode if mode in ('wrap', 'clip'): multi_index = (multi_index[0] - 1,) + multi_index[1:] # test with scalars and higher-dimensional indices if index_ndim == 0: multi_index = tuple(i[-1] for i in multi_index) elif index_ndim == 2: multi_index = tuple(i[:, np.newaxis] for i in multi_index) multi_index_symb = [theano.shared(i) for i in multi_index] # reference result ref = np.ravel_multi_index(multi_index, shape, mode, order) def fn(mi, s): return function([], ravel_multi_index(mi, s, mode, order)) # shape given as a tuple f_array_tuple = fn(multi_index, shape) f_symb_tuple = fn(multi_index_symb, shape) np.testing.assert_equal(ref, f_array_tuple()) np.testing.assert_equal(ref, f_symb_tuple()) # shape given as an array shape_array = np.array(shape) f_array_array = fn(multi_index, shape_array) np.testing.assert_equal(ref, f_array_array()) # shape given as a theano variable shape_symb = theano.shared(shape_array) f_array_symb = fn(multi_index, shape_symb) np.testing.assert_equal(ref, f_array_symb()) # shape testing self._compile_and_check([], [ravel_multi_index(multi_index, shape_symb, mode, order)], [], RavelMultiIndex) for mode in ('raise', 'wrap', 'clip'): for order in ('C', 'F'): for index_ndim in (0, 1, 2): check((3,), index_ndim, mode, order) check((3, 4), index_ndim, mode, order) check((3, 4, 5), index_ndim, mode, order) # must provide integers self.assertRaises(TypeError, ravel_multi_index, (theano.tensor.fvector(), theano.tensor.ivector()), (3, 4)) self.assertRaises(TypeError, ravel_multi_index, ((3, 4), theano.tensor.ivector()), (3.4, 3.2)) # dims must be a 1D sequence self.assertRaises(TypeError, ravel_multi_index, ((3, 4),), ((3, 4),)) Theano-1.0.0/theano/tensor/tests/test_opt_uncanonicalize.py00006000163562€ÿÿÿþ0000001543613203335532025527 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import unittest import numpy as np import theano from theano import function, config from theano import scalar from theano.gof import FunctionGraph from theano.gof.opt import out2in from theano.tensor.opt_uncanonicalize import ( local_alloc_dimshuffle, local_reshape_dimshuffle, local_dimshuffle_alloc, local_dimshuffle_subtensor, ) import theano.tensor as tensor #from theano.tensor import matrix,max_and_argmax,MaaxAndArgmax,neg from theano.tensor.elemwise import CAReduce, Elemwise, DimShuffle from theano.tests import unittest_tools as utt class T_max_and_argmax(unittest.TestCase): def test_optimization(self): # If we use only the max output, we should replace this op with # a faster one. mode = theano.compile.mode.get_default_mode().including( 'canonicalize', 'fast_run') for axis in [0, 1, -1]: data = np.asarray(np.random.rand(2, 3), dtype=config.floatX) n = tensor.matrix() f = function([n], tensor.max_and_argmax(n, axis)[0], mode=mode) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, CAReduce) f = function([n], tensor.max_and_argmax(n, axis), mode=mode) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, tensor.MaxAndArgmax) class T_min_max(unittest.TestCase): def setUp(self): utt.seed_rng() self.mode = theano.compile.mode.get_default_mode().including( 'canonicalize', 'fast_run') def test_optimization_max(self): data = np.asarray(np.random.rand(2, 3), dtype=config.floatX) n = tensor.matrix() for axis in [0, 1, -1]: f = function([n], tensor.max(n, axis), mode=self.mode) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, CAReduce) f(data) f = function([n], tensor.max(-n, axis), mode=self.mode) topo = f.maker.fgraph.toposort() assert len(topo) == 2 assert isinstance(topo[0].op, Elemwise) assert isinstance(topo[0].op.scalar_op, scalar.Neg) assert isinstance(topo[1].op, CAReduce) f(data) f = function([n], -tensor.max(n, axis), mode=self.mode) topo = f.maker.fgraph.toposort() assert len(topo) == 2 assert isinstance(topo[0].op, CAReduce) assert isinstance(topo[1].op, Elemwise) assert isinstance(topo[1].op.scalar_op, scalar.Neg) f(data) f = function([n], -tensor.max(-n, axis), mode=self.mode) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, CAReduce) # min f(data) def test_optimization_min(self): data = np.asarray(np.random.rand(2, 3), dtype=config.floatX) n = tensor.matrix() for axis in [0, 1, -1]: f = function([n], tensor.min(n, axis), mode=self.mode) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, CAReduce) f(data) # test variant with neg to make sure we optimize correctly f = function([n], tensor.min(-n, axis), mode=self.mode) topo = f.maker.fgraph.toposort() assert len(topo) == 2 assert isinstance(topo[0].op, CAReduce) # max assert isinstance(topo[1].op, Elemwise) assert isinstance(topo[1].op.scalar_op, scalar.Neg) f(data) f = function([n], -tensor.min(n, axis), mode=self.mode) topo = f.maker.fgraph.toposort() assert len(topo) == 2 assert isinstance(topo[0].op, Elemwise) assert isinstance(topo[0].op.scalar_op, scalar.Neg) assert isinstance(topo[1].op, CAReduce) # max f(data) f = function([n], -tensor.min(-n, axis), mode=self.mode) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, CAReduce) # max f(data) def test_local_alloc_dimshuffle(): alloc_dimshuffle = out2in(local_alloc_dimshuffle) x = tensor.vector('x') m = tensor.iscalar('m') y = x.dimshuffle('x', 0) out = tensor.alloc(y, m, 1, x.shape[0]) g = FunctionGraph([x, m], [out]) alloc_dimshuffle(g) topo = g.toposort() assert any([not isinstance(x, DimShuffle) for x in topo]) def test_local_reshape_dimshuffle(): reshape_dimshuffle = out2in(local_reshape_dimshuffle) x = tensor.matrix('x') y = x.dimshuffle('x', 0, 'x', 1) out = tensor.reshape(y, (1, x.shape[0] * x.shape[1], 1)) g = FunctionGraph([x], [out]) reshape_dimshuffle(g) topo = g.toposort() assert any([not isinstance(x, DimShuffle) for x in topo]) def test_local_dimshuffle_alloc(): reshape_dimshuffle = out2in(local_dimshuffle_alloc) x = tensor.vector('x') out = tensor.alloc(x, 3, 2).dimshuffle('x', 'x', 0, 1) g = FunctionGraph([x], [out]) reshape_dimshuffle(g) l = theano.gof.PerformLinker() l.accept(g) f = l.make_function() assert f([3, 4]).ndim == 4 topo = g.toposort() assert any([not isinstance(x, DimShuffle) for x in topo]) def test_local_dimshuffle_subtensor(): dimshuffle_subtensor = out2in(local_dimshuffle_subtensor) x = tensor.dtensor4('x') x = tensor.patternbroadcast(x, (False, True, False, False)) i = tensor.iscalar('i') out = x[:, :, 10:30, ::i].dimshuffle(0, 2, 3) g = FunctionGraph([x, i], [out]) dimshuffle_subtensor(g) topo = g.toposort() assert any([not isinstance(x, DimShuffle) for x in topo]) # Test dimshuffle remove dimensions the subtensor don't "see". x = tensor.tensor(broadcastable=(False, True, False), dtype='float64') out = x[i].dimshuffle(1) g = FunctionGraph([x, i], [out]) dimshuffle_subtensor(g) topo = g.toposort() assert any([not isinstance(x, DimShuffle) for x in topo]) # Test dimshuffle remove dimensions the subtensor don't "see" but # have in between dimensions. x = tensor.tensor(broadcastable=(False, True, False, True), dtype='float64') out = x[i].dimshuffle(1) f = theano.function([x, i], out) topo = f.maker.fgraph.toposort() assert any([not isinstance(x, DimShuffle) for x in topo]) assert f(np.random.rand(5, 1, 4, 1), 2).shape == (4,) # Test a corner case that had Theano return a bug. x = tensor.dtensor4('x') x = tensor.patternbroadcast(x, (False, True, False, False)) assert x[:,:, 0:3, ::-1].dimshuffle(0,2,3).eval({x: np.ones((5, 1, 6, 7))}).shape == (5, 3, 7) Theano-1.0.0/theano/tensor/tests/_test_mpi_roundtrip.py00006000163562€ÿÿÿþ0000000272113203335532024666 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division # Run using # mpiexec -np 2 python _test_mpi_roundtrip.py from mpi4py import MPI import theano from theano.configparser import change_flags from theano.tensor.io import send, recv, mpi_cmps from theano.gof.sched import sort_schedule_fn import numpy as np from sys import stdout, stderr, exit comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() if size != 2: stderr.write("mpiexec failed to create a world with two nodes.\n" "Closing with success message.") stdout.write("True") exit(0) shape = (2, 2) dtype = 'float32' scheduler = sort_schedule_fn(*mpi_cmps) mode = theano.Mode(optimizer=None, linker=theano.OpWiseCLinker(schedule=scheduler)) with change_flags(compute_test_value='off'): if rank == 0: x = theano.tensor.matrix('x', dtype=dtype) y = x + 1 send_request = send(y, 1, 11) z = recv(shape, dtype, 1, 12) f = theano.function([x], [send_request, z], mode=mode) xx = np.random.rand(*shape).astype(dtype) expected = (xx + 1) * 2 _, zz = f(xx) same = np.linalg.norm(zz - expected) < .001 # The parent test will look for "True" in the output stdout.write(str(same)) if rank == 1: y = recv(shape, dtype, 0, 11) z = y * 2 send_request = send(z, 0, 12) f = theano.function([], send_request, mode=mode) f() Theano-1.0.0/theano/tensor/tests/test_fourier.py00006000163562€ÿÿÿþ0000000453013203335532023307 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import numpy as np from numpy.testing import dec import theano from theano import tensor from theano.tests import unittest_tools as utt from theano.tensor.fourier import Fourier, fft class TestFourier(utt.InferShapeTester): rng = np.random.RandomState(43) def setUp(self): super(TestFourier, self).setUp() self.op_class = Fourier self.op = fft def test_perform(self): a = tensor.dmatrix() f = theano.function([a], self.op(a, n=10, axis=0)) a = np.random.rand(8, 6) assert np.allclose(f(a), np.fft.fft(a, 10, 0)) def test_infer_shape(self): a = tensor.dvector() self._compile_and_check([a], [self.op(a, 16, 0)], [np.random.rand(12)], self.op_class) a = tensor.dmatrix() for var in [self.op(a, 16, 1), self.op(a, None, 1), self.op(a, 16, None), self.op(a, None, None)]: self._compile_and_check([a], [var], [np.random.rand(12, 4)], self.op_class) b = tensor.iscalar() for var in [self.op(a, 16, b), self.op(a, None, b)]: self._compile_and_check([a, b], [var], [np.random.rand(12, 4), 0], self.op_class) @dec.skipif(True, "Complex grads not enabled, see #178") def test_gradient(self): def fft_test1(a): return self.op(a, None, None) def fft_test2(a): return self.op(a, None, 0) def fft_test3(a): return self.op(a, 4, None) def fft_test4(a): return self.op(a, 4, 0) pts = [np.random.rand(5, 2, 4, 3), np.random.rand(2, 3, 4), np.random.rand(2, 5), np.random.rand(5)] for fft_test in [fft_test1, fft_test2, fft_test3, fft_test4]: for pt in pts: theano.gradient.verify_grad(fft_test, [pt], n_tests=1, rng=TestFourier.rng, out_type='complex64') if __name__ == "__main__": t = TestFourier('setUp') t.setUp() t.test_perform() t.test_infer_shape() t.test_gradient() Theano-1.0.0/theano/tensor/tests/test_type_other.py00006000163562€ÿÿÿþ0000000261013203335532024013 0ustar boccoset00000000000000""" This file don't test everything. It only test one past crash error.""" from __future__ import absolute_import, print_function, division import theano from theano.gof import Constant from theano.tensor.type_other import MakeSlice, make_slice, NoneTypeT, NoneConst def test_make_slice_merge(): # In the past, this was crahsing during compilation. i = theano.tensor.iscalar() s1 = make_slice(0, i) s2 = make_slice(0, i) f = theano.function([i], [s1, s2]) nodes = f.maker.fgraph.apply_nodes assert len([n for n in nodes if isinstance(n.op, MakeSlice)]) == 1 theano.printing.debugprint(f) def test_none_Constant(): # Tests equals # We had an error in the past with unpickling o1 = Constant(NoneTypeT(), None, name='NoneConst') o2 = Constant(NoneTypeT(), None, name='NoneConst') assert o1.equals(o2) assert NoneConst.equals(o1) assert o1.equals(NoneConst) assert NoneConst.equals(o2) assert o2.equals(NoneConst) # This trigger equals that returned the wrong answer in the past. import six.moves.cPickle as pickle import theano from theano import tensor x = tensor.vector('x') y = tensor.argmax(x) kwargs = {} # We can't pickle DebugMode if theano.config.mode in ["DebugMode", "DEBUG_MODE"]: kwargs = {'mode': 'FAST_RUN'} f = theano.function([x], [y], **kwargs) pickle.loads(pickle.dumps(f)) Theano-1.0.0/theano/tensor/tests/test_gc.py00006000163562€ÿÿÿþ0000001016313203335532022224 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import numpy as np import six.moves.cPickle as pickle from six.moves import xrange import theano from theano import tensor as T import time def test_no_reuse(): x = T.lvector() y = T.lvector() f = theano.function([x, y], x + y) # provide both inputs in the first call f(np.ones(10, dtype='int64'), np.ones(10, dtype='int64')) try: f(np.ones(10)) except TypeError: return assert not 'should not get here' def test_gc_never_pickles_temporaries(): x = T.dvector() for i in xrange(2): # TODO: 30 causes like LONG compilation due to MERGE if i: r = r + r/10 else: r = x optimizer = None optimizer = 'fast_run' for f_linker, g_linker in [ (theano.PerformLinker(allow_gc=True), theano.PerformLinker(allow_gc=False)), (theano.OpWiseCLinker(allow_gc=True), theano.OpWiseCLinker(allow_gc=False))]: # f_linker has garbage collection # g_linker has no garbage collection f = theano.function([x], r, mode=theano.Mode(optimizer=optimizer, linker=f_linker)) g = theano.function([x], r, mode=theano.Mode(optimizer=optimizer, linker=g_linker)) pre_f = pickle.dumps(f) pre_g = pickle.dumps(g) len_pre_f = len(pre_f) len_pre_g = len(pre_g) # We can't compare the content or the length of the string # between f and g. 2 reason, we store some timming information # in float. They won't be the same each time. Different float # can have different lenght when printed. def a(fn): return len(pickle.dumps(fn.maker)) assert a(f) == a(f) # some sanity checks on the pickling mechanism assert a(g) == a(g) # some sanity checks on the pickling mechanism def b(fn): return len( pickle.dumps( theano.compile.function_module._pickle_Function( fn))) assert b(f) == b(f) # some sanity checks on the pickling mechanism def c(fn): return len(pickle.dumps(fn)) assert c(f) == c(f) # some sanity checks on the pickling mechanism assert c(g) == c(g) # some sanity checks on the pickling mechanism # now run the function once to create temporaries within the no-gc # linker f(np.ones(100, dtype='float64')) g(np.ones(100, dtype='float64')) # serialize the functions again post_f = pickle.dumps(f) post_g = pickle.dumps(g) len_post_f = len(post_f) len_post_g = len(post_g) # assert that f() didn't cause the function to grow # allow_gc should leave the function un-changed by calling assert len_pre_f == len_post_f, (len_pre_f, len_post_f) # assert that g() didn't cause g to grow because temporaries # that weren't collected shouldn't be pickled anyway # Allow for a couple of bytes of difference, since timing info, # for instance, can be represented as text of varying size. assert abs(len_post_f - len_post_g) < 256, ( f_linker, len_post_f, len_post_g) def test_merge_opt_runtime(): # In the original merge optimization, the following graph took # like caused the MERGE optimizer to exhibit really bad performance # (quadratic? exponential?) # # Ironically, there is actually no merging to do in this graph. x = T.dvector() for i in xrange(50): if i: r = r + r/10 else: r = x t = time.time() f = theano.function([x], r, mode='FAST_COMPILE') # FAST_RUN does in-place optimizer which requires a lot of # toposorting, which is actually pretty slow at the moment. This # test was designed to test MergeOptimizer... so I'm leaving # toposort optimizations for a later date. dt = time.time() - t # it should never take longer than 5 seconds to compile this graph assert dt < 5.0, dt Theano-1.0.0/theano/tensor/tests/test_blas_scipy.py00006000163562€ÿÿÿþ0000000512113203335532023761 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import sys import numpy as np import theano import theano.tensor as tensor from theano.tensor.blas_scipy import ScipyGer from .test_blas import TestCase, gemm_no_inplace, TestBlasStrides from theano.tests.unittest_tools import TestOptimizationMixin class TestScipyGer(TestCase, TestOptimizationMixin): def setUp(self): self.mode = theano.compile.get_default_mode() self.mode = self.mode.including('fast_run') self.mode = self.mode.excluding('c_blas') # c_blas trumps scipy Ops dtype = self.dtype = 'float64' # optimization isn't dtype-dependent self.A = tensor.tensor(dtype=dtype, broadcastable=(False, False)) self.a = tensor.tensor(dtype=dtype, broadcastable=()) self.x = tensor.tensor(dtype=dtype, broadcastable=(False,)) self.y = tensor.tensor(dtype=dtype, broadcastable=(False,)) self.Aval = np.ones((2, 3), dtype=dtype) self.xval = np.asarray([1, 2], dtype=dtype) self.yval = np.asarray([1.5, 2.7, 3.9], dtype=dtype) if not theano.tensor.blas_scipy.have_fblas: self.SkipTest() def function(self, inputs, outputs): return theano.function(inputs, outputs, self.mode) def run_f(self, f): f(self.Aval, self.xval, self.yval) f(self.Aval[::-1, ::-1], self.xval[::-1], self.yval[::-1]) def b(self, bval): return tensor.as_tensor_variable(np.asarray(bval, dtype=self.dtype)) def test_outer(self): f = self.function([self.x, self.y], tensor.outer(self.x, self.y)) self.assertFunctionContains(f, ScipyGer(destructive=True)) def test_A_plus_outer(self): f = self.function([self.A, self.x, self.y], self.A + tensor.outer(self.x, self.y)) self.assertFunctionContains(f, ScipyGer(destructive=False)) self.run_f(f) # DebugMode tests correctness def test_A_plus_scaled_outer(self): f = self.function([self.A, self.x, self.y], self.A + 0.1 * tensor.outer(self.x, self.y)) self.assertFunctionContains(f, ScipyGer(destructive=False)) self.run_f(f) # DebugMode tests correctness def test_scaled_A_plus_scaled_outer(self): f = self.function([self.A, self.x, self.y], 0.2 * self.A + 0.1 * tensor.outer(self.x, self.y)) self.assertFunctionContains(f, gemm_no_inplace) self.run_f(f) # DebugMode tests correctness class TestBlasStridesScipy(TestBlasStrides): mode = theano.compile.get_default_mode() mode = mode.including('fast_run').excluding('gpu', 'c_blas') Theano-1.0.0/theano/tensor/tests/test_var.py00006000163562€ÿÿÿþ0000000533613203335532022431 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import numpy as np from numpy.testing import assert_equal, assert_string_equal import theano import theano.tensor as tt import theano.tests.unittest_tools as utt from theano.tensor import (Subtensor, AdvancedSubtensor, AdvancedSubtensor1, IncSubtensor, AdvancedIncSubtensor, AdvancedIncSubtensor1) def test_numpy_method(): # This type of code is used frequently by PyMC3 users x = tt.dmatrix('x') data = np.random.rand(5, 5) x.tag.test_value = data for fct in [np.arccos, np.arccosh, np.arcsin, np.arcsinh, np.arctan, np.arctanh, np.ceil, np.cos, np.cosh, np.deg2rad, np.exp, np.exp2, np.expm1, np.floor, np.log, np.log10, np.log1p, np.log2, np.rad2deg, np.sin, np.sinh, np.sqrt, np.tan, np.tanh, np.trunc]: y = fct(x) f = theano.function([x], y) utt.assert_allclose(np.nan_to_num(f(data)), np.nan_to_num(fct(data))) def test_empty_list_indexing(): ynp = np.zeros((2, 2))[:, []] znp = np.zeros((2, 2))[:, ()] data = [[0, 0], [0, 0]] x = tt.dmatrix('x') y = x[:, []] z = x[:, ()] fy = theano.function([x], y) fz = theano.function([x], z) assert_equal(fy(data).shape, ynp.shape) assert_equal(fz(data).shape, znp.shape) def test_copy(): x = tt.dmatrix('x') data = np.random.rand(5, 5) y = x.copy(name='y') f = theano.function([x], y) assert_equal(f(data), data) assert_string_equal(y.name, 'y') def test_None_dimShuffle_replace(): # tests replacing None usage in subtensor with dimshuffle # # tests whenever None is used in subtensor to reshape a variable, it is # replaced by dimshuffle. If the replacement is done properly, Subtensor op # (or any of its variants) should not be used anymore. x = tt.dmatrix('x') y = x[:, None, :] f = theano.function([x], y) for elem in f.maker.fgraph.toposort(): assert type(elem.op) not in [Subtensor, AdvancedSubtensor, AdvancedSubtensor1, IncSubtensor, AdvancedIncSubtensor, AdvancedIncSubtensor1] x = tt.tensor3('x') y1 = x[:, :, None, :] y2 = x[None, :, :, None, :] y3 = x[:, :, None, :, None, None] f = theano.function([x], [y1, y2, y3]) for elem in f.maker.fgraph.toposort(): assert type(elem.op) not in [Subtensor, AdvancedSubtensor, AdvancedSubtensor1, IncSubtensor, AdvancedIncSubtensor, AdvancedIncSubtensor1] Theano-1.0.0/theano/tensor/tests/test_keepdims.py00006000163562€ÿÿÿþ0000001176313203335532023443 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import unittest import numpy as np from six import integer_types import theano from theano import tensor, function from theano.tests.unittest_tools import attr # this tests other ops to ensure they keep the dimensions of their # inputs correctly class TestKeepDims(unittest.TestCase): def makeKeepDims_local(self, x, y, axis): if axis is None: newaxis = list(range(x.ndim)) elif isinstance(axis, integer_types): if axis < 0: newaxis = [axis + x.type.ndim] else: newaxis = [axis] else: newaxis = [] for a in axis: if a < 0: a += x.type.ndim newaxis.append(a) i = 0 new_dims = [] for j, _ in enumerate(x.shape): if j in newaxis: new_dims.append('x') else: new_dims.append(i) i += 1 return tensor.DimShuffle(y.type.broadcastable, new_dims)(y) @attr('slow') def test_keepdims(self): x = tensor.dtensor3() a = np.random.rand(3, 2, 4) # We don't need to test all opt and C code, as this is tested # by the ops tests. mode = theano.compile.Mode(optimizer="fast_compile", linker="py") # 'max_and_argmax' has two outputs and can be specified with either # a single or every axis: for axis in [0, 1, 2, [0], [1], [2], None, [0, 1, 2], [-1], [-2], [-3], [-1, -2, -3], [0, -1, -2], [-2, -3, 2]]: op = tensor.max_and_argmax f = function([x], [op(x, axis=axis, keepdims=True)[0], self.makeKeepDims_local( x, op(x, axis=axis, keepdims=False)[0], axis)], mode=mode) ans1, ans2 = f(a) assert np.allclose(ans1, ans2) assert ans1.shape == ans2.shape f = function([x], [op(x, axis=axis, keepdims=True)[1], self.makeKeepDims_local( x, op(x, axis=axis, keepdims=False)[1], axis)], mode=mode) ans1, ans2 = f(a) assert np.allclose(ans1, ans2) assert ans1.shape == ans2.shape # the following ops can be specified with either a single axis or every # axis: for op in ([tensor.argmax, tensor.argmin]): for axis in [0, 1, 2, [0], [1], [2], None, [0, 1, 2], [-1], [-2], [-3], [-1, -2, -3], [0, -2, 2]]: f = function([x], [op(x, axis=axis, keepdims=True), self.makeKeepDims_local( x, op(x, axis=axis, keepdims=False), axis)], mode=mode) ans1, ans2 = f(a) assert np.allclose(ans1, ans2) assert ans1.shape == ans2.shape # the following ops can be specified with a freely specified axis # parameter for op in ([tensor.sum, tensor.prod, tensor.mean, tensor.var, tensor.std, tensor.all, tensor.any, tensor.max, tensor.min]): for axis in [0, 1, 2, [0], [1], [2], None, [0, 1], [1, 2], [0, 1, 2], [-1], [-2], [-3], [-1, -2], [-1, -2, -3], [0, -2, 2]]: f = function([x], [op(x, axis=axis, keepdims=True), self.makeKeepDims_local( x, op(x, axis=axis, keepdims=False), axis)], mode=mode) ans1, ans2 = f(a) assert np.allclose(ans1, ans2) assert ans1.shape == ans2.shape def test_norm(self): x = tensor.dtensor3() a = np.random.rand(3, 2, 4).astype(theano.config.floatX) mode = theano.compile.Mode(optimizer="fast_compile", linker="py") for axis in [0, 1, 2, [0], [1], [2], None, [0, 1], [1, 2], [0, 1, 2], [-1], [-2], [-3], [-1, -2], [-1, -2, -3], [0, -2, 2]]: f = function([x], [x.norm(L=1, axis=axis, keepdims=True), self.makeKeepDims_local(x, x.norm(L=1, axis=axis, keepdims=False), axis) ], mode=mode) ans1, ans2 = f(a) assert np.allclose(ans1, ans2) assert ans1.shape == ans2.shape g = function([x], [x.norm(L=2, axis=axis, keepdims=True), self.makeKeepDims_local(x, x.norm(L=2, axis=axis, keepdims=False), axis) ], mode=mode) ans1, ans2 = g(a) assert np.allclose(ans1, ans2) assert ans1.shape == ans2.shape Theano-1.0.0/theano/tensor/tests/test_merge.py00006000163562€ÿÿÿþ0000000426513203335532022740 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import numpy as np from theano.gof.type import Type from theano.gof.graph import Variable, Apply, Constant from theano.gof.op import Op from theano.gof.opt import * from theano.gof.fg import FunctionGraph as Env from theano.gof.toolbox import * import theano.tensor.basic as T def as_variable(x): if not isinstance(x, Variable): raise TypeError("not a Variable", x) return x class MyType(Type): def filter(self, data): return data def __eq__(self, other): return isinstance(other, MyType) class MyOp(Op): def __init__(self, name, dmap=None, x=None): if dmap is None: dmap = {} self.name = name self.destroy_map = dmap self.x = x def make_node(self, *inputs): inputs = list(map(as_variable, inputs)) for input in inputs: if not isinstance(input.type, MyType): raise Exception("Error 1") outputs = [MyType()()] return Apply(self, inputs, outputs) def __str__(self): return self.name def __repr__(self): return self.name def __eq__(self, other): return (self is other or isinstance(other, MyOp) and self.x is not None and self.x == other.x) def __hash__(self): if self.x is not None: return self.x else: return id(self) op1 = MyOp('Op1') def test_merge_with_weird_eq(): # numpy arrays don't compare equal like other python objects # SCALAR CASE x = T.constant(np.asarray(1), name='x') y = T.constant(np.asarray(1), name='y') g = Env([x, y], [x+y]) MergeOptimizer().optimize(g) assert len(g.apply_nodes) == 1 node = list(g.apply_nodes)[0] assert len(node.inputs) == 2 assert node.inputs[0] is node.inputs[1] # NONSCALAR CASE # This was created to test TensorConstantSignature x = T.constant(np.ones(5), name='x') y = T.constant(np.ones(5), name='y') g = Env([x, y], [x+y]) MergeOptimizer().optimize(g) assert len(g.apply_nodes) == 1 node = list(g.apply_nodes)[0] assert len(node.inputs) == 2 assert node.inputs[0] is node.inputs[1] Theano-1.0.0/theano/tensor/tests/__init__.py00006000163562€ÿÿÿþ0000000000013203335532022320 0ustar boccoset00000000000000Theano-1.0.0/theano/tensor/tests/test_fft.py00006000163562€ÿÿÿþ0000001465713203335532022426 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import numpy as np import unittest import theano from theano import tensor as T from theano.tests import unittest_tools as utt from theano.tensor import fft N = 16 class TestFFT(unittest.TestCase): def test_rfft_float(self): # Test that numpy's default float64 output is cast to theano input type eps = 1e-1 def f_rfft(inp): return fft.rfft(inp) inputs_val = np.random.random((1, N)).astype(theano.config.floatX) utt.verify_grad(f_rfft, [inputs_val], eps=eps) def f_irfft(inp): return fft.irfft(inp) inputs_val = np.random.random((1, N // 2 + 1, 2)).astype(theano.config.floatX) utt.verify_grad(f_irfft, [inputs_val], eps=eps) def test_1Drfft(self): inputs_val = np.random.random((1, N)).astype(theano.config.floatX) x = T.matrix('x') rfft = fft.rfft(x) f_rfft = theano.function([x], rfft) res_rfft = f_rfft(inputs_val) res_rfft_comp = (np.asarray(res_rfft[:, :, 0]) + 1j * np.asarray(res_rfft[:, :, 1])) rfft_ref = np.fft.rfft(inputs_val, axis=1) utt.assert_allclose(rfft_ref, res_rfft_comp) m = rfft.type() print(m.ndim) irfft = fft.irfft(m) f_irfft = theano.function([m], irfft) res_irfft = f_irfft(res_rfft) utt.assert_allclose(inputs_val, np.asarray(res_irfft)) # The numerical gradient of the FFT is sensitive, must set large # enough epsilon to get good accuracy. eps = 1e-1 def f_rfft(inp): return fft.rfft(inp) inputs_val = np.random.random((1, N)).astype(theano.config.floatX) utt.verify_grad(f_rfft, [inputs_val], eps=eps) def f_irfft(inp): return fft.irfft(inp) inputs_val = np.random.random((1, N // 2 + 1, 2)).astype(theano.config.floatX) utt.verify_grad(f_irfft, [inputs_val], eps=eps) def test_rfft(self): inputs_val = np.random.random((1, N, N)).astype(theano.config.floatX) inputs = theano.shared(inputs_val) rfft = fft.rfft(inputs) f_rfft = theano.function([], rfft) res_rfft = f_rfft() res_rfft_comp = (np.asarray(res_rfft[:, :, :, 0]) + 1j * np.asarray(res_rfft[:, :, :, 1])) rfft_ref = np.fft.rfftn(inputs_val, axes=(1, 2)) utt.assert_allclose(rfft_ref, res_rfft_comp, atol=1e-4, rtol=1e-4) def test_irfft(self): inputs_val = np.random.random((1, N, N)).astype(theano.config.floatX) inputs = theano.shared(inputs_val) rfft = fft.rfft(inputs) f_rfft = theano.function([], rfft) res_fft = f_rfft() m = rfft.type() irfft = fft.irfft(m) f_irfft = theano.function([m], irfft) res_irfft = f_irfft(res_fft) utt.assert_allclose(inputs_val, np.asarray(res_irfft)) inputs_val = np.random.random((1, N, N, 2)).astype(theano.config.floatX) inputs = theano.shared(inputs_val) irfft = fft.irfft(inputs) f_irfft = theano.function([], irfft) res_irfft = f_irfft() inputs_ref = inputs_val[..., 0] + inputs_val[..., 1] * 1j irfft_ref = np.fft.irfftn(inputs_ref, axes=(1, 2)) utt.assert_allclose(irfft_ref, res_irfft, atol=1e-4, rtol=1e-4) def test_norm_rfft(self): inputs_val = np.random.random((1, N, N)).astype(theano.config.floatX) inputs = theano.shared(inputs_val) # Unitary normalization rfft = fft.rfft(inputs, norm='ortho') f_rfft = theano.function([], rfft) res_rfft = f_rfft() res_rfft_comp = (np.asarray(res_rfft[:, :, :, 0]) + 1j * np.asarray(res_rfft[:, :, :, 1])) rfft_ref = np.fft.rfftn(inputs_val, axes=(1, 2)) utt.assert_allclose(rfft_ref / N, res_rfft_comp, atol=1e-4, rtol=1e-4) # No normalization rfft = fft.rfft(inputs, norm='no_norm') f_rfft = theano.function([], rfft) res_rfft = f_rfft() res_rfft_comp = (np.asarray(res_rfft[:, :, :, 0]) + 1j * np.asarray(res_rfft[:, :, :, 1])) utt.assert_allclose(rfft_ref, res_rfft_comp, atol=1e-4, rtol=1e-4) # Inverse FFT inputs inputs_val = np.random.random((1, N, N // 2 + 1, 2)).astype(theano.config.floatX) inputs = theano.shared(inputs_val) inputs_ref = inputs_val[..., 0] + 1j * inputs_val[..., 1] # Unitary normalization inverse FFT irfft = fft.irfft(inputs, norm='ortho') f_irfft = theano.function([], irfft) res_irfft = f_irfft() irfft_ref = np.fft.irfftn(inputs_ref, axes=(1, 2)) utt.assert_allclose(irfft_ref * N, res_irfft, atol=1e-4, rtol=1e-4) # No normalization inverse FFT irfft = fft.irfft(inputs, norm='no_norm') f_irfft = theano.function([], irfft) res_irfft = f_irfft() utt.assert_allclose(irfft_ref * N**2, res_irfft, atol=1e-4, rtol=1e-4) def test_params(self): inputs_val = np.random.random((1, N)).astype(theano.config.floatX) inputs = theano.shared(inputs_val) self.assertRaises(ValueError, fft.rfft, inputs, norm=123) inputs_val = np.random.random((1, N // 2 + 1, 2)).astype(theano.config.floatX) inputs = theano.shared(inputs_val) self.assertRaises(ValueError, fft.irfft, inputs, norm=123) self.assertRaises(ValueError, fft.irfft, inputs, is_odd=123) def test_grad_rfft(self): # The numerical gradient of the FFT is sensitive, must set large # enough epsilon to get good accuracy. eps = 1e-1 def f_rfft(inp): return fft.rfft(inp) inputs_val = np.random.random((1, N, N)).astype(theano.config.floatX) utt.verify_grad(f_rfft, [inputs_val], eps=eps) def f_irfft(inp): return fft.irfft(inp) inputs_val = np.random.random((1, N, N // 2 + 1, 2)).astype(theano.config.floatX) utt.verify_grad(f_irfft, [inputs_val], eps=eps) def f_rfft(inp): return fft.rfft(inp, norm='ortho') inputs_val = np.random.random((1, N, N)).astype(theano.config.floatX) utt.verify_grad(f_rfft, [inputs_val], eps=eps) def f_irfft(inp): return fft.irfft(inp, norm='no_norm') inputs_val = np.random.random((1, N, N // 2 + 1, 2)).astype(theano.config.floatX) utt.verify_grad(f_irfft, [inputs_val], eps=eps) Theano-1.0.0/theano/tensor/tests/test_blas.py00006000163562€ÿÿÿþ0000026133413203335532022564 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division from copy import copy from itertools import product as itertools_product from unittest import TestCase import numpy as np from numpy import (arange, array, common_type, complex64, complex128, float32, float64, newaxis, shape, transpose, zeros) from numpy.testing import assert_array_almost_equal from itertools import product from six.moves import xrange import theano import theano.tensor as T from theano import tensor, In, shared, config from theano.compat import exc_message from theano.printing import pp from theano.tensor.blas import (_dot22, _dot22scalar, res_is_a, _as_scalar, _is_real_matrix, _gemm_canonicalize, _factor_canonicalized, Gemm, Gemv, gemm_inplace, gemm_no_inplace, InconsistencyError, Ger, ger, ger_destructive) from theano.tests import unittest_tools from .test_basic import (as_tensor_variable, inplace_func, compile, inplace) import theano.tensor.blas_scipy from theano.tests.unittest_tools import attr if config.mode == 'FAST_COMPILE': mode_not_fast_compile = 'FAST_RUN' else: mode_not_fast_compile = config.mode mode_blas_opt = theano.compile.get_default_mode().including( 'BlasOpt', 'specialize', 'InplaceBlasOpt') mode_blas_opt = mode_blas_opt.excluding('c_blas') def test_dot_eq(): assert T.Dot() == T.Dot() def sharedX(x, name): return theano.shared(np.asarray(x, config.floatX), name=name) class t_gemm(TestCase): """ This test suite is supposed to establish that gemm works as it is supposed to. """ def setUp(self): unittest_tools.seed_rng() Gemm.debug = False @staticmethod def _gemm(z, a, x, y, b): assert a.shape == () assert b.shape == () return b * z + a * np.dot(x, y) @staticmethod def rand(*args): return np.random.rand(*args) def cmp(self, z_, a_, x_, y_, b_): for dtype in ['float32', 'float64', 'complex64', 'complex128']: z = np.asarray(z_, dtype=dtype) a = np.asarray(a_, dtype=dtype) x = np.asarray(x_, dtype=dtype) y = np.asarray(y_, dtype=dtype) b = np.asarray(b_, dtype=dtype) def cmp_linker(z, a, x, y, b, l): z, a, x, y, b = [np.asarray(p) for p in (z, a, x, y, b)] z_orig = z.copy() tz, ta, tx, ty, tb = [as_tensor_variable(p).type() for p in (z, a, x, y, b)] f = inplace_func([tz, ta, tx, ty, tb], gemm_inplace(tz, ta, tx, ty, tb), mode=compile.Mode(optimizer=None, linker=l)) f(z, a, x, y, b) z_after = self._gemm(z_orig, a, x, y, b) # print z_orig, z_after, z, type(z_orig), type(z_after), type(z) unittest_tools.assert_allclose(z_after, z) if a == 0.0 and b == 1.0: return elif z_orig.size == 0: self.assertTrue(z.size == 0) else: self.assertFalse(np.all(z_orig == z)) cmp_linker(copy(z), a, x, y, b, 'c|py') cmp_linker(copy(z), a, x, y, b, 'py') if (not dtype.startswith("complex") and theano.config.cxx): # If theano.config.blas.ldflags is empty, Theano will use # a NumPy C implementation of [sd]gemm_. cmp_linker(copy(z), a, x, y, b, 'c') def test0a(self): Gemm.debug = True try: gemm_no_inplace([1.], 1., [1.], [1.], 1.) except TypeError as e: if exc_message(e) is Gemm.E_rank: return self.fail() def test0(self): try: self.cmp(1., 0., 1.0, 1.0, 1.0) except TypeError as e: if exc_message(e) is Gemm.E_rank: return self.fail() def test2(self): try: self.cmp(2., 1.0, [3, 2, 1.], [[1], [2], [3.]], 1.0) except TypeError as e: self.assertTrue(exc_message(e) == Gemm.E_rank) return self.fail() def test4(self): self.cmp(self.rand(3, 4), 1.0, self.rand(3, 5), self.rand(5, 4), 0.0) def test5(self): self.cmp(self.rand(3, 4), 1.0, self.rand(3, 5), self.rand(5, 4), 1.0) def test6(self): self.cmp(self.rand(3, 4), 1.0, self.rand(3, 5), self.rand(5, 4), -1.0) def test7(self): self.cmp(self.rand(3, 4), 0.0, self.rand(3, 5), self.rand(5, 4), 0.0) def test8(self): self.cmp(self.rand(3, 4), 0.0, self.rand(3, 5), self.rand(5, 4), 0.6) def test9(self): self.cmp(self.rand(3, 4), 0.0, self.rand(3, 5), self.rand(5, 4), -1.0) def test10(self): self.cmp(self.rand(3, 4), -1.0, self.rand(3, 5), self.rand(5, 4), 0.0) def test11(self): self.cmp(self.rand(3, 4), -1.0, self.rand(3, 5), self.rand(5, 4), 1.0) def test12(self): self.cmp(self.rand(3, 4), -1.0, self.rand(3, 5), self.rand(5, 4), -1.0) def test_shape_0(self): self.cmp(self.rand(0, 4), -1.0, self.rand(0, 5), self.rand(5, 4), -1.0) self.cmp(self.rand(3, 0), -1.0, self.rand(3, 5), self.rand(5, 0), -1.0) self.cmp(self.rand(3, 4), -1.0, self.rand(3, 0), self.rand(0, 4), -1.0) self.cmp(self.rand(0, 0), -1.0, self.rand(0, 5), self.rand(5, 0), -1.0) self.cmp(self.rand(0, 0), -1.0, self.rand(0, 0), self.rand(0, 0), -1.0) def test_factorised_scalar(self): a = T.matrix() b = T.matrix() s = theano.shared(np.zeros((5, 5)).astype(config.floatX)) lr1 = T.constant(0.01).astype(config.floatX) lr2 = T.constant(2).astype(config.floatX) l2_reg = T.constant(0.0001).astype(config.floatX) # test constant merge with gemm f = theano.function([a, b], updates=[(s, lr1 * T.dot(a, b) + l2_reg * lr2 * s)], mode=mode_not_fast_compile).maker.fgraph.toposort() # [Gemm{inplace}(, 0.01, # , , # 2e-06)] assert len(f) == 1 assert f[0].op == gemm_inplace # test factored scalar with merge f = theano.function([a, b], updates=[(s, lr1 * (T.dot(a, b) - l2_reg * s))], mode=mode_not_fast_compile).maker.fgraph.toposort() # [Gemm{inplace}(, 0.01, # , , # -2e-06)] assert len(f) == 1 assert f[0].op == gemm_inplace # test factored scalar with merge and neg f = theano.function([a, b], updates=[(s, s - lr1 * (s * .0002 + T.dot(a, b)))], mode=mode_not_fast_compile).maker.fgraph.toposort() # [Gemm{inplace}(, -0.01, # , , # 0.999998)] assert len(f) == 1 assert f[0].op == gemm_inplace def test_destroy_map0(self): # test that only first input can be overwritten. Z = as_tensor_variable(self.rand(2, 2)) try: gemm_inplace(Z, 1.0, Z, Z, 1.0) except InconsistencyError as e: if exc_message(e) == Gemm.E_z_uniq: return self.fail() def test_destroy_map1(self): # test that only first input can be overwritten. Z = as_tensor_variable(self.rand(2, 2)) A = as_tensor_variable(self.rand(2, 2)) try: gemm_inplace(Z, 1.0, A, inplace.transpose_inplace(Z), 1.0) except InconsistencyError as e: if exc_message(e) == Gemm.E_z_uniq: return self.fail() def test_destroy_map2(self): # test that only first input can be overwritten. Z = as_tensor_variable(self.rand(2, 2)) A = as_tensor_variable(self.rand(2, 2)) try: gemm_inplace(Z, 1.0, inplace.transpose_inplace(Z), A, 1.0) except InconsistencyError as e: if exc_message(e) == Gemm.E_z_uniq: return self.fail() def test_destroy_map3(self): # test that only first input can be overwritten Z = as_tensor_variable(self.rand(2, 2)) A = as_tensor_variable(self.rand(2, 2)) try: gemm_inplace(Z, 1.0, Z, A, 1.0) except InconsistencyError as e: if exc_message(e) == Gemm.E_z_uniq: return self.fail() def test_destroy_map4(self): # test that dot args can be aliased Z = shared(self.rand(2, 2), name='Z') A = shared(self.rand(2, 2), name='A') one = T.constant(1.0).astype(Z.dtype) f = inplace_func([], gemm_inplace(Z, one, A, A, one)) f() f = inplace_func([], gemm_inplace(Z, one, A, A.T, one)) f() def test_transposes(self): # three square matrices which are not contiguous A = self.rand(4, 5)[:, :4] B = self.rand(4, 5)[:, :4] C = self.rand(4, 5)[:, :4] def t(z, x, y, a=1.0, b=0.0, l='c|py', dt='float64'): z, a, x, y, b = [theano._asarray(p, dtype=dt) for p in (z, a, x, y, b)] # z_orig = z.copy() z_after = self._gemm(z, a, x, y, b) tz, ta, tx, ty, tb = [shared(p) for p in (z, a, x, y, b)] # f = inplace_func([tz,ta,tx,ty,tb], gemm_inplace(tz,ta,tx,ty,tb), # mode = compile.Mode(optimizer = None, linker=l)) # f(z, a, x, y, b) f = inplace_func([], gemm_inplace(tz, ta, tx, ty, tb), mode=compile.Mode(optimizer=None, linker=l)) f() unittest_tools.assert_allclose(z_after, tz.get_value(borrow=True)) f() unittest_tools.assert_allclose(z_after, tz.get_value(borrow=True)) f() unittest_tools.assert_allclose(z_after, tz.get_value(borrow=True)) # tz.value *= 0 # clear z's value y_T = ty.get_value(borrow=True).T ty.set_value(tx.get_value(borrow=True).T, borrow=True) tx.set_value(y_T, borrow=True) f() # test that the transposed version of multiplication gives # same answer unittest_tools.assert_allclose(z_after, tz.get_value(borrow=True).T) t(C, A, B) t(C.T, A, B) t(C, A.T, B, dt='float32') t(C, A, B.T) t(C.T, A.T, B) t(C, A.T, B.T, dt='float32') t(C.T, A, B.T) t(C.T, A.T, B.T, dt='float32') t(C, A[:, :2], B[:2, :]) t(C.T, A[:, :2], B[:2, :], dt='float32') t(C, A[:2, :].T, B[:2, :]) t(C.T, A[:2, :].T, B[:2, :], dt='float32') t(C, A[:2, :].T, B[:, :2].T) t(C.T, A[:2, :].T, B[:, :2].T) try: t(C.T, A[:2, :], B[:, :2].T) except ValueError as e: if exc_message(e).find('aligned') >= 0: return self.fail() def test_non_contiguous(self): # Like test_transposes but with matrices without any # continuous dimension A = self.rand(4, 4, 3) B = self.rand(4, 4, 3) C = self.rand(4, 4, 3) def t(z, x, y, a=1.0, b=0.0, l='c|py', dt='float64'): z, a, x, y, b = [theano._asarray(p, dtype=dt) for p in (z, a, x, y, b)] z_orig = z.copy() z_after = np.zeros_like(z_orig) for i in xrange(3): z_after[:, :, i] = self._gemm(z[:, :, i], a, x[:, :, i], y[:, :, i], b) tz, ta, tx, ty, tb = [shared(p) for p in (z, a, x, y, b)] for i in xrange(3): f_i = inplace_func([], gemm_inplace(tz[:, :, i], ta, tx[:, :, i], ty[:, :, i], tb), mode=compile.Mode(optimizer=None, linker=l)) for j in xrange(3): # tz will not _always_ be overwritten, # and adding update={...} in the call to function() # will create cycles, so we update by hand. z_i = f_i() z = tz.get_value(borrow=True, return_internal_type=True) z[:, :, i] = z_i unittest_tools.assert_allclose(z_after[:, :, i], tz.get_value(borrow=True)[:, :, i]) tz_i = gemm_no_inplace(tz[:, :, i], ta, tx[ :, :, i], ty[:, :, i], tb) g_i = theano.function( [], tz_i, updates=[(tz, T.set_subtensor(tz[:, :, i], tz_i))], mode=compile.Mode(optimizer=None, linker=l)) for j in xrange(3): g_i() unittest_tools.assert_allclose(z_after[:, :, i], tz.get_value(borrow=True)[:, :, i]) t(C, A, B) t(C.transpose((1, 0, 2)), A, B) t(C, A.transpose((1, 0, 2)), B, dt='float32') t(C, A, B.transpose((1, 0, 2))) t(C.transpose((1, 0, 2)), A.transpose((1, 0, 2)), B) t(C, A.transpose((1, 0, 2)), B.transpose((1, 0, 2)), dt='float32') t(C.transpose((1, 0, 2)), A, B.transpose((1, 0, 2))) t(C.transpose((1, 0, 2)), A.transpose((1, 0, 2)), B.transpose(( 1, 0, 2)), dt='float32') class TestGemmNoFlags(object): gemm = gemm_no_inplace M = 4 N = 5 K = 6 slice_step = 3 def setUp(self): unittest_tools.seed_rng() def get_variable(self, V, to_transpose, to_slice): if to_transpose: V = V.T if to_slice: V = V[::self.slice_step] return V def get_function(self, dtype, transpose_A=False, transpose_B=False, transpose_C=False, slice_A=False, slice_B=False, slice_C=False): alpha = theano.tensor.scalar(dtype=dtype, name='alpha') beta = theano.tensor.scalar(dtype=dtype, name='beta') A = theano.tensor.matrix(dtype=dtype, name='A') B = theano.tensor.matrix(dtype=dtype, name='B') C = theano.tensor.matrix(dtype=dtype, name='C') A1 = self.get_variable(A, transpose_A, slice_A) B1 = self.get_variable(B, transpose_B, slice_B) C1 = self.get_variable(C, transpose_C, slice_C) return theano.function([alpha, A, B, beta, C], self.gemm(C1, alpha, A1, B1, beta)) def generate_value(self, dtype, width, height, to_transpose, to_slice): if to_slice: if to_transpose: shape = (height, width * self.slice_step) else: shape = (width * self.slice_step, height) else: if to_transpose: shape = (height, width) else: shape = (width, height) return np.random.random(shape).astype(dtype) def get_data(self, dtype, alpha, beta, transpose_A=False, transpose_B=False, transpose_C=False, slice_A=False, slice_B=False, slice_C=False): A = self.generate_value(dtype, self.M, self.N, transpose_A, slice_A) B = self.generate_value(dtype, self.N, self.K, transpose_B, slice_B) C = self.generate_value(dtype, self.M, self.K, transpose_C, slice_C) return (alpha, A, B, beta, C) def get_value(self, V, to_transpose, to_slice): if to_transpose: V = V.T if to_slice: V = V[::self.slice_step] return V def compute_ref(self, alpha, A, B, beta, C, transpose_A, transpose_B, transpose_C, slice_A, slice_B, slice_C): A = self.get_value(A, transpose_A, slice_A) B = self.get_value(B, transpose_B, slice_B) C = self.get_value(C, transpose_C, slice_C) return alpha * np.dot(A, B) + beta * C @theano.change_flags({'blas.ldflags': ''}) def run_gemm(self, dtype, ALPHA, BETA, transpose_A, transpose_B, transpose_C, slice_A, slice_B, slice_C): f = self.get_function(dtype, transpose_A, transpose_B, transpose_C, slice_A, slice_B, slice_C) values = self.get_data(dtype, ALPHA, BETA, transpose_A, transpose_B, transpose_C, slice_A, slice_B, slice_C) assert any(isinstance(node.op, Gemm) for node in f.maker.fgraph.apply_nodes) z_val = f(*values) assert z_val.dtype == dtype assert tuple(z_val.shape) == (self.M, self.K) ref_val = self.compute_ref(*(values + (transpose_A, transpose_B, transpose_C, slice_A, slice_B, slice_C))) unittest_tools.assert_allclose(ref_val, z_val) def test_gemm(self): dtypes = ('float32', 'float64') scalars = (0, 1, -2) booleans = (False, True) # dtype, alpha, beta, transA, transB, transC, sliceA, sliceB, sliceC iterables = [dtypes] + ([scalars] * 2) + ([booleans] * 6) for dtype, alpha, beta, tA, tB, tC, sA, sB, sC in product(*iterables): yield (self.run_gemm, dtype, alpha, beta, tA, tB, tC, sA, sB, sC) def test_res_is_a(): X, Y, Z, a, b = XYZab() assert not res_is_a(a, T.sqrt) assert not res_is_a(a + a, T.sqrt) assert res_is_a(T.sqrt(a + a), T.sqrt) # leave the maxclients stuff untested because it requires being in an fgraph. class t_as_scalar(TestCase): def test0(self): # Test that it works on scalar constants a = T.constant(2.5) b = T.constant(np.asarray([[[0.5]]])) b2 = b.dimshuffle() assert b2.ndim == 0 d_a = T.DimShuffle([], [])(a) d_b = T.DimShuffle([True, True, True], [0, 2, 1])(b) d_a2 = T.DimShuffle([], ['x', 'x', 'x'])(a) self.assertTrue(_as_scalar(a) == a) self.assertTrue(_as_scalar(b) != b) self.assertTrue(_as_scalar(d_a) != d_a) self.assertTrue(_as_scalar(d_b) != d_b) self.assertTrue(_as_scalar(d_a2) != d_a2) def test1(self): # Test that it fails on nonscalar constants a = T.constant(np.ones(5)) self.assertTrue(_as_scalar(a) is None) self.assertTrue(_as_scalar(T.DimShuffle([False], [0, 'x'])(a)) is None) def test2(self): # Test that it works on scalar variables a = T.dscalar() d_a = T.DimShuffle([], [])(a) d_a2 = T.DimShuffle([], ['x', 'x'])(a) self.assertTrue(_as_scalar(a) is a) self.assertTrue(_as_scalar(d_a) is a) self.assertTrue(_as_scalar(d_a2) is a) def test3(self): # Test that it fails on nonscalar variables a = T.matrix() self.assertTrue(_as_scalar(a) is None) self.assertTrue(_as_scalar(T.DimShuffle([False, False], [0, 'x', 1])(a)) is None) class T_real_matrix(TestCase): def test0(self): self.assertTrue(_is_real_matrix(T.DimShuffle([False, False], [1, 0])(T.matrix()))) self.assertTrue(not _is_real_matrix(T.DimShuffle([False], ['x', 0]) (T.dvector()))) def fail(msg): print('FAIL', msg) assert False """ This test suite ensures that Gemm is inserted where it belongs, and that the resulting functions compute the same things as the originals. """ def XYZab(): return T.matrix(), T.matrix(), T.matrix(), T.scalar(), T.scalar() class Failure(Exception): pass def just_gemm(i, o, ishapes=[(4, 3), (3, 5), (4, 5), (), ()], max_graphlen=0, expected_nb_gemm=1): try: f = inplace_func( [In(ii, mutable=True, allow_downcast=True) for ii in i], o, mode='FAST_RUN', on_unused_input='ignore') nb_gemm = 0 for node in f.maker.fgraph.apply_nodes: if isinstance(node.op, T.Dot): raise Failure('dot not changed to gemm_inplace in graph') if node.op == _dot22: raise Failure('_dot22 not changed to gemm_inplace in graph') if node.op == gemm_inplace: nb_gemm += 1 assert nb_gemm == expected_nb_gemm, (nb_gemm, expected_nb_gemm) g = inplace_func(i, o, mode=compile.Mode(linker='py', optimizer=None), allow_input_downcast=True, on_unused_input='ignore') for node in g.maker.fgraph.apply_nodes: if node.op == gemm_inplace: raise Exception('gemm_inplace in original graph') graphlen = len(f.maker.fgraph.toposort()) if max_graphlen and (graphlen <= max_graphlen): # theano.printing.debugprint(f) assert False, 'graphlen=%i>%i' % (graphlen, max_graphlen) rng = np.random.RandomState(unittest_tools.fetch_seed(234)) r0 = f(*[np.asarray(rng.randn(*sh), config.floatX) for sh in ishapes]) rng = np.random.RandomState(unittest_tools.fetch_seed(234)) r1 = g(*[np.asarray(rng.randn(*sh), config.floatX) for sh in ishapes]) max_abs_err = np.max(np.abs(r0[0] - r1[0])) eps = 1.0e-8 if config.floatX == 'float32': eps = 1.0e-6 if max_abs_err > eps: raise Failure('GEMM is computing the wrong output. max_rel_err =', max_abs_err) except Failure: for node in f.maker.fgraph.toposort(): print('GRAPH', node) raise @unittest_tools.assertFailure_fast def test_gemm_opt0(): # Many subgraphs whose dots can be eliminated X, Y, Z, a, b = XYZab() just_gemm([X, Y, Z, a, b], [T.dot(X, Y) * a + Z * b]) just_gemm([X, Y, Z, a, b], [a * T.dot(X, Y) + b * Z]) just_gemm([X, Y, Z, a, b], [b * Z + a * T.dot(X, Y)]) just_gemm([X, Y, Z, a, b], [T.dot(X, Y) * a - Z * b]) just_gemm([X, Y, Z, a, b], [a * T.dot(X, Y) - b * Z]) just_gemm([X, Y, Z, a, b], [b * Z - a * T.dot(X, Y)]) # with transposes (transposes should be pushed through dot in canonicalize) just_gemm([X, Y, Z, a, b], [b * Z.T - a * T.dot(Y.T, X.T)]) just_gemm([X, Y, Z, a, b], [b * Z.T + a * b * T.dot(X, Y).T]) just_gemm([X, Y, Z, a, b], [b * Z + a * T.dot(X, Y).T], ishapes=[(5, 3), (3, 4), (4, 5), (), ()]) # with N multiplications instead of just one just_gemm([X, Y, Z, a, b], [(b * b) * Z * a + (a * a) * T.dot(X, Y) * b]) just_gemm([X, Y, Z, a, b], [Z + T.dot(X, Y)]) just_gemm([X, Y, Z, a, b], [Z * b + T.dot(X, Y)]) just_gemm([X, Y, Z, a, b], [Z + a * b * a * T.dot(X, Y)]) just_gemm([X, Y, Z, a, b], [(b * b) * Z * a - (a * a) * T.dot(X, Y) * b]) just_gemm([X, Y, Z, a, b], [Z - T.dot(X, Y)]) just_gemm([X, Y, Z, a, b], [Z * b - T.dot(X, Y)]) just_gemm([X, Y, Z, a, b], [Z - a * b * a * T.dot(X, Y)]) @unittest_tools.assertFailure_fast def test_gemm_opt_double_gemm(): # This is the pattern that shows up in the autoencoder X, Y, Z, a, b = T.matrix(), T.matrix(), T.matrix(), T.scalar(), T.scalar() R, S, c = T.matrix(), T.matrix(), T.scalar() just_gemm([X, Y, Z, a, b, R, S, c], [Z * c + a * T.dot(X, Y) + b * T.dot(R, S).T], ishapes=[(4, 3), (3, 5), (4, 5), (), (), (5, 9), (9, 4), ()], expected_nb_gemm=2) ishapes = [(4, 3), (3, 5), (4, 5), (), (), (5, 9), (9, 4), ()] i = [X, Y, Z, a, b, R, S, c] o = [(a * T.dot(X, Y) + gemm_inplace(Z, b, S.T, R.T, T.constant(1.0).astype(config.floatX)))] try: f = inplace_func([In(ii, mutable=True) for ii in i], o, mode='FAST_RUN', on_unused_input='ignore') for node in f.maker.fgraph.apply_nodes: if isinstance(node.op, T.Dot): raise Failure('dot in graph') if node.op == _dot22: raise Failure('_dot22 in graph') g = inplace_func(i, o, mode=compile.Mode(linker='py', optimizer=None), on_unused_input='ignore') # for node in g.maker.fgraph.apply_nodes: # if node.op == gemm_inplace: raise Failure('gemm_inplace in graph') rng = np.random.RandomState(unittest_tools.fetch_seed(234)) r0 = f(*[np.asarray(rng.randn(*sh), config.floatX) for sh in ishapes]) rng = np.random.RandomState(unittest_tools.fetch_seed(234)) r1 = g(*[np.asarray(rng.randn(*sh), config.floatX) for sh in ishapes]) max_abs_err = np.max(np.abs(r0[0] - r1[0])) eps = 1.0e-8 if config.floatX == 'float32': eps = 1.0e-6 if max_abs_err > eps: raise Failure( 'GEMM is computing the wrong output. max_rel_err =', max_abs_err) except Failure: for node in f.maker.fgraph.toposort(): print('GRAPH', node) raise def test_gemm_canonicalize(): X, Y, Z, a, b = T.matrix('X'), T.matrix('Y'), T.matrix('Z'), T.scalar( 'a'), T.scalar('b') c, d = T.scalar('c'), T.scalar('d') u = T.row('u') v = T.vector('v') w = T.col('w') can = [] _gemm_canonicalize(X + Y + Z, 1.0, can, 0) assert can == [(1.0, X), (1.0, Y), (1.0, Z)] can = [] _gemm_canonicalize(X + Y + u, 1.0, can, 0) assert can == [(1.0, X), (1.0, Y), (1.0, u)], can can = [] _gemm_canonicalize(X + Y + v, 1.0, can, 0) # [(1.0, X), (1.0, Y), (1.0, InplaceDimShuffle{x,0}(v))] assert can[:2] == [(1.0, X), (1.0, Y)] assert isinstance(can[2], tuple) assert len(can[2]) == 2 assert can[2][0] == 1.0 assert can[2][1].owner assert isinstance(can[2][1].owner.op, T.DimShuffle) assert can[2][1].owner.inputs == [v] can = [] _gemm_canonicalize(X + Y + w, 1.0, can, 0) assert can == [(1.0, X), (1.0, Y), (1.0, w)], can can = [] _gemm_canonicalize(a * X + Y - b * Z * c, 1.0, can, 0) assert can[0] == (a, X) assert can[1] == (1.0, Y) assert can[2][0].owner.op == T.mul assert can[2][0].owner.inputs[0].owner.op == T.neg assert can[2][0].owner.inputs[0].owner.inputs[0] == c assert can[2][0].owner.inputs[1] == b can = [] _gemm_canonicalize((-d) * X - (a * X + Y - b * Z * c), 1.0, can, 0) # print can assert can[0][0].owner.op == T.neg assert can[0][0].owner.inputs[0] == d assert can[0][1] == X assert can[1][0].owner.op == T.neg assert can[1][0].owner.inputs[0] == a assert can[2] == (-1.0, Y) assert can[3][0].owner.op == T.mul assert can[3][0].owner.inputs == [c, b] def test_gemm_factor(): X, Y = T.matrix('X'), T.matrix('Y') assert [(1.0, X), (1.0, Y)] == _factor_canonicalized([(1.0, X), (1.0, Y)]) assert [(2.0, X)] == _factor_canonicalized([(1.0, X), (1.0, X)]) def test_upcasting_scalar_nogemm(): # Test that the optimization does not crash when the scale has an incorrect # dtype, and forces upcasting of the result v = T.fmatrix('v') w = T.fmatrix('w') t = T.fmatrix('t') alpha = T.dscalar('a') rval = T.dot(w, v) * alpha + t f = theano.function([w, v, t, alpha], rval) t = f.maker.fgraph.toposort() assert np.sum([isinstance(n.op, Gemm) for n in t]) == 0 # theano.printing.debugprint(f, print_type=True) v = T.fmatrix('v') w = T.fmatrix('w') t = T.fmatrix('t') alpha = T.cscalar('a') on_opt_error = config.on_opt_error try: config.on_opt_error = 'raise' rval = T.dot(w, v) * alpha + t f = theano.function([w, v, t, alpha], rval) finally: config.on_opt_error = on_opt_error t = f.maker.fgraph.toposort() assert np.sum([isinstance(n.op, Gemm) for n in t]) == 0 # theano.printing.debugprint(f, print_type=True) def test_gemm_nested(): X, Y, Z, a, b = T.matrix('X'), T.matrix('Y'), T.matrix('Z'), T.scalar( 'a'), T.scalar('b') R, S, U, c, d = T.matrix('R'), T.matrix('S'), T.matrix('U'), T.scalar( 'c'), T.scalar('d') just_gemm([X, Y, Z, R, S, U, a, b, c, d], [a * Z - b * (c * T.dot(X, Y) + d * Z)], ishapes=[(2, 3), (3, 4), (2, 4), (2, 3), (3, 4), (2, 4), (), (), (), ()], max_graphlen=1) # print "---------------------" just_gemm([X, Y, Z, R, S, U, a, b, c, d], [a * Z - b * (c * T.dot(X, Y) + d * Z + c * Z)], ishapes=[(2, 3), (3, 4), (2, 4), (2, 3), (3, 4), (2, 4), (), (), (), ()], max_graphlen=1) # print "---------------------" just_gemm([X, Y, Z, R, S, U, a, b, c, d], [a * Z - b * (c * T.dot(X, Y) + d * Z + c * U)], ishapes=[(2, 3), (3, 4), (2, 4), (2, 3), (3, 4), (2, 4), (), (), (), ()], max_graphlen=3) def test_gemm_opt_wishlist(): X, Y, Z, a, b = T.matrix(), T.matrix(), T.matrix(), T.scalar(), T.scalar() # with >2 additions of the same T.dot(X,Y term just_gemm([X, Y, Z, a, b], [(b * b) * Z * a + (a * a) * T.dot(X, Y) + b * T.dot(X, Y)]) just_gemm([X, Y, Z, a, b], [Z + T.dot(X, Y) + T.dot(X, Y)]) def test_gemm_with_vector(): # Many subgraphs whose dots can be eliminated. This adds a # vector two the previous test, which triggers the long-sought GEMM # bug. X, Y, Z, a, b = XYZab() v = T.vector() def my_just_gemm(o): i = [X, Y, Z, a, b, v] ishapes = [(4, 3), (3, 5), (4, 5), (), (), (5, )] just_gemm(i, o, ishapes=ishapes) my_just_gemm([v + T.dot(X, Y) * a + Z * b]) my_just_gemm([v + a * T.dot(X, Y) + b * Z]) my_just_gemm([v + b * Z + a * T.dot(X, Y)]) my_just_gemm([v + T.dot(X, Y) * a - Z * b]) my_just_gemm([v + a * T.dot(X, Y) - b * Z]) my_just_gemm([v + b * Z - a * T.dot(X, Y)]) # with N multiplications instead of just one my_just_gemm([v + (b * b) * Z * a + (a * a) * T.dot(X, Y) * b]) my_just_gemm([v + Z + T.dot(X, Y)]) my_just_gemm([v + Z * b + T.dot(X, Y)]) my_just_gemm([v + Z + a * b * a * T.dot(X, Y)]) my_just_gemm([v + (b * b) * Z * a - (a * a) * T.dot(X, Y) * b]) my_just_gemm([Z - T.dot(X, Y) + v]) my_just_gemm([Z * b - T.dot(X, Y) + v]) my_just_gemm([Z - a * b * a * T.dot(X, Y) + v]) def test_gemm_opt_vector_stuff(): X, Y, a = T.matrix(), T.matrix(), T.scalar() u, v = T.vector(), T.vector() f = inplace_func([a, u, v], a + T.dot(u, v), mode='FAST_RUN') if gemm_inplace in [n.op for n in f.maker.fgraph.apply_nodes]: raise Failure('gemm_inplace in graph') f = inplace_func([a, u, X, Y], a * u + T.dot(X, Y), mode='FAST_RUN') if (gemm_inplace in [n.op for n in f.maker.fgraph.apply_nodes]): raise Failure('gemm_inplace in graph') def test_gemm_unrolled(): # This test that the gemm optimizer remove the dot22 that was # present in the graph. Otherwise, this add a gemm, but still # compute the dot22. # This was not always the case in the with this the following code. batch_size = 100 rep_size = 40 rng = np.random.RandomState([1, 2, 3]) for num_rounds in range(1, 10): W = sharedX(rng.randn(rep_size, rep_size), name='W') V = sharedX(np.zeros((batch_size, rep_size)), name='V') H = sharedX(np.zeros((batch_size, rep_size)), name='H') G = sharedX(np.zeros((batch_size, rep_size)), name='G') cur_V = V cur_H = H def update_V(cur_H): return T.nnet.sigmoid(T.dot(cur_H, W.T)) def update_H(cur_V): return T.nnet.sigmoid(T.dot(cur_V, W) + T.dot(G, W.T)) for i in xrange(num_rounds): cur_V = update_V(cur_H) cur_H = update_H(cur_V) unrolled_theano = theano.function([], updates=[(V, cur_V), (H, cur_H)], name='unrolled_theano') nb_dot = sum([1 for node in unrolled_theano.maker.fgraph.toposort() if isinstance(node.op, (theano.tensor.Dot, theano.tensor.blas.Dot22, theano.tensor.blas.Gemm))]) # Each num_rounds add 3 dot, but one of them is always the same. # So the final graph should have 1 + 2* num_rounds dot varient op. assert nb_dot == num_rounds * 2 + 1, nb_dot unrolled_theano() def test_inplace0(): # should fail to insert gemm_inplace because gemm_inplace would # create cycles X, Y, Z, a, b = T.matrix('X'), T.matrix('Y'), T.matrix('Z'), T.scalar( 'a'), T.scalar('b') R, S, c = T.matrix('R'), T.matrix('S'), T.scalar('c') f = inplace_func([Z, b, R, S], [Z * (Z + b * T.dot(R, S).T)], mode='FAST_RUN') if (gemm_inplace in [n.op for n in f.maker.fgraph.apply_nodes]): print(pp(f.maker.fgraph.outputs[0])) raise Failure('gemm_inplace in graph') assert gemm_no_inplace in [n.op for n in f.maker.fgraph.apply_nodes] # gemm_inplace should be inserted here, to work in-place on Z*c f = inplace_func([X, Y, Z, a, b, R, S, c], [Z * (c * Z + a * T.dot(X, Y) + b * T.dot(R, S).T)], mode='FAST_RUN') if (gemm_inplace not in [n.op for n in f.maker.fgraph.apply_nodes]): theano.printing.debugprint(f) raise Failure('no gemm_inplace in graph') def test_inplace1(): X, Y, Z, a, b = XYZab() # with > 2 terms in the overall addition f = inplace_func([X, Y, Z], [Z + Z + T.dot(X, Y)], mode='FAST_RUN') # theano.printing.debugprint(f) # it doesn't work inplace because we didn't mark Z as mutable input assert [n.op for n in f.maker.fgraph.apply_nodes] == [gemm_no_inplace] def test_dot22(): for dtype1 in ['float32', 'float64', 'complex64', 'complex128']: a = T.matrix(dtype=dtype1) for dtype2 in ['float32', 'float64', 'complex64', 'complex128']: b = T.matrix(dtype=dtype2) f = theano.function([a, b], T.dot(a, b), mode=mode_blas_opt) topo = f.maker.fgraph.toposort() if dtype1 == dtype2: assert _dot22 in [x.op for x in topo], (dtype1, dtype2) else: check = [isinstance(x.op, T.Dot) for x in topo] assert any(check), (dtype1, dtype2) rng = np.random.RandomState(unittest_tools.fetch_seed()) def cmp(a_shp, b_shp): av = rng.uniform(size=a_shp).astype(dtype1) bv = rng.uniform(size=b_shp).astype(dtype2) f(av, bv) cmp((3, 4), (4, 5)) cmp((0, 4), (4, 5)) cmp((3, 0), (0, 5)) cmp((3, 4), (4, 0)) cmp((0, 4), (4, 0)) cmp((0, 0), (0, 0)) @attr('slow') def test_dot22scalar(): # including does not seem to work for 'local_dot_to_dot22' and # 'local_dot22_to_dot22scalar' # TODO: exclude other optimizations in BlasOpt? # m = theano.compile.get_default_mode().including('local_dot_to_dot22', # 'local_dot22_to_dot22scalar','specialize') # m = theano.compile.get_default_mode().including('BlasOpt', 'specialize') rng = np.random.RandomState(unittest_tools.fetch_seed()) for dtype1 in ['complex64', 'complex128']: a = T.matrix('a', dtype=dtype1) for dtype2 in ['complex64', 'complex128']: b = T.matrix('b', dtype=dtype2) for dtype3 in ['complex64', 'complex128']: c = T.matrix('c', dtype=dtype3) for dtype4 in ['complex64', 'complex128']: cst = theano.tensor.basic.constant(.2, dtype=dtype4) cst2 = theano.tensor.basic.constant(.1, dtype=dtype4) def check_dot22scalar(func, len_topo_scalar=-1): topo = func.maker.fgraph.toposort() ops = [x.op for x in topo] dtype4_upcast = theano.scalar.upcast(dtype4, dtype1, dtype2) if dtype1 == dtype2 == dtype3 == dtype4_upcast: if len_topo_scalar > 0: assert len(topo) == len_topo_scalar assert _dot22scalar in ops, (dtype1, dtype2, dtype3, dtype4) elif dtype1 == dtype2 == dtype4_upcast: if not (len_topo_scalar > 0): assert len(topo) == len_topo_scalar assert _dot22scalar in ops, (dtype1, dtype2, dtype3, dtype4) else: # Currently there is a problem of # optimization order The constant get # upcasted to float64 before we try to # merge it with the dot22 of # float32. So this prevent the merge. assert _dot22scalar in ops or _dot22 in ops, ( dtype1, dtype2, dtype3, dtype4) elif dtype1 == dtype2: assert _dot22 in ops, (dtype1, dtype2, dtype3, dtype4) else: check = [isinstance(o, T.Dot) for o in ops] assert any(check), (dtype1, dtype2, dtype3, dtype4) def cmp(a_shp, b_shp, c_shp, sqr_shp=(5, 5)): av = rng.uniform(size=a_shp).astype(dtype1) bv = rng.uniform(size=b_shp).astype(dtype2) cv = rng.uniform(size=c_shp).astype(dtype3) sv = rng.uniform(size=sqr_shp).astype(dtype1) if False: f = theano.function([a, b], cst * T.dot(a, b), mode=mode_blas_opt) f.maker.fgraph.toposort() check_dot22scalar(f, 1) f(av, bv) if True: f = theano.function([a, b, c], cst * c * T.dot(a, b), mode=mode_blas_opt) f.maker.fgraph.toposort() check_dot22scalar(f, 2) f(av, bv, cv) f = theano.function([a, b, c], c * cst * T.dot(a, b), mode=mode_blas_opt) f.maker.fgraph.toposort() check_dot22scalar(f, 2) f(av, bv, cv) # Here, canonicalize also seems needed # TODO: add only the optimizations needed? m2 = mode_blas_opt.including('canonicalize') f = theano.function([a, b, c], cst2 * c * cst * T.dot(a, b), mode=m2) f.maker.fgraph.toposort() check_dot22scalar(f, 2) f(av, bv, cv) if dtype1 == dtype2 == dtype3: f = theano.function([a, b, c], c * cst * a * T.dot(a, b), mode=m2) f.maker.fgraph.toposort() check_dot22scalar(f, 2) f(sv, sv, sv) f = theano.function([a, b, c], cst * c * a * T.dot(a, b), mode=mode_blas_opt) f.maker.fgraph.toposort() # currently the canonizer don't always # merge all Mul together... dot22scalar # optimizer does not do a recursive search # therefore, it doesn't find potential # matches of the scalar. TODO: combine # with the 'canonicalization' that is part # of the Gemm optimizer. # # assert _dot22scalar in [x.op for x in topo] # assert len(topo)==2 f(sv, sv, sv) f = theano.function([a, b, c], c * a * cst * T.dot(a, b), mode=m2) f.maker.fgraph.toposort() check_dot22scalar(f, 2) f(sv, sv, sv) cmp((3, 4), (4, 5), (3, 5)) cmp((0, 4), (4, 5), (0, 5)) cmp((3, 0), (0, 5), (3, 5)) cmp((3, 4), (4, 0), (3, 0), (0, 0)) cmp((0, 4), (4, 0), (0, 0)) cmp((0, 0), (0, 0), (0, 0)) def test_dot22scalar_cast(): # Test that in `dot22_to_dot22scalar` we properly cast integers to floats. # Note that this test was failing before d5ff6904. A = T.dmatrix() for scalar_int_type in T.int_dtypes: y = T.scalar(dtype=scalar_int_type) f = theano.function([A, y], T.dot(A, A) * y, mode=mode_blas_opt) assert _dot22scalar in [x.op for x in f.maker.fgraph.toposort()] A = T.fmatrix() for scalar_int_type in T.int_dtypes: y = T.scalar(dtype=scalar_int_type) f = theano.function([A, y], T.dot(A, A) * y, mode=mode_blas_opt) if scalar_int_type in ['int32', 'int64']: assert _dot22 in [x.op for x in f.maker.fgraph.toposort()] else: assert _dot22scalar in [x.op for x in f.maker.fgraph.toposort()] def test_local_dot22_to_dot22scalar(): # This test that the bug in gh-1507 is really fixed A = T.dmatrix() mode = theano.compile.mode.get_default_mode() opt = theano.tensor.opt.in2out( theano.tensor.blas.local_dot22_to_dot22scalar) mode = mode.__class__(optimizer=opt) x = T.dscalar() y = T.dscalar() z = T.dscalar() # make sure to don't have dimshuffle as we don't opt those cases m = T.dmatrix() r = T.drow() for idx, node in enumerate([ # Old working cases T.mul(_dot22(A, A), x), T.mul(_dot22(A, A), x, y), T.mul(_dot22(A, A), x, r), T.mul(_dot22(A, A), m, x), T.mul(_dot22(A, A), x, m), T.mul(_dot22(A, A), x, (m * y)), T.mul(_dot22(A, A), (m * y), x), T.mul(_dot22(A, A), x, (r * y)), T.mul(_dot22(A, A), (r * y), x), T.mul(_dot22(A, A), (x * y), (m * x)), T.mul(_dot22(A, A), (r * y), (y * x)), # Case that was raising an assert that is fixed in gh-1507 T.mul(_dot22(A, A), (m * y), m), T.mul(_dot22(A, A), m, (m * y)), T.mul(_dot22(A, A), (r * y), (m * x)), # assert fixed in gh-1507 and opt case added in gh-1515 T.mul(_dot22(A, A), (m * y * z), m), T.mul(_dot22(A, A), m, (m * y * z)), # Opt case added in gh-1515 T.mul(_dot22(A, A), T.mul(m, y, z), m), T.mul(_dot22(A, A), m, T.mul(m, y, z)), # Case that opt later in gh-1515 T.mul(_dot22(A, A), (r * m), (m * x)), ]): node2 = theano.tensor.blas.local_dot22_to_dot22scalar.transform( node.owner) assert node2 f = theano.function([x, y, z, m, r, A], node, mode=mode, on_unused_input='ignore') f(.1, .2, .3, [[1, 2], [3, 4]], [[5, 6]], [[7, 8], [9, 10]]) def test_dot_w_self(): # This can trigger problems in the optimization because what would # normally be a gemm must not be because the output is aliased to # one of the inputs. A = shared(value=np.ones((2, 2))) B = T.matrix() p = T.dot(A, A) * B grad = T.grad(T.mean(p), A) f = theano.function([B], p, updates=[(A, A - grad)]) # tests correctness in debugmode f(np.asarray([[0, 1], [2, 3]], dtype=config.floatX)) ############################################################################### # Tests for Gemv ############################################################################### class TestGemv(TestCase, unittest_tools.TestOptimizationMixin): def test_dot_vv(self): # Currently we generate a gemv for that case rng = np.random.RandomState(unittest_tools.fetch_seed()) v = theano.shared(np.array(rng.uniform(size=(2,)), dtype='float32')) w = theano.shared(np.array(rng.uniform(size=(2,)), dtype='float32')) f = theano.function([], theano.dot(v, w), mode=mode_blas_opt) # Assert that the dot was optimized somehow self.assertFunctionContains0(f, T.dot) self.assertFunctionContains1(f, Gemv(True)) # Assert they produce the same output assert np.allclose(f(), np.dot(v.get_value(), w.get_value())) def test_dot_vm(self): # Test vector dot matrix rng = np.random.RandomState(unittest_tools.fetch_seed()) v = theano.shared(np.array(rng.uniform(size=(2,)), dtype='float32')) m = theano.shared(np.array(rng.uniform(size=(2, 3)), dtype='float32')) f = theano.function([], theano.dot(v, m), mode=mode_blas_opt) # Assert that the dot was optimized somehow self.assertFunctionContains0(f, T.dot) self.assertFunctionContains1(f, Gemv(True)) # Assert they produce the same output assert np.allclose(f(), np.dot(v.get_value(), m.get_value())) # Assert it works when m has no contiguous dimension m.set_value(m.get_value(borrow=True)[::-1, ::-1], borrow=True) assert np.allclose(f(), np.dot(v.get_value(), m.get_value())) def test_dot_mv(self): # Test matrix dot vector rng = np.random.RandomState(unittest_tools.fetch_seed()) v = theano.shared(np.array(rng.uniform(size=(2,)), dtype='float32')) m = theano.shared(np.array(rng.uniform(size=(3, 2)), dtype='float32')) f = theano.function([], theano.dot(m, v), mode=mode_blas_opt) # Assert that the dot was optimized somehow self.assertFunctionContains0(f, T.dot) self.assertFunctionContains1(f, Gemv(True)) # Assert they produce the same output assert np.allclose(f(), np.dot(m.get_value(), v.get_value())) # Assert it works when m has no contiguous dimension m.set_value(m.get_value(borrow=True)[::-1, ::-1], borrow=True) assert np.allclose(f(), np.dot(m.get_value(), v.get_value())) @staticmethod def t_gemv1(m_shp): # test vector2+dot(matrix,vector1) rng = np.random.RandomState(unittest_tools.fetch_seed()) v1 = theano.shared(np.array(rng.uniform(size=(m_shp[1],)), dtype='float32')) v2_orig = np.array(rng.uniform(size=(m_shp[0],)), dtype='float32') v2 = theano.shared(v2_orig) m = theano.shared(np.array(rng.uniform(size=m_shp), dtype='float32')) f = theano.function([], v2 + theano.dot(m, v1), mode=mode_blas_opt) # Assert they produce the same output assert np.allclose(f(), np.dot(m.get_value(), v1.get_value()) + v2_orig) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, Gemv) assert topo[0].op.inplace is False # test the inplace version g = theano.function([], [], updates=[(v2, v2 + theano.dot(m, v1))], mode=mode_blas_opt) # Assert they produce the same output g() assert np.allclose(v2.get_value(), np.dot(m.get_value(), v1.get_value()) + v2_orig) topo = g.maker.fgraph.toposort() assert len(topo) == 1 assert isinstance(topo[0].op, Gemv) if config.mode != 'FAST_COMPILE': assert topo[0].op.inplace is True # Do the same tests with a matrix with strides in both dimensions m.set_value(m.get_value(borrow=True)[::-1, ::-1], borrow=True) v2.set_value(v2_orig) assert np.allclose(f(), np.dot(m.get_value(), v1.get_value()) + v2_orig) g() assert np.allclose(v2.get_value(), np.dot(m.get_value(), v1.get_value()) + v2_orig) @attr('slow') def test_gemv1(self): self.t_gemv1((3, 2)) self.t_gemv1((0, 2)) self.t_gemv1((3, 0)) self.t_gemv1((0, 0)) def test_gemv2(self): # test vector2+dot(vector1,matrix) rng = np.random.RandomState(unittest_tools.fetch_seed()) v1 = theano.shared(np.array(rng.uniform(size=(2,)), dtype='float32')) v2_orig = np.array(rng.uniform(size=(3,)), dtype='float32') v2 = theano.shared(v2_orig) m = theano.shared(np.array(rng.uniform(size=(2, 3)), dtype='float32')) f = theano.function([], v2 + theano.dot(v1, m), mode=mode_blas_opt) # Assert they produce the same output assert np.allclose(f(), np.dot(v1.get_value(), m.get_value()) + v2.get_value()) topo = f.maker.fgraph.toposort() assert sum(isinstance(node.op, Gemv) for node in topo) == 1 assert topo[-1].op.inplace is False # test the inplace version g = theano.function([], [], updates=[(v2, v2 + theano.dot(v1, m))], mode=mode_blas_opt) # Assert they produce the same output g() assert np.allclose(v2.get_value(), np.dot(v1.get_value(), m.get_value()) + v2_orig) topo = g.maker.fgraph.toposort() assert sum(isinstance(node.op, Gemv) for node in topo) == 1 if config.mode != 'FAST_COMPILE': assert topo[-1].op.inplace is True # Do the same tests with a matrix with strides in both dimensions m.set_value(m.get_value(borrow=True)[::-1, ::-1], borrow=True) v2.set_value(v2_orig) assert np.allclose(f(), np.dot(v1.get_value(), m.get_value()) + v2.get_value()) g() assert np.allclose(v2.get_value(), np.dot(v1.get_value(), m.get_value()) + v2_orig) def test_gemv_broadcast(self): # test gemv with some broadcasted input rng = np.random.RandomState(unittest_tools.fetch_seed()) v1 = theano.shared(np.array(rng.uniform(size=(2,)), dtype='float32')) v2_orig = np.array(rng.uniform(size=(1,)), dtype='float32') v2 = theano.shared(v2_orig) m = theano.shared(np.array(rng.uniform(size=(1, 2)), dtype='float32'), broadcastable=(True, False)) o = theano.dot(m, v1) f = theano.function([], o + v2, mode=mode_blas_opt) # Assert they produce the same output assert np.allclose( f(), np.dot(m.get_value(), v1.get_value()) + v2.get_value()) topo = f.maker.fgraph.toposort() assert sum(isinstance(node.op, Gemv) for node in topo) == 1 # call gemv directly for mixed broadcast pattern. o = theano.tensor.blas.gemv_no_inplace(v2, 0.5, m, v1, 0.25) f = theano.function([], o, mode=mode_blas_opt) assert np.allclose( f(), 0.5 * np.dot(m.get_value(), v1.get_value()) + 0.25 * v2.get_value()) topo = f.maker.fgraph.toposort() assert sum(isinstance(node.op, Gemv) for node in topo) == 1 def test_gemv_dimensions(self): A = T.matrix('A') x, y = T.vectors('x', 'y') alpha = theano.shared(theano._asarray(1.0, dtype=config.floatX), name='alpha') beta = theano.shared(theano._asarray(1.0, dtype=config.floatX), name='beta') z = beta * y + alpha * T.dot(A, x) f = theano.function([A, x, y], z) # Matrix value A_val = np.ones((5, 3), dtype=config.floatX) # Different vector length ones_3 = np.ones(3, dtype=config.floatX) ones_4 = np.ones(4, dtype=config.floatX) ones_5 = np.ones(5, dtype=config.floatX) ones_6 = np.ones(6, dtype=config.floatX) f(A_val, ones_3, ones_5) f(A_val[::-1, ::-1], ones_3, ones_5) self.assertRaises(ValueError, f, A_val, ones_4, ones_5) self.assertRaises(ValueError, f, A_val, ones_3, ones_6) self.assertRaises(ValueError, f, A_val, ones_4, ones_6) # The following gemv tests were added in March 2011 by Ian Goodfellow # and are based on the gemv tests from scipy # http://projects.scipy.org/scipy/browser/trunk/scipy/linalg/tests/test_fblas.py?rev=6803 # NOTE: At the time these tests were written, theano did not have a # conjugate function. If such a thing is ever added, the tests involving # conjugate should be ported over as well. def matrixmultiply(a, b): if len(b.shape) == 1: b_is_vector = True b = b[:, newaxis] else: b_is_vector = False assert a.shape[1] == b.shape[0] c = zeros((a.shape[0], b.shape[1]), common_type(a, b)) for i in xrange(a.shape[0]): for j in xrange(b.shape[1]): s = 0 for k in xrange(a.shape[1]): s += a[i, k] * b[k, j] c[i, j] = s if b_is_vector: c = c.reshape((a.shape[0],)) return c class BaseGemv(object): mode = mode_blas_opt # can be overridden with self.mode shared = staticmethod(theano.shared) def get_data(self, x_stride=1, y_stride=1): rng = np.random.RandomState(unittest_tools.fetch_seed()) mult = array(1, dtype=self.dtype) if self.dtype in [complex64, complex128]: mult = array(1 + 1j, dtype=self.dtype) alpha = array(1., dtype=self.dtype) * mult beta = array(1., dtype=self.dtype) * mult a = rng.randn(3, 3).astype(self.dtype) * mult x = arange(shape(a)[0] * x_stride, dtype=self.dtype) * mult y = arange(shape(a)[1] * y_stride, dtype=self.dtype) * mult return alpha, beta, a, x, y def test_simple(self): alpha, beta, a, x, y = [self.shared(value) for value in self.get_data()] desired_oy = alpha.get_value() * matrixmultiply(a.get_value(), x.get_value()) + beta.get_value() * y.get_value() oy = alpha * T.dot(a, x) + beta * y oy_func = theano.function([], oy, mode=self.mode) oy_func.maker.fgraph.toposort() self.assertFunctionContains1(oy_func, self.gemv) oy_val = oy_func() assert_array_almost_equal(desired_oy, oy_val) def test_default_beta_y(self): vs = self.get_data() alpha_v, beta_v, a_v, x_v, y_v = vs a = self.shared(a_v) x = self.shared(x_v) desired_oy = matrixmultiply(a_v, x_v) oy = T.dot(a, x) oy_func = theano.function([], oy, mode=self.mode) self.assertFunctionContains1(oy_func, self.gemv_inplace) oy_v = oy_func() assert_array_almost_equal(desired_oy, oy_v) def test_simple_transpose(self): vs = self.get_data() alpha_v, beta_v, a_v, x_v, y_v = vs alpha, beta, a, x, y = [self.shared(v) for v in vs] desired_oy = alpha_v * matrixmultiply(transpose(a_v), x_v) + beta_v * y_v oy = alpha * T.dot(a.T, x) + beta * y oy_func = theano.function([], oy, mode=self.mode) self.assertFunctionContains1(oy_func, self.gemv) oy_v = oy_func() assert_array_almost_equal(desired_oy, oy_v) def test_x_stride(self): vs = self.get_data(x_stride=2) alpha_v, beta_v, a_v, x_v, y_v = vs alpha, beta, a, x, y = [self.shared(v) for v in vs] desired_oy = alpha_v * matrixmultiply(a_v, x_v[::2]) + beta_v * y_v oy = alpha * T.dot(a, x[::2]) + beta * y oy_func = theano.function([], oy, mode=self.mode) self.assertFunctionContains1(oy_func, self.gemv) oy_v = oy_func() assert_array_almost_equal(desired_oy, oy_v) def test_x_stride_transpose(self): vs = self.get_data(x_stride=2) alpha_v, beta_v, a_v, x_v, y_v = vs alpha, beta, a, x, y = [self.shared(v) for v in vs] desired_oy = alpha_v * matrixmultiply(transpose(a_v), x_v[::2]) + \ beta_v * y_v oy = alpha * T.dot(a.T, x[::2]) + beta * y oy_func = theano.function([], oy, mode=self.mode) self.assertFunctionContains1(oy_func, self.gemv) oy_v = oy_func() assert_array_almost_equal(desired_oy, oy_v) def test_y_stride(self): vs = self.get_data(y_stride=2) alpha_v, beta_v, a_v, x_v, y_v = vs alpha, beta, a, x, y = [self.shared(v) for v in vs] desired_oy = alpha_v * matrixmultiply(a_v, x_v) + beta_v * y_v[::2] oy = alpha * T.dot(a, x) + beta * y[::2] oy_func = theano.function([], oy, mode=self.mode) self.assertFunctionContains1(oy_func, self.gemv) oy_v = oy_func() assert_array_almost_equal(desired_oy, oy_v) def test_y_stride_transpose(self): vs = self.get_data(y_stride=2) alpha_v, beta_v, a_v, x_v, y_v = vs alpha, beta, a, x, y = [self.shared(v) for v in vs] desired_oy = alpha_v * matrixmultiply(transpose(a_v), x_v) + beta_v * y_v[::2] oy = alpha * T.dot(a.T, x) + beta * y[::2] oy_func = theano.function([], oy, mode=self.mode) self.assertFunctionContains1(oy_func, self.gemv) oy_v = oy_func() assert_array_almost_equal(desired_oy, oy_v) def test_a_strides(self): vs = self.get_data() alpha_v, beta_v, a_v, x_v, y_v = vs alpha, beta, a, x, y = [self.shared(v) for v in vs] a_v = a_v[::-1, ::-1] a.set_value(a.get_value(borrow=True, return_internal_type=True)[::-1, ::-1], borrow=True) desired_oy = alpha_v * matrixmultiply(a_v, x_v) + beta_v * y_v oy = alpha * T.dot(a, x) + beta * y oy_func = theano.function([], oy, mode=self.mode) self.assertFunctionContains1(oy_func, self.gemv) oy_v = oy_func() assert_array_almost_equal(desired_oy, oy_v) def test_a_strides_transpose(self): vs = self.get_data() alpha_v, beta_v, a_v, x_v, y_v = vs alpha, beta, a, x, y = [self.shared(v) for v in vs] a_v = a_v[::-1, ::-1] a.set_value(a.get_value(borrow=True, return_internal_type=True)[::-1, ::-1], borrow=True) desired_oy = alpha_v * matrixmultiply(transpose(a_v), x_v) + beta_v * y_v oy = alpha * T.dot(a.T, x) + beta * y oy_func = theano.function([], oy, mode=self.mode) self.assertFunctionContains1(oy_func, self.gemv) oy_v = oy_func() assert_array_almost_equal(desired_oy, oy_v) def test_upcasting_scalar_nogemv(self): # Test that the optimization does not crash when the scale has # an incorrect dtype, and forces upcasting of the result # We put this test in this class to test it on the gpu too. vs = self.get_data() alpha_v, beta_v, a_v, x_v, y_v = vs alpha_v = alpha_v.astype("float64") a_v = a_v.astype("float32") x_v = x_v.astype("float32") y_v = y_v.astype("float32") alpha = T.dscalar('alpha') a = self.shared(a_v) x = self.shared(x_v) y = self.shared(y_v) rval = T.dot(a, x) * alpha + y f = theano.function([alpha], rval, mode=self.mode) # this function is currently optimized so that the gemv is # done inplace on a temporarily allocated-buffer, which is # then scaled by alpha and to t with a fused elemwise. n_gemvs = 0 # theano.printing.debugprint(f, print_type=True) for node in f.maker.fgraph.toposort(): if node.op == self.gemv_inplace: n_gemvs += 1 assert node.outputs[0].dtype == 'float32' assert n_gemvs == 1, n_gemvs self.assertFunctionContains1(f, self.gemv_inplace) f(alpha_v) class TestSgemv(TestCase, BaseGemv, unittest_tools.TestOptimizationMixin): dtype = float32 gemv = theano.tensor.blas.gemv_no_inplace gemv_inplace = theano.tensor.blas.gemv_inplace class TestDgemv(TestCase, BaseGemv, unittest_tools.TestOptimizationMixin): dtype = float64 gemv = theano.tensor.blas.gemv_no_inplace gemv_inplace = theano.tensor.blas.gemv_inplace # The optimization to put Gemv don't work for complex type for now. # See ticket 653. # class TestCgemv(TestCase, BaseGemv): # dtype = complex64 # class TestZgemv(TestCase, BaseGemv): # dtype = complex128 ############################################################################### # Tests for Ger ############################################################################### class TestGer_make_node(TestCase): def setUp(self): self.iv = T.tensor(dtype='int32', broadcastable=(False,)) self.fv = T.tensor(dtype='float32', broadcastable=(False,)) self.fv1 = T.tensor(dtype='float32', broadcastable=(True,)) self.dv = T.tensor(dtype='float64', broadcastable=(False,)) self.dv1 = T.tensor(dtype='float64', broadcastable=(True,)) self.cv = T.tensor(dtype='complex64', broadcastable=(False,)) self.zv = T.tensor(dtype='complex128', broadcastable=(False,)) self.fv_2 = T.tensor(dtype='float32', broadcastable=(False,)) self.fv1_2 = T.tensor(dtype='float32', broadcastable=(True,)) self.dv_2 = T.tensor(dtype='float64', broadcastable=(False,)) self.dv1_2 = T.tensor(dtype='float64', broadcastable=(True,)) self.cv_2 = T.tensor(dtype='complex64', broadcastable=(False,)) self.zv_2 = T.tensor(dtype='complex128', broadcastable=(False,)) self.fm = T.fmatrix() self.dm = T.dmatrix() self.cm = T.cmatrix() self.zm = T.zmatrix() self.fa = T.fscalar() self.da = T.dscalar() self.ca = T.cscalar() self.za = T.zscalar() def test_works_on_all_valid_dtypes(self): self.assertEqual(self.fm.type, ger(self.fm, self.fa, self.fv, self.fv_2).type) self.assertEqual(self.fm.type, ger(self.fm, self.fa, self.fv, self.fv_2).type) self.assertEqual(self.fm.type, ger(self.fm, self.fa, self.fv, self.fv_2).type) self.assertEqual(self.fm.type, ger(self.fm, self.fa, self.fv, self.fv_2).type) def test_fails_on_invalid_dtypes(self): self.assertRaises(TypeError, ger, T.imatrix(), T.iscalar(), T.ivector(), T.ivector()) def test_fails_for_nonscalar_alpha(self): self.assertRaises(TypeError, ger, self.fm, self.fm, self.fv, self.fv_2) # boundary case - fv1 has the right dtype and could be dimshuffled to a # scalar, but that's not make_node's job. self.assertRaises(TypeError, ger, self.fm, self.fv1, self.fv, self.fv_2) # actually doing the aforementioned dimshuffle makes it work self.assertEqual(self.fm.type, ger(self.fm, self.fv1.dimshuffle(), self.fv, self.fv_2).type) def test_fails_for_nonmatrix_A(self): self.assertRaises(TypeError, ger, self.fv, self.fa, self.fv, self.fv_2) def test_fails_for_nonvector_x_or_y(self): self.assertRaises(TypeError, ger, self.fm, self.fa, self.fv.dimshuffle('x', 0), self.fv_2) self.assertRaises(TypeError, ger, self.fm, self.fa, self.fv, self.fv_2.dimshuffle('x', 0)) def test_fails_for_mixed_dtypes(self): self.assertRaises(TypeError, ger, self.dm, self.fa, self.fv, self.fv_2) self.assertRaises(TypeError, ger, self.fm, self.da, self.fv, self.fv_2) self.assertRaises(TypeError, ger, self.fm, self.fa, self.dv, self.fv_2) self.assertRaises(TypeError, ger, self.fm, self.fa, self.fv, self.dv_2) self.assertRaises(TypeError, ger, self.cm, self.fa, self.fv, self.dv_2) self.assertRaises(TypeError, ger, self.cm, self.fa, self.fv, self.zv_2) class TestGer_OpContract(TestCase, unittest_tools.T_OpContractMixin): def setUp(self): self.ops = [ger, ger_destructive] def clone(self, op): return Ger(op.destructive) class TestGer(TestCase, unittest_tools.TestOptimizationMixin): shared = staticmethod(theano.shared) def setUp(self): self.mode = theano.compile.get_default_mode().including('fast_run') self.mode = self.mode.excluding('c_blas', 'scipy_blas') dtype = self.dtype = 'float64' # optimization isn't dtype-dependent self.A = T.tensor(dtype=dtype, broadcastable=(False, False)) self.a = T.tensor(dtype=dtype, broadcastable=()) self.x = T.tensor(dtype=dtype, broadcastable=(False,)) self.y = T.tensor(dtype=dtype, broadcastable=(False,)) self.ger = ger self.ger_destructive = ger_destructive self.gemm = gemm_no_inplace def function(self, inputs, outputs, updates=None): if updates is None: updates = [] return theano.function(inputs, outputs, self.mode, updates=updates) def b(self, bval): return T.as_tensor_variable(np.asarray(bval, dtype=self.dtype)) def test_b_0_triggers_ger(self): # test local_gemm_to_ger opt assert T.blas.local_gemm_to_ger.transform( gemm_no_inplace(self.A, self.a, self.x.dimshuffle(0, 'x'), self.y.dimshuffle('x', 0), self.b(0)).owner) def test_b_1_triggers_ger(self): # test local_gemm_to_ger opt assert T.blas.local_gemm_to_ger.transform( gemm_no_inplace(self.A, self.a, self.x.dimshuffle(0, 'x'), self.y.dimshuffle('x', 0), self.b(1)).owner) def test_b_other_does_not_triggers_ger(self): # test local_gemm_to_ger opt assert not T.blas.local_gemm_to_ger.transform( gemm_no_inplace(self.A, self.a, self.x.dimshuffle(0, 'x'), self.y.dimshuffle('x', 0), self.b(1.5)).owner) def test_b_nonconst_does_not_triggers_ger(self): # test local_gemm_to_ger opt assert not T.blas.local_gemm_to_ger.transform( gemm_no_inplace(self.A, self.a, self.x.dimshuffle(0, 'x'), self.y.dimshuffle('x', 0), self.a).owner) def test_outer(self): f = self.function([self.x, self.y], T.outer(self.x, self.y)) self.assertFunctionContains(f, self.ger_destructive) f(np.random.rand(5).astype(self.dtype), np.random.rand(4).astype(self.dtype)) def test_A_plus_outer(self): f = self.function([self.A, self.x, self.y], self.A + T.outer(self.x, self.y)) self.assertFunctionContains(f, self.ger) f(np.random.rand(5, 4).astype(self.dtype), np.random.rand(5).astype(self.dtype), np.random.rand(4).astype(self.dtype)) f(np.random.rand(5, 4).astype(self.dtype)[::-1, ::-1], np.random.rand(5).astype(self.dtype), np.random.rand(4).astype(self.dtype)) def test_A_plus_scaled_outer(self): f = self.function([self.A, self.x, self.y], self.A + 0.1 * T.outer(self.x, self.y)) self.assertFunctionContains(f, self.ger) f(np.random.rand(5, 4).astype(self.dtype), np.random.rand(5).astype(self.dtype), np.random.rand(4).astype(self.dtype)) f(np.random.rand(5, 4).astype(self.dtype)[::-1, ::-1], np.random.rand(5).astype(self.dtype), np.random.rand(4).astype(self.dtype)) def test_scaled_A_plus_scaled_outer(self): f = self.function([self.A, self.x, self.y], np.asarray(0.2, self.dtype) * self.A + np.asarray(0.1, self.dtype) * T.outer( self.x, self.y)) # Why gemm? This make the graph simpler did we test that it # make it faster? self.assertFunctionContains(f, self.gemm) f(np.random.rand(5, 4).astype(self.dtype), np.random.rand(5).astype(self.dtype), np.random.rand(4).astype(self.dtype)) f(np.random.rand(5, 4).astype(self.dtype)[::-1, ::-1], np.random.rand(5).astype(self.dtype), np.random.rand(4).astype(self.dtype)) def given_dtype(self, dtype, M, N): # test corner case shape and dtype f = self.function([self.A, self.x, self.y], self.A + 0.1 * T.outer(self.x, self.y)) self.assertFunctionContains(f, self.ger) f(np.random.rand(M, N).astype(self.dtype), np.random.rand(M).astype(self.dtype), np.random.rand(N).astype(self.dtype)) f(np.random.rand(M, N).astype(self.dtype)[::-1, ::-1], np.random.rand(M).astype(self.dtype), np.random.rand(N).astype(self.dtype)) def test_f32_0_0(self): return self.given_dtype('float32', 0, 0) def test_f32_1_0(self): return self.given_dtype('float32', 1, 0) def test_f32_0_1(self): return self.given_dtype('float32', 0, 1) def test_f32_1_1(self): return self.given_dtype('float32', 1, 1) def test_f32_4_4(self): return self.given_dtype('float32', 4, 4) def test_f32_7_1(self): return self.given_dtype('float32', 7, 1) def test_f32_1_2(self): return self.given_dtype('float32', 1, 2) def test_f64_4_5(self): return self.given_dtype('float64', 4, 5) def test_c64_7_1(self): return self.given_dtype('complex64', 7, 1) def test_c128_1_9(self): return self.given_dtype('complex128', 1, 9) def test_inplace(self): A = self.shared(np.random.rand(4, 5).astype(self.dtype)) f = self.function([self.x, self.y], [], updates=[(A, A + T.constant(0.1, dtype=self.dtype) * T.outer(self.x, self.y))]) self.assertFunctionContains(f, self.ger_destructive) f(np.random.rand(4).astype(self.dtype), np.random.rand(5).astype(self.dtype)) A.set_value( A.get_value(borrow=True, return_internal_type=True)[::-1, ::-1], borrow=True) f(np.random.rand(4).astype(self.dtype), np.random.rand(5).astype(self.dtype)) class TestBlasStrides(TestCase): dtype = 'float64' shared = staticmethod(tensor._shared) mode = theano.compile.get_default_mode() mode = mode.including('fast_run').excluding('gpu', 'c_blas', 'scipy_blas') rng = np.random.RandomState(seed=unittest_tools.fetch_seed()) def rand(self, *shape): return theano._asarray(self.rng.rand(*shape), dtype=self.dtype) def cmp_dot22(self, b_shp, c_shp): av = np.zeros((0, 0), dtype=self.dtype) bv = self.rand(*b_shp) cv = self.rand(*c_shp) a = self.shared(av, 'a') b = self.shared(bv, 'b') c = self.shared(cv, 'c') b_t = self.shared(bv.T, 'b.T') c_t = self.shared(cv.T, 'c.T') b_dev = b.get_value(borrow=False, return_internal_type=True) c_dev = c.get_value(borrow=False, return_internal_type=True) bt_dev = b_t.get_value(borrow=False, return_internal_type=True) ct_dev = c_t.get_value(borrow=False, return_internal_type=True) f_nn = theano.function([], [], updates=[(a, tensor.dot(b, c))], mode=self.mode) # print 'class name:', self.__class__.__name__ # theano.printing.debugprint(f_nn) f_nt = theano.function([], [], updates=[(a, tensor.dot(b, c_t.T))], mode=self.mode) f_tn = theano.function([], [], updates=[(a, tensor.dot(b_t.T, c))], mode=self.mode) f_tt = theano.function([], [], updates=[(a, tensor.dot(b_t.T, c_t.T))], mode=self.mode) # Try with all stride patterns, and all transposed pattern for step_signs in itertools_product((-1, 1), repeat=4): for step in (1, 2): b_step1, b_step2, c_step1, c_step2 = (s * step for s in step_signs) b.set_value(b_dev.copy()[::b_step1, ::b_step2], borrow=True) c.set_value(c_dev.copy()[::c_step1, ::c_step2], borrow=True) b_t.set_value(bt_dev.copy()[::b_step2, ::b_step1], borrow=True) c_t.set_value(ct_dev.copy()[::c_step2, ::c_step1], borrow=True) # Numpy result a_n = np.dot(bv[::b_step1, ::b_step2], cv[::c_step1, ::c_step2]) f_nn() assert np.allclose(a.get_value(), a_n) f_nt() assert np.allclose(a.get_value(), a_n) f_tn() assert np.allclose(a.get_value(), a_n) f_tt() assert np.allclose(a.get_value(), a_n) def test_dot22(self): self.cmp_dot22((3, 4), (4, 5)) self.cmp_dot22((1, 4), (4, 5)) self.cmp_dot22((3, 4), (4, 1)) self.cmp_dot22((3, 1), (1, 1)) self.cmp_dot22((1, 4), (4, 1)) self.cmp_dot22((3, 1), (1, 5)) self.cmp_dot22((0, 4), (4, 5)) self.cmp_dot22((0, 4), (4, 1)) self.cmp_dot22((0, 1), (1, 5)) self.cmp_dot22((3, 4), (4, 0)) self.cmp_dot22((3, 0), (0, 5)) self.cmp_dot22((0, 4), (4, 0)) self.cmp_dot22((0, 0), (0, 0)) def cmp_dot22scalar(self, b_shp, c_shp): av = np.zeros((0, 0), dtype=self.dtype) bv = self.rand(*b_shp) cv = self.rand(*c_shp) l = np.float32(0.2) a = self.shared(av, 'a') b = self.shared(bv, 'b') c = self.shared(cv, 'c') b_t = self.shared(bv.T, 'b.T') c_t = self.shared(cv.T, 'c.T') b_dev = b.get_value(borrow=False, return_internal_type=True) c_dev = c.get_value(borrow=False, return_internal_type=True) bt_dev = b_t.get_value(borrow=False, return_internal_type=True) ct_dev = c_t.get_value(borrow=False, return_internal_type=True) f_nn = theano.function([], [], updates=[(a, l * tensor.dot(b, c))], mode=self.mode) f_nt = theano.function([], [], updates=[(a, l * tensor.dot(b, c_t.T))], mode=self.mode) f_tn = theano.function([], [], updates=[(a, l * tensor.dot(b_t.T, c))], mode=self.mode) f_tt = theano.function([], [], updates=[(a, l * tensor.dot(b_t.T, c_t.T))], mode=self.mode) # Try with all stride patterns, and all transposed pattern for step_signs in itertools_product((-1, 1), repeat=4): for step in (1, 2): b_step1, b_step2, c_step1, c_step2 = (s * step for s in step_signs) b.set_value(b_dev.copy()[::b_step1, ::b_step2], borrow=True) c.set_value(c_dev.copy()[::c_step1, ::c_step2], borrow=True) b_t.set_value(bt_dev.copy()[::b_step2, ::b_step1], borrow=True) c_t.set_value(ct_dev.copy()[::c_step2, ::c_step1], borrow=True) # Numpy result a_n = l * np.dot(bv[::b_step1, ::b_step2], cv[::c_step1, ::c_step2]) f_nn() assert np.allclose(a.get_value(), a_n) f_nt() assert np.allclose(a.get_value(), a_n) f_tn() assert np.allclose(a.get_value(), a_n) f_tt() assert np.allclose(a.get_value(), a_n) def test_dot22scalar(self): self.cmp_dot22scalar((3, 4), (4, 5)) self.cmp_dot22scalar((1, 4), (4, 5)) self.cmp_dot22scalar((3, 4), (4, 1)) self.cmp_dot22scalar((3, 1), (1, 1)) self.cmp_dot22scalar((1, 4), (4, 1)) self.cmp_dot22scalar((3, 1), (1, 5)) self.cmp_dot22scalar((0, 4), (4, 5)) self.cmp_dot22scalar((0, 4), (4, 1)) self.cmp_dot22scalar((0, 1), (1, 5)) self.cmp_dot22scalar((3, 4), (4, 0)) self.cmp_dot22scalar((3, 0), (0, 5)) self.cmp_dot22scalar((0, 4), (4, 0)) self.cmp_dot22scalar((0, 0), (0, 0)) def cmp_gemm(self, a_shp, b_shp, c_shp): av = self.rand(*a_shp) bv = self.rand(*b_shp) cv = self.rand(*c_shp) l = np.float32(0.2) a = self.shared(av, 'a') b = self.shared(bv, 'b') c = self.shared(cv, 'c') a_t = self.shared(av.T, 'a.T') b_t = self.shared(bv.T, 'b.T') c_t = self.shared(cv.T, 'c.T') a_dev = a.get_value(borrow=False, return_internal_type=True) b_dev = b.get_value(borrow=False, return_internal_type=True) c_dev = c.get_value(borrow=False, return_internal_type=True) bt_dev = b_t.get_value(borrow=False, return_internal_type=True) ct_dev = c_t.get_value(borrow=False, return_internal_type=True) f_nnn = theano.function( [], [], updates=[(a, (l * a + tensor.dot(b, c)))], mode=self.mode) f_nnt = theano.function( [], [], updates=[(a, (l * a + tensor.dot(b, c_t.T)))], mode=self.mode) f_ntn = theano.function( [], [], updates=[(a, (l * a + tensor.dot(b_t.T, c)))], mode=self.mode) f_ntt = theano.function( [], [], updates=[(a, (l * a + tensor.dot(b_t.T, c_t.T)))], mode=self.mode) f_tnn = theano.function( [], [], updates=[(a_t, (l * a_t + tensor.dot(b, c).T))], mode=self.mode) f_tnt = theano.function( [], [], updates=[(a_t, (l * a_t + tensor.dot(b, c_t.T).T))], mode=self.mode) f_ttn = theano.function( [], [], updates=[(a_t, (l * a_t + tensor.dot(b_t.T, c).T))], mode=self.mode) f_ttt = theano.function( [], [], updates=[(a_t, (l * a_t + tensor.dot(b_t.T, c_t.T).T))], mode=self.mode) # Try with all stride patterns, and all transposed pattern for step_signs in itertools_product((-1, 1), repeat=6): for step in (1, 2): a_step1, a_step2, b_step1, b_step2, c_step1, c_step2 = \ (s * step for s in step_signs) b.set_value(b_dev.copy()[::b_step1, ::b_step2], borrow=True) c.set_value(c_dev.copy()[::c_step1, ::c_step2], borrow=True) b_t.set_value(bt_dev.copy()[::b_step2, ::b_step1], borrow=True) c_t.set_value(ct_dev.copy()[::c_step2, ::c_step1], borrow=True) # Numpy results a_n = (l * av[::a_step1, ::a_step2] + np.dot(bv[::b_step1, ::b_step2], cv[::c_step1, ::c_step2])) at_n = (l * av[::a_step1, ::a_step2].T + np.dot(bv[::b_step1, ::b_step2], cv[::c_step1, ::c_step2]).T) # a's value is updated, so we need to reinitialize it each time a.set_value(a_dev.copy()[::a_step1, ::a_step2], borrow=True) f_nnn() assert np.allclose(a.get_value(), a_n) a.set_value(a_dev.copy()[::a_step1, ::a_step2], borrow=True) f_nnt() assert np.allclose(a.get_value(), a_n) a.set_value(a_dev.copy()[::a_step1, ::a_step2], borrow=True) f_ntn() assert np.allclose(a.get_value(), a_n) a.set_value(a_dev.copy()[::a_step1, ::a_step2], borrow=True) f_ntt() assert np.allclose(a.get_value(), a_n) a_t.set_value(transpose(a_dev.copy())[::a_step2, ::a_step1], borrow=True) f_tnn() assert np.allclose(a_t.get_value(), at_n) a_t.set_value(transpose(a_dev.copy())[::a_step2, ::a_step1], borrow=True) f_tnt() assert np.allclose(a_t.get_value(), at_n) a_t.set_value(transpose(a_dev.copy())[::a_step2, ::a_step1], borrow=True) f_ttn() assert np.allclose(a_t.get_value(), at_n) a_t.set_value(transpose(a_dev.copy())[::a_step2, ::a_step1], borrow=True) f_ttt() assert np.allclose(a_t.get_value(), at_n) def test_gemm(self): self.cmp_gemm((3, 5), (3, 4), (4, 5)) self.cmp_gemm((1, 5), (1, 4), (4, 5)) self.cmp_gemm((3, 1), (3, 4), (4, 1)) self.cmp_gemm((3, 1), (3, 1), (1, 1)) self.cmp_gemm((1, 1), (1, 4), (4, 1)) self.cmp_gemm((3, 5), (3, 1), (1, 5)) self.cmp_gemm((0, 5), (0, 4), (4, 5)) self.cmp_gemm((0, 1), (0, 4), (4, 1)) self.cmp_gemm((0, 5), (0, 1), (1, 5)) self.cmp_gemm((3, 0), (3, 4), (4, 0)) self.cmp_gemm((3, 5), (3, 0), (0, 5)) self.cmp_gemm((0, 0), (0, 4), (4, 0)) self.cmp_gemm((0, 0), (0, 0), (0, 0)) def cmp_gemv(self, a_shp, b_shp, c_shp): av = self.rand(a_shp) bv = self.rand(*b_shp) cv = self.rand(c_shp) l = np.float32(0.2) a = self.shared(av, 'a') b = self.shared(bv, 'b') c = self.shared(cv, 'c') b_t = self.shared(bv.T, 'b.T') a_dev = a.get_value(borrow=False, return_internal_type=True) b_dev = b.get_value(borrow=False, return_internal_type=True) c_dev = c.get_value(borrow=False, return_internal_type=True) f_n = theano.function([], [], updates=[(a, (a + l * tensor.dot(b, c)))], mode=self.mode) f_t = theano.function([], [], updates=[(a, (a + l * tensor.dot(b_t.T, c)))], mode=self.mode) # Try with all stride patterns, and all transposed pattern for step_signs in itertools_product((1, -1), repeat=4): for step in (1, 2): a_step, b_step1, b_step2, c_step = (s * step for s in step_signs) a.set_value(a_dev.copy()[::a_step], borrow=True) b.set_value(b_dev.copy()[::b_step1, ::b_step2], borrow=True) b_t.set_value(transpose(b_dev.copy())[::b_step2, ::b_step1], borrow=True) c.set_value(c_dev.copy()[::c_step], borrow=True) a_n = (av[::a_step] + l * np.dot(bv[::b_step1, ::b_step2], cv[::c_step])) f_n() assert np.allclose(a.get_value(), a_n), (a.get_value(), a_n) a.set_value(a_dev.copy()[::a_step], borrow=True) f_t() assert np.allclose(a.get_value(), a_n), (a.get_value(), a_n) def test_gemv(self): self.cmp_gemv(3, (3, 5), 5) self.cmp_gemv(1, (1, 5), 5) self.cmp_gemv(3, (3, 1), 1) self.cmp_gemv(0, (0, 5), 5) self.cmp_gemv(3, (3, 0), 0) self.cmp_gemv(0, (0, 1), 1) self.cmp_gemv(1, (1, 0), 0) self.cmp_gemv(0, (0, 0), 0) def cmp_ger(self, a_shp, b_shp, c_shp): av = self.rand(*a_shp) bv = self.rand(b_shp) cv = self.rand(c_shp) l = np.float32(0.2) a = self.shared(av, 'a') b = self.shared(bv, 'b') c = self.shared(cv, 'c') a_t = self.shared(av.T, 'a.T') a_dev = a.get_value(borrow=False, return_internal_type=True) b_dev = b.get_value(borrow=False, return_internal_type=True) c_dev = c.get_value(borrow=False, return_internal_type=True) f_n = theano.function( [], [], updates=[(a, (a + l * tensor.outer(b, c)))], mode=self.mode) f_t = theano.function( [], [], updates=[(a_t, (a_t + l * tensor.outer(b, c).T))], mode=self.mode) # Try with all stride patterns, and all transposed patterns for step_signs in itertools_product((1, -1), repeat=4): for step in (1, 2): a_step1, a_step2, b_step, c_step = (s * step for s in step_signs) a.set_value(a_dev.copy()[::a_step1, ::a_step2], borrow=True) a_t.set_value(transpose(a_dev.copy())[::a_step1, ::a_step2], borrow=True) b.set_value(b_dev.copy()[::b_step], borrow=True) c.set_value(c_dev.copy()[::c_step], borrow=True) f_n() n_n = (av[::a_step1, ::a_step2] + l * np.outer(bv[::b_step], cv[::c_step])) assert np.allclose(a.get_value(), n_n), (a.get_value(), n_n) f_t() n_t = (av.T[::a_step1, ::a_step2] + l * np.outer(bv[::b_step], cv[::c_step]).T) assert np.allclose(a_t.get_value(), n_t), (a_t.get_value(), n_t) def test_ger_strides(self): self.cmp_ger((3, 5), 3, 5) self.cmp_ger((1, 5), 1, 5) self.cmp_ger((3, 1), 3, 1) self.cmp_ger((0, 5), 0, 5) self.cmp_ger((3, 0), 3, 0) self.cmp_ger((0, 1), 0, 1) self.cmp_ger((1, 0), 1, 0) self.cmp_ger((0, 0), 0, 0) def test_gemm_non_contiguous(self): # test_gemm_non_contiguous: Test if GEMM works well with non-contiguous matrices. aval = np.ones((6, 2)) bval = np.ones((2, 7)) cval = np.arange(7) + np.arange(0, .6, .1)[:, np.newaxis] a = theano.shared(aval[:3], borrow=True) b = theano.shared(bval[:, :5], borrow=True) c = theano.shared(cval[:3, :5], borrow=True) s = theano.tensor.scalar() upd_c = s * c + theano.tensor.dot(a, b) f = theano.function([s], [], updates={c: upd_c}) f(0) ref_output = np.ones((3, 5)) * 2 unittest_tools.assert_allclose(c.get_value(), ref_output) class test_infer_shape(unittest_tools.InferShapeTester): def test_dot22(self): x, y = T.matrices('xy') self._compile_and_check( [x, y], [T.blas._dot22(x, y)], [np.random.random((2, 3)).astype(config.floatX), np.random.random((3, 4)).astype(config.floatX)], T.blas.Dot22) def test_dot22scalar(self): x, y = T.matrices('xy') a = T.scalar('a') self._compile_and_check( [x, y, a], [T.blas._dot22scalar(x, y, a)], [np.random.random((2, 3)).astype(config.floatX), np.random.random((3, 4)).astype(config.floatX), np.asarray(0.5, dtype=config.floatX)], T.blas.Dot22Scalar) def test_gemm(self): x, y, z = T.matrices('xyz') a = T.scalar('a') b = T.scalar('b') self._compile_and_check( [x, y, a, z, b], [T.blas.gemm(z, a, x, y, b)], [np.random.random((2, 3)).astype(config.floatX), np.random.random((3, 4)).astype(config.floatX), np.asarray(0.5, dtype=config.floatX), np.random.random((2, 4)).astype(config.floatX), np.asarray(0.5, dtype=config.floatX)], T.blas.Gemm) def test_gemv(self): A = T.matrix('A') x, y = T.vectors('xy') a = T.scalar('a') b = T.scalar('b') self._compile_and_check( [y, a, A, x, b], [T.blas.gemv(y, a, A, x, b)], [np.random.random((2,)).astype(config.floatX), np.asarray(0.5, dtype=config.floatX), np.random.random((2, 3)).astype(config.floatX), np.random.random((3,)).astype(config.floatX), np.asarray(0.5, dtype=config.floatX)], T.blas.Gemv) def test_ger(self): A = T.matrix('A') x, y = T.vectors('xy') a = T.scalar('a') self._compile_and_check( [A, a, x, y], [T.blas.ger(A, a, x, y)], [np.random.random((2, 3)).astype(config.floatX), np.asarray(0.5, dtype=config.floatX), np.random.random((2,)).astype(config.floatX), np.random.random((3,)).astype(config.floatX)], T.blas.Ger) Theano-1.0.0/theano/tensor/tests/test_sort.py00006000163562€ÿÿÿþ0000003760613203335532022635 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division from itertools import product, chain from functools import reduce import unittest from theano.tests import unittest_tools as utt import numpy as np import theano from theano import tensor from theano.tensor.sort import sort, SortOp from theano.tensor.sort import argsort, ArgSortOp from theano.tensor.sort import topk, argtopk, topk_and_argtopk, TopKOp _all_dtypes = tensor.integer_dtypes + tensor.float_dtypes def gen_unique_vector(size, dtype): # generate a randomized vector with unique elements retval = np.arange(size) * 3. + np.random.uniform(-1., 1.) return (retval[np.random.permutation(size)] - size * 1.5).astype(dtype) class Test_sort(unittest.TestCase): def setUp(self): self.rng = np.random.RandomState(seed=utt.fetch_seed()) self.m_val = self.rng.rand(3, 2) self.v_val = self.rng.rand(4) def test1(self): a = tensor.dmatrix() w = sort(a) f = theano.function([a], w) utt.assert_allclose(f(self.m_val), np.sort(self.m_val)) def test2(self): a = tensor.dmatrix() axis = tensor.scalar() w = sort(a, axis) f = theano.function([a, axis], w) for axis_val in 0, 1: gv = f(self.m_val, axis_val) gt = np.sort(self.m_val, axis_val) utt.assert_allclose(gv, gt) def test3(self): a = tensor.dvector() w2 = sort(a) f = theano.function([a], w2) gv = f(self.v_val) gt = np.sort(self.v_val) utt.assert_allclose(gv, gt) def test4(self): a = tensor.dmatrix() axis = tensor.scalar() l = sort(a, axis, "mergesort") f = theano.function([a, axis], l) for axis_val in 0, 1: gv = f(self.m_val, axis_val) gt = np.sort(self.m_val, axis_val) utt.assert_allclose(gv, gt) def test5(self): a1 = SortOp("mergesort", []) a2 = SortOp("quicksort", []) # All the below should give true assert a1 != a2 assert a1 == SortOp("mergesort", []) assert a2 == SortOp("quicksort", []) def test_None(self): a = tensor.dmatrix() l = sort(a, None) f = theano.function([a], l) gv = f(self.m_val) gt = np.sort(self.m_val, None) utt.assert_allclose(gv, gt) def test_grad_vector(self): data = np.random.rand(10).astype(theano.config.floatX) utt.verify_grad(sort, [data]) def test_grad_none_axis(self): data = np.random.rand(10).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, None), [data]) utt.verify_grad(lambda x: sort(x, 0), [data]) data = np.random.rand(2, 3).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, None), [data]) data = np.random.rand(2, 3, 4).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, None), [data]) def test_grad_negative_axis_2d(self): data = np.random.rand(2, 3).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, -1), [data]) data = np.random.rand(2, 3).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, -2), [data]) def test_grad_negative_axis_3d(self): data = np.random.rand(2, 3, 4).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, -1), [data]) data = np.random.rand(2, 3, 4).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, -2), [data]) data = np.random.rand(2, 3, 4).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, -3), [data]) def test_grad_negative_axis_4d(self): data = np.random.rand(2, 3, 4, 2).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, -1), [data]) data = np.random.rand(2, 3, 4, 2).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, -2), [data]) data = np.random.rand(2, 3, 4, 2).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, -3), [data]) data = np.random.rand(2, 3, 4, 2).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, -4), [data]) def test_grad_nonnegative_axis_2d(self): data = np.random.rand(2, 3).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, 0), [data]) data = np.random.rand(2, 3).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, 1), [data]) def test_grad_nonnegative_axis_3d(self): data = np.random.rand(2, 3, 4).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, 0), [data]) data = np.random.rand(2, 3, 4).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, 1), [data]) data = np.random.rand(2, 3, 4).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, 2), [data]) def test_grad_nonnegative_axis_4d(self): data = np.random.rand(2, 3, 4, 2).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, 0), [data]) data = np.random.rand(2, 3, 4, 2).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, 1), [data]) data = np.random.rand(2, 3, 4, 2).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, 2), [data]) data = np.random.rand(2, 3, 4, 2).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, 3), [data]) class SortInferShapeTester(utt.InferShapeTester): def test_sort(self): x = tensor.matrix() self._compile_and_check( [x], [sort(x)], [np.random.randn(10, 40).astype(theano.config.floatX)], SortOp) self._compile_and_check( [x], [sort(x, axis=None)], [np.random.randn(10, 40).astype(theano.config.floatX)], SortOp) def test_argsort(): # Set up rng = np.random.RandomState(seed=utt.fetch_seed()) m_val = rng.rand(3, 2) v_val = rng.rand(4) # Example 1 a = tensor.dmatrix() w = argsort(a) f = theano.function([a], w) gv = f(m_val) gt = np.argsort(m_val) utt.assert_allclose(gv, gt) # Example 2 a = tensor.dmatrix() axis = tensor.lscalar() w = argsort(a, axis) f = theano.function([a, axis], w) for axis_val in 0, 1: gv = f(m_val, axis_val) gt = np.argsort(m_val, axis_val) utt.assert_allclose(gv, gt) # Example 3 a = tensor.dvector() w2 = argsort(a) f = theano.function([a], w2) gv = f(v_val) gt = np.argsort(v_val) utt.assert_allclose(gv, gt) # Example 4 a = tensor.dmatrix() axis = tensor.lscalar() l = argsort(a, axis, "mergesort") f = theano.function([a, axis], l) for axis_val in 0, 1: gv = f(m_val, axis_val) gt = np.argsort(m_val, axis_val) utt.assert_allclose(gv, gt) # Example 5 a = tensor.dmatrix() axis = tensor.lscalar() a1 = ArgSortOp("mergesort", []) a2 = ArgSortOp("quicksort", []) # All the below should give true assert a1 != a2 assert a1 == ArgSortOp("mergesort", []) assert a2 == ArgSortOp("quicksort", []) # Example 6: Testing axis=None a = tensor.dmatrix() w2 = argsort(a, None) f = theano.function([a], w2) gv = f(m_val) gt = np.argsort(m_val, None) utt.assert_allclose(gv, gt) def test_argsort_grad(): # Testing grad of argsort data = np.random.rand(2, 3).astype(theano.config.floatX) utt.verify_grad(lambda x: argsort(x, axis=-1), [data]) data = np.random.rand(2, 3, 4, 5).astype(theano.config.floatX) utt.verify_grad(lambda x: argsort(x, axis=-3), [data]) data = np.random.rand(2, 3, 3).astype(theano.config.floatX) utt.verify_grad(lambda x: argsort(x, axis=2), [data]) class Test_TopK(unittest.TestCase): def setUp(self): pass @utt.parameterized.expand(product( _all_dtypes, tensor.integer_dtypes, [-1, 0, None], [False])) def test_argtopk_sanity(self, dtype, idx_dtype, axis, sorted): x = tensor.vector(name='x', dtype=dtype) fn = theano.function([x], argtopk(x, 1, axis=axis, sorted=sorted, idx_dtype=idx_dtype)) xval = np.asarray([1]).astype(dtype) yval = fn(xval) assert yval == np.asarray([0], dtype=idx_dtype) assert yval.dtype == np.dtype(idx_dtype) @utt.parameterized.expand(product( _all_dtypes, [-1, 0, None], [False])) def test_topk_sanity(self, dtype, axis, sorted): x = tensor.vector(name='x', dtype=dtype) fn = theano.function([x], topk(x, 1, axis=axis, sorted=sorted)) xval = np.asarray([1]).astype(dtype) yval = fn(xval) assert yval == xval assert yval.dtype == xval.dtype @utt.parameterized.expand(product( _all_dtypes, tensor.integer_dtypes, [-1, 0, None], [False])) def test_combined_sanity(self, dtype, idx_dtype, axis, sorted): x = tensor.vector(name='x', dtype=dtype) yv, yi = topk_and_argtopk(x, 1, axis=axis, sorted=sorted, idx_dtype=idx_dtype) fn = theano.function([x], [yv, yi]) xval = np.asarray([1]).astype(dtype) yvval, yival = fn(xval) assert yival == np.asarray([0], dtype=idx_dtype) utt.assert_allclose(xval, yvval) assert yvval.dtype == xval.dtype assert yival.dtype == np.dtype(idx_dtype) @utt.parameterized.expand(chain( product( (16, 61, 257), (1, -1, -10, 'n//2', 'n-1', '-n', '1-n'), ('float64', 'float16', 'int16', 'int8'), (False,)), ((2049, 1337, 'float64', False),))) def test_topk_1d(self, size, k, dtype, sorted): if isinstance(k, str): k = eval(k.replace('n', str(size))) x = theano.tensor.vector(name='x', dtype=dtype) y = topk(x, k, sorted=sorted) fn = theano.function([x], y) # assert local_useless_topk opt is done properly assert 1 == len(fn.maker.fgraph.outputs[0].owner.outputs) # generate a all-unique array xval = gen_unique_vector(size, dtype) yval = fn(xval) idx = (slice(-k, None) if k > 0 else slice(-k)) goal = np.sort(xval)[idx] assert yval.dtype == goal.dtype utt.assert_allclose(np.sort(yval), goal) @utt.parameterized.expand(chain( product( (16, 61, 257), (1, -1, -10, 'n//2', 'n-1', '-n'), ('float32', 'int32'), (False,), ('int32', 'int64')), ((2049, 1337, 'float32', False, 'int32'),))) def test_argtopk_1d(self, size, k, dtype, sorted, idx_dtype): if isinstance(k, str): k = eval(k.replace('n', str(size))) x = theano.tensor.vector(name='x', dtype=dtype) y = argtopk(x, k, sorted=sorted, idx_dtype=idx_dtype) fn = theano.function([x], y) # assert local_useless_topk opt is done properly assert 1 == len(fn.maker.fgraph.outputs[0].owner.outputs) # generate a all-unique array xval = gen_unique_vector(size, dtype) yval = fn(xval) idx = (slice(-k, None) if k > 0 else slice(-k)) goal = np.argsort(xval)[idx].astype(idx_dtype) # due to uniqueness, we expect indices same assert np.all(xval[np.sort(yval)] == xval[np.sort(goal)]) @utt.parameterized.expand(chain( product( (16, 61, 257), (1, -1, 10, 'n//2', 'n-1', '1-n'), ('float32', 'int32'), (False,), ('int32', 'int64')), ((2049, 1337, 'float32', False, 'int32'),))) def test_combined_1d(self, size, k, dtype, sorted, idx_dtype): if isinstance(k, str): k = eval(k.replace('n', str(size))) x = theano.tensor.vector(name='x', dtype=dtype) yv, yi = topk_and_argtopk(x, k, sorted=sorted, idx_dtype=idx_dtype) fn = theano.function([x], [yv, yi]) # generate a all-unique array xval = gen_unique_vector(size, dtype) yvval, yival = fn(xval) idx = (slice(-k, None) if k > 0 else slice(-k)) goali = np.argsort(xval)[idx].astype(idx_dtype) goalv = xval[goali] # due to uniqueness, we expect indices same assert np.all(xval[np.sort(yival)] == xval[np.sort(goali)]) utt.assert_allclose(np.sort(yvval), goalv) @utt.parameterized.expand(chain( product( (18, 62, 258), (1, -1, 'n//2'), ('int32', 'float32'), (False,)), ((2048, 1337, 'float32', False),))) def test_argtopk_1d_collision(self, size, k, dtype, sorted): # with non-unique kth max value if isinstance(k, str): k = eval(k.replace('n', str(size))) x = theano.tensor.vector(name='x', dtype=dtype) y = argtopk(x, k, sorted=sorted, idx_dtype='int32') fn = theano.function([x], y) xval = np.repeat(np.random.uniform(-100., 100., size=size // 2).astype(dtype), 2) xval = xval[np.random.permutation(size)] yval = fn(xval) idx = (slice(-k, None) if k > 0 else slice(-k)) goal = np.argsort(xval)[idx].astype('int32') utt.assert_allclose(np.sort(xval[yval]), np.sort(xval[goal])) @utt.parameterized.expand(product( ((17, 15), (2, 3, 5, 7, 11), (2017, 5, 3)), (-1, '(1+n)//2', '-n', '1-n'), ('float32', 'int32'), (False,), ('int32', 'int64'))) def test_argtopk_nd(self, shp, k_, dtype, sorted, idx_dtype): ndim = len(shp) for axis in range(-ndim, ndim): if isinstance(k_, str): k = eval(k_.replace('n', str(shp[axis]))) else: k = k_ if k == 0: continue x = theano.tensor.tensor( name='x', broadcastable=(False,) * len(shp), dtype=dtype) y = argtopk(x, k, axis=axis, sorted=sorted, idx_dtype=idx_dtype) fn = theano.function([x], y) size = reduce(int.__mul__, shp) xval = gen_unique_vector(size, dtype).reshape(shp) yval = fn(xval) idx = slice(-k, None) if k > 0 else slice(-k) l = axis % ndim r = ndim - l idx = (slice(None),) * l + (idx,) + (slice(None),) * (r - 1) goal = np.argsort(xval, axis=axis)[idx].astype(idx_dtype) assert np.all(np.sort(yval, axis=axis) == np.sort(goal, axis=axis)) @utt.parameterized.expand(product( ((257,), (17, 15), (5, 3, 5, 3), (2, 3, 5, 7, 11)), (1, -1, '(1+n)//2', 'n-1', '-n', '1-n'), (False,))) def test_grad(self, shp, k_, sorted): ndim = len(shp) for axis in range(-ndim, ndim): if isinstance(k_, str): k = eval(k_.replace('n', str(shp[axis]))) else: k = k_ if k == 0: continue # make input away from undefined gradient (where some inputs are equal) xval = gen_unique_vector( reduce(int.__mul__, shp), dtype=theano.config.floatX ).reshape(shp) utt.verify_grad(lambda x: topk(x, k, axis=axis, sorted=sorted), [xval], eps=1e-2) class TopKInferShapeTester(utt.InferShapeTester): @utt.parameterized.expand(product( ((2, 3), (15, 17), (11, 7, 5), (2, 3, 5, 7, 11), (2, 4, 3, 1)), (1, '(1+n)//2', 'n-1', 'n'))) def test_combined_infer_shape(self, shp, k_): ndim = len(shp) for axis in range(-ndim, ndim): if isinstance(k_, str): k = eval(k_.replace('n', str(shp[axis]))) else: k = k_ if k == 0: continue x = theano.tensor.tensor( name='x', broadcastable=(False,) * len(shp), dtype=theano.config.floatX) yv, yi = topk_and_argtopk(x, k, axis=axis, sorted=False, idx_dtype='int32') size = reduce(int.__mul__, shp) xval = gen_unique_vector(size, theano.config.floatX).reshape(shp) self._compile_and_check( [x], [yv, yi], [xval], TopKOp) Theano-1.0.0/theano/tensor/tests/test_complex.py00006000163562€ÿÿÿþ0000000770613203335532023313 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import unittest from six.moves import xrange import theano import numpy as np from theano.tensor import * from theano.tests import unittest_tools as utt from numpy.testing import dec class TestRealImag(unittest.TestCase): def test0(self): x = zvector() rng = np.random.RandomState(23) xval = np.asarray(list(np.complex(rng.randn(), rng.randn()) for i in xrange(10))) assert np.all(xval.real == theano.function([x], real(x))(xval)) assert np.all(xval.imag == theano.function([x], imag(x))(xval)) def test_on_real_input(self): x = dvector() rng = np.random.RandomState(23) xval = rng.randn(10) np.all(0 == theano.function([x], imag(x))(xval)) np.all(xval == theano.function([x], real(x))(xval)) x = imatrix() xval = np.asarray(rng.randn(3, 3) * 100, dtype='int32') np.all(0 == theano.function([x], imag(x))(xval)) np.all(xval == theano.function([x], real(x))(xval)) def test_cast(self): x = zvector() self.assertRaises(TypeError, cast, x, 'int32') def test_complex(self): rng = np.random.RandomState(2333) m = fmatrix() c = complex(m[0], m[1]) assert c.type == cvector r, i = [real(c), imag(c)] assert r.type == fvector assert i.type == fvector f = theano.function([m], [r, i]) mval = np.asarray(rng.randn(2, 5), dtype='float32') rval, ival = f(mval) assert np.all(rval == mval[0]), (rval, mval[0]) assert np.all(ival == mval[1]), (ival, mval[1]) @dec.skipif(True, "Complex grads not enabled, see #178") def test_complex_grads(self): def f(m): c = complex(m[0], m[1]) return .5 * real(c) + .9 * imag(c) rng = np.random.RandomState(9333) mval = np.asarray(rng.randn(2, 5)) utt.verify_grad(f, [mval]) @dec.skipif(True, "Complex grads not enabled, see #178") def test_mul_mixed0(self): def f(a): ac = complex(a[0], a[1]) return abs((ac)**2).sum() rng = np.random.RandomState(9333) aval = np.asarray(rng.randn(2, 5)) try: utt.verify_grad(f, [aval]) except utt.verify_grad.E_grad as e: print(e.num_grad.gf) print(e.analytic_grad) raise @dec.skipif(True, "Complex grads not enabled, see #178") def test_mul_mixed1(self): def f(a): ac = complex(a[0], a[1]) return abs(ac).sum() rng = np.random.RandomState(9333) aval = np.asarray(rng.randn(2, 5)) try: utt.verify_grad(f, [aval]) except utt.verify_grad.E_grad as e: print(e.num_grad.gf) print(e.analytic_grad) raise @dec.skipif(True, "Complex grads not enabled, see #178") def test_mul_mixed(self): def f(a, b): ac = complex(a[0], a[1]) return abs((ac*b)**2).sum() rng = np.random.RandomState(9333) aval = np.asarray(rng.randn(2, 5)) bval = rng.randn(5) try: utt.verify_grad(f, [aval, bval]) except utt.verify_grad.E_grad as e: print(e.num_grad.gf) print(e.analytic_grad) raise @dec.skipif(True, "Complex grads not enabled, see #178") def test_polar_grads(self): def f(m): c = complex_from_polar(abs(m[0]), m[1]) return .5 * real(c) + .9 * imag(c) rng = np.random.RandomState(9333) mval = np.asarray(rng.randn(2, 5)) utt.verify_grad(f, [mval]) @dec.skipif(True, "Complex grads not enabled, see #178") def test_abs_grad(self): def f(m): c = complex(m[0], m[1]) return .5 * abs(c) rng = np.random.RandomState(9333) mval = np.asarray(rng.randn(2, 5)) utt.verify_grad(f, [mval]) Theano-1.0.0/theano/tensor/tests/test_inc_subtensor.py00006000163562€ÿÿÿþ0000001277713203335532024525 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import numpy as np import unittest from theano.tests import unittest_tools as utt import theano import theano.tensor as tt class Test_inc_subtensor(unittest.TestCase): """ Partial testing. What could be tested: - increment vs set - thing incremented: scalar, vector, matrix, - increment/set: constant, scalar, vector, matrix - indices: scalar vs slice, constant vs variable, out of bound, ... - inplace NOTE: these are the same tests as test_incsubtensor.py, but using the new (read: not deprecated) inc_subtensor, set_subtensor functions. """ def setUp(self): utt.seed_rng() def test_simple_2d(self): # Increments or sets part of a tensor by a scalar using full slice and # a partial slice depending on a scalar. a = tt.dmatrix() increment = tt.dscalar() sl1 = slice(None) sl2_end = tt.lscalar() sl2 = slice(sl2_end) for do_set in [False, True]: if do_set: resut = tt.set_subtensor(a[sl1, sl2], increment) else: resut = tt.inc_subtensor(a[sl1, sl2], increment) f = theano.function([a, increment, sl2_end], resut) val_a = np.ones((5, 5)) val_inc = 2.3 val_sl2_end = 2 result = f(val_a, val_inc, val_sl2_end) expected_result = np.copy(val_a) if do_set: expected_result[:, :val_sl2_end] = val_inc else: expected_result[:, :val_sl2_end] += val_inc utt.assert_allclose(result, expected_result) def test_wrong_dims(self): a = tt.matrix() increment = tt.matrix() index = 0 self.assertRaises(TypeError, tt.set_subtensor, a[index], increment) self.assertRaises(TypeError, tt.inc_subtensor, a[index], increment) def test_wrong_broadcast(self): a = tt.col() increment = tt.vector() # These symbolic graphs legitimate, as long as increment has exactly # one element. So it should fail at runtime, not at compile time. rng = np.random.RandomState(utt.fetch_seed()) def rng_randX(*shape): return rng.rand(*shape).astype(theano.config.floatX) for op in (tt.set_subtensor, tt.inc_subtensor): for base in (a[:], a[0]): out = op(base, increment) f = theano.function([a, increment], out) # This one should work f(rng_randX(3, 1), rng_randX(1)) # These ones should not self.assertRaises(ValueError, f, rng_randX(3, 1), rng_randX(2)) self.assertRaises(ValueError, f, rng_randX(3, 1), rng_randX(3)) self.assertRaises(ValueError, f, rng_randX(3, 1), rng_randX(0)) def test_simple_3d(self): # Increments or sets part of a tensor by a scalar using full slice and # a partial slice depending on a scalar. a = tt.dtensor3() increment = tt.dscalar() sl1 = slice(None) sl2_end = tt.lscalar() sl2 = slice(sl2_end) sl3 = 2 val_a = np.ones((5, 3, 4)) val_inc = 2.3 val_sl2_end = 2 for method in [tt.set_subtensor, tt.inc_subtensor]: print("MethodSet", method) resut = method(a[sl1, sl3, sl2], increment) f = theano.function([a, increment, sl2_end], resut) expected_result = np.copy(val_a) result = f(val_a, val_inc, val_sl2_end) if method is tt.set_subtensor: expected_result[:, sl3, :val_sl2_end] = val_inc else: expected_result[:, sl3, :val_sl2_end] += val_inc utt.assert_allclose(result, expected_result) # Test when we broadcast the result resut = method(a[sl1, sl2], increment) f = theano.function([a, increment, sl2_end], resut) expected_result = np.copy(val_a) result = f(val_a, val_inc, val_sl2_end) if method is tt.set_subtensor: expected_result[:, :val_sl2_end] = val_inc else: expected_result[:, :val_sl2_end] += val_inc utt.assert_allclose(result, expected_result) def test_grad_inc_set(self): def inc_slice(*s): def just_numeric_args(a, b): return tt.inc_subtensor(a[s], b) return just_numeric_args def set_slice(*s): def just_numeric_args(a, b): return tt.set_subtensor(a[s], b) return just_numeric_args for f_slice in [inc_slice, set_slice]: # vector utt.verify_grad( f_slice(slice(2, 4, None)), (np.asarray([0, 1, 2, 3, 4, 5.]), np.asarray([9, 9.]), )) # matrix utt.verify_grad( f_slice(slice(1, 2, None), slice(None, None, None)), (np.asarray([[0, 1], [2, 3], [4, 5.]]), np.asarray([[9, 9.]]), )) # single element utt.verify_grad( f_slice(2, 1), (np.asarray([[0, 1], [2, 3], [4, 5.]]), np.asarray(9.),)) # broadcast utt.verify_grad( f_slice(2), (np.asarray([[0, 1], [2, 3], [4, 5.]]), np.asarray(9.),)) Theano-1.0.0/theano/tensor/tests/test_nlinalg.py00006000163562€ÿÿÿþ0000004653213203335532023270 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import unittest import itertools import numpy as np import numpy.linalg from numpy.testing import assert_array_almost_equal from numpy.testing import dec, assert_array_equal, assert_allclose from numpy import inf from six.moves import xrange import theano from theano import tensor, function from theano.tensor.basic import _allclose from theano.tests.test_rop import break_op from theano.tests import unittest_tools as utt from theano import config from theano.tensor.nlinalg import ( MatrixInverse, matrix_inverse, MatrixPinv, pinv, AllocDiag, alloc_diag, ExtractDiag, extract_diag, diag, trace, Det, det, Eig, eig, Eigh, EighGrad, eigh, matrix_dot, _zero_disconnected, qr, matrix_power, norm, svd, SVD, TensorInv, tensorinv, tensorsolve) from nose.plugins.attrib import attr from nose.plugins.skip import SkipTest from nose.tools import assert_raises def test_pseudoinverse_correctness(): rng = np.random.RandomState(utt.fetch_seed()) d1 = rng.randint(4) + 2 d2 = rng.randint(4) + 2 r = rng.randn(d1, d2).astype(theano.config.floatX) x = tensor.matrix() xi = pinv(x) ri = function([x], xi)(r) assert ri.shape[0] == r.shape[1] assert ri.shape[1] == r.shape[0] assert ri.dtype == r.dtype # Note that pseudoinverse can be quite unprecise so I prefer to compare # the result with what np.linalg returns assert _allclose(ri, np.linalg.pinv(r)) def test_pseudoinverse_grad(): rng = np.random.RandomState(utt.fetch_seed()) d1 = rng.randint(4) + 2 d2 = rng.randint(4) + 2 r = rng.randn(d1, d2).astype(theano.config.floatX) utt.verify_grad(pinv, [r]) class test_MatrixInverse(utt.InferShapeTester): def setUp(self): super(test_MatrixInverse, self).setUp() self.op_class = MatrixInverse self.op = matrix_inverse self.rng = np.random.RandomState(utt.fetch_seed()) def test_inverse_correctness(self): r = self.rng.randn(4, 4).astype(theano.config.floatX) x = tensor.matrix() xi = self.op(x) ri = function([x], xi)(r) assert ri.shape == r.shape assert ri.dtype == r.dtype rir = np.dot(ri, r) rri = np.dot(r, ri) assert _allclose(np.identity(4), rir), rir assert _allclose(np.identity(4), rri), rri def test_infer_shape(self): r = self.rng.randn(4, 4).astype(theano.config.floatX) x = tensor.matrix() xi = self.op(x) self._compile_and_check([x], [xi], [r], self.op_class, warn=False) def test_matrix_dot(): rng = np.random.RandomState(utt.fetch_seed()) n = rng.randint(4) + 2 rs = [] xs = [] for k in xrange(n): rs += [rng.randn(4, 4).astype(theano.config.floatX)] xs += [tensor.matrix()] sol = matrix_dot(*xs) theano_sol = function(xs, sol)(*rs) numpy_sol = rs[0] for r in rs[1:]: numpy_sol = np.dot(numpy_sol, r) assert _allclose(numpy_sol, theano_sol) def test_qr_modes(): rng = np.random.RandomState(utt.fetch_seed()) A = tensor.matrix("A", dtype=theano.config.floatX) a = rng.rand(4, 4).astype(theano.config.floatX) f = function([A], qr(A)) t_qr = f(a) n_qr = np.linalg.qr(a) assert _allclose(n_qr, t_qr) for mode in ["reduced", "r", "raw"]: f = function([A], qr(A, mode)) t_qr = f(a) n_qr = np.linalg.qr(a, mode) if isinstance(n_qr, (list, tuple)): assert _allclose(n_qr[0], t_qr[0]) assert _allclose(n_qr[1], t_qr[1]) else: assert _allclose(n_qr, t_qr) try: n_qr = np.linalg.qr(a, "complete") f = function([A], qr(A, "complete")) t_qr = f(a) assert _allclose(n_qr, t_qr) except TypeError as e: assert "name 'complete' is not defined" in str(e) class test_SVD(utt.InferShapeTester): op_class = SVD dtype = 'float32' def setUp(self): super(test_SVD, self).setUp() self.rng = np.random.RandomState(utt.fetch_seed()) self.A = theano.tensor.matrix(dtype=self.dtype) self.op = svd def test_svd(self): A = tensor.matrix("A", dtype=self.dtype) U, S, VT = svd(A) fn = function([A], [U, S, VT]) a = self.rng.rand(4, 4).astype(self.dtype) n_u, n_s, n_vt = np.linalg.svd(a) t_u, t_s, t_vt = fn(a) assert _allclose(n_u, t_u) assert _allclose(n_s, t_s) assert _allclose(n_vt, t_vt) fn = function([A], svd(A, compute_uv=False)) t_s = fn(a) assert _allclose(n_s, t_s) def test_svd_infer_shape(self): self.validate_shape((4, 4), full_matrices=True, compute_uv=True) self.validate_shape((4, 4), full_matrices=False, compute_uv=True) self.validate_shape((2, 4), full_matrices=False, compute_uv=True) self.validate_shape((4, 2), full_matrices=False, compute_uv=True) self.validate_shape((4, 4), compute_uv=False) def validate_shape(self, shape, compute_uv=True, full_matrices=True): A = self.A A_v = self.rng.rand(*shape).astype(self.dtype) outputs = self.op(A, full_matrices=full_matrices, compute_uv=compute_uv) if not compute_uv: outputs = [outputs] self._compile_and_check([A], outputs, [A_v], self.op_class, warn=False) def test_tensorsolve(): rng = np.random.RandomState(utt.fetch_seed()) A = tensor.tensor4("A", dtype=theano.config.floatX) B = tensor.matrix("B", dtype=theano.config.floatX) X = tensorsolve(A, B) fn = function([A, B], [X]) # slightly modified example from np.linalg.tensorsolve docstring a = np.eye(2 * 3 * 4).astype(theano.config.floatX) a.shape = (2 * 3, 4, 2, 3 * 4) b = rng.rand(2 * 3, 4).astype(theano.config.floatX) n_x = np.linalg.tensorsolve(a, b) t_x = fn(a, b) assert _allclose(n_x, t_x) # check the type upcast now C = tensor.tensor4("C", dtype='float32') D = tensor.matrix("D", dtype='float64') Y = tensorsolve(C, D) fn = function([C, D], [Y]) c = np.eye(2 * 3 * 4, dtype='float32') c.shape = (2 * 3, 4, 2, 3 * 4) d = rng.rand(2 * 3, 4).astype('float64') n_y = np.linalg.tensorsolve(c, d) t_y = fn(c, d) assert _allclose(n_y, t_y) assert n_y.dtype == Y.dtype # check the type upcast now E = tensor.tensor4("E", dtype='int32') F = tensor.matrix("F", dtype='float64') Z = tensorsolve(E, F) fn = function([E, F], [Z]) e = np.eye(2 * 3 * 4, dtype='int32') e.shape = (2 * 3, 4, 2, 3 * 4) f = rng.rand(2 * 3, 4).astype('float64') n_z = np.linalg.tensorsolve(e, f) t_z = fn(e, f) assert _allclose(n_z, t_z) assert n_z.dtype == Z.dtype def test_inverse_singular(): singular = np.array([[1, 0, 0]] + [[0, 1, 0]] * 2, dtype=theano.config.floatX) a = tensor.matrix() f = function([a], matrix_inverse(a)) try: f(singular) except np.linalg.LinAlgError: return assert False def test_inverse_grad(): rng = np.random.RandomState(utt.fetch_seed()) r = rng.randn(4, 4) tensor.verify_grad(matrix_inverse, [r], rng=np.random) rng = np.random.RandomState(utt.fetch_seed()) r = rng.randn(4, 4) tensor.verify_grad(matrix_inverse, [r], rng=np.random) def test_det(): rng = np.random.RandomState(utt.fetch_seed()) r = rng.randn(5, 5).astype(config.floatX) x = tensor.matrix() f = theano.function([x], det(x)) assert np.allclose(np.linalg.det(r), f(r)) def test_det_grad(): rng = np.random.RandomState(utt.fetch_seed()) r = rng.randn(5, 5).astype(config.floatX) tensor.verify_grad(det, [r], rng=np.random) def test_det_shape(): rng = np.random.RandomState(utt.fetch_seed()) r = rng.randn(5, 5).astype(config.floatX) x = tensor.matrix() f = theano.function([x], det(x)) f_shape = theano.function([x], det(x).shape) assert np.all(f(r).shape == f_shape(r)) class test_diag(unittest.TestCase): """ Test that linalg.diag has the same behavior as numpy.diag. numpy.diag has two behaviors: (1) when given a vector, it returns a matrix with that vector as the diagonal. (2) when given a matrix, returns a vector which is the diagonal of the matrix. (1) and (2) are tested by test_alloc_diag and test_extract_diag respectively. test_diag test makes sure that linalg.diag instantiates the right op based on the dimension of the input. """ def __init__(self, name, mode=None, shared=tensor._shared, floatX=None, type=tensor.TensorType): self.mode = mode self.shared = shared if floatX is None: floatX = config.floatX self.floatX = floatX self.type = type super(test_diag, self).__init__(name) def test_alloc_diag(self): rng = np.random.RandomState(utt.fetch_seed()) x = theano.tensor.vector() g = alloc_diag(x) f = theano.function([x], g) # test "normal" scenario (5x5 matrix) and special cases of 0x0 and 1x1 for shp in [5, 0, 1]: m = rng.rand(shp).astype(self.floatX) v = np.diag(m) r = f(m) # The right matrix is created assert (r == v).all() # Test we accept only vectors xx = theano.tensor.matrix() ok = False try: alloc_diag(xx) except TypeError: ok = True assert ok # Test infer_shape f = theano.function([x], g.shape) topo = f.maker.fgraph.toposort() if config.mode != 'FAST_COMPILE': assert sum([node.op.__class__ == AllocDiag for node in topo]) == 0 for shp in [5, 0, 1]: m = rng.rand(shp).astype(self.floatX) assert (f(m) == m.shape).all() def test_alloc_diag_grad(self): rng = np.random.RandomState(utt.fetch_seed()) x = rng.rand(5) tensor.verify_grad(alloc_diag, [x], rng=rng) def test_diag(self): # test that it builds a matrix with given diagonal when using # vector inputs x = theano.tensor.vector() y = diag(x) assert y.owner.op.__class__ == AllocDiag # test that it extracts the diagonal when using matrix input x = theano.tensor.matrix() y = extract_diag(x) assert y.owner.op.__class__ == ExtractDiag # not testing the view=True case since it is not used anywhere. def test_extract_diag(self): rng = np.random.RandomState(utt.fetch_seed()) m = rng.rand(2, 3).astype(self.floatX) x = self.shared(m) g = extract_diag(x) f = theano.function([], g) assert [isinstance(node.inputs[0].type, self.type) for node in f.maker.fgraph.toposort() if isinstance(node.op, ExtractDiag)] == [True] for shp in [(2, 3), (3, 2), (3, 3), (1, 1), (0, 0)]: m = rng.rand(*shp).astype(self.floatX) x.set_value(m) v = np.diag(m) r = f() # The right diagonal is extracted assert (r == v).all() # Test we accept only matrix xx = theano.tensor.vector() ok = False try: extract_diag(xx) except TypeError: ok = True except ValueError: ok = True assert ok # Test infer_shape f = theano.function([], g.shape) topo = f.maker.fgraph.toposort() if config.mode != 'FAST_COMPILE': assert sum([node.op.__class__ == ExtractDiag for node in topo]) == 0 for shp in [(2, 3), (3, 2), (3, 3)]: m = rng.rand(*shp).astype(self.floatX) x.set_value(m) assert f() == min(shp) def test_extract_diag_grad(self): rng = np.random.RandomState(utt.fetch_seed()) x = rng.rand(5, 4).astype(self.floatX) tensor.verify_grad(extract_diag, [x], rng=rng) @attr('slow') def test_extract_diag_empty(self): c = self.shared(np.array([[], []], self.floatX)) f = theano.function([], extract_diag(c), mode=self.mode) assert [isinstance(node.inputs[0].type, self.type) for node in f.maker.fgraph.toposort() if isinstance(node.op, ExtractDiag)] == [True] def test_trace(): rng = np.random.RandomState(utt.fetch_seed()) x = theano.tensor.matrix() g = trace(x) f = theano.function([x], g) for shp in [(2, 3), (3, 2), (3, 3)]: m = rng.rand(*shp).astype(config.floatX) v = np.trace(m) assert v == f(m) xx = theano.tensor.vector() ok = False try: trace(xx) except TypeError: ok = True except ValueError: ok = True assert ok class test_Eig(utt.InferShapeTester): op_class = Eig op = eig dtype = 'float64' def setUp(self): super(test_Eig, self).setUp() self.rng = np.random.RandomState(utt.fetch_seed()) self.A = theano.tensor.matrix(dtype=self.dtype) self.X = np.asarray(self.rng.rand(5, 5), dtype=self.dtype) self.S = self.X.dot(self.X.T) def test_infer_shape(self): A = self.A S = self.S self._compile_and_check([A], # theano.function inputs self.op(A), # theano.function outputs # S must be square [S], self.op_class, warn=False) def test_eval(self): A = theano.tensor.matrix(dtype=self.dtype) self.assertEqual([e.eval({A: [[1]]}) for e in self.op(A)], [[1.0], [[1.0]]]) x = [[0, 1], [1, 0]] w, v = [e.eval({A: x}) for e in self.op(A)] assert_array_almost_equal(np.dot(x, v), w * v) class test_Eigh(test_Eig): op = staticmethod(eigh) def test_uplo(self): S = self.S a = theano.tensor.matrix(dtype=self.dtype) wu, vu = [out.eval({a: S}) for out in self.op(a, 'U')] wl, vl = [out.eval({a: S}) for out in self.op(a, 'L')] assert_array_almost_equal(wu, wl) assert_array_almost_equal(vu * np.sign(vu[0, :]), vl * np.sign(vl[0, :])) def test_grad(self): X = self.X # We need to do the dot inside the graph because Eigh needs a # matrix that is hermitian utt.verify_grad(lambda x: self.op(x.dot(x.T))[0], [X], rng=self.rng) utt.verify_grad(lambda x: self.op(x.dot(x.T))[1], [X], rng=self.rng) utt.verify_grad(lambda x: self.op(x.dot(x.T), 'U')[0], [X], rng=self.rng) utt.verify_grad(lambda x: self.op(x.dot(x.T), 'U')[1], [X], rng=self.rng) class test_Eigh_float32(test_Eigh): dtype = 'float32' def test_uplo(self): super(test_Eigh_float32, self).test_uplo() def test_grad(self): super(test_Eigh_float32, self).test_grad() class T_lstsq(unittest.TestCase): def test_correct_solution(self): x = tensor.lmatrix() y = tensor.lmatrix() z = tensor.lscalar() b = theano.tensor.nlinalg.lstsq()(x, y, z) f = function([x, y, z], b) TestMatrix1 = np.asarray([[2, 1], [3, 4]]) TestMatrix2 = np.asarray([[17, 20], [43, 50]]) TestScalar = np.asarray(1) f = function([x, y, z], b) m = f(TestMatrix1, TestMatrix2, TestScalar) self.assertTrue(np.allclose(TestMatrix2, np.dot(TestMatrix1, m[0]))) def test_wrong_coefficient_matrix(self): x = tensor.vector() y = tensor.vector() z = tensor.scalar() b = theano.tensor.nlinalg.lstsq()(x, y, z) f = function([x, y, z], b) self.assertRaises(np.linalg.linalg.LinAlgError, f, [2, 1], [2, 1], 1) def test_wrong_rcond_dimension(self): x = tensor.vector() y = tensor.vector() z = tensor.vector() b = theano.tensor.nlinalg.lstsq()(x, y, z) f = function([x, y, z], b) self.assertRaises(np.linalg.LinAlgError, f, [2, 1], [2, 1], [2, 1]) class Matrix_power(unittest.TestCase): def test_numpy_compare(self): rng = np.random.RandomState(utt.fetch_seed()) A = tensor.matrix("A", dtype=theano.config.floatX) Q = matrix_power(A, 3) fn = function([A], [Q]) a = rng.rand(4, 4).astype(theano.config.floatX) n_p = np.linalg.matrix_power(a, 3) t_p = fn(a) assert np.allclose(n_p, t_p) def test_non_square_matrix(self): rng = np.random.RandomState(utt.fetch_seed()) A = tensor.matrix("A", dtype=theano.config.floatX) Q = matrix_power(A, 3) f = function([A], [Q]) a = rng.rand(4, 3).astype(theano.config.floatX) self.assertRaises(ValueError, f, a) class T_NormTests(unittest.TestCase): def test_wrong_type_of_ord_for_vector(self): self.assertRaises(ValueError, norm, [2, 1], 'fro') def test_wrong_type_of_ord_for_matrix(self): self.assertRaises(ValueError, norm, [[2, 1], [3, 4]], 0) def test_non_tensorial_input(self): self.assertRaises(ValueError, norm, 3, None) def test_tensor_input(self): self.assertRaises(NotImplementedError, norm, np.random.rand(3, 4, 5), None) def test_numpy_compare(self): rng = np.random.RandomState(utt.fetch_seed()) M = tensor.matrix("A", dtype=theano.config.floatX) V = tensor.vector("V", dtype=theano.config.floatX) a = rng.rand(4, 4).astype(theano.config.floatX) b = rng.rand(4).astype(theano.config.floatX) A = ( [None, 'fro', 'inf', '-inf', 1, -1, None, 'inf', '-inf', 0, 1, -1, 2, -2], [M, M, M, M, M, M, V, V, V, V, V, V, V, V], [a, a, a, a, a, a, b, b, b, b, b, b, b, b], [None, 'fro', inf, -inf, 1, -1, None, inf, -inf, 0, 1, -1, 2, -2]) for i in range(0, 14): f = function([A[1][i]], norm(A[1][i], A[0][i])) t_n = f(A[2][i]) n_n = np.linalg.norm(A[2][i], A[3][i]) assert _allclose(n_n, t_n) class test_TensorInv(utt.InferShapeTester): def setUp(self): super(test_TensorInv, self).setUp() self.A = tensor.tensor4("A", dtype=theano.config.floatX) self.B = tensor.tensor3("B", dtype=theano.config.floatX) self.a = np.random.rand(4, 6, 8, 3).astype(theano.config.floatX) self.b = np.random.rand(2, 15, 30).astype(theano.config.floatX) self.b1 = np.random.rand(30, 2, 15).astype(theano.config.floatX) # for ind=1 since we need prod(b1.shape[:ind]) == prod(b1.shape[ind:]) def test_infer_shape(self): A = self.A Ai = tensorinv(A) self._compile_and_check([A], # theano.function inputs [Ai], # theano.function outputs [self.a], # value to substitute TensorInv) def test_eval(self): A = self.A Ai = tensorinv(A) n_ainv = np.linalg.tensorinv(self.a) tf_a = function([A], [Ai]) t_ainv = tf_a(self.a) assert _allclose(n_ainv, t_ainv) B = self.B Bi = tensorinv(B) Bi1 = tensorinv(B, ind=1) n_binv = np.linalg.tensorinv(self.b) n_binv1 = np.linalg.tensorinv(self.b1, ind=1) tf_b = function([B], [Bi]) tf_b1 = function([B], [Bi1]) t_binv = tf_b(self.b) t_binv1 = tf_b1(self.b1) assert _allclose(t_binv, n_binv) assert _allclose(t_binv1, n_binv1) Theano-1.0.0/theano/tensor/tests/mlp_test.py00006000163562€ÿÿÿþ0000003113513203335532022425 0ustar boccoset00000000000000""" This is a minimized version of the mlp.py in the tutorial. We removed stuff that make this mlp don't work. But this test a bug that we saw. This bug made the Shape_i object not being lifted, that caused the CrossentropySoftmax... op not being inserted. """ from __future__ import absolute_import, print_function, division __docformat__ = 'restructedtext en' from collections import OrderedDict import numpy as np import theano import theano.tensor as T def gen_data(): # generate the dataset train_set = (np.asarray(np.random.rand(10000, 784), dtype='float32'), np.asarray(np.random.rand(10000)*10, dtype='int64')) valid_set = (np.asarray(np.random.rand(10000, 784), dtype='float32'), np.asarray(np.random.rand(10000)*10, dtype='int64')) test_set = (np.asarray(np.random.rand(10000, 784), dtype='float32'), np.asarray(np.random.rand(10000)*10, dtype='int64')) def shared_dataset(data_xy): """ Function that loads the dataset into shared variables The reason we store our dataset in shared variables is to allow Theano to copy it into the GPU memory (when code is run on GPU). Since copying data into the GPU is slow, copying a minibatch everytime is needed (the default behaviour if the data is not in a shared variable) would lead to a large decrease in performance. """ data_x, data_y = data_xy shared_x = theano.shared(np.asarray(data_x, dtype=theano.config.floatX)) shared_y = theano.shared(np.asarray(data_y, dtype=theano.config.floatX)) # When storing data on the GPU it has to be stored as floats # therefore we will store the labels as ``floatX`` as well # (``shared_y`` does exactly that). But during our computations # we need them as ints (we use labels as index, and if they are # floats it doesn't make sense) therefore instead of returning # ``shared_y`` we will have to cast it to int. This little hack # lets ous get around this issue return shared_x, T.cast(shared_y, 'int32') test_set_x, test_set_y = shared_dataset(test_set) valid_set_x, valid_set_y = shared_dataset(valid_set) train_set_x, train_set_y = shared_dataset(train_set) rval = [(train_set_x, train_set_y), (valid_set_x, valid_set_y), (test_set_x, test_set_y)] return rval class LogisticRegression(object): """Multi-class Logistic Regression Class The logistic regression is fully described by a weight matrix :math:`W` and bias vector :math:`b`. Classification is done by projecting data points onto a set of hyperplanes, the distance to which is used to determine a class membership probability. """ def __init__(self, input, n_in, n_out, name_prefix=''): """ Initialize the parameters of the logistic regression :type input: theano.tensor.TensorType :param input: symbolic variable that describes the input of the architecture (one minibatch) :type n_in: int :param n_in: number of input units, the dimension of the space in which the datapoints lie :type n_out: int :param n_out: number of output units, the dimension of the space in which the labels lie """ # initialize with 0 the weights W as a matrix of shape (n_in, n_out) self.W = theano.shared(value=np.zeros((n_in, n_out), dtype=theano.config.floatX), name=name_prefix+'W') # compute vector of class-membership probabilities in symbolic form self.p_y_given_x = T.nnet.softmax(T.dot(input, self.W)) # compute prediction as class whose probability is maximal in # symbolic form self.y_pred = T.argmax(self.p_y_given_x, axis=1) # parameters of the model self.params = [self.W] def negative_log_likelihood(self, y): """Return the mean of the negative log-likelihood of the prediction of this model under a given target distribution. .. math:: \frac{1}{|\mathcal{D}|} \mathcal{L} (\theta=\{W,b\}, \mathcal{D}) = \frac{1}{|\mathcal{D}|} \sum_{i=0}^{|\mathcal{D}|} \log(P(Y=y^{(i)}|x^{(i)}, W,b)) \\ \ell (\theta=\{W,b\}, \mathcal{D}) :type y: theano.tensor.TensorType :param y: corresponds to a vector that gives for each example the correct label Note: we use the mean instead of the sum so that the learning rate is less dependent on the batch size """ # y.shape[0] is (symbolically) the number of rows in y, i.e., number of examples (call it n) in the minibatch # T.arange(y.shape[0]) is a symbolic vector which will contain [0,1,2,... n-1] # T.log(self.p_y_given_x) is a matrix of Log-Probabilities (call it LP) with one row per example and one column per class # LP[T.arange(y.shape[0]),y] is a vector v containing [LP[0,y[0]], LP[1,y[1]], LP[2,y[2]], ..., LP[n-1,y[n-1]]] # and T.mean(LP[T.arange(y.shape[0]),y]) is the mean (across minibatch examples) of the elements in v, # i.e., the mean log-likelihood across the minibatch. return T.log(self.p_y_given_x[T.arange(y.shape[0]), y]) class HiddenLayer(object): def __init__(self, rng, input, n_in, n_out, activation=T.tanh, name_prefix=''): """ Typical hidden layer of a MLP: units are fully-connected and have sigmoidal activation function. Weight matrix W is of shape (n_in,n_out) and the bias vector b is of shape (n_out,). NOTE : The nonlinearity used here is tanh Hidden unit activation is given by: tanh(dot(input,W) + b) :type rng: numpy.random.RandomState :param rng: a random number generator used to initialize weights :type input: theano.tensor.dmatrix :param input: a symbolic tensor of shape (n_examples, n_in) :type n_in: int :param n_in: dimensionality of input :type n_out: int :param n_out: number of hidden units :type activation: theano.Op or function :param activation: Non linearity to be applied in the hidden layer """ self.input = input # `W` is initialized with `W_values` which is uniformely sampled # from -6./sqrt(n_in+n_hidden) and 6./sqrt(n_in+n_hidden) # the output of uniform if converted using asarray to dtype # theano.config.floatX so that the code is runable on GPU W_values = np.asarray( rng.uniform( \ low=-np.sqrt(6./(n_in+n_out)), \ high=np.sqrt(6./(n_in+n_out)), \ size=(n_in, n_out)), dtype=theano.config.floatX) self.W = theano.shared(value=W_values, name=name_prefix+'W') self.output = T.dot(input, self.W) # parameters of the model self.params = [self.W] class MLP(object): """Multi-Layer Perceptron Class A multilayer perceptron is a feedforward artificial neural network model that has one layer or more of hidden units and nonlinear activations. Intermidiate layers usually have as activation function thanh or the sigmoid function (defined here by a ``SigmoidalLayer`` class) while the top layer is a softamx layer (defined here by a ``LogisticRegression`` class). """ def __init__(self, rng, input, n_in, n_hidden, n_out): """Initialize the parameters for the multilayer perceptron :type rng: numpy.random.RandomState :param rng: a random number generator used to initialize weights :type input: theano.tensor.TensorType :param input: symbolic variable that describes the input of the architecture (one minibatch) :type n_in: int :param n_in: number of input units, the dimension of the space in which the datapoints lie :type n_hidden: int :param n_hidden: number of hidden units :type n_out: int :param n_out: number of output units, the dimension of the space in which the labels lie """ # Since we are dealing with a one hidden layer MLP, this will # translate into a TanhLayer connected to the LogisticRegression # layer; this can be replaced by a SigmoidalLayer, or a layer # implementing any other nonlinearity self.hiddenLayer = HiddenLayer(rng=rng, input=input, n_in=n_in, n_out=n_hidden, activation=T.tanh, name_prefix='hid_') # The logistic regression layer gets as input the hidden units # of the hidden layer self.logRegressionLayer = LogisticRegression( input=self.hiddenLayer.output, n_in=n_hidden, n_out=n_out, name_prefix='log_') # negative log likelihood of the MLP is given by the negative # log likelihood of the output of the model, computed in the # logistic regression layer self.negative_log_likelihood = self.logRegressionLayer.negative_log_likelihood # the parameters of the model are the parameters of the two layer it is # made out of self.params = self.hiddenLayer.params + self.logRegressionLayer.params def test_mlp(): """ Demonstrate stochastic gradient descent optimization for a multilayer perceptron This is demonstrated on MNIST. :type learning_rate: float :param learning_rate: learning rate used (factor for the stochastic gradient :type n_epochs: int :param n_epochs: maximal number of epochs to run the optimizer :type dataset: string :param dataset: the path of the MNIST dataset file from http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz """ datasets = gen_data() train_set_x, train_set_y = datasets[0] valid_set_x, valid_set_y = datasets[1] test_set_x , test_set_y = datasets[2] batch_size = 100 # size of the minibatch # compute number of minibatches for training, validation and testing n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size ###################### # BUILD ACTUAL MODEL # ###################### # print '... building the model' # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch x = T.matrix('x') # the data is presented as rasterized images y = T.ivector('y') # the labels are presented as 1D vector of # [int] labels rng = np.random.RandomState(1234) # construct the MLP class classifier = MLP( rng=rng, input=x, n_in=28*28, n_hidden=500, n_out=10) # the cost we minimize during training is the negative log likelihood of # the model. # We take the mean of the cost over each minibatch. cost = classifier.negative_log_likelihood(y).mean() # compute the gradient of cost with respect to theta (stored in params) # the resulting gradients will be stored in a list gparams gparams = [] for param in classifier.params: gparam = T.grad(cost, param) gparams.append(gparam) # Some optimizations needed are tagged with 'fast_run' # TODO: refine that and include only those mode = theano.compile.get_default_mode().including('fast_run') updates2 = OrderedDict() updates2[classifier.hiddenLayer.params[0]] = T.grad(cost, classifier.hiddenLayer.params[0]) train_model = theano.function( inputs=[index], updates=updates2, givens={ x: train_set_x[index*batch_size:(index+1)*batch_size], y: train_set_y[index*batch_size:(index+1)*batch_size]}, mode=mode) # print 'MODEL 1' #theano.printing.debugprint(train_model, print_type=True) assert any([isinstance(i.op, T.nnet.CrossentropySoftmax1HotWithBiasDx) for i in train_model.maker.fgraph.toposort()]) # Even without FeatureShape train_model = theano.function( inputs=[index], updates=updates2, mode=mode.excluding('ShapeOpt'), givens={ x: train_set_x[index*batch_size:(index+1)*batch_size], y: train_set_y[index*batch_size:(index+1)*batch_size]}) # print # print 'MODEL 2' #theano.printing.debugprint(train_model, print_type=True) assert any([isinstance(i.op, T.nnet.CrossentropySoftmax1HotWithBiasDx) for i in train_model.maker.fgraph.toposort()]) if __name__ == '__main__': test_mlp() Theano-1.0.0/theano/tensor/tests/test_xlogx.py00006000163562€ÿÿÿþ0000000230213203335532022770 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division from theano.tensor.xlogx import xlogx, xlogy0 import unittest import theano from theano.tensor import as_tensor_variable from . import test_basic as TT import random import numpy.random from theano.tests import unittest_tools as utt class T_XlogX(unittest.TestCase): def setUp(self): utt.seed_rng() def test0(self): x = as_tensor_variable([1, 0]) y = xlogx(x) f = theano.function([], [y]) self.assertTrue(numpy.all(f() == numpy.asarray([0, 0.]))) def test1(self): # class Dummy(object): # def make_node(self, a): # return [xlogx(a)[:,2]] utt.verify_grad(xlogx, [numpy.random.rand(3, 4)]) class T_XlogY0(unittest.TestCase): def setUp(self): utt.seed_rng() def test2(self): utt.verify_grad(xlogy0, [numpy.random.rand(3, 4), numpy.random.rand(3, 4)]) def test3(self): x = as_tensor_variable([1, 0]) y = as_tensor_variable([1, 0]) z = xlogy0(x, y) f = theano.function([], z) self.assertTrue(numpy.all(f() == numpy.asarray([0, 0.]))) if __name__ == '__main__': unittest.main() Theano-1.0.0/theano/tensor/tests/test_misc.py00006000163562€ÿÿÿþ0000000566313203335532022577 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import copy import sys import numpy as np import theano from theano import tensor from theano.tensor.nnet import crossentropy_softmax_argmax_1hot_with_bias def test_bug_2009_06_02_trac_387(): y = tensor.lvector('y') f = theano.function([y], tensor.int_div( tensor.DimShuffle(y[0].broadcastable, ['x'])(y[0]), 2)) print(f(np.ones(1, dtype='int64') * 3)) # XXX: there is no assert, nor comment that DEBUGMODE is to do the # checking. What was the bug, and how is it being tested? def test_bug_2009_07_17_borrowed_output(): # Regression test for a bug where output was borrowed by mistake. a = theano.tensor.dmatrix() b = theano.tensor.dmatrix() # The output should *NOT* be borrowed. g = theano.function([a, b], theano.Out(theano.tensor.dot(a, b), borrow=False)) x = np.zeros((1, 2)) y = np.ones((2, 5)) z = g(x, y) print(z) # Should be zero. x.fill(1) print(g(x, y)) # Should be non-zero. print(z) # Should still be zero. assert np.linalg.norm(z) == 0 # The code above was supposed to fail when it was written (or, more # accurately, on the next revision, i.e. when it was merged with the # rest of the code, i.e. on revision cac9c9e9f08e). # However, for some reason, it does not fail anymore when at this revision. # Thus, a new test (below) was added that exhibits the same issue. Note # that it may better be moved into the test_nnet.py test file if it turns # out the bug was caused by 'crossentropy_softmax_argmax_1hot_with_bias', # and was not a more general issue. test_output_activation_no_bias = theano.tensor.dmatrix() test_b2 = theano.tensor.dvector() test_target = theano.tensor.ivector() nll_softmax_argmax = ( crossentropy_softmax_argmax_1hot_with_bias( test_output_activation_no_bias, test_b2, test_target)) output = nll_softmax_argmax[1] g = theano.function([test_output_activation_no_bias, test_b2, test_target], theano.Out(output, borrow=False)) a = np.zeros((1, 5)) b = np.ones(5) c = np.zeros(1, dtype=np.int32) z = g(a, b, c) z_backup = copy.copy(z) id_z = id(z) print(('Output z after first call: %s' % (z, ))) a[0, 0] = 1 id_other = id(g(a, b, c)) print(('Output z after second call: %s' % (z, ))) # Ensure that calling the function again returns a pointer towards a new # array. assert id_z != id_other # Just to be 100% sure, ensure that z was not altered. assert (z == z_backup).all() def test_deepcopied_type_filter(): a = copy.deepcopy(tensor.matrix()) # The following should run cleanly. # As of commit 731e2d2fa68487733320d341d08b454a50c90d12 # it was failing. a.type.filter( np.ones((2, 2), dtype=a.dtype), strict=True) Theano-1.0.0/theano/tensor/tests/test_blas_c.py00006000163562€ÿÿÿþ0000003527713203335532023073 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import sys import numpy as np from unittest import TestCase from nose.plugins.skip import SkipTest import theano import theano.tensor as tensor from theano.tensor.blas_c import CGer from theano.tensor.blas_scipy import ScipyGer from theano.tensor.blas import Ger from theano.tensor.blas_c import CGemv from theano.tensor.blas import Gemv from theano.tensor.blas_c import check_force_gemv_init from theano.tests import unittest_tools from theano.tests.unittest_tools import TestOptimizationMixin from theano.tensor.tests.test_blas import BaseGemv, TestBlasStrides mode_blas_opt = theano.compile.get_default_mode().including( 'BlasOpt', 'specialize', 'InplaceBlasOpt', 'c_blas') def skip_if_blas_ldflags_empty(*functions_detected): if theano.config.blas.ldflags == "": functions_string = "" if functions_detected: functions_string = " (at least " + (", ".join(functions_detected)) + ")" raise SkipTest("This test is useful only when Theano can access to BLAS functions" + functions_string + " other than [sd]gemm_.") class TestCGer(TestCase, TestOptimizationMixin): def setUp(self, dtype='float64'): # This tests can run even when theano.config.blas.ldflags is empty. self.dtype = dtype self.mode = theano.compile.get_default_mode().including('fast_run') self.A = tensor.tensor(dtype=dtype, broadcastable=(False, False)) self.a = tensor.tensor(dtype=dtype, broadcastable=()) self.x = tensor.tensor(dtype=dtype, broadcastable=(False,)) self.y = tensor.tensor(dtype=dtype, broadcastable=(False,)) self.Aval = np.ones((2, 3), dtype=dtype) self.xval = np.asarray([1, 2], dtype=dtype) self.yval = np.asarray([1.5, 2.7, 3.9], dtype=dtype) def function(self, inputs, outputs): return theano.function(inputs, outputs, mode=self.mode, # allow_inplace=True, ) def run_f(self, f): f(self.Aval, self.xval, self.yval) f(self.Aval[::-1, ::-1], self.xval, self.yval) def b(self, bval): return tensor.as_tensor_variable(np.asarray(bval, dtype=self.dtype)) def test_eq(self): self.assertTrue(CGer(True) == CGer(True)) self.assertTrue(CGer(False) == CGer(False)) self.assertTrue(CGer(False) != CGer(True)) self.assertTrue(CGer(True) != ScipyGer(True)) self.assertTrue(CGer(False) != ScipyGer(False)) self.assertTrue(CGer(True) != Ger(True)) self.assertTrue(CGer(False) != Ger(False)) # assert that eq works for non-CGer instances self.assertTrue(CGer(False) is not None) self.assertTrue(CGer(True) is not None) def test_hash(self): self.assertTrue(hash(CGer(True)) == hash(CGer(True))) self.assertTrue(hash(CGer(False)) == hash(CGer(False))) self.assertTrue(hash(CGer(False)) != hash(CGer(True))) def test_optimization_pipeline(self): skip_if_blas_ldflags_empty() f = self.function([self.x, self.y], tensor.outer(self.x, self.y)) self.assertFunctionContains(f, CGer(destructive=True)) f(self.xval, self.yval) # DebugMode tests correctness def test_optimization_pipeline_float(self): skip_if_blas_ldflags_empty() self.setUp('float32') f = self.function([self.x, self.y], tensor.outer(self.x, self.y)) self.assertFunctionContains(f, CGer(destructive=True)) f(self.xval, self.yval) # DebugMode tests correctness def test_int_fails(self): self.setUp('int32') f = self.function([self.x, self.y], tensor.outer(self.x, self.y)) self.assertFunctionContains0(f, CGer(destructive=True)) self.assertFunctionContains0(f, CGer(destructive=False)) def test_A_plus_outer(self): skip_if_blas_ldflags_empty() f = self.function([self.A, self.x, self.y], self.A + tensor.outer(self.x, self.y)) self.assertFunctionContains(f, CGer(destructive=False)) self.run_f(f) # DebugMode tests correctness def test_A_plus_scaled_outer(self): skip_if_blas_ldflags_empty() f = self.function([self.A, self.x, self.y], self.A + 0.1 * tensor.outer(self.x, self.y)) self.assertFunctionContains(f, CGer(destructive=False)) self.run_f(f) # DebugMode tests correctness class TestCGemv(TestCase, TestOptimizationMixin): """ Tests of CGemv specifically. Generic tests of Gemv-compatibility, including both dtypes are done below in TestCGemvFloat32 and TestCGemvFloat64 """ def setUp(self, dtype='float64'): # This tests can run even when theano.config.blas.ldflags is empty. self.dtype = dtype self.mode = theano.compile.get_default_mode().including('fast_run') # matrix self.A = tensor.tensor(dtype=dtype, broadcastable=(False, False)) self.Aval = np.ones((2, 3), dtype=dtype) # vector self.x = tensor.tensor(dtype=dtype, broadcastable=(False,)) self.y = tensor.tensor(dtype=dtype, broadcastable=(False,)) self.xval = np.asarray([1, 2], dtype=dtype) self.yval = np.asarray([1.5, 2.7, 3.9], dtype=dtype) # scalar self.a = tensor.tensor(dtype=dtype, broadcastable=()) def test_nan_beta_0(self): mode = self.mode.including() mode.check_isfinite = False f = theano.function([self.A, self.x, self.y, self.a], self.a*self.y + theano.dot(self.A, self.x), mode=mode) Aval = np.ones((3, 1), dtype=self.dtype) xval = np.ones((1,), dtype=self.dtype) yval = float('NaN') * np.ones((3,), dtype=self.dtype) zval = f(Aval, xval, yval, 0) assert not np.isnan(zval).any() def test_optimizations_vm(self): skip_if_blas_ldflags_empty() ''' Test vector dot matrix ''' f = theano.function([self.x, self.A], theano.dot(self.x, self.A), mode=self.mode) # Assert that the dot was optimized somehow self.assertFunctionContains0(f, tensor.dot) self.assertFunctionContains1( f, CGemv(inplace=True) ) # Assert they produce the same output assert np.allclose(f(self.xval, self.Aval), np.dot(self.xval, self.Aval)) # Test with negative strides on 2 dims assert np.allclose(f(self.xval, self.Aval[::-1, ::-1]), np.dot(self.xval, self.Aval[::-1, ::-1])) def test_optimizations_mv(self): skip_if_blas_ldflags_empty() ''' Test matrix dot vector ''' f = theano.function([self.A, self.y], theano.dot(self.A, self.y), mode=self.mode) # Assert that the dot was optimized somehow self.assertFunctionContains0(f, tensor.dot) self.assertFunctionContains1( f, CGemv(inplace=True) ) # Assert they produce the same output assert np.allclose(f(self.Aval, self.yval), np.dot(self.Aval, self.yval)) # Test with negative strides on 2 dims assert np.allclose(f(self.Aval[::-1, ::-1], self.yval), np.dot(self.Aval[::-1, ::-1], self.yval)) def test_force_gemv_init(self): if check_force_gemv_init(): sys.stderr.write( "WARNING: The current BLAS requires Theano to initialize" + " memory for some GEMV calls which will result in a minor" + " degradation in performance for such calls." ) def t_gemv1(self, m_shp): ''' test vector2 + dot(matrix, vector1) ''' rng = np.random.RandomState(unittest_tools.fetch_seed()) v1 = theano.shared(np.array(rng.uniform(size=(m_shp[1],)), dtype='float32')) v2_orig = np.array(rng.uniform(size=(m_shp[0],)), dtype='float32') v2 = theano.shared(v2_orig) m = theano.shared(np.array(rng.uniform(size=m_shp), dtype='float32')) f = theano.function([], v2 + tensor.dot(m, v1), mode=self.mode) # Assert they produce the same output assert np.allclose(f(), np.dot(m.get_value(), v1.get_value()) + v2_orig) topo = [n.op for n in f.maker.fgraph.toposort()] assert topo == [CGemv(inplace=False)], topo # test the inplace version g = theano.function([], [], updates=[(v2, v2 + theano.dot(m, v1))], mode=self.mode) # Assert they produce the same output g() assert np.allclose(v2.get_value(), np.dot(m.get_value(), v1.get_value()) + v2_orig) topo = [n.op for n in g.maker.fgraph.toposort()] assert topo == [CGemv(inplace=True)] # Do the same tests with a matrix with strides in both dimensions m.set_value( m.get_value(borrow=True)[::-1, ::-1], borrow=True) v2.set_value(v2_orig) assert np.allclose(f(), np.dot(m.get_value(), v1.get_value()) + v2_orig) g() assert np.allclose(v2.get_value(), np.dot(m.get_value(), v1.get_value()) + v2_orig) def test_gemv1(self): skip_if_blas_ldflags_empty() self.t_gemv1((3, 2)) self.t_gemv1((1, 2)) self.t_gemv1((0, 2)) self.t_gemv1((3, 1)) self.t_gemv1((3, 0)) self.t_gemv1((1, 0)) self.t_gemv1((0, 1)) self.t_gemv1((0, 0)) def test_gemv_dimensions(self, dtype='float32'): alpha = theano.shared(theano._asarray(1.0, dtype=dtype), name='alpha') beta = theano.shared(theano._asarray(1.0, dtype=dtype), name='beta') z = beta * self.y + alpha * tensor.dot(self.A, self.x) f = theano.function([self.A, self.x, self.y], z, mode=self.mode) # Matrix value A_val = np.ones((5, 3), dtype=dtype) # Different vector length ones_3 = np.ones(3, dtype=dtype) ones_4 = np.ones(4, dtype=dtype) ones_5 = np.ones(5, dtype=dtype) ones_6 = np.ones(6, dtype=dtype) f(A_val, ones_3, ones_5) f(A_val[::-1, ::-1], ones_3, ones_5) self.assertRaises(ValueError, f, A_val, ones_4, ones_5) self.assertRaises(ValueError, f, A_val, ones_3, ones_6) self.assertRaises(ValueError, f, A_val, ones_4, ones_6) def test_multiple_inplace(self): skip_if_blas_ldflags_empty() x = tensor.dmatrix('x') y = tensor.dvector('y') z = tensor.dvector('z') f = theano.function([x, y, z], [tensor.dot(y, x), tensor.dot(z,x)], mode=mode_blas_opt) vx = np.random.rand(3, 3) vy = np.random.rand(3) vz = np.random.rand(3) out = f(vx, vy, vz) assert np.allclose(out[0], np.dot(vy, vx)) assert np.allclose(out[1], np.dot(vz, vx)) assert len([n for n in f.maker.fgraph.apply_nodes if isinstance(n.op, tensor.AllocEmpty)]) == 2 class TestCGemvFloat32(TestCase, BaseGemv, TestOptimizationMixin): mode = mode_blas_opt dtype = 'float32' gemv = CGemv(inplace=False) gemv_inplace = CGemv(inplace=True) def setUp(self): skip_if_blas_ldflags_empty() class TestCGemvFloat64(TestCase, BaseGemv, TestOptimizationMixin): mode = mode_blas_opt dtype = 'float64' gemv = CGemv(inplace=False) gemv_inplace = CGemv(inplace=True) def setUp(self): skip_if_blas_ldflags_empty() class TestCGemvNoFlags(object): mode = mode_blas_opt gemv = CGemv(inplace=False) M = 4 N = 5 slice_step = 3 def setUp(self): unittest_tools.seed_rng() def get_function(self, dtype, transpose_A=False, slice_tensors=False): alpha = theano.tensor.scalar(dtype=dtype) beta = theano.tensor.scalar(dtype=dtype) A = theano.tensor.matrix(dtype=dtype) x = theano.tensor.vector(dtype=dtype) y = theano.tensor.vector(dtype=dtype) if transpose_A: A_1 = A.T else: A_1 = A if slice_tensors: A_2 = A_1[::-self.slice_step] x_2 = x[::-self.slice_step] y_2 = y[::-self.slice_step] else: A_2 = A_1 x_2 = x y_2 = y return theano.function([alpha, A, x, beta, y], self.gemv(y_2, alpha, A_2, x_2, beta), mode=self.mode) def get_data(self, dtype, alpha, beta, transpose_A=False, slice_tensors=False): if slice_tensors: if transpose_A: A_shape = (self.N, self.M * self.slice_step) else: A_shape = (self.M * self.slice_step, self.N) x_shape = (self.N * self.slice_step,) y_shape = (self.M * self.slice_step,) else: if transpose_A: A_shape = (self.N, self.M) else: A_shape = (self.M, self.N) x_shape = (self.N,) y_shape = (self.M,) A = np.random.random(A_shape).astype(dtype) x = np.random.random(x_shape).astype(dtype) y = np.random.random(y_shape).astype(dtype) return (alpha, A, x, beta, y) def compute_ref(self, alpha, A, x, beta, y, transpose_A, slice_tensors): if transpose_A: A = A.T if slice_tensors: A = A[::-self.slice_step] x = x[::-self.slice_step] y = y[::-self.slice_step] ref_val = alpha * np.dot(A, x) if beta != 0: ref_val += beta * y return ref_val @theano.change_flags({'blas.ldflags': ''}) def run_cgemv(self, dtype, ALPHA, BETA, transpose_A, slice_tensors): f = self.get_function(dtype, transpose_A=transpose_A, slice_tensors=slice_tensors) values = self.get_data(dtype, ALPHA, BETA, transpose_A=transpose_A, slice_tensors=slice_tensors) assert any(isinstance(node.op, CGemv) for node in f.maker.fgraph.apply_nodes) z_val = f(*values) assert z_val.dtype == dtype assert z_val.ndim == 1 assert z_val.shape[0] == self.M ref_val = self.compute_ref(*(values + (transpose_A, slice_tensors))) unittest_tools.assert_allclose(ref_val, z_val) def test_cgemv(self): for dtype in ('float32', 'float64'): for alpha in (0, 1, -2): for beta in (0, 1, -2): for transpose_A in (False, True): for slice_tensors in (False, True): yield (self.run_cgemv, dtype, alpha, beta, transpose_A, slice_tensors) class TestSdotNoFlags(TestCGemvNoFlags): M = 1 class TestBlasStridesC(TestBlasStrides): mode = mode_blas_opt Theano-1.0.0/theano/tensor/tests/test_casting.py00006000163562€ÿÿÿþ0000001017213203335532023263 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import unittest from theano import function from theano.tensor.basic import (_convert_to_int32, _convert_to_int8, _convert_to_int16, _convert_to_int64, _convert_to_float32, _convert_to_float64) from theano.tensor import * class test_casting(unittest.TestCase): def test_0(self): for op_fn in [_convert_to_int32, _convert_to_float32, _convert_to_float64]: for type_fn in bvector, ivector, fvector, dvector: x = type_fn() f = function([x], op_fn(x)) xval = theano._asarray(np.random.rand(10) * 10, dtype=type_fn.dtype) yval = f(xval) assert (str(yval.dtype) == op_fn.scalar_op.output_types_preference.spec[0].dtype) def test_illegal(self): try: x = zmatrix() function([x], cast(x, 'float64'))(np.ones((2, 3), dtype='complex128')) except TypeError: return assert 0 def test_basic(self): for type1 in ['uint8', 'uint16', 'uint32', 'uint64', 'int8', 'int16', 'int32', 'int64', 'float32', 'float64']: x = TensorType(dtype=type1, broadcastable=(False, ))() for type2, converter in zip(['int8', 'int16', 'int32', 'int64', 'float32', 'float64'], [_convert_to_int8, _convert_to_int16, _convert_to_int32, _convert_to_int64, _convert_to_float32, _convert_to_float64]): y = converter(x) f = function([compile.In(x, strict=True)], y) a = np.arange(10, dtype=type1) b = f(a) self.assertTrue(np.all(b == np.arange(10, dtype=type2))) def test_convert_to_complex(self): val64 = np.ones(3, dtype='complex64') + 0.5j val128 = np.ones(3, dtype='complex128') + 0.5j vec64 = TensorType('complex64', (False, ))() vec128 = TensorType('complex128', (False, ))() f = function([vec64], basic._convert_to_complex128(vec64)) # we need to compare with the same type. assert vec64.type.values_eq_approx(val128, f(val64)) f = function([vec128], basic._convert_to_complex128(vec128)) assert vec64.type.values_eq_approx(val128, f(val128)) f = function([vec64], basic._convert_to_complex64(vec64)) assert vec64.type.values_eq_approx(val64, f(val64)) f = function([vec128], basic._convert_to_complex64(vec128)) assert vec128.type.values_eq_approx(val64, f(val128)) # upcasting to complex128 for t in ['int8', 'int16', 'int32', 'int64', 'float32', 'float64']: a = theano.shared(np.ones(3, dtype=t)) b = theano.shared(np.ones(3, dtype='complex128')) f = function([], basic._convert_to_complex128(a)) assert a.type.values_eq_approx(b.get_value(), f()) # upcasting to complex64 for t in ['int8', 'int16', 'int32', 'int64', 'float32']: a = theano.shared(np.ones(3, dtype=t)) b = theano.shared(np.ones(3, dtype='complex64')) f = function([], basic._convert_to_complex64(a)) assert a.type.values_eq_approx(b.get_value(), f()) # downcast to complex64 for t in ['float64']: a = theano.shared(np.ones(3, dtype=t)) b = theano.shared(np.ones(3, dtype='complex64')) f = function([], basic._convert_to_complex64(a)) assert a.type.values_eq_approx(b.get_value(), f()) def test_bug_complext_10_august_09(self): v0 = dmatrix() v1 = basic._convert_to_complex128(v0) inputs = [v0] outputs = [v1] f = function(inputs, outputs) i = np.zeros((2, 2)) assert (f(i) == np.zeros((2, 2))).all() Theano-1.0.0/theano/tensor/xlogx.py00006000163562€ÿÿÿþ0000000362413203335532020577 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import numpy as np from theano.tensor.elemwise import Elemwise from theano import scalar class XlogX(scalar.UnaryScalarOp): """ Compute X * log(X), with special case 0 log(0) = 0. """ @staticmethod def st_impl(x): if x == 0.0: return 0.0 return x * np.log(x) def impl(self, x): return XlogX.st_impl(x) def grad(self, inputs, grads): x, = inputs gz, = grads return [gz * (1 + scalar.log(x))] def c_code(self, node, name, inputs, outputs, sub): x, = inputs z, = outputs if node.inputs[0].type in [scalar.float32, scalar.float64]: return """%(z)s = %(x)s == 0.0 ? 0.0 : %(x)s * log(%(x)s);""" % locals() raise NotImplementedError('only floatingpoint is implemented') scalar_xlogx = XlogX(scalar.upgrade_to_float, name='scalar_xlogx') xlogx = Elemwise(scalar_xlogx, name='xlogx') class XlogY0(scalar.BinaryScalarOp): """ Compute X * log(Y), with special case 0 log(0) = 0. """ @staticmethod def st_impl(x, y): if x == 0.0: return 0.0 return x * np.log(y) def impl(self, x, y): return XlogY0.st_impl(x, y) def grad(self, inputs, grads): x, y = inputs gz, = grads return [gz * scalar.log(y), gz * x / y] def c_code(self, node, name, inputs, outputs, sub): x, y = inputs z, = outputs if node.inputs[0].type in [scalar.float32, scalar.float64]: return """%(z)s = %(x)s == 0.0 ? 0.0 : %(x)s * log(%(y)s);""" % locals() raise NotImplementedError('only floatingpoint is implemented') scalar_xlogy0 = XlogY0(scalar.upgrade_to_float, name='scalar_xlogy0') xlogy0 = Elemwise(scalar_xlogy0, name='xlogy0') Theano-1.0.0/theano/tensor/elemwise.py00006000163562€ÿÿÿþ0000024334313203335532021254 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division from copy import copy import numpy as np from six import iteritems, integer_types from six.moves import xrange import theano from theano import gof from theano.compat import izip from theano import change_flags from theano.gof import Apply, Op, COp, OpenMPOp, ParamsType from theano import scalar from theano.scalar import get_scalar_type from theano.printing import pprint from theano.gradient import DisconnectedType from theano.gof.null_type import NullType from theano.tensor import elemwise_cgen as cgen from theano.misc.frozendict import frozendict config = theano.config _numpy_ver = [int(n) for n in np.__version__.split('.')[:2]] # tensor depends on elemwise to provide definitions for several ops # but elemwise needs to make TensorType instances, so we have these as # placeholders and the tensor module fills them def as_tensor_variable(data): raise Exception("Circular dependencies prevent using this" "here. import tensor before elemwise") def TensorType(*inputs, **kwargs): raise Exception("Circular dependencies prevent " "using this here. import tensor before elemwise") def TensorVariable(*inputs, **kwargs): raise Exception("Circular dependencies " "prevent using this here. import tensor before elemwise") def TensorConstant(*inputs, **kwargs): raise Exception("Circular dependencies " "prevent using this here. import tensor before elemwise") ################## # DimShuffle # ################## class DimShuffle(COp): """ Allows to reorder the dimensions of a tensor or insert or remove broadcastable dimensions. In the following examples, 'x' means that we insert a broadcastable dimension and a numerical index represents the dimension of the same rank in the tensor passed to perform. Parameters ---------- input_broadcastable The expected broadcastable pattern of the input new_order A list representing the relationship between the input's dimensions and the output's dimensions. Each element of the list can either be an index or 'x'. Indices must be encoded as python integers, not theano symbolic integers. inplace : bool, optional If True (default), the output will be a view of the input. Note ---- If `j = new_order[i]` is an index, the output's ith dimension will be the input's jth dimension. If `new_order[i]` is `x`, the output's ith dimension will be 1 and Broadcast operations will be allowed to do broadcasting over that dimension. If `input.broadcastable[i] == False` then `i` must be found in new_order. Broadcastable dimensions, on the other hand, can be discarded. Note ---- .. code-block:: python DimShuffle((False, False, False), ['x', 2, 'x', 0, 1]) This op will only work on 3d tensors with no broadcastable dimensions. The first dimension will be broadcastable, then we will have the third dimension of the input tensor as the second of the resulting tensor, etc. If the tensor has shape (20, 30, 40), the resulting tensor will have dimensions (1, 40, 1, 20, 30). (AxBxC tensor is mapped to 1xCx1xAxB tensor) .. code-block:: python DimShuffle((True, False), [1]) This op will only work on 2d tensors with the first dimension broadcastable. The second dimension of the input tensor will be the first dimension of the resulting tensor. If the tensor has shape (1, 20), the resulting tensor will have shape (20, ). Example ------- .. code-block:: python DimShuffle((), ['x']) # make a 0d (scalar) into a 1d vector DimShuffle((False, False), [0, 1]) # identity DimShuffle((False, False), [1, 0]) # inverts the 1st and 2nd dimensions DimShuffle((False,), ['x', 0]) # make a row out of a 1d vector # (N to 1xN) DimShuffle((False,), [0, 'x']) # make a column out of a 1d vector # (N to Nx1) DimShuffle((False, False, False), [2, 0, 1]) # AxBxC to CxAxB DimShuffle((False, False), [0, 'x', 1]) # AxB to Ax1xB DimShuffle((False, False), [1, 'x', 0]) # AxB to Bx1xA The reordering of the dimensions can be done with the numpy.transpose function. Adding, subtracting dimensions can be done with reshape. """ _f16_ok = True check_input = False __props__ = ("input_broadcastable", "new_order", "inplace") c_func_file = 'c_code/dimshuffle.c' c_func_name = 'APPLY_SPECIFIC(cpu_dimshuffle)' @property def params_type(self): # We can't directly create `params_type` as class attribute # because of importation issues related to TensorType. return ParamsType(input_broadcastable=TensorType(dtype='bool', broadcastable=(False,)), _new_order=theano.tensor.lvector, transposition=TensorType(dtype='uint32', broadcastable=(False,)), inplace=theano.scalar.bool) @property def _new_order(self): # Param for C code. # self.new_order may contain 'x', which is not a valid integer value. # We replace it with -1. return [(-1 if x == 'x' else x) for x in self.new_order] @property def transposition(self): return self.shuffle + self.drop def __init__(self, input_broadcastable, new_order, inplace=True): COp.__init__(self, [self.c_func_file], self.c_func_name) self.input_broadcastable = tuple(input_broadcastable) self.new_order = tuple(new_order) if inplace is True: self.inplace = inplace else: raise ValueError("DimShuffle is inplace by default and hence the inplace for DimShuffle must be true") for i, j in enumerate(new_order): if j != 'x': # There is a bug in numpy that results in # isinstance(x, integer_types) returning False for # numpy integers. See # . if not isinstance(j, (integer_types, np.integer)): raise TypeError( "DimShuffle indices must be python ints. " "Got: '%s' of type '%s'.", str(j), str(type(j))) if j >= len(input_broadcastable): raise ValueError(("new_order[%d] is %d, but the input " "only has %d axes.") % (i, j, len(input_broadcastable))) if j in new_order[(i + 1):]: raise ValueError("The same input dimension may not appear " "twice in the list of output dimensions", new_order) # list of dimensions of the input to drop self.drop = [] for i, b in enumerate(input_broadcastable): if i not in new_order: # we want to drop this dimension because it's not a value in # new_order if b == 1: # 1 aka True self.drop.append(i) else: # we cannot drop non-broadcastable dimensions raise ValueError( "You cannot drop a non-broadcastable dimension.", (input_broadcastable, new_order)) # this is the list of the original dimensions that we keep self.shuffle = [x for x in new_order if x != 'x'] # list of dimensions of the output that are broadcastable and were not # in the original input self.augment = [i for i, x in enumerate(new_order) if x == 'x'] if self.inplace: self.view_map = {0: [0]} def __setstate__(self, state): self.__dict__.update(state) if not hasattr(self, 'func_files'): # Perhaps we are loading an old `Op` version of DimShuffle. # Let's just build the COp. COp.__init__(self, [self.c_func_file], self.c_func_name) def make_node(self, _input): input = as_tensor_variable(_input) ib = tuple(input.type.broadcastable) if not ib == self.input_broadcastable: if len(ib) != len(self.input_broadcastable): raise TypeError(( "The number of dimensions of the " "input is incorrect for this op. Expected %s, got %s." % (self.input_broadcastable, ib))) for expected, b in zip(self.input_broadcastable, ib): if expected is True and b is False: raise TypeError(( "The broadcastable pattern of the " "input is incorrect for this op. Expected %s, got %s." % (self.input_broadcastable, ib))) # else, expected == b or expected is False and b is True # Both case are good. ob = [] for value in self.new_order: if value == 'x': ob.append(True) else: ob.append(ib[value]) output = TensorType(dtype=input.type.dtype, broadcastable=ob)() return Apply(self, [input], [output]) def __str__(self): if self.inplace: return "InplaceDimShuffle{%s}" % ",".join(str(x) for x in self.new_order) else: return "DimShuffle{%s}" % ",".join(str(x) for x in self.new_order) def perform(self, node, inp, out, params): input, = inp storage, = out # drop res = input if type(res) != np.ndarray and type(res) != np.memmap: raise TypeError(res) # transpose res = res.transpose(self.shuffle + self.drop) # augment shape = list(res.shape[:len(self.shuffle)]) for augm in self.augment: shape.insert(augm, 1) res = res.reshape(shape) # copy (if not inplace) if not self.inplace: res = np.copy(res) storage[0] = np.asarray(res) # asarray puts scalars back into array def infer_shape(self, node, shapes): ishp, = shapes # transpose rval = [ishp[i] for i in self.shuffle] # augment for augm in self.augment: rval.insert(augm, 1) return [rval] def R_op(self, inputs, eval_points): if None in eval_points: return [None] return self(*eval_points, **dict(return_list=True)) def grad(self, inp, grads): x, = inp gz, = grads gz = as_tensor_variable(gz) grad_order = ['x'] * len(x.type.broadcastable) for i, v in enumerate(self.new_order): if v != 'x': grad_order[v] = i # Do not make the DimShuffle inplace as an optimization at the # canonicalization optimization phase will remove the inplace. # The inplace will be reintroduced automatically later in the graph. if inp[0].dtype in theano.tensor.discrete_dtypes: return [inp[0].zeros_like(dtype=theano.config.floatX)] else: return [DimShuffle(gz.type.broadcastable, grad_order)( Elemwise(scalar.identity)(gz))] class DimShufflePrinter: def __p(self, new_order, pstate, r): if new_order != () and new_order[0] == 'x': return "%s" % self.__p(new_order[1:], pstate, r) # return "[%s]" % self.__p(new_order[1:], pstate, r) if list(new_order) == list(range(r.type.ndim)): return pstate.pprinter.process(r) if list(new_order) == list(reversed(range(r.type.ndim))): return "%s.T" % pstate.pprinter.process(r) return "DimShuffle{%s}(%s)" % (", ".join(map(str, new_order)), pstate.pprinter.process(r)) def process(self, r, pstate): if r.owner is None: raise TypeError("Can only print DimShuffle.") elif isinstance(r.owner.op, DimShuffle): ord = r.owner.op.new_order return self.__p(ord, pstate, r.owner.inputs[0]) else: raise TypeError("Can only print DimShuffle.") pprint.assign(DimShuffle, DimShufflePrinter()) ################ # Elemwise # ################ class Elemwise(OpenMPOp): """ Generalizes a scalar op to tensors. All the inputs must have the same number of dimensions. When the Op is performed, for each dimension, each input's size for that dimension must be the same. As a special case, it can also be 1 but only if the input's broadcastable flag is True for that dimension. In that case, the tensor is (virtually) replicated along that dimension to match the size of the others. The dtypes of the outputs mirror those of the scalar Op that is being generalized to tensors. In particular, if the calculations for an output are done inplace on an input, the output type must be the same as the corresponding input type (see the doc of scalar.ScalarOp to get help about controlling the output type) Parameters ---------- scalar_op An instance of a subclass of scalar.ScalarOp which works uniquely on scalars. inplace_pattern A dictionary that maps the index of an output to the index of an input so the output is calculated inplace using the input's storage. (Just like destroymap, but without the lists.) nfunc_spec Either None or a tuple of three elements, (nfunc_name, nin, nout) such that getattr(numpy, nfunc_name) implements this operation, takes nin inputs and nout outputs. Note that nin cannot always be inferred from the scalar op's own nin field because that value is sometimes 0 (meaning a variable number of inputs), whereas the numpy function may not have varargs. Note ---- | Elemwise(add) represents + on tensors (x + y) | Elemwise(add, {0 : 0}) represents the += operation (x += y) | Elemwise(add, {0 : 1}) represents += on the second argument (y += x) | Elemwise(mul)(rand(10, 5), rand(1, 5)) the second input is completed \ along the first dimension to match the first input | Elemwise(true_div)(rand(10, 5), rand(10, 1)) same but along the \ second dimension | Elemwise(int_div)(rand(1, 5), rand(10, 1)) the output has size (10, 5) | Elemwise(log)(rand(3, 4, 5)) """ __props__ = ("scalar_op", "inplace_pattern") def __init__(self, scalar_op, inplace_pattern=None, name=None, nfunc_spec=None, openmp=None): if inplace_pattern is None: inplace_pattern = frozendict({}) self.name = name self.scalar_op = scalar_op self.inplace_pattern = inplace_pattern self.destroy_map = dict((o, [i]) for o, i in self.inplace_pattern.items()) if nfunc_spec is None: nfunc_spec = getattr(scalar_op, 'nfunc_spec', None) self.nfunc_spec = nfunc_spec self.__setstate__(self.__dict__) super(Elemwise, self).__init__(openmp=openmp) def __getstate__(self): d = copy(self.__dict__) d.pop('ufunc') d.pop('nfunc') d.pop('__epydoc_asRoutine', None) return d def __setstate__(self, d): super(Elemwise, self).__setstate__(d) self.ufunc = None self.nfunc = None self.inplace_pattern = frozendict(self.inplace_pattern) def get_output_info(self, dim_shuffle, *inputs): """Return the outputs dtype and broadcastable pattern and the dimshuffled niputs. """ shadow = self.scalar_op.make_node( *[get_scalar_type(dtype=i.type.dtype).make_variable() for i in inputs]) target_length = max([input.type.ndim for input in inputs]) args = [] for input in inputs: length = input.type.ndim difference = target_length - length if not difference: args.append(input) else: # TODO: use LComplete instead args.append(dim_shuffle( input.type.broadcastable, ['x'] * difference + list(range(length)))(input)) inputs = args # HERE: all the broadcast dims have the same length now # cleverness: we iterate over the first, second, third broadcast flag # of all inputs in parallel... the all() gives us each output # broadcastable bit in turn. # it is multiplied by nout because Elemwise supports multiple outputs # (nout of them) out_broadcastables = [[all(bcast) for bcast in izip(*[input.type.broadcastable for input in inputs])]] * shadow.nout # inplace_pattern maps output idx -> input idx inplace_pattern = self.inplace_pattern if inplace_pattern: for overwriter, overwritten in iteritems(inplace_pattern): for ob, ib in izip(out_broadcastables[overwriter], inputs[overwritten].type.broadcastable): if ib and not ob: raise ValueError( "Operation cannot be done inplace on an input " "with broadcasted dimensions.") out_dtypes = [o.type.dtype for o in shadow.outputs] if any(inputs[i].type.dtype != out_dtypes[o] for o, i in inplace_pattern.items()): raise TypeError(( "Cannot do an inplace operation on incompatible data types.", ([i.type.dtype for i in inputs], out_dtypes, inplace_pattern))) assert len(out_dtypes) == len(out_broadcastables) return out_dtypes, out_broadcastables, inputs def make_node(self, *inputs): """ If the inputs have different number of dimensions, their shape is left-completed to the greatest number of dimensions with 1s using DimShuffle. """ inputs = list(map(as_tensor_variable, inputs)) out_dtypes, out_broadcastables, inputs = self.get_output_info( DimShuffle, *inputs) outputs = [TensorType(dtype=dtype, broadcastable=broadcastable)() for dtype, broadcastable in izip(out_dtypes, out_broadcastables)] return Apply(self, inputs, outputs) def __str__(self): if self.name is None: if self.inplace_pattern: items = list(self.inplace_pattern.items()) items.sort() return "Elemwise{%s}%s" % (self.scalar_op, str(items)) else: return "Elemwise{%s}" % (self.scalar_op) else: return self.name def R_op(self, inputs, eval_points): outs = self(*inputs, **dict(return_list=True)) rval = [None for x in outs] # For each output for idx, out in enumerate(outs): # make such that _bgrads computes only the gradients of the # current output on the inputs ( and not all outputs) ograds = [x.zeros_like() for x in outs] ograds[idx] = theano.tensor.ones_like(out) bgrads = self._bgrad(inputs, outs, ograds) rop_out = None for jdx, (inp, eval_point) in enumerate(izip(inputs, eval_points)): # if None, then we can just ignore this branch .. # what we do is to assume that for any non-differentiable # branch, the gradient is actually 0, which I think is not # the right thing to do .. have to talk to Ian and James # about it if bgrads[jdx] is None or \ isinstance(bgrads[jdx].type, DisconnectedType): pass elif eval_point is not None: if rop_out is None: rop_out = bgrads[jdx] * eval_point else: rop_out = rop_out + bgrads[jdx] * eval_point rval[idx] = rop_out return rval def connection_pattern(self, node): if hasattr(self.scalar_op, 'connection_pattern'): return self.scalar_op.connection_pattern(node) return [[True for output in node.outputs] for ipt in node.inputs] def L_op(self, inputs, outs, ograds): # compute grad with respect to broadcasted input rval = self._bgrad(inputs, outs, ograds) # TODO: make sure that zeros are clearly identifiable # to the gradient.grad method when the outputs have # some integer and some floating point outputs if any(out.type.dtype not in theano.tensor.continuous_dtypes for out in outs): # For integer output, return value may # only be zero or undefined # We don't bother with trying to check # that the scalar ops correctly # returned something that evaluates to 0, # we just make the return # value obviously zero so that gradient.grad # can tell this op did # the right thing. new_rval = [] for elem, ipt in izip(rval, inputs): if isinstance(elem.type, (NullType, DisconnectedType)): new_rval.append(elem) else: elem = ipt.zeros_like() if str(elem.type.dtype) not in theano.tensor.continuous_dtypes: elem = elem.astype(theano.config.floatX) assert str(elem.type.dtype) not in theano.tensor.discrete_dtypes new_rval.append(elem) return new_rval # sum out the broadcasted dimensions for i, ipt in enumerate(inputs): if isinstance(rval[i].type, (NullType, DisconnectedType)): continue # list of all the dimensions that are broadcastable for input[i] so # we can sum over them # todo: only count dimensions that were effectively broadcasted to_sum = [j for j, bcast in enumerate(ipt.type.broadcastable) if bcast and not outs[0].broadcastable[j]] if to_sum: sr = theano.tensor.basic.sum( rval[i], axis=to_sum, keepdims=True) rval[i] = sr # close if # close for return rval def _bgrad(self, inputs, outputs, ograds): # returns grad, with respect to broadcasted versions of inputs with change_flags(compute_test_value='off'): def as_scalar(t): if isinstance(t.type, (NullType, DisconnectedType)): return t return get_scalar_type(t.type.dtype)() scalar_inputs = list(map(as_scalar, inputs)) scalar_ograds = list(map(as_scalar, ograds)) scalar_outputs = self.scalar_op.make_node( *[get_scalar_type(dtype=i.type.dtype).make_variable() for i in inputs]).outputs scalar_igrads = self.scalar_op.L_op(scalar_inputs, scalar_outputs, scalar_ograds) for igrad in scalar_igrads: assert igrad is not None, self.scalar_op if not isinstance(scalar_igrads, (list, tuple)): raise TypeError('%s.grad returned %s instead of list or tuple' % (str(self.scalar_op), str(type(scalar_igrads)))) nd = len(inputs[0].type.broadcastable) # this is the same for everyone def transform(r): # From a graph of ScalarOps, make a graph of Broadcast ops. if isinstance(r.type, (NullType, DisconnectedType)): return r if r in scalar_inputs: return inputs[scalar_inputs.index(r)] if r in scalar_outputs: return outputs[scalar_outputs.index(r)] if r in scalar_ograds: return ograds[scalar_ograds.index(r)] node = r.owner if node is None: # the gradient contains a constant, translate it as # an equivalent TensorType of size 1 and proper number of # dimensions res = theano.tensor.constant(np.asarray(r.data), dtype=r.type.dtype) return DimShuffle((), ['x'] * nd)(res) new_r = Elemwise(node.op, {})( *[transform(ipt) for ipt in node.inputs]) return new_r ret = [] for scalar_igrad, ipt in izip(scalar_igrads, inputs): if scalar_igrad is None: # undefined gradient ret.append(None) continue ret.append(transform(scalar_igrad)) return ret def prepare_node(self, node, storage_map, compute_map, impl): # Postpone the ufunc building to the last minutes due to: # - NumPy ufunc support only up to 31 inputs. # But our c code support more. # - nfunc is reused for scipy and scipy is optional if getattr(self, 'nfunc_spec', None) and impl != 'c': self.nfunc = getattr(np, self.nfunc_spec[0], None) if self.nfunc is None: # Not inside NumPy. So probably another package like scipy. symb = self.nfunc_spec[0].split(".") for idx in range(1, len(self.nfunc_spec[0])): try: module = __import__('.'.join(symb[:idx])) except ImportError: break for sub in symb[1:]: try: module = getattr(module, sub) except AttributeError: module = None break self.nfunc = module if (len(node.inputs) < 32 and (self.nfunc is None or self.scalar_op.nin != len(node.inputs)) and self.ufunc is None and impl == 'py'): ufunc = np.frompyfunc(self.scalar_op.impl, len(node.inputs), self.scalar_op.nout) if self.scalar_op.nin > 0: # We can reuse it for many nodes self.ufunc = ufunc else: node.tag.ufunc = ufunc # Numpy ufuncs will sometimes perform operations in # float16, in particular when the input is int8. # This is not something that we want, and we do not # do it in the C code, so we specify that the computation # should be carried out in the returned dtype. # This is done via the "sig" kwarg of the ufunc, its value # should be something like "ff->f", where the characters # represent the dtype of the inputs and outputs. # NumPy 1.10.1 raise an error when giving the signature # when the input is complex. So add it only when inputs is int. out_dtype = node.outputs[0].dtype if (out_dtype in theano.tensor.float_dtypes and isinstance(self.nfunc, np.ufunc) and node.inputs[0].dtype in theano.tensor.discrete_dtypes): char = np.sctype2char(out_dtype) sig = char * node.nin + '->' + char * node.nout node.tag.sig = sig node.tag.fake_node = Apply( self.scalar_op, [get_scalar_type(dtype=input.type.dtype).make_variable() for input in node.inputs], [get_scalar_type(dtype=output.type.dtype).make_variable() for output in node.outputs]) self.scalar_op.prepare_node(node.tag.fake_node, None, None, impl) def perform(self, node, inputs, output_storage): if len(node.inputs) >= 32: # Some versions of NumPy will segfault, other will raise a # ValueError, if the number of inputs to a ufunc is 32 or more. # In that case, the C version should be used, or Elemwise fusion # should be disabled. super(Elemwise, self).perform(node, inputs, output_storage) for dims in izip(*[list(zip(input.shape, sinput.type.broadcastable)) for input, sinput in zip(inputs, node.inputs)]): if max(d for d, b in dims) != 1 and (1, False) in dims: # yes there may be more compact ways to write this code, # but please maintain python 2.4 compatibility # (no "x if c else y") msg = [] assert len(inputs) == len(node.inputs) for input, sinput in zip(inputs, node.inputs): assert len(input.shape) == len(sinput.type.broadcastable) msg2 = [] for d, b in zip(input.shape, sinput.type.broadcastable): if b: msg2 += ['*'] else: msg2 += [str(d)] msg.append('(%s)' % ", ".join(msg2)) base_exc_str = 'Dimension mismatch; shapes are %s' % ( ', '.join(msg)) raise ValueError(base_exc_str) # Determine the shape of outputs out_shape = [] for values in izip(*[input.shape for input in inputs]): if any(v == 0 for v in values): # All non-broadcasted dimensions should be zero assert max(values) <= 1 out_shape.append(0) else: out_shape.append(max(values)) out_shape = tuple(out_shape) ufunc_args = inputs ufunc_kwargs = {} # We supported in the past calling manually op.perform. # To keep that support we need to sometimes call self.prepare_node if self.nfunc is None and self.ufunc is None: self.prepare_node(node, None, None, 'py') if self.nfunc and len(inputs) == self.nfunc_spec[1]: ufunc = self.nfunc nout = self.nfunc_spec[2] if hasattr(node.tag, 'sig'): ufunc_kwargs['sig'] = node.tag.sig # Unfortunately, the else case does not allow us to # directly feed the destination arguments to the nfunc # since it sometimes requires resizing. Doing this # optimization is probably not worth the effort, since we # should normally run the C version of the Op. else: # the second calling form is used because in certain versions of # numpy the first (faster) version leads to segfaults if self.ufunc: ufunc = self.ufunc elif not hasattr(node.tag, 'ufunc'): # It happen that make_thunk isn't called, like in # get_scalar_constant_value self.prepare_node(node, None, None, 'py') # prepare_node will add ufunc to self or the tag # depending if we can reuse it or not. So we need to # test both again. if self.ufunc: ufunc = self.ufunc else: ufunc = node.tag.ufunc else: ufunc = node.tag.ufunc nout = ufunc.nout variables = ufunc(*ufunc_args, **ufunc_kwargs) if nout == 1: variables = [variables] i = 0 for variable, storage, nout in izip(variables, output_storage, node.outputs): if getattr(variable, "dtype", "") == 'object': # Since numpy 1.6, function created with numpy.frompyfunc # always return an ndarray with dtype object variable = np.asarray(variable, dtype=nout.dtype) if i in self.inplace_pattern: odat = inputs[self.inplace_pattern[i]] odat[...] = variable storage[0] = odat # Sometimes NumPy return a Python type. # Some Theano op return a different dtype like floor, ceil, # trunc, eq, ... elif (not isinstance(variable, np.ndarray) or variable.dtype != nout.dtype): variable = np.asarray(variable, nout.dtype) # The next line is needed for numpy 1.9. Otherwise # there are tests that fail in DebugMode. # Normally we would call theano.misc._asarray, but it # is faster to inline the code. We know that the dtype # are the same string, just different typenum. if np.dtype(nout.dtype).num != variable.dtype.num: variable = variable.view(dtype=nout.dtype) storage[0] = variable # numpy.real return a view! elif not variable.flags.owndata: storage[0] = variable.copy() else: storage[0] = variable i += 1 def infer_shape(self, node, i_shapes): rval = [] for o in node.outputs: oshp = [] for dim, b in enumerate(o.type.broadcastable): b_dim = None if b: # this is broadcastable b_dim = 1 else: # there must be some input that is not broadcastable in # dimension 'dim' for ishp, i in izip(i_shapes, node.inputs): if isinstance(i.type, theano.scalar.Scalar): continue # we skip scalar if not i.type.broadcastable[dim]: # input i is not broadcastable in position dim # therefore if its shape is known, we can use it # as the output shape if ishp[dim]: b_dim = ishp[dim] break # b_dim might still be None, if every input's shape was unknown # in dimension 'dim' oshp.append(b_dim) # TODO: it would be interesting to return the constraining # information that if one of the inputs shape[dim] is known # and another input's shape[dim] is not, that we can now assume # that the other input's shape[dim] is the same as the first. rval.append(tuple(oshp)) return rval def _c_all(self, node, nodename, inames, onames, sub): # Some ops call directly the Elemwise._c_all or Elemwise.c_code # To not request all of them to call prepare_node(), do it here. # There is no harm if it get called multile time. if not hasattr(node.tag, 'fake_node'): self.prepare_node(node, None, None, 'c') _inames = inames _onames = onames inames = gof.utils.uniq(inames) inputs = gof.utils.uniq(node.inputs) # assert that inames and inputs order stay consistent. # This is to protect again futur change of uniq. assert len(inames) == len(inputs) ii, iii = list(zip(*gof.utils.uniq(list(zip(_inames, node.inputs))))) assert all([x == y for x, y in zip(ii, inames)]) assert all([x == y for x, y in zip(iii, inputs)]) defines = "" undefs = "" # The destroy map is a map of output indices to input indices # that overwrite them. We just convert them to the actual # Variables. dmap = dict([(node.outputs[o], [node.inputs[i]]) for o, i in iteritems(self.inplace_pattern)]) # dtypes of the inputs idtypes = [input.type.dtype_specs()[1] for input in inputs] # These are the outputs that we will need to allocate # (output, name, name of the c type), transposed real = list(zip(*[(r, s, r.type.dtype_specs()[1]) for r, s in izip(node.outputs, onames) if r not in dmap])) if real: real_outputs, real_onames, real_odtypes = real else: real_outputs, real_onames, real_odtypes = [], [], [] # Outputs that are aliased with an input (inplace) # (output, name), transposed (c type name not needed since we don't # need to allocate. aliased = list(zip(*[(r, s) for (r, s) in izip(node.outputs, onames) if r in dmap])) if aliased: aliased_outputs, aliased_onames = aliased else: aliased_outputs, aliased_onames = [], [] # for each input: # same as range(ndim), but with 'x' at all broadcastable positions orders = [[x and 'x' or i for i, x in enumerate(input.type.broadcastable)] for input in inputs] # number of nested loops we will need (all inputs have same # dimensionality) nnested = len(orders[0]) sub = dict(sub) for i, (input, iname) in enumerate(izip(inputs, inames)): # the c generators will substitute the input names for # references to loop variables lv0, lv1, ... sub['lv%i' % i] = iname decl = cgen.make_declare(orders, idtypes, sub) checks = cgen.make_checks(orders, idtypes, sub) # Check if all inputs (except broadcasted scalar) are fortran. # In that case, create an fortran output ndarray. z = list(zip(inames, inputs)) alloc_fortran = ' && '.join(["PyArray_ISFORTRAN(%s)" % arr for arr, var in z if not all(var.broadcastable)]) # If it is a scalar, make it c contig to prevent problem with # NumPy C and F contig not always set as both of them. if len(alloc_fortran) == 0: alloc_fortran = '0' alloc = "" # We loop over the "real" outputs, i.e., those that are not # inplace (must be allocated) and we declare/allocate/check # them for output, oname, odtype in izip( real_outputs, real_onames, real_odtypes): i += 1 # before this loop, i = number of inputs sub['lv%i' % i] = oname sub['olv'] = oname alloc += cgen.make_declare([list(range(nnested))], [odtype], dict(sub, lv0=oname)) alloc += cgen.make_alloc(orders, odtype, sub, fortran=alloc_fortran) alloc += cgen.make_checks([list(range(nnested))], [odtype], dict(sub, lv0=oname)) olv_index = i # index of the last output # We loop over the "aliased" outputs, i.e., those that are # inplace (overwrite the contents of one of the inputs) and # make the output pointers point to their corresponding input # pointers. for output, oname in izip(aliased_outputs, aliased_onames): olv_index = inputs.index(dmap[output][0]) iname = inames[olv_index] # We make the output point to the corresponding input and # decrease the reference of whatever the output contained # prior to this alloc += """ if (%(oname)s) { Py_XDECREF(%(oname)s); } %(oname)s = %(iname)s; Py_XINCREF(%(oname)s); """ % locals() # We alias the scalar variables defines += "#define %(oname)s_i %(iname)s_i\n" % locals() undefs += "#undef %(oname)s_i\n" % locals() # Note: here, olv_index is either the index of the last output # which is allocated, OR, if there are any aliased outputs, # the index of the last of these aliased outputs. # We generate the C code of the inner loop using the scalar op if self.openmp: # If we are using openmp, we need to get rid of the "goto" # statement in sub['fail']. For now we recreate it here. fail = gof.cc.failure_code(sub, use_goto=False) else: fail = sub['fail'] task_code = self.scalar_op.c_code( node.tag.fake_node, nodename + '_scalar_', ["%s_i" % s for s in _inames], ["%s_i" % s for s in onames], dict(sub, fail=fail)) code = """ { %(defines)s %(task_code)s %(undefs)s } """ % locals() loop_orders = orders + [list(range(nnested))] * len(real_onames) dtypes = (idtypes + list(real_odtypes)) if all([o.ndim <= 1 for o in node.outputs] or # Use simpler code when output ndim == 0 or 1 # or for broadcated scalar. all(node.outputs[0].broadcastable)): if nnested: all_code = [("", "")] * (nnested - 1) + [("", code)] + [""] else: all_code = [code] if len(all_code) == 1: # No loops task_decl = "".join([ "%s& %s_i = *%s_iter;\n" % (dtype, name, name) for name, dtype in izip(inames + list(real_onames), idtypes + list(real_odtypes))]) preloops = {} for i, (loop_order, dtype) in enumerate(zip(loop_orders, dtypes)): for j, index in enumerate(loop_order): if index != 'x': preloops.setdefault(j, "") preloops[j] += ("%%(lv%(i)s)s_iter = (%(dtype)s*)" "(PyArray_DATA(%%(lv%(i)s)s));\n" % locals()) % sub break else: # all broadcastable preloops.setdefault(0, "") preloops[0] += ("%%(lv%(i)s)s_iter = (%(dtype)s*)" "(PyArray_DATA(%%(lv%(i)s)s));\n" % locals()) % sub init_array = preloops.get(0, " ") loop = """ { %(defines)s %(init_array)s %(task_decl)s %(task_code)s %(undefs)s } """ % locals() else: loop = cgen.make_loop( loop_orders=loop_orders, dtypes=dtypes, loop_tasks=all_code, sub=sub, openmp=self.openmp) else: loop = cgen.make_reordered_loop( init_loop_orders=loop_orders, olv_index=olv_index, dtypes=dtypes, inner_task=code, sub=sub, openmp=self.openmp) # If all inputs and outputs are contiguous # and the scalar op define optimized code for that case # use it! The scalar_op need to check the broadcast flag himself. if (all([o.ndim >= 1 for o in node.outputs]) and # Don't use the contig code for broadcasted scalar. not all(node.outputs[0].broadcastable)): contig = None try: contig = self.scalar_op.c_code_contiguous( node, nodename + '_scalar_contig_', _inames, onames, sub) except theano.gof.utils.MethodNotDefined: # Try to make one generic version, this will help the # compiler to vectorize the code as their won't be as # many ptr and the stride will be hard coded. if all([io.broadcastable == node.outputs[0].broadcastable or all(io.broadcastable) for io in node.inputs + node.outputs]): z = onames[0] contig = """ // All output have the same size npy_intp n = PyArray_SIZE(%(z)s); """ % locals() index = "" for x, var in zip(inames + onames, inputs + node.outputs): if not all(var.broadcastable): contig += """ dtype_%(x)s * %(x)s_ptr = (dtype_%(x)s*) PyArray_DATA(%(x)s); """ % locals() index += """ dtype_%(x)s& %(x)s_i = %(x)s_ptr[i]; """ % locals() else: contig += """ dtype_%(x)s& %(x)s_i = ((dtype_%(x)s*) PyArray_DATA(%(x)s))[0]; """ % locals() if self.openmp: contig += """#pragma omp parallel for if(n>=%d) """ % (config.openmp_elemwise_minsize) contig += """ for(int i=0; i', ''] def c_support_code(self): return self.scalar_op.c_support_code() def c_support_code_apply(self, node, nodename): support_code = self.scalar_op.c_support_code_apply(node, nodename + '_scalar_') return support_code def c_code_cache_version_apply(self, node): version = [13] # the version corresponding to the c code in this Op # now we insert versions for the ops on which we depend... scalar_node = Apply( self.scalar_op, [get_scalar_type(dtype=input.type.dtype).make_variable() for input in node.inputs], [get_scalar_type(dtype=output.type.dtype).make_variable() for output in node.outputs]) version.append(self.scalar_op.c_code_cache_version_apply(scalar_node)) for i in node.inputs + node.outputs: version.append( get_scalar_type(dtype=i.type.dtype).c_code_cache_version()) version.append(('openmp', self.openmp)) if all(version): return tuple(version) else: return () def python_constant_folding(self, node): """ Return True if we do not want to compile c code when doing constant folding of this node. """ # The python code don't support 32 inputs or more. return node.outputs[0].ndim == 0 and len(node.inputs) < 32 ################ # CAReduce # ################ class CAReduce(Op): """ CAReduce = Commutative Associative Reduce Reduces a scalar operation along the specified axis(es). (The scalar op should be both commutative and assocative) The output will have the same shape as the input minus the reduced dimensions. It will contain the variable of accumulating all values over the reduced dimensions using the specified scalar op. Parameters ---------- scalar_op A binary scalar op with only one output. It must be commutative and associative. axis - The dimension along which we want to reduce - List of dimensions that we want to reduce - If None, all dimensions are reduced Note ---- .. code-block:: python CAReduce(add) # sum (ie, acts like the numpy sum operation) CAReduce(mul) # product CAReduce(maximum) # max CAReduce(minimum) # min CAReduce(or_) # any # not lazy CAReduce(and_) # all # not lazy CAReduce(xor) # a bit at 1 tell that there was an odd number of # bit at that position that where 1. 0 it was an # even number ... In order to (eventually) optimize memory usage patterns, CAReduce makes zero guarantees on the order in which it iterates over the dimensions and the elements of the array(s). Therefore, to ensure consistent variables, the scalar operation represented by the reduction must be both commutative and associative (eg add, multiply, maximum, binary or/and/xor - but not subtract, divide or power). """ __props__ = ("scalar_op", "axis") def __init__(self, scalar_op, axis=None): if scalar_op.nin not in [-1, 2] or scalar_op.nout != 1: raise NotImplementedError(( "CAReduce only supports binary functions with a single " "output.")) self.scalar_op = scalar_op if axis is None: self.axis = axis # There is a bug in numpy that results in isinstance(x, # integer_types) returning False for numpy integers. See # . elif isinstance(axis, (integer_types, np.integer)): self.axis = (axis,) elif isinstance(axis, np.ndarray) and axis.ndim == 0: self.axis = (int(axis),) else: self.axis = list(set(int(a) for a in axis)) self.axis.sort() self.axis = tuple(self.axis) self.set_ufunc(scalar_op) def set_ufunc(self, scalar_op): # This is probably a speed up of the implementation if isinstance(scalar_op, theano.scalar.basic.Add): self.ufunc = np.add elif isinstance(scalar_op, theano.scalar.basic.Mul): self.ufunc = np.multiply elif isinstance(scalar_op, theano.scalar.basic.Maximum): self.ufunc = np.maximum elif isinstance(scalar_op, theano.scalar.basic.Minimum): self.ufunc = np.minimum elif (isinstance(scalar_op, theano.scalar.basic.AND) and _numpy_ver >= [1, 12]): # numpy.bitwise_and.identity was incorrect for versions before # 1.12 (it was 1 instead of -1), so we skip it in that case. # We will fall back to the "else:" case, which defines a # ufunc without identity. self.ufunc = np.bitwise_and elif isinstance(scalar_op, theano.scalar.basic.OR): self.ufunc = np.bitwise_or elif isinstance(scalar_op, theano.scalar.basic.XOR): self.ufunc = np.bitwise_xor else: self.ufunc = np.frompyfunc(scalar_op.impl, 2, 1) def _output_dtype(self, input_dtype): return input_dtype def make_node(self, input): input = as_tensor_variable(input) if self.axis is not None: for axis in self.axis: if (axis >= input.type.ndim or (axis < 0 and abs(axis) > input.type.ndim)): raise ValueError(( 'Not enough dimensions on %s to reduce on axis %s' % (input, axis))) input = as_tensor_variable(input) axis = self.axis if axis is None: axis = list(range(len(input.type.broadcastable))) if any(a < 0 for a in axis): axis2 = [] for a in self.axis: if a < 0: axis2.append(a + input.type.ndim) else: axis2.append(a) assert len(axis) == len(axis2) axis = tuple(axis2) # We can't call self.__class__() as there is a class that # inherits from CAReduce that doesn't have the same signature op = copy(self) op.set_ufunc(op.scalar_op) op.axis = axis else: op = self broadcastable = [x for i, x in enumerate(input.type.broadcastable) if i not in axis] output = TensorType(dtype=self._output_dtype(input.type.dtype), broadcastable=broadcastable)() return Apply(op, [input], [output]) def __getstate__(self): d = copy(self.__dict__) d.pop('ufunc', None) return d def __setstate__(self, d): self.__dict__.update(d) self.set_ufunc(self.scalar_op) def __str__(self): if self.axis is not None: return "Reduce{%s}{%s}" % ( self.scalar_op, ", ".join(str(x) for x in self.axis)) else: return "Reduce{%s}" % self.scalar_op def perform(self, node, inp, out): input, = inp output, = out axis = self.axis if axis is None: axis = list(range(input.ndim)) variable = input to_reduce = reversed(sorted(axis)) if hasattr(self, 'acc_dtype') and self.acc_dtype is not None: acc_dtype = self.acc_dtype else: acc_dtype = node.outputs[0].type.dtype if to_reduce: for dimension in to_reduce: # If it's a zero-sized array, use scalar_op.identity # if available if variable.shape[dimension] == 0: if hasattr(self.scalar_op, 'identity'): # Compute the shape of the output v_shape = list(variable.shape) del v_shape[dimension] variable = np.empty(tuple(v_shape), dtype=acc_dtype) variable.fill(self.scalar_op.identity) else: raise ValueError(( "Input (%s) has zero-size on axis %s, but " "self.scalar_op (%s) has no attribute 'identity'" % (variable, dimension, self.scalar_op))) else: variable = self.ufunc.reduce(variable, dimension, dtype=acc_dtype) variable = np.asarray(variable) if np.may_share_memory(variable, input): # perhaps numpy is clever for reductions of size 1? # We don't want this. variable = variable.copy() output[0] = theano._asarray(variable, dtype=node.outputs[0].type.dtype) else: # Force a copy output[0] = np.array(variable, copy=True, dtype=node.outputs[0].type.dtype) def infer_shape(self, node, shapes): ishape, = shapes axis = self.axis if axis is None: return (), return [ishape[i] for (i, b) in enumerate(node.inputs[0].type.broadcastable) if i not in axis], def _c_all(self, node, name, inames, onames, sub): input = node.inputs[0] output = node.outputs[0] iname = inames[0] oname = onames[0] idtype = input.type.dtype_specs()[1] odtype = output.type.dtype_specs()[1] if hasattr(self, 'acc_dtype') and self.acc_dtype is not None: if self.acc_dtype == 'float16': raise theano.gof.utils.MethodNotDefined("no c_code for " "float16") acc_type = TensorType( broadcastable=node.outputs[0].broadcastable, dtype=self.acc_dtype) adtype = acc_type.dtype_specs()[1] else: adtype = odtype axis = self.axis if axis is None: axis = list(range(len(input.type.broadcastable))) if len(axis) == 0: # The acc_dtype is never a downcast compared to the input dtype # So we just need a cast to the output dtype. var = theano.tensor.cast(input, node.outputs[0].dtype) if var is input: var = Elemwise(scalar.identity)(input) assert var.dtype == node.outputs[0].dtype return var.owner.op._c_all(var.owner, name, inames, onames, sub) order1 = [i for i in xrange(input.type.ndim) if i not in axis] order = order1 + list(axis) nnested = len(order1) sub = dict(sub) for i, (input, iname) in enumerate(izip(node.inputs, inames)): sub['lv%i' % i] = iname decl = "" if adtype != odtype: # Create an accumulator variable different from the output aname = "acc" decl = acc_type.c_declare(aname, sub) decl += acc_type.c_init(aname, sub) else: # the output is the accumulator variable aname = oname decl += cgen.make_declare([order], [idtype], sub) checks = cgen.make_checks([order], [idtype], sub) alloc = "" i += 1 sub['lv%i' % i] = oname sub['olv'] = oname # Allocate output buffer alloc += cgen.make_declare( [list(range(nnested)) + ['x'] * len(axis)], [odtype], dict(sub, lv0=oname)) alloc += cgen.make_alloc([order1], odtype, sub) alloc += cgen.make_checks( [list(range(nnested)) + ['x'] * len(axis)], [odtype], dict(sub, lv0=oname)) if adtype != odtype: # Allocate accumulation buffer sub['lv%i' % i] = aname sub['olv'] = aname alloc += cgen.make_declare( [list(range(nnested)) + ['x'] * len(axis)], [adtype], dict(sub, lv0=aname)) alloc += cgen.make_alloc([order1], adtype, sub) alloc += cgen.make_checks( [list(range(nnested)) + ['x'] * len(axis)], [adtype], dict(sub, lv0=aname)) if hasattr(self.scalar_op, 'identity'): identity = self.scalar_op.identity elif self.scalar_op in [scalar.maximum, scalar.minimum]: if self.scalar_op == scalar.maximum: scal_name = 'maximum' if input.type.dtype in ["float32", "float64"]: identity = "-__builtin_inf()" elif input.type.dtype.startswith("uint") or input.type.dtype == 'bool': # numpy does not define NPY_MIN_UINT* and NPY_MIN_BOOL identity = "0" else: identity = "NPY_MIN_" + str(input.type.dtype).upper() if self.scalar_op == scalar.minimum: scal_name = 'minimum' if input.type.dtype in ["float32", "float64"]: identity = "__builtin_inf()" elif input.type.dtype == 'bool': # numpy does not define NPY_MAX_BOOL identity = "1" else: identity = "NPY_MAX_" + str(input.type.dtype).upper() fail = sub["fail"] pattern = [0] * len(node.inputs[0].broadcastable) axis = self.axis if axis is None: axis = list(range(len(pattern))) for i in axis: pattern[i] = 1 pattern_ = str(pattern)[1:-1] decl += """int tosum[]={%(pattern_)s};""" % locals() alloc += """ for(int i=0;i', ''] def c_code_cache_version_apply(self, node): # the version corresponding to the c code in this Op version = [8] # now we insert versions for the ops on which we depend... scalar_node = Apply( self.scalar_op, [get_scalar_type(dtype=input.type.dtype).make_variable() for input in node.inputs], [get_scalar_type(dtype=output.type.dtype).make_variable() for output in node.outputs]) version.append(self.scalar_op.c_code_cache_version_apply(scalar_node)) for i in node.inputs + node.outputs: version.append( get_scalar_type(dtype=i.type.dtype).c_code_cache_version()) if all(version): return tuple(version) else: return () class All(CAReduce): """ Applies `logical and` to all the values of a tensor along the specified axis(es). """ __props__ = ("axis",) def __init__(self, axis=None): CAReduce.__init__(self, scalar.and_, axis) def _output_dtype(self, idtype): return "bool" def __str__(self): if self.axis is None: return "All" else: return "All{%s}" % ", ".join(map(str, self.axis)) def make_node(self, input): input = as_tensor_variable(input) if input.dtype != "bool": input = theano.tensor.neq(input, 0) ret = super(All, self).make_node(input) return ret def grad(self, inp, grads): x, = inp return [x.zeros_like(theano.config.floatX)] class Any(CAReduce): """ Applies `bitwise or` to all the values of a tensor along the specified axis(es). """ __props__ = ("axis", ) def __init__(self, axis=None): CAReduce.__init__(self, scalar.or_, axis) def _output_dtype(self, idtype): return "bool" def __str__(self): if self.axis is None: return "Any" else: return "Any{%s}" % ", ".join(map(str, self.axis)) def make_node(self, input): input = as_tensor_variable(input) if input.dtype != "bool": input = theano.tensor.neq(input, 0) ret = super(Any, self).make_node(input) return ret def grad(self, inp, grads): x, = inp return [x.zeros_like(theano.config.floatX)] class CAReduceDtype(CAReduce): """ Reduces a scalar operation along the specified axis(es). This subclass of CAReduce accepts an additional "dtype" parameter, that specifies which dtype the output should be. It also accepts an optional "acc_dtype", which specify the dtype that will be used for the accumulation. So, the accumulation will be done into a tensor of dtype "acc_dtype", then it will be casted into "dtype" and returned. If no dtype is provided, one will be inferred so as not to lose too much precision. Parameters ---------- scalar_op A binary scalar op with only one output. It must be commutative and associative. axis * the dimension along which we want to reduce * list of dimensions that we want to reduce * if None, all dimensions are reduced dtype The dtype of the returned tensor. If None, then we use the default dtype which is the same as the input tensor's dtype except when: * the input dtype is a signed integer of precision < 64 bit, in which case we use int64 * the input dtype is an unsigned integer of precision < 64 bit, in which case we use uint64 This default dtype does _not_ depend on the value of "acc_dtype". This behavior is similar in spirit to that of numpy (except numpy uses the default machine integer while we always use 64 bit integers to avoid platform-dependent behavior). acc_dtype The dtype of the internal accumulator. If None (default), we use the dtype in the list below, or the input dtype if its precision is higher: * for int dtypes, we use at least int64; * for uint dtypes, we use at least uint64; * for float dtypes, we use at least float64; * for complex dtypes, we use at least complex128. """ __props__ = ("scalar_op", "axis", "dtype", "acc_dtype") def __init__(self, scalar_op, axis=None, dtype=None, acc_dtype=None): CAReduce.__init__(self, scalar_op, axis=axis) self.dtype = dtype self.acc_dtype = acc_dtype def __setstate__(self, d): super(CAReduceDtype, self).__setstate__(d) if not hasattr(self, "dtype"): # This is needed as old pickled will crash otherwise. # We need to keep the old dtype behavior as the op # could be in an apply node with a specified dtype. self.dtype = "OLD" if not hasattr(self, "acc_dtype"): # acc_dtype is not used by any external Op, so we do not # need to keep the previous behaviour here. self.acc_dtype = None def _output_dtype(self, idtype): dtype = self.dtype if dtype == "OLD": return dict( int8='int32', int16='int32', int32='int64', uint8='uint32', uint16='uint32', uint32='uint64').get(idtype, idtype) if dtype is None: # If input has a discrete dtype, upcast it to 64 return dict( bool='int64', int8='int64', int16='int64', int32='int64', uint8='uint64', uint16='uint64', uint32='uint64').get(idtype, idtype) else: # The important is that the accumulator dtype does not # lose precision. Then, the result can be downcasted. return dtype def _acc_dtype(self, idtype): acc_dtype = self.acc_dtype if acc_dtype is None: return dict( bool='int64', int8='int64', int16='int64', int32='int64', uint8='uint64', uint16='uint64', uint32='uint64', float16='float32', float32='float64', complex64='complex128').get(idtype, idtype) elif (acc_dtype in theano.tensor.continuous_dtypes and idtype in theano.tensor.discrete_dtypes): # Specifying a continuous accumulator for discrete input is OK return acc_dtype else: # The conversion has to be considered an upcast. upcasted_dtype = scalar.upcast(idtype, acc_dtype) if acc_dtype != upcasted_dtype: raise TypeError( 'Cannot build %s node with input dtype %s ' 'and acc_dtype %s, as precision would be lost. ' 'To correct this error, you can:\n' ' - not specify acc_dtype, or\n' ' - use an acc_dtype at least as precise as %s.\n' ' - specify "dtype" instead of "acc_dtype", so ' 'the reduction will be precise, but the result will ' 'be casted into "dtype" at the end.\n' 'If you are expecting the precision loss, you can ' 'use tensor.cast(..., dtype="%s"), on your input.' % (self, idtype, acc_dtype, upcasted_dtype, acc_dtype)) return acc_dtype def make_node(self, input): # We need to redefine make_node so that, if self.dtype is None, # we can infer what dtype should be, and create a node from an Op # of the appropriate dtype. input = as_tensor_variable(input) dtype = self._output_dtype(input.dtype) acc_dtype = self._acc_dtype(input.dtype) assert dtype is not None assert acc_dtype is not None if dtype == self.dtype and acc_dtype == self.acc_dtype: # Don't build another instance op = self else: op = copy(self) op.set_ufunc(self.scalar_op) op.dtype = dtype op.acc_dtype = acc_dtype assert op.acc_dtype is not None return CAReduce.make_node(op, input) def __str__(self): name = self.__class__.__name__ if self.__class__.__name__ == "CAReduceDtype": name = "ReduceDtype{%s}" % self.scalar_op, axis = "" if self.axis is not None: axis = ", ".join(str(x) for x in self.axis) axis = "axis=[%s], " % axis return "%s{%sacc_dtype=%s}" % ( name, axis, str(self.acc_dtype) ) class Sum(CAReduceDtype): """ Sums all the values of a tensor along the specified axis(es). Equivalent to `CAReduceDtype(scalar.add, axis=axis, dtype=dtype)`, with the difference that this defines the gradient of sum wrt its tensor input. Parameters ---------- axis Axis(es) along which the tensor should be summed (use None to sum over all axes, and a list or tuple to sum along more than one axis). dtype The dtype of the internal accumulator and returned tensor. If None, then we use the default dtype which is the same as the input tensor's dtype except when: - the input dtype is a signed integer of precision < 64 bit, in which case we use int64 - the input dtype is an unsigned integer of precision < 64 bit, in which case we use uint64 This value does not depend on the value of "acc_dtype". acc_dtype The dtype of the internal accumulator. If None (default), we use the dtype in the list below, or the input dtype if its precision is higher: - for int dtypes, we use at least int64; - for uint dtypes, we use at least uint64; - for float dtypes, we use at least float64; - for complex dtypes, we use at least complex128. """ __props__ = ("axis", "dtype", "acc_dtype") def __init__(self, axis=None, dtype=None, acc_dtype=None): CAReduceDtype.__init__(self, scalar.add, axis=axis, dtype=dtype, acc_dtype=acc_dtype) def __str__(self): name = self.__class__.__name__ axis = "" if self.axis is not None: axis = ", ".join(str(x) for x in self.axis) axis = "axis=[%s], " % axis return "%s{%sacc_dtype=%s}" % ( name, axis, str(self.acc_dtype) ) def L_op(self, inp, out, grads): x, = inp if out[0].dtype not in theano.tensor.continuous_dtypes: return [x.zeros_like(dtype=theano.config.floatX)] gz, = grads gz = as_tensor_variable(gz) axis = self.axis if axis is None: axis = list(range(x.type.ndim)) if axis == (): return gz, new_dims = [] i = 0 for j, _ in enumerate(x.type.broadcastable): if j in axis: new_dims.append('x') else: new_dims.append(i) i += 1 ds_op = DimShuffle(gz.type.broadcastable, new_dims) gx = Elemwise(scalar.second)(x, ds_op(gz)) return [gx] def R_op(self, inputs, eval_points): # There is just one element in inputs and eval_points, the axis are # part of self if None in eval_points: return [None] return self(*eval_points, **dict(return_list=True)) class Prod(CAReduceDtype): """ Multiplies all the values of a tensor along the specified axis(es). Equivalent to `CAReduce(scalar.prod, axis = axis)`, with the difference that this defines the gradient of prod wrt its tensor input. """ __props__ = ("axis", "dtype", "acc_dtype") def __init__(self, axis=None, dtype=None, acc_dtype=None, no_zeros_in_input=False): CAReduceDtype.__init__(self, scalar.mul, axis=axis, dtype=dtype, acc_dtype=acc_dtype) self.no_zeros_in_input = no_zeros_in_input def __setstate__(self, dct): super(Prod, self).__setstate__(dct) # Add default value to be able to reload old pickled objects. if 'no_zeros_in_input' not in dct: self.no_zeros_in_input = False def L_op(self, inp, out, grads): """ The grad of this Op could be very easy, if it is was not for the case where zeros are present in a given "group" (ie. elements reduced together to form the product). If no zeros are found in the elements of the product, then the partial derivative of the product relative to one of the elements (one of the inputs) is simply the product of the other elements. That's easy to see from the chain rule. Now the trick (with no zeros) is to take the overall product, then for every original element, the partial derivative is given by this product divided by the element itself (which equals the product of the other terms). This is easy to do by broadcasting the original product. (Note that we also need to broadcast-multiply by the "incoming gradient", ie. the gradient of the cost relative to the output/product). With zeros, things get more complicated. For a given group, we have 3 cases: * No zeros in the group. Use previous trick. * If only one zero is present, then the gradient for that element is non-zero, but is zero for all others. * If more than one zero is present, then all the derivatives are zero. For the last two cases (with 1 or more zeros), we can't use the division trick, as this gives divisions by 0. Implementing that case-by-case logic is not as trivial, so a bunch of hacks are piled down here to do it. Notably, for the "only one zero" case, there's a special Op that computes the product of the elements in the group, minus the zero (see ProdWithoutZero). The trick is then to use the division trick for groups with no zero, to use the ProdWithoutZeros op where there's only one zero, and to output a derivative of zero for any element part of a group with more than one zero. I do this by first counting the number of zeros in each group (see the "T.eq()" bits), then taking this or that behavior (see T.switch) based on the result of this count. """ prod_in, = inp gz, = grads if (out[0].dtype in theano.tensor.discrete_dtypes or self.acc_dtype in theano.tensor.discrete_dtypes): # There is an int conversion in the way return [prod_in.zeros_like(dtype=theano.config.floatX)] # Prepare the broadcasting that is used everywhere to broadcast # over the original groups (ie. broadcast over the elements of a given # product) gz = as_tensor_variable(gz) axis = self.axis if axis is None: axis = list(range(prod_in.type.ndim)) if axis == (): return gz, new_dims = [] i = 0 for j, _ in enumerate(prod_in.type.broadcastable): if j in axis: new_dims.append('x') else: new_dims.append(i) i += 1 # result of the product, broadcastable over groups prod_out = self(prod_in).dimshuffle(new_dims) # incoming gradient, broadcastable over groups gz = gz.dimshuffle(new_dims) # division trick if we don't have zeros. This will contain # NaNs to be eliminated in the T.switch if we do have zeros. grad_case_without_zeros = (gz * prod_out / prod_in) if self.no_zeros_in_input: # this handles inputs with zeros, but only certain input shapes return [grad_case_without_zeros] else: T = theano.tensor where_zeros = T.eq(prod_in, 0.0) sum_where_zeros = T.sum(where_zeros, axis=self.axis) groups_with_single_zero = T.eq(sum_where_zeros, 1).dimshuffle( new_dims) # tensor with 0 everywhere except for those places where # a 0 part of a group with a single zero was to be found where_single_zero = groups_with_single_zero * where_zeros # further optimization to avoid computing ProdWithoutZeros # if the incoming gradient is 0 where_gz_not_zero = T.neq(gz, 0.0) # only take ProdWithoutZeros for the groups with single zeros # with non-null incoming gradient where_to_take_prod_without_zeros = ( groups_with_single_zero * where_gz_not_zero) # preprocess the original input so that we set 0 everywhere # except for groups that contain a single zero, to avoid computing # multiplications on other groups prod_without_zeros_in = where_to_take_prod_without_zeros * prod_in # TODO: put lazy switch here, if it'd work # this is pretty efficient already (no multiplication if 0), but # it'd be even better if we had a lazy if per element prod_without_zeros = ProdWithoutZeros(axis=self.axis)( prod_without_zeros_in) prod_without_zeros = prod_without_zeros.dimshuffle(new_dims) groups_without_zeros = T.eq(sum_where_zeros, 0).dimshuffle( new_dims) final_grad = T.switch( groups_without_zeros, grad_case_without_zeros, T.switch(where_single_zero, prod_without_zeros, 0.0) * gz) return [final_grad] def c_code_cache_version(self): return (1,) class MulWithoutZeros(scalar.BinaryScalarOp): # "identity" here is zero, as in Reduce we don't want to start # with reducing (1, something_else): this leads to the erronous # case where a vector of zeros is reduced by binary reductions # of (1, 0), which always ends up as 1 (ie. the result for # the c version, for the product of [0,0,0], is 1.0) identity = 0. commutative = True associative = True def impl(self, x, y): if x == 0: return y if y == 0: return x return x * y def c_code(self, node, name, inp, out, sub): x, y = inp z, = out return (("%(z)s = ((%(x)s == 0) ? (%(y)s) : " + "((%(y)s == 0) ? (%(x)s) : ((%(y)s)*(%(x)s))) );") % locals()) def c_code_cache_version(self): return (1,) mul_without_zeros = MulWithoutZeros(scalar.upcast_out, name='mul_without_zeros') class ProdWithoutZeros(CAReduceDtype): __props__ = ("axis", "dtype", "acc_dtype") def __init__(self, axis=None, dtype=None, acc_dtype=None): CAReduceDtype.__init__(self, mul_without_zeros, axis=axis, dtype=dtype, acc_dtype=acc_dtype) def grad(self, inp, grads): a, = inp a_grad = theano.gradient.grad_not_implemented( self, 0, a, "2nd derivatives of `product(a)` is not currently supported." "If `a` is guarenteed to contains no zeros, use " "`product(a, no_zeros_in_input=True)`.") return [a_grad] Theano-1.0.0/theano/tensor/inplace.py00006000163562€ÿÿÿþ0000001701013203335532021043 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division from theano import scalar as scal from . import elemwise from theano import printing from theano.printing import pprint def _scal_inplace(symbol): """Replace a symbol definition with an elementwise version of the corresponding scalar Op""" symbolname = symbol.__name__ inplace = symbolname.endswith('_inplace') if inplace: scalar_op = getattr(scal, symbolname[:-len('_inplace')]) inplace_scalar_op = scalar_op.__class__(scal.transfer_type(0)) rval = elemwise.Elemwise(inplace_scalar_op, {0: 0}, name=symbolname) else: scalar_op = getattr(scal, symbolname) rval = elemwise.Elemwise(scalar_op, name=symbolname) if getattr(symbol, '__doc__', False): rval.__doc__ = symbol.__doc__ + '\n' + rval.__doc__ # for the meaning of this see the ./epydoc script # it makes epydoc display rval as if it were a function, not an object rval.__epydoc_asRoutine = symbol rval.__module__ = 'theano.tensor.inplace' pprint.assign(rval, printing.FunctionPrinter(symbolname.replace('_inplace', '='))) return rval @_scal_inplace def lt_inplace(a, b): """a < b (inplace on a)""" @_scal_inplace def gt_inplace(a, b): """a > b (inplace on a)""" @_scal_inplace def le_inplace(a, b): """a <= b (inplace on a)""" @_scal_inplace def ge_inplace(a, b): """a >= b (inplace on a)""" @_scal_inplace def eq_inplace(a, b): """a == b (inplace on a)""" @_scal_inplace def neq_inplace(a, b): """a != b (inplace on a)""" @_scal_inplace def and__inplace(a, b): """bitwise a & b (inplace on a)""" @_scal_inplace def or__inplace(a, b): """bitwise a | b (inplace on a)""" @_scal_inplace def xor_inplace(a, b): """bitwise a ^ b (inplace on a)""" @_scal_inplace def invert_inplace(a): """bitwise ~a (inplace on a)""" @_scal_inplace def abs__inplace(a): """|`a`| (inplace on `a`)""" @_scal_inplace def exp_inplace(a): """e^`a` (inplace on `a`)""" @_scal_inplace def exp2_inplace(a): """2^`a` (inplace on `a`)""" @_scal_inplace def expm1_inplace(a): """e^`a` - 1 (inplace on `a`)""" @_scal_inplace def neg_inplace(a): """-a (inplace on a)""" @_scal_inplace def inv_inplace(a): """1.0/a (inplace on a)""" @_scal_inplace def log_inplace(a): """base e logarithm of a (inplace on a)""" @_scal_inplace def log1p_inplace(a): """log(1+a)""" @_scal_inplace def log2_inplace(a): """base 2 logarithm of a (inplace on a)""" @_scal_inplace def log10_inplace(a): """base 10 logarithm of a (inplace on a)""" @_scal_inplace def sgn_inplace(a): """sign of `a` (inplace on `a`)""" @_scal_inplace def ceil_inplace(a): """ceil of `a` (inplace on `a`)""" @_scal_inplace def floor_inplace(a): """floor of `a` (inplace on `a`)""" @_scal_inplace def trunc_inplace(a): """trunc of `a` (inplace on `a`)""" @_scal_inplace def round_half_to_even_inplace(a): """round_half_to_even_inplace(a) (inplace on `a`)""" @_scal_inplace def round_half_away_from_zero_inplace(a): """round_half_away_from_zero_inplace(a) (inplace on `a`)""" @_scal_inplace def sqr_inplace(a): """square of `a` (inplace on `a`)""" @_scal_inplace def sqrt_inplace(a): """square root of `a` (inplace on `a`)""" @_scal_inplace def deg2rad_inplace(a): """convert degree `a` to radian(inplace on `a`)""" @_scal_inplace def rad2deg_inplace(a): """convert radian `a` to degree(inplace on `a`)""" @_scal_inplace def cos_inplace(a): """cosine of `a` (inplace on `a`)""" @_scal_inplace def arccos_inplace(a): """arccosine of `a` (inplace on `a`)""" @_scal_inplace def sin_inplace(a): """sine of `a` (inplace on `a`)""" @_scal_inplace def arcsin_inplace(a): """arcsine of `a` (inplace on `a`)""" @_scal_inplace def tan_inplace(a): """tangent of `a` (inplace on `a`)""" @_scal_inplace def arctan_inplace(a): """arctangent of `a` (inplace on `a`)""" @_scal_inplace def arctan2_inplace(a, b): """arctangent of `a` / `b` (inplace on `a`)""" @_scal_inplace def cosh_inplace(a): """hyperbolic cosine of `a` (inplace on `a`)""" @_scal_inplace def arccosh_inplace(a): """hyperbolic arc cosine of `a` (inplace on `a`)""" @_scal_inplace def sinh_inplace(a): """hyperbolic sine of `a` (inplace on `a`)""" @_scal_inplace def arcsinh_inplace(a): """hyperbolic arc sine of `a` (inplace on `a`)""" @_scal_inplace def tanh_inplace(a): """hyperbolic tangent of `a` (inplace on `a`)""" @_scal_inplace def arctanh_inplace(a): """hyperbolic arc tangent of `a` (inplace on `a`)""" @_scal_inplace def erf_inplace(a): """error function""" @_scal_inplace def erfc_inplace(a): """complementary error function""" @_scal_inplace def erfcx_inplace(a): """scaled complementary error function""" @_scal_inplace def gamma_inplace(a): """gamma function""" @_scal_inplace def gammaln_inplace(a): """log gamma function""" @_scal_inplace def psi_inplace(a): """derivative of log gamma function""" @_scal_inplace def tri_gamma_inplace(a): """second derivative of the log gamma function""" @_scal_inplace def chi2sf_inplace(x, k): """chi squared survival function""" @_scal_inplace def j0_inplace(x): """Bessel function of the first kind of order 0.""" @_scal_inplace def j1_inplace(x): """Bessel function of the first kind of order 1.""" @_scal_inplace def jv_inplace(v, x): """Bessel function of the first kind of order v (real).""" @_scal_inplace def i0_inplace(x): """Modified Bessel function of the first kind of order 0.""" @_scal_inplace def i1_inplace(x): """Modified Bessel function of the first kind of order 1.""" @_scal_inplace def iv_inplace(v, x): """Modified Bessel function of the first kind of order v (real).""" @_scal_inplace def second_inplace(a): """Fill `a` with `b`""" fill_inplace = second_inplace pprint.assign(fill_inplace, printing.FunctionPrinter('fill=')) @_scal_inplace def maximum_inplace(a, b): """elementwise addition (inplace on `a`)""" @_scal_inplace def minimum_inplace(a, b): """elementwise addition (inplace on `a`)""" @_scal_inplace def add_inplace(a, b): """elementwise addition (inplace on `a`)""" @_scal_inplace def sub_inplace(a, b): """elementwise subtraction (inplace on `a`)""" @_scal_inplace def mul_inplace(a, b): """elementwise multiplication (inplace on `a`)""" @_scal_inplace def true_div_inplace(a, b): """elementwise division (inplace on `a`)""" @_scal_inplace def int_div_inplace(a, b): """elementwise division (inplace on `a`)""" @_scal_inplace def mod_inplace(a, b): """elementwise modulo (inplace on `a`)""" @_scal_inplace def pow_inplace(a, b): """elementwise power (inplace on `a`)""" @_scal_inplace def conj_inplace(a): """elementwise conjugate (inplace on `a`)""" pprint.assign(add_inplace, printing.OperatorPrinter('+=', -2, 'either')) pprint.assign(mul_inplace, printing.OperatorPrinter('*=', -1, 'either')) pprint.assign(sub_inplace, printing.OperatorPrinter('-=', -2, 'left')) pprint.assign(neg_inplace, printing.OperatorPrinter('-=', 0, 'either')) pprint.assign(true_div_inplace, printing.OperatorPrinter('/=', -1, 'left')) pprint.assign(int_div_inplace, printing.OperatorPrinter('//=', -1, 'left')) pprint.assign(pow_inplace, printing.OperatorPrinter('**=', 1, 'right')) def transpose_inplace(x, **kwargs): "Perform a transpose on a tensor without copying the underlying storage" dims = list(range(x.ndim - 1, -1, -1)) return elemwise.DimShuffle(x.broadcastable, dims, inplace=True)(x) Theano-1.0.0/theano/tensor/__init__.py00006000163562€ÿÿÿþ0000000344713203335532021200 0ustar boccoset00000000000000"""Define the tensor toplevel""" from __future__ import absolute_import, print_function, division __docformat__ = "restructuredtext en" import warnings from theano.tensor.basic import * from theano.tensor.subtensor import * from theano.tensor.type_other import * from theano.tensor.var import ( AsTensorError, _tensor_py_operators, TensorVariable, TensorConstantSignature, TensorConstant) from theano.tensor import opt from theano.tensor import opt_uncanonicalize from theano.tensor import blas from theano.tensor import blas_scipy from theano.tensor import blas_c from theano.tensor import xlogx from theano.tensor import nlinalg # These imports cannot be performed here because the modules depend on tensor. This is done at the # end of theano.__init__.py instead. #from theano.tensor import raw_random #from theano.tensor import shared_randomstreams from theano.tensor.elemwise import DimShuffle, Elemwise, CAReduce from theano.tensor import sharedvar # adds shared-variable constructors # We import as `_shared` instead of `shared` to avoid confusion between # `theano.shared` and `tensor._shared`. from theano.tensor.sharedvar import tensor_constructor as _shared from theano.tensor.io import * from theano.tensor import nnet # used for softmax, sigmoid, etc. from theano.gradient import Rop, Lop, grad, numeric_grad, verify_grad, \ jacobian, hessian, consider_constant from theano.tensor.sort import sort, argsort, topk, argtopk, topk_and_argtopk from theano.tensor.extra_ops import (DiffOp, bincount, squeeze, repeat, bartlett, fill_diagonal, fill_diagonal_offset, cumsum, cumprod, unravel_index, ravel_multi_index) # SpecifyShape is defined in theano.compile, but should be available in tensor from theano.compile import SpecifyShape, specify_shape Theano-1.0.0/theano/tensor/nlinalg.py00006000163562€ÿÿÿþ0000005612613203335532021067 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import logging import warnings import numpy as np from six.moves import xrange import theano from theano.tensor import as_tensor_variable from theano.gof import Op, Apply from theano.gradient import DisconnectedType from theano.tensor import basic as tensor from theano.tensor.basic import ExtractDiag logger = logging.getLogger(__name__) class MatrixPinv(Op): """Computes the pseudo-inverse of a matrix :math:`A`. The pseudo-inverse of a matrix :math:`A`, denoted :math:`A^+`, is defined as: "the matrix that 'solves' [the least-squares problem] :math:`Ax = b`," i.e., if :math:`\\bar{x}` is said solution, then :math:`A^+` is that matrix such that :math:`\\bar{x} = A^+b`. Note that :math:`Ax=AA^+b`, so :math:`AA^+` is close to the identity matrix. This method is not faster than `matrix_inverse`. Its strength comes from that it works for non-square matrices. If you have a square matrix though, `matrix_inverse` can be both more exact and faster to compute. Also this op does not get optimized into a solve op. """ __props__ = () def __init__(self): pass def make_node(self, x): x = as_tensor_variable(x) assert x.ndim == 2 return Apply(self, [x], [x.type()]) def perform(self, node, inputs, outputs): (x,) = inputs (z,) = outputs z[0] = np.linalg.pinv(x).astype(x.dtype) def L_op(self, inputs, outputs, g_outputs): r"""The gradient function should return .. math:: V\frac{\partial X^+}{\partial X}, where :math:`V` corresponds to ``g_outputs`` and :math:`X` to ``inputs``. According to `Wikipedia `_, this corresponds to .. math:: (-X^+ V^T X^+ + X^+ X^{+T} V (I - X X^+) + (I - X^+ X) V X^{+T} X^+)^T. """ x, = inputs z, = outputs gz, = g_outputs x_dot_z = theano.tensor.dot(x, z) z_dot_x = theano.tensor.dot(z, x) grad = (-matrix_dot(z, gz.T, z) + matrix_dot(z, z.T, gz, (theano.tensor.identity_like(x_dot_z) - x_dot_z)) + matrix_dot((theano.tensor.identity_like(z_dot_x) - z_dot_x), gz, z.T, z)).T return [grad] pinv = MatrixPinv() class MatrixInverse(Op): """Computes the inverse of a matrix :math:`A`. Given a square matrix :math:`A`, ``matrix_inverse`` returns a square matrix :math:`A_{inv}` such that the dot product :math:`A \cdot A_{inv}` and :math:`A_{inv} \cdot A` equals the identity matrix :math:`I`. Notes ----- When possible, the call to this op will be optimized to the call of ``solve``. """ __props__ = () def __init__(self): pass def make_node(self, x): x = as_tensor_variable(x) assert x.ndim == 2 return Apply(self, [x], [x.type()]) def perform(self, node, inputs, outputs): (x,) = inputs (z,) = outputs z[0] = np.linalg.inv(x).astype(x.dtype) def grad(self, inputs, g_outputs): r"""The gradient function should return .. math:: V\frac{\partial X^{-1}}{\partial X}, where :math:`V` corresponds to ``g_outputs`` and :math:`X` to ``inputs``. Using the `matrix cookbook `_, one can deduce that the relation corresponds to .. math:: (X^{-1} \cdot V^{T} \cdot X^{-1})^T. """ x, = inputs xi = self(x) gz, = g_outputs # TT.dot(gz.T,xi) return [-matrix_dot(xi, gz.T, xi).T] def R_op(self, inputs, eval_points): r"""The gradient function should return .. math:: \frac{\partial X^{-1}}{\partial X}V, where :math:`V` corresponds to ``g_outputs`` and :math:`X` to ``inputs``. Using the `matrix cookbook `_, one can deduce that the relation corresponds to .. math:: X^{-1} \cdot V \cdot X^{-1}. """ x, = inputs xi = self(x) ev, = eval_points if ev is None: return [None] return [-matrix_dot(xi, ev, xi)] def infer_shape(self, node, shapes): return shapes matrix_inverse = MatrixInverse() def matrix_dot(*args): """ Shorthand for product between several dots. Given :math:`N` matrices :math:`A_0, A_1, .., A_N`, ``matrix_dot`` will generate the matrix product between all in the given order, namely :math:`A_0 \cdot A_1 \cdot A_2 \cdot .. \cdot A_N`. """ rval = args[0] for a in args[1:]: rval = theano.tensor.dot(rval, a) return rval class AllocDiag(Op): """ Allocates a square matrix with the given vector as its diagonal. """ __props__ = () def make_node(self, _x): warnings.warn("DeprecationWarning: theano.tensor.nlinalg.AllocDiag" "is deprecated, please use theano.tensor.AllocDiag" "instead.", category=DeprecationWarning) x = as_tensor_variable(_x) if x.type.ndim != 1: raise TypeError('AllocDiag only works on vectors', _x) return Apply(self, [x], [theano.tensor.matrix(dtype=x.type.dtype)]) def grad(self, inputs, g_outputs): return [extract_diag(g_outputs[0])] def perform(self, node, inputs, outputs): (x,) = inputs (z,) = outputs if x.ndim != 1: raise TypeError(x) z[0] = np.diag(x) def infer_shape(self, node, shapes): x_s, = shapes return [(x_s[0], x_s[0])] alloc_diag = AllocDiag() extract_diag = ExtractDiag() # TODO: optimization to insert ExtractDiag with view=True def diag(x): """ Numpy-compatibility method If `x` is a matrix, return its diagonal. If `x` is a vector return a matrix with it as its diagonal. * This method does not support the `k` argument that numpy supports. """ xx = as_tensor_variable(x) if xx.type.ndim == 1: return alloc_diag(xx) elif xx.type.ndim == 2: return extract_diag(xx) else: raise TypeError('diag requires vector or matrix argument', x) def trace(X): """ Returns the sum of diagonal elements of matrix X. Notes ----- Works on GPU since 0.6rc4. """ return extract_diag(X).sum() class Det(Op): """ Matrix determinant. Input should be a square matrix. """ __props__ = () def make_node(self, x): x = as_tensor_variable(x) assert x.ndim == 2 o = theano.tensor.scalar(dtype=x.dtype) return Apply(self, [x], [o]) def perform(self, node, inputs, outputs): (x,) = inputs (z,) = outputs try: z[0] = np.asarray(np.linalg.det(x), dtype=x.dtype) except Exception: print('Failed to compute determinant', x) raise def grad(self, inputs, g_outputs): gz, = g_outputs x, = inputs return [gz * self(x) * matrix_inverse(x).T] def infer_shape(self, node, shapes): return [()] def __str__(self): return "Det" det = Det() class Eig(Op): """ Compute the eigenvalues and right eigenvectors of a square array. """ _numop = staticmethod(np.linalg.eig) __props__ = () def make_node(self, x): x = as_tensor_variable(x) assert x.ndim == 2 w = theano.tensor.vector(dtype=x.dtype) v = theano.tensor.matrix(dtype=x.dtype) return Apply(self, [x], [w, v]) def perform(self, node, inputs, outputs): (x,) = inputs (w, v) = outputs w[0], v[0] = [z.astype(x.dtype) for z in self._numop(x)] def infer_shape(self, node, shapes): n = shapes[0][0] return [(n,), (n, n)] eig = Eig() class Eigh(Eig): """ Return the eigenvalues and eigenvectors of a Hermitian or symmetric matrix. """ _numop = staticmethod(np.linalg.eigh) __props__ = ('UPLO',) def __init__(self, UPLO='L'): assert UPLO in ['L', 'U'] self.UPLO = UPLO def make_node(self, x): x = as_tensor_variable(x) assert x.ndim == 2 # Numpy's linalg.eigh may return either double or single # presision eigenvalues depending on installed version of # LAPACK. Rather than trying to reproduce the (rather # involved) logic, we just probe linalg.eigh with a trivial # input. w_dtype = self._numop([[np.dtype(x.dtype).type()]])[0].dtype.name w = theano.tensor.vector(dtype=w_dtype) v = theano.tensor.matrix(dtype=x.dtype) return Apply(self, [x], [w, v]) def perform(self, node, inputs, outputs): (x,) = inputs (w, v) = outputs w[0], v[0] = self._numop(x, self.UPLO) def grad(self, inputs, g_outputs): r"""The gradient function should return .. math:: \sum_n\left(W_n\frac{\partial\,w_n} {\partial a_{ij}} + \sum_k V_{nk}\frac{\partial\,v_{nk}} {\partial a_{ij}}\right), where [:math:`W`, :math:`V`] corresponds to ``g_outputs``, :math:`a` to ``inputs``, and :math:`(w, v)=\mbox{eig}(a)`. Analytic formulae for eigensystem gradients are well-known in perturbation theory: .. math:: \frac{\partial\,w_n} {\partial a_{ij}} = v_{in}\,v_{jn} .. math:: \frac{\partial\,v_{kn}} {\partial a_{ij}} = \sum_{m\ne n}\frac{v_{km}v_{jn}}{w_n-w_m} """ x, = inputs w, v = self(x) # Replace gradients wrt disconnected variables with # zeros. This is a work-around for issue #1063. gw, gv = _zero_disconnected([w, v], g_outputs) return [EighGrad(self.UPLO)(x, w, v, gw, gv)] def _zero_disconnected(outputs, grads): l = [] for o, g in zip(outputs, grads): if isinstance(g.type, DisconnectedType): l.append(o.zeros_like()) else: l.append(g) return l class EighGrad(Op): """ Gradient of an eigensystem of a Hermitian matrix. """ __props__ = ('UPLO',) def __init__(self, UPLO='L'): assert UPLO in ['L', 'U'] self.UPLO = UPLO if UPLO == 'L': self.tri0 = np.tril self.tri1 = lambda a: np.triu(a, 1) else: self.tri0 = np.triu self.tri1 = lambda a: np.tril(a, -1) def make_node(self, x, w, v, gw, gv): x, w, v, gw, gv = map(as_tensor_variable, (x, w, v, gw, gv)) assert x.ndim == 2 assert w.ndim == 1 assert v.ndim == 2 assert gw.ndim == 1 assert gv.ndim == 2 out_dtype = theano.scalar.upcast(x.dtype, w.dtype, v.dtype, gw.dtype, gv.dtype) out = theano.tensor.matrix(dtype=out_dtype) return Apply(self, [x, w, v, gw, gv], [out]) def perform(self, node, inputs, outputs): """ Implements the "reverse-mode" gradient for the eigensystem of a square matrix. """ x, w, v, W, V = inputs N = x.shape[0] outer = np.outer def G(n): return sum(v[:, m] * V.T[n].dot(v[:, m]) / (w[n] - w[m]) for m in xrange(N) if m != n) g = sum(outer(v[:, n], v[:, n] * W[n] + G(n)) for n in xrange(N)) # Numpy's eigh(a, 'L') (eigh(a, 'U')) is a function of tril(a) # (triu(a)) only. This means that partial derivative of # eigh(a, 'L') (eigh(a, 'U')) with respect to a[i,j] is zero # for i < j (i > j). At the same time, non-zero components of # the gradient must account for the fact that variation of the # opposite triangle contributes to variation of two elements # of Hermitian (symmetric) matrix. The following line # implements the necessary logic. out = self.tri0(g) + self.tri1(g).T # Make sure we return the right dtype even if NumPy performed # upcasting in self.tri0. outputs[0][0] = np.asarray(out, dtype=node.outputs[0].dtype) def infer_shape(self, node, shapes): return [shapes[0]] def eigh(a, UPLO='L'): return Eigh(UPLO)(a) class QRFull(Op): """ Full QR Decomposition. Computes the QR decomposition of a matrix. Factor the matrix a as qr, where q is orthonormal and r is upper-triangular. """ _numop = staticmethod(np.linalg.qr) __props__ = ('mode',) def __init__(self, mode): self.mode = mode def make_node(self, x): x = as_tensor_variable(x) assert x.ndim == 2, "The input of qr function should be a matrix." q = theano.tensor.matrix(dtype=x.dtype) if self.mode != 'raw': r = theano.tensor.matrix(dtype=x.dtype) else: r = theano.tensor.vector(dtype=x.dtype) return Apply(self, [x], [q, r]) def perform(self, node, inputs, outputs): (x,) = inputs (q, r) = outputs assert x.ndim == 2, "The input of qr function should be a matrix." q[0], r[0] = self._numop(x, self.mode) class QRIncomplete(Op): """ Incomplete QR Decomposition. Computes the QR decomposition of a matrix. Factor the matrix a as qr and return a single matrix R. """ _numop = staticmethod(np.linalg.qr) __props__ = ('mode',) def __init__(self, mode): self.mode = mode def make_node(self, x): x = as_tensor_variable(x) assert x.ndim == 2, "The input of qr function should be a matrix." r = theano.tensor.matrix(dtype=x.dtype) return Apply(self, [x], [r]) def perform(self, node, inputs, outputs): (x,) = inputs (r,) = outputs assert x.ndim == 2, "The input of qr function should be a matrix." r[0] = self._numop(x, self.mode) def qr(a, mode="reduced"): """ Computes the QR decomposition of a matrix. Factor the matrix a as qr, where q is orthonormal and r is upper-triangular. Parameters ---------- a : array_like, shape (M, N) Matrix to be factored. mode : {'reduced', 'complete', 'r', 'raw'}, optional If K = min(M, N), then 'reduced' returns q, r with dimensions (M, K), (K, N) 'complete' returns q, r with dimensions (M, M), (M, N) 'r' returns r only with dimensions (K, N) 'raw' returns h, tau with dimensions (N, M), (K,) Note that array h returned in 'raw' mode is transposed for calling Fortran. Default mode is 'reduced' Returns ------- q : matrix of float or complex, optional A matrix with orthonormal columns. When mode = 'complete' the result is an orthogonal/unitary matrix depending on whether or not a is real/complex. The determinant may be either +/- 1 in that case. r : matrix of float or complex, optional The upper-triangular matrix. """ x = [[2, 1], [3, 4]] if isinstance(np.linalg.qr(x, mode), tuple): return QRFull(mode)(a) else: return QRIncomplete(mode)(a) class SVD(Op): """ Parameters ---------- full_matrices : bool, optional If True (default), u and v have the shapes (M, M) and (N, N), respectively. Otherwise, the shapes are (M, K) and (K, N), respectively, where K = min(M, N). compute_uv : bool, optional Whether or not to compute u and v in addition to s. True by default. """ # See doc in the docstring of the function just after this class. _numop = staticmethod(np.linalg.svd) __props__ = ('full_matrices', 'compute_uv') def __init__(self, full_matrices=True, compute_uv=True): self.full_matrices = full_matrices self.compute_uv = compute_uv def make_node(self, x): x = as_tensor_variable(x) assert x.ndim == 2, "The input of svd function should be a matrix." s = theano.tensor.vector(dtype=x.dtype) if self.compute_uv: u = theano.tensor.matrix(dtype=x.dtype) vt = theano.tensor.matrix(dtype=x.dtype) return Apply(self, [x], [u, s, vt]) else: return Apply(self, [x], [s]) def perform(self, node, inputs, outputs): (x,) = inputs assert x.ndim == 2, "The input of svd function should be a matrix." if self.compute_uv: u, s, vt = outputs u[0], s[0], vt[0] = self._numop(x, self.full_matrices, self.compute_uv) else: s, = outputs s[0] = self._numop(x, self.full_matrices, self.compute_uv) def infer_shape(self, node, shapes): x_shape, = shapes M, N = x_shape K = tensor.minimum(M, N) s_shape = (K, ) if self.compute_uv: u_shape = (M, M) if self.full_matrices else (M, K) vt_shape = (N, N) if self.full_matrices else (K, N) return [u_shape, s_shape, vt_shape] else: return [s_shape] def svd(a, full_matrices=1, compute_uv=1): """ This function performs the SVD on CPU. Parameters ---------- full_matrices : bool, optional If True (default), u and v have the shapes (M, M) and (N, N), respectively. Otherwise, the shapes are (M, K) and (K, N), respectively, where K = min(M, N). compute_uv : bool, optional Whether or not to compute u and v in addition to s. True by default. Returns ------- U, V, D : matrices """ return SVD(full_matrices, compute_uv)(a) class lstsq(Op): __props__ = () def make_node(self, x, y, rcond): x = theano.tensor.as_tensor_variable(x) y = theano.tensor.as_tensor_variable(y) rcond = theano.tensor.as_tensor_variable(rcond) return theano.Apply(self, [x, y, rcond], [theano.tensor.matrix(), theano.tensor.dvector(), theano.tensor.lscalar(), theano.tensor.dvector()]) def perform(self, node, inputs, outputs): zz = np.linalg.lstsq(inputs[0], inputs[1], inputs[2]) outputs[0][0] = zz[0] outputs[1][0] = zz[1] outputs[2][0] = np.array(zz[2]) outputs[3][0] = zz[3] def matrix_power(M, n): """ Raise a square matrix to the (integer) power n. Parameters ---------- M : Tensor variable n : Python int """ result = 1 for i in xrange(n): result = theano.dot(result, M) return result def norm(x, ord): x = as_tensor_variable(x) ndim = x.ndim if ndim == 0: raise ValueError("'axis' entry is out of bounds.") elif ndim == 1: if ord is None: return tensor.sum(x**2)**0.5 elif ord == 'inf': return tensor.max(abs(x)) elif ord == '-inf': return tensor.min(abs(x)) elif ord == 0: return x[x.nonzero()].shape[0] else: try: z = tensor.sum(abs(x**ord))**(1. / ord) except TypeError: raise ValueError("Invalid norm order for vectors.") return z elif ndim == 2: if ord is None or ord == 'fro': return tensor.sum(abs(x**2))**(0.5) elif ord == 'inf': return tensor.max(tensor.sum(abs(x), 1)) elif ord == '-inf': return tensor.min(tensor.sum(abs(x), 1)) elif ord == 1: return tensor.max(tensor.sum(abs(x), 0)) elif ord == -1: return tensor.min(tensor.sum(abs(x), 0)) else: raise ValueError(0) elif ndim > 2: raise NotImplementedError("We don't support norm witn ndim > 2") class TensorInv(Op): """ Class wrapper for tensorinv() function; Theano utilization of numpy.linalg.tensorinv; """ _numop = staticmethod(np.linalg.tensorinv) __props__ = ('ind',) def __init__(self, ind=2): self.ind = ind def make_node(self, a): a = as_tensor_variable(a) out = a.type() return Apply(self, [a], [out]) def perform(self, node, inputs, outputs): (a,) = inputs (x,) = outputs x[0] = self._numop(a, self.ind) def infer_shape(self, node, shapes): sp = shapes[0][self.ind:] + shapes[0][:self.ind] return [sp] def tensorinv(a, ind=2): """ Does not run on GPU; Theano utilization of numpy.linalg.tensorinv; Compute the 'inverse' of an N-dimensional array. The result is an inverse for `a` relative to the tensordot operation ``tensordot(a, b, ind)``, i. e., up to floating-point accuracy, ``tensordot(tensorinv(a), a, ind)`` is the "identity" tensor for the tensordot operation. Parameters ---------- a : array_like Tensor to 'invert'. Its shape must be 'square', i. e., ``prod(a.shape[:ind]) == prod(a.shape[ind:])``. ind : int, optional Number of first indices that are involved in the inverse sum. Must be a positive integer, default is 2. Returns ------- b : ndarray `a`'s tensordot inverse, shape ``a.shape[ind:] + a.shape[:ind]``. Raises ------ LinAlgError If `a` is singular or not 'square' (in the above sense). """ return TensorInv(ind)(a) class TensorSolve(Op): """ Theano utilization of numpy.linalg.tensorsolve Class wrapper for tensorsolve function. """ _numop = staticmethod(np.linalg.tensorsolve) __props__ = ('axes', ) def __init__(self, axes=None): self.axes = axes def make_node(self, a, b): a = as_tensor_variable(a) b = as_tensor_variable(b) out_dtype = theano.scalar.upcast(a.dtype, b.dtype) x = theano.tensor.matrix(dtype=out_dtype) return Apply(self, [a, b], [x]) def perform(self, node, inputs, outputs): (a, b,) = inputs (x,) = outputs x[0] = self._numop(a, b, self.axes) def tensorsolve(a, b, axes=None): """ Theano utilization of numpy.linalg.tensorsolve. Does not run on GPU! Solve the tensor equation ``a x = b`` for x. It is assumed that all indices of `x` are summed over in the product, together with the rightmost indices of `a`, as is done in, for example, ``tensordot(a, x, axes=len(b.shape))``. Parameters ---------- a : array_like Coefficient tensor, of shape ``b.shape + Q``. `Q`, a tuple, equals the shape of that sub-tensor of `a` consisting of the appropriate number of its rightmost indices, and must be such that ``prod(Q) == prod(b.shape)`` (in which sense `a` is said to be 'square'). b : array_like Right-hand tensor, which can be of any shape. axes : tuple of ints, optional Axes in `a` to reorder to the right, before inversion. If None (default), no reordering is done. Returns ------- x : ndarray, shape Q Raises ------ LinAlgError If `a` is singular or not 'square' (in the above sense). """ return TensorSolve(axes)(a, b) Theano-1.0.0/theano/tensor/inc_code.py00006000163562€ÿÿÿþ0000001111713203335532021175 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division def inc_code(): types = ['npy_' + t for t in ['int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', 'float16', 'float32', 'float64']] complex_types = ['npy_' + t for t in ['complex32', 'complex64', 'complex128']] inplace_map_template = """ #if defined(%(typen)s) static void %(type)s_inplace_add(PyArrayMapIterObject *mit, PyArrayIterObject *it, int inc_or_set) { int index = mit->size; while (index--) { %(op)s PyArray_MapIterNext(mit); PyArray_ITER_NEXT(it); } } #endif """ floatadd = ("((%(type)s*)mit->dataptr)[0] = " "(inc_or_set ? ((%(type)s*)mit->dataptr)[0] : 0)" " + ((%(type)s*)it->dataptr)[0];") complexadd = """ ((%(type)s*)mit->dataptr)[0].real = (inc_or_set ? ((%(type)s*)mit->dataptr)[0].real : 0) + ((%(type)s*)it->dataptr)[0].real; ((%(type)s*)mit->dataptr)[0].imag = (inc_or_set ? ((%(type)s*)mit->dataptr)[0].imag : 0) + ((%(type)s*)it->dataptr)[0].imag; """ fns = ''.join([inplace_map_template % {'type': t, 'typen': t.upper(), 'op': floatadd % {'type': t}} for t in types] + [inplace_map_template % {'type': t, 'typen': t.upper(), 'op': complexadd % {'type': t}} for t in complex_types]) def gen_binop(type, typen): return """ #if defined(%(typen)s) %(type)s_inplace_add, #endif """ % dict(type=type, typen=typen) fn_array = ("static inplace_map_binop addition_funcs[] = {" + ''.join([gen_binop(type=t, typen=t.upper()) for t in types + complex_types]) + "NULL};\n") def gen_num(typen): return """ #if defined(%(typen)s) %(typen)s, #endif """ % dict(type=type, typen=typen) type_number_array = ("static int type_numbers[] = {" + ''.join([gen_num(typen=t.upper()) for t in types + complex_types]) + "-1000};") code = (""" typedef void (*inplace_map_binop)(PyArrayMapIterObject *, PyArrayIterObject *, int inc_or_set); """ + fns + fn_array + type_number_array + """ static int map_increment(PyArrayMapIterObject *mit, PyArrayObject *op, inplace_map_binop add_inplace, int inc_or_set) { PyArrayObject *arr = NULL; PyArrayIterObject *it; PyArray_Descr *descr; if (mit->ait == NULL) { return -1; } descr = PyArray_DESCR(mit->ait->ao); Py_INCREF(descr); arr = (PyArrayObject *)PyArray_FromAny((PyObject *)op, descr, 0, 0, NPY_ARRAY_FORCECAST, NULL); if (arr == NULL) { return -1; } if ((mit->subspace != NULL) && (mit->consec)) { PyArray_MapIterSwapAxes(mit, (PyArrayObject **)&arr, 0); if (arr == NULL) { return -1; } } it = (PyArrayIterObject*) PyArray_BroadcastToShape((PyObject*)arr, mit->dimensions, mit->nd); if (it == NULL) { Py_DECREF(arr); return -1; } (*add_inplace)(mit, it, inc_or_set); Py_DECREF(arr); Py_DECREF(it); return 0; } static int inplace_increment(PyArrayObject *a, PyObject *index, PyArrayObject *inc, int inc_or_set) { inplace_map_binop add_inplace = NULL; int type_number = -1; int i = 0; PyArrayMapIterObject * mit; if (PyArray_FailUnlessWriteable(a, "input/output array") < 0) { return -1; } if (PyArray_NDIM(a) == 0) { PyErr_SetString(PyExc_IndexError, "0-d arrays can't be indexed."); return -1; } type_number = PyArray_TYPE(a); while (type_numbers[i] >= 0 && addition_funcs[i] != NULL){ if (type_number == type_numbers[i]) { add_inplace = addition_funcs[i]; break; } i++ ; } if (add_inplace == NULL) { PyErr_SetString(PyExc_TypeError, "unsupported type for a"); return -1; } mit = (PyArrayMapIterObject *) PyArray_MapIterArray(a, index); if (mit == NULL) { goto fail; } if (map_increment(mit, inc, add_inplace, inc_or_set) != 0) { goto fail; } Py_DECREF(mit); Py_INCREF(Py_None); return 0; fail: Py_XDECREF(mit); return -1; } """) return code Theano-1.0.0/theano/tensor/signal/00007500163562€ÿÿÿþ0000000000013203336030020331 5ustar boccoset00000000000000Theano-1.0.0/theano/tensor/signal/__init__.py00006000163562€ÿÿÿþ0000000000013203335532022433 0ustar boccoset00000000000000Theano-1.0.0/theano/tensor/signal/pool.py00007000163562€ÿÿÿþ0000027745613203335532021705 0ustar boccoset00000000000000 """ Ops for downsampling images. Planned: Pool, DownsampleAvg, DownsampleSoftmax. """ from __future__ import absolute_import, print_function, division # This file should move along with conv.py import warnings import itertools import numpy as np from six.moves import xrange import six.moves.builtins as builtins import theano from theano import gof, OpenMPOp, tensor, Variable, Apply from theano.gof import ParamsType, EnumList from theano.gradient import DisconnectedType from theano.scalar import bool as bool_t def max_pool_2d_same_size(input, patch_size): """ Takes as input a 4-D tensor. It sets all non maximum values of non-overlapping patches of size (patch_size[0],patch_size[1]) to zero, keeping only the maximum values. The output has the same dimensions as the input. Parameters ---------- input : 4-D theano tensor of input images Input images. Max pooling will be done over the 2 last dimensions. patch_size : tuple of length 2 or theano vector of ints of size 2. Size of the patch (patch height, patch width). (2,2) will retain only one non-zero value per patch of 4 values. """ output = Pool(True)(input, patch_size) outs = MaxPoolGrad(True)(input, output, output, patch_size) return outs def pool_2d(input, ws=None, ignore_border=None, stride=None, pad=(0, 0), mode='max', ds=None, st=None, padding=None): """Downscale the input by a specified factor Takes as input a N-D tensor, where N >= 2. It downscales the input image by the specified factor, by keeping only the maximum value of non-overlapping patches of size (ws[0],ws[1]) Parameters ---------- input : N-D theano tensor of input images Input images. Max pooling will be done over the 2 last dimensions. ws : tuple of length 2 or theano vector of ints of size 2. Factor by which to downscale (vertical ws, horizontal ws). (2,2) will halve the image in each dimension. ignore_border : bool (default None, will print a warning and set to False) When True, (5,5) input with ws=(2,2) will generate a (2,2) output. (3,3) otherwise. stride : tuple of two ints or theano vector of ints of size 2. Stride size, which is the number of shifts over rows/cols to get the next pool region. If stride is None, it is considered equal to ws (no overlap on pooling regions), eg: stride=(1,1) will shifts over one row and one col for every iteration. pad : tuple of two ints or theano vector of ints of size 2. (pad_h, pad_w), pad zeros to extend beyond four borders of the images, pad_h is the size of the top and bottom margins, and pad_w is the size of the left and right margins. mode : {'max', 'sum', 'average_inc_pad', 'average_exc_pad'} Operation executed on each window. `max` and `sum` always exclude the padding in the computation. `average` gives you the choice to include or exclude it. ds *deprecated*, use parameter ws instead. st *deprecated*, use parameter stride instead. padding *deprecated*, use parameter pad instead. """ # check for deprecated parameter names if ds is not None: if ws is not None: raise ValueError( "You can't provide a tuple value to both 'ws' and 'ds'." " Please provide a value only to 'ws'." ) else: warnings.warn( "DEPRECATION: the 'ds' parameter is not going to exist" " anymore as it is going to be replaced by the parameter" " 'ws'.", stacklevel=2 ) ws = ds elif ds is None and ws is None: raise ValueError( "You must provide a tuple value for the window size." ) if st is not None: if stride is not None: raise ValueError( "You can't provide a tuple value to both 'st and 'stride'." " Please provide a value only to 'stride'." ) else: warnings.warn( "DEPRECATION: the 'st' parameter is not going to exist" " anymore as it is going to be replaced by the parameter" " 'stride'.", stacklevel=2 ) stride = st if padding is not None: if pad not in {None, (0, 0)}: raise ValueError( "You can't provide a tuple value to both 'padding' and pad." " Please provide a value only to pad." ) else: warnings.warn( "DEPRECATION: the 'padding' parameter is not going to exist" " anymore as it is going to be replaced by the parameter" " 'pad'.", stacklevel=2 ) pad = padding if input.ndim < 2: raise NotImplementedError('pool_2d requires a dimension >= 2') if ignore_border is None: warnings.warn( "pool_2d() will have the parameter ignore_border" " default value changed to True (currently" " False). To have consistent behavior with all Theano" " version, explicitly add the parameter ignore_border=True." " On the GPU, using ignore_border=True is needed to use cuDNN." " When using ignore_border=False and not using cuDNN, the only" " GPU combination supported is when" " `ws == stride and pad == (0, 0) and mode == 'max'`." " Otherwise, the convolution will be executed on CPU.", stacklevel=2) ignore_border = False op = Pool(ignore_border, ndim=2, mode=mode) output = op(input, ws, stride, pad) return output def pool_3d(input, ws=None, ignore_border=None, stride=None, pad=(0, 0, 0), mode='max', ds=None, st=None, padding=None): """Downscale the input by a specified factor Takes as input a N-D tensor, where N >= 3. It downscales the input image by the specified factor, by keeping only the maximum value of non-overlapping patches of size (ws[0],ws[1],ws[2]) Parameters ---------- input : N-D theano tensor of input images Input images. Max pooling will be done over the 3 last dimensions. ws : tuple of length 3 or theano vector of ints of size 3 Factor by which to downscale (vertical ws, horizontal ws, depth ws). (2,2,2) will halve the image in each dimension. ignore_border : bool (default None, will print a warning and set to False) When True, (5,5,5) input with ws=(2,2,2) will generate a (2,2,2) output. (3,3,3) otherwise. st : tuple of three ints or theano vector of ints of size 3 Stride size, which is the number of shifts over rows/cols/slices to get the next pool region. If st is None, it is considered equal to ws (no overlap on pooling regions). pad : tuple of two ints or theano vector of ints of size 3 (pad_h, pad_w, pad_d), pad zeros to extend beyond six borders of the images, pad_h is the size of the top and bottom margins, pad_w is the size of the left and right margins, and pad_d is the size of the front and back margins mode : {'max', 'sum', 'average_inc_pad', 'average_exc_pad'} Operation executed on each window. `max` and `sum` always exclude the padding in the computation. `average` gives you the choice to include or exclude it. ds *deprecated*, use parameter ws instead. st *deprecated*, use parameter st instead. padding *deprecated*, use parameter pad instead. """ # check for deprecated parameter names if ds is not None: if ws is not None: raise ValueError( "You can't provide a tuple value to both 'ws' and 'ds'." " Please provide a value only to 'ws'." ) else: warnings.warn( "DEPRECATION: the 'ds' parameter is not going to exist" " anymore as it is going to be replaced by the parameter" " 'ws'.", stacklevel=2 ) ws = ds elif ds is None and ws is None: raise ValueError( "You must provide a tuple value for the window size." ) if st is not None: if stride is not None: raise ValueError( "You can't provide a tuple value to both 'st and 'stride'." " Please provide a value only to 'stride'." ) else: warnings.warn( "DEPRECATION: the 'st' parameter is not going to exist" " anymore as it is going to be replaced by the parameter" " 'stride'.", stacklevel=2 ) stride = st if padding is not None: if pad not in {None, (0, 0, 0)}: raise ValueError( "You can't provide a tuple value to both 'padding' and pad." " Please provide a value only to pad." ) else: warnings.warn( "DEPRECATION: the 'padding' parameter is not going to exist" " anymore as it is going to be replaced by the parameter" " 'pad'.", stacklevel=2 ) pad = padding if input.ndim < 3: raise NotImplementedError('pool_3d requires a dimension >= 3') if ignore_border is None: warnings.warn( "pool_3d() will have the parameter ignore_border" " default value changed to True (currently" " False). To have consistent behavior with all Theano" " version, explicitly add the parameter ignore_border=True." " On the GPU, using ignore_border=True is needed to use cuDNN." " When using ignore_border=False and not using cuDNN, the only" " GPU combination supported is when" " `ws == stride and pad == (0, 0, 0) and mode == 'max'`." " Otherwise, the convolution will be executed on CPU.", stacklevel=2) ignore_border = False op = Pool(ignore_border, ndim=3, mode=mode) output = op(input, ws, stride, pad) return output # NB: This enum type is currently used in gpuarray/pool.py. # It may be used later as op param in this current file. # Enum name and constants names are inspired from cuDNN type `cudnnPoolingMode_t` # (cf. `theano/gpuarray/cudnn_defs.py`). PoolingMode_t = EnumList(('POOLING_MAX', 'max'), ('POOLING_SUM', 'sum'), ('POOLING_AVERAGE_COUNT_INCLUDE_PADDING', 'average_inc_pad'), ('POOLING_AVERAGE_COUNT_EXCLUDE_PADDING', 'average_exc_pad')) class Pool(OpenMPOp): """ sum or average over different patches. Parameters ---------- ws : list or tuple of N ints Downsample factor over rows, columns etc. ws indicates the size of the pooling region. ignore_border : bool If ws doesn't divide imgshape, do we include an extra row/col/slice of partial downsampling (False) or ignore it (True). stride : list or tuple of N ints or None Stride size, which is the number of shifts over rows/cols/slices to get the next pool region. If stride is None, it is considered equal to ws (no overlap on pooling regions). pad : tuple of N ints or None For each downsampling dimension, this specifies the number of zeros to add as padding on both sides. For 2D and (pad_h, pad_w), pad_h specifies the size of the top and bottom margins, pad_w specifies the size of the left and right margins. No padding is added if pad is None. mode : {'max', 'sum', 'average_inc_pad', 'average_exc_pad'} ('average_inc_pad' excludes the padding from the count, 'average_exc_pad' include it) ndim : int The number of pooling dimensions N. The default is 2. ds *deprecated*, use parameter ws instead. st *deprecated*, use parameter st instead. padding *deprecated*, use parameter pad instead. """ __props__ = ('ignore_border', 'mode', 'ndim') params_type = ParamsType(ignore_border=bool_t,) @staticmethod def out_shape(imgshape, ws=None, ignore_border=False, stride=None, pad=None, ndim=2, ds=None, st=None, padding=None): """ Return the shape of the output from this op, for input of given shape and flags. Parameters ---------- imgshape : tuple, list, or similar of integer or scalar Theano variable The shape of a tensor of images. The last N elements are interpreted as the number of rows, and the number of cols. ws : list or tuple of N ints Downsample factor over rows and column. ws indicates the pool region size. ignore_border : bool If ws doesn't divide imgshape, do we include an extra row/col/slice of partial downsampling (False) or ignore it (True). stride : list or tuple of N ints or None Stride size, which is the number of shifts over rows/cols/slices to get the next pool region. If stride is None, it is considered equal to ws (no overlap on pooling regions). pad : tuple of N ints or None For each downsampling dimension, this specifies the number of zeros to add as padding on both sides. For 2D and (pad_h, pad_w), pad_h specifies the size of the top and bottom margins, pad_w specifies the size of the left and right margins. No padding is added if pad is None. ndim : int The number of pooling dimensions N. The default is 2. ds *deprecated*, use parameter ws instead. st *deprecated*, use parameter st instead. padding *deprecated*, use parameter pad instead. Returns ------- list The shape of the output from this op, for input of given shape. This will have the same length as imgshape, but with last N elements reduced as per the downsampling & ignore_border flags. """ # check for deprecated parameter names if ds is not None: if ws is not None: raise ValueError( "You can't provide a tuple value to both 'ws' and 'ds'." " Please provide a value only to 'ws'." ) else: warnings.warn( "DEPRECATION: the 'ds' parameter is not going to exist" " anymore as it is going to be replaced by the parameter" " 'ws'.", stacklevel=2 ) ws = ds elif ds is None and ws is None: raise ValueError( "You must provide a tuple value for the window size." ) if st is not None: if stride is not None: raise ValueError( "You can't provide a tuple value to both 'st and 'stride'." " Please provide a value only to 'stride'." ) else: warnings.warn( "DEPRECATION: the 'st' parameter is not going to exist" " anymore as it is going to be replaced by the parameter" " 'stride'.", stacklevel=2 ) stride = st if padding is not None: zero_pad = (0,) * ndim if pad not in {None, zero_pad}: raise ValueError( "You can't provide a tuple value to both 'padding' and pad." " Please provide a value only to pad." ) else: warnings.warn( "DEPRECATION: the 'padding' parameter is not going to" " exist anymore as it is going to be replaced by the" " parameter 'pad'.", stacklevel=2 ) pad = padding if ndim is None: ndim = 2 assert ndim > 0 if len(imgshape) < ndim: raise TypeError('imgshape must have at least {} dimensions'.format(ndim)) if stride is None: stride = ws if pad is None: pad = (0,) * ndim patch_shape = tuple(tensor.extract_constant(imgshape[-ndim + i]) + pad[i] * 2 for i in xrange(ndim)) def compute_out(v, downsample, stride): if ignore_border: if downsample == stride: return v // stride else: out = (v - downsample) // stride + 1 if isinstance(out, theano.Variable): return tensor.maximum(out, 0) else: return np.maximum(out, 0) else: if isinstance(v, theano.Variable): return tensor.switch(tensor.ge(stride, downsample), (v - 1) // stride + 1, tensor.maximum(0, (v - 1 - downsample) // stride + 1) + 1) elif stride >= downsample: return (v - 1) // stride + 1 else: return max(0, (v - 1 - downsample + stride) // stride) + 1 out_shape = [compute_out(patch_shape[i], ws[i], stride[i]) for i in xrange(ndim)] rval = list(imgshape[:-ndim]) + out_shape return rval def __init__(self, ignore_border=False, mode='max', ndim=2, openmp=None): super(Pool, self).__init__(openmp=openmp) self.ndim = ndim self.ignore_border = ignore_border if mode == 'max_deterministic': # It seems max pool algo is already deterministic in CPU. mode = 'max' if mode not in ['max', 'average_inc_pad', 'average_exc_pad', 'sum']: raise ValueError( "Pool mode parameter only support 'max', 'sum'," " 'average_inc_pad' and 'average_exc_pad'. Got %s" % mode) self.mode = mode def prepare_node(self, node, storage_map, compute_map, impl): if len(node.inputs) == 1: # Old interface self.ndim = len(node.op.ds) self.mode = node.op.mode ws = theano.tensor.constant(node.op.ds) st = theano.tensor.constant(node.op.st) pad = theano.tensor.constant(node.op.padding) node.inputs.append(ws) node.inputs.append(st) node.inputs.append(pad) if isinstance(ws, theano.Constant): storage_map[ws] = [ws.data] compute_map[ws] = [True] else: storage_map[ws] = [None] compute_map[ws] = [False] if isinstance(st, theano.Constant): storage_map[st] = [st.data] compute_map[st] = [True] else: storage_map[st] = [None] compute_map[st] = [False] if isinstance(pad, theano.Constant): storage_map[pad] = [pad.data] compute_map[pad] = [True] else: storage_map[pad] = [None] compute_map[pad] = [False] def make_node(self, x, ws, stride=None, pad=None): # TODO: consider restricting the dtype? x = tensor.as_tensor_variable(x) nd = self.ndim if stride is None: stride = ws if pad is None: pad = (0,) * nd elif isinstance(pad, (tuple, list)): if max(pad) != 0 and not self.ignore_border: raise NotImplementedError( 'padding works only with ignore_border=True') if isinstance(ws, (tuple, list)): if any(pad[i] >= ws[i] for i in range(nd)): raise NotImplementedError( 'padding must be smaller than strides') ws = tensor.as_tensor_variable(ws) stride = tensor.as_tensor_variable(stride) pad = tensor.as_tensor_variable(pad) assert ws.ndim == 1 assert stride.ndim == 1 assert pad.ndim == 1 if x.type.ndim < nd: raise TypeError() if ws.dtype not in tensor.int_dtypes: raise TypeError('Pool downsample parameters must be ints.') if stride.dtype not in tensor.int_dtypes: raise TypeError('Stride parameters must be ints.') if pad.dtype not in tensor.int_dtypes: raise TypeError('Padding parameters must be ints.') # If the input shape are broadcastable we can have 0 in the output shape broad = x.broadcastable[:-nd] + (False,) * nd out = tensor.TensorType(x.dtype, broad) return gof.Apply(self, [x, ws, stride, pad], [out()]) def perform(self, node, inp, out, params): x, ws, stride, pad = inp z, = out nd = self.ndim assert ws.shape == stride.shape == pad.shape == (nd,) if len(x.shape) < nd: raise NotImplementedError( 'Pool requires input with {} or more dimensions'.format(nd)) z_shape = self.out_shape(x.shape, ws, params.ignore_border, stride, pad, nd) if not params.ignore_border: assert all(z > 0 for z in z_shape[-nd:]) if (z[0] is None) or (z[0].shape != z_shape): z[0] = np.empty(z_shape, dtype=x.dtype) zz = z[0] # size of pooling output pool_out_shp = zz.shape[-nd:] img_shp = tuple(x.shape[-nd + i] + 2 * pad[i] for i in xrange(nd)) inc_pad = self.mode == 'average_inc_pad' # pad the image if max(pad) != 0: y = np.zeros(x.shape[:-nd] + img_shp, dtype=x.dtype) y[(slice(None),) * (len(x.shape) - nd) + tuple(slice(pad[i], img_shp[i] - pad[i]) for i in xrange(nd))] = x else: y = x func = np.max if self.mode == 'sum': func = np.sum elif self.mode != 'max': func = np.average # precompute the region boundaries for each dimension region_slices = [[] for i in xrange(nd)] for i in xrange(nd): for j in xrange(pool_out_shp[i]): start = j * stride[i] end = builtins.min(start + ws[i], img_shp[i]) if not inc_pad: start = builtins.max(start, pad[i]) end = builtins.min(end, img_shp[i] - pad[i]) region_slices[i].append(slice(start, end)) # iterate over non-pooling dimensions for k in np.ndindex(*x.shape[:-nd]): zzk = zz[k] yk = y[k] # iterate over pooling regions for r in np.ndindex(*pool_out_shp): zzk[r] = func( yk[[region_slices[i][r[i]] for i in xrange(nd)]]) def infer_shape(self, node, in_shapes): ws, stride, pad = [node.inputs[1], node.inputs[2], node.inputs[3]] shp = self.out_shape(in_shapes[0], ws, self.ignore_border, stride, pad, self.ndim) return [shp] def L_op(self, inputs, outputs, grads): x, ws, stride, pad = inputs gz, = grads disc = [DisconnectedType()() for i in inputs[1:]] if self.mode == 'max': return [MaxPoolGrad(ndim=self.ndim, ignore_border=self.ignore_border)( x, outputs[0], gz, ws=ws, stride=stride, pad=pad)] + disc else: return [AveragePoolGrad(ndim=self.ndim, ignore_border=self.ignore_border, mode=self.mode)( x, gz, ws=ws, stride=stride, pad=pad)] + disc def connection_pattern(self, node): return [[1], [0], [0], [0]] def R_op(self, inputs, eval_points): if self.mode != 'max': # Rop for average or sum is simply pooling evaluated at eval point eval_inputs = [eval_points[0]] + inputs[1:] return [self(*eval_inputs)] # R_op can receive None as eval_points. # That mean there is no diferientiable path through that input # If this imply that you cannot compute some outputs, # return None for those. if eval_points[0] is None: return [None] z = self(*inputs) x, ws, stride, pad = inputs return [ DownsampleFactorMaxGradGrad(self.ignore_border, self.mode, self.ndim)(x, z, eval_points[0], ws, stride, pad) ] def c_headers(self): headers = [''] headers += super(Pool, self).c_headers() return headers def c_code(self, node, name, inp, out, sub): if self.mode not in ('max', 'sum', 'average_exc_pad', 'average_inc_pad'): raise theano.gof.utils.MethodNotDefined() x, ws, stride, pad = inp z, = out nd = self.ndim total_ndim = node.inputs[0].ndim non_pool_ndim = total_ndim - nd fail = sub['fail'] params = sub['params'] if self.openmp: # run in parallel over each pooling block omp_parallel = '#pragma omp parallel for private(r_st, r_end, r_idx, i_idx, o_idx, collector) schedule(static)' else: omp_parallel = '' ccode = """ int typenum = PyArray_ObjectType((PyObject*)%(x)s, 0); if(PyArray_NDIM(%(x)s)!=%(total_ndim)s) { PyErr_SetString(PyExc_ValueError, "x must be a %(total_ndim)sD ndarray"); %(fail)s; } if(PyArray_DIM(%(ws)s, 0)!=%(nd)s) { PyErr_SetString(PyExc_ValueError, "ws must be a vector of size %(nd)s"); %(fail)s; } if(PyArray_DIM(%(stride)s, 0)!=%(nd)s) { PyErr_SetString(PyExc_ValueError, "stride must be a vector of size %(nd)s"); %(fail)s; } if(PyArray_DIM(%(pad)s, 0)!=%(nd)s) { PyErr_SetString(PyExc_ValueError, "pad must be a vector of size %(nd)s"); %(fail)s; } npy_intp z[%(nd)s]; // shape of the output npy_intp r[%(nd)s]; // shape of the padded_input npy_intp ws[%(nd)s]; npy_intp st[%(nd)s]; npy_intp pd[%(nd)s]; int nonzero_padding; nonzero_padding = 0; for (int i=0; i<%(nd)s; i++) { ws[i] = *((dtype_%(ws)s*)PyArray_GETPTR1(%(ws)s, i)); st[i] = *((dtype_%(stride)s*)PyArray_GETPTR1(%(stride)s, i)); pd[i] = *((dtype_%(pad)s*)PyArray_GETPTR1(%(pad)s, i)); r[i] = PyArray_DIMS(%(x)s)[%(non_pool_ndim)s + i] + 2 * pd[i]; if (pd[i]>0) nonzero_padding = 1; } if (!%(params)s->ignore_border && nonzero_padding) { PyErr_SetString(PyExc_ValueError, "padding must be zero when ignore border is False"); %(fail)s; } if (%(params)s->ignore_border) { for (int i=0; i<%(nd)s; i++) { // '/' in C is different from '/' in python if (r[i] - ws[i] < 0) { z[i] = 0; } else { z[i] = (r[i] - ws[i]) / st[i] + 1; } } } else { for (int i=0; i<%(nd)s; i++) { // decide how many rows/cols the output has if (st[i] >= ws[i]) { z[i] = (r[i] - 1) / st[i] + 1; } else { z[i] = std::max((npy_intp)0, (r[i] - 1 - ws[i] + st[i]) / st[i]) + 1; } assert(z[i] > 0); } } // memory allocation of z if necessary int mem_nec; mem_nec = 0; if ((!%(z)s) || *PyArray_DIMS(%(z)s)!=%(total_ndim)s) { mem_nec = 1; } if (!mem_nec) { for (int i=0; i<%(non_pool_ndim)s; i++) { if (PyArray_DIMS(%(z)s)[i] != PyArray_DIMS(%(x)s)[i]) { mem_nec = 1; break; } } } if (!mem_nec) { for (int i=0; i<%(nd)s; i++) { if (PyArray_DIMS(%(z)s)[%(non_pool_ndim)s + i] != z[i]) { mem_nec = 1; break; } } } if (mem_nec) { if (%(z)s) Py_XDECREF(%(z)s); npy_intp dims[%(total_ndim)s]; for (int i=0; i<%(non_pool_ndim)s; i++) { dims[i] = PyArray_DIMS(%(x)s)[i]; } for (int i=0; i<%(nd)s; i++) { dims[%(non_pool_ndim)s + i] = z[i]; } //TODO: zeros not necessary %(z)s = (PyArrayObject*) PyArray_ZEROS(%(total_ndim)s, dims, typenum,0); } // initialize temp var for the value in a region dtype_%(x)s collector; npy_intp z_prod; // do not run if any z[i] is zero z_prod = 1; for (int i=0; i<%(nd)s; i++) { z_prod *= z[i]; } if (z_prod) { // will be used to hold start and end index of a region npy_intp r_st[%(nd)s]; npy_intp r_end[%(nd)s]; // index for iterating over the pooling regions npy_intp r_idx[%(nd)s]; // placeholder for PyArray indexing (output) npy_intp o_idx[%(total_ndim)s]; // placeholder for PyArray indexing (input) npy_intp i_idx[%(total_ndim)s]; // loop over non-pooling dimensions npy_intp non_pooling_prod = 1; for (int i=0; i<%(non_pool_ndim)s; i++) { non_pooling_prod *= PyArray_DIMS(%(x)s)[i]; } %(omp_parallel)s // first loop over non-pooling dimensions for (npy_intp t=0; t (r[%(i)s] - pd[%(i)s]) ? r[%(i)s] - pd[%(i)s] : r_end[%(i)s]; // from padded_img space to img space r_st[%(i)s] -= pd[%(i)s]; r_end[%(i)s] -= pd[%(i)s]; // handle the case where no padding, ignore border is True if (%(params)s->ignore_border) { r_end[%(i)s] = r_end[%(i)s] > r[%(i)s] ? r[%(i)s] : r_end[%(i)s]; } // use the index to find the correct position in the output o_idx[%(non_pool_ndim)s + %(i)s] = r_idx[%(i)s]; """ % dict(i=i, non_pool_ndim=non_pool_ndim, params=sub['params']) ccode += """ // get a pointer to the correct position in the output dtype_%(z)s * z; if (%(total_ndim)s == 4) z = ((dtype_%(z)s*)(PyArray_GETPTR4(%(z)s, o_idx[0], o_idx[1], o_idx[2], o_idx[3]))); else z = ((dtype_%(z)s*)(PyArray_GetPtr(%(z)s, o_idx))); """ if self.mode == 'max': for i in xrange(nd): ccode += """ // set the first index of dimension %(i)s i_idx[%(non_pool_ndim)s + %(i)s] = r_st[%(i)s]; """ % dict(i=i, non_pool_ndim=non_pool_ndim) ccode += """ // use the first element as the initial value of collector if (%(total_ndim)s == 4) collector = ((dtype_%(x)s*)(PyArray_GETPTR4(%(x)s,i_idx[0],i_idx[1],i_idx[2],i_idx[3])))[0]; else collector = ((dtype_%(x)s*)(PyArray_GetPtr(%(x)s,i_idx)))[0]; """ for i in xrange(nd): ccode += """ // go through the pooled region in the unpadded input for(npy_intp m%(i)s=r_st[%(i)s]; m%(i)s collector) ? a : collector; """ for i in xrange(nd): ccode += """ } // for loop over region """ ccode += """ z[0] = collector; """ elif self.mode in ('sum', 'average_exc_pad', 'average_inc_pad'): ccode += """ // initialize the sum at zero collector = ((dtype_%(x)s)(0)); """ for i in xrange(nd): ccode += """ // go through the pooled region in the unpadded input for(npy_intp m%(i)s=r_st[%(i)s]; m%(i)s= downsample: return (v - 1) // stride + 1 else: return max(0, (v - 1 - downsample) // stride + 1) + 1 out_shape = [compute_out(patch_shape[i], ws[i], stride[i]) for i in xrange(ndim)] rval = list(imgshape[:-ndim]) + out_shape return rval def __init__(self, ignore_border, mode='max', ndim=2, openmp=None): self.ndim = ndim self.ignore_border = ignore_border if mode == 'max_deterministic': # It seems max pool grad algo is already deterministic in CPU. mode = 'max' if mode not in ['max', 'sum', 'average_inc_pad', 'average_exc_pad']: raise ValueError( "Pool mode parameter only support 'max', 'sum'," " 'average_inc_pad' and 'average_exc_pad'. Got %s" % mode) self.mode = mode super(PoolGrad, self).__init__(openmp=openmp) def prepare_node(self, node, storage_map, compute_map, impl): if len(node.inputs) < 5: # 5 for AveragePoolGrad, 6 for MaxPoolGrad # Old interface self.ndim = len(node.op.ds) self.mode = node.op.mode ws = theano.tensor.constant(node.op.ds) st = theano.tensor.constant(node.op.st) pad = theano.tensor.constant(node.op.padding) node.inputs.append(ws) node.inputs.append(st) node.inputs.append(pad) if isinstance(ws, theano.Constant): storage_map[ws] = [ws.data] compute_map[ws] = [True] else: storage_map[ws] = [None] compute_map[ws] = [False] if isinstance(st, theano.Constant): storage_map[st] = [st.data] compute_map[st] = [True] else: storage_map[st] = [None] compute_map[st] = [False] if isinstance(pad, theano.Constant): storage_map[pad] = [pad.data] compute_map[pad] = [True] else: storage_map[pad] = [None] compute_map[pad] = [False] def infer_shape(self, node, in_shapes): return [in_shapes[0]] class MaxPoolGrad(PoolGrad): # params_type ignore_border don't change c code def __init__(self, ignore_border, ndim=2, openmp=None): PoolGrad.__init__(self, ignore_border, mode='max', ndim=ndim, openmp=openmp) def make_node(self, x, maxout, gz, ws, stride=None, pad=None): # make_node should only be called by the grad function of # Pool, so these asserts should not fail. x = tensor.as_tensor_variable(x) maxout = tensor.as_tensor_variable(maxout) gz = tensor.as_tensor_variable(gz) nd = self.ndim if stride is None: stride = ws if pad is None: pad = (0,) * nd ws = tensor.as_tensor_variable(ws) stride = tensor.as_tensor_variable(stride) pad = tensor.as_tensor_variable(pad) assert isinstance(x, Variable) and x.ndim >= nd assert isinstance(maxout, Variable) and maxout.ndim >= nd assert isinstance(gz, Variable) and gz.ndim >= nd assert isinstance(ws, Variable) and ws.ndim == 1 assert isinstance(stride, Variable) and stride.ndim == 1 assert isinstance(pad, Variable) and pad.ndim == 1 assert x.ndim == maxout.ndim == gz.ndim >= nd if ws.dtype not in tensor.int_dtypes: raise TypeError('Pool downsample parameters must be ints.') if stride.dtype not in tensor.int_dtypes: raise TypeError('Stride parameters must be ints.') if pad.dtype not in tensor.int_dtypes: raise TypeError('Padding parameters must be ints.') return Apply(self, [x, maxout, gz, ws, stride, pad], [x.type()]) def perform(self, node, inp, out): assert self.mode == 'max' x, maxout, gz, ws, stride, pad = inp gx_stg, = out nd = self.ndim assert ws.shape == stride.shape == pad.shape == (nd,) if len(x.shape) < nd: raise NotImplementedError( 'MaxPoolGrad requires input with {} or more dimensions'.format(nd)) pool_out_shp = maxout.shape[-nd:] img_shp = tuple(x.shape[-nd + i] + 2 * pad[i] for i in xrange(nd)) # pad the image if max(pad) != 0: y = np.zeros(x.shape[:-nd] + img_shp, dtype=x.dtype) y[(slice(None),) * (len(x.shape) - nd) + tuple(slice(pad[i], img_shp[i] - pad[i]) for i in xrange(nd))] = x else: y = x gx = np.zeros_like(y) # precompute the region boundaries for each dimension region_ranges = [[] for i in xrange(nd)] for i in xrange(nd): for j in xrange(pool_out_shp[i]): start = builtins.max(j * stride[i], pad[i]) end = builtins.min(start + ws[i], img_shp[i]) region_ranges[i].append(xrange(start, end)) # iterate over non-pooling dimensions for k in np.ndindex(*x.shape[:-nd]): gxk = gx[k] gzk = gz[k] yk = y[k] maxoutk = maxout[k] # iterate over pooling regions for r in np.ndindex(*pool_out_shp): maxout_value = maxoutk[r] # iterate inside region for c in itertools.product(*[region_ranges[i][r[i]] for i in xrange(nd)]): if maxout_value == yk[c]: gxk[c] += gzk[r] # unpad the image gx = gx[(slice(None),) * (len(x.shape) - nd) + tuple(slice(pad[i], img_shp[i] - pad[i]) for i in xrange(nd))] gx_stg[0] = gx def grad(self, inp, grads): x, maxout, gz, ws, stride, pad = inp ggx, = grads return ([theano.tensor.zeros_like(x), theano.tensor.zeros_like(maxout), DownsampleFactorMaxGradGrad(ndim=self.ndim, ignore_border=self.ignore_border)( x, maxout, ggx, ws, stride, pad)] + [DisconnectedType()() for i in inp[3:]]) def connection_pattern(self, node): return [[1], [1], [1], [0], [0], [0]] def c_code(self, node, name, inp, out, sub): assert self.mode == 'max' x, z, gz, ws, stride, pad = inp gx, = out nd = self.ndim total_ndim = node.inputs[0].ndim non_pool_ndim = total_ndim - nd fail = sub['fail'] if self.openmp: # run in parallel over each pooling block omp_parallel = '#pragma omp parallel for private(r_st, r_end, r_idx, i_idx, o_idx, maximum) schedule(static)' else: omp_parallel = '' ccode = """ // sanity checks int x_typenum = PyArray_ObjectType((PyObject*)%(x)s, 0); int z_typenum = PyArray_ObjectType((PyObject*)%(z)s, 0); int gz_typenum = PyArray_ObjectType((PyObject*)%(gz)s, 0); if ((x_typenum != z_typenum) || (x_typenum != gz_typenum)) { PyErr_SetString(PyExc_ValueError, "input types must all match"); %(fail)s; } if(PyArray_NDIM(%(x)s)!=%(total_ndim)s) { PyErr_SetString(PyExc_ValueError, "x must be a %(total_ndim)sD ndarray"); %(fail)s; } if(PyArray_NDIM(%(z)s)!=%(total_ndim)s) { PyErr_SetString(PyExc_ValueError, "z must be a %(total_ndim)sD ndarray"); %(fail)s; } if(PyArray_NDIM(%(gz)s)!=%(total_ndim)s) { PyErr_SetString(PyExc_ValueError, "gz must be a %(total_ndim)sD ndarray"); %(fail)s; } if(PyArray_DIM(%(ws)s, 0)!=%(nd)s) { PyErr_SetString(PyExc_ValueError, "ws must be a vector of size %(nd)s"); %(fail)s; } if(PyArray_DIM(%(stride)s, 0)!=%(nd)s) { PyErr_SetString(PyExc_ValueError, "stride must be a vector of size %(nd)s"); %(fail)s; } if(PyArray_DIM(%(pad)s, 0)!=%(nd)s) { PyErr_SetString(PyExc_ValueError, "pad must be a vector of size %(nd)s"); %(fail)s; } npy_intp z[%(nd)s]; // shape of the output npy_intp r[%(nd)s]; // shape of the padded_input npy_intp ws[%(nd)s]; npy_intp st[%(nd)s]; npy_intp pd[%(nd)s]; int nonzero_padding; nonzero_padding = 0; for (int i=0; i<%(nd)s; i++) { ws[i] = *((dtype_%(ws)s*)PyArray_GETPTR1(%(ws)s, i)); st[i] = *((dtype_%(stride)s*)PyArray_GETPTR1(%(stride)s, i)); pd[i] = *((dtype_%(pad)s*)PyArray_GETPTR1(%(pad)s, i)); z[i] = PyArray_DIMS(%(z)s)[%(non_pool_ndim)s + i]; r[i] = PyArray_DIMS(%(x)s)[%(non_pool_ndim)s + i] + 2 * pd[i]; if (pd[i]>0) nonzero_padding = 1; } // allocating memory for output, if necessary int mem_nec; mem_nec = 0; if ((!%(gx)s) || !PyArray_ISCONTIGUOUS(%(gx)s) || *PyArray_DIMS(%(gx)s)!=%(total_ndim)s) { mem_nec = 1; } if (!mem_nec) { for (int i=0; i<%(total_ndim)s; i++) { if (PyArray_DIMS(%(gx)s)[i] != PyArray_DIMS(%(x)s)[i]) { mem_nec = 1; break; } } } if (mem_nec) { Py_XDECREF(%(gx)s); %(gx)s = (PyArrayObject*) PyArray_ZEROS(%(total_ndim)s, PyArray_DIMS(%(x)s), x_typenum,0); } else { PyArray_FILLWBYTE(%(gx)s, 0); } dtype_%(z)s maximum; // temp var for maximum value in a region npy_intp z_prod; // do not run if any z[i] is zero z_prod = 1; for (int i=0; i<%(nd)s; i++) { z_prod *= z[i]; } if (z_prod) { // will be used to hold start and end index of a region npy_intp r_st[%(nd)s]; npy_intp r_end[%(nd)s]; // index for iterating over the pooling regions npy_intp r_idx[%(nd)s]; // placeholder for PyArray indexing (output) npy_intp o_idx[%(total_ndim)s]; // placeholder for PyArray indexing (input) npy_intp i_idx[%(total_ndim)s]; // loop over non-pooling dimensions npy_intp non_pooling_prod = 1; for (int i=0; i<%(non_pool_ndim)s; i++) { non_pooling_prod *= PyArray_DIMS(%(x)s)[i]; } %(omp_parallel)s // first loop over non-pooling dimensions for (npy_intp t=0; t (r[%(i)s] - pd[%(i)s]) ? r[%(i)s] - pd[%(i)s] : r_end[%(i)s]; // from padded_img space to img space r_st[%(i)s] -= pd[%(i)s]; r_end[%(i)s] -= pd[%(i)s]; // use the index to find the correct position in the output o_idx[%(non_pool_ndim)s + %(i)s] = r_idx[%(i)s]; """ % dict(i=i, non_pool_ndim=non_pool_ndim) ccode += """ dtype_%(gz)s * gz; if (%(total_ndim)s == 4) { // the maximum value maximum = ((dtype_%(z)s*)(PyArray_GETPTR4(%(z)s,o_idx[0],o_idx[1],o_idx[2],o_idx[3])))[0]; // the gradient corresponding to this maximum value in z gz = ((dtype_%(gz)s*)(PyArray_GETPTR4(%(gz)s, o_idx[0],o_idx[1],o_idx[2],o_idx[3]))); } else { // the maximum value maximum = ((dtype_%(z)s*)(PyArray_GetPtr(%(z)s,o_idx)))[0]; // the gradient corresponding to this maximum value in z gz = ((dtype_%(gz)s*)(PyArray_GetPtr(%(gz)s, o_idx))); } """ for i in xrange(nd): ccode += """ // go through the pooled region in the unpadded input for(npy_intp m%(i)s=r_st[%(i)s]; m%(i)s= nd assert isinstance(gz, Variable) and gz.ndim >= nd assert isinstance(ws, Variable) and ws.ndim == 1 assert isinstance(stride, Variable) and stride.ndim == 1 assert x.ndim == gz.ndim >= nd assert isinstance(pad, Variable) and pad.ndim == 1 if ws.dtype not in tensor.int_dtypes: raise TypeError('Pool downsample parameters must be ints.') if stride.dtype not in tensor.int_dtypes: raise TypeError('Stride parameters must be ints.') if pad.dtype not in tensor.int_dtypes: raise TypeError('Padding parameters must be ints.') return Apply(self, [x, gz, ws, stride, pad], [x.type()]) def perform(self, node, inp, out): x, gz, ws, stride, pad = inp gx_stg, = out nd = self.ndim assert ws.shape == stride.shape == pad.shape == (nd,) if len(x.shape) < nd: raise NotImplementedError( 'AveragePoolGrad requires input with {} or more dimensions'.format(nd)) if self.mode == 'average_exc_pad' and max(pad) != 0: raise NotImplementedError() z_shape = self.out_shape(x.shape, ws, self.ignore_border, stride, pad, nd) if (gx_stg[0] is None) or (gx_stg[0].shape != z_shape): gx_stg[0] = np.empty(z_shape, dtype=x.dtype) zz = gx_stg[0] # size of pooling output pool_out_shp = zz.shape[-nd:] img_shp = tuple(x.shape[-nd + i] + 2 * pad[i] for i in xrange(nd)) inc_pad = self.mode == 'average_inc_pad' sum_mode = self.mode == 'sum' # initialize the padded output gx = np.zeros((x.shape[:-nd] + img_shp), dtype=x.dtype) # precompute the region boundaries and sizes for each dimension region_slices = [[] for i in xrange(nd)] region_sizes = [[] for i in xrange(nd)] for i in xrange(nd): for j in xrange(pool_out_shp[i]): if sum_mode or inc_pad: start = j * stride[i] else: start = builtins.max(j * stride[i], pad[i]) end = builtins.min(start + ws[i], img_shp[i]) region_slices[i].append(slice(start, end)) region_sizes[i].append(end - start) # iterate over non-pooling dimensions region_slice = [None] * nd for k in np.ndindex(*x.shape[:-nd]): gzk = gz[k] gxk = gx[k] # iterate over pooling regions for r in np.ndindex(*pool_out_shp): region_size = 1 for i in xrange(nd): region_slice[i] = region_slices[i][r[i]] region_size *= region_sizes[i][r[i]] if sum_mode: val = gzk[r] else: # divide by region size val = gzk[r] / region_size gxk[region_slice] += val # unpad the image gx = gx[(slice(None),) * (len(x.shape) - nd) + tuple(slice(pad[i], img_shp[i] - pad[i]) for i in xrange(nd))] gx_stg[0] = gx def grad(self, inp, grads): x, gz, ws, stride, pad = inp ggx, = grads return ([theano.tensor.zeros_like(x), Pool(ignore_border=self.ignore_border, ndim=self.ndim, mode=self.mode)(ggx, ws, stride, pad)] + [DisconnectedType()() for i in inp[2:]]) def connection_pattern(self, node): return [[1], [1], [0], [0], [0]] def c_code(self, node, name, inp, out, sub): x, gz, ws, stride, pad = inp gx, = out nd = self.ndim total_ndim = node.inputs[0].ndim non_pool_ndim = total_ndim - nd fail = sub['fail'] inc_pad = int(self.mode == 'average_inc_pad') sum_mode = int(self.mode == 'sum') if self.openmp: # run in parallel over each pooling block omp_parallel = '#pragma omp parallel for private(r_st, r_end, r_pad_width, r_idx, i_idx, o_idx) schedule(static)' else: omp_parallel = '' ccode = """ // sanity checks int x_typenum = PyArray_ObjectType((PyObject*)%(x)s, 0); int gz_typenum = PyArray_ObjectType((PyObject*)%(gz)s, 0); if (x_typenum != gz_typenum) { PyErr_SetString(PyExc_ValueError, "input types must all match"); %(fail)s; } if(PyArray_NDIM(%(x)s)!=%(total_ndim)s) { PyErr_SetString(PyExc_ValueError, "x must be a %(total_ndim)sD ndarray"); %(fail)s; } if(PyArray_NDIM(%(gz)s)!=%(total_ndim)s) { PyErr_SetString(PyExc_ValueError, "gz must be a %(total_ndim)sD ndarray"); %(fail)s; } if(PyArray_DIM(%(ws)s, 0)!=%(nd)s) { PyErr_SetString(PyExc_ValueError, "ws must be a vector of size %(nd)s"); %(fail)s; } if(PyArray_DIM(%(stride)s, 0)!=%(nd)s) { PyErr_SetString(PyExc_ValueError, "stride must be a vector of size %(nd)s"); %(fail)s; } if(PyArray_DIM(%(pad)s, 0)!=%(nd)s) { PyErr_SetString(PyExc_ValueError, "pad must be a vector of size %(nd)s"); %(fail)s; } npy_intp z[%(nd)s]; // shape of the output npy_intp r[%(nd)s]; // shape of the padded_input npy_intp ws[%(nd)s]; npy_intp st[%(nd)s]; npy_intp pd[%(nd)s]; int nonzero_padding; nonzero_padding = 0; for (int i=0; i<%(nd)s; i++) { ws[i] = *((dtype_%(ws)s*)PyArray_GETPTR1(%(ws)s, i)); st[i] = *((dtype_%(stride)s*)PyArray_GETPTR1(%(stride)s, i)); pd[i] = *((dtype_%(pad)s*)PyArray_GETPTR1(%(pad)s, i)); z[i] = PyArray_DIMS(%(gz)s)[%(non_pool_ndim)s + i]; r[i] = PyArray_DIMS(%(x)s)[%(non_pool_ndim)s + i] + 2 * pd[i]; if (pd[i]>0) nonzero_padding = 1; } if (!%(inc_pad)s && !%(sum_mode)s && nonzero_padding) { PyErr_SetString(PyExc_ValueError, "padding must be zero for average_exc_pad"); %(fail)s; } // allocating memory for output, if necessary int mem_nec; mem_nec = 0; if ((!%(gx)s) || !PyArray_ISCONTIGUOUS(%(gx)s) || *PyArray_DIMS(%(gx)s)!=%(total_ndim)s) { mem_nec = 1; } if (!mem_nec) { for (int i=0; i<%(total_ndim)s; i++) { if (PyArray_DIMS(%(gx)s)[i] != PyArray_DIMS(%(x)s)[i]) { mem_nec = 1; break; } } } if (mem_nec) { Py_XDECREF(%(gx)s); %(gx)s = (PyArrayObject*) PyArray_ZEROS(%(total_ndim)s, PyArray_DIMS(%(x)s), x_typenum,0); } else { PyArray_FILLWBYTE(%(gx)s, 0); } npy_intp z_prod; // do not run if any z[i] is zero z_prod = 1; for (int i=0; i<%(nd)s; i++) { z_prod *= z[i]; } if (z_prod) { // will be used to hold start and end index of a region npy_intp r_st[%(nd)s]; npy_intp r_end[%(nd)s]; // padded region size npy_intp r_pad_width[%(nd)s]; // index for iterating over the pooling regions npy_intp r_idx[%(nd)s]; // placeholder for PyArray indexing (output) npy_intp o_idx[%(total_ndim)s]; // placeholder for PyArray indexing (input) npy_intp i_idx[%(total_ndim)s]; // loop over non-pooling dimensions npy_intp non_pooling_prod = 1; for (int i=0; i<%(non_pool_ndim)s; i++) { non_pooling_prod *= PyArray_DIMS(%(x)s)[i]; } %(omp_parallel)s // first loop over non-pooling dimensions for (npy_intp t=0; t r[%(i)s] ? r[%(i)s] : r_end[%(i)s]; r_pad_width[%(i)s] = r_end[%(i)s] - r_st[%(i)s]; // from padded_img space to img space r_st[%(i)s] = r_st[%(i)s] - pd[%(i)s] > 0 ? r_st[%(i)s] - pd[%(i)s] : 0; r_end[%(i)s] = r_end[%(i)s] > r[%(i)s] - pd[%(i)s] ? r[%(i)s] - 2 * pd[%(i)s] : r_end[%(i)s] - pd[%(i)s]; // use the index to find the correct position in the output o_idx[%(non_pool_ndim)s + %(i)s] = r_idx[%(i)s]; """ % dict(i=i, sum_mode=sum_mode, inc_pad=inc_pad, non_pool_ndim=non_pool_ndim) ccode += """ dtype_%(gz)s * gz; dtype_%(gz)s val; if (%(total_ndim)s == 4) { // the gradient for this region gz = ((dtype_%(gz)s*)(PyArray_GETPTR4(%(gz)s, o_idx[0],o_idx[1],o_idx[2],o_idx[3]))); } else { // the gradient for this region gz = ((dtype_%(gz)s*)(PyArray_GetPtr(%(gz)s, o_idx))); } // compute the contribution if (%(sum_mode)s) { val = gz[0]; } else { val = gz[0] / (%(region_size)s); } """ region_size = ' * '.join('r_pad_width[%d]' % i for i in xrange(nd)) for i in xrange(nd): ccode += """ // go through the pooled region in the unpadded input for(npy_intp m%(i)s=r_st[%(i)s]; m%(i)s= ws[i] for i in range(nd)): raise NotImplementedError( 'padding must be smaller than strides') ws = tensor.as_tensor_variable(ws) stride = tensor.as_tensor_variable(stride) pad = tensor.as_tensor_variable(pad) assert ws.ndim == 1 assert stride.ndim == 1 assert pad.ndim == 1 assert x.ndim == maxout.ndim == gz.ndim >= nd if ws.dtype not in tensor.int_dtypes: raise TypeError('Pool downsample parameters must be ints.') if stride.dtype not in tensor.int_dtypes: raise TypeError('Stride parameters must be ints.') if pad.dtype not in tensor.int_dtypes: raise TypeError('Padding parameters must be ints.') return Apply(self, [x, maxout, gz, ws, stride, pad], [x.type()]) def perform(self, node, inp, out): x, maxout, ggx, ws, stride, pad = inp z, = out nd = self.ndim assert ws.shape == stride.shape == pad.shape == (nd,) if len(x.shape) < nd: raise NotImplementedError( 'DownsampleFactorMaxGradGrad requires input ' 'with {} or more dimensions'.format(nd)) if (z[0] is None) or (z[0].shape != maxout.shape): z[0] = np.zeros(maxout.shape, dtype=x.dtype) ggz = z[0] # grad wrt maxout_grad has the same shape as maxout # size of pooling output pool_out_shp = ggz.shape[-nd:] img_shp = tuple(x.shape[-nd + i] + 2 * pad[i] for i in xrange(nd)) # pad the image and its gradients if max(pad) > 0: y_padded = np.zeros(x.shape[:-nd] + img_shp, dtype=x.dtype) y_padded[(slice(None),) * (len(x.shape) - nd) + tuple(slice(pad[i], img_shp[i] - pad[i]) for i in xrange(nd))] = x ggx_padded = np.zeros(x.shape[:-nd] + img_shp, dtype=x.dtype) ggx_padded[(slice(None),) * (len(x.shape) - nd) + tuple(slice(pad[i], img_shp[i] - pad[i]) for i in xrange(nd))] = ggx else: y_padded = x ggx_padded = ggx # precompute the region boundaries for each dimension region_ranges = [[] for i in xrange(nd)] for i in xrange(nd): for j in xrange(pool_out_shp[i]): start = j * stride[i] end = builtins.min(start + ws[i], img_shp[i]) region_ranges[i].append(xrange(start, end)) # iterate over non-pooling dimensions for k in np.ndindex(*x.shape[:-nd]): ggxk = ggx_padded[k] ggzk = ggz[k] yk = y_padded[k] maxoutk = maxout[k] # iterate over pooling regions for r in np.ndindex(*pool_out_shp): # iterate inside region maxout_value = maxoutk[r] for c in itertools.product(*[region_ranges[i][r[i]] for i in xrange(nd)]): if maxout_value == yk[c]: ggzk[r] += ggxk[c] def infer_shape(self, node, in_shapes): return [in_shapes[1]] def grad(self, inp, grads): x, maxout, ggx, ws, stride, pad = inp gz, = grads return [theano.tensor.zeros_like(x), theano.tensor.zeros_like(maxout), MaxPoolGrad(ignore_border=self.ignore_border, ndim=self.ndim)(x, maxout, gz, ws, stride, pad), DisconnectedType()(), DisconnectedType()(), DisconnectedType()()] def connection_pattern(self, node): return [[1], [1], [1], [0], [0], [0]] def c_code(self, node, name, inp, out, sub): if self.mode != 'max': raise theano.gof.utils.MethodNotDefined() x, maxout, ggx, ws, stride, pad = inp z, = out # the grad of grad nd = self.ndim total_ndim = node.inputs[0].ndim non_pool_ndim = total_ndim - nd fail = sub['fail'] if self.openmp: # run in parallel over each pooling block omp_parallel = '#pragma omp parallel for private(r_st, r_end, r_idx, i_idx, o_idx, maximum) schedule(static)' else: omp_parallel = '' ccode = """ int z_typenum = PyArray_ObjectType((PyObject*)%(maxout)s, 0); npy_intp z[%(nd)s]; // shape of the output npy_intp r[%(nd)s]; // shape of the padded_input npy_intp ws[%(nd)s]; npy_intp st[%(nd)s]; npy_intp pd[%(nd)s]; if(PyArray_DIM(%(ws)s, 0)!=%(nd)s) { PyErr_SetString(PyExc_ValueError, "ws must be a vector of size %(nd)s"); %(fail)s; } if(PyArray_DIM(%(stride)s, 0)!=%(nd)s) { PyErr_SetString(PyExc_ValueError, "stride must be a vector of size %(nd)s"); %(fail)s; } if(PyArray_DIM(%(pad)s, 0)!=%(nd)s) { PyErr_SetString(PyExc_ValueError, "pad must be a vector of size %(nd)s"); %(fail)s; } for (int i=0; i<%(nd)s; i++) { ws[i] = *((dtype_%(ws)s*)PyArray_GETPTR1(%(ws)s, i)); st[i] = *((dtype_%(stride)s*)PyArray_GETPTR1(%(stride)s, i)); pd[i] = *((dtype_%(pad)s*)PyArray_GETPTR1(%(pad)s, i)); z[i] = PyArray_DIMS(%(maxout)s)[%(non_pool_ndim)s + i]; r[i] = PyArray_DIMS(%(x)s)[%(non_pool_ndim)s + i] + 2 * pd[i]; } // allocating memory for output, if necessary int mem_nec; mem_nec = 0; if ((!%(z)s) || !PyArray_ISCONTIGUOUS(%(z)s) || *PyArray_DIMS(%(z)s)!=%(total_ndim)s) { mem_nec = 1; } if (!mem_nec) { for (int i=0; i<%(total_ndim)s; i++) { if (PyArray_DIMS(%(z)s)[i] != PyArray_DIMS(%(maxout)s)[i]) { mem_nec = 1; break; } } } if (mem_nec) { Py_XDECREF(%(z)s); %(z)s = (PyArrayObject*) PyArray_ZEROS(%(total_ndim)s, PyArray_DIMS(%(maxout)s), z_typenum,0); } else { PyArray_FILLWBYTE(%(z)s, 0); } dtype_%(maxout)s maximum; // temp var for maximum value in a region // will be used to hold start and end index of a region npy_intp r_st[%(nd)s]; npy_intp r_end[%(nd)s]; // index for iterating over the pooling regions npy_intp r_idx[%(nd)s]; // placeholder for PyArray indexing (output) npy_intp o_idx[%(total_ndim)s]; // placeholder for PyArray indexing (input) npy_intp i_idx[%(total_ndim)s]; // loop over non-pooling dimensions npy_intp non_pooling_prod; non_pooling_prod = 1; for (int i=0; i<%(non_pool_ndim)s; i++) { non_pooling_prod *= PyArray_DIMS(%(x)s)[i]; } %(omp_parallel)s // first loop over non-pooling dimensions for (npy_intp t=0; t (r[%(i)s] - pd[%(i)s]) ? r[%(i)s] - pd[%(i)s] : r_end[%(i)s]; // from padded_img space to img space r_st[%(i)s] -= pd[%(i)s]; r_end[%(i)s] -= pd[%(i)s]; // use the index to find the correct position in the output o_idx[%(non_pool_ndim)s + %(i)s] = r_idx[%(i)s]; """ % dict(i=i, non_pool_ndim=non_pool_ndim) ccode += """ dtype_%(z)s * z; if (%(total_ndim)s == 4) { // the maximum value maximum = ((dtype_%(maxout)s*)(PyArray_GETPTR4(%(maxout)s,o_idx[0],o_idx[1],o_idx[2],o_idx[3])))[0]; // z at this position z = ((dtype_%(z)s*)(PyArray_GETPTR4(%(z)s,o_idx[0],o_idx[1],o_idx[2],o_idx[3]))); } else { // the maximum value maximum = ((dtype_%(maxout)s*)(PyArray_GetPtr(%(maxout)s,o_idx)))[0]; // z at this position z = ((dtype_%(z)s*)(PyArray_GetPtr(%(z)s,o_idx))); } """ for i in xrange(nd): ccode += """ // go through the pooled region in the unpadded input for(npy_intp m%(i)s=r_st[%(i)s]; m%(i)s= ws[i] for i in range(nd)): raise NotImplementedError( 'padding must be smaller than strides') ws = tensor.as_tensor_variable(ws) stride = tensor.as_tensor_variable(stride) pad = tensor.as_tensor_variable(pad) assert ws.ndim == 1 assert stride.ndim == 1 assert pad.ndim == 1 if x.type.ndim < nd: raise TypeError() if not ws.dtype.startswith('int'): raise TypeError('Pool downsample parameters must be ints.') if not stride.dtype.startswith('int'): raise TypeError('Stride parameters must be ints.') if not pad.dtype.startswith('int'): raise TypeError('Padding parameters must be ints.') # If the input shape are broadcastable we can have 0 in the output shape broad = x.broadcastable[:-nd] + (False,) * nd out = tensor.TensorType(eval_point.dtype, broad) return gof.Apply(self, [x, eval_point, ws, stride, pad], [out()]) def perform(self, node, inp, out, params): x, ex, ws, stride, pad = inp z, = out nd = self.ndim assert ws.shape == stride.shape == pad.shape == (nd,) if len(x.shape) < nd: raise NotImplementedError( 'Pool requires input with {} or more dimensions'.format(nd)) z_shape = Pool.out_shape(x.shape, ws, params.ignore_border, stride, pad, nd) if not self.ignore_border: assert all(z > 0 for z in z_shape[-nd:]) if (z[0] is None) or (z[0].shape != z_shape): z[0] = np.empty(z_shape, dtype=x.dtype) zz = z[0] # size of pooling output pool_out_shp = zz.shape[-nd:] img_shp = tuple(x.shape[-nd + i] + 2 * pad[i] for i in xrange(nd)) inc_pad = self.mode == 'average_inc_pad' # pad the image and the eval point if max(pad) != 0: y = np.zeros(x.shape[:-nd] + img_shp, dtype=x.dtype) y[(slice(None),) * (len(x.shape) - nd) + tuple(slice(pad[i], img_shp[i] - pad[i]) for i in xrange(nd))] = x ey = np.zeros(ex.shape[:-nd] + img_shp, dtype=ex.dtype) ey[(slice(None),) * (len(ex.shape) - nd) + tuple(slice(pad[i], img_shp[i] - pad[i]) for i in xrange(nd))] = ex else: y = x ey = ex # precompute the region boundaries for each dimension region_slices = [[] for i in xrange(nd)] for i in xrange(nd): for j in xrange(pool_out_shp[i]): start = j * stride[i] end = builtins.min(start + ws[i], img_shp[i]) if not inc_pad: start = builtins.max(start, pad[i]) end = builtins.min(end, img_shp[i] - pad[i]) region_slices[i].append(slice(start, end)) # iterate over non-pooling dimensions for k in np.ndindex(*x.shape[:-nd]): zzk = zz[k] yk = y[k] eyk = ey[k] # iterate over pooling regions for r in np.ndindex(*pool_out_shp): # current slice in padded input ykslice = yk[[region_slices[i][r[i]] for i in xrange(nd)]] # current slice in eval points eykslice = eyk[[region_slices[i][r[i]] for i in xrange(nd)]] # indices of maximum idx = np.unravel_index(np.argmax(ykslice), ykslice.shape) zzk[r] = eykslice[idx] def c_headers(self): headers = [''] headers += super(MaxPoolRop, self).c_headers() return headers def c_code(self, node, name, inp, out, sub): if self.mode != 'max': raise theano.gof.utils.MethodNotDefined() x, ex, ws, stride, pad = inp z, = out nd = self.ndim total_ndim = node.inputs[0].ndim non_pool_ndim = total_ndim - nd fail = sub['fail'] params = sub['params'] if self.openmp: # run in parallel over each pooling block omp_parallel = '#pragma omp parallel for private(r_st, r_end, r_idx, i_idx, o_idx, collector, eval_collector) schedule(static)' else: omp_parallel = '' ccode = """ int typenum = PyArray_ObjectType((PyObject*)%(x)s, 0); if(PyArray_NDIM(%(x)s)!=%(total_ndim)s) { PyErr_SetString(PyExc_ValueError, "x must be a %(total_ndim)sD ndarray"); %(fail)s; } if(PyArray_NDIM(%(ex)s)!=%(total_ndim)s) { PyErr_SetString(PyExc_ValueError, "eval_point must be a %(total_ndim)sD ndarray"); %(fail)s; } if(PyArray_DIM(%(ws)s, 0)!=%(nd)s) { PyErr_SetString(PyExc_ValueError, "ws must be a vector of size %(nd)s"); %(fail)s; } if(PyArray_DIM(%(stride)s, 0)!=%(nd)s) { PyErr_SetString(PyExc_ValueError, "stride must be a vector of size %(nd)s"); %(fail)s; } if(PyArray_DIM(%(pad)s, 0)!=%(nd)s) { PyErr_SetString(PyExc_ValueError, "pad must be a vector of size %(nd)s"); %(fail)s; } npy_intp z[%(nd)s]; // shape of the output npy_intp r[%(nd)s]; // shape of the padded_input npy_intp ws[%(nd)s]; npy_intp st[%(nd)s]; npy_intp pd[%(nd)s]; int nonzero_padding; nonzero_padding = 0; for (int i=0; i<%(nd)s; i++) { ws[i] = *((dtype_%(ws)s*)PyArray_GETPTR1(%(ws)s, i)); st[i] = *((dtype_%(stride)s*)PyArray_GETPTR1(%(stride)s, i)); pd[i] = *((dtype_%(pad)s*)PyArray_GETPTR1(%(pad)s, i)); r[i] = PyArray_DIMS(%(x)s)[%(non_pool_ndim)s + i] + 2 * pd[i]; if (pd[i]>0) nonzero_padding = 1; } if (!%(params)s->ignore_border && nonzero_padding) { PyErr_SetString(PyExc_ValueError, "padding must be zero when ignore border is False"); %(fail)s; } if (%(params)s->ignore_border) { for (int i=0; i<%(nd)s; i++) { // '/' in C is different from '/' in python if (r[i] - ws[i] < 0) { z[i] = 0; } else { z[i] = (r[i] - ws[i]) / st[i] + 1; } } } else { for (int i=0; i<%(nd)s; i++) { // decide how many rows/cols the output has if (st[i] >= ws[i]) { z[i] = (r[i] - 1) / st[i] + 1; } else { z[i] = std::max((npy_intp)0, (r[i] - 1 - ws[i] + st[i]) / st[i]) + 1; } assert(z[i] > 0); } } // memory allocation of z if necessary int mem_nec; mem_nec = 0; if ((!%(z)s) || *PyArray_DIMS(%(z)s)!=%(total_ndim)s) { mem_nec = 1; } if (!mem_nec) { for (int i=0; i<%(non_pool_ndim)s; i++) { if (PyArray_DIMS(%(z)s)[i] != PyArray_DIMS(%(x)s)[i]) { mem_nec = 1; break; } } } if (!mem_nec) { for (int i=0; i<%(nd)s; i++) { if (PyArray_DIMS(%(z)s)[%(non_pool_ndim)s + i] != z[i]) { mem_nec = 1; break; } } } if (mem_nec) { if (%(z)s) Py_XDECREF(%(z)s); npy_intp dims[%(total_ndim)s]; for (int i=0; i<%(non_pool_ndim)s; i++) { dims[i] = PyArray_DIMS(%(x)s)[i]; } for (int i=0; i<%(nd)s; i++) { dims[%(non_pool_ndim)s + i] = z[i]; } //TODO: zeros not necessary %(z)s = (PyArrayObject*) PyArray_ZEROS(%(total_ndim)s, dims, typenum,0); } // initialize temp var for the value in a region dtype_%(x)s collector; dtype_%(ex)s eval_collector; npy_intp z_prod; // do not run if any z[i] is zero z_prod = 1; for (int i=0; i<%(nd)s; i++) { z_prod *= z[i]; } if (z_prod) { // will be used to hold start and end index of a region npy_intp r_st[%(nd)s]; npy_intp r_end[%(nd)s]; // index for iterating over the pooling regions npy_intp r_idx[%(nd)s]; // placeholder for PyArray indexing (output) npy_intp o_idx[%(total_ndim)s]; // placeholder for PyArray indexing (input) npy_intp i_idx[%(total_ndim)s]; // loop over non-pooling dimensions npy_intp non_pooling_prod = 1; for (int i=0; i<%(non_pool_ndim)s; i++) { non_pooling_prod *= PyArray_DIMS(%(x)s)[i]; } %(omp_parallel)s // first loop over non-pooling dimensions for (npy_intp t=0; t (r[%(i)s] - pd[%(i)s]) ? r[%(i)s] - pd[%(i)s] : r_end[%(i)s]; // from padded_img space to img space r_st[%(i)s] -= pd[%(i)s]; r_end[%(i)s] -= pd[%(i)s]; // handle the case where no padding, ignore border is True if (%(params)s->ignore_border) { r_end[%(i)s] = r_end[%(i)s] > r[%(i)s] ? r[%(i)s] : r_end[%(i)s]; } // use the index to find the correct position in the output o_idx[%(non_pool_ndim)s + %(i)s] = r_idx[%(i)s]; """ % dict(i=i, params=sub['params'], non_pool_ndim=non_pool_ndim) ccode += """ // get a pointer to the correct position in the output dtype_%(z)s * z; if (%(total_ndim)s == 4) z = ((dtype_%(z)s*)(PyArray_GETPTR4(%(z)s, o_idx[0], o_idx[1], o_idx[2], o_idx[3]))); else z = ((dtype_%(z)s*)(PyArray_GetPtr(%(z)s, o_idx))); """ for i in xrange(nd): ccode += """ // set the first index of dimension %(i)s i_idx[%(non_pool_ndim)s + %(i)s] = r_st[%(i)s]; """ % dict(i=i, non_pool_ndim=non_pool_ndim) ccode += """ // use the first element as the initial value of collector if (%(total_ndim)s == 4) { collector = ((dtype_%(x)s*)(PyArray_GETPTR4(%(x)s,i_idx[0],i_idx[1],i_idx[2],i_idx[3])))[0]; eval_collector = ((dtype_%(ex)s*)(PyArray_GETPTR4(%(ex)s,i_idx[0],i_idx[1],i_idx[2],i_idx[3])))[0]; } else { collector = ((dtype_%(x)s*)(PyArray_GetPtr(%(x)s,i_idx)))[0]; eval_collector = ((dtype_%(ex)s*)(PyArray_GetPtr(%(ex)s,i_idx)))[0]; } """ for i in xrange(nd): ccode += """ // go through the pooled region in the unpadded input for(npy_intp m%(i)s=r_st[%(i)s]; m%(i)s collector) { collector = a; eval_collector = ea; } """ for i in xrange(nd): ccode += """ } // for loop over region """ ccode += """ z[0] = eval_collector; """ for i in xrange(nd): ccode += """ } // loop over pooling dimension """ ccode += """ } // for loop over non-pooling dimensions } // if z_prod """ return ccode % locals() def c_code_cache_version(self): return (2, self.openmp) Theano-1.0.0/theano/tensor/signal/tests/00007500163562€ÿÿÿþ0000000000013203336030021473 5ustar boccoset00000000000000Theano-1.0.0/theano/tensor/signal/tests/old_pool_interface.pkl00006000163562€ÿÿÿþ0000007611113203335532026043 0ustar boccoset00000000000000€ctheano.compile.function_module _constructor_Function qctheano.compile.function_module _constructor_FunctionMaker q}q(UprofileqNUfunction_builderqctheano.compile.function_module Function qUinputsq]qctheano.compile.io In q )q }q (Unameq NUupdateq NUvalueqNUborrowqNUstrictq‰Uimplicitq‰Umutableq‰Uvariableqctheano.tensor.var TensorVariable q)q}q(U auto_nameqUauto_3qUindexqNUtagq(ctheano.gof.utils scratchpad qoq}qUtraceq]q]q (Ucreate_pickle_old_interface.pyq!KUq"Ux = theano.tensor.ftensor4()tq#aasbh NUownerq$NUtypeq%ctheano.tensor.type TensorType q&)q'}q((U broadcastableq)(‰‰‰‰tUdtypeq*Ufloat32q+U numpy_dtypeq,cnumpy dtype q-Uf4KK‡Rq.(KUh7ah2h4Uoutputsq?]q@h)qA}qB(hKh Nh2h4h9]qC(h;)qD}qE(h]qF(h7hAh)qG}qH(hKh Nh2h4h9]qI(h;)qJ}qK(h]qL(h7hGeh2h4h?]qMh)qN}qO(hKh Nh2h4h9]qPUoutputqQK†qRahUauto_67qSh(hoqT}qUh]qV]qW((h!K h"U(dz_dx = theano.gradient.grad(z.sum(), x)tqX(U\/u/laurent/.local/lib/python2.7/site-packages/Theano-0.8.0.dev0-py2.7.egg/theano/gradient.pyqYM1UgradqZUgrad_dict, wrt, cost_name)tq[(U\/u/laurent/.local/lib/python2.7/site-packages/Theano-0.8.0.dev0-py2.7.egg/theano/gradient.pyq\M,U_populate_grad_dictq]U0rval = [access_grad_cache(elem) for elem in wrt]tq^(U\/u/laurent/.local/lib/python2.7/site-packages/Theano-0.8.0.dev0-py2.7.egg/theano/gradient.pyq_MÿUaccess_grad_cacheq`U#term = access_term_cache(node)[idx]tqa(U\/u/laurent/.local/lib/python2.7/site-packages/Theano-0.8.0.dev0-py2.7.egg/theano/gradient.pyqbMYUaccess_term_cacheqcU4input_grads = node.op.grad(inputs, new_output_grads)tqdeasbh$hJh%h'ubah(ho}qebUdepsqf}Uopqgctheano.tensor.signal.pool AveragePoolGrad qh)qi}qj(UstqkKK†qlUpaddingqmKK†qnUmodeqoUaverage_exc_padqpU ignore_borderqqˆU_op_use_c_codeqrU /usr/bin/g++qsUdsqthlububK†quhDK†qvehUauto_793qwh(hoqx}qyh]qz]q{((h!K h"U(dz_dx = theano.gradient.grad(z.sum(), x)tq|(hYM1hZUgrad_dict, wrt, cost_name)tq}(h\M,h]U0rval = [access_grad_cache(elem) for elem in wrt]tq~(h_Mÿh`U#term = access_term_cache(node)[idx]tq(hbMÍhcU?output_grads = [access_grad_cache(var) for var in node.outputs]tq€(h_Mÿh`U#term = access_term_cache(node)[idx]tq(hbMYhcU4input_grads = node.op.grad(inputs, new_output_grads)tq‚easbh$h;)qƒ}q„(h]q…(ctheano.tensor.var TensorConstant q†)q‡}qˆ(hNh Nh2h4h%h&)q‰}qŠ(h)(ˆˆˆˆth*Ufloat32q‹h,h.h/‰h Nubh9]qŒhƒK†qahUauto_791qŽh(hoq}q(h]q‘]q’((h!K h"U(dy_dx = theano.gradient.grad(y.sum(), x)tq“(hYM1hZUgrad_dict, wrt, cost_name)tq”(h\M,h]U0rval = [access_grad_cache(elem) for elem in wrt]tq•(h_Mÿh`U#term = access_term_cache(node)[idx]tq–(hbMÍhcU?output_grads = [access_grad_cache(var) for var in node.outputs]tq—(h_Mÿh`U#term = access_term_cache(node)[idx]tq˜(hbMYhcU4input_grads = node.op.grad(inputs, new_output_grads)tq™eaU unique_valueqšcnumpy.core.multiarray scalar q›h.U€?†RqœubUdataqcnumpy.core.multiarray _reconstruct qžcnumpy ndarray qŸK…Ub‡Rq (K(KKKKth.‰U€?tbubh)q¡}q¢(hKh Nh2h4h9]q£hƒK†q¤ahUauto_70q¥h(hoq¦}q§h]q¨sbh$h;)q©}qª(h]q«h7ah2h4h?]q¬h¡ah(ho}q­bhf}hgctheano.compile.ops Shape_i q®)q¯}q°(UiKhrhsububh%h&)q±}q²(h))h*Uint64q³h,h-Ui8KK‡Rq´(KU(hbMYhcU4input_grads = node.op.grad(inputs, new_output_grads)tr?easbh$hDh%h'ubah(ho}r@bhf}hgctheano.tensor.signal.pool MaxPoolGrad rA)rB}rC(hkKK†rDhmhnhoUmaxrEhqˆhrhshtjDububK†rFhQK†rGehUauto_56rHh(horI}rJh]rK]rL(h!Kh"U6y = pool_2d(x, (2, 2), mode='max', ignore_border=True)trMaasbh$h(h)r?}r@(hKh Nh2h4h9]rAhUauto_836rBh(horC}rDh]sbh$h;)rE}rF(h]rGj2ah2h4h?]rHj?ah(ho}rIbhf}hgj.ubh%h'ubj½]…RrJe]rK(h)rL}rM(hKh Nh2h4h9]rNhUauto_837rOh(horP}rQh]sbh$h;)rR}rS(h]rThNah2h4h?]rUjLah(ho}rVbhf}hgj.ubh%h'ubj½]…RrWe]rX(h÷j½]rY]rZ(hƒKea…Rr[e]r\(hÄj½]r]]r^(hƒKea…Rr_ee…Rr`Uroot_destroyerra}rb(hÏhÌhçhäuUdebug_all_appsrcj—)rd}re(jšj›)rf]rg(j›)rh]ri(j›)rj]rk(j›)rl]rm(j›)rn]ro(j›)rp]rq(j›)rr]rs(j›)rt]ru(j›)rv]rw(j›)rx]ry(j›)rz]r{(j›)r|]r}(jfjzhïebjxh×ebjvh½ebjth©ebjrh(h)(ˆˆˆˆth*h+h,h.h/‰h Nubub(jÀjÀjÀjÀth)r?}r@(hKh Nh2h4h9]rAhUauto_792rBh(horC}rDh]rE]rF((h!K h"U(dy_dx = theano.gradient.grad(y.sum(), x)trG(hYM1hZUgrad_dict, wrt, cost_name)trH(h\M,h]U0rval = [access_grad_cache(elem) for elem in wrt]trI(h_Mÿh`U#term = access_term_cache(node)[idx]trJ(hbMÍhcU?output_grads = [access_grad_cache(var) for var in node.outputs]trK(h_Mÿh`U#term = access_term_cache(node)[idx]trL(hbMYhcU4input_grads = node.op.grad(inputs, new_output_grads)trMeasbh$h;)rN}rO(h]rP(h‡h¡hµjjêeh2h4h?]rQj?ah(ho}rRbhf}hgj-ubh%h&)rS}rT(h)(‰‰‰‰th*h‹h,h.h/‰h Nubub(h¡hµhÄh÷th)rU}rV(hKh Nh2h4h9]rWhUauto_119rXh(horY}rZh]sbh$h;)r[}r\(h]r](h)r^}r_(hKh Nh2h4h9]r`hUauto_114rah(horb}rch]rdh’asbh$h;)re}rf(h]rgh)rh}ri(hKh Nh2h4h9]rjhUauto_59rkh(horl}rmh]rn]ro(h!K h"U(dy_dx = theano.gradient.grad(y.sum(), x)trpaasbh$h;)rq}rr(h]rshAah2h4h?]rtjhah(ho}rubhf}hgj.)rv}rw(j1j2h*h+hrhsjjßj3Nububh%h&)rx}ry(h))h*h+h,h.h/‰h Nububah2h4h?]rzj^ah(ho}r{bhf}hgj±)r|}r}(j´]r~j¶]rj¸]r€(KKKKejº)j»‰j¼jhrhsububh%h&)r}r‚(h)(ˆˆˆˆth*h+h,h.h/‰h Nububjeh2h4h?]rƒjUah(ho}r„bhf}hgj)r…}r†(j÷jøjctheano.scalar.basic Second r‡)rˆ}r‰(j j )rŠ}r‹j K…sbhrhsh UsecondrŒubh UElemwise{second,no_inplace}rhrhsj}rŽjNj}rj‰jTƒCreate a matrix by filling the shape of a with b Generalizes a scalar op to tensors. All the inputs must have the same number of dimensions. When the Op is performed, for each dimension, each input's size for that dimension must be the same. As a special case, it can also be 1 but only if the input's broadcastable flag is True for that dimension. In that case, the tensor is (virtually) replicated along that dimension to match the size of the others. The dtypes of the outputs mirror those of the scalar Op that is being generalized to tensors. In particular, if the calculations for an output are done inplace on an input, the output type must be the same as the corresponding input type (see the doc of scalar.ScalarOp to get help about controlling the output type) Parameters ---------- scalar_op An instance of a subclass of scalar.ScalarOp which works uniquely on scalars. inplace_pattern A dictionary that maps the index of an output to the index of an input so the output is calculated inplace using the input's storage. (Just like destroymap, but without the lists.) nfunc_spec Either None or a tuple of three elements, (nfunc_name, nin, nout) such that getattr(numpy, nfunc_name) implements this operation, takes nin inputs and nout outputs. Note that nin cannot always be inferred from the scalar op's own nin field because that value is sometimes 0 (meaning a variable number of inputs), whereas the numpy function may not have varargs. Examples -------- Elemwise(add) # represents + on tensors (x + y) Elemwise(add, {0 : 0}) # represents the += operation (x += y) Elemwise(add, {0 : 1}) # represents += on the second argument (y += x) Elemwise(mul)(rand(10, 5), rand(1, 5)) # the second input is completed # along the first dimension to match the first input Elemwise(true_div)(rand(10, 5), rand(10, 1)) # same but along the # second dimension Elemwise(int_div)(rand(1, 5), rand(10, 1)) # the output has size (10, 5) Elemwise(log)(rand(3, 4, 5)) rububh%h&)r‘}r’(h)(ˆˆˆˆth*h+h,h.h/‰h Nubub(jÀjÀjÀjÀth)r“}r”(hKh Nh2h4h9]r•hUauto_797r–h(hor—}r˜h]sbh$h;)r™}rš(h]r›(hÏhÏeh2h4h?]rœj“ah(ho}rbhf}hgj…ubh%h&)rž}rŸ(h))h*h³h,h´h/‰h Nubub)h)r }r¡(hKh Nh2h4h9]r¢hUauto_108r£h(hor¤}r¥h]sbh$h;)r¦}r§(h]r¨(hÏhÞeh2h4h?]r©j ah(ho}rªbhf}hgjõubh%h&)r«}r¬(h))h*h³h,h´h/‰h Nubub)j)jê)j )hç)h)r­}r®(hKh Nh2h4h9]r¯hUauto_125r°h(hor±}r²h]sbh$h;)r³}r´(h]rµ(jjeh2h4h?]r¶j­ah(ho}r·bhf}hgj…ubh%h&)r¸}r¹(h)(ˆˆˆˆth*h+h,h.h/‰h Nubub(jÀjÀjÀjÀth‡(jÀjÀjÀjÀtj¦)h)rº}r»(hKh Nh2h4h9]r¼hUauto_61r½h(hor¾}r¿h]rÀ]rÁ((h!K h"U(dz_dx = theano.gradient.grad(z.sum(), x)trÂ(hYMÙhZUg_cost = _float_ones_like(cost)trÃ(U\/u/laurent/.local/lib/python2.7/site-packages/Theano-0.8.0.dev0-py2.7.egg/theano/gradient.pyrÄMAU_float_ones_likerÅUrval = tensor.ones_like(x)trÆeasbh$h;)rÇ}rÈ(h]rÉ(j j¦eh2h4h?]rÊjºah(ho}rËbhf}hgj…ubh%h&)rÌ}rÍ(h))h*h+h,h.h/‰h Nubub)h)rÎ}rÏ(hKh Nh2h4h9]rÐhUauto_821rÑh(horÒ}rÓh]sbh$h;)rÔ}rÕ(h]rÖ(hçhçeh2h4h?]r×jÎah(ho}rØbhf}hgj…ubh%h&)rÙ}rÚ(h))h*h³h,h´h/‰h Nubub)h)rÛ}rÜ(hKh Nh2h4h9]rÝhUauto_813rÞh(horß}ràh]sbh$h;)rá}râ(h]rã(hçhçeh2h4h?]räjÛah(ho}råbhf}hgj…ubh%h&)ræ}rç(h))h*h³h,h´h/‰h Nubub)h)rè}ré(hKh Nh2h4h9]rêhUauto_62rëh(horì}ríh]rî]rï((h!K h"U(dy_dx = theano.gradient.grad(y.sum(), x)trð(hYMÙhZUg_cost = _float_ones_like(cost)trñ(jÄMAjÅUrval = tensor.ones_like(x)tròeasbh$h;)ró}rô(h]rõ(jhj¦eh2h4h?]röjèah(ho}r÷bhf}hgj…ubh%h&)rø}rù(h))h*h+h,h.h/‰h Nubub)h)rú}rû(hKh Nh2h4h9]rühUauto_93rýh(horþ}rÿh]sbh$h;)r}r(h]r(hçhÞeh2h4h?]rjúah(ho}rbhf}hgjõubh%h&)r}r(h))h*h³h,h´h/‰h Nubub)h÷)h)r}r(hKh Nh2h4h9]r hUauto_825r h(hor }r h]sbh$h;)r }r(h]r(jÑjÎeh2h4h?]rjah(ho}rbhf}hgj…ubh%h&)r}r(h))h*h³h,h´h/‰h Nubub)j^(jÀjÀjÀjÀth†)r}r(hUauto_94rhNh(hor}rhšNsbh Nh%h&)r}r(h))h*Uint8h,hâh/‰h Nubh9]rhhžhŸK…Ub‡Rr(K)hâ‰Utbub)jh)h)r}r(hKh Nh2h4h9]rhUauto_103r h(hor!}r"h]sbh$h;)r#}r$(h]r%(hçh†)r&}r'(hUauto_99r(hNh(hor)}r*hšNsbh Nh%h&)r+}r,(h))h*Uint8h,hâh/‰h Nubh9]r-hhžhŸK…Ub‡Rr.(K)hâ‰Utbubeh2h4h?]r/jah(ho}r0bhf}hgjÜubh%h&)r1}r2(h))h*h³h,h´h/‰h Nubub)h¡)h)r3}r4(hKh Nh2h4h9]r5hUauto_78r6h(hor7}r8h]sbh$h;)r9}r:(h]r;(hÏjòeh2h4h?]r<j3ah(ho}r=bhf}hgjÜubh%h&)r>}r?(h))h*h³h,h´h/‰h Nubub)h)r@}rA(hKh Nh2h4h9]rBhUauto_64rCh(horD}rEh]rFh’asbh$h;)rG}rH(h]rIjèah2h4h?]rJj@ah(ho}rKbhf}hgj±)rL}rM(j´]rNj¶]rOj¸]rP(KKKKejº)j»‰j¼jhrhsububh%h&)rQ}rR(h)(ˆˆˆˆth*h+h,h.h/‰h Nubub(jÀjÀjÀjÀtjÑ)h)rS}rT(hKh Nh2h4h9]rUhUauto_801rVh(horW}rXh]sbh$h;)rY}rZ(h]r[(jj“eh2h4h?]r\jSah(ho}r]bhf}hgj…ubh%h&)r^}r_(h))h*h³h,h´h/‰h Nubub)h)r`}ra(hKh Nh2h4h9]rbhUauto_65rch(hord}reh]rfh{asbh$h;)rg}rh(h]ri(jYh‡eh2h4h?]rjj`ah(ho}rkbhf}hgj)rl}rm(h Njjˆj}rnjNj}roj‰hrhsububh%h&)rp}rq(h)(‰‰‰‰th*h+h,h.h/‰h Nubub(h¡hµhÄh÷th†)rr}rs(hUauto_89rthNh(horu}rvhšNsbh Nh%h&)rw}rx(h))h*Uint8h,hâh/‰h Nubh9]ryhhžhŸK…Ub‡Rrz(K)hâ‰Utbub)jL(h¡hµhÏhçth)r{}r|(hKh Nh2h4h9]r}hUauto_809r~h(hor}r€h]sbh$h;)r}r‚(h]rƒ(jòh)r„}r…(hKh Nh2h4h9]r†hUauto_805r‡h(horˆ}r‰h]sbh$h;)rŠ}r‹(h]rŒ(hÏhÏeh2h4h?]rj„ah(ho}rŽbhf}hgj…ubh%h&)r}r(h))h*h³h,h´h/‰h Nububeh2h4h?]r‘j{ah(ho}r’bhf}hgj…ubh%h&)r“}r”(h))h*h³h,h´h/‰h Nubub)h)r•}r–(hKh Nh2h4h9]r—hUauto_66r˜h(hor™}ršh]r›jFasbh$h;)rœ}r(h]rž(hAh‡eh2h4h?]rŸj•ah(ho}r bhf}hgj)r¡}r¢(h Njjˆj}r£jNj}r¤j‰hrhsububh%h&)r¥}r¦(h)(‰‰‰‰th*h+h,h.h/‰h Nubub(h¡hµhÄh÷tj„)h)r§}r¨(hKh Nh2h4h9]r©hUauto_98rªh(hor«}r¬h]sbh$h;)r­}r®(h]r¯(hÏjeh2h4h?]r°j§ah(ho}r±bhf}hgjÜubh%h&)r²}r³(h))h*h³h,h´h/‰h Nubub)hÄ)hN(h¡hµhÏhçthG(h¡hµhÄh÷thÏ)j&)j2(h¡hµhÏhçth)r´}rµ(hKh Nh2h4h9]r¶hUauto_817r·h(hor¸}r¹h]sbh$h;)rº}r»(h]r¼(j&jÛeh2h4h?]r½j´ah(ho}r¾bhf}hgj…ubh%h&)r¿}rÀ(h))h*h³h,h´h/‰h Nubub)h)rÁ}rÂ(hKh Nh2h4h9]rÃhUauto_63rÄh(horÅ}rÆh]rÇj›asbh$h;)rÈ}rÉ(h]rÊjºah2h4h?]rËjÁah(ho}rÌbhf}hgj±)rÍ}rÎ(j´]rÏj¶]rÐj¸]rÑ(KKKKejº)j»‰j¼j½hrhsububh%h&)rÒ}rÓ(h)(ˆˆˆˆth*h+h,h.h/‰h Nubub(jÀjÀjÀjÀtuU lscalar_onerÔjÀUshape_of_reverse_indexrÕ}rÖ(j”jƒ]…Rr×jÈjƒ]…RrØj?jƒ]…RrÙjéjƒ]…RrÚjjƒ]…RrÛjYjƒ]…RrÜj"jƒ]…RrÝj{jƒ]…RrÞj“jƒ]…RrßhAjƒ]…Rràj?jƒ]…RrájÎjƒ]…Rrâjjƒ]…RrãjÛjƒ]…RräjLjƒ]…Rråj3jƒ]…Rræjrjƒ]…Rrçhçjƒ]rè(j?h7hNjLj2e…Rréj jƒ]…RrêhÏjƒ]rë(j?h7hNjLj2e…Rrìjjƒ]…Rríjûjƒ]…Rrîjêjƒ]…Rrïjújƒ]…Rrðj2jƒ]…Rrñh÷jƒ]rò(hAj`j2j•j?hGjYj"e…Rrójjƒ]…RrôjÁjƒ]…Rrõj­jƒ]…Rröh¡jƒ]r÷(hAj`j?j?j•j2h7hNjLjYj"j2hGe…Rrøj@jƒ]…RrùjSjƒ]…Rrúj`jƒ]…RrûhÄjƒ]rü(hAj`j2j•j?hGjYj"e…RrýjÀjƒ]rþ(j”jjUh‡j^j­jj@jÁe…Rrÿj•jƒ]…Rrj2jƒ]…Rrj§jƒ]…RrhNjƒ]…RrjUjƒ]…Rrj„jƒ]…Rrj´jƒ]…Rrhµjƒ]r(hAj`j?j?j•j2h7hNjLjYj"j2hGe…RruubU apply_nodesr jƒ]r (h©hÌhäh×jUh½hïh(hh)r?}r@(hUauto_43rAhKh(horB}rCh]rD]rE((h!K h"U(dy_dx = theano.gradient.grad(y.sum(), x)trF(hYM1hZUgrad_dict, wrt, cost_name)trG(h\M,h]U0rval = [access_grad_cache(elem) for elem in wrt]trH(h_Mÿh`U#term = access_term_cache(node)[idx]trI(hbMYhcU4input_grads = node.op.grad(inputs, new_output_grads)trJeasbh Nh$h;)rK}rL(h]rMhah(ho}rNbh?]rOj?ahgjRubh%h&)rP}rQ(h)(‰‰‰‰th*h+h,h.h/‰h Nububh)rR}rS(hUauto_47rThKh(horU}rVhj›sbh Nh$h;)rW}rX(h]rY(jh)rZ}r[(hUauto_46r\hKh(hor]}r^hjFsbh Nh$h;)r_}r`(h]rah)rb}rc(hUauto_45rdhKh(hore}rfhjîsbh Nh$h;)rg}rh(h]ri(h)rj}rk(hUauto_44rlhKh(horm}rnhjnsbh Nh$h;)ro}rp(h]rqjah(ho}rrbh?]rsjjahgjvubh%jxubh†)rt}ru(hUauto_7rvhNh(horw}rxhšNsbh NUcachedryˆh%j«hj®ubeh(ho}rzbh?]r{jbahgj…ubh%jøubah(ho}r|bh?]r}jZahgjLubh%jQubeh(ho}r~bh?]rjRahgj¡ubh%j¥ubeh(ho}r€bh?]rj7ahgjBubh%h'ubh‰ubj)r‚}rƒ(hh)r„}r…(hUauto_53r†hKh(hor‡}rˆhhVsbh Nh$h;)r‰}rŠ(h]r‹(hh)rŒ}r(hUauto_52rŽhKh(hor}rhjfsbh Nh$h;)r‘}r’(h]r“(j+h)r”}r•(hUauto_51r–hKh(hor—}r˜hjÇsbh Nh$h;)r™}rš(h]r›h)rœ}r(hUauto_50ržhKh(horŸ}r hjÀsbh Nh$h;)r¡}r¢(h]r£(h)r¤}r¥(hUauto_49r¦hKh(hor§}r¨hj&sbh Nh$h;)r©}rª(h]r«j+ah(ho}r¬bh?]r­j¤ahgj/ubh%j4ubjteh(ho}r®bh?]r¯jœahgj…ubh%jÌubah(ho}r°bh?]r±j”ahgjÍubh%jÒubeh(ho}r²bh?]r³jŒahgjlubh%jpubeh(ho}r´bh?]rµj„ahghiubh%h'ubh‰ubeUon_unused_inputr¶NUaccept_inplacer·‰hoctheano.compile.mode Mode r¸)r¹Ucvmrºctheano.gof.optdb Query r»)r¼}r½(h U OPT_FAST_RUNr¾Urequirer¿j—)rÀ}rÁ(jšj›)rÂ]rÃ(jÂjÂebjž}ubUsubqueryrÄ}Uposition_cutoffrÅNUexcluderÆj—)rÇ}rÈ(jšj›)rÉ]rÊ(jÉjÉebjž}ubUincluderËj—)rÌ}rÍ(jšj›)rÎ]rÏ(j›)rÐ]rÑ(jÎjÎUfast_runrÒebjÐebjž}rÓjÒjÐsubUextra_optimizationsrÔ]rÕub†bu…RrÖ]r×ctheano.gof.link Container rØ)rÙ}rÚ(UprovidedrÛKh NUrequiredr܈UstoragerÝ]rÞNah‰h‰Ureadonlyr߉h%h'h1Nuba]ràNa‡Rrá.Theano-1.0.0/theano/tensor/signal/tests/test_conv.py00006000163562€ÿÿÿþ0000001024213203335532024053 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import unittest from nose.plugins.skip import SkipTest import numpy as np import theano import theano.tensor as T from theano.tests import unittest_tools as utt from theano.tensor.signal import conv from theano.tensor.basic import _allclose class TestSignalConv2D(unittest.TestCase): def setUp(self): utt.seed_rng() def validate(self, image_shape, filter_shape, out_dim, verify_grad=True): image_dim = len(image_shape) filter_dim = len(filter_shape) input = T.TensorType('float64', [False] * image_dim)() filters = T.TensorType('float64', [False] * filter_dim)() bsize = image_shape[0] if image_dim != 3: bsize = 1 nkern = filter_shape[0] if filter_dim != 3: nkern = 1 # THEANO IMPLEMENTATION ############ # we create a symbolic function so that verify_grad can work def sym_conv2d(input, filters): return conv.conv2d(input, filters) output = sym_conv2d(input, filters) assert output.ndim == out_dim theano_conv = theano.function([input, filters], output) # initialize input and compute result image_data = np.random.random(image_shape) filter_data = np.random.random(filter_shape) theano_output = theano_conv(image_data, filter_data) # REFERENCE IMPLEMENTATION ############ out_shape2d = np.array(image_shape[-2:]) - np.array(filter_shape[-2:]) + 1 ref_output = np.zeros(tuple(out_shape2d)) # reshape as 3D input tensors to make life easier image_data3d = image_data.reshape((bsize,) + image_shape[-2:]) filter_data3d = filter_data.reshape((nkern,) + filter_shape[-2:]) # reshape theano output as 4D to make life easier theano_output4d = theano_output.reshape((bsize, nkern,) + theano_output.shape[-2:]) # loop over mini-batches (if required) for b in range(bsize): # loop over filters (if required) for k in range(nkern): image2d = image_data3d[b, :, :] filter2d = filter_data3d[k, :, :] output2d = np.zeros(ref_output.shape) for row in range(ref_output.shape[0]): for col in range(ref_output.shape[1]): output2d[row, col] += ( image2d[row:row + filter2d.shape[0], col:col + filter2d.shape[1]] * filter2d[::-1, ::-1] ).sum() self.assertTrue(_allclose(theano_output4d[b, k, :, :], output2d)) # TEST GRADIENT ############ if verify_grad: utt.verify_grad(sym_conv2d, [image_data, filter_data]) def test_basic(self): # Basic functionality of nnet.conv.ConvOp is already tested by # its own test suite. We just have to test whether or not # signal.conv.conv2d can support inputs and filters of type # matrix or tensor3. if(not theano.tensor.nnet.conv.imported_scipy_signal and theano.config.cxx == ""): raise SkipTest("conv2d tests need SciPy or a c++ compiler") self.validate((1, 4, 5), (2, 2, 3), out_dim=4, verify_grad=True) self.validate((7, 5), (5, 2, 3), out_dim=3, verify_grad=False) self.validate((3, 7, 5), (2, 3), out_dim=3, verify_grad=False) self.validate((7, 5), (2, 3), out_dim=2, verify_grad=False) def test_fail(self): # Test that conv2d fails for dimensions other than 2 or 3. self.assertRaises(Exception, conv.conv2d, T.dtensor4(), T.dtensor3()) self.assertRaises(Exception, conv.conv2d, T.dtensor3(), T.dvector()) def test_bug_josh_reported(self): # Test refers to a bug reported by Josh, when due to a bad merge these # few lines of code failed. See # http://groups.google.com/group/theano-dev/browse_thread/thread/8856e7ca5035eecb m1 = theano.tensor.matrix() m2 = theano.tensor.matrix() conv.conv2d(m1, m2) Theano-1.0.0/theano/tensor/signal/tests/test_pool.py00006000163562€ÿÿÿþ0000014416513203335532024073 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division from nose.plugins.skip import SkipTest from parameterized import parameterized from itertools import product import os import unittest from six import reraise from six.moves import cPickle import six.moves.builtins as builtins import sys import numpy as np import theano import theano.tensor as tensor from theano.tests import unittest_tools as utt from theano.tensor.signal.pool import (Pool, pool_2d, pool_3d, MaxPoolGrad, AveragePoolGrad, max_pool_2d_same_size, DownsampleFactorMaxGradGrad) from theano import function class TestDownsampleFactorMax(utt.InferShapeTester): def test_out_shape(self): assert Pool.out_shape((9, 8, 6), (2, 2)) == [9, 4, 3] assert Pool.out_shape((8, 6), (2, 2)) == [4, 3] @staticmethod def numpy_max_pool_2d(input, ws, ignore_border=False, mode='max'): '''Helper function, implementing pool_2d in pure numpy''' if len(input.shape) < 2: raise NotImplementedError('input should have at least 2 dim,' ' shape is %s' % str(input.shape)) xi = 0 yi = 0 if not ignore_border: if input.shape[-2] % ws[0]: xi += 1 if input.shape[-1] % ws[1]: yi += 1 out_shp = list(input.shape[:-2]) out_shp.append(input.shape[-2] // ws[0] + xi) out_shp.append(input.shape[-1] // ws[1] + yi) output_val = np.zeros(out_shp) func = np.max if mode == 'sum': func = np.sum elif mode != 'max': func = np.average for k in np.ndindex(*input.shape[:-2]): for i in range(output_val.shape[-2]): ii = i * ws[0] for j in range(output_val.shape[-1]): jj = j * ws[1] patch = input[k][ii:ii + ws[0], jj:jj + ws[1]] output_val[k][i, j] = func(patch) return output_val @staticmethod def numpy_max_pool_nd(input, ws, ignore_border=False, mode='max'): '''Helper function, implementing pool_nd in pure numpy''' if len(input.shape) < len(ws): raise NotImplementedError('input should have at least %s dim,' ' shape is %s' % (str(ws), str(input.shape))) nd = len(ws) si = [0] * nd if not ignore_border: for i in range(nd): if input.shape[-nd + i] % ws[i]: si[i] += 1 out_shp = list(input.shape[:-nd]) for i in range(nd): out_shp.append(input.shape[-nd + i] // ws[i] + si[i]) output_val = np.zeros(out_shp) func = np.max if mode == 'sum': func = np.sum elif mode != 'max': func = np.average for l in np.ndindex(*input.shape[:-nd]): for r in np.ndindex(*output_val.shape[-nd:]): patch = input[l][tuple(slice(r[i] * ws[i], (r[i] + 1) * ws[i]) for i in range(nd))] output_val[l][r] = func(patch) return output_val @staticmethod def numpy_max_pool_2d_stride_pad( x, ws, ignore_border=True, stride=None, pad=(0, 0), mode='max'): assert ignore_border pad_h = pad[0] pad_w = pad[1] h = x.shape[-2] w = x.shape[-1] assert ws[0] > pad_h assert ws[1] > pad_w def pad_img(x): y = np.zeros( (x.shape[0], x.shape[1], x.shape[2] + pad_h * 2, x.shape[3] + pad_w * 2), dtype=x.dtype) y[:, :, pad_h:(x.shape[2] + pad_h), pad_w:(x.shape[3] + pad_w)] = x return y img_rows = h + 2 * pad_h img_cols = w + 2 * pad_w out_r = (img_rows - ws[0]) // stride[0] + 1 out_c = (img_cols - ws[1]) // stride[1] + 1 out_shp = list(x.shape[:-2]) out_shp.append(out_r) out_shp.append(out_c) ws0, ws1 = ws stride0, stride1 = stride output_val = np.zeros(out_shp) y = pad_img(x) func = np.max if mode == 'sum': func = np.sum elif mode != 'max': func = np.average inc_pad = mode == 'average_inc_pad' for k in np.ndindex(*x.shape[:-2]): for i in range(output_val.shape[-2]): ii_stride = i * stride[0] ii_end = builtins.min(ii_stride + ws[0], img_rows) if not inc_pad: ii_stride = builtins.max(ii_stride, pad_h) ii_end = builtins.min(ii_end, h + pad_h) for j in range(output_val.shape[-1]): jj_stride = j * stride[1] jj_end = builtins.min(jj_stride + ws[1], img_cols) if not inc_pad: jj_stride = builtins.max(jj_stride, pad_w) jj_end = builtins.min(jj_end, w + pad_w) patch = y[k][ii_stride:ii_end, jj_stride:jj_end] output_val[k][i, j] = func(patch) return output_val @staticmethod def numpy_max_pool_nd_stride_pad( input, ws, ignore_border=True, stride=None, pad=None, mode='max'): assert ignore_border nd = len(ws) if pad is None: pad = (0,) * nd if stride is None: stride = (0,) * nd assert len(pad) == len(ws) == len(stride) assert all(ws[i] > pad[i] for i in range(nd)) def pad_img(x): # initialize padded input y = np.zeros( x.shape[0:-nd] + tuple(x.shape[-nd + i] + pad[i] * 2 for i in range(nd)), dtype=x.dtype) # place the unpadded input in the center block = ((slice(None),) * (len(x.shape) - nd) + tuple(slice(pad[i], x.shape[-nd + i] + pad[i]) for i in range(nd))) y[block] = x return y pad_img_shp = list(input.shape[:-nd]) out_shp = list(input.shape[:-nd]) for i in range(nd): padded_size = input.shape[-nd + i] + 2 * pad[i] pad_img_shp.append(padded_size) out_shp.append((padded_size - ws[i]) // stride[i] + 1) output_val = np.zeros(out_shp) padded_input = pad_img(input) func = np.max if mode == 'sum': func = np.sum elif mode != 'max': func = np.average inc_pad = mode == 'average_inc_pad' for l in np.ndindex(*input.shape[:-nd]): for r in np.ndindex(*output_val.shape[-nd:]): region = [] for i in range(nd): r_stride = r[i] * stride[i] r_end = builtins.min(r_stride + ws[i], pad_img_shp[-nd + i]) if not inc_pad: r_stride = builtins.max(r_stride, pad[i]) r_end = builtins.min(r_end, input.shape[-nd + i] + pad[i]) region.append(slice(r_stride, r_end)) patch = padded_input[l][region] output_val[l][r] = func(patch) return output_val @staticmethod def numpy_max_pool_2d_stride(input, ws, ignore_border=False, stride=None, mode='max'): '''Helper function, implementing pool_2d in pure numpy this function provides stride input to indicate the stide size for the pooling regions. if not indicated, stride == ws.''' if len(input.shape) < 2: raise NotImplementedError('input should have at least 2 dim,' ' shape is %s' % str(input.shape)) if stride is None: stride = ws img_rows = input.shape[-2] img_cols = input.shape[-1] out_r = 0 out_c = 0 if img_rows - ws[0] >= 0: out_r = (img_rows - ws[0]) // stride[0] + 1 if img_cols - ws[1] >= 0: out_c = (img_cols - ws[1]) // stride[1] + 1 if not ignore_border: if out_r > 0: if img_rows - ((out_r - 1) * stride[0] + ws[0]) > 0: rr = img_rows - out_r * stride[0] if rr > 0: out_r += 1 else: if img_rows > 0: out_r += 1 if out_c > 0: if img_cols - ((out_c - 1) * stride[1] + ws[1]) > 0: cr = img_cols - out_c * stride[1] if cr > 0: out_c += 1 else: if img_cols > 0: out_c += 1 out_shp = list(input.shape[:-2]) out_shp.append(out_r) out_shp.append(out_c) func = np.max if mode == 'sum': func = np.sum elif mode != 'max': func = np.average output_val = np.zeros(out_shp) for k in np.ndindex(*input.shape[:-2]): for i in range(output_val.shape[-2]): ii_stride = i * stride[0] ii_end = builtins.min(ii_stride + ws[0], img_rows) for j in range(output_val.shape[-1]): jj_stride = j * stride[1] jj_end = builtins.min(jj_stride + ws[1], img_cols) patch = input[k][ii_stride:ii_end, jj_stride:jj_end] output_val[k][i, j] = func(patch) return output_val @staticmethod def numpy_max_pool_nd_stride(input, ws, ignore_border=False, stride=None, mode='max'): '''Helper function, implementing pooling in pure numpy this function provides stride input to indicate the stide size for the pooling regions. if not indicated, stride == ws.''' nd = len(ws) if stride is None: stride = ws assert len(stride) == len(ws) out_shp = list(input.shape[:-nd]) for i in range(nd): out = 0 if input.shape[-nd + i] - ws[i] >= 0: out = (input.shape[-nd + i] - ws[i]) // stride[i] + 1 if not ignore_border: if out > 0: if input.shape[-nd + i] - ((out - 1) * stride[i] + ws[i]) > 0: if input.shape[-nd + i] - out * stride[i] > 0: out += 1 else: if input.shape[-nd + i] > 0: out += 1 out_shp.append(out) func = np.max if mode == 'sum': func = np.sum elif mode != 'max': func = np.average output_val = np.zeros(out_shp) for l in np.ndindex(*input.shape[:-nd]): for r in np.ndindex(*output_val.shape[-nd:]): region = [] for i in range(nd): r_stride = r[i] * stride[i] r_end = builtins.min(r_stride + ws[i], input.shape[-nd + i]) region.append(slice(r_stride, r_end)) patch = input[l][region] output_val[l][r] = func(patch) return output_val def test_DownsampleFactorMax(self): rng = np.random.RandomState(utt.fetch_seed()) # maxpool, input size examples = ( ((2,), (16,)), ((2,), (4, 16,)), ((2,), (4, 2, 16,)), ((1, 1), (4, 2, 16, 16)), ((2, 2), (4, 2, 16, 16)), ((3, 3), (4, 2, 16, 16)), ((3, 2), (4, 2, 16, 16)), ((3, 2, 2), (3, 2, 16, 16, 16)), ((2, 2, 3, 2), (3, 2, 6, 6, 6, 5)), ) for example, ignore_border, mode in product(examples, [True, False], ['max', 'sum', 'average_inc_pad', 'average_exc_pad']): (maxpoolshp, inputsize) = example imval = rng.rand(*inputsize) images = theano.shared(imval) # Pure Numpy computation numpy_output_val = self.numpy_max_pool_nd(imval, maxpoolshp, ignore_border, mode=mode) # The pool_2d or pool_3d helper methods if len(maxpoolshp) == 2: output = pool_2d(images, maxpoolshp, ignore_border, mode=mode) f = function([], [output, ]) output_val = f() utt.assert_allclose(output_val, numpy_output_val) elif len(maxpoolshp) == 3: output = pool_3d(images, maxpoolshp, ignore_border, mode=mode) f = function([], [output, ]) output_val = f() utt.assert_allclose(output_val, numpy_output_val) # Pool op maxpool_op = Pool(ndim=len(maxpoolshp), ignore_border=ignore_border, mode=mode)(images, maxpoolshp) output_shape = Pool.out_shape(imval.shape, maxpoolshp, ndim=len(maxpoolshp), ignore_border=ignore_border) utt.assert_allclose(np.asarray(output_shape), numpy_output_val.shape) f = function([], maxpool_op) output_val = f() utt.assert_allclose(output_val, numpy_output_val) def test_DownsampleFactorMaxStride(self): rng = np.random.RandomState(utt.fetch_seed()) # maxpool, stride, ignore_border, input, output sizes examples = ( ((1, 1), (1, 1), True, (4, 10, 16, 16), (4, 10, 16, 16)), ((1, 1), (5, 7), True, (4, 10, 16, 16), (4, 10, 4, 3)), ((1, 1), (1, 1), False, (4, 10, 16, 16), (4, 10, 16, 16)), ((1, 1), (5, 7), False, (4, 10, 16, 16), (4, 10, 4, 3)), ((3, 3), (1, 1), True, (4, 10, 16, 16), (4, 10, 14, 14)), ((3, 3), (3, 3), True, (4, 10, 16, 16), (4, 10, 5, 5)), ((3, 3), (5, 7), True, (4, 10, 16, 16), (4, 10, 3, 2)), ((3, 3), (1, 1), False, (4, 10, 16, 16), (4, 10, 14, 14)), ((3, 3), (3, 3), False, (4, 10, 16, 16), (4, 10, 6, 6)), ((3, 3), (5, 7), False, (4, 10, 16, 16), (4, 10, 4, 3)), ((5, 3), (1, 1), True, (4, 10, 16, 16), (4, 10, 12, 14)), ((5, 3), (3, 3), True, (4, 10, 16, 16), (4, 10, 4, 5)), ((5, 3), (5, 7), True, (4, 10, 16, 16), (4, 10, 3, 2)), ((5, 3), (1, 1), False, (4, 10, 16, 16), (4, 10, 12, 14)), ((5, 3), (3, 3), False, (4, 10, 16, 16), (4, 10, 5, 6)), ((5, 3), (5, 7), False, (4, 10, 16, 16), (4, 10, 4, 3)), ((16, 16), (1, 1), True, (4, 10, 16, 16), (4, 10, 1, 1)), ((16, 16), (5, 7), True, (4, 10, 16, 16), (4, 10, 1, 1)), ((16, 16), (1, 1), False, (4, 10, 16, 16), (4, 10, 1, 1)), ((16, 16), (5, 7), False, (4, 10, 16, 16), (4, 10, 1, 1)), ((3,), (5,), True, (16,), (3,)), ((3,), (5,), True, (2, 16,), (2, 3,)), ((5,), (3,), True, (2, 3, 16,), (2, 3, 4,)), ((5, 1, 3), (3, 3, 3), True, (2, 16, 16, 16), (2, 4, 6, 5)), ((5, 1, 3), (3, 3, 3), True, (4, 2, 16, 16, 16), (4, 2, 4, 6, 5)), ) for example, mode in product(examples, ['max', 'sum', 'average_inc_pad', 'average_exc_pad']): (maxpoolshp, stride, ignore_border, inputshp, outputshp) = example # generate random images imval = rng.rand(*inputshp) images = theano.shared(imval) # Pool op numpy_output_val = \ self.numpy_max_pool_nd_stride(imval, maxpoolshp, ignore_border, stride, mode) assert numpy_output_val.shape == outputshp, ( "outshape is %s, calculated shape is %s" % (outputshp, numpy_output_val.shape)) maxpool_op = \ Pool(ndim=len(maxpoolshp), ignore_border=ignore_border, mode=mode)(images, maxpoolshp, stride) f = function([], maxpool_op) output_val = f() utt.assert_allclose(output_val, numpy_output_val) def test_DownsampleFactorMaxStrideExtra(self): rng = np.random.RandomState(utt.fetch_seed()) maxpoolshps = ((5, 3), (5, 3), (5, 3), (5, 5), (3, 2), (7, 7), (9, 9)) stridesizes = ((3, 2), (7, 5), (10, 6), (1, 1), (2, 3), (10, 10), (1, 1)) imvsizs = ((16, 16), (16, 16), (16, 16), (8, 5), (8, 5), (8, 5), (8, 5)) outputshps = ((4, 10, 4, 7), (4, 10, 5, 8), (4, 10, 2, 3), (4, 10, 3, 4), (4, 10, 2, 3), (4, 10, 2, 3), (4, 10, 4, 1), (4, 10, 4, 1), (4, 10, 3, 2), (4, 10, 4, 2), (4, 10, 1, 0), (4, 10, 1, 1), (4, 10, 0, 0), (4, 10, 1, 1)) images = tensor.dtensor4() for indx in np.arange(len(maxpoolshps)): imvsize = imvsizs[indx] imval = rng.rand(4, 10, imvsize[0], imvsize[1]) stride = stridesizes[indx] maxpoolshp = maxpoolshps[indx] for ignore_border, mode in product([True, False], ['max', 'sum', 'average_inc_pad', 'average_exc_pad']): indx_out = indx * 2 if not ignore_border: indx_out += 1 outputshp = outputshps[indx_out] # Pool op numpy_output_val = \ self.numpy_max_pool_2d_stride(imval, maxpoolshp, ignore_border, stride, mode) assert numpy_output_val.shape == outputshp, ( "outshape is %s, calculated shape is %s" % (outputshp, numpy_output_val.shape)) maxpool_op = \ Pool(ignore_border=ignore_border, ndim=len(maxpoolshp), mode=mode)( images, maxpoolshp, stride) f = function([images], maxpool_op) output_val = f(imval) utt.assert_allclose(output_val, numpy_output_val) def test_DownsampleFactorMaxPaddingStride(self): ignore_border = True # padding does not support ignore_border=False rng = np.random.RandomState(utt.fetch_seed()) # maxpool, stride, pad, input sizes examples = ( ((3,), (2,), (2,), (5,)), ((3,), (2,), (2,), (4, 5)), ((3,), (2,), (2,), (4, 2, 5, 5)), ((3, 3), (2, 2), (2, 2), (4, 2, 5, 5)), ((4, 4), (2, 2), (1, 2), (4, 2, 5, 5)), ((3, 4), (1, 1), (2, 1), (4, 2, 5, 6)), ((4, 3), (1, 2), (0, 0), (4, 2, 6, 5)), ((2, 2), (2, 2), (1, 1), (4, 2, 5, 5)), ((4, 3, 2), (1, 2, 2), (0, 2, 1), (4, 6, 6, 5)), ((4, 3, 2), (1, 2, 2), (0, 2, 1), (4, 2, 6, 5, 5)), ) for example, mode in product(examples, ['max', 'sum', 'average_inc_pad', 'average_exc_pad']): (maxpoolshp, stridesize, padsize, inputsize) = example imval = rng.rand(*inputsize) - 0.5 images = theano.shared(imval) numpy_output_val = self.numpy_max_pool_nd_stride_pad( imval, maxpoolshp, ignore_border, stridesize, padsize, mode) maxpool_op = Pool( ndim=len(maxpoolshp), ignore_border=ignore_border, mode=mode )(images, maxpoolshp, stridesize, padsize) f = function([], maxpool_op) output_val = f() utt.assert_allclose(output_val, numpy_output_val) def test_DownsampleFactorMaxPaddingStride_grad(self): rng = np.random.RandomState(utt.fetch_seed()) # maxpool, stride, pad, input sizes examples = ( ((10,), (5,), (3,), (2,)), ((10,), (5,), (3,), (2, 2)), ((10,), (5,), (3,), (1, 1, 2)), ((10, 10), (5, 3), (3, 2), (1, 1, 2, 2)), ((10, 5), (3, 5), (2, 3), (1, 1, 2, 1)), ((5, 5), (3, 3), (3, 3), (1, 1, 2, 2)), ((5, 5, 5), (3, 3, 3), (3, 3, 3), (1, 1, 2, 2, 2)), ) # average_inc_pad and average_exc_pad do not # support grad with padding for mode in ['max', 'sum']: for example in examples: (maxpoolshp, stridesize, padsize, inputsize) = example imval = rng.rand(*inputsize) * 10.0 def mp(input): return Pool( ndim=len(maxpoolshp), ignore_border=True, mode=mode, )(input, maxpoolshp, stridesize, padsize) utt.verify_grad(mp, [imval], rng=rng) def test_DownsampleFactorMax_grad(self): rng = np.random.RandomState(utt.fetch_seed()) # maxpool, input sizes examples = ( ((2,), (3,)), ((2,), (2, 3)), ((2,), (2, 3, 3)), ((1, 1), (2, 3, 3, 4)), ((3, 2), (2, 3, 3, 4)), ((2, 3), (2, 3, 3, 4)), ((1, 1, 1), (2, 3, 3)), ((3, 2, 2), (2, 3, 3, 4)), ((2, 2, 3), (2, 3, 3, 4, 4)), ) for example, ignore_border, mode in product(examples, [True, False], ['max', 'sum', 'average_inc_pad', 'average_exc_pad']): (maxpoolshp, inputsize) = example imval = rng.rand(*inputsize) * 10.0 # more variance means numeric gradient will be more accurate def mp(input): return Pool(ndim=len(maxpoolshp), ignore_border=ignore_border, mode=mode)(input, maxpoolshp) utt.verify_grad(mp, [imval], rng=rng) # pool, stride, input sizes pool_grad_stride_examples = ( ((1,), (1,), (16,)), ((1,), (3,), (1, 16)), ((1,), (5,), (1, 2, 16)), ((2,), (1,), (16,)), ((2,), (3,), (1, 16)), ((2,), (5,), (1, 2, 16)), ((1, 1), (1, 1), (1, 2, 16, 16)), ((1, 1), (3, 3), (1, 2, 16, 16)), ((1, 1), (5, 7), (1, 2, 16, 16)), ((3, 3), (1, 1), (1, 2, 16, 16)), ((3, 3), (3, 3), (1, 2, 16, 16)), ((3, 3), (5, 7), (1, 2, 16, 16)), ((5, 3), (1, 1), (1, 2, 16, 16)), ((5, 3), (3, 3), (1, 2, 16, 16)), ((5, 3), (5, 7), (1, 2, 16, 16)), ((5, 1, 2), (1, 1, 1), (16, 3, 16)), ((5, 1, 2), (3, 1, 2), (1, 16, 3, 16)), ((5, 1, 2), (5, 1, 4), (1, 2, 16, 3, 16)), ((5, 3), (3, 2), (1, 2, 16, 16)), ((5, 3), (7, 5), (1, 2, 16, 16)), ((5, 3), (10, 6), (1, 2, 16, 16)), ((5, 5), (1, 1), (1, 2, 8, 5)), ((3, 2), (2, 3), (1, 2, 8, 5)), ((7, 7), (10, 10), (1, 2, 8, 5)), ((9, 9), (1, 1), (1, 2, 8, 5)), ) @parameterized.expand(product(pool_grad_stride_examples, [True, False], ['max', 'sum', 'average_inc_pad', 'average_exc_pad']), testcase_func_name=utt.custom_name_func) def test_DownsampleFactorMax_grad_stride(self, example, ignore_border, mode): # checks the gradient for the case that stride is used rng = np.random.RandomState(utt.fetch_seed()) (maxpoolshp, stridesize, inputsize) = example imval = rng.rand(*inputsize) def mp(input): return Pool(ndim=len(maxpoolshp), ignore_border=ignore_border, mode=mode)(input, maxpoolshp, stridesize) utt.verify_grad(mp, [imval], rng=rng) def test_DownsampleFactorMaxGrad_grad(self): rng = np.random.RandomState(utt.fetch_seed()) # maxpool, input sizes examples = ( ((2,), (2,)), ((2,), (2, 3)), ((1, 1), (2, 3, 3, 4)), ((3, 2), (2, 3, 3, 4)), ((2, 3), (2, 3, 3, 4)), ((1, 1, 1), (2, 3, 3, 4)), ((3, 2, 2), (2, 3, 3, 4)), ((2, 3, 2), (2, 3, 3, 4)), ((2, 2, 3), (2, 3, 3, 4)), ((2, 2, 3), (2, 1, 3, 3, 4)), ) for (maxpoolshp, inputsize) in examples: imval = rng.rand(*inputsize) * 10.0 # more variance means numeric gradient will be more accurate for ignore_border in [True, False]: # print 'maxpoolshp =', maxpoolshp # print 'ignore_border =', ignore_border # The shape of the gradient will be the shape of the output grad_shape = Pool.out_shape( imval.shape, maxpoolshp, ndim=len(maxpoolshp), ignore_border=ignore_border) grad_val = rng.rand(*grad_shape) * 10.0 def mp(input, grad): out = Pool( ndim=len(maxpoolshp), ignore_border=ignore_border)(input, maxpoolshp) grad_op = MaxPoolGrad( ndim=len(maxpoolshp), ignore_border=ignore_border) return grad_op(input, out, grad, maxpoolshp) utt.verify_grad(mp, [imval, grad_val], rng=rng) def test_AveragePoolGrad_grad(self): rng = np.random.RandomState(utt.fetch_seed()) # avgpool, input sizes examples = ( ((2,), (2,)), ((2,), (2, 3)), ((1, 1), (2, 3, 3, 4)), ((3, 2), (2, 3, 3, 4)), ((2, 3), (2, 3, 3, 4)), ((3, 2, 2), (2, 3, 3, 4)), ((2, 2, 3), (2, 3, 3, 4)), ) for (avgpoolshp, inputsize) in examples: imval = rng.rand(*inputsize) * 10.0 # more variance means numeric gradient will be more accurate for ignore_border in [True, False]: for mode in ['sum', 'average_inc_pad', 'average_exc_pad']: # print 'maxpoolshp =', maxpoolshp # print 'ignore_border =', ignore_border # The shape of the gradient will be the shape of the output grad_shape = Pool.out_shape( imval.shape, avgpoolshp, ndim=len(avgpoolshp), ignore_border=ignore_border) grad_val = rng.rand(*grad_shape) * 10.0 def mp(input, grad): grad_op = AveragePoolGrad( ndim=len(avgpoolshp), ignore_border=ignore_border, mode=mode) return grad_op(input, grad, avgpoolshp) utt.verify_grad(mp, [imval, grad_val], rng=rng) @parameterized.expand(product(pool_grad_stride_examples, [True, False]), testcase_func_name=utt.custom_name_func) def test_DownsampleFactorMaxGrad_grad_stride(self, example, ignore_border): # checks the gradient of the gradient for # the case that stride is used rng = np.random.RandomState(utt.fetch_seed()) (maxpoolshp, stride, inputsize) = example imval = rng.rand(*inputsize) grad_shape = Pool.out_shape( imval.shape, maxpoolshp, ndim=len(maxpoolshp), ignore_border=ignore_border, stride=stride) # skip the grad verification when the output is empty if np.prod(grad_shape) != 0: grad_val = rng.rand(*grad_shape) def mp(input, grad): out = Pool( ndim=len(maxpoolshp), ignore_border=ignore_border)(input, maxpoolshp, stride) grad_op = MaxPoolGrad( ndim=len(maxpoolshp), ignore_border=ignore_border) return grad_op(input, out, grad, maxpoolshp, stride) utt.verify_grad(mp, [imval, grad_val], rng=rng) @parameterized.expand(product(pool_grad_stride_examples, [True, False], ['sum', 'average_inc_pad', 'average_exc_pad']), testcase_func_name=utt.custom_name_func) def test_AveragePoolGrad_grad_stride(self, example, ignore_border, mode): # checks the gradient of the gradient for # the case that stride is used rng = np.random.RandomState(utt.fetch_seed()) (avgpoolshp, stride, inputsize) = example imval = rng.rand(*inputsize) grad_shape = Pool.out_shape( imval.shape, avgpoolshp, ndim=len(avgpoolshp), ignore_border=ignore_border, stride=stride) # skip the grad verification when the output is empty if np.prod(grad_shape) != 0: grad_val = rng.rand(*grad_shape) def mp(input, grad): grad_op = AveragePoolGrad( ndim=len(avgpoolshp), ignore_border=ignore_border, mode=mode) return grad_op(input, grad, avgpoolshp, stride) utt.verify_grad(mp, [imval, grad_val], rng=rng) def test_DownsampleFactorMaxPaddingStride_grad_grad(self): rng = np.random.RandomState(utt.fetch_seed()) # maxpool, stride, pad, input sizes examples = ( ((3,), (2,), (2,), (10,)), ((3,), (2,), (2,), (2, 10,)), ((3,), (2,), (2,), (2, 1, 10,)), ((5, 3), (3, 2), (2, 2), (1, 1, 10, 10)), ((3, 5), (2, 3), (2, 1), (1, 1, 10, 5)), ((5, 3, 3), (3, 2, 2), (2, 2, 2), (1, 1, 10, 5, 5)), ((3, 3, 5), (2, 2, 3), (2, 2, 1), (1, 1, 5, 5, 10)), ) for (maxpoolshp, stridesize, padsize, inputsize) in examples: imval = rng.rand(*inputsize) * 10.0 grad_shape = Pool.out_shape(imval.shape, maxpoolshp, ndim=len(maxpoolshp), stride=stridesize, ignore_border=True, pad=padsize) grad_val = rng.rand(*grad_shape) * 10.0 def mp(input, grad): out = Pool( ndim=len(maxpoolshp), ignore_border=True, )(input, maxpoolshp, stridesize, padsize) grad_op = MaxPoolGrad(ndim=len(maxpoolshp), ignore_border=True) return grad_op(input, out, grad, maxpoolshp, stridesize, padsize) utt.verify_grad(mp, [imval, grad_val], rng=rng) def test_AveragePoolPaddingStride_grad_grad(self): rng = np.random.RandomState(utt.fetch_seed()) # avgpool, stride, pad, input sizes examples = ( ((3,), (2,), (2,), (10,)), ((3,), (2,), (2,), (2, 10,)), ((3,), (2,), (2,), (2, 1, 10,)), ((5, 3), (3, 2), (2, 2), (1, 1, 10, 10)), ((3, 5), (2, 3), (2, 1), (1, 1, 10, 5)), ((5, 3, 2), (3, 2, 1), (2, 2, 2), (1, 1, 10, 5, 5)), ) for (avgpoolshp, stridesize, padsize, inputsize) in examples: imval = rng.rand(*inputsize) * 10.0 # 'average_exc_pad' with non-zero padding is not implemented for mode in ['sum', 'average_inc_pad']: grad_shape = Pool.out_shape(imval.shape, avgpoolshp, ndim=len(avgpoolshp), stride=stridesize, ignore_border=True, pad=padsize) grad_val = rng.rand(*grad_shape) * 10.0 def mp(input, grad): grad_op = AveragePoolGrad(ndim=len(avgpoolshp), ignore_border=True, mode=mode) return grad_op(input, grad, avgpoolshp, stridesize, padsize) utt.verify_grad(mp, [imval, grad_val], rng=rng) def test_DownsampleFactorMax_hessian(self): # Example provided by Frans Cronje, see # https://groups.google.com/d/msg/theano-users/qpqUy_3glhw/JMwIvlN5wX4J x_vec = tensor.vector('x') z = tensor.dot(x_vec.dimshuffle(0, 'x'), x_vec.dimshuffle('x', 0)) y = pool_2d(input=z, ws=(2, 2), ignore_border=True) C = tensor.exp(tensor.sum(y)) grad_hess = tensor.hessian(cost=C, wrt=x_vec) fn_hess = function(inputs=[x_vec], outputs=grad_hess) # The value has been manually computed from the theoretical gradient, # and confirmed by the implementation. assert np.allclose(fn_hess([1, 2]), [[0., 0.], [0., 982.7667]]) def test_DownsampleFactorMaxGradGrad_grad(self): rng = np.random.RandomState(utt.fetch_seed()) # maxpool, stride, pad, input sizes examples = ( ((3,), (2,), (2,), (10,)), ((3,), (2,), (2,), (2, 10,)), ((3,), (2,), (2,), (2, 1, 10,)), ((5, 3), (3, 2), (2, 2), (1, 1, 10, 10)), ((3, 5), (2, 3), (2, 1), (1, 1, 10, 5)), ((3, 3), (3, 3), (2, 2), (1, 1, 5, 5)), ((5, 3, 3), (3, 2, 2), (2, 2, 2), (1, 1, 10, 5, 5)), ((3, 3, 5), (2, 2, 3), (2, 2, 1), (1, 1, 5, 5, 10)), ) for (maxpoolshp, stridesize, padsize, inputsize) in examples: imval1 = rng.rand(*inputsize) * 10.0 imval2 = rng.rand(*inputsize) * 10.0 def mp(input1, input2): op1 = Pool(ndim=len(maxpoolshp), ignore_border=True) pooled_out = op1(input1, maxpoolshp, stridesize, padsize) op2 = DownsampleFactorMaxGradGrad( ndim=len(maxpoolshp), ignore_border=True) out = op2(input1, pooled_out, input2, maxpoolshp, stridesize, padsize) return out utt.verify_grad(mp, [imval1, imval2], rng=rng) def test_max_pool_2d_2D(self): rng = np.random.RandomState(utt.fetch_seed()) maxpoolshps = ((1, 1), (3, 2)) imval = rng.rand(4, 5) images = tensor.dmatrix() for maxpoolshp, ignore_border, mode in product(maxpoolshps, [True, False], ['max', 'sum', 'average_inc_pad', 'average_exc_pad']): # print 'maxpoolshp =', maxpoolshp # print 'ignore_border =', ignore_border numpy_output_val = self.numpy_max_pool_2d(imval, maxpoolshp, ignore_border, mode=mode) output = pool_2d(images, maxpoolshp, ignore_border, mode=mode) output_val = function([images], output)(imval) utt.assert_allclose(output_val, numpy_output_val) def mp(input): return pool_2d(input, maxpoolshp, ignore_border, mode=mode) utt.verify_grad(mp, [imval], rng=rng) def test_max_pool_3d_3D(self): rng = np.random.RandomState(utt.fetch_seed()) maxpoolshps = ((1, 1, 1), (3, 2, 1)) imval = rng.rand(4, 5, 6) images = tensor.dtensor3() for maxpoolshp, ignore_border, mode in product(maxpoolshps, [True, False], ['max', 'sum', 'average_inc_pad', 'average_exc_pad']): # print 'maxpoolshp =', maxpoolshp # print 'ignore_border =', ignore_border numpy_output_val = self.numpy_max_pool_nd(imval, maxpoolshp, ignore_border, mode=mode) output = pool_3d(images, maxpoolshp, ignore_border, mode=mode) output_val = function([images], output)(imval) utt.assert_allclose(output_val, numpy_output_val) def mp(input): return pool_3d(input, maxpoolshp, ignore_border, mode=mode) utt.verify_grad(mp, [imval], rng=rng) def test_max_pool_3d_3D_deprecated_interface(self): rng = np.random.RandomState(utt.fetch_seed()) maxpoolshps = ((1, 1, 1), (3, 2, 1)) imval = rng.rand(4, 5, 6) images = tensor.dtensor3() for maxpoolshp, ignore_border, mode in product(maxpoolshps, [True, False], ['max', 'sum', 'average_inc_pad', 'average_exc_pad']): # print 'maxpoolshp =', maxpoolshp # print 'ignore_border =', ignore_border numpy_output_val = self.numpy_max_pool_nd(imval, maxpoolshp, ignore_border, mode=mode) output = pool_3d(input=images, ds=maxpoolshp, ignore_border=ignore_border, st=maxpoolshp, padding=(0, 0, 0), mode=mode) output_val = function([images], output)(imval) utt.assert_allclose(output_val, numpy_output_val) def mp(input): return pool_3d(input, maxpoolshp, ignore_border, mode=mode) def test_max_pool_2d_2D_same_size(self): rng = np.random.RandomState(utt.fetch_seed()) test_input_array = np.array([[[ [1., 2., 3., 4.], [5., 6., 7., 8.] ]]]).astype(theano.config.floatX) test_answer_array = np.array([[[ [0., 0., 0., 0.], [0., 6., 0., 8.] ]]]).astype(theano.config.floatX) input = tensor.tensor4(name='input') patch_size = (2, 2) op = max_pool_2d_same_size(input, patch_size) op_output = function([input], op)(test_input_array) utt.assert_allclose(op_output, test_answer_array) def mp(input): return max_pool_2d_same_size(input, patch_size) utt.verify_grad(mp, [test_input_array], rng=rng) def test_max_pool_2d_3D(self): rng = np.random.RandomState(utt.fetch_seed()) maxpoolshps = [(1, 2)] imval = rng.rand(2, 3, 4) images = tensor.dtensor3() for maxpoolshp, ignore_border, mode in product(maxpoolshps, [True, False], ['max', 'sum', 'average_inc_pad', 'average_exc_pad']): # print 'maxpoolshp =', maxpoolshp # print 'ignore_border =', ignore_border numpy_output_val = self.numpy_max_pool_2d(imval, maxpoolshp, ignore_border, mode) output = pool_2d(images, maxpoolshp, ignore_border, mode=mode) output_val = function([images], output)(imval) utt.assert_allclose(output_val, numpy_output_val) # removed as already tested in test_max_pool_2d_2D # This make test in debug mode too slow. # def mp(input): # return pool_2d(input, maxpoolshp, ignore_border) # utt.verify_grad(mp, [imval], rng=rng) def test_max_pool_2d_6D(self): rng = np.random.RandomState(utt.fetch_seed()) maxpoolshps = [(3, 2)] imval = rng.rand(2, 1, 1, 1, 3, 4) images = tensor.TensorType('float64', [False] * 6)() for maxpoolshp, ignore_border, mode in product(maxpoolshps, [True, False], ['max', 'sum', 'average_inc_pad', 'average_exc_pad']): # print 'maxpoolshp =', maxpoolshp # print 'ignore_border =', ignore_border numpy_output_val = self.numpy_max_pool_2d(imval, maxpoolshp, ignore_border, mode=mode) output = pool_2d(images, maxpoolshp, ignore_border, mode=mode) output_val = function([images], output)(imval) utt.assert_allclose(output_val, numpy_output_val) # removed as already tested in test_max_pool_2d_2D # This make test in debug mode too slow. # def mp(input): # return pool_2d(input, maxpoolshp, ignore_border) # utt.verify_grad(mp, [imval], rng=rng) def test_infer_shape(self): image = tensor.dtensor4() maxout = tensor.dtensor4() gz = tensor.dtensor4() rng = np.random.RandomState(utt.fetch_seed()) maxpoolshps = ((1, 1), (2, 2), (3, 3), (2, 3), (3, 2)) image_val = rng.rand(4, 6, 7, 9) out_shapes = [[[[4, 6, 7, 9], [4, 6, 7, 9]], [[4, 6, 3, 4], [4, 6, 4, 5]], [[4, 6, 2, 3], [4, 6, 3, 3]], [[4, 6, 3, 3], [4, 6, 4, 3]], [[4, 6, 2, 4], [4, 6, 3, 5]]], [[None, None], [[4, 6, 4, 5], None], [[4, 6, 3, 3], None], [[4, 6, 4, 3], None], [[4, 6, 3, 5], None]], [[None, None], [None, None], [[4, 6, 3, 4], None], [[4, 6, 4, 4], None], [None, None]]] for i, maxpoolshp in enumerate(maxpoolshps): for j, ignore_border in enumerate([True, False]): for k, pad in enumerate([(0, 0), (1, 1), (1, 2)]): if out_shapes[k][i][j] is None: continue # checking shapes generated by Pool self._compile_and_check([image], [Pool(ignore_border=ignore_border) (image, maxpoolshp, pad=pad)], [image_val], Pool) # checking shapes generated by MaxPoolGrad maxout_val = rng.rand(*out_shapes[k][i][j]) gz_val = rng.rand(*out_shapes[k][i][j]) self._compile_and_check([image, maxout, gz], [MaxPoolGrad( ignore_border=ignore_border) (image, maxout, gz, maxpoolshp, pad=pad)], [image_val, maxout_val, gz_val], MaxPoolGrad, warn=False) # checking with broadcastable input image = tensor.tensor(dtype='float64', broadcastable=(False, False, True, True)) image_val = rng.rand(4, 6, 1, 1) self._compile_and_check( [image], [Pool(ignore_border=True)(image, (2, 2), pad=(0, 0))], [image_val], Pool) def test_pooling_with_tensor_vars(self): x = tensor.ftensor4() window_size = tensor.ivector() stride = tensor.ivector() padding = tensor.ivector() data = np.random.normal(0, 1, (1, 1, 5, 5)).astype('float32') # checking variable params vs fixed params for ignore_border in [True, False]: for mode in ['max', 'sum', 'average_inc_pad', 'average_exc_pad']: y = pool_2d(x, window_size, ignore_border, stride, padding, mode) dx = theano.gradient.grad(y.sum(), x) var_fct = theano.function([x, window_size, stride, padding], [y, dx]) for ws in (4, 2, 5): for st in (2, 3): for pad in (0, 1): if (pad > st or st > ws or (pad != 0 and not ignore_border) or (mode == 'average_exc_pad' and pad != 0)): continue y = pool_2d(x, (ws, ws), ignore_border, (st, st), (pad, pad), mode) dx = theano.gradient.grad(y.sum(), x) fix_fct = theano.function([x], [y, dx]) var_y, var_dx = var_fct(data, (ws, ws), (st, st), (pad, pad)) fix_y, fix_dx = fix_fct(data) utt.assert_allclose(var_y, fix_y) utt.assert_allclose(var_dx, fix_dx) def test_pooling_with_tensor_vars_deprecated_interface(self): x = tensor.ftensor4() window_size = tensor.ivector() stride = tensor.ivector() padding = tensor.ivector() data = np.random.normal(0, 1, (1, 1, 5, 5)).astype('float32') # checking variable params vs fixed params for ignore_border in [True, False]: for mode in ['max', 'sum', 'average_inc_pad', 'average_exc_pad']: y = pool_2d(input=x, ds=window_size, ignore_border=ignore_border, st=stride, padding=padding, mode=mode) dx = theano.gradient.grad(y.sum(), x) var_fct = theano.function([x, window_size, stride, padding], [y, dx]) ws = 5 st = 3 pad = 1 if (pad > st or st > ws or (pad != 0 and not ignore_border) or (mode == 'average_exc_pad' and pad != 0)): continue y = pool_2d(input=x, ds=(ws, ws), ignore_border=ignore_border, st=(st, st), padding=(pad, pad), mode=mode) dx = theano.gradient.grad(y.sum(), x) fix_fct = theano.function([x], [y, dx]) var_y, var_dx = var_fct(data, (ws, ws), (st, st), (pad, pad)) fix_y, fix_dx = fix_fct(data) utt.assert_allclose(var_y, fix_y) utt.assert_allclose(var_dx, fix_dx) def test_old_pool_interface(self): if sys.version_info[0] != 3: # Only tested with python 3 because of pickling issues. raise SkipTest('Skip old pool interface with python 2.x') # 1. Load the old version testfile_dir = os.path.dirname(os.path.realpath(__file__)) fname = 'old_pool_interface.pkl' with open(os.path.join(testfile_dir, fname), 'rb') as fp: try: old_fct = cPickle.load(fp, encoding='latin1') except ImportError: # Windows sometimes fail with nonsensical errors like: # ImportError: No module named type # ImportError: No module named copy_reg # when "type" and "copy_reg" are builtin modules. if sys.platform == 'win32': exc_type, exc_value, exc_trace = sys.exc_info() reraise(SkipTest, exc_value, exc_trace) raise # 2. Create the new version x = theano.tensor.ftensor4() y = pool_2d(x, (2, 2), mode='max', ignore_border=True) z = pool_2d(x, (2, 2), mode='average_exc_pad', ignore_border=True) dy_dx = theano.gradient.grad(y.sum(), x) dz_dx = theano.gradient.grad(z.sum(), x) new_fct = theano.function([x], [y, z, dy_dx, dz_dx]) # 3. Assert that the answer is the same rng = np.random.RandomState(utt.fetch_seed()) image_val = rng.rand(4, 6, 7, 9).astype(np.float32) old_out = old_fct(image_val) new_out = new_fct(image_val) for o, n in zip(old_out, new_out): utt.assert_allclose(o, n) if __name__ == '__main__': unittest.main() Theano-1.0.0/theano/tensor/signal/tests/__init__.py00006000163562€ÿÿÿþ0000000000013203335532023575 0ustar boccoset00000000000000Theano-1.0.0/theano/tensor/signal/conv.py00006000163562€ÿÿÿþ0000000715713203335532021665 0ustar boccoset00000000000000""" Contains a wrapper function for tensor.nnet.ConvOp, which can be used to perform generic 2D convolution. """ from __future__ import absolute_import, print_function, division import warnings import theano import theano.tensor as tensor from theano.tensor.nnet import conv import logging __docformat__ = "restructuredtext en" _logger = logging.getLogger("theano.tensor.signal.conv") def conv2d(input, filters, image_shape=None, filter_shape=None, border_mode='valid', subsample=(1, 1), **kargs): """ signal.conv.conv2d performs a basic 2D convolution of the input with the given filters. The input parameter can be a single 2D image or a 3D tensor, containing a set of images. Similarly, filters can be a single 2D filter or a 3D tensor, corresponding to a set of 2D filters. Shape parameters are optional and will result in faster execution. Parameters ---------- input : Symbolic theano tensor for images to be filtered. Dimensions: ([num_images], image height, image width) filters : Symbolic theano tensor for convolution filter(s). Dimensions: ([num_filters], filter height, filter width) border_mode: {'valid', 'full'} See scipy.signal.convolve2d. subsample Factor by which to subsample output. image_shape : tuple of length 2 or 3 ([num_images,] image height, image width). filter_shape : tuple of length 2 or 3 ([num_filters,] filter height, filter width). kwargs See theano.tensor.nnet.conv.conv2d. Returns ------- symbolic 2D,3D or 4D tensor Tensor of filtered images, with shape ([number images,] [number filters,] image height, image width). """ assert input.ndim in (2, 3) assert filters.ndim in (2, 3) # use shape information if it is given to us ### if filter_shape and image_shape: if input.ndim == 3: bsize = image_shape[0] else: bsize = 1 imshp = (1,) + tuple(image_shape[-2:]) if filters.ndim == 3: nkern = filter_shape[0] else: nkern = 1 kshp = filter_shape[-2:] else: nkern, kshp = None, None bsize, imshp = None, None # reshape tensors to 4D, for compatibility with ConvOp ### if input.ndim == 3: sym_bsize = input.shape[0] else: sym_bsize = 1 if filters.ndim == 3: sym_nkern = filters.shape[0] else: sym_nkern = 1 new_input_shape = tensor.join(0, tensor.stack([sym_bsize, 1]), input.shape[-2:]) input4D = tensor.reshape(input, new_input_shape, ndim=4) new_filter_shape = tensor.join(0, tensor.stack([sym_nkern, 1]), filters.shape[-2:]) filters4D = tensor.reshape(filters, new_filter_shape, ndim=4) # perform actual convolution ### op = conv.ConvOp(output_mode=border_mode, dx=subsample[0], dy=subsample[1], imshp=imshp, kshp=kshp, nkern=nkern, bsize=bsize, **kargs) output = op(input4D, filters4D) # flatten to 3D tensor if convolving with single filter or single image if input.ndim == 2 and filters.ndim == 2: if theano.config.warn.signal_conv2d_interface: warnings.warn( "theano.tensor.signal.conv2d() now outputs a 2d tensor when both" " inputs are 2d. To disable this warning, set the Theano flag" " warn.signal_conv2d_interface to False", stacklevel=3) output = tensor.flatten(output.T, ndim=2).T elif input.ndim == 2 or filters.ndim == 2: output = tensor.flatten(output.T, ndim=3).T return output Theano-1.0.0/theano/tensor/blas_c.py00006000163562€ÿÿÿþ0000006373313203335532020670 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division from theano import config from theano.gof.params_type import ParamsType from theano.scalar import bool as bool_t from theano.tensor.opt import in2out from theano.tensor.blas import ldflags, blas_header_text, blas_header_version from theano.tensor.blas import blas_optdb, optdb, local_optimizer from theano.tensor.blas import Ger, ger, ger_destructive from theano.tensor.blas import Gemv, gemv_inplace, gemv_no_inplace from theano.tensor import basic as T class BaseBLAS(object): def c_libraries(self): return ldflags() def c_compile_args(self): return ldflags(libs=False, flags=True) def c_lib_dirs(self): return ldflags(libs=False, libs_dir=True) def c_header_dirs(self): return ldflags(libs=False, include_dir=True) def c_support_code(self): return blas_header_text() # ##### ####### ####### # GER # ##### ####### ####### def ger_c_code(A, a, x, y, Z, fail, params): return """ int elemsize ; if (PyArray_NDIM(%(A)s) != 2) {PyErr_SetString(PyExc_NotImplementedError, "rank(A) != 2"); %(fail)s;} if (PyArray_NDIM(%(x)s) != 1) {PyErr_SetString(PyExc_NotImplementedError, "rank(x) != 1"); %(fail)s;} if (PyArray_NDIM(%(y)s) != 1) {PyErr_SetString(PyExc_NotImplementedError, "rank(y) != 1"); %(fail)s;} if (PyArray_NDIM(%(a)s) != 0) {PyErr_SetString(PyExc_NotImplementedError, "rank(a) != 0"); %(fail)s;} if (PyArray_DESCR(%(A)s)->type_num != PyArray_DESCR(%(x)s)->type_num) { PyErr_SetString(PyExc_TypeError, "A vs. x"); %(fail)s; } if (PyArray_DESCR(%(A)s)->type_num != PyArray_DESCR(%(y)s)->type_num) { PyErr_SetString(PyExc_TypeError, "A vs. y"); %(fail)s; } if (PyArray_DIMS(%(A)s)[0] != PyArray_DIMS(%(x)s)[0]) { PyErr_SetString(PyExc_ValueError, "Shape mismatch: A.shape[0] != x.shape[0]"); %(fail)s; } if (PyArray_DIMS(%(A)s)[1] != PyArray_DIMS(%(y)s)[0]) { PyErr_SetString(PyExc_ValueError, "Shape mismatch: A.shape[1] != y.shape[0]"); %(fail)s; } if (PyArray_DESCR(%(A)s)->type_num == NPY_DOUBLE) { elemsize = 8; } else if (PyArray_DESCR(%(A)s)->type_num == NPY_FLOAT) { elemsize = 4;} else { PyErr_SetString(PyExc_NotImplementedError, "complex CGer"); %(fail)s; } // copy A if !self.destructive or A is fully strided if (!%(params)s->destructive || (PyArray_STRIDES(%(A)s)[0] < 0) || (PyArray_STRIDES(%(A)s)[1] < 0) || ((PyArray_STRIDES(%(A)s)[0] != elemsize) && (PyArray_STRIDES(%(A)s)[1] != elemsize))) { npy_intp dims[2]; dims[0] = PyArray_DIMS(%(A)s)[0]; dims[1] = PyArray_DIMS(%(A)s)[1]; if ((NULL == %(Z)s) || (PyArray_DIMS(%(Z)s)[0] != PyArray_DIMS(%(A)s)[0]) || (PyArray_DIMS(%(Z)s)[1] != PyArray_DIMS(%(A)s)[1]) || (PyArray_STRIDES(%(Z)s)[0] < 0) || (PyArray_STRIDES(%(Z)s)[1] < 0) || ((PyArray_STRIDES(%(Z)s)[0] != elemsize) && (PyArray_STRIDES(%(Z)s)[1] != elemsize))) { Py_XDECREF(%(Z)s); %(Z)s = (PyArrayObject*) PyArray_SimpleNew(2, dims, PyArray_TYPE(%(A)s)); if(!%(Z)s) { PyErr_SetString(PyExc_MemoryError, "failed to alloc ger output"); %(fail)s } } if (%(Z)s == %(A)s) { PyErr_SetString(PyExc_AssertionError, "%(Z)s != %(A)s"); %(fail)s } if (PyArray_DESCR(%(Z)s)->type_num == NPY_FLOAT) { float * zoutdata = (float*)PyArray_DATA(%(Z)s); const float * zdata = (float*)PyArray_DATA(%(A)s); const float * xdata = (float*)PyArray_DATA(%(x)s); const float * ydata = (float*)PyArray_DATA(%(y)s); const float * adata = (float*)PyArray_DATA(%(a)s); const float alpha = adata[0]; float tmp, xx; int Ai = PyArray_STRIDES(%(A)s)[0]/sizeof(float); int Aj = PyArray_STRIDES(%(A)s)[1]/sizeof(float); int Zi = PyArray_STRIDES(%(Z)s)[0]/sizeof(float); int Zj = PyArray_STRIDES(%(Z)s)[1]/sizeof(float); int xi = PyArray_STRIDES(%(x)s)[0]/sizeof(float); int yj = PyArray_STRIDES(%(y)s)[0]/sizeof(float); for (int i = 0; i < dims[0]; ++i) { xx = alpha * xdata[xi * i]; for (int j = 0; j < dims[1]; ++j) { tmp = zdata[Ai*i+Aj*j]; tmp += xx * ydata[yj * j]; zoutdata[Zi*i+Zj*j] = tmp; } } } else if (PyArray_DESCR(%(Z)s)->type_num == NPY_DOUBLE) { double * zoutdata = (double*) PyArray_DATA(%(Z)s); const double * zdata = (double*)PyArray_DATA(%(A)s); const double * xdata = (double*)PyArray_DATA(%(x)s); const double * ydata = (double*)PyArray_DATA(%(y)s); const double * adata = (double*)PyArray_DATA(%(a)s); const double alpha = adata[0]; double tmp, xx; int Ai = PyArray_STRIDES(%(A)s)[0]/sizeof(double); int Aj = PyArray_STRIDES(%(A)s)[1]/sizeof(double); int Zi = PyArray_STRIDES(%(Z)s)[0]/sizeof(double); int Zj = PyArray_STRIDES(%(Z)s)[1]/sizeof(double); int xi = PyArray_STRIDES(%(x)s)[0]/sizeof(double); int yj = PyArray_STRIDES(%(y)s)[0]/sizeof(double); for (int i = 0; i < dims[0]; ++i) { xx = alpha * xdata[xi * i]; for (int j = 0; j < dims[1]; ++j) { tmp = zdata[Ai*i+Aj*j]; tmp += xx * ydata[yj * j]; zoutdata[Zi*i+Zj*j] = tmp; } } } else { PyErr_SetString(PyExc_AssertionError, "neither float nor double dtype"); %(fail)s } } else { if (%(Z)s != %(A)s) { if (%(Z)s) { Py_DECREF(%(Z)s); } %(Z)s = %(A)s; Py_INCREF(%(Z)s); } npy_intp dims[2]; dims[0] = PyArray_DIMS(%(A)s)[0]; dims[1] = PyArray_DIMS(%(A)s)[1]; if ((dims[0] * dims[1]) < 100000) { if (PyArray_DESCR(%(Z)s)->type_num == NPY_FLOAT) { float * zoutdata = (float*)PyArray_DATA(%(Z)s); const float * xdata = (float*)PyArray_DATA(%(x)s); const float * ydata = (float*)PyArray_DATA(%(y)s); const float * adata = (float*)PyArray_DATA(%(a)s); const float alpha = adata[0]; float tmp, axi; int Zi = PyArray_STRIDES(%(Z)s)[0]/sizeof(float); int Zj = PyArray_STRIDES(%(Z)s)[1]/sizeof(float); int xi = PyArray_STRIDES(%(x)s)[0]/sizeof(float); int yj = PyArray_STRIDES(%(y)s)[0]/sizeof(float); for (int i = 0; i < dims[0]; ++i) { axi = alpha * xdata[xi * i]; for (int j = 0; j < dims[1]; ++j) { zoutdata[Zi*i+Zj*j] += axi * ydata[yj * j]; } } } else if (PyArray_DESCR(%(Z)s)->type_num == NPY_DOUBLE) { double * zoutdata = (double*) PyArray_DATA(%(Z)s); const double * xdata = (double*)PyArray_DATA(%(x)s); const double * ydata = (double*)PyArray_DATA(%(y)s); const double * adata = (double*)PyArray_DATA(%(a)s); const double alpha = adata[0]; double tmp, axi; int Zi = PyArray_STRIDES(%(Z)s)[0]/sizeof(double); int Zj = PyArray_STRIDES(%(Z)s)[1]/sizeof(double); int xi = PyArray_STRIDES(%(x)s)[0]/sizeof(double); int yj = PyArray_STRIDES(%(y)s)[0]/sizeof(double); for (int i = 0; i < dims[0]; ++i) { axi = alpha * xdata[xi * i]; for (int j = 0; j < dims[1]; ++j) { zoutdata[Zi*i+Zj*j] += axi * ydata[yj * j]; } } } } else { int Nz0 = PyArray_DIMS(%(Z)s)[0]; int Nz1 = PyArray_DIMS(%(Z)s)[1]; int Sx = PyArray_STRIDES(%(x)s)[0] / elemsize; int Sy = PyArray_STRIDES(%(y)s)[0] / elemsize; /* create appropriate strides for Z, if it is a row or column matrix. * In that case, the value of the stride does not really matter, but * some versions of BLAS insist that: * - they are not smaller than the number of elements in the array, * - they are not 0. */ int Sz0 = (Nz0 > 1) ? (PyArray_STRIDES(%(Z)s)[0] / elemsize) : (Nz1 + 1); int Sz1 = (Nz1 > 1) ? (PyArray_STRIDES(%(Z)s)[1] / elemsize) : (Nz0 + 1); dtype_%(x)s* x_data = (dtype_%(x)s*) PyArray_DATA(%(x)s); dtype_%(y)s* y_data = (dtype_%(y)s*) PyArray_DATA(%(y)s); // gemv expects pointers to the beginning of memory arrays, // but numpy provides provides a pointer to the first element, // so when the stride is negative, we need to get the last one. if (Sx < 0) x_data += (Nz0 - 1) * Sx; if (Sy < 0) y_data += (Nz1 - 1) * Sy; if (PyArray_STRIDES(%(Z)s)[0] == elemsize) { if (PyArray_DESCR(%(Z)s)->type_num == NPY_FLOAT) { float alpha = ((dtype_%(a)s*)PyArray_DATA(%(a)s))[0]; sger_(&Nz0, &Nz1, &alpha, (float*)x_data, &Sx, (float*)y_data, &Sy, (float*)(PyArray_DATA(%(Z)s)), &Sz1); } else if (PyArray_DESCR(%(Z)s)->type_num == NPY_DOUBLE) { double alpha = ((dtype_%(a)s*)PyArray_DATA(%(a)s))[0]; dger_(&Nz0, &Nz1, &alpha, (double*)x_data, &Sx, (double*)y_data, &Sy, (double*)(PyArray_DATA(%(Z)s)), &Sz1); } else { PyErr_SetString(PyExc_NotImplementedError, "not float nor double"); %(fail)s } } else if (PyArray_STRIDES(%(Z)s)[1] == elemsize) { if (PyArray_DESCR(%(Z)s)->type_num == NPY_FLOAT) { float alpha = ((dtype_%(a)s*)(PyArray_DATA(%(a)s)))[0]; sger_(&Nz1, &Nz0, &alpha, (float*)y_data, &Sy, (float*)x_data, &Sx, (float*)(PyArray_DATA(%(Z)s)), &Sz0); } else if (PyArray_DESCR(%(Z)s)->type_num == NPY_DOUBLE) { double alpha = ((dtype_%(a)s*)PyArray_DATA(%(a)s))[0]; dger_(&Nz1, &Nz0, &alpha, (double*)y_data, &Sy, (double*)x_data, &Sx, (double*)(PyArray_DATA(%(Z)s)), &Sz0); } else { PyErr_SetString(PyExc_NotImplementedError, "not float nor double"); %(fail)s } } else { PyErr_SetString(PyExc_AssertionError, "A is a double-strided matrix, and should have been copied " "into a memory-contiguous one."); %(fail)s } } } """ % locals() class CGer(BaseBLAS, Ger): params_type = ParamsType(destructive=bool_t,) def c_code(self, node, name, inp, out, sub): A, a, x, y = inp Z, = out code = ger_c_code(A, a, x, y, Z, fail=sub['fail'], params=sub['params']) return code def c_code_cache_version(self): return (11, blas_header_version()) cger_inplace = CGer(True) cger_no_inplace = CGer(False) @local_optimizer([ger, ger_destructive]) def use_c_ger(node): if not config.blas.ldflags: return # Only float32 and float64 are supported for now. if (node.op == ger and node.outputs[0].dtype in ['float32', 'float64']): return [CGer(False)(*node.inputs)] if (node.op == ger_destructive and node.outputs[0].dtype in ['float32', 'float64']): return [CGer(True)(*node.inputs)] @local_optimizer([CGer(False)]) def make_c_ger_destructive(node): if isinstance(node.op, CGer) and not node.op.destructive: return [cger_inplace(*node.inputs)] # ##### ####### ####### # GEMV # ##### ####### ####### def gemv_c_code(y, A, x, z, alpha, beta, fail, force_init_beta=False, params=None): """ z <- beta * y + alpha * dot(A, x) where A is a matrix, y and x are vectors (ergo z is vector) """ code = """ int elemsize; float fbeta; double dbeta; if (PyArray_DIMS(%(A)s)[0] != PyArray_DIMS(%(y)s)[0]) { PyErr_SetString(PyExc_ValueError, "Shape mismatch: A.shape[0] != y.shape[0]"); %(fail)s; } if (PyArray_DIMS(%(A)s)[1] != PyArray_DIMS(%(x)s)[0]) { PyErr_SetString(PyExc_ValueError, "Shape mismatch: A.shape[1] != x.shape[0]"); %(fail)s; } if (PyArray_DESCR(%(y)s)->type_num == NPY_DOUBLE) { elemsize = 8; } else if (PyArray_DESCR(%(y)s)->type_num == NPY_FLOAT) { elemsize = 4;} else { PyErr_SetString(PyExc_NotImplementedError, "complex Gemv"); %(fail)s; } fbeta = dbeta = ((dtype_%(beta)s*)PyArray_DATA(%(beta)s))[0]; // copy y if not destructive if (!%(params)s->inplace) { if ((NULL == %(z)s) || (PyArray_DIMS(%(z)s)[0] != PyArray_DIMS(%(y)s)[0])) { Py_XDECREF(%(z)s); %(z)s = (PyArrayObject*)PyArray_SimpleNew(1, PyArray_DIMS(%(y)s), PyArray_TYPE(%(y)s)); if(!%(z)s) { PyErr_SetString(PyExc_MemoryError, "failed to alloc gemv output"); %(fail)s } } if (dbeta != 0) { if (PyArray_CopyInto(%(z)s, %(y)s) != 0) { %(fail)s } } else if (%(force_init_beta)d) { PyObject *zero = PyFloat_FromDouble(0.); if (zero == NULL) %(fail)s; if (PyArray_FillWithScalar(%(z)s, zero) != 0) %(fail)s; Py_DECREF(zero); } } else { if (%(z)s != %(y)s) { Py_XDECREF(%(z)s); %(z)s = %(y)s; Py_INCREF(%(z)s); } } { char TRANS = 'T'; char NOTRANS = 'N'; int NA0 = PyArray_DIMS(%(A)s)[0]; int NA1 = PyArray_DIMS(%(A)s)[1]; /* This formula is needed in the case where A is actually a row or * column matrix, because BLAS sometimes insists that the strides: * - are not smaller than the number of elements in the array * - are not 0. */ int SA0 = (NA0 > 1) ? (PyArray_STRIDES(%(A)s)[0] / elemsize) : (NA1 + 1); int SA1 = (NA1 > 1) ? (PyArray_STRIDES(%(A)s)[1] / elemsize) : (NA0 + 1); int Sz = PyArray_STRIDES(%(z)s)[0] / elemsize; int Sx = PyArray_STRIDES(%(x)s)[0] / elemsize; dtype_%(x)s* x_data = (dtype_%(x)s*) PyArray_DATA(%(x)s); dtype_%(z)s* z_data = (dtype_%(z)s*) PyArray_DATA(%(z)s); // gemv expects pointers to the beginning of memory arrays, // but numpy provides a pointer to the first element, // so when the stride is negative, we need to get the last one. if (Sx < 0) x_data += (NA1 - 1) * Sx; if (Sz < 0) z_data += (NA0 - 1) * Sz; if (NA0 * NA1) { // If A is neither C- nor F-contiguous, we make a copy. // TODO: // - if one stride is equal to "- elemsize", we can still call // gemv on reversed matrix and vectors // - if the copy is too long, maybe call vector/vector dot on // each row instead if ((PyArray_STRIDES(%(A)s)[0] < 0) || (PyArray_STRIDES(%(A)s)[1] < 0) || ((PyArray_STRIDES(%(A)s)[0] != elemsize) && (PyArray_STRIDES(%(A)s)[1] != elemsize))) { npy_intp dims[2]; dims[0] = NA0; dims[1] = NA1; PyArrayObject * A_copy = (PyArrayObject *) PyArray_Copy( %(A)s); if (!A_copy) %(fail)s Py_XDECREF(%(A)s); %(A)s = A_copy; SA0 = (NA0 > 1) ? (PyArray_STRIDES(%(A)s)[0] / elemsize) : (NA1 + 1); SA1 = (NA1 > 1) ? (PyArray_STRIDES(%(A)s)[1] / elemsize) : (NA0 + 1); } if (PyArray_STRIDES(%(A)s)[0] == elemsize) { if (PyArray_DESCR(%(A)s)->type_num == NPY_FLOAT) { float alpha = ((dtype_%(alpha)s*)PyArray_DATA(%(alpha)s))[0]; sgemv_(&NOTRANS, &NA0, &NA1, &alpha, (float*)(PyArray_DATA(%(A)s)), &SA1, (float*)x_data, &Sx, &fbeta, (float*)z_data, &Sz); } else if (PyArray_DESCR(%(A)s)->type_num == NPY_DOUBLE) { double alpha = ((dtype_%(alpha)s*)PyArray_DATA(%(alpha)s))[0]; dgemv_(&NOTRANS, &NA0, &NA1, &alpha, (double*)(PyArray_DATA(%(A)s)), &SA1, (double*)x_data, &Sx, &dbeta, (double*)z_data, &Sz); } else { PyErr_SetString(PyExc_AssertionError, "neither float nor double dtype"); %(fail)s } } else if (PyArray_STRIDES(%(A)s)[1] == elemsize) { if (PyArray_DESCR(%(A)s)->type_num == NPY_FLOAT) { float alpha = ((dtype_%(alpha)s*)PyArray_DATA(%(alpha)s))[0]; // Check for vector-vector dot (NA0 == 1). The code may work // for SA1 != 1 as well, but has not been tested for this case, // so SA1 == 1 is required for safety. if (NA0 == 1 && SA1 == 1) { if (fbeta != 0.f) { z_data[0] = fbeta*z_data[0]; } else { z_data[0] = 0.f; } z_data[0] += alpha*sdot_(&NA1, (float*)(PyArray_DATA(%(A)s)), &SA1, (float*)x_data, &Sx); } else { sgemv_(&TRANS, &NA1, &NA0, &alpha, (float*)(PyArray_DATA(%(A)s)), &SA0, (float*)x_data, &Sx, &fbeta, (float*)z_data, &Sz); } } else if (PyArray_DESCR(%(A)s)->type_num == NPY_DOUBLE) { double alpha = ((dtype_%(alpha)s*)PyArray_DATA(%(alpha)s))[0]; // Check for vector-vector dot (NA0 == 1). The code may work // for SA1 != 1 as well, but has not been tested for this case, // so SA1 == 1 is required for safety. if (NA0 == 1 && SA1 == 1) { if (dbeta != 0.) { z_data[0] = dbeta*z_data[0]; } else { z_data[0] = 0.; } z_data[0] += alpha*ddot_(&NA1, (double*)(PyArray_DATA(%(A)s)), &SA1, (double*)x_data, &Sx); } else { dgemv_(&TRANS, &NA1, &NA0, &alpha, (double*)(PyArray_DATA(%(A)s)), &SA0, (double*)x_data, &Sx, &dbeta, (double*)z_data, &Sz); } } else { PyErr_SetString(PyExc_AssertionError, "neither float nor double dtype"); %(fail)s } } else { PyErr_SetString(PyExc_AssertionError, "xx is a double-strided matrix, and should have been " "copied into a memory-contiguous one."); %(fail)s } } else if (dbeta != 1.0) { // the matrix has at least one dim of length 0 // so we do this loop, which either iterates over 0 elements // or else it does the right thing for length-0 A. dtype_%(z)s * zptr = (dtype_%(z)s*)(PyArray_DATA(%(z)s)); for (int i = 0; i < NA0; ++i) { zptr[i * Sz] = (dbeta == 0.0 ? 0.0 : zptr[i * Sz] * dbeta); } } } """ return code % locals() class CGemv(BaseBLAS, Gemv): params_type = ParamsType(inplace=bool_t,) def __init__(self, inplace): super(CGemv, self).__init__(inplace) def c_code(self, node, name, inp, out, sub): y, alpha, A, x, beta = inp z, = out code = gemv_c_code( y, A, x, z, alpha, beta, fail=sub['fail'], force_init_beta=check_force_gemv_init(), params=sub['params'], ) return code def c_code_cache_version(self): return (14, blas_header_version(), check_force_gemv_init()) cgemv_inplace = CGemv(inplace=True) cgemv_no_inplace = CGemv(inplace=False) def check_force_gemv_init(): if check_force_gemv_init._force_init_beta is None: from theano.gof.cmodule import GCC_compiler """ Test issue 1569. Namely when evaluating beta*y + alpha*dot(A, x) where we set y * beta = zeros of the correct dimensions we do not actually set y = zeros and instead let the BLAS perform beta*y with uninitialized memory for speed. Occasionally the memory contains values that are equivalent to NaN in which case the product beta*y contains NaN's for correctly implemented BLAS libraries. In this situation, since we are introducing the NaN's, we need to test whether the BLAS performs correctly. If it *does*, i.e. it actually performs the multiplication beta*y which will result in NaN's in the result, then we need intialize the memory to zeros. """ test_code = """ #include extern "C" void dgemv_(char*, const int*, const int*, const double *, const double *, const int*, const double *, const int*, const double *, double *, const int *); int main() { double A[2][2] = {{1., 1.}, {1., 1.}}; double x[2] = {1., 1.}; double y[2] = {NAN, NAN}; const int s = 2; const int inc = 1; const double alpha = 1.0; const double beta = 0.0; dgemv_("T", &s, &s, &alpha, A, &s, x, &inc, &beta, &y, &inc); return (isnan(y[0]) || isnan(y[1]) ? 1 : 0; } """ res = GCC_compiler.try_compile_tmp(test_code, tmp_prefix='check_beta_', flags=ldflags(libs=True, flags=True, libs_dir=True), try_run=True) if res: if res[0]: check_force_gemv_init._force_init_beta = res[1] else: check_force_gemv_init._force_init_beta = False else: check_force_gemv_init._force_init_beta = False return check_force_gemv_init._force_init_beta check_force_gemv_init._force_init_beta = None @local_optimizer([gemv_inplace, gemv_no_inplace]) def use_c_gemv(node): if not config.blas.ldflags: return # Only float32 and float64 are supported for now. if (node.op == gemv_no_inplace and node.outputs[0].dtype in ['float32', 'float64']): return [cgemv_no_inplace(*node.inputs)] if (node.op == gemv_inplace and node.outputs[0].dtype in ['float32', 'float64']): return [cgemv_inplace(*node.inputs)] @local_optimizer([CGemv(inplace=False)]) def make_c_gemv_destructive(node): if isinstance(node.op, CGemv) and not node.op.inplace: inputs = list(node.inputs) dest = inputs[0] if (dest.owner and isinstance(dest.owner.op, T.AllocEmpty) and len(dest.clients) > 1): inputs[0] = T.AllocEmpty(dest.dtype)(*dest.owner.inputs) return [cgemv_inplace(*inputs)] # ##### ####### ####### # Optimizers # ##### ####### ####### blas_optdb.register('use_c_blas', in2out(use_c_ger, use_c_gemv), 20, 'fast_run', 'c_blas') # this matches the InplaceBlasOpt defined in blas.py optdb.register('c_blas_destructive', in2out(make_c_ger_destructive, make_c_gemv_destructive, name="c_blas_destructive"), 70.0, 'fast_run', 'inplace', 'c_blas') Theano-1.0.0/theano/tensor/fourier.py00006000163562€ÿÿÿþ0000001457013203335532021113 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import numpy as np import math from theano import gof, tensor class Fourier(gof.Op): """ WARNING: for officially supported FFTs, use theano.tensor.fft, which provides real-input FFTs. Gradients are supported, as well as optimization transfers to GPU ops. An instance of this class returns a finite fourier transform calcutated along one dimension of an input array. inputs: a : Array of at least one dimension. Can be complex. n : Integer, optional. Length of the transformed axis of the output. If n is smaller than the length of the input, the input is cropped. If it is larger, the input is padded with zeros. If n is not given, the length of the input (along the axis specified by axis) is used. axis : Integer, optional. Axis over which to compute the FFT. If not supplied, the last axis is used. output: Complex array. The input array, transformed along the axis indicated by 'axis' or along the last axis if 'axis' is not specified. It is truncated or zero-padded as required if 'n' is specified. (From numpy.fft.fft's documentation:) The values in the output follow so-called standard order. If A = fft(a, n), then A[0] contains the zero-frequency term (the mean of the signal), which is always purely real for real inputs. Then A[1:n/2] contains the positive-frequency terms, and A[n/2+1:] contains the negative-frequency terms, in order of decreasingly negative frequency. For an even number of input points, A[n/2] represents both positive and negative Nyquist frequency, and is also purely real for real input. For an odd number of input points, A[(n-1)/2] contains the largest positive frequency, while A[(n+1)/2] contains the largest negative frequency. """ __props__ = () def make_node(self, a, n, axis): a = tensor.as_tensor_variable(a) if a.ndim < 1: raise TypeError('%s: input must be an array, not a scalar' % self.__class__.__name__) if axis is None: axis = a.ndim - 1 axis = tensor.as_tensor_variable(axis) else: axis = tensor.as_tensor_variable(axis) if axis.dtype not in tensor.integer_dtypes: raise TypeError('%s: index of the transformed axis must be' ' of type integer' % self.__class__.__name__) elif axis.ndim != 0 or (isinstance(axis, tensor.TensorConstant) and (axis.data < 0 or axis.data > a.ndim - 1)): raise TypeError('%s: index of the transformed axis must be' ' a scalar not smaller than 0 and smaller than' ' dimension of array' % self.__class__.__name__) if n is None: n = a.shape[axis] n = tensor.as_tensor_variable(n) else: n = tensor.as_tensor_variable(n) if n.dtype not in tensor.integer_dtypes: raise TypeError('%s: length of the transformed axis must be' ' of type integer' % self.__class__.__name__) elif n.ndim != 0 or (isinstance(n, tensor.TensorConstant) and n.data < 1): raise TypeError('%s: length of the transformed axis must be a' ' strictly positive scalar' % self.__class__.__name__) return gof.Apply(self, [a, n, axis], [tensor.TensorType('complex128', a.type.broadcastable)()]) def infer_shape(self, node, in_shapes): shape_a = in_shapes[0] n = node.inputs[1] axis = node.inputs[2] if len(shape_a) == 1: return [(n,)] elif isinstance(axis, tensor.TensorConstant): out_shape = (list(shape_a[0: axis.data.item()]) + [n] + list(shape_a[axis.data + 1:])) else: l = len(shape_a) shape_a = tensor.stack(shape_a) out_shape = tensor.concatenate((shape_a[0: axis], [n], shape_a[axis + 1:])) n_splits = [1] * l out_shape = tensor.split(out_shape, n_splits, l) out_shape = [a[0] for a in out_shape] return [out_shape] def perform(self, node, inputs, output_storage): a = inputs[0] n = inputs[1] axis = inputs[2] output_storage[0][0] = np.fft.fft(a, n=int(n), axis=axis.item()) def grad(self, inputs, cost_grad): """ In defining the gradient, the Finite Fourier Transform is viewed as a complex-differentiable function of a complex variable """ a = inputs[0] n = inputs[1] axis = inputs[2] grad = cost_grad[0] if not isinstance(axis, tensor.TensorConstant): raise NotImplementedError('%s: gradient is currently implemented' ' only for axis being a Theano constant' % self.__class__.__name__) axis = int(axis.data) # notice that the number of actual elements in wrto is independent of # possible padding or truncation: elem = tensor.arange(0, tensor.shape(a)[axis], 1) # accounts for padding: freq = tensor.arange(0, n, 1) outer = tensor.outer(freq, elem) pow_outer = tensor.exp(((-2 * math.pi * 1j) * outer) / (1. * n)) res = tensor.tensordot(grad, pow_outer, (axis, 0)) # This would be simpler but not implemented by theano: # res = tensor.switch(tensor.lt(n, tensor.shape(a)[axis]), # tensor.set_subtensor(res[...,n::], 0, False, False), res) # Instead we resort to that to account for truncation: flip_shape = list(np.arange(0, a.ndim)[::-1]) res = res.dimshuffle(flip_shape) res = tensor.switch(tensor.lt(n, tensor.shape(a)[axis]), tensor.set_subtensor(res[n::, ], 0, False, False), res) res = res.dimshuffle(flip_shape) # insures that gradient shape conforms to input shape: out_shape = list(np.arange(0, axis)) + [a.ndim - 1] +\ list(np.arange(axis, a.ndim - 1)) res = res.dimshuffle(*out_shape) return [res, None, None] fft = Fourier() Theano-1.0.0/theano/tensor/opt.py00006000163562€ÿÿÿþ0000111534713203335532020247 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division """ Tensor optimizations addressing the ops in basic.py. """ # TODO: intelligent merge for mul/add # TODO: 0*x -> 0 from collections import defaultdict import logging import itertools import operator import sys import time import traceback import warnings import numpy as np from six import integer_types, iteritems from six.moves import reduce, xrange import theano from theano import gof from theano.compat import izip from theano.gof import opt, InconsistencyError, TopoOptimizer, graph from theano.gof import Variable, Constant from theano.gof.opt import copy_stack_trace, in2out from theano.gof.utils import MethodNotDefined from theano.gradient import DisconnectedType from theano import config from theano.tensor.elemwise import Elemwise, DimShuffle from theano.tensor.subtensor import (get_idx_list, get_canonical_form_slice, Subtensor, IncSubtensor, make_constant, AdvancedIncSubtensor1, AdvancedIncSubtensor, AdvancedSubtensor1, advanced_subtensor, advanced_subtensor1, advanced_inc_subtensor1) from theano.tensor.sort import TopKOp from theano import scalar from theano.scalar import basic from theano.tensor import basic as T from theano import compile # to register the optimizer built by this file from theano.compile.ops import Shape, Shape_i from theano.tensor.type import (values_eq_approx_remove_inf, values_eq_approx_remove_nan, values_eq_approx_remove_inf_nan) from theano.gof.opt import (Optimizer, pre_constant_merge, pre_greedy_local_optimizer) from theano.gof import toolbox from theano.tensor.basic import (Alloc, get_scalar_constant_value, ShapeError, extract_constant, NotScalarConstantError, Reshape) from six import StringIO _logger = logging.getLogger('theano.tensor.opt') # Utilities def _fill_chain(new_out, orig_inputs): for i in orig_inputs: new_out = T.fill(i, new_out) return [new_out] def encompasses_broadcastable(b1, b2): """ Parameters ---------- b1 The broadcastable attribute of a tensor type. b2 The broadcastable attribute of a tensor type. Returns ------- bool True if the broadcastable patterns b1 and b2 are such that b2 is broadcasted to b1's shape and not the opposite. """ if len(b1) < len(b2): return False b1 = b1[-len(b2):] return not any(v1 and not v2 for v1, v2 in zip(b1, b2)) def merge_broadcastables(broadcastables): return [all(bcast) for bcast in zip(*broadcastables)] def scalarconsts_rest(inputs, elemwise=True, only_process_constants=False): """Partition a list of variables into two kinds: scalar constants, and the rest.""" consts = [] origconsts = [] nonconsts = [] for i in inputs: try: v = get_scalar_constant_value(i, elemwise=elemwise, only_process_constants=only_process_constants) consts.append(v) origconsts.append(i) except NotScalarConstantError: nonconsts.append(i) return consts, origconsts, nonconsts def broadcast_like(value, template, fgraph, dtype=None): """ Return a Variable with the same shape and dtype as the template, filled by broadcasting value through it. `value` will be cast as necessary. """ value = T.as_tensor_variable(value) if value.type == template.type: return value if template not in fgraph.variables: raise NotImplementedError('broadcast_like currently requires the ' 'template Variable to be in the fgraph already') if dtype is None: dtype = template.dtype value = T.cast(value, dtype) if value.type == template.type: return value if hasattr(fgraph, 'shape_feature'): new_shape = fgraph.shape_feature.shape_of[template] else: new_shape = template.shape rval = T.alloc(value, *new_shape) # the template may have 1s in its shape without being broadcastable if rval.broadcastable != template.broadcastable: rval = T.unbroadcast(rval, *[i for i in xrange(rval.ndim) if rval.broadcastable[i] and not template.broadcastable[i]]) assert rval.type.dtype == dtype if rval.type.broadcastable != template.broadcastable: raise AssertionError("rval.type.broadcastable is " + str(rval.type.broadcastable) + " but template.broadcastable is" + str(template.broadcastable)) return rval class InplaceElemwiseOptimizer(Optimizer): """ We parametrise it to make it work for Elemwise and GpuElemwise op. """ def __init__(self, OP): self.op = OP def add_requirements(self, fgraph): fgraph.attach_feature(theano.gof.destroyhandler.DestroyHandler()) @staticmethod def print_profile(stream, prof, level=0): blanc = (' ' * level) print(blanc, "InplaceElemwiseOptimizer ", prof['opt'].op, file=stream) for k in ['node_before', 'nb_call_replace', 'nb_call_validate', 'nb_inconsistent']: print(blanc, k, prof[k], file=stream) ndim = prof['ndim'] if ndim: print(blanc, "ndim", "nb", file=stream) for n in sorted(ndim.keys()): print(blanc, n, ndim[n], file=stream) def apply(self, fgraph): """ Usage: InplaceElemwiseOptimizer(op).optimize(fgraph) Attempts to replace all Broadcast ops by versions of them that operate inplace. It operates greedily: for each Broadcast Op that is encountered, for each output, tries each input to see if it can operate inplace on that input. If so, makes the change and go to the next output or Broadcast Op. Examples -------- `x + y + z -> x += y += z` `(x + y) * (x * y) -> (x += y) *= (x * y) or (x + y) *= (x *= y)` """ # We should not validate too often as this takes too much time to # execute! # It is the _dfs_toposort() fct in theano/gof/destroyhandler.py # that takes so much time. # Should we try to use another lib that does toposort? # igraph: http://igraph.sourceforge.net/ # networkx: https://networkx.lanl.gov/ # Should we try to use cython? # Compiling only that fct is not enough, should we try to add the # deque class too? # And init the deque and other list to an upper bound number of # elements? # Maybe Theano should do online toposort as in # http://code.google.com/p/acyclic # # The next longest optimizer is the canonizer phase. # Then I think it is the [io_?]toposort (need to validate) so check if # the solution is also applicable there. # We execute `validate` after this number of change. prof = {'opt': self, 'node_before': len(fgraph.apply_nodes), 'nb_call_replace': 0, 'nb_call_validate': 0, 'nb_inconsistent': 0, 'ndim': defaultdict(lambda: 0)} check_each_change = config.tensor.insert_inplace_optimizer_validate_nb if check_each_change == -1: if len(fgraph.apply_nodes) > 500: check_each_change = 10 else: check_each_change = 1 nb_change_no_validate = 0 chk = fgraph.checkpoint() if fgraph.update_mapping: update_outs = [fgraph.outputs[i] for i in fgraph.update_mapping] else: update_outs = [] protected_inputs = [ f.protected for f in fgraph._features if isinstance(f, theano.compile.function_module.Supervisor)] protected_inputs = sum(protected_inputs, []) # flatten the list protected_inputs.extend(fgraph.outputs) for node in list(graph.io_toposort(fgraph.inputs, fgraph.outputs)): op = node.op # gpuarray GpuElemwise inherit from Elemwise if not type(op) == self.op: continue # If big graph and the outputs are scalar, do not make it # inplace. if (check_each_change != 1 and # If multiple outputs, they must all have the same size, # so only check the first. getattr(node.outputs[0].type, 'ndim', -1) == 0): continue if op.inplace_pattern: # Maybe this isn't needed anymore, but I don't want to # rish regression now. This case only happen if the # original node add already some inplace patter and we # still try to add more pattern. baseline = op.inplace_pattern candidate_outputs = [i for i in xrange(len(node.outputs)) if i not in baseline] # node inputs that are Constant, already destroyed, # or fgraph protected inputs and fgraph outputs can't be used as # inplace target. # Remove here as faster. candidate_inputs = [i for i in xrange(len(node.inputs)) if i not in baseline.values() and not isinstance(node.inputs[i], Constant) and # the next line should not be costly most of the time. not fgraph.has_destroyers([node.inputs[i]]) and node.inputs[i] not in protected_inputs] else: baseline = [] candidate_outputs = list(range(len(node.outputs))) # node inputs that are Constant, already destroyed, # fgraph protected inputs and fgraph outputs can't be used as inplace # target. # Remove here as faster. candidate_inputs = [i for i in xrange(len(node.inputs)) if not isinstance(node.inputs[i], Constant) and not fgraph.has_destroyers([node.inputs[i]]) and node.inputs[i] not in protected_inputs] verbose = False raised_warning = not verbose for candidate_output in candidate_outputs: # If the output of the node can be established as an update # output of the fgraph, visit the candidate_inputs in an order # that will improve the chances of making the node operate # inplace on the input it's meant to update candidate_out_var = node.outputs[candidate_output] sorted_candidate_inputs = candidate_inputs if candidate_out_var in update_outs: # The candidate output is an update. Sort the # variables in candidate_inputs in the following order: # - Vars corresponding to the actual updated input # (best case scenario is for the node that procudes # an update to operate inplace on the variable to # update) # - Vars computed inplace on the updates input (second # best scenario if for the node to work inplace on # a variable obtained by a chain of inplace on the # variable to update. In some cases, this will be # equivalent to operating inplace on the variable to # update) # - Remaining variables updated_inputs = [] for i, f_out in enumerate(fgraph.outputs): if (f_out is candidate_out_var and i in fgraph.update_mapping): updated_inp_idx = fgraph.update_mapping[i] updated_inputs.append(fgraph.inputs[updated_inp_idx]) updated_vars = [] vars_from_inplace = [] other_vars = [] for inp_idx in candidate_inputs: inp = node.inputs[inp_idx] if inp in updated_inputs: # the candidate input is the actual updated input updated_vars.append(inp_idx) elif (hasattr(fgraph, 'destroy_handler') and inp.owner and any([fgraph.destroy_handler.root_destroyer.get(up_inp, None) is inp.owner for up_inp in updated_inputs])): # the candidate input is a variable computed # inplace on the updated input via a sequence of # one or more inplace operations vars_from_inplace.append(inp_idx) else: other_vars.append(inp_idx) sorted_candidate_inputs = (updated_vars + vars_from_inplace + other_vars) for candidate_input in sorted_candidate_inputs: # remove inputs that don't have the same dtype as the output if node.inputs[candidate_input].type != node.outputs[ candidate_output].type: continue inplace_pattern = dict(baseline) inplace_pattern[candidate_output] = candidate_input try: if hasattr(op.scalar_op, "make_new_inplace"): new_scal = op.scalar_op.make_new_inplace( scalar.transfer_type( *[inplace_pattern.get(i, o.dtype) for i, o in enumerate(node.outputs)])) else: new_scal = op.scalar_op.__class__( scalar.transfer_type( *[inplace_pattern.get(i, None) for i in xrange(len(node.outputs))])) new_outputs = self.op(new_scal, inplace_pattern)( *node.inputs, **dict(return_list=True)) new_node = new_outputs[0].owner for r, new_r in zip(node.outputs, new_outputs): prof['nb_call_replace'] += 1 fgraph.replace(r, new_r, reason="inplace_elemwise_optimizer") nb_change_no_validate += 1 prof['ndim'][candidate_out_var.ndim] += 1 if nb_change_no_validate >= check_each_change: prof['nb_call_validate'] += 1 fgraph.validate() chk = fgraph.checkpoint() nb_change_no_validate = 0 except (ValueError, InconsistencyError) as e: prof['nb_inconsistent'] += 1 if check_each_change != 1 and not raised_warning: print(("Some inplace optimization was not " "performed due to unexpected error:"), file=sys.stderr) print(e, file=sys.stderr) raised_warning = True fgraph.revert(chk) continue candidate_inputs.remove(candidate_input) node = new_node baseline = inplace_pattern break if nb_change_no_validate > 0: try: fgraph.validate() except Exception: if not raised_warning: print(("Some inplace optimization was not " "performed due to unexpected error"), file=sys.stderr) fgraph.revert(chk) return prof def print_summary(self, stream=sys.stdout, level=0, depth=-1): print("%s%s (%s)" % ( (' ' * level), self.__class__.__name__, self.op), file=stream) return inplace_elemwise_optimizer inplace_elemwise_optimizer = InplaceElemwiseOptimizer(T.Elemwise) compile.optdb.register('inplace_elemwise_opt', inplace_elemwise_optimizer, 75, 'inplace_opt', # for historic reason 'inplace_elemwise_optimizer', 'fast_run', 'inplace') def register_useless(lopt, *tags, **kwargs): if type(lopt) == str: def register(inner_lopt): return register_useless(inner_lopt, lopt, *tags, **kwargs) return register else: name = kwargs.pop('name', None) or lopt.__name__ compile.mode.local_useless.register(name, lopt, 'last', 'fast_run', *tags, **kwargs) return lopt def register_canonicalize(lopt, *tags, **kwargs): if type(lopt) == str: def register(inner_lopt): return register_canonicalize(inner_lopt, lopt, *tags, **kwargs) return register else: name = kwargs.pop('name', None) or lopt.__name__ compile.optdb['canonicalize'].register(name, lopt, 'fast_run', *tags, **kwargs) return lopt def register_stabilize(lopt, *tags, **kwargs): if type(lopt) == str: def register(inner_lopt): return register_stabilize(inner_lopt, lopt, *tags, **kwargs) return register else: name = kwargs.pop('name', None) or lopt.__name__ compile.optdb['stabilize'].register(name, lopt, 'fast_run', *tags, **kwargs) return lopt def register_specialize(lopt, *tags, **kwargs): if type(lopt) == str: def register(inner_lopt): return register_specialize(inner_lopt, lopt, *tags, **kwargs) return register else: name = kwargs.pop('name', None) or lopt.__name__ compile.optdb['specialize'].register(name, lopt, 'fast_run', *tags, **kwargs) return lopt def register_uncanonicalize(lopt, *tags, **kwargs): if type(lopt) == str: def register(inner_lopt): return register_uncanonicalize(inner_lopt, lopt, *tags, **kwargs) return register else: name = (kwargs and kwargs.pop('name', None)) or lopt.__name__ compile.optdb['uncanonicalize'].register(name, lopt, 'fast_run', *tags, **kwargs) return lopt def register_specialize_device(lopt, *tags, **kwargs): if type(lopt) == str: def register(inner_lopt): return register_specialize_device(inner_lopt, lopt, *tags, **kwargs) return register else: name = (kwargs and kwargs.pop('name', None)) or lopt.__name__ compile.optdb['specialize_device'].register(name, lopt, 'fast_run', *tags, **kwargs) return lopt ##################### # Dot optimizations # ##################### @register_canonicalize @register_stabilize @gof.local_optimizer([T.Dot]) def local_0_dot_x(node): if not isinstance(node.op, T.Dot): return False x = node.inputs[0] y = node.inputs[1] replace = False try: if get_scalar_constant_value(x, only_process_constants=True) == 0: replace = True except NotScalarConstantError: pass try: if get_scalar_constant_value(y, only_process_constants=True) == 0: replace = True except NotScalarConstantError: pass if replace: constant_zero = T.constant(0, dtype=node.outputs[0].type.dtype) if x.ndim == 2 and y.ndim == 2: constant_zero = assert_(constant_zero, T.eq(x.shape[1], y.shape[0])) return [T.alloc(constant_zero, x.shape[0], y.shape[1])] elif x.ndim == 1 and y.ndim == 2: constant_zero = assert_(constant_zero, T.eq(x.shape[0], y.shape[0])) return [T.alloc(constant_zero, y.shape[1])] elif x.ndim == 2 and y.ndim == 1: constant_zero = assert_(constant_zero, T.eq(x.shape[1], y.shape[0])) return [T.alloc(constant_zero, x.shape[0])] elif x.ndim == 1 and y.ndim == 1: constant_zero = assert_(constant_zero, T.eq(x.shape[0], y.shape[0])) return [constant_zero] else: _logger.warning("Optimization Warning: " "Optimization theano/opt.py:local_0_dot_x Found " "that it could apply, but was not implemented " "for dot product with these input types:\n" "(%s, %s)", x.type, y.type) ###################### # DimShuffle lifters # ###################### def apply_local_dimshuffle_lift(var): # return var # lift recursively if not var.owner: return var new = local_dimshuffle_lift.transform(var.owner) if new: return new[0] return var # Checks for two types of useless dimshuffles: # 1 - dimshuffle all dimensions in order. # 2 - dimshuffle a broadcastable dimension. def is_dimshuffle_useless(new_order, input): is_useless = True if len(new_order) == input.type.ndim: all_broadcastable_dims = [i for (i, is_broadcastable) in enumerate(input.type.broadcastable) if is_broadcastable] + ['x'] for i in range(input.type.ndim): if (new_order[i] == i or (i in all_broadcastable_dims and new_order[i] in all_broadcastable_dims)): is_useless = True else: is_useless = False break else: is_useless = False return is_useless @gof.local_optimizer([DimShuffle]) def local_dimshuffle_lift(node): """ "Lifts" DimShuffle through Elemwise operations and merges consecutive DimShuffles. Basically, applies the following transformations on the whole graph: DimShuffle(Elemwise(x, y)) => Elemwise(DimShuffle(x), DimShuffle(y)) DimShuffle(DimShuffle(x)) => DimShuffle(x) DimShuffle{0,1,...}(x) => x (when the dimshuffle do nothing) After this transform, clusters of Elemwise operations are void of DimShuffle operations. """ op = node.op if not isinstance(op, DimShuffle): return False input = node.inputs[0] inode = input.owner new_order = op.new_order if inode and isinstance(inode.op, Elemwise) and (len(input.clients) == 1): # Don't use make_node to have tag.test_value set. new_inputs = [] for inp in inode.inputs: new_inp = op.__class__(inp.type.broadcastable, op.new_order)(inp) new_inputs.append(apply_local_dimshuffle_lift(new_inp)) copy_stack_trace(node.outputs[0], new_inputs) ret = inode.op(*new_inputs, **dict(return_list=True)) return ret if inode and isinstance(inode.op, DimShuffle): new_order = [x == 'x' and 'x' or inode.op.new_order[x] for x in new_order] input = inode.inputs[0] if is_dimshuffle_useless(new_order, input): return [input] elif inode and isinstance(inode.op, DimShuffle): ret = op.__class__(input.type.broadcastable, new_order)(input) ret = apply_local_dimshuffle_lift(ret) copy_stack_trace(node.outputs[0], ret) return [ret] @register_canonicalize @gof.local_optimizer([Reshape]) def local_useless_dimshuffle_in_reshape(node): """ Removes useless DimShuffle operation inside Reshape: reshape(vector.dimshuffle('x', 0), shp) => reshape(vector, shp) reshape(matrix.dimshuffle('x', 0, 'x', 1), shp) => reshape(matrix, shp) reshape(row.dimshuffle(1, 'x'), shp) => reshape(row, shp) reshape(col.dimshuffle(0), shp) => reshape(col, shp) """ op = node.op if not isinstance(op, Reshape): return False if not (node.inputs[0].owner is not None and isinstance(node.inputs[0].owner.op, DimShuffle)): return False new_order = node.inputs[0].owner.op.new_order input = node.inputs[0].owner.inputs[0] broadcastables = node.inputs[0].broadcastable new_order_of_nonbroadcast = [] for i, bd in zip(new_order, broadcastables): if not bd: new_order_of_nonbroadcast.append(i) no_change_in_order = all( new_order_of_nonbroadcast[i] <= new_order_of_nonbroadcast[i + 1] for i in xrange(len(new_order_of_nonbroadcast) - 1)) if no_change_in_order: shape = node.inputs[1] ret = op.__class__(node.outputs[0].ndim)(input, shape) copy_stack_trace(node.outputs[0], ret) return [ret] @register_canonicalize @gof.local_optimizer([DimShuffle]) def local_lift_transpose_through_dot(node): """ dot(x,y).T -> dot(y.T, x.T) These optimizations "lift" (propagate towards the inputs) DimShuffle through dot product. It allows to put the graph in a more standard shape, and to later merge consecutive DimShuffles. The transformation should be apply whether or not the transpose is inplace. The newly-introduced transpositions are not inplace, this will be taken care of in a later optimization phase. """ if not (isinstance(node.op, T.DimShuffle) and node.op.new_order == (1, 0)): return False if not (node.inputs[0].owner and isinstance(node.inputs[0].owner.op, T.Dot)): return False x, y = node.inputs[0].owner.inputs if x.ndim == y.ndim == 2: # Output is dot product of transposed inputs in reverse order ret = [T.dot(y.T, x.T)] # Copy over stack trace to output from result of dot-product copy_stack_trace(node.inputs[0], ret) return ret register_canonicalize(local_dimshuffle_lift) register_specialize(local_dimshuffle_lift) ###################### # Casting operations # ###################### @register_canonicalize @register_specialize @gof.local_optimizer([T.TensorFromScalar]) def local_tensor_scalar_tensor(node): '''tensor_from_scalar(scalar_from_tensor(x)) -> x''' if isinstance(node.op, T.TensorFromScalar): s = node.inputs[0] if s.owner and isinstance(s.owner.op, T.ScalarFromTensor): t = s.owner.inputs[0] # We don't need to copy over any stack traces here return [t] @register_canonicalize @register_specialize @gof.local_optimizer([T.ScalarFromTensor]) def local_scalar_tensor_scalar(node): '''scalar_from_tensor(tensor_from_scalar(x)) -> x''' if isinstance(node.op, T.ScalarFromTensor): t = node.inputs[0] if t.owner and isinstance(t.owner.op, T.TensorFromScalar): s = t.owner.inputs[0] # We don't need to copy over any stack traces here return [s] ##################################### # ShapeFeature, Shape optimizations ##################################### class MakeVector(T.Op): """Concatenate a number of scalars together into a vector. This is a simple version of stack() that introduces far less cruft into the graph. Should work with 0 inputs. The constant_folding optimization will remove it. """ __props__ = ("dtype",) def __init__(self, dtype='int64'): self.dtype = dtype def make_node(self, *inputs): inputs = list(map(T.as_tensor_variable, inputs)) if (not all(a.type == inputs[0].type for a in inputs) or (len(inputs) > 0 and inputs[0].dtype != self.dtype)): dtype = theano.scalar.upcast(self.dtype, *[i.dtype for i in inputs]) # upcast the input to the determined dtype, # but don't downcast anything assert dtype == self.dtype, ( "The upcast of the inputs to MakeVector should match the " "dtype given in __init__.") if not all(self.dtype == T.cast(i, dtype=dtype).dtype for i in inputs): raise TypeError("MakeVector.make_node expected inputs" " upcastable to %s. got %s" % (self.dtype, str([i.dtype for i in inputs]))) inputs = [T.cast(i, dtype=dtype) for i in inputs] assert all(self.dtype == a.dtype for a in inputs) assert all(a.ndim == 0 for a in inputs) if inputs: dtype = inputs[0].type.dtype else: dtype = self.dtype # bcastable = (len(inputs) == 1) bcastable = False otype = T.TensorType(broadcastable=(bcastable,), dtype=dtype) return T.Apply(self, inputs, [otype()]) def perform(self, node, inputs, out_): out, = out_ # not calling theano._asarray as optimization if (out[0] is None) or (out[0].size != len(inputs)): out[0] = theano._asarray(inputs, dtype=node.outputs[0].dtype) else: # assume that out has correct dtype. there is no cheap way to check out[0][...] = inputs def c_code_cache_version(self): return (2,) def c_code(self, node, name, inp, out_, sub): out, = out_ # Shouldn't use PyArray_TYPE(inp[0]) for the dtype # when len(inp) == 0 (we need to support this case. # So there will be (1 * nb_dtype) + ((nb len(inp) - 1 )) # different c code with the following algo out_shape = len(inp) out_num = np.dtype(node.outputs[0].dtype).num # don't use dtype_%(out)s as when check_input=False, it isn't defined. out_dtype = node.outputs[0].type.dtype_specs()[1] if len(inp) > 0: assert self.dtype == node.inputs[0].dtype out_num = 'PyArray_TYPE(%s)' % inp[0] ret = """ npy_intp dims[1]; dims[0] = %(out_shape)s; if(!%(out)s || PyArray_DIMS(%(out)s)[0] != %(out_shape)s){ Py_XDECREF(%(out)s); %(out)s = (PyArrayObject*)PyArray_EMPTY(1, dims, %(out_num)s, 0); } """ % locals() for idx, i in enumerate(inp): ret += """ *((%(out_dtype)s *)PyArray_GETPTR1(%(out)s, %(idx)s)) = *((%(out_dtype)s *) PyArray_DATA(%(i)s)); """ % locals() return ret def infer_shape(self, node, ishapes): return [(len(ishapes),)] def grad(self, inputs, output_gradients): # If the output is of an integer dtype, no gradient shall pass if self.dtype in theano.tensor.discrete_dtypes: return [ipt.zeros_like().astype(theano.config.floatX) for ipt in inputs] grads = [] for i, inp in enumerate(inputs): grads.append(output_gradients[0][i]) return grads def R_op(self, inputs, eval_points): if None in eval_points: return [None] return self.make_node(*eval_points).outputs make_vector = MakeVector() class MakeVectorPrinter: def process(self, r, pstate): if r.owner is None: raise TypeError("Can only print make_vector.") elif isinstance(r.owner.op, MakeVector): old_precedence = getattr(pstate, 'precedence', None) try: pstate.precedence = 1000 s = [pstate.pprinter.process(input) for input in r.owner.inputs] finally: pstate.precedence = old_precedence return "[%s]" % ", ".join(s) else: raise TypeError("Can only print make_vector.") T.pprint.assign(MakeVector, MakeVectorPrinter()) class ShapeFeature(object): """Graph optimizer for removing all calls to shape(). This optimizer replaces all Shapes and Subtensors of Shapes with Shape_i and MakeVector Ops. This optimizer has several goals: 1. to 'lift' Shapes to as close to the inputs as possible. 2. to infer the shape of every node in the graph in terms of the input shapes. 3. remove all fills (T.second, T.fill) from the graph Lifting shapes as close to the inputs as possible is important for canonicalization because it is very bad form to have to compute something just to know how big it will be. Firstly, it is a waste of time to compute such outputs. But it is important to get rid of these outputs as early as possible in the compilation process because the extra computations make it appear as if many internal graph nodes have multiple clients. Many optimizations refuse to work on nodes with multiple clients. Lifting is done by using an `.infer_shape` function if one is present, or else using a conservative default. An Op that supports shape-lifting should define a infer_shape(self, node, input_shapes) function. The argument input_shapes is a tuple of tuples... there is an interior tuple for each input to the node. The tuple has as many elements as dimensions. The element in position i of tuple j represents the i'th shape component of the j'th input. The function should return a tuple of tuples. One output tuple for each node.output. Again, the i'th element of the j'th output tuple represents the output[j].shape[i] of the function. If an output is not a TensorType, then None should be returned instead of a tuple for that output. For example the infer_shape for a matrix-matrix product would accept input_shapes=((x0,x1), (y0,y1)) and return ((x0, y1),). Inferring the shape of internal nodes in the graph is important for doing size-driven optimizations. If we know how big various intermediate results will be, we can estimate the cost of many Ops accurately, and generate c-code that is specific [e.g. unrolled] to particular sizes. In cases where you cannot figure out the shape, raise a ShapeError. Notes ----- Right now there is only the ConvOp that could really take advantage of this shape inference, but it is worth it even just for the ConvOp. All that's necessary to do shape inference is 1) to mark shared inputs as having a particular shape, either via a .tag or some similar hacking; and 2) to add an optional In() argument to promise that inputs will have a certain shape (or even to have certain shapes in certain dimensions). We can't automatically infer the shape of shared variables as they can change of shape during the execution by default. (NOT IMPLEMENTED YET, BUT IS IN TRAC) **Using Shape information in Optimizations** To use this shape information in OPTIMIZATIONS, use the ``shape_of`` dictionary. For example: .. code-block:: python try: shape_of = node.fgraph.shape_feature.shape_of except AttributeError: # This can happen when the mode doesn't include the ShapeFeature. return shape_of_output_zero = shape_of[node.output[0]] The ``shape_of_output_zero`` symbol will contain a tuple, whose elements are either integers or symbolic integers. TODO: check to see if the symbols are necessarily non-constant... or are integer literals sometimes Theano constants?? That would be confusing. """ def get_node_infer_shape(self, node): try: shape_infer = node.op.infer_shape except AttributeError: shape_infer = self.default_infer_shape try: o_shapes = shape_infer(node, [self.shape_of[r] for r in node.inputs]) except ShapeError: o_shapes = self.default_infer_shape(node, [self.shape_of[r] for r in node.inputs]) except NotImplementedError as e: raise NotImplementedError( 'Code called by infer_shape failed raising a ' 'NotImplementedError. Raising NotImplementedError to ' 'indicate that a shape cannot be computed is no longer ' 'supported, and one should now use tensor.ShapeError ' 'instead. The original exception message is: %s' % e) except Exception as e: msg = ('Failed to infer_shape from Op %s.\nInput shapes: ' '%s\nException encountered during infer_shape: ' '%s\nException message: %s\nTraceback: %s') % ( node.op, [self.shape_of[r] for r in node.inputs], type(e), str(e), traceback.format_exc()) if config.on_shape_error == "raise": raise Exception(msg) else: _logger.warning(msg) o_shapes = self.default_infer_shape( node, [self.shape_of[r] for r in node.inputs]) return o_shapes def get_shape(self, var, idx): """ Optimization can call this to get the current shape_i It is better to call this then use directly shape_of[var][idx] as this method should update shape_of if needed. TODO: Up to now, we don't update it in all cases. Update in all cases. """ r = self.shape_of[var][idx] if (r.owner and isinstance(r.owner.op, Shape_i) and r.owner.inputs[0] not in var.fgraph.variables): assert var.owner node = var.owner # recur on inputs for i in node.inputs: if getattr(i, 'ndim', None) > 0: self.get_shape(i, 0) o_shapes = self.get_node_infer_shape(node) assert len(o_shapes) == len(node.outputs) # Only change the variables and dimensions that would introduce # extra computation for new_shps, out in zip(o_shapes, node.outputs): if not hasattr(out, 'ndim'): continue merged_shps = list(self.shape_of[out]) changed = False for i in range(out.ndim): n_r = merged_shps[i] if (n_r.owner and isinstance(n_r.owner.op, Shape_i) and n_r.owner.inputs[0] not in var.fgraph.variables): changed = True merged_shps[i] = new_shps[i] if changed: self.set_shape(out, merged_shps, override=True) r = self.shape_of[var][idx] return r def shape_ir(self, i, r): """Return symbolic r.shape[i] for tensor variable r, int i.""" if hasattr(r.type, "broadcastable") and r.type.broadcastable[i]: return self.lscalar_one else: # Do not call make_node for test_value s = Shape_i(i)(r) try: s = get_scalar_constant_value(s) except NotScalarConstantError: pass return s def shape_tuple(self, r): """Return a tuple of symbolic shape vars for tensor variable r.""" if not hasattr(r, 'ndim'): # This happen for NoneConst. return None return tuple([self.shape_ir(i, r) for i in xrange(r.ndim)]) def default_infer_shape(self, node, i_shapes): """Return a list of shape tuple or None for the outputs of node. This function is used for Ops that don't implement infer_shape. Ops that do implement infer_shape should use the i_shapes parameter, but this default implementation ignores it. """ rval = [] for r in node.outputs: try: rval.append(self.shape_tuple(r)) except AttributeError: rval.append(None) return rval def unpack(self, s_i, var): """Return a symbolic integer scalar for the shape element s_i. The s_i argument was produced by the infer_shape() of an Op subclass. var: the variable that correspond to s_i. This is just for error reporting. """ # unpack the s_i that the Op returned assert s_i is not None if s_i == 1: # don't make the optimizer merge a zillion ones together # by always returning the same object to represent 1 return self.lscalar_one if type(s_i) is float and int(s_i) == s_i: s_i = int(s_i) if (type(s_i) in integer_types or isinstance(s_i, np.integer) or (isinstance(s_i, np.ndarray) and s_i.ndim == 0)): # this shape is a constant if s_i < 0: msg = "There is a negative shape in the graph!" msg += gof.utils.get_variable_trace_string(var) # The rest of the pipeline don't handle correctly this # case. So we have 2 choices, stop compilation or # consider the shape as unknow. As we have more # chance to give the stack trace here then later, I # choose that options as it would give better error # message. raise AssertionError(msg) return T.constant(s_i, dtype='int64') if type(s_i) in (tuple, list): # this dimension is the same as many of the inputs # which tells us that if one of the inputs is known, # the others all become known. # TODO: should be implemented in Elemwise, and Dot # # worst case, we loop over shape_of and replace things raise NotImplementedError(s_i) # s_i is x.shape[i] for some x, we change it to shape_of[x][i] if (s_i.owner and isinstance(s_i.owner.op, Subtensor) and s_i.owner.inputs[0].owner and isinstance(s_i.owner.inputs[0].owner.op, T.Shape)): assert s_i.ndim == 0 assert len(s_i.owner.op.idx_list) == 1 # The current Subtensor always put constant index in the graph. # This was not True in the past. So call the Subtensor function # that will return the right index. idx = get_idx_list(s_i.owner.inputs, s_i.owner.op.idx_list) assert len(idx) == 1 idx = idx[0] try: i = get_scalar_constant_value(idx) except NotScalarConstantError: pass else: # Executed only if no exception was raised x = s_i.owner.inputs[0].owner.inputs[0] # x should already have been imported, and should be in shape_of. s_i = self.shape_of[x][i] if s_i.type.dtype in theano.tensor.integer_dtypes: if getattr(s_i.type, 'ndim', 0): raise TypeError('Shape element must be scalar', s_i) return s_i else: raise TypeError('Unsupported shape element', s_i, type(s_i), getattr(s_i, 'type', None)) def set_shape(self, r, s, override=False): """Assign the shape `s` to previously un-shaped variable `r`. Parameters ---------- r : a variable s : None or a tuple of symbolic integers override : If False, it mean r is a new object in the fgraph. If True, it mean r is already in the fgraph and we want to override its shape. """ if not override: assert r not in self.shape_of, 'r already in shape_of' if s is None: self.shape_of[r] = s else: if not isinstance(s, (tuple, list)): raise TypeError('shapes must be tuple/list', (r, s)) if r.ndim != len(s): sio = StringIO() theano.printing.debugprint(r, file=sio, print_type=True) raise AssertionError( "Something inferred a shape with %d dimensions " "for a variable with %d dimensions" " for the variable:\n%s" % ( len(s), r.ndim, sio.getvalue())) shape_vars = [] for i in xrange(r.ndim): if (hasattr(r.type, 'broadcastable') and r.type.broadcastable[i]): shape_vars.append(self.lscalar_one) else: shape_vars.append(self.unpack(s[i], r)) assert all([not hasattr(r.type, "broadcastable") or not r.type.broadcastable[i] or # The two following comparison are a speed optimization # But we never timed this speed optimization! self.lscalar_one.equals(shape_vars[i]) or self.lscalar_one.equals( T.extract_constant(shape_vars[i])) for i in xrange(r.ndim)]) self.shape_of[r] = tuple(shape_vars) for sv in shape_vars: self.shape_of_reverse_index.setdefault(sv, set()).add(r) def update_shape(self, r, other_r): """Replace shape of r by shape of other_r. If, on some dimensions, the shape of other_r is not informative, keep the shape of r on those dimensions. """ # other_r should already have a shape assert other_r in self.shape_of, ('other_r not in shape_of', other_r) other_shape = self.shape_of[other_r] # If other_shape has no information, call is pointless. if other_shape is None: return if r in self.shape_of: r_shape = self.shape_of[r] else: # If no info is known on r's shape, use other_shape self.set_shape(r, other_shape) return if (other_r.owner and r.owner and other_r.owner.inputs == r.owner.inputs and other_r.owner.op == r.owner.op): # We are doing a merge. So the 2 shapes graph will be the # same. This is only a speed optimization to call # ancestors() less frequently. return # Merge other_shape with r_shape, giving the priority to other_shape merged_shape = [] for i, ps in enumerate(other_shape): if r_shape is None and other_shape: merged_shape.append(other_shape[i]) elif (ps.owner and isinstance(getattr(ps.owner, 'op', None), Shape_i) and ps.owner.op.i == i and ps.owner.inputs[0] in (r, other_r)): # If other_shape[i] is uninformative, use r_shape[i]. # For now, we consider 2 cases of uninformative other_shape[i]: # - Shape_i(i)(other_r); # - Shape_i(i)(r). merged_shape.append(r_shape[i]) elif isinstance(r_shape[i], (Constant, integer_types)): # We do this to call less often ancestors and make # sure we have the simplest shape possible. merged_shape.append(r_shape[i]) elif isinstance(other_shape[i], (Constant, integer_types)): # We do this to call less often ancestors and make # sure we have the simplest shape possible. merged_shape.append(other_shape[i]) elif other_shape[i] == r_shape[i]: # This mean the shape is equivalent # We do not want to do the ancestor check in those cases merged_shape.append(r_shape[i]) elif r_shape[i] in theano.gof.graph.ancestors([other_shape[i]]): # Another case where we want to use r_shape[i] is when # other_shape[i] actually depends on r_shape[i]. In that case, # we do not want to substitute an expression with another that # is strictly more complex. Such a substitution could also lead # to cycles: if (in the future) r_shape[i] gets replaced by an # expression of other_shape[i], other_shape[i] may end up # depending on itself. merged_shape.append(r_shape[i]) else: merged_shape.append(other_shape[i]) assert all([(not hasattr(r.type, "broadcastable") or not r.type.broadcastable[i] and not other_r.type.broadcastable[i]) or # The two following comparison are a speed optimization # But we never timed this speed optimization! self.lscalar_one.equals(merged_shape[i]) or self.lscalar_one.equals( T.extract_constant(merged_shape[i], only_process_constants=True)) for i in xrange(r.ndim)]) self.shape_of[r] = tuple(merged_shape) for sv in self.shape_of[r]: self.shape_of_reverse_index.setdefault(sv, set()).add(r) def set_shape_i(self, r, i, s_i): '''Replace element i of shape_of[r] by s_i''' assert r in self.shape_of prev_shape = self.shape_of[r] # prev_shape is a tuple, so we cannot change it inplace, # so we build another one. new_shape = [] for j, s_j in enumerate(prev_shape): if j == i: new_shape.append(self.unpack(s_i, r)) else: new_shape.append(s_j) assert all([not hasattr(r.type, "broadcastable") or not r.type.broadcastable[idx] or # The two following comparison are a speed optimization # But we never timed this speed optimization! self.lscalar_one.equals(new_shape[idx]) or self.lscalar_one.equals(T.extract_constant(new_shape[idx])) for idx in xrange(r.ndim)]) self.shape_of[r] = tuple(new_shape) for sv in self.shape_of[r]: self.shape_of_reverse_index.setdefault(sv, set()).add(r) def init_r(self, r): '''Register r's shape in the shape_of dictionary.''' if r not in self.shape_of: try: self.set_shape(r, self.shape_tuple(r)) except AttributeError: # XXX: where would this come from? self.set_shape(r, None) def make_vector_shape(self, r): return make_vector(*self.shape_of[r]) # # Feature interface # # def on_attach(self, fgraph): assert not hasattr(fgraph, 'shape_feature') fgraph.shape_feature = self # Must be local to the object as otherwise we reuse the same # variable for multiple fgraph! self.lscalar_one = T.constant(1, dtype='int64') assert self.lscalar_one.type == T.lscalar self.shape_of = {} # Variable -> tuple(scalars) or None (All tensor vars map to tuple) self.scheduled = {} # Variable -> self.shape_of_reverse_index = {} # shape var -> graph v for node in fgraph.toposort(): self.on_import(fgraph, node, reason='on_attach') def on_detach(self, fgraph): self.shape_of = {} self.scheduled = {} self.shape_of_reverse_index = {} del fgraph.shape_feature def on_import(self, fgraph, node, reason): if node.outputs[0] in self.shape_of: # this is a revert, not really an import for r in node.outputs + node.inputs: assert r in self.shape_of return for i, r in enumerate(node.inputs): # make sure we have shapes for the inputs self.init_r(r) o_shapes = self.get_node_infer_shape(node) # this is packed information # an element of o_shapes is either None or a tuple # elements of the tuple can be either strings, or ints if len(o_shapes) != len(node.outputs): raise Exception( ('The infer_shape method for the Op "%s" returned a list ' + 'with the wrong number of element: len(o_shapes) = %d ' + ' != len(node.outputs) = %d') % (str(node.op), len(o_shapes), len(node.outputs))) # Ensure shapes are in 'int64'. This is to make sure the assert # found in the `local_useless_subtensor` optimization does not fail. for sh_idx, sh in enumerate(o_shapes): if sh is None: continue if not isinstance(sh, (list, tuple)): raise ValueError("infer_shape of %s didn't return a list of" " list. It returned '%s'" % (str(node), str(o_shapes))) new_shape = [] for i, d in enumerate(sh): # Note: we ignore any shape element that is not typed (i.e., # does not have a 'dtype' attribute). This means there may # still remain int elements that are int32 on 32-bit platforms, # but this works with `local_useless_subtensor`, so for now we # keep it this way. See #266 for a better long-term fix. if getattr(d, 'dtype', 'int64') != 'int64': assert d.dtype in theano.tensor.discrete_dtypes, (node, d.dtype) assert str(d.dtype) != 'uint64', node new_shape += sh[len(new_shape):i + 1] if isinstance(d, T.Constant): casted_d = T.constant(d.data, dtype='int64') else: casted_d = theano.tensor.cast(d, 'int64') new_shape[i] = casted_d if new_shape: # We replace the shape with wrong dtype by the one with # 'int64'. new_shape += sh[len(new_shape):] o_shapes[sh_idx] = tuple(new_shape) for r, s in izip(node.outputs, o_shapes): self.set_shape(r, s) def on_change_input(self, fgraph, node, i, r, new_r, reason): if new_r not in self.shape_of: # It happen that the fgraph didn't called on_import for some # new_r. This happen when new_r don't have an # owner(i.e. it is a constant or an input of the graph) # update_shape suppose that r and new_r are in shape_of. self.init_r(new_r) # This tells us that r and new_r must have the same shape if # we didn't know that the shapes are related, now we do. self.update_shape(new_r, r) # change_input happens in two cases: # 1) we are trying to get rid of r, or # 2) we are putting things back after a failed transaction. # In case 1, if r has a shape_i client, we will want to # replace the shape_i of r with the shape of new_r. Say that # r is *scheduled*. # At that point, node is no longer a client of r, but of new_r for (shpnode, idx) in (r.clients + [(node, i)]): if isinstance(getattr(shpnode, 'op', None), Shape_i): idx = shpnode.op.i repl = self.shape_of[new_r][idx] if repl.owner is shpnode: # This mean the replacement shape object is # exactly the same as the current shape object. So # no need for replacement. This happen for example # with the InputToGpuOptimizer optimizer. continue if (repl.owner and repl.owner.inputs[0] is shpnode.inputs[0] and isinstance(repl.owner.op, Shape_i) and repl.owner.op.i == shpnode.op.i): # The replacement is a shape_i of the same # input. So no need to do this equivalent # replacement. continue if shpnode.outputs[0] in theano.gof.graph.ancestors([repl]): raise InconsistencyError( "This substitution would insert a cycle in the graph:" "node: %s, i: %i, r: %s, new_r: %s" % (node, i, r, new_r)) self.scheduled[shpnode] = new_r # In case 2, if r is a variable that we've scheduled for shape update, # then we should cancel it. unscheduled = [k for k, v in self.scheduled.items() if v == r] for k in unscheduled: del self.scheduled[k] # In either case, r could be in shape_of.values(), that is, r itself # is the shape of something. In that case, we want to update # the value in shape_of, to keep it up-to-date. for v in self.shape_of_reverse_index.get(r, []): # The reverse index is only approximate. It is not updated on # deletion of variables, or on change_input so it might be the # case that there are a few extra `v`'s in it that no longer have # a shape of r or possibly have been deleted from shape_of # entirely. The important thing is that it permits to recall # all variables with r in their shape. for ii, svi in enumerate(self.shape_of.get(v, [])): if svi == r: self.set_shape_i(v, ii, new_r) self.shape_of_reverse_index[r] = set() def same_shape(self, x, y, dim_x=None, dim_y=None): """Return True if we are able to assert that x and y have the same shape. dim_x and dim_y are optional. If used, they should be an index to compare only 1 dimension of x and y. """ sx = self.shape_of[x] sy = self.shape_of[y] if sx is None or sy is None: return False if dim_x is not None: sx = [sx[dim_x]] if dim_y is not None: sy = [sy[dim_y]] assert len(sx) == len(sy) # We look on each dimensions we want to compare. # If any of them can't be asserted to be equal, return False. # Otherwise, we return True at the end. for dx, dy in zip(sx, sy): if dx is dy: continue # Need to try to find that they are the same shape. We # need to compare the full graph. It could be slow. So I # just implement for now the case of Shape_i. if not dx.owner or not dy.owner: return False if (not isinstance(dx.owner.op, Shape_i) or not isinstance(dy.owner.op, Shape_i)): return False opx = dx.owner.op opy = dy.owner.op if not (opx.i == opy.i): return False # FB I'm not sure if this handle correctly constants. if dx.owner.inputs[0] == dy.owner.inputs[0]: continue # To be sure to cover all case, call equal_computation. # Can't use theano.gof.graph.is_same_graph(dx, dy) # As it currently expect that dx and dy aren't in a FunctionGraph from theano.scan_module.scan_utils import equal_computations if not equal_computations([dx], [dy]): return False return True class ShapeOptimizer(Optimizer): """Optimizer that serves to add ShapeFeature as an fgraph feature.""" def add_requirements(self, fgraph): fgraph.attach_feature(ShapeFeature()) def apply(self, fgraph): pass class UnShapeOptimizer(Optimizer): """Optimizer remove ShapeFeature as an fgraph feature.""" def apply(self, fgraph): for feature in fgraph._features: if isinstance(feature, ShapeFeature): fgraph.remove_feature(feature) # Register it after merge1 optimization at 0. We don't want to track # the shape of merged node. theano.compile.mode.optdb.register('ShapeOpt', ShapeOptimizer(), 0.1, 'fast_run', 'fast_compile') # Not enabled by default for now. Some crossentropy opt use the # shape_feature. They are at step 2.01. uncanonicalize is at step # 3. After it goes to 48.5 that move to the gpu. So 10 seem resonable. theano.compile.mode.optdb.register('UnShapeOpt', UnShapeOptimizer(), 10) def local_elemwise_alloc_op(ElemwiseOP, AllocOP, DimShuffleOP): def local_elemwise_alloc(node): """ elemwise(alloc(x, shp), ..., y.TensorType(BROADCAST CONDITION)) -> elemwise(x, y.TensorType(BROADCAST CONDITION)) elemwise(dimshuffle(alloc(x, shp)),... ,y.TensorType(BROADCAST CONDITION)) -> elemwise(x.dimshuffle(...), y.TensorType(BROADCAST CONDITION)) BROADCAST CONDITION: the condition is that the one input that are not to be optimized to have the same broadcast pattern as the output. We can change the alloc by a dimshuffle as the elemwise already have the shape info. The dimshuffle will be faster to exec. """ if not isinstance(node.op, ElemwiseOP): return False if len(node.outputs) > 1: # Ensure all outputs have the same broadcast pattern # This is a supposition that I'm not sure is always true. assert all([o.type.broadcastable == node.outputs[0].type.broadcastable for o in node.outputs[1:]]) # The broadcast pattern of the ouptut must match the broadcast # pattern of at least one of the inputs. if not any([i.type.broadcastable == node.outputs[0].type.broadcastable for i in node.inputs]): return False def dimshuffled_alloc(i): return (isinstance(i.owner.op, DimShuffleOP) and i.owner.inputs[0].owner and isinstance(i.owner.inputs[0].owner.op, AllocOP)) # At least one input must have an owner that is either a AllocOP or a # DimShuffleOP with an owner that is a AllocOP -- otherwise there is # nothing to optimize. if not any([i.owner and (isinstance(i.owner.op, AllocOP) or dimshuffled_alloc(i)) for i in node.inputs]): return False # Search for input that we can use as a baseline for the dimensions. assert_op_idx = -1 for idx, i in enumerate(node.inputs): if i.type.broadcastable == node.outputs[0].type.broadcastable: # Prefer an input that is not a AllocOP nor a DimShuffleOP of a # AllocOP so that all allocs can be optimized. if not (i.owner and (isinstance(i.owner.op, AllocOP) or dimshuffled_alloc(i))): assert_op_idx = idx break # It may be the case that only AllocOP and DimShuffleOP of AllocOP exist. if assert_op_idx < 0: # We want to optimize as many allocs as possible. When # there is more than one then do all but one. number of # inputs with alloc or dimshuffle alloc l2 = [i for i in node.inputs if (i.owner and (isinstance(i.owner.op, AllocOP) or dimshuffled_alloc(i)))] # If only 1 alloc or dimshuffle alloc, it is the one we # will use for the shape. So no alloc would be removed. if len(l2) > 1: # l containt inputs with alloc or dimshuffle alloc # only. Its length will always be at least one, as we # checked that before l = [idx for idx, i in enumerate(node.inputs) if i.broadcastable == node.outputs[0].broadcastable] assert_op_idx = l[0] # The first one is as good as any to use. else: # Nothing would be optimized! return False assert_op = node.inputs[assert_op_idx] cmp_op = assert_op new_i = [] same_shape = node.fgraph.shape_feature.same_shape for i in node.inputs: # Remove alloc if (i.owner and isinstance(i.owner.op, AllocOP) and i.owner.inputs[0].type != i.owner.outputs[0].type): # when i.owner.inputs[0].type == i.owner.outputs[0].type we # will remove that alloc later assert i.type.ndim == cmp_op.ndim if theano.config.experimental.local_alloc_elemwise_assert: get_shape = node.fgraph.shape_feature.get_shape cond = [] for idx in xrange(i.type.ndim): if (not i.type.broadcastable[idx] and not same_shape(i, cmp_op, idx, idx)): i_shp = get_shape(i, idx) cmp_shp = get_shape(cmp_op, idx) cond.append(T.eq(i_shp, cmp_shp)) if cond: assert_op = assert_(assert_op, *cond) new_i.append(i.owner.inputs[0]) # Remove Alloc in DimShuffle elif i.owner and dimshuffled_alloc(i): assert i.type.ndim == cmp_op.type.ndim if theano.config.experimental.local_alloc_elemwise_assert: assert_cond = [T.eq(i.shape[idx], cmp_op.shape[idx]) for idx in xrange(i.type.ndim) if not i.type.broadcastable[idx] and not same_shape(i, cmp_op, idx, idx)] if assert_cond: assert_op = assert_(assert_op, *assert_cond) alloc_input = i.owner.inputs[0].owner.inputs[0] if alloc_input.ndim != i.owner.inputs[0].ndim: # The alloc can add dimension to the value # We add a dimshuffle to add them. # We let later optimization merge the multiple dimshuffle nb_dim_to_add = i.owner.inputs[0].ndim - alloc_input.ndim alloc_input = alloc_input.dimshuffle( ['x'] * nb_dim_to_add + list(range(alloc_input.ndim))) # We need to keep the dimshuffle. It could swap axes or # add dimensions anywhere. r_i = i.owner.op(alloc_input) # Copy stack trace from i to new_i copy_stack_trace(i, r_i) new_i.append(r_i) else: new_i.append(i) new_i[assert_op_idx] = assert_op ret = node.op(*new_i, return_list=True) # Copy over stack trace from previous outputs to new outputs. copy_stack_trace(node.outputs, ret) return ret return local_elemwise_alloc # TODO, global optimizer that lift the assert to the beginning of the graph. # TODO, optimize all inputs when possible -- currently when all inputs have # an alloc all but one is optimized. local_elemwise_alloc = register_specialize( gof.local_optimizer([T.Elemwise])( local_elemwise_alloc_op(T.Elemwise, T.Alloc, T.DimShuffle)), 'local_alloc_elemwise') @gof.local_optimizer([T.Elemwise]) def local_fill_sink(node): """ f(fill(a, b), fill(c, d), e) -> fill(c, fill(a, f(b, d, e))) f need to be an elemwise that isn't a fill. """ if (not hasattr(node, 'op') or not isinstance(node.op, T.Elemwise) or node.op == T.fill): return False models = [] inputs = [] for input in node.inputs: if input.owner and input.owner.op == T.fill: models.append(input.owner.inputs[0]) inputs.append(input.owner.inputs[1]) else: inputs.append(input) if not models: return False c = node.op(*inputs) for model in models: if model.type != c.type: c = T.fill(model, c) # The newly created node c doesn't has 'clients', # so this iteration is took place with node.outputs[0] replacements = {node.outputs[0]: c} for client, cl_idx in node.outputs[0].clients: if (hasattr(client, 'op') and isinstance(client.op, T.Elemwise) and not client.op == T.fill): client_inputs = client.inputs[:] client_inputs[cl_idx] = c new_client = client.op(*client_inputs) # Add clients to new_client new_client.owner.outputs[0].clients = client.outputs[0].clients r = local_fill_sink.transform(new_client.owner) if not r: continue replacements.update(r) return replacements register_canonicalize(local_fill_sink) @register_specialize @register_stabilize # @register_canonicalize # We make full pass after the canonizer phase. @gof.local_optimizer([T.fill]) def local_fill_to_alloc(node): """fill(s,v) -> alloc(v, shape(s)) This is an important optimization because with the shape_to_shape_i optimization, the dependency on 's' is often removed. """ if node.op == T.fill: r, v = node.inputs if v.type == node.outputs[0].type: # this is a useless fill, erase it. rval = [v] elif v.type.broadcastable == node.outputs[0].type.broadcastable: # this is a cast rval = [T.cast(v, node.outputs[0].type.dtype)] elif r.type.broadcastable == node.outputs[0].type.broadcastable: # we are broadcasting v somehow, but not r o = broadcast_like(v, r, node.fgraph, dtype=v.dtype) copy_stack_trace(node.outputs[0], o) rval = [o] else: # we are broadcasting both v and r, # the output shape must be computed # # TODO: implement this case (including a test!) # # I think the strategy should be to extend the shorter # shape vector with 1s (how?) and then take the # elementwise max of the two. - how to flag an error of # shape mismatch where broadcasting should be illegal? return # TODO: cut out un-necessary dimshuffles of v assert rval[0].type == node.outputs[0].type, ( 'rval', rval[0].type, 'orig', node.outputs[0].type, 'node', node,) # theano.printing.debugprint(node.outputs[0], file='str')) return rval # Register this after stabilize at 1.5 to make sure stabilize don't # get affected by less canonicalized graph due to alloc. compile.optdb.register('local_fill_to_alloc', in2out(local_fill_to_alloc), 1.51, 'fast_run') # Needed to clean some extra alloc added by local_fill_to_alloc compile.optdb.register('local_elemwise_alloc', in2out(local_elemwise_alloc), 1.52, 'fast_run') @register_canonicalize("fast_compile") @register_useless @gof.local_optimizer([T.fill]) def local_useless_fill(node): """fill(s,v) -> v This optimization is only needed in FAST_COMPILE to make the code more readable. Normally, it is done by the local_fill_to_alloc opt. """ if node.op == T.fill: r, v = node.inputs if v.type == node.outputs[0].type: # this is a useless fill, erase it. # also, we don't need to copy over any stack traces here return [v] @register_specialize @register_stabilize @register_canonicalize @register_useless @gof.local_optimizer([T.alloc]) def local_useless_alloc(node): """ If the input type is the same as the output type (dtype and broadcast) there is no change in the shape of the input. So this is just a simple copy of the input. This is not needed. """ op = node.op if not isinstance(op, Alloc): return False input = node.inputs[0] output = node.outputs[0] # Check if dtype and broadcast remain the same. if input.type == output.type: # We don't need to copy over any stack traces here return [input] @register_specialize @register_stabilize @register_canonicalize @gof.local_optimizer([T.alloc]) def local_canonicalize_alloc(node): """If the input type is the same as the output type (dtype and broadcast) there is no change in the shape of the input. So this is just a simple copy of the input. This is not needed. (as local_useless_alloc) Also, it will canonicalize alloc by creating Dimshuffle after the alloc to introduce the dimensions of constant size 1. See https://github.com/Theano/Theano/issues/4072 to know why this is needed. """ op = node.op if not isinstance(op, Alloc): return False input = node.inputs[0] output = node.outputs[0] # Check if dtype and broadcast remain the same. if input.type == output.type: # We don't need to copy over any stack traces here return [input] # Allow local_merge_alloc to do its work first clients = getattr(output, 'clients', []) for client, i in clients: if client != "output" and isinstance(client.op, Alloc): return # Check if alloc adds a broadcastable dimension with shape 1. output_shape = node.inputs[1:] num_dims_with_size_1_added_to_left = 0 for i in range(len(output_shape) - input.ndim): if extract_constant(output_shape[i], only_process_constants=True) == 1: num_dims_with_size_1_added_to_left += 1 else: break new_output_shape = output_shape[num_dims_with_size_1_added_to_left:] if num_dims_with_size_1_added_to_left > 0 and len(new_output_shape) >= input.ndim: if output.broadcastable[num_dims_with_size_1_added_to_left:] == input.broadcastable: inner = input else: inner = op(*([input] + new_output_shape)) dimshuffle_new_order = (['x'] * num_dims_with_size_1_added_to_left + list(xrange(len(new_output_shape)))) return [DimShuffle(inner.type.broadcastable, dimshuffle_new_order)(inner)] # Don't register by default. @gof.local_optimizer([T.AllocEmpty]) def local_alloc_empty_to_zeros(node): """This convert AllocEmpty to Alloc of 0. This help investigate NaN with NanGuardMode. Not registered by default. To activate it, use the Theano flag optimizer_including=alloc_empty_to_zeros. This also enable the GPU version of this optimizations. """ if isinstance(node.op, T.AllocEmpty): return [T.zeros(node.inputs, dtype=node.outputs[0].dtype)] compile.optdb.register('local_alloc_empty_to_zeros', in2out(local_alloc_empty_to_zeros), # After move to gpu and merge2, before inplace. 49.3, 'alloc_empty_to_zeros',) @register_specialize @register_canonicalize @gof.local_optimizer([T.Shape]) def local_shape_to_shape_i(node): if node.op == T.shape: # This optimization needs ShapeOpt and fgraph.shape_feature if not hasattr(node.fgraph, 'shape_feature'): return shape_feature = node.fgraph.shape_feature ret = shape_feature.make_vector_shape(node.inputs[0]) # We need to copy over stack trace from input to output copy_stack_trace(node.outputs[0], ret) return [ret] # TODO: Not sure what type of node we are expecting here @register_specialize @register_canonicalize @gof.local_optimizer(None) def local_track_shape_i(node): try: shape_feature = node.fgraph.shape_feature except AttributeError: return if node in shape_feature.scheduled: # Don't unschedule node as it could be reinserted in the # fgraph as we don't change it in the shapefeature internal # structure. assert isinstance(node.op, Shape_i) replacement = shape_feature.scheduled[node] return [shape_feature.shape_of[replacement][node.op.i]] @register_specialize @register_canonicalize @gof.local_optimizer([Subtensor]) def local_subtensor_inc_subtensor(node): """ Subtensor(SetSubtensor(x, y, idx), idx) -> y """ if isinstance(node.op, Subtensor): x = node.inputs[0] if not x.owner or not isinstance(x.owner.op, IncSubtensor): return if not x.owner.op.set_instead_of_inc: return if (x.owner.inputs[2:] == node.inputs[1:] and tuple(x.owner.op.idx_list) == tuple(node.op.idx_list)): out = node.outputs[0] y = x.owner.inputs[1] # If the dtypes differ, cast y into x.dtype if x.dtype != y.dtype: y = y.astype(x.dtype) if out.type == y.type: # if x[idx] and y have the same type, directly return y return [y] else: # The difference is related to broadcasting pattern assert out.broadcastable != y.broadcastable # We have to alloc y to the shape of x[idx] x_subtensor = node.op(x.owner.inputs[0], *x.owner.inputs[2:]) return [T.alloc(y, *x_subtensor.shape)] else: return @register_specialize @register_canonicalize @gof.local_optimizer([Subtensor]) def local_subtensor_remove_broadcastable_index(node): """ Remove broadcastable dimension with index 0 or -1 a[:,:,:,0] -> a.dimshuffle(0,1,2), when a.broadcastable = (False, False, False, True) a[0,:,-1,:] -> a.dimshuffle(1,3), when a.broadcastable = (True, False, True, False) """ if isinstance(node.op, Subtensor): idx = node.op.idx_list else: return remove_dim = [] node_inputs_idx = 1 for dim, elem in enumerate(idx): if isinstance(elem, (scalar.Scalar)): # The idx is a Scalar, ie a Type. This means the actual index # is contained in node.inputs[1] dim_index = node.inputs[node_inputs_idx] if type(dim_index) == theano.scalar.basic.ScalarConstant: dim_index = dim_index.value if dim_index in [0, -1] and node.inputs[0].broadcastable[dim]: remove_dim.append(dim) node_inputs_idx += 1 else: return elif isinstance(elem, slice): if elem != slice(None): return elif isinstance(elem, (integer_types, np.integer)): if elem in [0, -1] and node.inputs[0].broadcastable[dim]: remove_dim.append(dim) else: raise TypeError('case not expected') if len(remove_dim) == 0: return else: all_dim = range(node.inputs[0].ndim) remain_dim = [x for x in all_dim if x not in remove_dim] return [node.inputs[0].dimshuffle(tuple(remain_dim))] @register_specialize @register_canonicalize('fast_compile_gpu') @register_useless @gof.local_optimizer([Subtensor, AdvancedSubtensor1]) def local_subtensor_make_vector(node): """ Replace all subtensor(make_vector) like: [a,b,c][0] -> a [a,b,c][0:2] -> [a,b] Replace all AdvancedSubtensor1(make_vector) like: [a,b,c][[0,2]] -> [a,c] We can do this for constant indexes. """ x = node.inputs[0] if not x.owner or x.owner.op != make_vector: return if isinstance(node.op, Subtensor): # This optimization needs ShapeOpt and fgraph.shape_feature try: idx, = node.op.idx_list except Exception: # 'how can you have multiple indexes into a shape?' raise if isinstance(idx, (scalar.Scalar, T.TensorType)): # The idx is a Scalar, ie a Type. This means the actual index # is contained in node.inputs[1] old_idx, idx = idx, node.inputs[1] assert idx.type == old_idx elif isinstance(node.op, AdvancedSubtensor1): idx = node.inputs[1] else: return if isinstance(idx, (integer_types, np.integer)): # We don't need to copy over any stack traces here return [x.owner.inputs[idx]] elif isinstance(idx, Variable): if idx.ndim == 0: # if it is a constant we can do something with it try: v = get_scalar_constant_value(idx, only_process_constants=True) if isinstance(v, np.integer): # Python 2.4 wants to index only with Python integers v = int(v) # We don't need to copy over any stack traces here try: ret = [x.owner.inputs[v]] except IndexError: raise NotScalarConstantError("Bad user graph!") return ret except NotScalarConstantError: pass elif idx.ndim == 1 and isinstance(idx, T.Constant): values = list(map(int, list(idx.value))) ret = make_vector(*[x.owner.inputs[v] for v in values]) # Copy over stack trace from previous output to new output copy_stack_trace(node.outputs[0], ret) ret = T.patternbroadcast(ret, node.outputs[0].broadcastable) return [ret] else: raise TypeError('case not expected') elif isinstance(idx, slice): # it is a slice of ints and/or Variables # check subtensor to see if it can contain constant variables, and if # it can, then try to unpack them. try: const_slice = node.op.get_constant_idx(node.inputs, allow_partial=False)[0] ret = make_vector(*x.owner.inputs[const_slice]) # Copy over stack trace from previous outputs to new output copy_stack_trace(node.outputs, ret) ret = T.patternbroadcast(ret, node.outputs[0].broadcastable) return [ret] except NotScalarConstantError: pass else: raise TypeError('case not expected') # TODO: the other optimization for and, or, xor, le and ge see ticket #496. @register_useless @register_canonicalize('fast_compile') @register_specialize @gof.local_optimizer([T.Elemwise]) def local_useless_elemwise(node): """ eq(x, x) -> 1 neq(x, x) -> 0 mul(x) -> x add(x) -> x identity(x) -> x and(x, 1) -> x (if x.dtype == 'bool') and(x, 0) -> zeros_like(x) or(x, 0) -> x or(x, 1) -> ones_like(x) (if x.dtype == 'bool') xor(x, x) -> zeros_like(x) """ if isinstance(node.op, T.Elemwise): # We call zeros_like and one_like with opt=True to generate a # cleaner graph. dtype = node.outputs[0].dtype if node.op.scalar_op == theano.scalar.eq and len(node.inputs) == 2: if node.inputs[0] == node.inputs[1]: # it is the same var in the graph. That will always be true ret = T.ones_like(node.inputs[0], dtype=dtype, opt=True) # Copy stack trace from input to constant output copy_stack_trace(node.outputs[0], ret) return [ret] elif node.op.scalar_op == theano.scalar.neq and len(node.inputs) == 2: if node.inputs[0] == node.inputs[1]: # it is the same var in the graph. That will always be false ret = T.zeros_like(node.inputs[0], dtype=dtype, opt=True) # Copy stack trace from input to constant output copy_stack_trace(node.outputs[0], ret) return [ret] elif node.op.scalar_op == theano.scalar.mul and len(node.inputs) == 1: # No need to copy over any stack trace return [node.inputs[0]] elif node.op.scalar_op == theano.scalar.add and len(node.inputs) == 1: # No need to copy over any stack trace return [node.inputs[0]] elif (node.op.scalar_op == theano.scalar.identity and len(node.inputs) == 1): return [node.inputs[0]] elif (isinstance(node.op.scalar_op, scalar.AND) and len(node.inputs) == 2): if isinstance(node.inputs[0], T.TensorConstant): const_val = T.extract_constant(node.inputs[0], only_process_constants=True) if not isinstance(const_val, Variable): if const_val == 0: return [T.zeros_like(node.inputs[1], dtype=dtype, opt=True)] elif node.outputs[0].dtype == 'bool': # If the output is not Boolean, it is the bitwise AND, # and this optimization would be wrong return [node.inputs[1].astype(node.outputs[0].dtype)] if isinstance(node.inputs[1], T.TensorConstant): const_val = T.extract_constant(node.inputs[1], only_process_constants=True) if not isinstance(const_val, Variable): if const_val == 0: return [T.zeros_like(node.inputs[0], dtype=dtype, opt=True)] elif node.outputs[0].dtype == 'bool': # If the output is not Boolean, it is the bitwise AND, # and this optimization would be wrong return [node.inputs[0].astype(node.outputs[0].dtype)] elif (isinstance(node.op.scalar_op, scalar.OR) and len(node.inputs) == 2): if isinstance(node.inputs[0], T.TensorConstant): const_val = T.extract_constant(node.inputs[0], only_process_constants=True) if not isinstance(const_val, Variable): if const_val == 0: return [node.inputs[1].astype(node.outputs[0].dtype)] elif node.outputs[0].dtype == 'bool': # If the output is not Boolean, it is the bitwise OR, # and this optimization would be wrong return [T.ones_like(node.inputs[1], dtype=dtype, opt=True)] if isinstance(node.inputs[1], T.TensorConstant): const_val = T.extract_constant(node.inputs[1], only_process_constants=True) if not isinstance(const_val, Variable): if const_val == 0: return [node.inputs[0].astype(node.outputs[0].dtype)] elif node.outputs[0].dtype == 'bool': # If the output is not Boolean, it is the bitwise OR, # and this optimization would be wrong return [T.ones_like(node.inputs[0], dtype=dtype, opt=True)] elif (isinstance(node.op.scalar_op, scalar.XOR) and len(node.inputs) == 2): if node.inputs[0] is node.inputs[1]: return [T.zeros_like(node.inputs[0], dtype=dtype, opt=True)] @register_specialize @gof.local_optimizer([T.Elemwise]) def local_alloc_unary(node): """unary(alloc(x, shp)) -> alloc(unary(x), shp)""" if isinstance(node.op, T.Elemwise) and len(node.inputs) == 1: a = node.inputs[0] if a.owner and isinstance(a.owner.op, T.Alloc): x = a.owner.inputs[0] shp = a.owner.inputs[1:] v = node.op(x) # T.alloc does not preserve the stacktrace of v, # so we need to copy it over from x. copy_stack_trace(node.outputs[0], v) ret = T.alloc(T.cast(v, node.outputs[0].dtype), *shp) # T.cast does not preserve the stacktrace of x, # so we need to copy it over to the output. copy_stack_trace([node.outputs[0], a], ret) return [ret] @register_canonicalize @register_specialize @gof.local_optimizer([T.Elemwise]) def local_cast_cast(node): """cast(cast(x, dtype1), dtype2) when those contrain: dtype1 == dtype2 OR the base dtype is the same (int, uint, float, complex) and the first cast cause an upcast. """ if (not isinstance(node.op, T.Elemwise) or not isinstance(node.op.scalar_op, scalar.Cast)): return x = node.inputs[0] if (not x.owner or not isinstance(x.owner.op, T.Elemwise) or not isinstance(x.owner.op.scalar_op, scalar.Cast)): return type1 = x.owner.op.scalar_op.o_type type2 = node.op.scalar_op.o_type base = x.owner.inputs[0] if type1 == type2: # We don't need to copy over any stack traces here return [x] if(is_an_upcast(base.dtype, type1.dtype)): # Checking for further redundancy. Eg: int8 -> int32 -> int8 if(type2.dtype == base.dtype): return x.owner.inputs else: # Apply the second cast only v = node.op(base) # Copy stack trace from the output of the original cast copy_stack_trace(node.outputs[0], v) return [v] def is_an_upcast(type1, type2): """Given two data types (as strings), check if converting to type2 from type1 constitutes an upcast. Differs from theano.scalar.upcast """ category = { # The first number in the pair is the dtype (bool, uint, int, float, # complex). Conversion from higher to lower is never an upcast. # The second number roughly indicates the precision. Again, conversion # from higher to lower is never an upcast. 'bool': (0, 0), 'uint8': (1, 1), 'uint16': (1, 2), 'uint32': (1, 3), 'uint64': (1, 4), 'int8': (2, 1), 'int16': (2, 2), 'int32': (2, 3), 'int64': (2, 4), 'float16': (3, 1.5), 'float32': (3, 2.5), 'float64': (3, 3.5), 'complex64': (4, 3), 'complex128': (4, 4) } cat1 = category[type1] cat2 = category[type2] if(cat2[0] >= cat1[0] and cat2[1] > cat1[1]): return True else: return False @register_canonicalize @register_specialize @gof.local_optimizer([T.Elemwise]) def local_func_inv(node): """ Check for two consecutive operations that are functional inverses and remove them from the function graph. """ inv_pairs = ( (basic.Deg2Rad, basic.Rad2Deg), (basic.Cosh, basic.ArcCosh), (basic.Tanh, basic.ArcTanh), (basic.Sinh, basic.ArcSinh), (basic.Conj, basic.Conj), (basic.Neg, basic.Neg), (basic.Inv, basic.Inv), ) x = node.inputs[0] if not isinstance(node.op, T.Elemwise): return if (not x.owner or not isinstance(x.owner.op, T.Elemwise)): return prev_op = x.owner.op.scalar_op node_op = node.op.scalar_op for inv_pair in inv_pairs: if is_inverse_pair(node_op, prev_op, inv_pair): # We don't need to copy stack trace, because the optimization # is trivial and maintains the earlier stack trace return x.owner.inputs return def is_inverse_pair(node_op, prev_op, inv_pair): """ Given two consecutive operations, check if they are the provided pair of inverse functions. """ node_is_op0 = isinstance(node_op, inv_pair[0]) node_is_op1 = isinstance(node_op, inv_pair[1]) prev_is_op0 = isinstance(prev_op, inv_pair[0]) prev_is_op1 = isinstance(prev_op, inv_pair[1]) return (node_is_op0 and prev_is_op1) or (node_is_op1 and prev_is_op0) class Assert(T.Op): """ Implements assertion in a computational graph. Returns the first parameter if the condition is true, otherwise, triggers AssertionError. Notes ----- This Op is a debugging feature. It can be removed from the graph because of optimizations, and can hide some possible optimizations to the optimizer. Specifically, removing happens if it can be determined that condition will always be true. Also, the output of the Op must be used in the function computing the graph, but it doesn't have to be returned. Examples -------- >>> import theano >>> T = theano.tensor >>> x = T.vector('x') >>> assert_op = T.opt.Assert() >>> func = theano.function([x], assert_op(x, x.size<2)) """ _f16_ok = True __props__ = ('msg',) view_map = {0: [0]} check_input = False def __init__(self, msg="Theano Assert failed!"): self.msg = msg def __setstate__(self, attrs): self.__dict__.update(attrs) if not hasattr(self, 'msg'): self.msg = "Theano Assert failed!" def make_node(self, value, *conds): if not isinstance(value, Variable): value = T.as_tensor_variable(value) cond = [T.as_tensor_variable(c) for c in conds] assert np.all([c.type.ndim == 0 for c in cond]) return gof.Apply(self, [value] + cond, [value.type()]) def perform(self, node, inputs, out_): out, = out_ v = inputs[0] out[0] = v assert np.all(inputs[1:]), self.msg def grad(self, input, output_gradients): return output_gradients + [DisconnectedType()()] * (len(input) - 1) def connection_pattern(self, node): return [[1]] + [[0]] * (len(node.inputs) - 1) def c_code(self, node, name, inames, onames, sub): value = inames[0] out = onames[0] check = [] fail = sub['fail'] msg = self.msg.replace('"', '\\"').replace('\n', '\\n') for idx in xrange(len(inames) - 1): i = inames[idx + 1] dtype = node.inputs[idx + 1].dtype check.append('if(!((npy_%(dtype)s*)PyArray_DATA(%(i)s))[0])' '{PyErr_SetString(PyExc_AssertionError,"%(msg)s");' '%(fail)s}' % locals()) check = "\n".join(check) return """ %(check)s Py_XDECREF(%(out)s); %(out)s = %(value)s; Py_INCREF(%(value)s); """ % locals() def c_code_cache_version(self): return (3, 0) def infer_shape(self, node, input_shapes): return [input_shapes[0]] assert_ = Assert() # Unittest.assert_ is a deprecated name for assertTrue. # 2to3 convert theano.tensor.opt.assert_ to theano.tensor.opt.assertTrue # So I define a new name as a work around. assert_op = assert_ @register_specialize @gof.local_optimizer([Assert]) def local_remove_useless_assert(node): if isinstance(node.op, Assert): cond = [] for c in node.inputs[1:]: try: const = get_scalar_constant_value(c) if 0 != const.ndim or const == 0: # Should we raise an error here? How to be sure it # is not catched? cond.append(c) except NotScalarConstantError: cond.append(c) if len(cond) == 0: # We don't need to copy over any stack traces here return [node.inputs[0]] if len(cond) != len(node.inputs) - 1: ret = assert_(node.inputs[0], *cond) # We copy over stack trace from the output of the original assert copy_stack_trace(node.outputs[0], ret) return [ret] @gof.local_optimizer([Assert]) def local_remove_all_assert(node): """An optimization disabled by default that removes all asserts from the graph. Notes ----- See the :ref:`unsafe` section to know how to enable it. """ if not isinstance(node.op, Assert): return # We don't need to copy over any stack traces here return [node.inputs[0]] # Disabled by default compile.optdb['canonicalize'].register('local_remove_all_assert', local_remove_all_assert, 'unsafe', use_db_name_as_tag=False) compile.optdb['stabilize'].register('local_remove_all_assert', local_remove_all_assert, 'unsafe', use_db_name_as_tag=False) compile.optdb['specialize'].register('local_remove_all_assert', local_remove_all_assert, 'unsafe', use_db_name_as_tag=False) compile.optdb['useless'].register('local_remove_all_assert', local_remove_all_assert, 'unsafe', use_db_name_as_tag=False) ####################### # Constant Canonicalization ############################ @register_canonicalize @gof.local_optimizer([T.Elemwise]) def local_upcast_elemwise_constant_inputs(node): """This explicitly upcasts constant inputs to elemwise Ops, when those Ops do implicit upcasting anyway. Rationale: it helps merge things like (1-x) and (1.0 - x). """ if len(node.outputs) > 1: return try: shape_i = node.fgraph.shape_feature.shape_i except AttributeError: shape_i = None if isinstance(node.op, T.Elemwise): scalar_op = node.op.scalar_op # print "aa", scalar_op.output_types_preference if (getattr(scalar_op, 'output_types_preference', None) in (T.scal.upgrade_to_float, T.scal.upcast_out)): # this is the kind of op that we can screw with the input # dtypes by upcasting explicitly output_dtype = node.outputs[0].type.dtype new_inputs = [] for i in node.inputs: if i.type.dtype == output_dtype: new_inputs.append(i) else: try: # works only for scalars cval_i = get_scalar_constant_value(i, only_process_constants=True) if all(i.broadcastable): new_inputs.append(T.shape_padleft( T.cast(cval_i, output_dtype), i.ndim)) else: if shape_i is None: return new_inputs.append( T.alloc(T.cast(cval_i, output_dtype), *[shape_i(d)(i) for d in xrange(i.ndim)])) # print >> sys.stderr, "AAA", # *[Shape_i(d)(i) for d in xrange(i.ndim)] except NotScalarConstantError: # for the case of a non-scalar if isinstance(i, T.TensorConstant): new_inputs.append(T.cast(i, output_dtype)) else: new_inputs.append(i) if new_inputs != node.inputs: rval = [node.op(*new_inputs)] if rval[0].type != node.outputs[0].type: # This can happen for example when floatX=float32 # and we do the true division between and int64 # and a constant that will get typed as int8. # As this is just to allow merging more case, if # the upcast don't work, we can just skip it. return # Copy over output stacktrace from before upcasting copy_stack_trace(node.outputs[0], rval) return rval ################## # Subtensor opts # ################## @register_useless @register_canonicalize @register_specialize @gof.local_optimizer([IncSubtensor]) def local_useless_inc_subtensor(node): """ Remove IncSubtensor, when we overwrite the full inputs with the new value. """ if not isinstance(node.op, IncSubtensor): return if node.op.set_instead_of_inc is False: # This is an IncSubtensor, so the init value must be zeros try: c = get_scalar_constant_value(node.inputs[0], only_process_constants=True) if c != 0: return except NotScalarConstantError: return if (node.inputs[0].ndim != node.inputs[1].ndim or node.inputs[0].broadcastable != node.inputs[1].broadcastable): # FB: I didn't check if this case can happen, but this opt # don't support it. return # We have a SetSubtensor or an IncSubtensor on zeros # If is this IncSubtensor useful? # Check that we keep all the original data. # Put the constant inputs in the slice. idx_cst = get_idx_list(node.inputs[1:], node.op.idx_list) if all(isinstance(e, slice) and e.start is None and e.stop is None and (e.step is None or T.extract_constant(e.step, only_process_constants=True) == -1) for e in idx_cst): # IncSubtensor broadcast node.inputs[1] on node.inputs[0] # based on run time shapes, so we must check they are the same. if not hasattr(node.fgraph, 'shape_feature'): return if not node.fgraph.shape_feature.same_shape(node.inputs[0], node.inputs[1]): return # There is no reverse, so we don't need a replacement. if all(e.step is None for e in node.op.idx_list): # They are the same shape, so we can remore this IncSubtensor return [node.inputs[1]] ret = Subtensor(node.op.idx_list)(*node.inputs[1:]) # Copy over previous output stacktrace copy_stack_trace(node.outputs, ret) return [ret] @register_canonicalize @gof.local_optimizer([AdvancedIncSubtensor1]) def local_set_to_inc_subtensor(node): """ AdvancedIncSubtensor1(x, x[ilist]+other, ilist, set_instead_of_inc=True) -> AdvancedIncSubtensor1(x, other, ilist, set_instead_of_inc=False) """ if (isinstance(node.op, AdvancedIncSubtensor1) and node.op.set_instead_of_inc and node.inputs[1].owner and isinstance(node.inputs[1].owner.op, Elemwise) and isinstance(node.inputs[1].owner.op.scalar_op, scalar.Add)): addn = node.inputs[1].owner subn = None other = None if (addn.inputs[0].owner and isinstance(addn.inputs[0].owner.op, AdvancedSubtensor1)): subn = addn.inputs[0].owner other = addn.inputs[1] elif (addn.inputs[1].owner and isinstance(addn.inputs[1].owner.op, AdvancedSubtensor1)): subn = addn.inputs[1].owner other = addn.inputs[0] else: return if (subn.inputs[1] != node.inputs[2] or subn.inputs[0] != node.inputs[0]): return ret = advanced_inc_subtensor1(node.inputs[0], other, node.inputs[2]) # Copy over previous output stacktrace # Julian: I'm not sure about this at all... copy_stack_trace(node.outputs, ret) return [ret] @register_useless @register_canonicalize @register_specialize @gof.local_optimizer([Subtensor]) def local_useless_slice(node): """ Remove Subtensor of the form X[0, :] -> X[0] """ if isinstance(node.op, Subtensor): slices = get_idx_list(node.inputs, node.op.idx_list) last_slice = len(slices) for s in slices[::-1]: # check if slice and then check slice indices if (isinstance(s, slice) and s.start is None and s.stop is None and (s.step is None or T.extract_constant(s.step, only_process_constants=True) == 1)): last_slice -= 1 else: break # check if we removed something if last_slice < len(slices): subtens = Subtensor(slices[:last_slice]) sl_ins = Subtensor.collapse(slices[:last_slice], lambda x: isinstance(x, T.Variable)) out = subtens(node.inputs[0], *sl_ins) # Copy over previous output stacktrace copy_stack_trace(node.outputs, out) return [out] @register_canonicalize @register_specialize @gof.local_optimizer([Subtensor, AdvancedSubtensor1]) def local_useless_subtensor(node): """ Remove Subtensor/AdvancedSubtensor1 if it takes the full input. In the AdvancedSubtensor1 case, the full input is taken when the indices are equivalent to `arange(0, input.shape[0], 1)` using either an explicit list/vector or the ARange op. """ # If the optimization is tried over a node that is not a part of graph before if not hasattr(node, 'fgraph'): return # This optimization needs ShapeOpt and fgraph.shape_feature if not hasattr(node.fgraph, 'shape_feature'): return shape_of = node.fgraph.shape_feature.shape_of if isinstance(node.op, Subtensor): cdata = node.op.get_constant_idx(node.inputs, allow_partial=True, only_process_constants=True) for pos, idx in enumerate(cdata): if not isinstance(idx, slice): # If idx is not a slice, this means we remove this dimension # from the output, so the subtensor is not useless return False if idx.start is not None and idx.start != 0: # If the start of the slice is different from 0, or is a # variable, then we assume the subtensor is not useless return False if idx.step is not None and idx.step != 1: # If we are going backwards, or skipping elements, then this # is not a useless subtensor return False for pos, idx in enumerate(cdata): length_pos = shape_of[node.inputs[0]][pos] if isinstance(idx.stop, (integer_types, np.integer)): length_pos_data = sys.maxsize try: length_pos_data = get_scalar_constant_value(length_pos, only_process_constants=True) except NotScalarConstantError: pass if idx.stop < length_pos_data: return False elif isinstance(idx.stop, gof.Variable): length_pos_shape_i = idx.stop # length_pos is a tensor variable, but length_pos_shape_i # is a scalar variable. We try to see if they represent # the same underlying variable. if (length_pos_shape_i.owner and isinstance(length_pos_shape_i.owner.op, T.ScalarFromTensor)): length_pos_shape_i = length_pos_shape_i.owner.inputs[0] elif (length_pos.owner and isinstance(length_pos.owner.op, T.TensorFromScalar)): length_pos = length_pos.owner.inputs[0] else: # We did not find underlying variables of the same type return False # The type can be different: int32 vs int64. length_pos # should always be int64 as that is what the shape # tracker keep. Subtensor accept any scalar int{8,16,32,64} # as index type. assert str(length_pos.type.dtype) == "int64" assert str(length_pos_shape_i.type.dtype) in ["int8", "int16", "int32", "int64"] # length_pos_shape_i cannot be None if length_pos_shape_i != length_pos: return False elif idx.stop is None: pass else: return False elif isinstance(node.op, AdvancedSubtensor1): # get length of the indexed tensor along the first axis try: length = get_scalar_constant_value(shape_of[node.inputs[0]][0], only_process_constants=True) except NotScalarConstantError: return False # get index (which must be a vector by definition) idx = node.inputs[1] # `idx` must be equivalent to [0,1,...,shape[0] - 1] to qualify for # this optimization if isinstance(idx, T.Constant): idx = idx.value if len(idx) != length: return False if np.any(idx != np.arange(length)): return False elif idx.owner is not None and isinstance(idx.owner.op, T.ARange): try: start, stop, step = map(lambda x: get_scalar_constant_value(x, only_process_constants=True), idx.owner.inputs) except NotScalarConstantError: return False if start != 0: return False if stop != length: return False if step != 1: return False else: return False else: return False # We don't need to copy over any stacktrace here, # because previous stacktrace should suffice. return [node.inputs[0]] # fast_compile to allow opt subtensor(cast{float32}(make_vector)) @register_canonicalize('fast_compile') @gof.local_optimizer([Subtensor]) def local_subtensor_lift(node): """ unary(x)[idx] -> unary(x[idx])#any broadcast pattern. Handles the following unary ops: elemwise(x,...)[idx] -> elemwise(x[idx],...) when x,... are broadcasted scalar or not broadcasted at all rebroadcast(x)[idx] => rebroadcast(x[idx]) """ if isinstance(node.op, Subtensor): u = node.inputs[0] if not u.owner or len(u.clients) > 1: return False if isinstance(u.owner.op, T.Elemwise) and len(u.owner.inputs) == 1: idx = node.inputs[1:] x_idx = node.op(u.owner.inputs[0], *idx) # Copy over previous output stacktrace copy_stack_trace(node.outputs, x_idx) ret = u.owner.op(x_idx) # Copy over previous output stacktrace # and stacktrace from previous unary operation copy_stack_trace([node.outputs[0], node.inputs[0]], ret) return [ret] if isinstance(u.owner.op, T.Elemwise): new_inputs = [] if all([sum(i.type.broadcastable) == 0 for i in u.owner.inputs]): # There is no broadcastable in the inputs idx = node.inputs[1:] new_inputs = [node.op(i, *idx) for i in u.owner.inputs] # Copy over previous output stacktrace copy_stack_trace(node.outputs[0], new_inputs) ret = u.owner.op(*new_inputs) # Copy over previous output stacktrace # and stacktrace from previous unary operation copy_stack_trace([node.outputs[0], node.inputs[0]], ret) return [ret] elif all([sum(i.type.broadcastable) in [i.ndim, 0] for i in u.owner.inputs]): # There is no broadcastable in the inputs or it is scalar idx = node.inputs[1:] new_inputs = [] for i in u.owner.inputs: if sum(i.type.broadcastable) == 0: new_inputs.append(node.op(i, *idx)) else: # If the subtensor remove some dims, we must # lower the number of dimensions of this scalar. if node.outputs[0].ndim == i.ndim: new_inputs.append(i) else: new_inputs.append( i.dimshuffle(['x'] * node.outputs[0].ndim)) # Copy over previous output stacktrace copy_stack_trace(node.outputs[0], new_inputs) ret = u.owner.op(*new_inputs) # Copy over previous output stacktrace # and stacktrace from previous unary operation copy_stack_trace([node.outputs[0], node.inputs[0]], ret) return [ret] if isinstance(u.owner.op, T.Rebroadcast): # make sure that Rebroadcast has only 1 input assert len(u.owner.inputs) == 1 # Subtensor might reduce dim., adapt broadcast pattern accordingly new_axis = [] # loop through indices being subtensor-ed # i indexes broadcastable pattern before subtensor # j indexes broadcastable pattern after subtensor j = 0 for (i, x) in enumerate(node.op.idx_list): # if its not a slice, it will reduce the dimension, should # not appear in the broascastable dimensions if isinstance(x, slice): new_axis += [(j, u.broadcastable[i])] j += 1 # now keep the broadcastable pattern of all # items not appearing in subtensor list for i in xrange(len(node.op.idx_list), len(u.broadcastable)): new_axis += [(j, u.broadcastable[i])] j += 1 subt_x = node.op(u.owner.inputs[0], *node.inputs[1:]) # Copy over previous output stacktrace copy_stack_trace(node.outputs[0], subt_x) rbcast_subt_x = T.Rebroadcast(*new_axis)(subt_x) # Copy over previous output stacktrace # and stacktrace from previous unary operation copy_stack_trace([node.outputs[0], node.inputs[0]], rbcast_subt_x) return [rbcast_subt_x] def merge_two_slices(slice1, len1, slice2, len2): """ This function merges two slices into a single slice. The code works on the assumption that: a) slice1 is actually a slice and not an index, while slice2 can be just an index. b) the two slices **have been applied consecutively** on the same tensor The output slice is **not** in canonical form, but actually just a slice that can be applied to a tensor to produce the same output as applying the two consecutive slices. ``len1`` is the length of the tensor **before** applying the first slice, while ``len2`` is the length **after** applying the first slice. """ list_opt = [local_abs_merge, local_mul_switch_sink, local_upcast_elemwise_constant_inputs, local_useless_switch, constant_folding] if type(slice1) is not slice: raise ValueError(('First provided slice should actually be of type' 'slice and not an index !'), slice1) sl1, reverse1 = get_canonical_form_slice(slice1, len1) sl2, reverse2 = get_canonical_form_slice(slice2, len2) if type(sl2) is not slice: if reverse1 is None: # The first slice is not in reverse, which makes things a lot # more clear. # In this case we need to take care only of the special cases: # len2 <=0 -> throw index error regardless of sl2 # sl2 > len2 -> throw index error # sl2 < -len2 -> throw index error # To get a index error we simply use len1+1 to indicate we are # out of bounds, because passing this index through the formula # of getting the mixed slice is not guaranteed to result in an # index error. The **issue though** if that the error will # complain about accessing element len1+1 which is probably not # too intuitive for the user val = sl1.start + sl2 * sl1.step val = T.switch(T.le(len2, 0), len1 + 1, val) val = T.switch(T.ge(sl2, len2), len1 + 1, val) val = T.switch(T.lt(sl2, 0), - len1 - 1, val) if sl1.step: val = T.switch(T.eq(sl1.step, 0), len1 + 1, val) val = pre_greedy_local_optimizer(list_opt, val) return val else: # We are in the more complex case when we do not actually know # if the first slice was in reverse or not. # in case it was not in reverse: p_val = sl1.start + sl2 * sl1.step # case it was in reverse we need to realize that we do not want # the k-th element from sl.start but the k-th element from # sl.stop backwards n_val = sl1.stop - 1 - sl2 * sl1.step if config.warn.subtensor_merge_bug: warnings.warn(( 'Your current code is fine, but Theano versions ' 'prior to 0.5rc2 might have given an incorrect result. ' 'To disable this warning, set the Theano flag ' 'warn.subtensor_merge_bug to False.')) # we need to pick either n_val or p_val and then follow same # steps as above for covering the index error cases val = T.switch(T.lt(reverse1, 0), n_val, p_val) val = T.switch(T.le(len2, 0), len1 + 1, val) val = T.switch(T.ge(sl2, len2), len1 + 1, val) val = T.switch(T.lt(sl2, 0), - len1 - 1, val) if sl1.step: val = T.switch(T.eq(sl1.step, 0), len1 + 1, val) val = pre_greedy_local_optimizer(list_opt, val) return val else: # We are deleaing with two slices that need to be put together # according to the two steps we have 4 different combinations of # positive/negative. I will denote the case I'm looking at by # suffixes to the variables (nn,np,pn,pp): flen = sl2.stop - sl2.start p_step = sl1.step * sl2.step n_step = sl1.step * sl2.step * -1 pp_start = T.minimum(sl1.start + sl2.start * sl1.step, sl1.stop) pp_stop = T.minimum(sl1.start + sl2.stop * sl1.step, sl1.stop) pn_stop = sl1.start + (sl2.start - 1) * sl1.step pn_stop = T.switch(T.and_(T.lt(pn_stop, 0), T.gt(flen, 0)), -len1 - 1, T.minimum(pn_stop, sl1.stop)) pn_start = sl1.start + (sl2.stop - 1) * sl1.step pn_start = T.minimum(pn_start, sl1.stop) pn_start = T.maximum(pn_start, 0) np_stop = sl1.stop - sl2.stop * sl1.step - 1 np_stop = T.switch(T.and_(T.lt(np_stop, 0), T.gt(flen, 0)), -len1 - 1, T.maximum(sl1.start - 1, np_stop)) np_start = T.maximum(sl1.start, sl1.stop - sl2.start * sl1.step - 1) nn_start = T.maximum(sl1.start, (sl1.stop - 1) - (sl2.stop - 1) * sl1.step) nn_stop = T.maximum(sl1.start, sl1.stop - sl2.start * sl1.step) start = T.switch(T.lt(reverse2 * reverse1, 0), T.switch(T.lt(reverse1, 0), np_start, pn_start), T.switch(T.lt(reverse1, 0), nn_start, pp_start)) stop = T.switch(T.lt(reverse2 * reverse1, 0), T.switch(T.lt(reverse1, 0), np_stop, pn_stop), T.switch(T.lt(reverse1, 0), nn_stop, pp_stop)) step = T.switch(T.lt(reverse2 * reverse1, 0), n_step, p_step) start = T.switch(T.le(flen, 0), 0, start) stop = T.switch(T.le(flen, 0), 0, stop) # The canonical form of the slice is pretty complicated # and is not simplified. We simplify it in advance here # as otherwise this create too many useless optimization that # DebugMode must check. start = pre_greedy_local_optimizer(list_opt, start) stop = pre_greedy_local_optimizer(list_opt, stop) step = pre_greedy_local_optimizer(list_opt, step) start = pre_greedy_local_optimizer(list_opt, start) stop = pre_greedy_local_optimizer(list_opt, stop) step = pre_greedy_local_optimizer(list_opt, step) # Pre merge constant for the same reason. start, stop, step = pre_constant_merge([start, stop, step]) return slice(start, stop, step) @register_canonicalize @register_specialize @gof.local_optimizer([Subtensor]) def local_subtensor_merge(node): """ Refactored optimization to deal with all cases of tensor merging. Given a subgraph of the form Subtensor(Subtensor(u)), the optimization expresses all slices in a canonical form, and then merges them together. """ if isinstance(node.op, Subtensor): u = node.inputs[0] if u.owner and isinstance(u.owner.op, Subtensor): # We can merge :) # x actual tensor on which we are picking slices x = u.owner.inputs[0] # slices of the first applied subtensor slices1 = get_idx_list(u.owner.inputs, u.owner.op.idx_list) slices2 = get_idx_list(node.inputs, node.op.idx_list) # Get the shapes of the vectors ! try: # try not to introduce new shape into the graph xshape = node.fgraph.shape_feature.shape_of[x] ushape = node.fgraph.shape_feature.shape_of[u] except AttributeError: # Following the suggested use of shape_feature which should # consider the case when the compilation mode doesn't # include the ShapeFeature xshape = x.shape ushape = u.shape merged_slices = [] pos_2 = 0 pos_1 = 0 while (pos_1 < len(slices1)) and (pos_2 < len(slices2)): slice1 = slices1[pos_1] if type(slice1) is slice: merged_slices.append( merge_two_slices(slice1, xshape[pos_1], slices2[pos_2], ushape[pos_2])) pos_2 += 1 else: merged_slices.append(slice1) pos_1 += 1 if pos_2 < len(slices2): merged_slices += slices2[pos_2:] else: merged_slices += slices1[pos_1:] merged_slices = make_constant(merged_slices) subtens = Subtensor(merged_slices) sl_ins = Subtensor.collapse( merged_slices, lambda x: isinstance(x, T.Variable)) # Do not call make_node for test_value out = subtens(x, *sl_ins) # Copy over previous output stacktrace # and stacktrace from previous slicing operation. # Why? Because, the merged slicing operation could have failed # because of either of the two original slicing operations orig_out = node.outputs[0] copy_stack_trace([orig_out, node.inputs[0]], out) # Restore original broadcastable dimensions that `subtens()` may # have been unable to infer again if out.type != orig_out.type: assert out.dtype == orig_out.dtype assert out.ndim == orig_out.ndim out = T.patternbroadcast(out, orig_out.broadcastable) copy_stack_trace([orig_out, node.inputs[0]], out) return [out] @register_useless @register_canonicalize @register_specialize @gof.local_optimizer([Subtensor]) def local_subtensor_of_alloc(node): """ alloc(val)[x:y] -> alloc(val[...]) alloc(val)[x:y] -> alloc(val) This can be seen as a lift, but it also reduce the number of computation/memory. """ if not isinstance(node.op, Subtensor): return False u = node.inputs[0] if u.owner is None: return False if not isinstance(u.owner.op, T.Alloc): return False slices = get_idx_list(node.inputs, node.op.idx_list) val = u.owner.inputs[0] dims = u.owner.inputs[1:] assert len(slices) <= len(dims) # Number of dimensions added to val n_added_dims = u.ndim - val.ndim # Dimensions of the returned alloc nw_dims = [] # Slices to take from val val_slices = [] for i, (sl, dim) in enumerate(zip(slices, dims)): # If val was not copied over that dim, # we need to take the appropriate subtensor on it. if i >= n_added_dims: # We check that the corresponding val dimensions was # not a broadcasted dimensions. if (val.type.ndim > (i - n_added_dims) and val.type.broadcastable[i - n_added_dims]): val_slices.append(slice(None)) else: val_slices.append(sl) csl, _ = get_canonical_form_slice(sl, dim) if type(csl) is not slice: # That dimension is removed. pass else: nw_dim = csl.stop - csl.start if csl.step != 1: # Do not add the ceil_intdiv() graphs in the graphs # when this is not needed as it prevent detecting the # correct broadcast pattern. nw_dim = T.ceil_intdiv(nw_dim, csl.step) nw_dims += [nw_dim] nw_val = val[tuple(val_slices)] nw_dims += dims[len(slices):] if nw_val.ndim > len(nw_dims): return False rval = T.alloc(nw_val, *nw_dims) if type(rval) not in (list, tuple): rval = [rval] if rval[0].type != node.outputs[0].type: # It happen that the make_node() isn't able to infer the same pattern. # We know it is safe, so fix that. rval[0] = T.patternbroadcast(rval[0], node.outputs[0].broadcastable) return rval @register_canonicalize @register_stabilize @register_specialize @gof.local_optimizer([Subtensor]) def local_subtensor_of_dot(node): """ This optimization translates T.dot(A, B)[idxs] into T.dot(A[idxs_a], B[idxs_b]), where idxs_a and idxs_b are defined appropriately. idxs_a is the first A.ndim-1 entries of idxs, and idxs_b is the remaining entries of idxs (if any), modified to skip the second-to-last dimension of B (because dot sums over this dimension). """ if not isinstance(node.op, Subtensor): return if (not node.inputs[0].owner or not isinstance(node.inputs[0].owner.op, T.Dot)): return # If there is other node that use the outputs of the dot # We don't want to compute twice the sub part. if len(node.inputs[0].clients) > 1: return a = node.inputs[0].owner.inputs[0] b = node.inputs[0].owner.inputs[1] idx_list = get_idx_list(node.inputs, node.op.idx_list) num_a_indices = min(a.ndim - 1, len(idx_list)) a_indices = idx_list[:num_a_indices] b_indices = idx_list[num_a_indices:] # This is necessary because np.dot sums the last index of a with the second to last of b # so we want to skip the second-to-last index into b. # This wasn't necessary for a, because we just ommitted the last index. # We skip this if b.ndim = 1, since then we just want b_sub = b, not b_sub = b[:] # (dot also handles b.ndim < 2 as a special case) if b.ndim > 1 and len(b_indices) >= b.ndim - 1: b_indices = (b_indices[:b.ndim - 2] + (slice(None, None, None),) + b_indices[b.ndim - 2:]) a_sub = a.__getitem__(tuple(a_indices)) b_sub = b.__getitem__(tuple(b_indices)) if b_indices else b # Copy over previous output stacktrace to a_sub and b_sub, # because an error in the subtensor operation (e.g. an index error) # on either a or b must correspond to an error in the # subtensor operation on their dot product. copy_stack_trace(node.outputs[0], [a_sub, b_sub]) # Copy over previous output stacktrace and previous dot product stacktrace, # because an error here may correspond to an either in either the original # dot product, or in the dot product after the subtensor operation. r = T.dot(a_sub, b_sub) copy_stack_trace([node.outputs[0], node.inputs[0]], r) return [r] @register_canonicalize @gof.local_optimizer([T.add]) def local_IncSubtensor_serialize(node): """ When using Subtensor, gradient graphs can be ugly. If we ask for grad(f(a[0]), a), we are going to get something like IncSubtensor(Elemwise{second}(a, 0), g(f(a[0])), [0]) This might be ugly, but at least it's as fast as you could want. If we ask for grad(f(a[0], a[1], a[2]), a), it's much worse... Elemwise{Add} IncSubtensor(Elemwise{second}(a, 0), g(f(a[0])), [0]) IncSubtensor(Elemwise{second}(a, 0), g(f(a[1])), [1]) IncSubtensor(Elemwise{second}(a, 0), g(f(a[2])), [2]) This is much worse because this time we have to produce 3 matrices the size of 'a', just so we can add them together. This Op rearranges IncSubtensor's that all work on the same initial argument (here, Elemwise{second}(a,0)) into a chain. The advantage of the chain structure is that each one can be optimized later in the pipeline to operate inplace. Ideally, the op will do something like this: # # add(x, incsubtensor(b, c), incsubtensor(b, d)) # -> incsubtensor(incsubtensor(add(x,b,b), c), d) """ def movable(i): # Return True iff this is a incsubtensor that we can move return (i.owner and isinstance(i.owner.op, (IncSubtensor, AdvancedIncSubtensor1, AdvancedIncSubtensor,)) and i.type == o_type and len(i.clients) == 1 and not i.owner.op.set_instead_of_inc) if node.op == T.add: o_type = node.outputs[0].type movable_inputs = [i for i in node.inputs if movable(i)] if movable_inputs: new_inputs = ([i for i in node.inputs if not movable(i)] + [mi.owner.inputs[0] for mi in movable_inputs]) if len(new_inputs) == 0: new_add = new_inputs[0] else: new_add = T.add(*new_inputs) # Copy over stacktrace from original output, as an error # (e.g. an index error) in this add operation should # correspond to an error in the original add operation. copy_stack_trace(node.outputs[0], new_add) # stack up the new incsubtensors tip = new_add for mi in movable_inputs: assert tip.type == o_type assert tip.type == mi.owner.inputs[0].type tip = mi.owner.op(tip, *mi.owner.inputs[1:]) # Copy over stacktrace from outputs of the original # "movable" operation to the new operation. copy_stack_trace(node.outputs + mi.owner.outputs, tip) return [tip] # print incsub_inputs, [id(i.owner.inputs[0]) for i in incsub_inputs] # We register it in a TopoOptimizer inside the canonizer EQ optimizer. # Otherwise in some cases it was making the EQ optimizer use 45. In # the TopoOptimizer, the EQ only use 5 passes. compile.optdb.register('pre_local_IncSubtensor_serialize', in2out(local_IncSubtensor_serialize), # Just before canonizer 0.99, 'fast_run') # after priority 50 Destructive inplace operations # gemm is the first one now, at priority 70 @gof.local_optimizer([IncSubtensor], inplace=True) def local_inplace_setsubtensor(node): """ Also work for GpuIncSubtensor. """ if isinstance(node.op, IncSubtensor) and not node.op.inplace: dta = node.op.destroyhandler_tolerate_aliased new_op = node.op.__class__( node.op.idx_list, inplace=True, set_instead_of_inc=node.op.set_instead_of_inc, destroyhandler_tolerate_aliased=dta) new_node = new_op(*node.inputs) val = getattr(node.outputs[0].tag, 'nan_guard_mode_check', True) new_node.tag.nan_guard_mode_check = val # Copy stacktrace from original outputs to new outputs. # This is sensible, because the new operation is the # same as the old one, but now with different attributes. copy_stack_trace(node.outputs, new_node) return [new_node] return False compile.optdb.register('local_inplace_setsubtensor', TopoOptimizer( local_inplace_setsubtensor, failure_callback=TopoOptimizer.warn_inplace), 60, 'fast_run', 'inplace') # DEBUG @gof.local_optimizer([AdvancedIncSubtensor1], inplace=True) def local_inplace_incsubtensor1(node): """ Also work for GpuAdvancedIncSubtensor1. """ if isinstance(node.op, AdvancedIncSubtensor1) and not node.op.inplace: new_op = node.op.clone_inplace() new_node = new_op(*node.inputs) # Copy stacktrace from original outputs to new outputs. # This is sensible, because the new operation is the # same as the old one, but now with different attributes. copy_stack_trace(node.outputs, new_node) return [new_node] return False compile.optdb.register('local_inplace_incsubtensor1', TopoOptimizer( local_inplace_incsubtensor1, failure_callback=TopoOptimizer.warn_inplace), 60, 'fast_run', 'inplace') # DEBUG # Register old name @register_canonicalize("local_incsubtensor_of_allocs") @register_stabilize("local_incsubtensor_of_allocs") @gof.local_optimizer([IncSubtensor, AdvancedIncSubtensor, AdvancedIncSubtensor1]) def local_incsubtensor_of_zeros(node): """ IncSubtensor(x, zeros, idx) -> x """ if (isinstance(node.op, (IncSubtensor, AdvancedIncSubtensor, AdvancedIncSubtensor1)) and not node.op.set_instead_of_inc): x = node.inputs[0] y = node.inputs[1] try: # Don't use only_process_constants=True. We need to # investigate Alloc of 0s but with non constant shape. if get_scalar_constant_value(y, elemwise=False) == 0: # No need to copy over the stacktrace, # because x should already have a stacktrace return [x] except NotScalarConstantError: return @register_canonicalize @register_specialize @gof.local_optimizer([IncSubtensor]) def local_incsubtensor_of_zeros_to_setsubtensor(node): """ IncSubtensor(zeros, x, ...) -> SetSubtensor(zeros, x, ...) """ if (isinstance(node.op, (IncSubtensor)) and not node.op.set_instead_of_inc): x = node.inputs[0] if isinstance(x, T.Constant) and not np.any(x.data): return [IncSubtensor(node.op.idx_list, node.op.inplace, set_instead_of_inc=True, destroyhandler_tolerate_aliased=node.op.destroyhandler_tolerate_aliased, )(*node.inputs)] @register_canonicalize('local_setsubtensor_of_allocs') @register_stabilize('local_setsubtensor_of_allocs') @gof.local_optimizer([IncSubtensor]) def local_setsubtensor_of_constants(node): """ SetSubtensor(x, x[idx], idx) -> x when x is constant or alloc. """ if isinstance(node.op, IncSubtensor) and node.op.set_instead_of_inc: x = node.inputs[0] y = node.inputs[1] # Don't use only_process_constants=True. We need to # investigate Alloc of 0s but with non constant shape. try: replace_x = get_scalar_constant_value(x, elemwise=False) except NotScalarConstantError: return try: replace_y = get_scalar_constant_value(y, elemwise=False) except NotScalarConstantError: return if replace_x == replace_y: # No need to copy over the stacktrace, # because x should already have a stacktrace return [x] else: return False @register_canonicalize @register_stabilize @gof.local_optimizer([AdvancedSubtensor1]) def local_adv_sub1_adv_inc_sub1(node): """Optimize the possible AdvSub1(AdvSetSub1(...), ...). AdvancedSubtensor1(AdvancedSetSubtensor1(x, y, idx), idx) -> y Notes ----- This opt add AssertOp. Otherwise, it would remove shape and index error. If you want to get rid of them, see the :ref:`unsafe_optimization` section. WARNING: A previous version of this optimization also matched AdvancedSubtensor1(AdvancedIncSubtensor1(0s, y, idx), idx) -> y This is incorrect when there are duplicate indices. The current version warns the user about potential past issues. """ if not isinstance(node.op, AdvancedSubtensor1): return inp = node.inputs[0] if (not inp.owner or not isinstance(inp.owner.op, AdvancedIncSubtensor1)): return idx = node.inputs[1] idx2 = inp.owner.inputs[2] x = inp.owner.inputs[0] y = inp.owner.inputs[1] if idx is not idx2: return if (not inp.owner.op.set_instead_of_inc and # Don't use only_process_constants=True. We need to # investigate Alloc of 0s but with non constant shape. T.extract_constant(x, elemwise=False) != 0): return if not inp.owner.op.set_instead_of_inc: if config.warn.inc_subtensor1_opt: warnings.warn( 'Your current code is fine, but Theano versions ' 'between 0.7rc1 and 0.10 (or development versions ' 'between Nov. 2014 and May 2017) ' 'might have given incorrect results. This graph has ' 'following pattern: inc_subtensor(zeros[idx], x)[idx], ' 'where idx is an array of integers. This used to be ' 'optimized to "x", which is incorrect if there are ' 'duplicated indices in idx. ' 'To disable this warning, set the Theano flag ' 'warn.inc_subtensor1_opt to False.') return cond = [T.all(T.and_(T.lt(idx, x.shape[0]), T.ge(idx, -x.shape[0])))] if not node.fgraph.shape_feature.same_shape(idx, y, 0, 0): cond.append(T.eq(idx.shape[0], y.shape[0])) r = Assert("Bad indexing or shapes in a AdvancedIncSubtensor1 " "that was optimized away")(y, *cond) copy_stack_trace(y, r) if r.dtype == node.outputs[0].dtype: return [r] # It is possible that y is upcast or downcast to x.dtype. # In all case, as we set or add with 0, we can just cast y. r2 = T.cast(r, node.outputs[0].dtype) # Copy over stacktrace from before casting, since # we don't expect problems in the casting operation, # and any problems in the indexing would have been spotted above. copy_stack_trace(r, r2) return [r2] @register_specialize @register_stabilize @register_canonicalize @register_useless @gof.local_optimizer([IncSubtensor, AdvancedIncSubtensor, AdvancedIncSubtensor1]) def local_useless_inc_subtensor_alloc(node): """ Replaces an [Advanced]IncSubtensor[1], whose increment is an `alloc` of a fully or partially broadcastable variable, by one that skips the intermediate `alloc` where possible. """ if isinstance(node.op, (IncSubtensor, AdvancedIncSubtensor, AdvancedIncSubtensor1)): x = node.inputs[0] y = node.inputs[1] i = node.inputs[2:] if y.owner is not None and isinstance(y.owner.op, T.Alloc): # `z` is the input of the Alloc op, i.e. T.alloc(z, ) z = y.owner.inputs[0] try: shape_feature = node.fgraph.shape_feature except AttributeError: # The shape feature may not be available in some mode, but we # need it for this optimization, so don't continue. return False shape_of = shape_feature.shape_of same_shape = shape_feature.same_shape # Get the subtensor of `x` indexed by `i` in order to compare # shapes later. if isinstance(node.op, IncSubtensor): xi = Subtensor(node.op.idx_list)(x, *i) elif isinstance(node.op, AdvancedIncSubtensor): xi = advanced_subtensor(x, *i) elif isinstance(node.op, AdvancedIncSubtensor1): xi = advanced_subtensor1(x, *i) else: raise Exception('Should never happen!') reason = 'local_useless_incsubtensor_alloc' # Add `xi` to the shape feature `fgraph`. This is important for # shape inference later because the variable must be part of the # function graph in order to call `same_shape` on it. if xi not in shape_of: shape_feature.on_import(node.fgraph, xi.owner, '%s: add `xi`' % reason) # `xi` may have more dimensions than `y` since the subtensor ops # do automatic broadcasting of the increment internally. Thus, we # need to make the leading implicitly broadcasted dimensions # explicit for shape comparison later. if xi.ndim > y.ndim: y = T.shape_padleft(y, xi.ndim - y.ndim) if y not in shape_of: shape_feature.on_import(node.fgraph, y.owner, '%s: add `y`' % reason) # Build `z_broad` explicitly to include extra implicit dimensions. z_broad = ((True,) * (xi.ndim - z.ndim) + z.broadcastable) cond = [ # The shapes of `y` and `xi` must either agree or `y` may # also have shape equal to 1 which may be treated as a # broadcastable dimension by the subtensor op. T.or_(T.eq(y.shape[k], 1), T.eq(y.shape[k], xi.shape[k])) # Loop over all dimensions. for k in xrange(xi.ndim) # We need to check the above shapes, if # * the pre-alloc increment `z` is broadcastable in # dimension `k` (if it isn't, then the shapes of `z` and # `y` are the same by the definition of the `Alloc` op in # this dimension and replacing `y` by `z` will not hide a # shape error), and # * `xi` and `y` do not have the same shape in dimension # `k` or we cannot infer the shape statically (if the # shapes of `xi` and `y` are not the same, then replacing # `y` by `z` will hide the shape error of `y`), and # * the shape of `y` is not equal to 1 or we cannot infer # the shape statically (if the shape of `y` is equal to # 1, then `y` is broadcasted by the inc_subtensor op # internally, so the shapes of `xi` and `y` do not need # to match in dimension `k`; else we need to check at # runtime that the shape of `y` is either 1 or the same # as `xi` or otherwise replacing `y` by `z` will hide a # shape error). if (z_broad[k] and not same_shape(xi, y, dim_x=k, dim_y=k) and shape_of[y][k] != 1)] if len(cond) > 0: msg = '`x[i]` and `y` do not have the same shape.' z = Assert(msg)(z, *cond) r = node.op(x, z, *i) # Copy over stacktrace from previous output, since # we don't expect problems when removing the intermediate # alloc operation and so we still want to point at the line # of the inc_subtensor operation. copy_stack_trace(node.outputs, r) return [r] #################### # Rebroadcast opts # #################### @register_useless @register_canonicalize @register_specialize @gof.local_optimizer([T.Rebroadcast]) def local_useless_rebroadcast(node): """ Remove Rebroadcast if id does not actually change the broadcasting pattern. """ if isinstance(node.op, T.Rebroadcast): x = node.inputs[0] if np.all(x.broadcastable == node.outputs[0].broadcastable): # No broadcastable flag was modified # No need to copy over stack trace, # because x should already have a stack trace. return [x] else: # Keep the flags that modify something new_axis = {} for dim, bc in list(node.op.axis.items()): if x.broadcastable[dim] != bc: new_axis[dim] = bc if new_axis == node.op.axis: # All flags are useful return else: r = T.Rebroadcast(*list(new_axis.items()))(x) # Copy over stacktrace from previous output copy_stack_trace(node.outputs, r) return [r] @register_canonicalize @register_specialize @gof.local_optimizer([T.Rebroadcast]) def local_rebroadcast_lift(node): """ Lifts Rebroadcast through unary Elemwise operations, and merges consecutive Rebroadcasts. Rebroadcast(Elemwise(x)) => Elemwise(Rebroadcast(x)) Rebroadcast(Rebroadcast(x)) => Rebroadcast(x) """ op = node.op if not isinstance(op, T.Rebroadcast): return False input = node.inputs[0] inode = input.owner if inode and isinstance(inode.op, Elemwise) and len(inode.inputs) == 1: # It may happen that `input` has no client because this optimization # is called from `apply_rebroadcast_opt`, which in particular is used # by the `unbroadcast` function before we are in the actual function # compilation phase. if hasattr(input, 'clients') and len(input.clients) == 1: rebroadcasted = T.Rebroadcast(*list(op.axis.items()))( inode.inputs[0]) # Copy over stacktrace from previous output (after rebroadcasting) # to new output, because an error in the new graph right after # rebroadcasting must have been caused by the previous rebroadcasting. copy_stack_trace(node.outputs, rebroadcasted) rval = inode.op.make_node(rebroadcasted).outputs # Copy over stacktrace from previous output (after rebroadcasting) # and input (after elemwise operation) to new output, because an # error in the new graph could have been caused by either of the # two ops. copy_stack_trace(node.outputs + node.inputs, rval) return rval if inode and isinstance(inode.op, T.Rebroadcast): # the "axis" specification in the outer Rebroadcast overrides # the axis of the inner one axis = inode.op.axis.copy() axis.update(op.axis) iinput = inode.inputs[0] rval = [T.Rebroadcast(*list(axis.items()))(iinput)] # Copy over stacktrace from previous output (after second rebroadcast) # and from previous input (after first rebroadcast op) because an error in # the new graph could have been caused by either of the two # rebroadcast ops. copy_stack_trace(node.outputs + node.inputs, rval) return rval def apply_rebroadcast_opt(rval): """ Apply as many times as required the optimization local_useless_rebroadcast and local_rebroadcast_lift. Parameters ---------- rval: a Variable Returns ------- A Variable (the same if no optimization can be applied) """ changed = True while changed and rval.owner: changed = False rval2 = theano.tensor.opt.local_useless_rebroadcast.transform( rval.owner) if rval2: assert len(rval2) == 1 rval = rval2[0] changed = True if rval.owner: rval2 = theano.tensor.opt.local_rebroadcast_lift.transform( rval.owner) if rval2: assert len(rval2) == 1 rval = rval2[0] changed = True return rval ############# # Join opts # ############# @register_specialize @register_canonicalize @register_useless @gof.local_optimizer([T.Join]) def local_join_1(node): """Join(i, x) => x Remove Join() when only one element is joined. """ if not isinstance(node.op, T.Join): return tensors = node.inputs[1:] if len(tensors) == 1: # We don't need to copy over any stacktrace here, because the # input variable should already have its own stacktrace. return [tensors[0]] # TODO: merge in local_useless_join @register_useless @register_specialize @register_canonicalize @gof.local_optimizer([T.Join]) def local_join_empty(node): """Join(i, x, y, empty) => Join(i, x, y) Remove empty inputs to joins. The empty inputs can be anywhere. """ if not isinstance(node.op, T.Join): return new_inputs = [] try: join_idx = get_scalar_constant_value(node.inputs[0], only_process_constants=True) except NotScalarConstantError: return for idx in xrange(1, len(node.inputs)): inp = node.inputs[idx] # We can not use size == 0,, as this can change shape from 3,0 # to 2,0. This trigger DebugMode error. This happen with # stack(...,[]) as this add a dimshuffle on [], that add a # dimensions with shape 1. if isinstance(inp, theano.Constant) and inp.data.shape[join_idx] == 0: continue new_inputs.append(inp) if len(new_inputs) < len(node.inputs) - 1: if len(new_inputs) == 0: # T.join do not work in that case. # constant folding will take care of this case. return ret = T.join(node.inputs[0], *new_inputs) o = node.outputs[0] if ret.dtype != o.dtype: # Join can upcast some inputs return # Copy over stacktrace from previous output (after join op) # to new output, because an error in the new op must be caused # by an error in the old join op. copy_stack_trace(node.outputs, ret) if ret.type != o.type: assert ret.dtype == o.dtype assert ret.ndim == o.ndim ret = T.patternbroadcast(ret, node.outputs[0].broadcastable) # Copy over stacktrace from previous output # (after patternbroadcast op) for same reasons as before. copy_stack_trace(node.outputs, ret) return [ret] @register_specialize @register_canonicalize @register_useless @gof.local_optimizer([T.Join]) def local_join_make_vector(node): """Join(0, make_vector1, make_vector2, ...) => Join(0, make_vector12, ...) Merge MakeVector inputs to Join. This can make the join completly disapear with the local_join_1 opt. """ if not isinstance(node.op, T.Join) or node.outputs[0].ndim != 1: return new_inputs = [node.inputs[1]] for idx in xrange(2, len(node.inputs)): inp = node.inputs[idx] if (inp.owner and isinstance(inp.owner.op, MakeVector) and new_inputs[-1].owner and isinstance(new_inputs[-1].owner.op, MakeVector) and # MakeVector have a dtype parameter inp.owner.op == new_inputs[-1].owner.op): inps = new_inputs[-1].owner.inputs + inp.owner.inputs new_inputs[-1] = inp.owner.op(*inps) # Copy over stacktrace from previous output (after join op) # to new intermediate output, because an error in the intermediate # op must be caused by an error in the old join op. copy_stack_trace(node.outputs, new_inputs[-1]) else: new_inputs.append(inp) if len(new_inputs) < len(node.inputs) - 1: ret = T.join(node.inputs[0], *new_inputs) # Copy over stacktrace from previous output (after join op) # to new output, because an error in the new op must be caused # by an error in the old join op. copy_stack_trace(node.outputs, ret) return [ret] ################# # speed/memory # ################# @register_canonicalize @register_specialize @gof.local_optimizer([T.elemwise.Sum]) def local_sumsqr2dot(node): """ This optimization detects T.sqr( W.dimshuffle('x',0,1) * G.dimshuffle(0,'x',1) ).sum(axis=(1,2)) and converts this to T.dot(T.sqr(G), T.sqr(W).sum(axis=0)). """ if (isinstance(node.op, T.elemwise.Sum) and isinstance(node.op.scalar_op, theano.scalar.basic.Add) and node.op.axis == (1, 2)): in1 = node.inputs[0] out = node.outputs[0] if (in1.owner and isinstance(in1.owner.op, T.Elemwise) and isinstance(in1.owner.op.scalar_op, theano.scalar.basic.Sqr)): in_sqr = in1.owner.inputs[0] if (in_sqr.owner and isinstance(in_sqr.owner.op, T.Elemwise) and isinstance(in_sqr.owner.op.scalar_op, theano.scalar.basic.Mul) and len(in_sqr.owner.inputs) == 2): in_mul1, in_mul2 = in_sqr.owner.inputs if (isinstance(in_mul1.owner.op, T.elemwise.DimShuffle) and in_mul1.owner.op.new_order == ('x', 0, 1) and isinstance(in_mul2.owner.op, T.elemwise.DimShuffle) and in_mul2.owner.op.new_order == (0, 'x', 1)): W = in_mul1.owner.inputs[0] G = in_mul2.owner.inputs[0] new_out = T.dot(T.sqr(G), T.sqr(W).sum(axis=0)) if new_out.dtype != out.dtype: new_out = T.cast(new_out, dtype=out.dtype) return [new_out] ################# # Exp stability # ################# @register_stabilize @register_specialize @register_canonicalize @gof.local_optimizer([T.Elemwise]) def local_expm1(node): """ This optimization detects exp(a)-1 and converts this to expm1(a). """ if (isinstance(node.op, T.Elemwise) and isinstance(node.op.scalar_op, theano.scalar.basic.Sub)): in1, in2 = node.inputs out = node.outputs[0] if (in1.owner and isinstance(in1.owner.op, T.Elemwise) and isinstance(in1.owner.op.scalar_op, theano.scalar.basic.Exp) and T.extract_constant(in2, only_process_constants=False) == 1): in11 = in1.owner.inputs[0] new_out = T.expm1(in11) if new_out.dtype != out.dtype: new_out = T.cast(new_out, dtype=out.dtype) if new_out.type != out.type: return return [new_out] ############### # Switch opts # ############### @register_useless('local_remove_switch_const_cond') @register_canonicalize('fast_compile', 'local_remove_switch_const_cond') @register_specialize @gof.local_optimizer([T.Elemwise]) def local_useless_switch(node): """ This optimization makes the following changes in the graph: T.switch(cond,left,right) --> if cond is constant and cond == 0: right if cond is constant and cond != 0: left if left is right -> left T.switch(le(shape_i{id}(X), 0), 0, shape_i{id}(X)) -> shape_i{id}(X) """ if (isinstance(node.op, T.Elemwise) and isinstance(node.op.scalar_op, scalar.basic.Switch)): cond = T.extract_constant(node.inputs[0], only_process_constants=True) if ((type(cond) is np.ndarray and cond.ndim == 0) or isinstance(cond, np.number)): if cond == 0: correct_out = node.inputs[2] else: correct_out = node.inputs[1] if correct_out.ndim != node.outputs[0].ndim: # TODO: broadcast? return False if correct_out.dtype != node.outputs[0].dtype: out = T.cast(correct_out, node.outputs[0].dtype) else: out = correct_out if out.type.broadcastable != node.outputs[0].type.broadcastable: # We need to copy data to the new dimensions during execution # We should not depend on node.outputs as this would # make the new node depend on the old one that will # get optimized again. So this create a cycle. shps = [] for idx, (b1, b2), in enumerate(zip(out.type.broadcastable, node.outputs[0].type.broadcastable)): if b1 == b2: shps.append(out.shape[idx]) elif not node.inputs[1].type.broadcastable[idx]: shps.append(node.inputs[1].shape[idx]) else: shps.append(node.inputs[2].shape[idx]) out = T.alloc(out, *shps) else: out = out # Copy over stacktrace from selected output to new output copy_stack_trace(node.outputs + correct_out, out) return [out] # if left is right -> left if node.inputs[1] is node.inputs[2]: # Note: No need to copy over stacktrace, because the input node # already has its own stacktrace if cond.type == node.inputs[1].type: return [node.inputs[1]] ret = T.fill(cond, node.inputs[1]) # Copy over stacktrace from switch output and correct branch copy_stack_trace(node.outputs + node.inputs[1], ret) return [ret] # This case happens with scan. # Elemwise{switch}(le(shape_i{id}(X), 0), 0, shape_i{id}(X)) -> shape_i{id}(X) left = node.inputs[1] right = node.inputs[2] cond_var = node.inputs[0] if cond_var.owner and \ isinstance(cond_var.owner.op, T.Elemwise) and \ isinstance(cond_var.owner.op.scalar_op, scalar.LE) and \ cond_var.owner.inputs[0].owner and \ isinstance(cond_var.owner.inputs[0].owner.op, Shape_i) and \ T.extract_constant(cond_var.owner.inputs[1], only_process_constants=True) == 0 and \ T.extract_constant(left, only_process_constants=True) == 0 and \ right is cond_var.owner.inputs[0]: assert right.type == node.outputs[0].type # No need to copy over stacktrace, because the right input node # already has its own stacktrace return [right] return False return False @register_specialize @register_canonicalize @gof.local_optimizer([T.mul]) def local_mul_switch_sink(node): """ This optimization makes the folowing changes in the graph: T.mul(A,T.switch(cond,0,iff),B) --> T.switch(cond,0,T.mul(A,B,iff)) T.mul(A,T.switch(cond,ift,0),B) --> T.switch(cond,T.mul(A,B,ift),0) A and B being several (or none) symbolic variables. This is useful because A and B may not be numerically stable and give NaN or inf values for cases where the switch returns 0. With this optimization T.grad(T.switch(...)) has the right behavior. Examples -------- x -> f(x) x -> g(x) y = T.switch(cond,f(x),g(x)) **without the optimization T.grad(y,x) -> grad(f(x),x) * grad(y,f(x)) + grad(g(x),x) * grad(y,g(x)) **with the optimization T.grad(y,x) -> switch(cond,grad(f(x),x), 0) + switch(cond,0,grad(g(x),x)) This will be particularly useful for the lazyif because we skip an entire part of the graph. """ if node.op != T.mul: return False for idx, i in enumerate(node.inputs): if i.owner and i.owner.op == T.switch: switch = i.owner try: if (get_scalar_constant_value( switch.inputs[1], only_process_constants=True) == 0.): listmul = node.inputs[:idx] + node.inputs[idx + 1:] fmul = T.mul(*(listmul + [switch.inputs[2]])) # Copy over stacktrace for elementwise multiplication op # from previous elementwise multiplication op. # An error in the multiplication (e.g. errors due to # inconsistent shapes), will point to the # multiplication op. copy_stack_trace(node.outputs, fmul) fct = [T.switch(switch.inputs[0], 0, fmul)] fct[0].tag.values_eq_approx = values_eq_approx_remove_nan # Copy over stacktrace for switch op from both previous # elementwise multiplication op and previous switch op, # because an error in this part can be caused by either # of the two previous ops. copy_stack_trace(node.outputs + switch.outputs, fct) return fct except NotScalarConstantError: pass try: if (get_scalar_constant_value( switch.inputs[2], only_process_constants=True) == 0.): listmul = node.inputs[:idx] + node.inputs[idx + 1:] fmul = T.mul(*(listmul + [switch.inputs[1]])) # Copy over stacktrace for elementwise multiplication op # from previous elementwise multiplication op. # An error in the multiplication (e.g. errors due to # inconsistent shapes), will point to the # multiplication op. copy_stack_trace(node.outputs, fmul) fct = [T.switch(switch.inputs[0], fmul, 0)] fct[0].tag.values_eq_approx = values_eq_approx_remove_nan # Copy over stacktrace for switch op from both previous # elementwise multiplication op and previous switch op, # because an error in this part can be caused by either # of the two previous ops. copy_stack_trace(node.outputs + switch.outputs, fct) return fct except NotScalarConstantError: pass return False @register_canonicalize @gof.local_optimizer([T.true_div, T.int_div]) def local_div_switch_sink(node): """ This optimization makes the folowing changes in the graph: T.div(T.switch(cond,0,iff),A) --> T.switch(cond,0,T.div(iff,A)) T.div(T.switch(cond,ift,0),A) --> T.switch(cond,T.div(ift,A),0) A being a symbolic variable. This is useful because A may not be numerically stable and give NaN or inf values for cases where the switch returns 0. See local_mul_switch_sink for more details. """ if (node.op != T.true_div and node.op != T.int_div): return False op = node.op if node.inputs[0].owner and node.inputs[0].owner.op == T.switch: switch = node.inputs[0].owner try: if get_scalar_constant_value(switch.inputs[1], only_process_constants=True) == 0.: fdiv = op(switch.inputs[2], node.inputs[1]) # Copy over stacktrace for elementwise division op # from previous elementwise multiplication op. # An error in the division (e.g. errors due to # inconsistent shapes or division by zero), # will point to the new division op. copy_stack_trace(node.outputs, fdiv) fct = [T.switch(switch.inputs[0], 0, fdiv)] fct[0].tag.values_eq_approx = values_eq_approx_remove_nan # Copy over stacktrace for switch op from both previous # elementwise division op and previous switch op, # because an error in this part can be caused by either # of the two previous ops. copy_stack_trace(node.outputs + switch.outputs, fct) return fct except NotScalarConstantError: pass try: if get_scalar_constant_value(switch.inputs[2], only_process_constants=True) == 0.: fdiv = op(switch.inputs[1], node.inputs[1]) # Copy over stacktrace for elementwise division op # from previous elementwise multiplication op. # An error in the division (e.g. errors due to # inconsistent shapes or division by zero), # will point to the new division op. copy_stack_trace(node.outputs, fdiv) fct = [T.switch(switch.inputs[0], fdiv, 0)] fct[0].tag.values_eq_approx = values_eq_approx_remove_nan # Copy over stacktrace for switch op from both previous # elementwise division op and previous switch op, # because an error in this part can be caused by either # of the two previous ops. copy_stack_trace(node.outputs + switch.outputs, fct) return fct except NotScalarConstantError: pass return False # Merge add/sub/mul/div/minimum/maximum/... of switches sharing the same # condition, to enable further simplification of their branches # Example: switch(c, a, b) + switch(c, x, y) -> switch(c, a+x, b+y) @register_canonicalize @gof.local_optimizer([T.Elemwise]) def local_merge_switch_same_cond(node): scal = theano.scalar # node must be binary elemwise or add or mul if not isinstance(node.op, T.Elemwise) or not isinstance( node.op.scalar_op, (scal.BinaryScalarOp, scal.Add, scal.Mul)): return # all inputs must be switch if not all(s.owner and isinstance(s.owner.op, T.Elemwise) and isinstance(s.owner.op.scalar_op, scal.Switch) for s in node.inputs): return # all switch conditions must be the same cond = node.inputs[0].owner.inputs[0] if not all(s.owner.inputs[0] is cond for s in node.inputs[1:]): return # pull out switch return [T.switch(cond, node.op(*[s.owner.inputs[1] for s in node.inputs]), node.op(*[s.owner.inputs[2] for s in node.inputs]))] ############# # Tile Opts # ############# @register_useless @register_canonicalize @register_stabilize @gof.local_optimizer([T.Tile]) def local_useless_tile(node): """Tile(x, (1,)*N) -> x This is useless tile. (1,)*N, just mean a vector with all element being 1. """ if isinstance(node.op, T.Tile): try: a = T.get_scalar_constant_value(node.inputs[1], only_process_constants=True) if a == 1: try: l = T.get_vector_length(node.inputs[1]) if l == node.inputs[0].ndim: # No need to copy over any stacktrace as previous # input variable already has a stacktrace return [node.inputs[0]] elif l < node.inputs[0].ndim: # The Op don't support that case, so we can't # implement the opt and test it. return return [node.inputs[0]] else: # The Op don't support that case, so we can't # implement the opt and test it. return x_nd = node.inputs[0].ndim broad = ['x'] * (l - x_nd) + xrange(x_nd) ret = node.inputs[0].dimshuffle(broad) # Copy over stacktrace from previous output node, # and from node before tiling operation. copy_stack_trace(node.outputs + node.inputs[0], ret) return [ret] except ValueError: return except NotScalarConstantError: return ############## # Split Opts # ############## @register_useless @register_canonicalize @register_specialize @gof.local_optimizer([T.Split]) def local_useless_split(node): """ Split{n_splits=1}(x, y) -> x Remove Split with only 1 split. """ if isinstance(node.op, T.Split): if node.op.len_splits == 1: x, axis, splits = node.inputs out = assert_op(x, T.eq(splits.shape[0], 1)) # Copy over stacktrace from previous output node. copy_stack_trace(node.outputs, out) out2 = assert_op(out, T.eq(x.shape[axis], splits[0])) # Copy over stacktrace from previous output node. copy_stack_trace(out, out2) return [out2] ################ # Flatten Opts # ################ @register_canonicalize @register_stabilize @gof.local_optimizer([T.Flatten]) def local_flatten_lift(node): """ Flatten(UnaryElemwise(x)) -> UnaryElemwise(Flatten(x)) This optimization is needed by optimization nnet/sigm.py:log1msigm_to_softplus to get applied when there is a flatten. """ if (isinstance(node.op, T.Flatten) and node.inputs[0].owner and isinstance(node.inputs[0].owner.op, T.Elemwise) and len(node.inputs[0].owner.inputs) == 1): f = node.op(node.inputs[0].owner.inputs[0]) # Copy over stacktrace from previous output node (flatten op), # since this is the op which may cause an error for f. copy_stack_trace(node.outputs, f) e = node.inputs[0].owner.op(f) # Copy over stacktrace from previous output node and from unary # elementwise output node since if there was an error, it would # probably have come from that operation. copy_stack_trace(node.outputs + [node.inputs[0]], e) return [e] ################## # Reshape opts # ################## def local_reshape_chain(op): @gof.local_optimizer([op]) def f(node): """ Reshape(Reshape(shape1),shape2) -> Reshape(shape2) """ if not opt.check_chain(node, op, op): return False # TODO: this can permit a failing program to run by eliminating # the lower reshape rval = node.op(node.inputs[0].owner.inputs[0], node.inputs[1]) # Copy over stacktrace from previous output node, as any error # in new computational graph would have been caused by last op # in the old computational graph. copy_stack_trace(node.outputs, rval) # It might happen that the desired output of this node has a # broadcastable pattern that does not match that of 'rval'. This is # when originally, we were able to figure out that one of the # dimensions of the reshape is one, but some other transformation # replaced the shape by one for which this cannot be guessed. # We should try to figure out why we lost the information about this # constant value... but in the meantime, better not apply this # optimization. if rval.broadcastable == node.outputs[0].broadcastable: return [rval] else: return False return f register_canonicalize(local_reshape_chain(T.Reshape), name='local_reshape_chain') @register_useless @register_canonicalize @register_stabilize @gof.local_optimizer([T.Reshape]) def local_useless_reshape(node): """ Remove two kinds of useless reshape. Remove Reshape when both the input and output have a single dimension. Remove Reshape when reshaping to the shape of the input. """ op = node.op if not isinstance(op, Reshape): return False input = node.inputs[0] output = node.outputs[0] output_shape = node.inputs[1] if input.ndim != output.ndim: return False # Simple case: both input and output have a single dimension. # This could hide errors if the user provides inconsistent shapes. if (input.ndim == 1 and output.ndim == 1 and input.broadcastable == output.broadcastable): return [input] # Second case: all the shapes match the input shape # Match Reshape(x, x.shape) if output_shape.owner and isinstance(output_shape.owner.op, Shape): shape_input = output_shape.owner.inputs[0] if shape_input == input: return [input] # Match Reshape(x, [x.shape[0], ..., x.shape[-1]]), accounting for # broadcastable and constant dimensions if output_shape.owner and isinstance(output_shape.owner.op, MakeVector): output_shape_is = output_shape.owner.inputs if not hasattr(node, 'fgraph'): shape_feature = None else: shape_feature = getattr(node.fgraph, 'shape_feature', None) nb_m1 = 0 shape_match = [False] * input.ndim for dim in xrange(input.ndim): outshp_i = output_shape_is[dim] # Match Shape_i{dim}(input) if (outshp_i.owner and isinstance(outshp_i.owner.op, Shape_i) and outshp_i.owner.op.i == dim and outshp_i.owner.inputs[0] == input): shape_match[dim] = True continue # Match Shape(input)[dim] if (outshp_i.owner and isinstance(outshp_i.owner.op, Subtensor) and len(outshp_i.owner.inputs) == 2 and extract_constant(outshp_i.owner.inputs[1]) == dim): subtensor_inp = outshp_i.owner.inputs[0] if (subtensor_inp.owner and isinstance(subtensor_inp.owner.op, Shape)): shape_input_i = subtensor_inp.owner.inputs[0] if shape_input_i == input: shape_match[dim] = True continue # Match 1 if input.broadcastable[dim] is True cst_outshp_i = extract_constant(outshp_i, only_process_constants=1) if input.broadcastable[dim] and cst_outshp_i == 1: shape_match[dim] = True continue # Match -1 if cst_outshp_i == -1: shape_match[dim] = True nb_m1 += 1 continue # Match shape_of[input][dim] or its constant equivalent if shape_feature: inpshp_i = shape_feature.get_shape(input, dim) if (inpshp_i == outshp_i or (extract_constant(inpshp_i, only_process_constants=1) == extract_constant(outshp_i, only_process_constants=1))): shape_match[dim] = True continue if all(shape_match) and nb_m1 <= 1: return [input] # TODO later: if all the shapes except one match, we may want to # consider it useless as well, like we do in the 1-dim case. @register_canonicalize @gof.local_optimizer([T.Reshape]) def local_reshape_to_dimshuffle(node): """ Broadcastable dimensions in Reshape are replaced with dimshuffle. The goal is to avoid using reshape to add or remove broadcastable dimensions, but use dimshuffle instead, so dimshuffles can cancel out or be removed later on. For example: - reshape(x, (1, n)) --> dimshuffle{x,0}(reshape(x, (n,)) - reshape(x, (1, m, 1, n, 1, 1)) --> dimshuffle{x,0,x,1,x,x}(reshape(x, (m, n))) """ op = node.op if not isinstance(op, Reshape): return False input = node.inputs[0] output = node.outputs[0] output_shape = node.inputs[1] dimshuffle_new_order = [] new_output_shape = [] index = 0 # index over the output of the new reshape for i in xrange(output.ndim): # Since output_shape is a symbolic vector, we trust extract_constant # to go through however it is formed to see if its i-th element is 1. # We need only_process_constants=False for that. dim = extract_constant(output_shape[i], only_process_constants=False, elemwise=False) if dim == 1: dimshuffle_new_order.append('x') else: dimshuffle_new_order.append(index) new_output_shape.append(dim) index = index + 1 if index != output.ndim: inner = op.__class__(len(new_output_shape))(input, new_output_shape) copy_stack_trace(output, inner) new_node = [DimShuffle(inner.type.broadcastable, dimshuffle_new_order)(inner)] copy_stack_trace(output, new_node) return new_node @register_canonicalize @register_stabilize @gof.local_optimizer([T.Reshape]) def local_reshape_lift(node): """ Reshape(UnaryElemwise(x)) -> UnaryElemwise(Reshape(x)) This optimization is needed by optimization nnet/sigm.py:log1msigm_to_softplus to get applied when there is a reshape. """ if (isinstance(node.op, T.Reshape) and node.inputs[0].owner and isinstance(node.inputs[0].owner.op, T.Elemwise) and len(node.inputs[0].owner.inputs) == 1): r = node.op(node.inputs[0].owner.inputs[0], node.inputs[1]) # Copy stacktrace from previous Reshape op, as an error in new # Reshape op could only have been caused by old one. copy_stack_trace(node.outputs, r) e = node.inputs[0].owner.op(r) # Copy stacktrace from both previous Reshape and UnaryElemwise op # because an error in new cg could have been caused by either ops. copy_stack_trace(node.outputs + node.inputs, e) # In rare case the original broadcast was (False, True), but # the new one is (False, False). So don't crash in that case. if e.type != node.outputs[0].type: re = T.patternbroadcast(e, node.outputs[0].broadcastable) # Copy over stack trace. # If the graph fails it is usually due to the fact that a dimension # that should be broadcastable does not actually have length 1, copy_stack_trace(e, re) else: re = e return [re] ################## # Middleman cuts # ################## register_canonicalize(gof.OpRemove(T.tensor_copy), name='remove_tensor_copy') ################ # Canonization # ################ class Canonizer(gof.LocalOptimizer): """ Simplification tool. The variable is a local_optimizer. It is best used with a TopoOptimizer in in_to_out order. Usage: Canonizer(main, inverse, reciprocal, calculate) Parameters ---------- main A suitable Op class that is commutative, associative and takes one to an arbitrary number of inputs, e.g. add or mul inverse An Op class such that inverse(main(x, y), y) == x e.g. sub or true_div reciprocal A function such that main(x, reciprocal(y)) == inverse(x, y) e.g. neg or inv calculate Function that takes a list of numpy.ndarray instances for the numerator, another list for the denumerator, and calculates inverse(main(\*num), main(\*denum)). It takes a keyword argument, aslist. If True, the value should be returned as a list of one element, unless the value is such that value = main(). In that case, the return value should be an empty list. Examples -------- >>> import theano.tensor as T >>> from theano.tensor.opt import Canonizer >>> add_canonizer = Canonizer(T.add, T.sub, T.neg, \\ ... lambda n, d: sum(n) - sum(d)) >>> mul_canonizer = Canonizer(T.mul, T.true_div, T.inv, \\ ... lambda n, d: prod(n) / prod(d)) Examples of optimizations mul_canonizer can perform: | x / x -> 1 | (x * y) / x -> y | x / y / x -> 1 / y | x / y / z -> x / (y * z) | x / (y / z) -> (x * z) / y | (a / b) * (b / c) * (c / d) -> a / d | (2.0 * x) / (4.0 * y) -> (0.5 * x) / y | 2 * x / 2 -> x | x * y * z -> Elemwise(T.mul){x,y,z} #only one pass over the memory. | !-> Elemwise(T.mul){x,Elemwise(T.mul){y,z}} """ def __init__(self, main, inverse, reciprocal, calculate, use_reciprocal=True): self.main = main self.inverse = inverse self.reciprocal = reciprocal self.calculate = calculate self.use_reciprocal = use_reciprocal self.external_simplifiers = [] def add_simplifier(self, simplifier, reason): self.external_simplifiers.append((reason, simplifier)) def tracks(self): return [self.main, self.inverse, self.reciprocal] def get_num_denum(self, input): """ This extract two lists, num and denum, such that the input is: self.inverse(self.main(\*num), self.main(\*denum)). It returns the two lists in a (num, denum) pair. For example, for main, inverse and reciprocal = \*, / and inv(), | input -> returned value (num, denum) | x*y -> ([x, y], []) | inv(x) -> ([], [x]) | inv(x) * inv(y) -> ([], [x, y]) | x*y/z -> ([x, y], [z]) | log(x) / y * (z + x) / y -> ([log(x), z + x], [y, y]) | (((a / b) * c) / d) -> ([a, c], [b, d]) | a / (b / c) -> ([a, c], [b]) | log(x) -> ([log(x)], []) | x**y -> ([x**y], []) | x * y * z -> ([x, y, z], []) """ # This function is recursive. The idea is that there is a # get_num_denum recursion in which the internal ops are all # one of (main, inverse, reciprocal, DimShuffle) and the # internal data nodes all have the dtype of the 'input' # argument. The leaf-Variables of the graph covered by the # recursion may be of any Variable type. if input.owner is None or input.owner.op not in [ self.main, self.inverse, self.reciprocal]: if input.owner and isinstance(input.owner.op, T.DimShuffle): # If input is a DimShuffle of some input which does # something like this: # * change a vector of length N into a 1xN row matrix # * change a scalar into a 1x1x1 tensor # * in general, complete the shape of a tensor # with broadcastable 1s to the *left* # Then we will simply discard the DimShuffle and return # the num/denum of its input dsn = input.owner # dimshuffle node dsop = dsn.op # dimshuffle op # the first input of the dimshuffle i.e. the ndarray to redim dsi0 = dsn.inputs[0] # The compatible order is a DimShuffle "new_order" of the form: # ('x', ..., 'x', 0, 1, 2, ..., dimshuffle_input.type.ndim) # That kind of DimShuffle only adds broadcastable # dimensions on the left, without discarding any # existing broadcastable dimension and is inserted # automatically by Elemwise when the inputs have # different numbers of dimensions (hence why we can # discard its information - we know we can retrieve it # later on). compatible_order = (('x',) * (input.type.ndim - dsi0.type.ndim) + tuple(range(dsi0.type.ndim))) if dsop.new_order == compatible_order: # If the "new_order" is the one we recognize, # we return the num_denum of the dimshuffled input. return self.get_num_denum(input.owner.inputs[0]) else: # This is when the input isn't produced by main, # inverse or reciprocal. return [input], [] else: return [input], [] num = [] denum = [] parent = input.owner # We get the (num, denum) pairs for each input # pairs = [self.get_num_denum(input2) if input2.type.dtype == # input.type.dtype else ([input2], []) for input2 in # parent.inputs] pairs = [self.get_num_denum(input2) for input2 in parent.inputs] if parent.op == self.main: # If we have main(x, y, ...), numx, denumx, numy, denumy, ... # then num is concat(numx, numy, num...) and denum is # concat(denumx, denumy, denum...) note that main() can have any # number of arguments >= 0 concat is list concatenation num = reduce(list.__iadd__, map(operator.itemgetter(0), pairs)) denum = reduce(list.__iadd__, map(operator.itemgetter(1), pairs)) elif parent.op == self.inverse: # If we have inverse(x, y), numx, denumx, numy and denumy # then num is concat(numx, denumy) and denum is # concat(denumx, numy) note that inverse() is binary num = pairs[0][0] + pairs[1][1] denum = pairs[0][1] + pairs[1][0] elif parent.op == self.reciprocal: # If we have reciprocal(x), numx, denumx # then num is denumx and denum is numx # note that reciprocal() is unary num = pairs[0][1] denum = pairs[0][0] return num, denum def merge_num_denum(self, num, denum): """ Utility function which takes two lists, num and denum, and returns something which is equivalent to inverse(main(\*num), main(\*denum)), but depends on the length of num and the length of denum (in order to minimize the number of operations). Let n = len(num) and d = len(denum): | n=0, d=0: neutral element (given by self.calculate([], [])) | (for example, this would be 0 if main is addition | and 1 if main is multiplication) | n=1, d=0: num[0] | n=0, d=1: reciprocal(denum[0]) | n=1, d=1: inverse(num[0], denum[0]) | n=0, d>1: reciprocal(main(\*denum)) | n>1, d=0: main(\*num) | n=1, d>1: inverse(num[0], main(\*denum)) | n>1, d=1: inverse(main(\*num), denum[0]) | n>1, d>1: inverse(main(\*num), main(\*denum)) Given the values of n and d to which they are associated, all of the above are equivalent to: inverse(main(\*num), main(\*denum)) """ ln, ld = len(num), len(denum) if not ln and not ld: return T.as_tensor_variable(self.calculate([], [])) if not ln: if self.use_reciprocal: return self.reciprocal(self.merge_num_denum(denum, [])) else: ln = [self.calculate([], [], aslist=False)] if not ld: if ln == 1: # num[0] should always be a variable assert isinstance(num[0], gof.Variable) return num[0] else: return self.main(*num) return self.inverse(self.merge_num_denum(num, []), self.merge_num_denum(denum, [])) @staticmethod def get_constant(v): """ Returns ------- object A numeric constant if v is a Constant or, well, a numeric constant. If v is a plain Variable, returns None. """ if isinstance(v, Constant): if getattr(v.tag, 'unique_value', None) is not None: data = v.tag.unique_value else: data = v.data if data.ndim == 0: return data else: return None elif isinstance(v, Variable): return None else: return v def simplify(self, num, denum, out_type): """ Shorthand for: .. code-block:: python self.simplify_constants(*self.simplify_factors(num, denum)) """ rval = self.simplify_constants(*self.simplify_factors(num, denum), out_type=out_type) for reason, simplifier in self.external_simplifiers: # TODO: document that 'reason' is associated with this # simplification to help auditing when things go # wrong rval = simplifier(*rval) return rval def simplify_factors(self, num, denum): """ For any Variable r which is both in num and denum, removes it from both lists. Modifies the lists inplace. Returns the modified lists. For example: | [x], [x] -> [], [] | [x, y], [x] -> [y], [] | [a, b], [c, d] -> [a, b], [c, d] """ ln = len(num) ld = len(denum) if (ld > 2 and ln > 2): # Faster version for "big" inputs. while True: s = set(num) # Inputs can appear multiple times redo = len(s) != len(num) inter = s.intersection(denum) for v in inter: num.remove(v) denum.remove(v) if not redo or not inter: break else: for v in list(num): if v in denum: num.remove(v) denum.remove(v) return num, denum def simplify_constants(self, orig_num, orig_denum, out_type=None): """ Find all constants and put them together into a single constant. Finds all constants in orig_num and orig_denum (using get_constant) and puts them together into a single constant. The constant is inserted as the first element of the numerator. If the constant is the neutral element, it is removed from the numerator. Examples -------- Let main be multiplication: | [2, 3, x], [] -> [6, x], [] | [x, y, 2], [4, z] -> [0.5, x, y], [z] | [x, 2, y], [z, 2] -> [x, y], [z] """ # Lists representing the numerator and denumerator num, denum = [], [] # Lists representing the *constant* elements of num and denum numct, denumct = [], [] for v in orig_num: ct = self.get_constant(v) if ct is not None: # We found a constant in the numerator! # We add it to numct numct.append(ct) else: num.append(v) for v in orig_denum: ct = self.get_constant(v) if ct is not None: denumct.append(ct) else: denum.append(v) if self.use_reciprocal or num: # This will calculate either: # [inverse(main(*numct), main(*denumct))] # [] - if inverse(main(*numct), main(*denumct)) is the # neutral element ct = self.calculate(numct, denumct, aslist=True, out_type=out_type) else: # This happens if we don't allow the reciprocal and the # numerator is empty. That means we will need to represent # reciprocal(x) like inverse(neutral_element, x) so # we can't allow ct == [] # TODO: why is this branch needed when merge_num_denum # does it for us? ct = [self.calculate(numct, denumct, aslist=False, out_type=out_type)] # Wrapping ct in a Constant with the right dtype ct = [T.constant(c, dtype=out_type.dtype) for c in ct] if orig_num and len(numct) == 1 and len(denumct) == 0 and ct: # In that case we should only have one constant in `ct`. assert len(ct) == 1 first_num_ct = self.get_constant(orig_num[0]) if first_num_ct is not None and ct[0].type.values_eq(ct[0].data, first_num_ct): # This is an important trick :( if it so happens that: # * there's exactly one constant on the numerator and none on # the denominator # * it's not the neutral element (ct is an empty list in that # case) # * the constant is the same as the first argument in the # numerator (we only check the first argument because the # canonizer puts the computed constants first) # -> then we return very exactly the original num/denum. # If we don't do that the optimizer will just loop # infinitely because it will not catch on that there are # no changes to be made and everytime it will want to # replace something by the same thing... # Note that it is important to use `values_eq` instead of # the == operator, to handle NaN values correctly. return orig_num, orig_denum return ct + num, denum def transform(self, node): op = node.op if op not in [self.main, self.inverse, self.reciprocal]: return False assert len(node.outputs) == 1 out = node.outputs[0] # out won't have a clients field when we didn't commit a # started change in the graph. We can't do the check if we # want to skip it, so we force the skip it. It should be # reapplied later. if not hasattr(out, 'clients'): return # check if any of the clients of this node would be part of # this canonized graph... if so, we do nothing and wait for # them to be transformed. for c, c_idx in out.clients: if c == 'output': continue while (isinstance(getattr(c, 'op', None), DimShuffle) and len(c.outputs[0].clients) <= 1): c = c.outputs[0].clients[0][0] if getattr(c, 'op', '') in [self.main, self.inverse, self.reciprocal]: return False # Here we make the canonical version of the graph around this node # See the documentation of get_num_denum and simplify orig_num, orig_denum = self.get_num_denum(node.outputs[0]) num, denum = self.simplify(list(orig_num), list(orig_denum), out.type) def same(x, y): return len(x) == len(y) and all(np.all(xe == ye) for xe, ye in zip(x, y)) if same(orig_num, num) and same(orig_denum, denum): # We return False if there are no changes return False new = self.merge_num_denum(num, denum) if new.type.dtype != out.type.dtype: new = T.cast(new, out.type.dtype) assert (new.type == out.type) == (not (new.type != out.type)) if not (new.type == out.type): new = _fill_chain(new, node.inputs)[0] if new.type == out.type: # This happen with test # theano/tensor/tests/test_opt.py:T_local_switch_sink new.tag.values_eq_approx = values_eq_approx_remove_inf_nan # We need to implement the copy over of the stacktrace. # See issue #5104. return [new] else: _logger.warning(' '.join(('CANONIZE FAILED: new, out = ', new, ',', out, 'types', new.type, ',', out.type))) return False def __str__(self): return getattr(self, 'name', 'Canonizer(%s, %s, %s)' % ( self.main, self.inverse, self.reciprocal)) def mul_calculate(num, denum, aslist=False, out_type=None): if not num and not denum: # Smallest 1 possible. if aslist: return [] else: return np.int8(1) # Make sure we do not accidently upcast data types. if out_type is None: out_dtype = scalar.upcast(*[v.dtype for v in (num + denum)]) else: out_dtype = out_type.dtype one = theano._asarray(1, dtype=out_dtype) v = reduce(np.multiply, num, one) / reduce(np.multiply, denum, one) if aslist: if np.all(v == 1): return [] else: return [v] return v local_mul_canonizer = Canonizer(T.mul, T.true_div, T.inv, mul_calculate, False) register_canonicalize(local_mul_canonizer, name='local_mul_canonizer') @gof.local_optimizer([T.neg]) def local_neg_to_mul(node): if node.op == T.neg: return [T.mul(np.array(-1, dtype=node.inputs[0].dtype), node.inputs[0])] register_canonicalize(local_neg_to_mul) @register_specialize @gof.local_optimizer([T.Sum, T.elemwise.Prod]) def local_sum_prod_mul_by_scalar(node): """ sum(scalar * smth) -> scalar * sum(smth) sum(-smth) -> -sum(smth) or prod(scalar * smth) -> scalar ** size(smth) * prod(smth) prod(-smth) -> -1 ** size(smth) * prod(smth) """ # TODO: if the the thing inside the Sum is a division, # we should get at the numerator.... if isinstance(node.op, (T.Sum, T.elemwise.Prod)): node_inps, = node.inputs if node_inps.owner and node_inps.owner.op == T.mul: terms = node_inps.owner.inputs scalars = [t.dimshuffle() for t in terms if np.all(t.type.broadcastable)] if len(scalars) == 0: # Nothing to optimize here return non_scalars = [t for t in terms if not np.all(t.broadcastable)] # Perform the op only on the non-scalar inputs, if applicable if len(non_scalars) == 0: new_op_input_nb_elements = 1 new_op_output = 1 elif len(non_scalars) == 1: new_op_input_nb_elements = non_scalars[0].size new_op_output = node.op(non_scalars[0]) else: new_op_input = T.mul(*non_scalars) # We assume that errors always come from the prod/mul op in the # original computational graph, and therefore need to only # copy over its output stacktrace. copy_stack_trace(node.outputs, new_op_input) new_op_input_nb_elements = new_op_input.size new_op_output = node.op(new_op_input) if not len(non_scalars) == 0: # Copy over stacktrace from previous output to new mul op, # for same reason as above. copy_stack_trace(node.outputs, new_op_output) # If node.op is a T.elemwise.Prod, then the scalars need to be # raised to the power of the number of elements in the input # to the Prod if (isinstance(node.op, T.elemwise.Prod) and new_op_input_nb_elements != 1): scalars = [s ** new_op_input_nb_elements for s in scalars] # Scale the output of the op by the scalars and return as # replacement for the original output mul_inputs = scalars if new_op_input_nb_elements != 1: mul_inputs.append(new_op_output) if len(mul_inputs) == 1: # Copy over stacktrace from previous output to new mul op, # for same reason as above. copy_stack_trace(node.outputs, mul_inputs) return mul_inputs else: ret = T.mul(*mul_inputs) # Copy over stacktrace from previous output to new mul op, # for same reason as above. copy_stack_trace(node.outputs, [ret] + mul_inputs) return [ret] if isinstance(node.op, T.Sum) and node_inps.owner and node_inps.owner.op == T.neg: s = node.op(node_inps.owner.inputs[0]) ret = T.neg(s) # There are never errors in the negative op, thus # we need only to copy over stacktrace from previous output node to # the two new ops. copy_stack_trace(node.outputs, [s, ret]) return [ret] @register_specialize @gof.local_optimizer([T.Elemwise]) def local_elemwise_sub_zeros(node): """ Elemwise{sub}(X,X) -> zeros_like(X) """ if (isinstance(node.op, T.Elemwise) and node.op.scalar_op.nin == 2 and node.op.scalar_op == scalar.sub and node.inputs[0] == node.inputs[1]): res = T.zeros_like(node.inputs[0]) # Copy over stacktrace from previous output. # This could help for failures due to out-of-memory. copy_stack_trace(node.outputs, res) return [res] @register_useless @register_specialize @register_stabilize @register_canonicalize @gof.local_optimizer([T.Elemwise]) def local_useless_elemwise_comparison(node): """... :note: These cases appear in the graph generated by scan. These optimizations will make the graph easier to read. # Comparing to itself is constant Elemwise[{LT,GT}](X, X) -> Elemwise[zeros](X) Elemwise[{LE,GE}](X, X) -> Elemwise[ones](X) Elemwise[{minimum,maximum}](X, X) -> X # Comparing shape to 0 can be constant Elemwise[LT](X.shape[i], 0) -> Elemwise[zeros](X) Elemwise[GE](X.shape[i], 0) -> Elemwise[ones](X) Elemwise[maximum](X.shape[i], 0) -> X.shape[i] Elemwise[maximum](0, X.shape[i]) -> X.shape[i] Elemwise[minimum](X.shape[i], 0) -> 0 Elemwise[minimum](0, X.shape[i]) -> 0 # The shape can be replaced with sum of shapes Elemwise[LT](add([anything that is shapes]), 0) -> Elemwise[zeros](X) Elemwise[GE](add([anything that is shapes]), 0) -> Elemwise[ones](X) # Shapes are never negative # Needed by Reshape.infer_shape Elemwise[EQ](Subtensor(Shape(x)), -N) -> Elemwise[zeros](X) """ if not isinstance(node.op, T.Elemwise): return if node.op.scalar_op.nin != 2: return # We call zeros_like and one_like with opt=True to generate a # cleaner graph. dtype = node.outputs[0].dtype # Elemwise[{LT,GT}](X, X) -> Elemwise[zeros](X) if isinstance(node.op.scalar_op, (scalar.LT, scalar.GT)) and \ node.inputs[0] is node.inputs[1]: res = T.zeros_like(node.inputs[0], dtype=dtype, opt=True) # Copy over stacktrace from previous output. copy_stack_trace(node.outputs, res) return [res] # Elemwise[{LE,GE}](X, X) -> Elemwise[ones](X) if isinstance(node.op.scalar_op, (scalar.LE, scalar.GE)) and \ node.inputs[0] is node.inputs[1]: res = T.ones_like(node.inputs[0], dtype=dtype, opt=True) # Copy over stacktrace from previous output. copy_stack_trace(node.outputs, res) return [res] # Elemwise[{minimum,maximum}](X, X) -> X if isinstance(node.op.scalar_op, (scalar.Minimum, scalar.Maximum)) and \ node.inputs[0] is node.inputs[1]: res = node.inputs[0] # Copy over stacktrace from previous output. copy_stack_trace(node.outputs, res) return [res] # Elemwise[LT](X.shape[i], 0) -> Elemwise[zeros](X) if isinstance(node.op.scalar_op, scalar.LT) and \ node.inputs[0].owner and \ isinstance(node.inputs[0].owner.op, Shape_i) and \ T.extract_constant(node.inputs[1], only_process_constants=True) == 0: res = T.zeros_like(node.inputs[0], dtype=dtype, opt=True) # Copy over stacktrace from previous output. copy_stack_trace(node.outputs, res) return [res] # Elemwise[GE](X.shape[i], 0) -> Elemwise[ones](X) if isinstance(node.op.scalar_op, scalar.GE) and \ node.inputs[0].owner and \ isinstance(node.inputs[0].owner.op, Shape_i) and \ T.extract_constant(node.inputs[1], only_process_constants=True) == 0: res = T.ones_like(node.inputs[0], dtype=dtype, opt=True) # Copy over stacktrace from previous output. copy_stack_trace(node.outputs, res) return [res] # Elemwise[maximum](X.shape[i], 0) -> X.shape[i] if isinstance(node.op.scalar_op, scalar.Maximum) and \ node.inputs[0].owner and \ isinstance(node.inputs[0].owner.op, Shape_i) and \ T.extract_constant(node.inputs[1], only_process_constants=True) == 0: # No need to copy over stacktrace. return [node.inputs[0]] # Elemwise[maximum](0, X.shape[i]) -> X.shape[i] if isinstance(node.op.scalar_op, scalar.Maximum) and \ T.extract_constant(node.inputs[0], only_process_constants=True) == 0 and \ node.inputs[1].owner and \ isinstance(node.inputs[1].owner.op, Shape_i): # No need to copy over stacktrace. return [node.inputs[1]] # Elemwise[minimum](X.shape[i], 0) -> 0 if isinstance(node.op.scalar_op, scalar.Minimum) and \ node.inputs[0].owner and \ isinstance(node.inputs[0].owner.op, Shape_i) and \ T.extract_constant(node.inputs[1], only_process_constants=True) == 0: res = T.zeros_like(node.inputs[0], dtype=dtype, opt=True) # Copy over stacktrace from previous output. copy_stack_trace(node.outputs, res) return [res] # Elemwise[minimum](0, X.shape[i]) -> 0 if isinstance(node.op.scalar_op, scalar.Minimum) and \ T.extract_constant(node.inputs[0], only_process_constants=True) == 0 and \ node.inputs[1].owner and \ isinstance(node.inputs[1].owner.op, Shape_i): res = T.zeros_like(node.inputs[1], dtype=dtype, opt=True) # Copy over stacktrace from previous output. copy_stack_trace(node.outputs, res) return [res] # Elemwise[LT](add([anything that is shapes]), 0) -> Elemwise[zeros](X) if isinstance(node.op.scalar_op, scalar.LT) and \ node.inputs[0].owner and \ isinstance(node.inputs[0].owner.op, Elemwise) and \ isinstance(node.inputs[0].owner.op.scalar_op, scalar.Add) and \ all([isinstance(var.owner and var.owner.op, Shape_i) for var in node.inputs[0].owner.inputs]) and \ T.extract_constant(node.inputs[1], only_process_constants=True) == 0: res = T.zeros_like(node.inputs[0], dtype=dtype, opt=True) # Copy over stacktrace from previous output. copy_stack_trace(node.outputs, res) return [res] # Elemwise[GE](add([anything that is shapes]), 0) -> Elemwise[ones](X) if isinstance(node.op.scalar_op, scalar.GE) and \ node.inputs[0].owner and \ isinstance(node.inputs[0].owner.op, Elemwise) and \ isinstance(node.inputs[0].owner.op.scalar_op, scalar.Add) and \ all([isinstance(var.owner and var.owner.op, Shape_i) for var in node.inputs[0].owner.inputs]) and \ T.extract_constant(node.inputs[1], only_process_constants=True) == 0: res = T.ones_like(node.inputs[0], dtype=dtype, opt=True) # Copy over stacktrace from previous output. copy_stack_trace(node.outputs, res) return [res] # Elemwise[EQ](Subtensor(Shape(x)), -N) # Elemwise[EQ](somegraph that only depend of shape, -N) # TODO: handle the case where the -N is on either side """ |Elemwise{eq,no_inplace} [id B] '' | |Subtensor{int64} [id C] '' | | |Join [id D] '' | | | |TensorConstant{0} [id E] | | | |Subtensor{int64:int64:} [id F] '' | | | | |Shape [id G] '' """ def investigate(node): " Return True if values will be shapes, so >= 0" if isinstance(node.op, (T.Shape, Shape_i)): return True elif isinstance(node.op, Subtensor) and node.inputs[0].owner: return investigate(node.inputs[0].owner) elif isinstance(node.op, T.Join): return all(v.owner and investigate(v.owner) for v in node.inputs[1:]) elif isinstance(node.op, MakeVector): return all(v.owner and investigate(v.owner) for v in node.inputs) if (isinstance(node.op.scalar_op, scalar.EQ) and node.inputs[0].owner and investigate(node.inputs[0].owner)): try: cst = get_scalar_constant_value(node.inputs[1], only_process_constants=True) res = T.zeros_like(node.inputs[0], dtype=dtype, opt=True) if cst < 0: # Copy over stacktrace from previous output. copy_stack_trace(node.outputs, res) return [res] except NotScalarConstantError: pass return @register_canonicalize @register_specialize @gof.local_optimizer([T.Sum, T.elemwise.Prod]) def local_sum_prod_div_dimshuffle(node): """ sum(a / dimshuffle{...}(b), axis=l) -> sum(a, axis={...}) / b, if dimension l of the DimShuffle is 'x' or prod(a / dimshuffle{...}(b), axis=l) -> prod(a, axis={...}) / b ** a.shape[l], if dimension l of the DimShuffle is 'x' """ # It does not make much sense now to extend it to the case where the # dimshuffle is in the numerator, since elemwise inversion of the # denominator would still be needed before the summation or production. if isinstance(node.op, (T.Sum, T.elemwise.Prod)): axis = node.op.axis if axis is None: axis = list(range(node.inputs[0].ndim)) node_input = node.inputs[0] if node_input.owner and node_input.owner.op == T.true_div: numerator, denominator = node_input.owner.inputs # Old, bugged logic, reproduced here only to warn users if (config.warn.sum_div_dimshuffle_bug and isinstance(node.op, T.Sum) and numerator.owner and isinstance(numerator.owner.op, T.DimShuffle)): # Check compatibility new_order = numerator.owner.op.new_order compatible_dims = True for ax in axis: if len(new_order) <= ax or new_order[ax] != 'x': compatible_dims = False break if compatible_dims: _logger.warn('WARNING: Your current code is fine, but' ' Theano versions between ' 'rev. 3bd9b789f5e8 (2010-06-16) and' ' cfc6322e5ad4 (2010-08-03) would ' 'have given an incorrect result. ' 'To disable this warning, set the Theano' ' flag warn.sum_div_dimshuffle_bug to' ' False.') if denominator.owner and isinstance(denominator.owner.op, T.DimShuffle): dimshuffle_input = denominator.owner.inputs[0] dimshuffle_order = denominator.owner.op.new_order compatible_dims = [] incompatible_dims = [] for ax in axis: if (ax < len(dimshuffle_order) and dimshuffle_order[ax] == 'x'): compatible_dims.append(ax) else: incompatible_dims.append(ax) reordered_incompatible_dims = [] for ic_ax in incompatible_dims: reordered_incompatible_dims.append( ic_ax - sum( [1 for c_ax in compatible_dims if c_ax < ic_ax])) if len(compatible_dims) > 0: optimized_dimshuffle_order = list( ax for i, ax in enumerate(dimshuffle_order) if (i not in axis) or (ax != 'x')) # Removing leading 'x' (since it will be done automatically) while (len(optimized_dimshuffle_order) > 0 and optimized_dimshuffle_order[0] == 'x'): del optimized_dimshuffle_order[0] # if optimized_dimshuffle_order is sorted with # not 'x', then dimshuffle is useless. if all(i == e for i, e in enumerate(optimized_dimshuffle_order)): optimized_dimshuffle = dimshuffle_input else: optimized_dimshuffle = T.DimShuffle( dimshuffle_input.type.broadcastable, optimized_dimshuffle_order)(dimshuffle_input) if (config.warn.sum_div_dimshuffle_bug and isinstance(node.op, T.Sum)): _logger.warn('WARNING: Your current code is fine,' ' but Theano versions between ' 'rev. 3bd9b789f5e8 (2010-06-16) and' ' cfc6322e5ad4 (2010-08-03) would ' 'have given an incorrect result. ' 'To disable this warning, set the' ' Theano flag ' 'warn.sum_div_dimshuffle_bug' ' to False.') if isinstance(node.op, T.Sum): op_on_compatible_dims = T.sum( numerator, axis=compatible_dims) rval = T.true_div( op_on_compatible_dims, optimized_dimshuffle) if len(reordered_incompatible_dims) > 0: rval = T.sum(rval, axis=reordered_incompatible_dims) elif isinstance(node.op, T.elemwise.Prod): op_on_compatible_dims = T.prod( numerator, axis=compatible_dims) dtype = numerator.dtype rval = T.true_div( op_on_compatible_dims, (optimized_dimshuffle ** T.prod([numerator.shape[ax].astype(dtype) for ax in compatible_dims]))) if len(reordered_incompatible_dims) > 0: rval = T.prod(rval, axis=reordered_incompatible_dims) return [rval] @register_canonicalize @gof.local_optimizer([T.Sum, T.elemwise.Prod]) def local_sum_prod_all_to_none(node): """ Sum{0,1,...N} -> Sum{} or Prod{0,1,...N} -> Prod{} """ if isinstance(node.op, T.Sum) or isinstance(node.op, T.elemwise.Prod): opt_type = T.Sum if isinstance(node.op, T.Sum) else T.elemwise.Prod # if all the axes are named, then use None as a shorthand # this permits more merging if node.op.axis is None: return if set(node.op.axis) == set(range(node.inputs[0].type.ndim)): return [opt_type(axis=None, dtype=node.op.dtype)(node.inputs[0])] @register_canonicalize @gof.local_optimizer([T.Sum, T.elemwise.Prod]) def local_op_of_op(node): """ Prod(Prod()) -> single Prod() or Sum(Sum()) -> single Sum() """ if isinstance(node.op, T.elemwise.Prod) or isinstance(node.op, T.Sum): opt_type = T.Sum if isinstance(node.op, T.Sum) else T.elemwise.Prod node_inps, = node.inputs out_dtype = node.op.dtype # We manipulate the graph so this is done to make sure the opt # doesn't affect other computations. if len(node_inps.clients) == 1: if (node_inps.owner and (isinstance(node_inps.owner.op, node.op.__class__))): # check to see either the inner or outer prod is doing a # product over all axis, in which case we can remove it if node_inps.owner.op.axis is None or node.op.axis is None: return [opt_type(None, dtype=out_dtype)( node_inps.owner.inputs[0])] # figure out which axes were in the original sum newaxis = list(tuple(node_inps.owner.op.axis)) for i in node.op.axis: new_i = i for ii in node_inps.owner.op.axis: if new_i >= ii: new_i += 1 assert new_i not in newaxis newaxis.append(new_i) assert len(newaxis) == len(list(node_inps.owner.op.axis) + list(node.op.axis)) # The old bugged logic. We keep it there to generate a warning # when we generated bad code. alldims = list(range(node_inps.owner.inputs[0].type.ndim)) alldims = [d for i, d in enumerate(alldims) if i in node_inps.owner.op.axis] alldims = [d for i, d in enumerate(alldims) if i in node.op.axis] newaxis_old = [i for i in xrange(node_inps.owner.inputs[0].type.ndim) if i not in alldims] if (theano.config.warn.sum_sum_bug and newaxis != newaxis_old and len(newaxis) == len(newaxis_old)): _logger.warn( "WARNING (YOUR CURRENT CODE IS FINE): Theano " "versions between version 9923a40c7b7a and August " "2nd, 2010 generated bugged code in this case. " "This happens when there are two consecutive sums " "in the graph and the intermediate sum is not " "used elsewhere in the code. Some safeguard " "removed some bad code, but not in all cases. You " "are in one such case. To disable this warning " "(that you can safely ignore since this bug has " "been fixed) set the theano flag " "`warn.sum_sum_bug` to False.") combined = opt_type(newaxis, dtype=out_dtype) return [combined(node_inps.owner.inputs[0])] ALL_REDUCE = [T.elemwise.CAReduce, T.elemwise.All, T.elemwise.Any, T.elemwise.Sum, T.elemwise.Prod, T.elemwise.ProdWithoutZeros] @register_canonicalize @register_uncanonicalize # Needed for MaxAndArgmax -> CAReduce @gof.local_optimizer(ALL_REDUCE) def local_reduce_join(node): """ Reduce{scalar.op}(Join(axis=0, a, b), axis=0) -> Elemwise{scalar.op}(a, b) Notes ----- Supported scalar.op are Maximum, Mimimum in some cases and Add and Mul in all cases. Currently we must reduce on axis 0. It is probably extensible to the case where we join and reduce on the same set of axis. """ if (isinstance(node.op, T.CAReduce) and node.inputs[0].owner and isinstance(node.inputs[0].owner.op, T.Join)): join = node.inputs[0].owner if T.extract_constant(join.inputs[0], only_process_constants=True) != 0: return if isinstance(node.op.scalar_op, (scalar.Maximum, scalar.Minimum)): # Support only 2 inputs for now if len(join.inputs) != 3: return elif not isinstance(node.op.scalar_op, (scalar.Add, scalar.Mul)): return elif len(join.inputs) <= 2: # This is a useless join, that will get removed by another opt. return new_inp = [] for inp in join.inputs[1:]: inp = inp.owner if not inp: return if (not isinstance(inp.op, DimShuffle) or inp.op.new_order != ('x',) + tuple(range(inp.inputs[0].ndim))): return new_inp.append(inp.inputs[0]) ret = Elemwise(node.op.scalar_op)(*new_inp) if ret.dtype != node.outputs[0].dtype: # The reduction do something about the dtype. return reduce_axis = node.op.axis if reduce_axis is None: reduce_axis = tuple(xrange(node.inputs[0].ndim)) # I put this warning late to don't add extra warning. if len(reduce_axis) != 1 or 0 not in reduce_axis: if theano.config.warn.reduce_join: warnings.warn(( 'Your current code is fine, but Theano versions ' 'prior to 0.7 (or this development version Sept 2014) ' 'might have given an incorrect result for this code. ' 'To disable this warning, set the Theano flag ' 'warn.reduce_join to False. The problem was an ' 'optimization, that modified the pattern ' '"Reduce{scalar.op}(Join(axis=0, a, b), axis=0)", ' 'did not check the reduction axis. So if the ' 'reduction axis was not 0, you got a wrong answer.')) return # We add the new check late to don't add extra warning. try: join_axis = get_scalar_constant_value(join.inputs[0], only_process_constants=True) if join_axis != reduce_axis[0]: return except NotScalarConstantError: return return [ret] @register_canonicalize('fast_compile', 'local_cut_useless_reduce') @register_useless('local_cut_useless_reduce') @gof.local_optimizer(ALL_REDUCE) def local_useless_reduce(node): """Sum(a, axis=[]) -> a """ if isinstance(node.op, T.CAReduce): summed, = node.inputs # if reduce were doing anything, the output ndim would be reduced if summed.type == node.outputs[0].type: return [summed] @register_canonicalize @register_uncanonicalize @register_specialize @gof.local_optimizer(ALL_REDUCE) def local_reduce_broadcastable(node): """Remove reduction over broadcastable dimensions.""" if isinstance(node.op, T.CAReduce): reduced, = node.inputs odtype = node.outputs[0].dtype if node.op.axis is None: if all(reduced.broadcastable): return [reduced.dimshuffle().astype(odtype)] else: axis = list(node.op.axis) cuttable = [a for a in axis if reduced.broadcastable[a]] if cuttable: # -- we can remove some axes of summation, # which simplifies the codegen for sum, especially on GPU new_axis = [] pattern = [] ii = 0 for p in xrange(reduced.ndim): if p not in cuttable: if p in axis: new_axis.append(ii) pattern.append(p) ii += 1 new_reduced = reduced.dimshuffle(*pattern) if new_axis: if type(node.op) == theano.tensor.elemwise.CAReduce: # This happen for tensor.max(), tensor.min() new_op = node.op.__class__(node.op.scalar_op, axis=new_axis) else: new_op = node.op.__class__(axis=new_axis) return [new_op(new_reduced)] else: # -- in this case we can remove the reduction completely return [new_reduced.astype(odtype)] @register_specialize @gof.local_optimizer([T.Sum, T.elemwise.Prod]) def local_opt_alloc(node): """ sum(alloc(constant,shapes...)) => constant*prod(shapes) or prod(alloc(constant,shapes...)) => constant**prod(shapes) """ if isinstance(node.op, T.Sum) or isinstance(node.op, T.elemwise.Prod): node_inps, = node.inputs if node_inps.owner and isinstance(node_inps.owner.op, T.Alloc): input = node_inps.owner.inputs[0] shapes = node_inps.owner.inputs[1:] try: val = get_scalar_constant_value(input, only_process_constants=True) assert val.size == 1 val = val.reshape(1)[0] # check which type of op size = T.mul(*shapes) if input.dtype in ["float16", "float32"]: # shapes are ints and normally int64. # We don't want to have a float64 upcast # We don't want to downcast to float16 # as we fear it could loose too much precision # that will be amplified by the mul/pow below. size = size.astype('float32') if (node.op.axis is None or node.op.axis == tuple(range(input.ndim))): if isinstance(node.op, T.Sum): val = val * size else: val = val ** size # Sum can change the input dtype (upcast or bool # -> float32) by default or by user request. # We can ignore the acc_dtype, as there is only 1 # elemwise we will do and not a sequence, so there is no # accumulation of errors. # So mostly, we just need to cast the output to the old # dtype. val = val.astype(node.outputs[0].dtype) return [val] to_prod = [shapes[i] for i in xrange(len(shapes)) if i in node.op.axis] if to_prod: size = T.mul(*to_prod) if isinstance(node.op, T.Sum): val *= size else: val = val ** size # See comments above. val = val.astype(node.outputs[0].dtype) return [T.alloc(val, *[shapes[i] for i in xrange(len(shapes)) if i not in node.op.axis])] except NotScalarConstantError: pass @register_specialize @gof.local_optimizer([T.neg]) def local_neg_neg(node): # other specializations shouldn't put this in, # but sometimes they do if node.op == T.neg: if node.inputs[0].owner and node.inputs[0].owner.op == T.neg: return [node.inputs[0].owner.inputs[0]] @register_specialize @gof.local_optimizer([T.neg]) def local_neg_div_neg(node): """ - (-a / b) -> a / b Also performs - (c / b) -> ((-c) / b) when c is a scalar constant. """ if node.op == T.neg: if node.inputs[0].owner and node.inputs[0].owner.op == T.true_div: frac = node.inputs[0] num, denom = frac.owner.inputs if num.owner and num.owner.op == T.neg: if len(frac.clients) == 1: # No other clients of the original division new_num = num.owner.inputs[0] return [T.true_div(new_num, denom)] elif np.all(num.broadcastable) and isinstance(num, Constant): if len(frac.clients) == 1: new_num = -num.data return [T.true_div(new_num, denom)] @gof.local_optimizer([T.mul]) def local_mul_zero(node): """ As part of canonicalization, we replace multiplication by zero with zero. """ if node.op == T.mul: otype = node.outputs[0].type for i in node.inputs: try: value = get_scalar_constant_value(i) except NotScalarConstantError: continue # print 'MUL by value', value, node.inputs if value == 0: # print '... returning zeros' return _fill_chain(theano._asarray(0, dtype=otype.dtype), node.inputs) register_canonicalize(local_mul_zero) @gof.local_optimizer([T.true_div]) def local_div_to_inv(node): if node.op == T.true_div and np.all( local_mul_canonizer.get_constant(node.inputs[0]) == 1.0): out = node.outputs[0] new_out = T.inv(local_mul_canonizer.merge_num_denum(node.inputs[1:], [])) # The ones could have forced upcasting if new_out.dtype != out.dtype: new_out = T.cast(new_out, dtype=out.dtype) # The ones could have forced a specific length if new_out.type != out.type: new_out = broadcast_like(new_out, out, node.fgraph) return [new_out] else: return False register_specialize(local_div_to_inv) @gof.local_optimizer([T.inv]) def local_inv_canon(node): if node.op == T.inv: return [T.pow(node.inputs[0], -1.0)] else: return False register_canonicalize(local_inv_canon) @gof.local_optimizer([T.pow]) def local_pow_canonicalize(node): if node.op == T.pow: cst = local_mul_canonizer.get_constant(node.inputs[1]) if cst == 0: return [broadcast_like(1, node.outputs[0], node.fgraph)] if cst == 1: return [broadcast_like(node.inputs[0], node.outputs[0], node.fgraph)] else: return False register_canonicalize(local_pow_canonicalize) @register_specialize @gof.local_optimizer([T.mul]) def local_mul_to_sqr(node): """ x*x -> sqr(x) This is faster on the GPU when memory fetching is a big part of the computation time. """ if node.op == T.mul: if len(node.inputs) == 2: if node.inputs[0] is node.inputs[1]: return [T.sqr(node.inputs[0])] @register_canonicalize @gof.local_optimizer([T.int_div]) def local_intdiv_by_one(node): """x // 1 -> x """ if node.op in [T.int_div]: if isinstance(node.inputs[1], T.TensorConstant) and \ np.all(node.inputs[1].value == 1): return [node.inputs[0].astype(node.outputs[0].dtype)] @register_canonicalize @register_specialize @gof.local_optimizer([T.int_div, T.true_div]) def local_zero_div(node): """0 / x -> 0 """ if isinstance(node.op, T.Elemwise) and isinstance( node.op.scalar_op, (theano.scalar.IntDiv, theano.scalar.TrueDiv)): if local_mul_canonizer.get_constant(node.inputs[0]) == 0: ret = broadcast_like(0, node.outputs[0], node.fgraph) ret.tag.values_eq_approx = values_eq_approx_remove_nan return [ret] @gof.local_optimizer([T.pow]) def local_pow_specialize(node): # here, we are past the point of canonicalization, so we don't want # to put in un-necessary fills. if node.op == T.pow: # the idea here is that we have pow(x, y) odtype = node.outputs[0].dtype xsym = node.inputs[0] ysym = node.inputs[1] y = local_mul_canonizer.get_constant(ysym) if (y is not None) \ and encompasses_broadcastable(xsym.type.broadcastable, ysym.type.broadcastable): rval = None if np.all(y == 2): rval = [T.sqr(xsym)] if np.all(y == 1): rval = [xsym] if np.all(y == 0): rval = [T.fill(xsym, np.asarray(1, dtype=odtype))] if np.all(y == 0.5): rval = [T.sqrt(xsym)] if np.all(y == -0.5): rval = [T.inv(T.sqrt(xsym))] if np.all(y == -1): rval = [T.inv(xsym)] if np.all(y == -2): rval = [T.inv(T.sqr(xsym))] if rval: rval[0] = T.cast(rval[0], odtype) assert rval[0].type == node.outputs[0].type, ( rval, node.outputs) return rval else: return False register_specialize(local_pow_specialize) @register_specialize_device @gof.local_optimizer([T.pow]) def local_pow_specialize_device(node): """ This optimization is not the same on all device. We do it only on cpu here. """ if node.op == T.pow: # the idea here is that we have pow(x, y) odtype = node.outputs[0].dtype xsym = node.inputs[0] ysym = node.inputs[1] y = local_mul_canonizer.get_constant(ysym) # the next line is needed to fix a strange case that I don't # know how to make a separate test. # That happen in the test_opt.py:test_log_erfc test. # y is a ndarray with dtype int8 and value 2,4 or 6. This make # the abs(y) <= 512 fail! # taking the value outside ndarray solve the problem. # it could be that in that case, numpy make the comparaison # into the wrong type(do in int8 that overflow.) if isinstance(y, np.ndarray): assert y.size == 1 try: y = y[0] except IndexError: pass if (y is not None) \ and encompasses_broadcastable(xsym.type.broadcastable, ysym.type.broadcastable): rval = None # 512 is too small for the cpu and too big for some gpu! if abs(y) == int(abs(y)) and abs(y) <= 512: pow2 = [xsym] pow2_scal = [theano.scalar.get_scalar_type(xsym.dtype)()] y_to_do = abs(y) for i in xrange(int(np.log2(y_to_do))): pow2.append(T.sqr(pow2[i])) pow2_scal.append(theano.scalar.sqr(pow2_scal[i])) rval1 = None rval1_scal = None while y_to_do > 0: log_to_do = int(np.log2(y_to_do)) if rval1: rval1 *= pow2[log_to_do] rval1_scal *= pow2_scal[log_to_do] else: rval1 = pow2[log_to_do] rval1_scal = pow2_scal[log_to_do] y_to_do -= 2 ** log_to_do if abs(y) > 2: # We fuse all the pow together here to make # compilation faster rval1 = Elemwise( theano.scalar.Composite( [pow2_scal[0]], [rval1_scal])).make_node(xsym) if y < 0: rval = [T.inv(rval1)] else: rval = [rval1] if rval: rval[0] = T.cast(rval[0], odtype) assert rval[0].type == node.outputs[0].type, ( rval, node.outputs) return rval @gof.local_optimizer([T.mul]) def local_mul_specialize(node): """ Remove special-case constants from mul arguments and useless neg in inputs. mul(-1, x) -> neg(x) mul(1, x, y) -> mul(x, y) mul(0, ...) -> alloc(0, shapes...) This is not done if we would add more nodes in the graph, like with: mul(-1, x, y) -/-> neg(mul(x, y)) """ # here, we are past the point of canonicalization, so we don't # want to put in un-necessary fills. # # at this point [post canonicalize], mul() may have many inputs. if node.op == T.mul: # the idea here is that we have pow(x, y) neg = False new_inputs = [] nb_neg_node = 0 nb_cst = 0 for input in node.inputs: # remove any neg arguments while input.owner and input.owner.op == T.neg: neg ^= True input = input.owner.inputs[0] nb_neg_node += 1 # remove special case arguments of 1, -1 or 0 y = local_mul_canonizer.get_constant(input) if y == 1.0: nb_cst += 1 elif y == -1.0: nb_cst += 1 neg ^= True # toggles elif y == 0.0: # if we find any zero, we just return right away return [broadcast_like(0, node.outputs[0], node.fgraph)] else: new_inputs.append(input) if new_inputs != node.inputs: if new_inputs: if len(new_inputs) == 1: if neg: if new_inputs[0].dtype in (T.uint_dtypes + ['bool']): return else: rval = -new_inputs[0] else: rval = new_inputs[0] else: # The next case would cause a replace by an equivalent case. if (neg and nb_neg_node == 0 and nb_cst == 1): return elif neg: # Don't add an extra neg node as we can't # fully replace this mul by a neg. m1 = np.asarray(-1, dtype=node.outputs[0].dtype) new_inputs = [m1] + new_inputs rval = T.mul(*new_inputs) return [broadcast_like(rval, node.outputs[0], node.fgraph)] else: # there are no variable inputs to mul # N.B. this could have been constant-folded... if neg: return [broadcast_like(-1, node.outputs[0], node.fgraph)] else: return [broadcast_like(1, node.outputs[0], node.fgraph)] register_specialize(local_mul_specialize) @gof.local_optimizer([T.add]) def local_add_specialize(node): def fill_chain(v): out = _fill_chain(v, node.inputs) return out # here, we are past the point of canonicalization, so we don't want # to put in un-necessary fills. if node.op == T.add: new_inputs = [] for input in node.inputs: try: y = get_scalar_constant_value(input) except NotScalarConstantError: y = input if np.all(y == 0.0): continue new_inputs.append(input) if len(new_inputs) < len(node.inputs): dtype = node.outputs[0].type.dtype if len(new_inputs) == 0: # we got rid of the entire expression! ndim = node.outputs[0].type.ndim # Reuse call to constant for cache() cst = T.constant(np.zeros((1,) * ndim, dtype=dtype)) assert cst.type.broadcastable == (True,) * ndim return fill_chain(cst) if len(new_inputs) == 1: ret = fill_chain(new_inputs[0]) else: ret = fill_chain(T.add(*new_inputs)) # The dtype should not be changed. It can happen if the input # that was forcing upcasting was equal to 0. if ret[0].dtype != dtype: ret = [T.cast(ret[0], dtype)] return ret else: return False register_specialize(local_add_specialize) mul_canonizer = in2out(gof.LocalOptGroup(local_mul_canonizer, local_fill_sink, apply_all_opts=True), name='mul_canonizer_groups') def check_for_x_over_absX(numerators, denominators): """Convert x/abs(x) into sign(x). """ # TODO: this function should dig/search through dimshuffles # This won't catch a dimshuffled absolute value for den in list(denominators): if (den.owner and den.owner.op == T.abs_ and den.owner.inputs[0] in numerators): if den.owner.inputs[0].type.dtype.startswith('complex'): # TODO: Make an Op that projects a complex number to # have unit length but projects 0 to 0. That # would be a weird Op, but consistent with the # special case below. I heard there's some # convention in Matlab that is similar to # this... but not sure. pass else: denominators.remove(den) numerators.remove(den.owner.inputs[0]) numerators.append(T.sgn(den.owner.inputs[0])) return numerators, denominators local_mul_canonizer.add_simplifier(check_for_x_over_absX, 'X_over_absX') @register_canonicalize @gof.local_optimizer([T.abs_]) def local_abs_lift(node): """ Move the abs toward the input. This is needed for check_for_x_over_absX to apply in more case. """ if node.op == T.abs_ and node.inputs[0].owner: assert node.nin == 1 if node.inputs[0].owner.op == T.mul: return [T.mul(*[T.abs_(i) for i in node.inputs[0].owner.inputs])] if node.inputs[0].owner.op == T.true_div: i = node.inputs[0].owner.inputs return [T.true_div(T.abs_(i[0]), T.abs_(i[1]))] @register_specialize @gof.local_optimizer([T.mul, T.true_div]) def local_abs_merge(node): """ Merge abs generated by local_abs_lift when the canonizer don't need it anymore """ if node.op == T.mul and sum([i.owner.op == T.abs_ for i in node.inputs if i.owner]) > 1: inputs = [] for i in node.inputs: if i.owner and i.owner.op == T.abs_: inputs.append(i.owner.inputs[0]) elif isinstance(i, Constant): try: const = get_scalar_constant_value(i, only_process_constants=True) except NotScalarConstantError: return False if not (const >= 0).all(): return False inputs.append(i) else: return False return [T.abs_(T.mul(*inputs))] if node.op == T.true_div and sum([i.owner.op == T.abs_ for i in node.inputs if i.owner]) == 2: return [T.abs_(T.true_div(node.inputs[0].owner.inputs[0], node.inputs[1].owner.inputs[0]))] @register_stabilize @register_specialize @gof.local_optimizer([T.log]) def local_log1p(node): # log(1+x) -> log1p(x) # log(1-x) -> log1p(-x) if node.op == T.log: log_arg, = node.inputs if log_arg.owner and log_arg.owner.op == T.add: scalars, scalar_inputs, nonconsts = scalarconsts_rest( log_arg.owner.inputs, only_process_constants=True) # scalar_inputs are potentially dimshuffled and fill'd scalars if scalars and np.allclose(np.sum(scalars), 1): if nonconsts: if len(nonconsts) > 1: ninp = T.add(*nonconsts) else: ninp = nonconsts[0] if ninp.dtype != log_arg.type.dtype: ninp = ninp.astype(node.outputs[0].dtype) return _fill_chain(T.log1p(ninp), scalar_inputs) elif log_arg.owner and log_arg.owner.op == T.sub: one = T.extract_constant(log_arg.owner.inputs[0], only_process_constants=True) if one != 1: return other = log_arg.owner.inputs[1] if other.dtype != log_arg.dtype: other = other.astype(log_arg.dtype) return [T.log1p(T.neg(other))] # TODO: in canonicalize, change log10 and log2 -> log @register_stabilize @register_specialize @gof.local_optimizer([T.log]) def local_log_add(node): # log(exp(x)+exp(y)) # # Suppose x >= y # log(exp(x) + exp(y)) # log(exp(x) * (1 + exp(y)/exp(x))) # x + log(1 + exp(y)/exp(x)) # x + log1p(exp(y)/exp(x)) # x + log1p(exp(y-x)) if node.op == T.log: z = node.inputs[0] if z.owner and z.owner.op == T.add: zi = z.owner.inputs if len(zi) != 2: # -- upgrading Maximum to handle multiple inputs wasn't trivial # TODO # raise NotImplementedError() return pre_exp = [x.owner.inputs[0] for x in zi if x.owner and x.owner.op == T.exp] if len(pre_exp) == len(zi): # all arguments to add are exp() max_pre = T.maximum(*pre_exp) ret = max_pre + T.log1p(T.exp(T.add(*[p - max_pre for p in pre_exp]))) ret.tag.values_eq_approx = values_eq_approx_remove_inf return [ret] @gof.local_optimizer([T.log]) def local_log_sum_exp(node): # log(sum_i(exp(x_i))) = x_max + log(sum_i(exp(x_i - x_max))) if node.op != T.log: return sum_node = node.inputs[0].owner # If the sum has keepdims=True, there might be a dimshuffle if sum_node and isinstance(sum_node.op, T.DimShuffle): dimshuffle_op = sum_node.op sum_node = sum_node.inputs[0].owner else: dimshuffle_op = None if not sum_node or not isinstance(sum_node.op, T.Sum): return exp_node, axis = sum_node.inputs[0].owner, sum_node.op.axis if not exp_node or not ( isinstance(exp_node.op, Elemwise) and isinstance(exp_node.op.scalar_op, scalar.Exp)): return pre_exp = exp_node.inputs[0] max_pre_exp = T.max(pre_exp, axis=axis) max_pre_exp_keepdims = T.makeKeepDims(pre_exp, max_pre_exp, axis) ret = (max_pre_exp + T.log(T.sum(T.exp(pre_exp - max_pre_exp_keepdims), axis=axis))) # Restore the dimshuffle op, if any. if dimshuffle_op: ret = dimshuffle_op(ret) return [ret] compile.optdb.register('local_log_sum_exp', in2out(local_log_sum_exp, ignore_newtrees=True), 1.6, 'fast_run') def add_calculate(num, denum, aslist=False, out_type=None): # TODO: make sure that this function and mul_calculate are similar if out_type is None: zero = 0.0 else: zero = theano._asarray(0, dtype=out_type.dtype) # zero = 0.0 if out_type is None else theano._asarray(0, # dtype=out_type.dtype) v = reduce(np.add, num, zero) - reduce(np.add, denum, zero) if aslist: if np.all(v == 0): return [] else: return [v] return v local_add_canonizer = Canonizer(T.add, T.sub, T.neg, add_calculate) add_canonizer = in2out(gof.LocalOptGroup(local_add_canonizer, local_fill_sink, apply_all_opts=True), name='add_canonizer_group') register_canonicalize(local_add_canonizer, name='local_add_canonizer') ################## # Distributivity # ################## def distribute_greedy(pos_pairs, neg_pairs, num, denum, out_type, minscore=0): # each pair in pos_pairs and neg_pairs is a num/denum pair. this # function attempts to add num and denum to the corresponding parts # of each pair, and counts how many multiplications/divisions can # be saved in that way. # each division is counted like div_cost multiplications # (typically, division costs more so we are willing to multiply more # in order to divide less) # 1.5 was obtained through an informal test and may very well be # platform dependent div_cost = 1.5 # score is number of operations saved, higher is better score = len(num) + div_cost * len(denum) new_pos_pairs = list(itertools.starmap(local_mul_canonizer.simplify, [(n + num, d + denum, out_type) for (n, d) in pos_pairs])) new_neg_pairs = list(itertools.starmap(local_mul_canonizer.simplify, [(n + num, d + denum, out_type) for (n, d) in neg_pairs])) for (n, d), (nn, dd) in zip(pos_pairs + neg_pairs, new_pos_pairs + new_neg_pairs): # We calculate how many operations we are saving with the new # num and denum score += len(n) + div_cost * len(d) - len(nn) - div_cost * len(dd) if score <= minscore: # the change is not applied because it adds too many operations return False, pos_pairs, neg_pairs return True, new_pos_pairs, new_neg_pairs def attempt_distribution(factor, num, denum, out_type): # we try to insert each num and each denum in the factor # returns: changes?, new_factor, new_num, new_denum # if there are changes, new_num and new_denum contain all the numerators # and denumerators that could not be distributed in the factor pos, neg = local_add_canonizer.get_num_denum(factor) if len(pos) == 1 and not neg: return False, factor, num, denum pos_pairs = list(map(local_mul_canonizer.get_num_denum, pos)) neg_pairs = list(map(local_mul_canonizer.get_num_denum, neg)) change = False for n in list(num): success, pos_pairs, neg_pairs = distribute_greedy(pos_pairs, neg_pairs, [n], [], out_type) if success: change = True num.remove(n) for d in list(denum): success, pos_pairs, neg_pairs = distribute_greedy(pos_pairs, neg_pairs, [], [d], out_type) if success: change = True denum.remove(d) if not change: return change, factor, num, denum else: return change, local_add_canonizer.merge_num_denum( list(itertools.starmap(local_mul_canonizer.merge_num_denum, pos_pairs)), list(itertools.starmap(local_mul_canonizer.merge_num_denum, neg_pairs))), num, denum @register_canonicalize @register_stabilize @gof.local_optimizer([T.mul, T.true_div, T.inv]) def local_greedy_distributor(node): """ Optimize by reducing the number of multiplications and/or divisions. This optimization tries to apply distributivity of multiplication to addition in order to reduce the number of multiplications and/or divisions that must be done. The algorithm weighs division more than multiplication to account for the former's slightly greater computational cost. The following expressions are simplified: 1. ((a/x + b/y) * x * y) --> a*y + b*x 2. ((a/x + b) * x) --> a + b*x 3. There are other forms too where node is a true_div. The following expressions are not simplified: 4. ((a + b) * x) -/-> a*x + b*x This optimization aims to reduce computational cost. It may also increase numerical stability, e.g. when x and/or y tend to 0 in example 1. """ out = node.outputs[0] num, denum = local_mul_canonizer.get_num_denum(out) if len(num) == 1 and not denum: return False new_num, new_denum = [], [] change = False out_type = out.type for candidate in list(num): if candidate not in num: continue num.remove(candidate) _change, candidate, num, denum = attempt_distribution( candidate, num, denum, out_type,) change |= _change new_num.append(candidate) for candidate in list(denum): if candidate not in denum: continue denum.remove(candidate) _change, candidate, denum, num = attempt_distribution( candidate, denum, num, out_type) change |= _change new_denum.append(candidate) if not change: return False new_num += num new_denum += denum rval = local_mul_canonizer.merge_num_denum(new_num, new_denum) if not (rval.type == out.type): # WHY DOES THIS HAPPEN? return False return [rval] @gof.local_optimizer(None) def constant_folding(node): for input in node.inputs: if not isinstance(input, Constant): return False # condition: all inputs are constant if not node.op.do_constant_folding(node): # The op asks not to be constant folded. return False storage_map = dict([(i, [i.data]) for i in node.inputs]) compute_map = dict([(i, [True]) for i in node.inputs]) for o in node.outputs: storage_map[o] = [None] compute_map[o] = [False] impl = None if (hasattr(node.op, 'python_constant_folding') and node.op.python_constant_folding(node)): impl = 'py' thunk = node.op.make_thunk(node, storage_map, compute_map, no_recycling=[], impl=impl) required = thunk() assert not required # a node whose inputs are all provided should always # return successfully rval = [] for output in node.outputs: assert compute_map[output][0], (output, storage_map[output][0]) try: constant = output.type.Constant except AttributeError: constant = Constant v = constant(output.type, storage_map[output][0]) copy_stack_trace(output, v) rval.append(v) return rval topo_constant_folding = in2out(constant_folding, ignore_newtrees=True, name="topo_constant_folding") register_canonicalize(topo_constant_folding, 'fast_compile', final_opt=True) register_uncanonicalize(topo_constant_folding, 'fast_compile', final_opt=True) register_stabilize(topo_constant_folding, 'fast_compile', final_opt=True) register_specialize(topo_constant_folding, 'fast_compile', final_opt=True) def get_clients(node): """ Used by erf/erfc opt to track less frequent op. """ return [c for c, i in node.outputs[0].clients if c != "output"] def get_clients2(node): """ Used by erf/erfc opt to track less frequent op. """ l = [] for c, i in node.outputs[0].clients: if c != "output": for var in c.outputs: l.extend([cc for cc, ii in var.clients if cc != "output"]) return l # 1+erf(x)=>erfc(-x) local_one_plus_erf = gof.PatternSub((T.add, 1, (T.erf, 'x')), (T.erfc, (T.neg, 'x')), allow_multiple_clients=True, name='local_one_plus_erf', tracks=[T.erf], get_nodes=get_clients) register_canonicalize(local_one_plus_erf) register_stabilize(local_one_plus_erf) register_specialize(local_one_plus_erf) # 1-erf(x)=>erfc(x) local_one_minus_erf = gof.PatternSub((T.sub, 1, (T.erf, 'x')), (T.erfc, 'x'), allow_multiple_clients=True, name='local_one_minus_erf',) register_canonicalize(local_one_minus_erf) register_stabilize(local_one_minus_erf) register_specialize(local_one_minus_erf) local_one_minus_erf2 = gof.PatternSub((T.add, 1, (T.mul, -1, (T.erf, 'x'))), (T.erfc, 'x'), allow_multiple_clients=True, name='local_one_minus_erf2') register_canonicalize(local_one_minus_erf2) register_stabilize(local_one_minus_erf2) register_specialize(local_one_minus_erf2) # 1+(-erf(x))=>erfc(x) This is a different graph then the previous as # the canonicalize don't work completly local_one_plus_neg_erf = gof.PatternSub((T.add, 1, (T.neg, (T.erf, 'x'))), (T.erfc, 'x'), allow_multiple_clients=True, name='local_one_plus_neg_erf', tracks=[T.erf], get_nodes=get_clients2) register_canonicalize(local_one_plus_neg_erf) register_stabilize(local_one_plus_neg_erf) register_specialize(local_one_plus_neg_erf) # (-1)+erf(x) => -erfc(x) don't need erf(x)+(-1) as the canonicalize # will put the -1 as the first argument. local_erf_minus_one = gof.PatternSub((T.add, -1, (T.erf, 'x')), (T.neg, (T.erfc, 'x')), allow_multiple_clients=True, name='local_erf_minus_one', tracks=[T.erf], get_nodes=get_clients) register_canonicalize(local_erf_minus_one) register_stabilize(local_erf_minus_one) register_specialize(local_erf_minus_one) # 1-erfc(x) => erf(x) local_one_minus_erfc = gof.PatternSub((T.sub, 1, (T.erfc, 'x')), (T.erf, 'x'), allow_multiple_clients=True, name='local_one_minus_erfc', tracks=[T.erfc], get_nodes=get_clients) register_canonicalize(local_one_minus_erfc) register_stabilize(local_one_minus_erfc) register_specialize(local_one_minus_erfc) local_one_minus_erfc2 = gof.PatternSub((T.add, 1, (T.neg, (T.erfc, 'x'))), (T.erf, 'x'), allow_multiple_clients=True, name='local_one_minus_erfc2', tracks=[T.erfc], get_nodes=get_clients2) register_canonicalize(local_one_minus_erfc2) register_stabilize(local_one_minus_erfc2) register_specialize(local_one_minus_erfc2) local_one_minus_erfc3 = gof.PatternSub((T.add, 1, (T.mul, -1, (T.erfc, 'x'))), (T.erf, 'x'), allow_multiple_clients=True, name='local_one_minus_erfc3', tracks=[T.erfc], get_nodes=get_clients2) register_canonicalize(local_one_minus_erfc3) register_stabilize(local_one_minus_erfc3) register_specialize(local_one_minus_erfc3) # 1+(-erfc(x)) => erf(x) This is a different graph then the previous as # the canonicalize don't work completly local_one_add_neg_erfc = gof.PatternSub((T.add, 1, (T.neg, (T.erfc, 'x'))), (T.erf, 'x'), allow_multiple_clients=True, name='local_one_add_neg_erfc', tracks=[T.erfc], get_nodes=get_clients2) register_canonicalize(local_one_add_neg_erfc) register_stabilize(local_one_add_neg_erfc) register_specialize(local_one_add_neg_erfc) # (-1)+erfc(-x)=>erf(x) local_erf_neg_minus_one = gof.PatternSub((T.add, -1, (T.erfc, (T.neg, 'x'))), (T.erf, 'x'), allow_multiple_clients=True, name='local_erf_neg_minus_one', tracks=[T.erfc], get_nodes=get_clients) register_canonicalize(local_erf_neg_minus_one) register_stabilize(local_erf_neg_minus_one) register_specialize(local_erf_neg_minus_one) # (-1)+erfc(-1*x)=>erf(x) local_erf_neg_minus_one2 = gof.PatternSub((T.add, -1, (T.erfc, (T.mul, -1, 'x'))), (T.erf, 'x'), allow_multiple_clients=True, name='local_erf_neg_minus_one2', tracks=[T.erfc], get_nodes=get_clients) register_canonicalize(local_erf_neg_minus_one2) register_stabilize(local_erf_neg_minus_one2) register_specialize(local_erf_neg_minus_one2) # Stability optimization # log(erfc(x)) => when x>threashold, # -x**2-log(x)-.5*log(pi)+log(1-1/(2*x**2)+3/(4*x**4)-15/(8*x**6)) # for float64: threshold=26.641747557 was choosed with: # [(i,numpy.log(scipy.special.erfc(numpy.asarray([i],dtype='float64')))) # for i in numpy.arange(26.641747557,26.6417475571,.00000000001)] # for float32: threshold=10.0541949, [(i,numpy.log(scipy.special.erfc( # numpy.asarray([i],dtype='float32')))) for i in numpy.arange( # 10.0541948,10.0541951,.0000001)] @register_stabilize @register_specialize @gof.local_optimizer([T.log]) def local_log_erfc(node): if node.op != T.log: return False if not node.inputs[0].owner or node.inputs[0].owner.op != T.erfc: return False if hasattr(node.tag, 'local_log_erfc_applied'): # We use that flag to don't apply the optimization recursively return False node.tag.local_log_erfc_applied = True x = node.inputs[0].owner.inputs[0] stab_value = (-x ** 2 - T.log(x) - .5 * T.log(np.pi) + T.log(1 - 1 / (2 * x ** 2) + 3 / (4 * x ** 4) - 15 / (8 * x ** 6))) if (node.outputs[0].dtype == 'float32' or node.outputs[0].dtype == 'float16'): threshold = 10.0541949 elif node.outputs[0].dtype == 'float64': threshold = 26.641747557 ret = T.switch(x < threshold, node.outputs[0], stab_value) ret.tag.values_eq_approx = values_eq_approx_remove_inf return [ret] # Stability optimization of the grad of log(erfc(x)) # ([y*]exp(-(x**2)))/erfc(x) # The y* is optional # ([y*]exp(x**2))/erfc(-x) => [y*](when x>threashold, # sqrt(pi)*-x/(1-1/(2*x**2)+3/(4*x**4)-15/(8*x**6))) # for float64: threshold=26.63 see at the end of the fct for the explaination # for float32: threshold=9.3 see at the end of the fct for the explaination # TODO: remove the contraint that there are only 2 inputs to exp(x**2) # is the second. # TODO: at the test point 10 in float32, there is instability in the original # value. The original gives -30.0, the stab -20.1 and in float64 -18.1. # Make it so that the test does not generate an error in that case! @register_stabilize @register_specialize @gof.local_optimizer([T.true_div]) def local_grad_log_erfc_neg(node): if node.op != T.true_div: return False if not node.inputs[1].owner or node.inputs[1].owner.op != T.erfc: return False erfc = node.inputs[1] erfc_x = erfc.owner.inputs[0] if not node.inputs[0].owner: return False # The mul is optional. if node.inputs[0].owner.op != T.mul: mul = None y = [] if not node.inputs[0].owner or node.inputs[0].owner.op != T.exp: return False exp = node.inputs[0] else: mul = node.inputs[0] exp = None for idx, inp in enumerate(mul.owner.inputs): if inp.owner and inp.owner.op == T.exp: exp = inp break if len(mul.owner.inputs) == 2: y = [mul.owner.inputs[1 - idx]] else: y = mul.owner.inputs[:] del y[idx] del mul if not exp.owner.inputs[0].owner: return False if exp.owner.inputs[0].owner.op == T.neg: neg = exp.owner.inputs[0] if (not neg.owner.inputs[0].owner or neg.owner.inputs[0].owner.op != T.sqr): return False sqr = neg.owner.inputs[0] x = sqr.owner.inputs[0] elif exp.owner.inputs[0].owner.op == T.mul: # We should compare that -(erfc_x**2) is equivalent to mul_neg. # There is currently no easy way to do this in the general case, # so we implement some common case for now. # In many cases the neg are replaced by mul in the graph. # This also allows to stabilize log(erfc(cst*x)). mul_neg = exp.owner.inputs[0] # In case that multiple mul are not fused together, we do it here. def check_input(inputs): new_inputs = [] for i in inputs: if i.owner and i.owner.op == T.mul: new_inputs.extend(check_input(i.owner.inputs)) else: new_inputs.append(i) return new_inputs mul_inputs = check_input(mul_neg.owner.inputs) # Put the constant first. for i in xrange(len(mul_inputs)): if isinstance(i, Constant): if i == 0: break else: tmp = mul_inputs[0] mul_inputs[0] = mul_inputs[i] mul_inputs[i] = tmp break mul_neg = T.mul(*mul_inputs) try: cst2 = get_scalar_constant_value(mul_neg.owner.inputs[0], only_process_constants=True) except NotScalarConstantError: return False if len(mul_neg.owner.inputs) == 2: if (not mul_neg.owner.inputs[1].owner or mul_neg.owner.inputs[1].owner.op != T.sqr): return False sqr = mul_neg.owner.inputs[1] x = sqr.owner.inputs[0] elif len(mul_neg.owner.inputs) == 3: if mul_neg.owner.inputs[1] is not mul_neg.owner.inputs[2]: return False x = mul_neg.owner.inputs[1] else: return False if cst2 != -1: if (not erfc_x.owner or erfc_x.owner.op != T.mul or len(erfc_x.owner.inputs) != 2): # todo implement that case return False if erfc_x.owner.inputs[1] is not mul_neg.owner.inputs[1]: return False x = erfc_x try: cst = get_scalar_constant_value(erfc_x.owner.inputs[0], only_process_constants=True) except NotScalarConstantError: return False if cst2 != -cst * 2: return False # The constant is valid. Must check that the elif erfc_x is not x: return False else: return False if hasattr(node.tag, 'local_grad_log_erfc_neg'): # We use that flag to don't apply the optimization recursively return False # we move the y outside the div. true_div_no_mul = T.true_div(exp, erfc) true_div_no_mul.owner.tag.local_grad_log_erfc_neg = True # aaron value stab_value = (x * T.pow(1 - 1 / (2 * (x ** 2)) + 3 / (4 * (x ** 4)) - 15 / (8 * (x ** 6)), -1) * T.cast(T.sqrt(np.pi), dtype=x.dtype)) if x.dtype == 'float32' or x.dtype == 'float16': threshold = 9.3 # threshold = 10.1 elif x.dtype == 'float64': threshold = 26.641747557 ret = T.switch(x < threshold, true_div_no_mul, stab_value) if y: ret = T.mul(ret, *y) ret.tag.values_eq_approx = values_eq_approx_remove_inf_nan return [ret] """ The libm used for the test is amdlibm #([y*]exp(-(x**2)))/erfc(x) # The mul is optional #exp(x**2)/erfc(-x) => when x>threashold, #-x*(1-1/(2*x**2)+3/(4*x**4)-15/(8*x**6))*sqrt(pi) for float64: #threshold=26.63 see below for float32: threshold=9.3 see below TODO #remove the contraint that there are only 2 inputs to mul TODO: should #we cast numpy.pi to x.dtype? #float32 threshold 9.3 as the approximation is more precise at that #point and more stable. import numpy, scipy.special r = numpy.arange(9,10.06,.01) p64=[(numpy.exp(-(x**2)))/scipy.special.erfc(x) for x in r] p32=[(numpy.exp(-(x**2)))/scipy.special.erfc(x) for x in numpy.asarray(r,dtype='float32')] a64=[x*((1-1/(2*x**2)+3/(4*x**4)-15/(8*x**6))**(-1))*numpy.sqrt(numpy.pi) for x in r] a32=[x*((1-1/(2*x**2)+3/(4*x**4)-15/(8*x**6))**(-1)) * numpy.float32(numpy.sqrt(numpy.pi)) for x in numpy.asarray(r,dtype='float32')] for idx,(a,b,c,d,e) in enumerate(zip(r,p64,p32,a64,a32)):print a,b,c,d,e,c-b,e-b,numpy.absolute(c-b) 1: # We don't support the fusion for node with multiple outputs. return inputs = [] # inputs of the new Elemwise op. s_inputs = [] # inputs of the new scalar op used by the Composite. # Inputs of the new scalar op that represents the current node. s_g = [] # There is a hard limit of 256 bytes for the formal argument list to a # GPU kernel function. max_nb_input = max_input_fct(node) # The number of inputs to the new fused op if we do not fuse more # inputs. new_nb_input = len(node.inputs) # Did we fuse something? # Needed as we can fuse unary op that don't change the number of # inputs. # And there is a case where the inputs are the same as the current # node. That won't change the number of inputs of the new op. fused = False for i in node.inputs: do_fusion = False catch = False # Will store inputs of the fused node that are not currently inputs # of the node we want to create (to avoid duplicating inputs). tmp_input = [] # Same as tmp_input, but for scalars. tmp_scalar = [] # We should not check the number of inputs here # As fusing op don't always change the number of input. # If a variable is used as multiple into to the same node, # we still want to fusion. So we take the set. if (i.owner and isinstance(i.owner.op, OP) and len(set([n for n, idx in i.clients])) == 1 and # Do not merge elemwise that don't have the same # broadcastable pattern to don't redo duplicate # computation due to broadcast. i.owner.outputs[0].broadcastable == node.outputs[0].broadcastable): do_fusion = True try: tmp_s_input = [] # we should not put duplicate input into s_inputs and inputs for ii in i.owner.inputs: if ii in inputs: tmp_s_input.append(s_inputs[inputs.index(ii)]) elif ii in tmp_input: tmp_s_input.append(tmp_scalar[tmp_input.index(ii)]) else: tmp = scalar.get_scalar_type(ii.dtype).make_variable() try: tv = gof.op.get_test_value(ii) if tv.size > 0: tmp.tag.test_value = tv.flatten()[0] else: tmp.tag.test_value = tv except AttributeError: pass tmp_s_input.append(tmp) tmp_input.append(ii) tmp_scalar.append(tmp_s_input[-1]) s_op = i.owner.op.scalar_op(*tmp_s_input, return_list=True) # if the scalar_op don't have a c implementation, # we skip its fusion to allow the fusion of the # other ops. i.owner.op.scalar_op.c_code(s_op[0].owner, "test_presence_of_c_code", ["x" for x in i.owner.inputs], ["z" for z in i.owner.outputs], {"fail": "%(fail)s"}) except MethodNotDefined: catch = True except NotImplementedError: catch = True if catch: _logger.info(("%s does not implement the c_code function." " As well as being potentially slow, this" " disables loop fusion of this op.") % str(i.owner.op.scalar_op)) do_fusion = False # Compute the number of inputs in case we fuse this input. # We subtract 1 because we replace the existing input with the new # inputs from `tmp_input`. new_nb_input_ = new_nb_input + len(tmp_input) - 1 # If the new input is already an input of the current node, it was # already counted when `new_nb_input` was initialized to # len(node.inputs). # This can happen when a variable is used both by the Elemwise to # fuse and the current node. for x in tmp_input: if x in node.inputs: new_nb_input_ -= 1 if do_fusion and (new_nb_input_ <= max_nb_input): fused = True new_nb_input = new_nb_input_ inputs.extend(tmp_input) s_inputs.extend(tmp_scalar) s_g.extend(s_op) else: # We must support the case where the same variable appear many # time in the inputs if inputs.count(i) == node.inputs.count(i): s = s_inputs[inputs.index(i)] else: s = scalar.get_scalar_type(i.dtype).make_variable() try: if theano.config.compute_test_value != 'off': v = gof.op.get_test_value(i) if v.size > 0: s.tag.test_value = v.flatten()[0] except AttributeError: pass inputs.append(i) s_inputs.append(s) s_g.append(s) if not fused: return False if new_nb_input != len(inputs) or len(s_inputs) != len(inputs): raise Exception("""Something has gone wrong with the elemwise fusion optimization. We skip this optimization. You can ignore this message, your code will run correctly, but may be slower.""") s_new_out = node.op.scalar_op(*s_g, return_list=True) try: s_new_out[0].owner.op.c_code(s_new_out[0].owner, "test_presence_of_c_code", ["x" for x in s_g], ["z" for x in s_new_out], {"fail": "%(fail)s"}) except MethodNotDefined: _logger.info(("%s does not implement the c_code function." " As well as being potentially slow, this disables " "loop fusion of this op.") % str( s_new_out[0].owner.op)) return False except NotImplementedError: _logger.info(("%s does not implement the c_code function. As well" " as being potentially slow, this disables loop" " fusion of this op.") % str(s_new_out[0].owner.op)) return False # create the composite op. C = scalar.Composite(s_inputs, s_new_out) # create the new node. # Do not call make_node to have test_value n = maker(node, C)(*inputs).owner assert len(n.outputs) == 1 assert node.outputs[0].dtype == n.outputs[0].dtype if len(n.inputs) > max_nb_input: _logger.info('loop fusion failed because Op would exceed' ' kernel argument limit.') return False # we fuse as many that we can at the same time to make debug mode faster # debug mode will be faster as it won't test all intermediate step. while True: ret = local_fuse(n) if ret is not False and ret is not None: # print n,ret assert len(ret) == len(n.outputs) assert len(ret) == 1 n = ret[0].owner else: break return n.outputs return local_fuse def elemwise_max_input_fct(node): # The Elemwise.perform use numpy ufunc and they are limited to 31 # inputs. if not theano.config.cxx: return 31 return 1024 local_elemwise_fusion = local_elemwise_fusion_op(T.Elemwise, elemwise_max_input_fct) class FusionOptimizer(Optimizer): """Graph optimizer for Fusion of elemwise operations.""" def __init__(self, local_optimizer): Optimizer.__init__(self) self.optimizer = local_optimizer def add_requirements(self, fgraph): fgraph.attach_feature(toolbox.ReplaceValidate()) def apply(self, fgraph): did_something = True nb_iter = 0 nb_replacement = 0 nb_inconsistency_replace = 0 time_toposort = 0 if fgraph.profile: validate_before = fgraph.profile.validate_time callbacks_before = fgraph.execute_callbacks_times.copy() callback_before = fgraph.execute_callbacks_time while did_something: t0 = time.time() nodelist = list(fgraph.toposort()) time_toposort += time.time() - t0 nodelist.reverse() did_something = False for node in nodelist: # Don't try to fuse node that have already been fused. if node in fgraph.apply_nodes: new_outputs = self.optimizer(node) if new_outputs: assert len(new_outputs) == len(node.outputs) try: fgraph.replace_all_validate( list(zip(node.outputs, new_outputs)), reason=self.__class__.__name__) did_something = True nb_replacement += 1 except InconsistencyError: nb_inconsistency_replace += 1 pass nb_iter += 1 if fgraph.profile: validate_time = fgraph.profile.validate_time - validate_before callback_time = fgraph.execute_callbacks_time - callback_before callbacks_time = {} for k, v in iteritems(fgraph.execute_callbacks_times): if k in callbacks_before: callbacks_time[k] = v - callbacks_before[k] else: callbacks_time[k] = v else: validate_time = None callback_time = None callbacks_time = {} return (self, nb_iter, nb_replacement, nb_inconsistency_replace, validate_time, callback_time, callbacks_time, time_toposort) @staticmethod def print_profile(stream, prof, level=0): blanc = (' ' * level) print(blanc, "FusionOptimizer", file=stream) print(blanc, " nb_iter", prof[1], file=stream) print(blanc, " nb_replacement", prof[2], file=stream) print(blanc, " nb_inconsistency_replace", prof[3], file=stream) print(blanc, " validate_time", prof[4], file=stream) print(blanc, " callback_time", prof[5], file=stream) if prof[5] > 1: print(blanc, " callbacks_time", file=stream) for i in sorted(iteritems(prof[6]), key=lambda a: a[1])[::-1]: if i[1] > 0: print(blanc, " ", i) print(blanc, " time_toposort", prof[7], file=stream) def local_add_mul_fusion(node): """Fuse consecutive add or mul in one such node with more inputs. It is better to fuse add/mul that way then in a Composite node as this make the inner graph of the Composite smaller. This allow to put more computation in a Composite before hitting the max recusion limit when pickling Composite. """ if (not isinstance(node.op, Elemwise) or not isinstance(node.op.scalar_op, (scalar.Add, scalar.Mul))): return False s_op = node.op.scalar_op.__class__ new_inp = [] fused = False nb_inputs = len(node.inputs) max_inputs = float('inf') if hasattr(node.op, 'max_inputs'): max_inputs = node.op.max_inputs(node) for inp in node.inputs: if (inp.owner and isinstance(inp.owner.op, Elemwise) and isinstance(inp.owner.op.scalar_op, s_op) and # Do not duplicate the operation. len(inp.clients) == 1 and (nb_inputs + len(inp.owner.inputs) - 1) <= max_inputs): new_inp.extend(inp.owner.inputs) fused = True else: new_inp.append(inp) # We can not compare the number of inputs as Mul and Add could have # 0 or 1 inputs in some corner cases. if fused: output = node.op(*new_inp) copy_stack_trace(node.outputs[0], output) # Do the recursion here to help lower the number of # FusionOptimizer iteration. if output.owner: output2 = local_add_mul_fusion(output.owner) if output2: return output2 return [output] if config.tensor.local_elemwise_fusion: _logger.debug("enabling optimization fusion elemwise in fast_run") # Must be after gpu(48.5) and before AddDestroyHandler(49.5) fuse_seqopt = gof.SequenceDB() fuse_seqopt.register('local_add_mul_fusion', FusionOptimizer(local_add_mul_fusion), 0, 'fast_run', 'fusion') fuse_seqopt.register('composite_elemwise_fusion', FusionOptimizer(local_elemwise_fusion), 1, 'fast_run', 'fusion') compile.optdb.register('elemwise_fusion', fuse_seqopt, 49, 'fast_run', 'fusion', 'local_elemwise_fusion', 'FusionOptimizer') else: _logger.debug("not enabling optimization fusion elemwise in fast_run") compile.optdb.register('elemwise_fusion', FusionOptimizer(local_elemwise_fusion), 49, 'fusion', 'local_elemwise_fusion', 'FusionOptimizer') @register_canonicalize @gof.local_optimizer([Elemwise]) def local_useless_composite(node): """For elemwise Composite that have multiple outputs, remove the outputs that are not used. """ if (not isinstance(node.op, Elemwise) or not isinstance(node.op.scalar_op, scalar.Composite)): return comp = node.op.scalar_op idx = [i for i, o_extern in enumerate(node.outputs) if o_extern.clients] if len(idx) < len(node.outputs): new_outputs = [comp.outputs[i] for i in idx] c = scalar.Composite(inputs=comp.inputs, outputs=new_outputs) e = Elemwise(scalar_op=c)(*node.inputs, return_list=True) return dict(zip([node.outputs[i] for i in idx], e)) # ############################ # # Remove consider_constant # # ############################ # Although the ops ConsiderConstant, ZeroGrad and DisconnectedGrad # just returns the input, it should be removed from the graph to @register_canonicalize('fast_compile') @register_useless('fast_compile') @gof.local_optimizer(None) def local_view_op(node): if isinstance(node.op, theano.compile.ops.ViewOp): return node.inputs @register_useless @register_canonicalize @register_stabilize @register_specialize @gof.local_optimizer([T.Alloc]) def local_merge_alloc(node): # This opt takes care of several cases: # Alloc(Alloc(m, x, 1, 1, 1), x, y, z, w) -> Alloc(m, x, y, z, w) # Alloc(Alloc(m, y, 1, 1), x, y, z, w) -> Alloc(m, x, y, z, w) # Alloc(Alloc(m, y1, 1, 1), x, y2, z, w) -> Alloc(m, x, assert(y1, y1==y2), z, w) if not isinstance(node.op, T.Alloc): return False if not node.inputs[0].owner or not isinstance( node.inputs[0].owner.op, T.Alloc): return False inputs_outer = node.inputs inputs_inner = node.inputs[0].owner.inputs dims_outer = inputs_outer[1:] dims_inner = inputs_inner[1:] dims_outer_rev = dims_outer[::-1] dims_inner_rev = dims_inner[::-1] # check if the pattern of broadcasting is matched, in the reversed ordering. # The reverse ordering is needed when an Alloc add an implicit new # broadcasted dimensions to its inputs[0]. Eg: # Alloc(Alloc(m, y, 1, 1), x, y, z, w) -> Alloc(m, x, y, z, w) i = 0 for dim_inner, dim_outer in zip(dims_inner_rev, dims_outer_rev): if dim_inner != dim_outer: if isinstance(dim_inner, Constant) and dim_inner.data == 1: pass else: dims_outer[-1 - i] = Assert( "You have a shape error in your graph. To see a better" " error message and a stack trace of where in your code" " the error is created, use the Theano flags" " optimizer=None or optimizer=fast_compile.")( dim_outer, T.eq(dim_outer, dim_inner)) i += 1 return [T.alloc(inputs_inner[0], *dims_outer)] @register_useless('fast_compile') @gof.local_optimizer([TopKOp]) def local_useless_topk(node): """ TopKOp generates two outputs by default This opt removes the useless ones """ op = node.op if not isinstance(op, TopKOp): return if not (op.return_values and op.return_indices): return False x, k = node.inputs ret_val = bool(node.outputs[0].clients) ret_idx = bool(node.outputs[1].clients) if not (ret_val ^ ret_idx): # both true -> nothing to remove # both false -> let pruner handle return False old_output = node.outputs[ret_idx] new_output = TopKOp( axis=op.axis, idx_dtype=op.idx_dtype, return_values=ret_val, return_indices=ret_idx)(x, k) return {old_output: new_output} Theano-1.0.0/theano/tensor/io.py00006000163562€ÿÿÿþ0000001664213203335532020051 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import numpy as np from theano import gof from theano.gof import Constant, Generic, Op from theano.gof.sched import key_to_cmp from theano.tensor import tensor import theano ########################## # Disk Access ########################## class LoadFromDisk(Op): """ An operation to load an array from disk. See Also -------- load Notes ----- Non-differentiable. """ __props__ = ("dtype", "broadcastable", "mmap_mode") def __init__(self, dtype, broadcastable, mmap_mode=None): self.dtype = np.dtype(dtype) # turn "float64" into np.float64 self.broadcastable = broadcastable if mmap_mode not in (None, 'c'): raise ValueError("The only supported values for mmap_mode " "are None and 'c', got %s" % mmap_mode) self.mmap_mode = mmap_mode def make_node(self, path): if isinstance(path, str): path = Constant(Generic(), path) return gof.Apply(self, [path], [tensor(self.dtype, broadcastable=self.broadcastable)]) def perform(self, node, inp, out): path = inp[0] if (path.split('.')[-1] == 'npz'): raise ValueError("Expected a .npy file, got %s instead" % path) result = np.load(path, mmap_mode=self.mmap_mode) if result.dtype != self.dtype: raise TypeError("Expected an array of type %s, got %s instead" % (self.dtype, result.dtype)) out[0][0] = result def __str__(self): return ("Load{dtype: %s, broadcastable: %s, mmep: %s}" % (self.dtype, self.broadcastable, self.mmap_mode)) def load(path, dtype, broadcastable, mmap_mode=None): """ Load an array from an .npy file. Parameters ---------- path A Generic symbolic variable, that will contain a string dtype : data-type The data type of the array to be read. broadcastable The broadcastable pattern of the loaded array, for instance, (False,) for a vector, (False, True) for a column, (False, False) for a matrix. mmap_mode How the file will be loaded. None means that the data will be copied into an array in memory, 'c' means that the file will be mapped into virtual memory, so only the parts that are needed will be actually read from disk and put into memory. Other modes supported by numpy.load ('r', 'r+', 'w+') cannot be supported by Theano. Examples -------- >>> from theano import * >>> path = Variable(Generic()) >>> x = tensor.load(path, 'int64', (False,)) >>> y = x*2 >>> fn = function([path], y) >>> fn("stored-array.npy") # doctest: +SKIP array([0, 2, 4, 6, 8], dtype=int64) """ return LoadFromDisk(dtype, broadcastable, mmap_mode)(path) ########################## # MPI ########################## try: from mpi4py import MPI except ImportError: mpi_enabled = False else: comm = MPI.COMM_WORLD mpi_enabled = True class MPIRecv(Op): """ An operation to asynchronously receive an array to a remote host using MPI. See Also -------- MPIRecv MPIWait Notes ----- Non-differentiable. """ __props__ = ("source", "tag", "shape", "dtype") def __init__(self, source, tag, shape, dtype): self.source = source self.tag = tag self.shape = shape self.dtype = np.dtype(dtype) # turn "float64" into numpy.float64 self.broadcastable = (False,) * len(shape) def make_node(self): return gof.Apply(self, [], [theano.Variable(Generic()), tensor(self.dtype, broadcastable=self.broadcastable)]) def perform(self, node, inp, out): data = np.zeros(self.shape, dtype=self.dtype) request = comm.Irecv(data, self.source, self.tag) out[0][0] = request out[1][0] = data def __str__(self): return ("MPIRecv{source: %d, tag: %d, shape: %s, dtype: %s}" % (self.source, self.tag, self.shape, self.dtype)) def infer_shape(self, node, shapes): return [None, self.shape] def do_constant_folding(self, node): return False class MPIRecvWait(Op): """ An operation to wait on a previously received array using MPI. See Also -------- MPIRecv Notes ----- Non-differentiable. """ __props__ = ("tag",) def __init__(self, tag): self.tag = tag def make_node(self, request, data): return gof.Apply(self, [request, data], [tensor(data.dtype, broadcastable=data.broadcastable)]) def perform(self, node, inp, out): request = inp[0] data = inp[1] request.wait() out[0][0] = data def infer_shape(self, node, shapes): return [shapes[1]] view_map = {0: [1]} class MPISend(Op): """ An operation to asynchronously Send an array to a remote host using MPI. See Also -------- MPIRecv MPISendWait Notes ----- Non-differentiable. """ __props__ = ("dest", "tag") def __init__(self, dest, tag): self.dest = dest self.tag = tag def make_node(self, data): return gof.Apply(self, [data], [theano.Variable(Generic()), data.type()]) view_map = {1: [0]} def perform(self, node, inp, out): data = inp[0] request = comm.Isend(data, self.dest, self.tag) out[0][0] = request out[1][0] = data def __str__(self): return "MPISend{dest: %d, tag: %d}" % (self.dest, self.tag) class MPISendWait(Op): """ An operation to wait on a previously sent array using MPI. See Also -------- MPISend Notes ----- Non-differentiable. """ __props__ = ("tag",) def __init__(self, tag): self.tag = tag def make_node(self, request, data): return gof.Apply(self, [request, data], [theano.Variable(Generic())]) def perform(self, node, inp, out): request = inp[0] request.wait() out[0][0] = True def isend(var, dest, tag): """ Non blocking send. """ return MPISend(dest, tag)(var) def send(var, dest, tag): """ Blocking send. """ return MPISendWait(tag)(*isend(var, dest, tag)) def irecv(shape, dtype, source, tag): """ Non-blocking receive. """ return MPIRecv(source, tag, shape, dtype)() def recv(shape, dtype, source, tag): """ Blocking receive. """ return MPIRecvWait(tag)(*irecv(shape, dtype, source, tag)) # Ordering keys for scheduling def mpi_send_wait_key(a): """Wait as long as possible on Waits, Start Send/Recvs early.""" if isinstance(a.op, (MPIRecvWait, MPISendWait)): return 1 if isinstance(a.op, (MPIRecv, MPISend)): return -1 return 0 def mpi_tag_key(a): """Break MPI ties by using the variable tag - prefer lower tags first.""" if isinstance(a.op, (MPISend, MPIRecv, MPIRecvWait, MPISendWait)): return a.op.tag else: return 0 mpi_send_wait_cmp = key_to_cmp(mpi_send_wait_key) mpi_tag_cmp = key_to_cmp(mpi_tag_key) mpi_keys = (mpi_send_wait_key, mpi_tag_key) mpi_cmps = (mpi_send_wait_cmp, mpi_tag_cmp) Theano-1.0.0/theano/tensor/sort.py00006000163562€ÿÿÿþ0000004252013203335532020423 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import numpy as np import theano from theano.tensor.basic import mul, arange from theano.gradient import grad_undefined from theano.tensor.subtensor import set_subtensor def _variable_is_none(var): return isinstance(var, theano.Constant) and var.data is None def _check_tensor_is_scalar(var): ''' Checks if a tensor variable is scalar, raise ValueError otherwise ''' msg = '%(var)s is expected to be 0d tensor, got %(ndim)d' if var.ndim != 0: raise ValueError( msg % (var, var.ndim)) class SortOp(theano.Op): """ This class is a wrapper for numpy sort function. """ __props__ = ("kind", "order") def __init__(self, kind, order=None): self.kind = kind self.order = order def __str__(self): return self.__class__.__name__ + "{%s, %s}" % (self.kind, str(self.order)) def make_node(self, input, axis=-1): input = theano.tensor.as_tensor_variable(input) axis = theano.tensor.as_tensor_variable(axis) out_type = input.type() return theano.Apply(self, [input, axis], [out_type]) def perform(self, node, inputs, output_storage): a = inputs[0] axis = inputs[1] z = output_storage[0] z[0] = np.sort(a, axis, self.kind, self.order) def infer_shape(self, node, inputs_shapes): if _variable_is_none(node.inputs[1]): # That means axis = None, # So the array is flattened before being sorted return [(mul(*inputs_shapes[0]),)] # axis should not be None # So there should be the same number of dimensions # in the input and output assert node.inputs[0].ndim == node.outputs[0].ndim assert inputs_shapes[1] == () return [inputs_shapes[0]] def grad(self, inputs, output_grads): a, axis = inputs indices = self.__get_argsort_indices(a, axis) inp_grad = output_grads[0][tuple(indices)] axis_grad = grad_undefined( self, 1, axis, "The gradient of sort is not defined " "with respect to the integer axes itself") return [inp_grad, axis_grad] def __get_expanded_dim(self, a, axis, i): index_shape = [1] * a.ndim index_shape[i] = a.shape[i] # it's a way to emulate # numpy.ogrid[0: a.shape[0], 0: a.shape[1], 0: a.shape[2]] index_val = arange(a.shape[i]).reshape(index_shape) return index_val def __get_argsort_indices(self, a, axis): """ Calculates indices which can be used to reverse sorting operation of "a" tensor along "axis". Returns ------- 1d array if axis is None list of lenght len(a.shape) otherwise """ # The goal is to get gradient wrt input from gradient # wrt sort(input, axis) idx = argsort(a, axis, kind=self.kind, order=self.order) # rev_idx is the reverse of previous argsort operation rev_idx = argsort(idx, axis, kind=self.kind, order=self.order) indices = [] axis_data = theano.tensor.switch(theano.tensor.ge(axis.data, 0), axis.data, a.ndim + axis.data) for i in range(a.ndim): index_val = theano.tensor.switch(theano.tensor.eq(i, axis_data), rev_idx, self.__get_expanded_dim(a, axis, i)) indices.append(index_val) return indices """ def R_op(self, inputs, eval_points): # R_op can receive None as eval_points. # That mean there is no diferientiable path through that input # If this imply that you cannot compute some outputs, # return None for those. if eval_points[0] is None: return eval_points return self.grad(inputs, eval_points) """ def sort(a, axis=-1, kind='quicksort', order=None): """ Parameters ---------- a : Tensor Tensor to be sorted axis : Tensor Axis along which to sort. If None, the array is flattened before sorting. kind : {'quicksort', 'mergesort', 'heapsort'}, optional Sorting algorithm. Default is 'quicksort'. order : list, optional When `a` is a structured array, this argument specifies which fields to compare first, second, and so on. This list does not need to include all of the fields. Returns ------- array A sorted copy of an array. """ if axis is None: a = a.flatten() axis = 0 return SortOp(kind, order)(a, axis) class ArgSortOp(theano.Op): """ This class is a wrapper for numpy argsort function. """ __props__ = ("kind", "order") def __init__(self, kind, order=None): self.kind = kind self.order = order def __str__(self): return (self.__class__.__name__ + "{%s, %s}" % (self.kind, str(self.order))) def make_node(self, input, axis=-1): input = theano.tensor.as_tensor_variable(input) axis = theano.tensor.as_tensor_variable(axis) bcast = input.type.broadcastable return theano.Apply(self, [input, axis], [theano.tensor.TensorType( dtype="int64", broadcastable=bcast)()]) def perform(self, node, inputs, output_storage): a = inputs[0] axis = inputs[1] z = output_storage[0] z[0] = theano._asarray(np.argsort(a, axis, self.kind, self.order), dtype=node.outputs[0].dtype) def infer_shape(self, node, inputs_shapes): if _variable_is_none(node.inputs[1]): return [(mul(*inputs_shapes[0]),)] # axis should not be None, so there should be the same number of # dimensions in the input and output assert node.inputs[0].ndim == node.outputs[0].ndim assert inputs_shapes[1] == () return [inputs_shapes[0]] def grad(self, inputs, output_grads): # No grad defined for intergers. inp, axis = inputs inp_grad = inp.zeros_like() axis_grad = grad_undefined( self, 1, axis, "argsort is not defined for non-integer axes so" " argsort(x, axis+eps) is undefined") return [inp_grad, axis_grad] """ def R_op(self, inputs, eval_points): # R_op can receive None as eval_points. # That mean there is no diferientiable path through that input # If this imply that you cannot compute some outputs, # return None for those. if eval_points[0] is None: return eval_points return self.grad(inputs, eval_points) """ def argsort(a, axis=-1, kind='quicksort', order=None): """ Returns the indices that would sort an array. Perform an indirect sort along the given axis using the algorithm specified by the kind keyword. It returns an array of indices of the same shape as a that index data along the given axis in sorted order. """ if axis is None: a = a.flatten() axis = 0 return ArgSortOp(kind, order)(a, axis) def _topk_py_impl(op, x, k, axis, idx_dtype): ndim = x.ndim assert -ndim <= axis < ndim axis %= ndim if k == 0: raise ValueError('topk: kth cannot be zero') elif k > x.shape[axis]: raise ValueError( 'topk: kth cannot be larger than the size of specified axis %d' % axis) if abs(k) == 1: # negative k means min instead of max fn_max = [None, np.max, np.min][k] fn_argmax = [None, np.argmax, np.argmin][k] if not op.return_indices: return np.expand_dims(fn_max(x, axis=axis), axis) elif op.return_values: zi = np.expand_dims( fn_argmax(x, axis=axis), axis) idx2 = tuple( np.arange(s).reshape( (s,) + (1,) * (ndim - i - 1) ) if i != axis else zi for i, s in enumerate(x.shape)) zv = x[idx2] return zv, zi.astype(idx_dtype) else: zi = np.expand_dims( fn_argmax(x, axis=axis), axis) return zi.astype(idx_dtype) if x.shape[axis] == abs(k): if not op.return_indices: return x.copy() else: l = axis r = ndim - l reps = list(x.shape) reps[axis] = 1 zi = np.arange(abs(k), dtype=idx_dtype) zi = zi.reshape((1,) * l + (k,) + (1,) * (r - 1)) zi = np.tile(zi, reps) if op.return_values: return x.copy(), zi else: return zi idx = [slice(None)] * ndim idx[axis] = (slice(-k, None) if k > 0 else slice(-k)) if not op.return_indices: zv = np.partition(x, -k, axis=axis)[idx] return zv elif op.return_values: zi = np.argpartition(x, -k, axis=axis)[idx] idx2 = tuple( np.arange(s).reshape( (s,) + (1,) * (ndim - i - 1) ) if i != axis else zi for i, s in enumerate(x.shape)) zv = x[idx2] return zv, zi.astype(idx_dtype) else: zi = np.argpartition(x, -k, axis=axis)[idx] return zi.astype(idx_dtype) class TopKOp(theano.Op): """ Operations related to finding k-largest elements. Parameters ---------- axis: integer Defaults to ``-1``. The axis to perform the operation. Must be in range ``[-ndim, ndim)``, where ``ndim`` is the dimensionality of input tensor. idx_dtype: string Specify output dtype for indices, defaults to ``int64``, must be integer type. sorted: bool NOTE: NOT IMPLEMENTED YET Defaults to ``True`` If True, the result array would be sorted in descending order. Notes ----- - By default, this Op give two outputs: values and indices. However optimizer may remove a certain output if not needed. - Computing gradient is only possible when both values and indices are computed in forward pass. - If the top-k-th value is not unique, we cannot guarantee the output indices being deterministically chosen. See Also -------- topk argtopk argtopk_and_topk """ # TODO more params ''' only_top_kth: bool Defaults to ``False`` If ``True``, will only find one exact top k-th element on given axis. ''' # TODO c_code # TODO add opt, if k==1, use max/min reduce # also if k is axis size, just copy input tensor # TODO add opt, to merge argtopk / topk __props__ = ('axis', 'sorted', 'return_values', 'return_indices', 'idx_dtype') def __init__( self, axis=-1, sorted=True, idx_dtype='int64', return_values=True, return_indices=True ): # numpy always uses int64 as output dtype for arg*() routines # however, we add "idx_dtype" param as memory is more precious on gpu if not isinstance(axis, int): raise TypeError( '"axis" parameter must be integer, got "%s"' % type(axis)) if idx_dtype not in theano.tensor.integer_dtypes: raise TypeError( '"idx_dtype" parameter must be an integer dtype, got "%s"' % idx_dtype) if not (return_indices or return_values): raise ValueError( "Neither return_values nor return_indices is True, this isn't allowed") self.axis = axis self.sorted = sorted self.return_values = return_values self.return_indices = return_indices self.idx_dtype = idx_dtype def __str__(self): return '%(op)s{axis=%(axis)d, sorted=%(sorted)s}' % dict( op=self.__class__.__name__, axis=self.axis, sorted=self.sorted) def make_node(self, inp, kth): inp = theano.tensor.as_tensor_variable(inp) ndim = inp.ndim if ndim == 0: raise ValueError('Cannot take scalar as input') if not -ndim <= self.axis < ndim: raise IndexError( '"axis" parameter out of range,' ' expected integer within [%d, %d]' % (-ndim, ndim - 1)) kth = theano.tensor.as_tensor_variable(kth) _check_tensor_is_scalar(kth) bcast = inp.type.broadcastable outs = [] if self.return_values: outs.append(inp.type()) if self.return_indices: outs.append(theano.tensor.TensorType( dtype=self.idx_dtype, broadcastable=bcast)()) return theano.Apply(self, [inp, kth], outs) def perform(self, node, inputs, output_storage): x, k = inputs axis = self.axis if not self.return_indices: pzv = output_storage[0] pzv[0] = _topk_py_impl(self, x, k, axis, None) elif self.return_values: pzv = output_storage[0] pzi = output_storage[1] pzv[0], pzi[0] = _topk_py_impl( self, x, k, axis, node.outputs[1].dtype) else: pzi = output_storage[0] pzi[0] = _topk_py_impl(self, x, k, axis, node.outputs[0].dtype) def infer_shape(self, node, inp_shapes): shp = list(inp_shapes[0]) shp[self.axis] = np.abs(node.inputs[1]) shp = tuple(shp) return [shp for i in [self.return_values, self.return_indices] if i] def L_op(self, inputs, outputs, out_grads): x, k = inputs k_grad = grad_undefined(self, 1, k, 'topk: k is not differentiable') if not (self.return_indices or self.return_values): x_grad = grad_undefined( self, 0, x, 'topk: cannot get gradient' ' without both indices and values') else: x_shp = theano.tensor.shape(x) z_grad = out_grads[0] ndim = x.ndim axis = self.axis % ndim grad_indices = [ arange(x_shp[i]).dimshuffle([0] + ['x'] * (ndim - i - 1)) if i != axis else outputs[-1] for i in range(ndim)] x_grad = x.zeros_like(dtype=z_grad.dtype) x_grad = set_subtensor(x_grad[tuple(grad_indices)], z_grad) return [x_grad, k_grad] def topk(x, kth, axis=-1, sorted=True, idx_dtype='int64'): """ Returns the k-largest elements along an axis. Parameters ---------- x: tensor instance kth: integer constant/variable Must not be 0. If negative, gives k-smallest elements instead. axis: integer or ``None`` Upon which axis shall the operation be performed on. If ``None``, works on flattened array. sorted: bool NOTE: NOT IMPLEMENTED YET, USE ``False`` FOR NOW. Defaults to ``True`` If True, the result array would be sorted in descending order. idx_dtype: string Specify output dtype used in indices, defaults to ``int64``, must be integer type. This option is here because indices are needed for gradient. Returns ------- Tensor variable with same dtype as `x`. Notes ----- - ``sorted=True`` is not supported yet. """ if sorted: raise NotImplementedError( "We are still working on sorted topk. Use sorted=False for now.") if axis is None: x = theano.tensor.flatten(x) axis = 0 return TopKOp( axis=axis, sorted=sorted, idx_dtype=idx_dtype)(x, kth)[0] def argtopk(x, kth, axis=-1, sorted=True, idx_dtype='int64'): """ Returns the indices of k-largest elements along an axis. Parameters ---------- x: tensor instance kth: integer constant/variable Must not be 0. If negative, gives k-smallest elements instead. sorted: bool NOTE: NOT IMPLEMENTED YET, USE ``False`` FOR NOW. Defaults to ``True`` If True, the result array of corresponding indices would be sorted in descending order. axis: integer, tuple/list of integers, or ``None`` Upon which axis shall the operation be performed on. If ``None``, works on flattened array. idx_dtype: string Specify output dtype, defaults to ``int64``, must be integer type. Returns ------- Tensor variable with dtype specified in `idx_dtype`. Notes ----- - ``sorted=True`` is not supported yet. - If the top-k-th value is not unique, we cannot guarantee the output indices are deterministically chosen. """ if sorted: raise NotImplementedError( "We are still working on sorted topk. Use sorted=False for now.") if axis is None: x = theano.tensor.flatten(x) axis = 0 return TopKOp( axis=axis, sorted=sorted, idx_dtype=idx_dtype)(x, kth)[1] def topk_and_argtopk(x, kth, axis=-1, sorted=True, idx_dtype='int64'): """ Returns the results of both topk() and argtopk() in one Op. See the respective documentation for details. Returns ------- tuple: (values, indices) """ if sorted: raise NotImplementedError( "We are still working on sorted topk. Use sorted=False for now.") if axis is None: x = theano.tensor.flatten(x) axis = 0 return TopKOp( axis=axis, sorted=sorted, idx_dtype=idx_dtype)(x, kth) Theano-1.0.0/theano/tensor/blas.py00006000163562€ÿÿÿþ0000026522013203335532020361 0ustar boccoset00000000000000"""Ops and optimizations for using BLAS calls BLAS = Basic Linear Algebra Subroutines Learn more about BLAS here: http://www.netlib.org/blas/blast-forum/ The standard BLAS libraries implement what is called "legacy BLAS" in that document. This documentation describes Theano's BLAS optimization pipeline. Where there is a discrepancy between how things do work and how they *should* work, both aspects should be documented. There are four kinds of BLAS Ops in Theano: - Python implementations (this file) - SciPy-based (blas_scipy) - C-based (blas_c) - GPU-based (theano.gpuarray) Notes ----- Unfortunately (because it's confusing) this file currently contains Ops that contain both Python and C versions. I think it would be better to move the C implementations to blas_c so that this file is pure Python. -JB Ops === GEMM: Dot22, Dot22Scalar, GemmRelated, Gemm ------------------------------------------- The BLAS GEMM operation implements Z <- a X Y + b Z, where Z, X and Y are matrices, and a and b are scalars. Dot22 is a GEMM where a=1, b=0, and Z is allocated every time. Dot22Scalar is a GEMM where b=0 and Z is allocated every time. Gemm is a GEMM in all its generality. In the future we can refactor the GemmRelated, Gemm, Dot22 and Dot22Scalar Ops into a single Op. That new Op (Gemm2) is basically a normal Gemm, but with an additional configuration variable that says to ignore the input Z. Setting that configuration variable to True would make Gemm2 equivalent to the current Dot22 and Dot22Scalar. This would make the file a lot easier to read, and save a few hundred lines of library, to say nothing of testing and documentation. GEMV: Gemv ---------- The BLAS GEMV operation implements Z <- a X Y + b Z, where X is a matrix, Y, and Z are vectors, and a and b are scalars. GER: Ger -------- The BLAS GER operation implements Z <- a X' Y + Z, where X and Y are vectors, and matrix Z gets a rank-1 update. Other Notable BLAS-related Ops ------------------------------ SYRK is another useful special case of GEMM. Particularly SYRK preserves symmetry in the matrix that it updates. See how the linear-algebra module uses symmetry hints before implementing this Op, so that this Op is compatible with that system. Optimizations ============= The optimization pipeline works something like this: 1. identify dot22 from dot 2. identify gemm from dot22 3. identify dot22scalar from dot22 that are not gemm 4. specialize gemm to gemv where applicable 5. specialize gemm to ger where applicable 6. specialize dot22 -> gemv or ger where applicable :note: GEMM is the most canonical BLAS signature that we deal with so far, it would be good to turn most things into GEMM (dot, inner, outer, dot22, dot22scalar), and then to specialize from gemm to the various other L2 and L3 operations. Identify Dot22 -------------- Numpy's dot supports arguments that are of any rank, and we should support that too (just for compatibility). The BLAS optimizations work with Dot Ops whose inputs are each either vector or matrix. So the first part of the optimization pipeline is to transform qualifying Dot Ops to Dot22 Ops. Dot22 Ops may be transformed further, but they will get implemented by a BLAS call. More precisely, Dot nodes whose inputs are all vectors or matrices and whose inputs both have the same dtype, and whose dtype is float or complex, become Dot22. This is implemented in `local_dot_to_dot22`. Identify Gemm from Dot22 ------------------------ This is complicated, done in GemmOptimizer. Identify Dot22Scalar from Dot22 ------------------------------- Dot22 Ops that remain after the GemmOptimizer is done have not qualified as GEMM Ops. Still they might be scaled by a factor, in which case we use Dot22Scalar which is like Gemm, but without the b and the Z. In the future it would be good to merge this into the GemmOptimizer. Specialize Gemm to Gemv ----------------------- If arguments to GEMM are dimshuffled vectors, then we can use GEMV instead. This optimization is `local_gemm_to_gemv`. """ from __future__ import absolute_import, print_function, division import copy import logging import os import time import numpy as np import numpy.distutils try: import numpy.distutils.__config__ # noqa except ImportError: pass from six import iteritems from six.moves import reduce, xrange from theano import config from theano.gof import (utils, Op, view_roots, local_optimizer, Optimizer, InconsistencyError, toolbox, SequenceDB, EquilibriumOptimizer, Apply, ReplacementDidntRemovedError) from theano.gof.params_type import ParamsType from theano.gof.opt import inherit_stack_trace from theano.printing import pprint, FunctionPrinter, debugprint from theano.compile.mode import optdb import theano.scalar from theano.scalar import bool as bool_t from theano.tensor import basic as T from theano.tensor.blas_headers import blas_header_text from theano.tensor.blas_headers import blas_header_version from theano.tensor.opt import in2out, local_dimshuffle_lift from theano.tensor.type import values_eq_approx_remove_inf_nan _logger = logging.getLogger('theano.tensor.blas') try: import scipy.linalg.blas have_fblas = True try: fblas = scipy.linalg.blas.fblas except AttributeError: # A change merged in Scipy development version on 2012-12-02 replaced # `scipy.linalg.blas.fblas` with `scipy.linalg.blas`. # See http://github.com/scipy/scipy/pull/358 fblas = scipy.linalg.blas _blas_gemv_fns = {np.dtype('float32'): fblas.sgemv, np.dtype('float64'): fblas.dgemv, np.dtype('complex64'): fblas.cgemv, np.dtype('complex128'): fblas.zgemv} except ImportError as e: have_fblas = False # This is used in Gemv and ScipyGer. We use CGemv and CGer # when theano.config.blas.ldflags is defined. So we don't need a # warning in that case. if not config.blas.ldflags: _logger.warning('Failed to import scipy.linalg.blas, and ' 'Theano flag blas.ldflags is empty. ' 'Falling back on slower implementations for ' 'dot(matrix, vector), dot(vector, matrix) and ' 'dot(vector, vector) (%s)', str(e)) # If check_init_y() == True we need to initialize y when beta == 0. def check_init_y(): if check_init_y._result is None: if not have_fblas: check_init_y._result = False y = float('NaN') * np.ones((2,)) x = np.ones((2,)) A = np.ones((2, 2)) gemv = _blas_gemv_fns[y.dtype] gemv(1.0, A.T, x, 0.0, y, overwrite_y=True, trans=True) check_init_y._result = np.isnan(y).any() return check_init_y._result check_init_y._result = None class Gemv(Op): """ expression is beta * y + alpha * A x A is matrix x, y are vectors alpha, beta are scalars output is a vector that can be inplace on y """ __props__ = ("inplace",) def __init__(self, inplace): self.inplace = inplace if inplace: self.destroy_map = {0: [0]} def __str__(self): if self.inplace: return '%s{inplace}' % self.__class__.__name__ else: return '%s{no_inplace}' % self.__class__.__name__ def make_node(self, y, alpha, A, x, beta): y = T.as_tensor_variable(y) x = T.as_tensor_variable(x) A = T.as_tensor_variable(A) alpha = T.as_tensor_variable(alpha) beta = T.as_tensor_variable(beta) if y.dtype != A.dtype or y.dtype != x.dtype: raise TypeError('Gemv requires matching dtypes', (y.dtype, A.dtype, x.dtype)) if A.ndim != 2: raise TypeError('gemv requires matrix for A', A.type) if x.ndim != 1: raise TypeError('gemv requires vector for x', x.type) if y.ndim != 1: raise TypeError('gemv requires vector for y', y.type) return Apply(self, [y, alpha, A, x, beta], [y.type()]) def perform(self, node, inputs, out_storage, params=None): y, alpha, A, x, beta = inputs if (have_fblas and y.shape[0] != 0 and x.shape[0] != 0 and y.dtype in _blas_gemv_fns): gemv = _blas_gemv_fns[y.dtype] if (A.shape[0] != y.shape[0] or A.shape[1] != x.shape[0]): raise ValueError( 'Incompatible shapes for gemv ' '(beta * y + alpha * dot(A, x)). y: %s, A: %s, x: %s ' % (y.shape, A.shape, x.shape)) if beta == 0 and check_init_y(): y.fill(0) # Here I suppose that A is in c order. If we don't make it # explicitly as fortran order, scipy 0.7.2 seam to create # a copy in fortran order instead of just reshaping it # and using the trans flag. # If A is already in fortran order, make it in c order and using the # trans flag don't seam to cause slowdown. # out_storage[0][0] = gemv(alpha, A, x, beta, y, # overwrite_y=self.inplace) out_storage[0][0] = gemv(alpha, A.T, x, beta, y, overwrite_y=self.inplace, trans=True) else: out = np.dot(A, x) if alpha != 1: out *= alpha if beta != 0: if beta != 1: out += beta * y else: out += y out_storage[0][0] = np.asarray(out, dtype=y.dtype) def infer_shape(self, node, input_shapes): return [input_shapes[0]] gemv_no_inplace = Gemv(inplace=False) gemv_inplace = Gemv(inplace=True) # For the user interface. Opt will make them inplace later gemv = gemv_no_inplace class Ger(Op): """ BLAS defines general rank-1 update GER as A <- A + alpha x y' for matrix A, scalar alpha, vectors x and y. This interface to GER allows non-destructive operation on A via the `destructive` argument to the constructor. """ __props__ = ("destructive",) def __init__(self, destructive): self.destructive = destructive if destructive: self.destroy_map = {0: [0]} def __str__(self): if self.destructive: return '%s{destructive}' % self.__class__.__name__ else: return '%s{non-destructive}' % self.__class__.__name__ def make_node(self, A, alpha, x, y): A = T.as_tensor_variable(A) y = T.as_tensor_variable(y) x = T.as_tensor_variable(x) alpha = T.as_tensor_variable(alpha) if not(A.dtype == x.dtype == y.dtype == alpha.dtype): raise TypeError('ger requires matching dtypes', (A.dtype, alpha.dtype, x.dtype, y.dtype)) if alpha.ndim != 0: raise TypeError('ger requires scalar alpha', alpha.type) if A.ndim != 2: raise TypeError('ger requires matrix for A', A.type) if x.ndim != 1: raise TypeError('ger requires vector for x', x.type) if y.ndim != 1: raise TypeError('ger requires vector for y', y.type) if x.dtype not in ('float32', 'float64', 'complex64', 'complex128'): raise TypeError('only float and complex types supported', x.dtype) return Apply(self, [A, alpha, x, y], [A.type()]) def perform(self, node, inp, out, params=None): cA, calpha, cx, cy = inp cZ, = out if self.destructive: A = cA else: A = cA.copy() if calpha != 1: A += calpha * np.outer(cx, cy) else: A += np.outer(cx, cy) cZ[0] = A def infer_shape(self, node, input_shapes): return [input_shapes[0]] ger = Ger(destructive=False) ger_destructive = Ger(destructive=True) def ldflags(libs=True, flags=False, libs_dir=False, include_dir=False): """Extract a list of compilation flags from config.blas.ldflags. Depending on the options, different type of flags will be kept. It returns a list of libraries against which an Op's object file should be linked to benefit from a BLAS implementation. Parameters ---------- libs : bool, optional Extract flags starting with "-l" (the default is True). libs_dir : bool, optional Extract flags starting with "-L" (the default is False). include_dir : bool, optional Extract flags starting with "-I" (the default is False). flags: bool, optional Extract all the other flags (the default is False). Returns ------- list of strings Extracted flags. """ ldflags_str = theano.config.blas.ldflags return _ldflags(ldflags_str=ldflags_str, libs=libs, flags=flags, libs_dir=libs_dir, include_dir=include_dir) @utils.memoize def _ldflags(ldflags_str, libs, flags, libs_dir, include_dir): """Extract list of compilation flags from a string. Depending on the options, different type of flags will be kept. Parameters ---------- ldflags_str : string The string to process. Typically, this will be the content of `theano.config.blas.ldflags`. libs : bool Extract flags starting with "-l". flags: bool Extract all the other flags. libs_dir: bool Extract flags starting with "-L". include_dir: bool Extract flags starting with "-I". Returns ------- list of strings Extracted flags. """ rval = [] if libs_dir: found_dyn = False dirs = [x[2:] for x in ldflags_str.split() if x.startswith('-L')] l = _ldflags(ldflags_str=ldflags_str, libs=True, flags=False, libs_dir=False, include_dir=False) for d in dirs: for f in os.listdir(d.strip('"')): if (f.endswith('.so') or f.endswith('.dylib') or f.endswith('.dll')): if any([f.find(ll) >= 0 for ll in l]): found_dyn = True if not found_dyn and dirs: _logger.warning( "We did not find a dynamic library in the " "library_dir of the library we use for blas. If you use " "ATLAS, make sure to compile it with dynamics library.") for t in ldflags_str.split(): # Remove extra quote. if (t.startswith("'") and t.endswith("'")) or (t.startswith('"') and t.endswith('"')): t = t[1:-1] try: t0, t1, t2 = t[0:3] assert t0 == '-' except Exception: raise ValueError('invalid token "%s" in ldflags_str: "%s"' % (t, ldflags_str)) if libs_dir and t1 == 'L': rval.append(t[2:]) elif include_dir and t1 == 'I': raise ValueError('Include dirs are not used for blas. We disable' ' this as this can hide other headers and this' ' is not wanted.', t) rval.append(t[2:]) elif libs and t1 == 'l': # example -lmkl rval.append(t[2:]) elif flags and t1 not in ['L', 'I', 'l']: # example -openmp rval.append(t) elif flags and t1 == 'L': # to find it when we load the compiled op if the env of the # used is not well configured. rval.append('-Wl,-rpath,' + t[2:]) return rval class GemmRelated(Op): """Base class for Gemm and Dot22. This class provides a kind of templated gemm Op. """ __props__ = () def c_support_code(self): # return cblas_header_text() mod_str = """ #ifndef MOD #define MOD % #endif static double time_time() // a time function like time.time() { struct timeval tv; gettimeofday(&tv, 0); return (double) tv.tv_sec + (double) tv.tv_usec / 1000000.0; } """ return blas_header_text() + mod_str def c_headers(self): # std.cout doesn't require the '%' symbol to print stuff... # so it works much better with python's string-substitution stuff. return ['', '', ''] def c_libraries(self): return ldflags() # code_cache_version is built by subclasses from # build_gemm_version def c_compile_args(self): return ldflags(libs=False, flags=True) def c_lib_dirs(self): return ldflags(libs=False, libs_dir=True) def c_header_dirs(self): return ldflags(libs=False, include_dir=True) declare_NS = """ int unit = 0; int type_num = PyArray_DESCR(%(_x)s)->type_num; int type_size = PyArray_DESCR(%(_x)s)->elsize; // in bytes npy_intp* Nx = PyArray_DIMS(%(_x)s); npy_intp* Ny = PyArray_DIMS(%(_y)s); npy_intp* Nz = 0; //PyArray_DIMS(%(_zout)s); npy_intp* Sx = PyArray_STRIDES(%(_x)s); npy_intp* Sy = PyArray_STRIDES(%(_y)s); npy_intp* Sz = 0; //PyArray_STRIDES(%(_zout)s); //strides for x, y, z in dimensions 0, 1 int sx_0, sx_1, sy_0, sy_1, sz_0, sz_1; """ # implement if you don't have an inplace props # setup_z_Nz_Sz = None # otherwise implement # setup_z_Nz_Sz_inplace = None # setup_z_Nz_Sz_outplace = None check_xyz_rank2 = """ if (PyArray_NDIM(%(_x)s) != 2) { PyErr_Format(PyExc_NotImplementedError, "rank(x) != 2. rank(x) is %%d.", PyArray_NDIM(%(_x)s)); %(fail)s; } if (PyArray_NDIM(%(_y)s) != 2) { PyErr_Format(PyExc_NotImplementedError, "rank(y) != 2. rank(y) is %%d.", PyArray_NDIM(%(_y)s)); %(fail)s; } if (%(_zout)s && PyArray_NDIM(%(_zout)s) != 2) { PyErr_Format(PyExc_NotImplementedError, "rank(z) != 2. rank(z) is %%d.", PyArray_NDIM(%(_zout)s)); %(fail)s; } """ check_xyz_double_or_float = """ if ((PyArray_DESCR(%(_x)s)->type_num != NPY_DOUBLE) && (PyArray_DESCR(%(_x)s)->type_num != NPY_FLOAT)) {PyErr_SetString(PyExc_NotImplementedError, "type(x) is not double or float"); %(fail)s;} if ((PyArray_DESCR(%(_y)s)->type_num != NPY_DOUBLE) && (PyArray_DESCR(%(_y)s)->type_num != NPY_FLOAT)) {PyErr_SetString(PyExc_NotImplementedError, "type(y) is not double or float"); %(fail)s;} if ((PyArray_DESCR(%(_zout)s)->type_num != NPY_DOUBLE) && (PyArray_DESCR(%(_zout)s)->type_num != NPY_FLOAT)) {PyErr_SetString(PyExc_NotImplementedError, "type(z) is not double or float"); %(fail)s;} if ((PyArray_DESCR(%(_x)s)->type_num != PyArray_DESCR(%(_y)s)->type_num) ||(PyArray_DESCR(%(_x)s)->type_num != PyArray_DESCR(%(_zout)s)->type_num)) { PyErr_SetString(PyExc_NotImplementedError, "type(x), type(y), type(z) are not all the same"); %(fail)s; } """ # it is not necessary that a or b have the same type as x,y,z check_ab_double_or_float = """ if ((PyArray_DESCR(%(_a)s)->type_num != NPY_DOUBLE) && (PyArray_DESCR(%(_a)s)->type_num != NPY_FLOAT)) {PyErr_SetString(PyExc_NotImplementedError, "type(a) is not double or float"); %(fail)s;} if ((PyArray_DESCR(%(_b)s)->type_num != NPY_DOUBLE) && (PyArray_DESCR(%(_b)s)->type_num != NPY_FLOAT)) {PyErr_SetString(PyExc_NotImplementedError, "type(b) is not double or float"); %(fail)s;} """ check_dims = """ if (Nx[0] != Nz[0]) { PyErr_Format(PyExc_ValueError, "Shape mismatch: x has %%ld rows but z has %%ld rows", (long int)Nx[0], (long int)Nz[0]); %(fail)s; } if (Nx[1] != Ny[0]) { PyErr_Format(PyExc_ValueError, "Shape mismatch: x has %%ld cols (and %%ld rows) but y has %%ld rows (and %%ld cols)", (long int)Nx[1], (long int)Nx[0], (long int)Ny[0], (long int)Ny[1]); %(fail)s; } if (Ny[1] != Nz[1]) { PyErr_Format(PyExc_ValueError, "Shape mismatch: y has %%ld cols but z has %%ld cols", (long int)Ny[1], (long int)Nz[1]); %(fail)s; } // We must not raise an error when Nx[1] == 0. This would disable cases // that numpy.dot accept. """ check_strides = """ /* If some matrices are not contiguous on either dimensions, or have invalid strides, copy their content into a contiguous one */ if ((Sx[0] < 1) || (Sx[1] < 1) || (Sx[0] MOD type_size) || (Sx[1] MOD type_size) || ((Sx[0] != type_size) && (Sx[1] != type_size))) { PyArrayObject * _x_copy = (PyArrayObject *) PyArray_Copy(%(_x)s); if (!_x_copy) %(fail)s Py_XDECREF(%(_x)s); %(_x)s = _x_copy; Sx = PyArray_STRIDES(%(_x)s); } if ((Sy[0] < 1) || (Sy[1] < 1) || (Sy[0] MOD type_size) || (Sy[1] MOD type_size) || ((Sy[0] != type_size) && (Sy[1] != type_size))) { PyArrayObject * _y_copy = (PyArrayObject *) PyArray_Copy(%(_y)s); if (!_y_copy) %(fail)s Py_XDECREF(%(_y)s); %(_y)s = _y_copy; Sy = PyArray_STRIDES(%(_y)s); } if ((Sz[0] < 1) || (Sz[1] < 1) || (Sz[0] MOD type_size) || (Sz[1] MOD type_size) || ((Sz[0] != type_size) && (Sz[1] != type_size))) { PyArrayObject * _z_copy = (PyArrayObject *) PyArray_Copy(%(_zout)s); if (!_z_copy) %(fail)s Py_XDECREF(%(_zout)s); %(_zout)s = _z_copy; Sz = PyArray_STRIDES(%(_zout)s); } """ encode_strides_in_unit = """ /* encode the stride structure of _x,_y,_zout into a single integer */ unit |= ((Sx[1] == type_size || Nx[1]==1) ? 0x0 : (Sx[0] == type_size || Nx[0]==1) ? 0x1 : 0x2) << 8; unit |= ((Sy[1] == type_size || Ny[1]==1) ? 0x0 : (Sy[0] == type_size || Ny[0]==1) ? 0x1 : 0x2) << 4; unit |= ((Sz[1] == type_size || Nz[1]==1) ? 0x0 : (Sz[0] == type_size || Nz[0]==1) ? 0x1 : 0x2) << 0; """ compute_strides = """ /* create appropriate strides for malformed matrices that are row or column * vectors, or empty matrices. * In that case, the value of the stride does not really matter, but * some versions of BLAS insist that: * - they are not smaller than the number of elements in the array, * - they are not 0. */ sx_0 = (Nx[0] > 1) ? Sx[0]/type_size : (Nx[1] + 1); sx_1 = (Nx[1] > 1) ? Sx[1]/type_size : (Nx[0] + 1); sy_0 = (Ny[0] > 1) ? Sy[0]/type_size : (Ny[1] + 1); sy_1 = (Ny[1] > 1) ? Sy[1]/type_size : (Ny[0] + 1); sz_0 = (Nz[0] > 1) ? Sz[0]/type_size : (Nz[1] + 1); sz_1 = (Nz[1] > 1) ? Sz[1]/type_size : (Nz[0] + 1); """ begin_switch_typenum = """ switch (type_num) { """ case_float = """ case NPY_FLOAT: { """ # case_float_ab_constants = None case_float_gemm = """ float* x = (float*)PyArray_DATA(%(_x)s); float* y = (float*)PyArray_DATA(%(_y)s); float* z = (float*)PyArray_DATA(%(_zout)s); char N = 'N'; char T = 'T'; int Nz0 = Nz[0], Nz1 = Nz[1], Nx1 = Nx[1]; //std::cerr << (unit/256) MOD 16 << (unit / 16) MOD 16 << unit MOD 16<< '\\n'; //double t0 = time_time(); switch(unit) { case 0x000: sgemm_(&N, &N, &Nz1, &Nz0, &Nx1, &a, y, &sy_0, x, &sx_0, &b, z, &sz_0); break; case 0x100: sgemm_(&N, &T, &Nz1, &Nz0, &Nx1, &a, y, &sy_0, x, &sx_1, &b, z, &sz_0); break; case 0x010: sgemm_(&T, &N, &Nz1, &Nz0, &Nx1, &a, y, &sy_1, x, &sx_0, &b, z, &sz_0); break; case 0x110: sgemm_(&T, &T, &Nz1, &Nz0, &Nx1, &a, y, &sy_1, x, &sx_1, &b, z, &sz_0); break; case 0x001: sgemm_(&T, &T, &Nz0, &Nz1, &Nx1, &a, x, &sx_0, y, &sy_0, &b, z, &sz_1); break; case 0x101: sgemm_(&N, &T, &Nz0, &Nz1, &Nx1, &a, x, &sx_1, y, &sy_0, &b, z, &sz_1); break; case 0x011: sgemm_(&T, &N, &Nz0, &Nz1, &Nx1, &a, x, &sx_0, y, &sy_1, &b, z, &sz_1); break; case 0x111: sgemm_(&N, &N, &Nz0, &Nz1, &Nx1, &a, x, &sx_1, y, &sy_1, &b, z, &sz_1); break; default: PyErr_SetString(PyExc_ValueError, "some matrix has no unit stride"); %(fail)s; }; //fprintf(stderr, "Calling sgemm %%i %%i %%i %%i took %%f\\n", unit, Nz1, Nz0, Nx1, time_time() - t0); """ case_double = """ } break; case NPY_DOUBLE: { """ # case_double_ab_constants = None case_double_gemm = """ double* x = (double*)PyArray_DATA(%(_x)s); double* y = (double*)PyArray_DATA(%(_y)s); double* z = (double*)PyArray_DATA(%(_zout)s); char N = 'N'; char T = 'T'; int Nz0 = Nz[0], Nz1 = Nz[1], Nx1 = Nx[1]; //std::cerr << (unit/256) MOD 16 << (unit / 16) MOD 16 << unit MOD 16<< '\\n'; //double t0 = time_time(); //fprintf(stderr, "unit=%%x N= %%i %%i %%i S = %%i %%i %%i %%i %%i %%i\\n", unit, //Nz1, Nz0, Nx1, //sy_0, sy_1, //sx_0, sx_1, //sz_0, sz_1 //); switch(unit) { case 0x000: dgemm_(&N, &N, &Nz1, &Nz0, &Nx1, &a, y, &sy_0, x, &sx_0, &b, z, &sz_0); break; case 0x100: dgemm_(&N, &T, &Nz1, &Nz0, &Nx1, &a, y, &sy_0, x, &sx_1, &b, z, &sz_0); break; case 0x010: dgemm_(&T, &N, &Nz1, &Nz0, &Nx1, &a, y, &sy_1, x, &sx_0, &b, z, &sz_0); break; case 0x110: dgemm_(&T, &T, &Nz1, &Nz0, &Nx1, &a, y, &sy_1, x, &sx_1, &b, z, &sz_0); break; case 0x001: dgemm_(&T, &T, &Nz0, &Nz1, &Nx1, &a, x, &sx_0, y, &sy_0, &b, z, &sz_1); break; case 0x101: dgemm_(&N, &T, &Nz0, &Nz1, &Nx1, &a, x, &sx_1, y, &sy_0, &b, z, &sz_1); break; case 0x011: dgemm_(&T, &N, &Nz0, &Nz1, &Nx1, &a, x, &sx_0, y, &sy_1, &b, z, &sz_1); break; case 0x111: dgemm_(&N, &N, &Nz0, &Nz1, &Nx1, &a, x, &sx_1, y, &sy_1, &b, z, &sz_1); break; default: PyErr_SetString(PyExc_ValueError, "some matrix has no unit stride"); %(fail)s; }; //fprintf(stderr, "Calling dgemm %%i %%i %%i %%i took %%f\\n", // unit, Nz1, Nz0, Nx1, time_time()- t0); """ end_switch_typenum = """ } break; } """ def build_gemm_call(self): if hasattr(self, 'inplace'): setup_z_Nz_Sz = "if(%%(params)s->inplace){%s}else{%s}" % ( self.setup_z_Nz_Sz_inplace, self.setup_z_Nz_Sz_outplace) else: setup_z_Nz_Sz = self.setup_z_Nz_Sz return reduce(str.__add__, ( self.declare_NS, self.check_xyz_rank2, setup_z_Nz_Sz, self.check_xyz_double_or_float, self.check_ab_double_or_float, self.check_dims, self.check_strides, self.encode_strides_in_unit, self.compute_strides, self.begin_switch_typenum, self.case_float, self.case_float_ab_constants, self.case_float_gemm, self.case_double, self.case_double_ab_constants, self.case_double_gemm, self.end_switch_typenum), '') def build_gemm_version(self): return (13, blas_header_version()) class Gemm(GemmRelated): """In-place version of matrix-matrix multiplication (with accumulation). When a and b are scalars and x, y, and z are matrices, then gemm(z,a,x,y,b) is similar to b*z + a*dot(x,y) The difference between the two is that the top form is destructive on z, whereas the bottom form is not. Gemm works in-place on the storage associated with z, and the L{Variable} returned by Gemm has a storage that will be aliased to the storage of the z argument. Because of this in-place computation, an L{Apply} of this op will destroy the L{Variable} z on which it operates. (See L{DestructiveOps} for an explanation of what destroying means in the context of theano graphs. See L{BlasLapackSupport} for more optimized linear algebra operations.) """ E_rank = 'gemm only works for rank 2' E_scalar = 'gemm requires scalar argument' E_z_uniq = 'argument z aliased to x or y' # TODO: justify / delete this E_mixed = 'gemm requires matching dtypes' E_float = 'gemm requires floating-point dtypes' __props__ = ('inplace',) params_type = ParamsType(inplace=bool_t,) check_input = False def __init__(self, inplace): self.inplace = inplace if self.inplace: self.destroy_map = {0: [0]} def __str__(self): if self.inplace: inplace_str = 'inplace' else: inplace_str = 'no_inplace' return '%s{%s}' % (self.__class__.__name__, inplace_str) def __setstate__(self, dct): self.__dict__.update(dct) # Correctly reload older pickles where destroy_map were not # saved if 'destroy_map' not in self.__dict__ and self.inplace: self.destroy_map = {0: [0]} def __getstate__(self): rval = self.__dict__.copy() # Do not serialize the setup code, it will be restored in __setstate__ # depending on the value of 'inplace' rval.pop('setup_z_Nz_Sz', None) return rval def make_node(self, *inputs): inputs = list(map(T.as_tensor_variable, inputs)) if len(inputs) != 5: raise TypeError( "Wrong number of inputs for %s (expected 5, got %s)" % (self, len(inputs))) z, a, x, y, b = inputs zr, xr, yr = [set(view_roots(i)) for i in (z, x, y)] # We want the gemm to be inplace. When this op is inplace, it # declare to be inplace only on z. So to make it safe, we # raise an error if z can be a view on x or y. # I don't know if Theano currently can support that case. As # this case don't happen in our code, I won't spent time # investigating this. So the assert is for safety. I also # think there is another mechanism that would prevent this, # but I don't what to modify old code and have chance to break # something. if self.inplace: if zr.intersection(xr): raise InconsistencyError(Gemm.E_z_uniq, (z, x)) if zr.intersection(yr): raise InconsistencyError(Gemm.E_z_uniq, (z, y)) if z.ndim != 2: raise TypeError(Gemm.E_rank, z) if a.ndim != 0: raise TypeError(Gemm.E_scalar, a) if x.ndim != 2: raise TypeError(Gemm.E_rank, x) if y.ndim != 2: raise TypeError(Gemm.E_rank, y) if b.ndim != 0: raise TypeError(Gemm.E_scalar, b) if not (z.dtype == a.dtype == x.dtype == y.dtype == b.dtype): raise TypeError(Gemm.E_mixed, (z.dtype, a.dtype, x.dtype, y.dtype, b.dtype)) if (not z.dtype.startswith('float') and not z.dtype.startswith('complex')): raise TypeError(Gemm.E_float, (z.dtype)) output = z.type() return Apply(self, inputs, [output]) def perform(self, node, inp, out, params): z, a, x, y, b = inp zout, = out assert a.shape == () assert b.shape == () if not params.inplace: z = z.copy() # the original z will not be changed if z.shape == (): z.itemset(z * a + b * np.dot(x, y)) zout[0] = z else: if b == 0.0: if a == 1.0: z[:] = np.dot(x, y) elif a == -1.0: z[:] = -np.dot(x, y) else: z[:] = a * np.dot(x, y) elif b == 1.0: if a == 1.0: z += np.dot(x, y) elif a == -1.0: z -= np.dot(x, y) else: z += a * np.dot(x, y) else: z *= b z += a * np.dot(x, y) zout[0] = z def infer_shape(self, node, input_shapes): return [input_shapes[0]] setup_z_Nz_Sz_inplace = """ if (%(_zout)s != %(_z)s) { if (%(_zout)s) { Py_DECREF(%(_zout)s); } %(_zout)s = %(_z)s; Py_INCREF(%(_zout)s); } Nz = PyArray_DIMS(%(_z)s); Sz = PyArray_STRIDES(%(_z)s); """ setup_z_Nz_Sz_outplace = """ if ((NULL == %(_zout)s) || (PyArray_DIMS(%(_zout)s)[0] != PyArray_DIMS(%(_z)s)[0]) || (PyArray_DIMS(%(_zout)s)[1] != PyArray_DIMS(%(_z)s)[1]) || (PyArray_STRIDES(%(_zout)s)[0] <= 0) || (PyArray_STRIDES(%(_zout)s)[1] <= 0) || (PyArray_STRIDES(%(_zout)s)[0] MOD type_size) || (PyArray_STRIDES(%(_zout)s)[1] MOD type_size) || ((PyArray_STRIDES(%(_zout)s)[0] != type_size) && (PyArray_STRIDES(%(_zout)s)[1] != type_size))) { Py_XDECREF(%(_zout)s); npy_intp dims[2]; dims[0] = PyArray_DIMS(%(_z)s)[0]; dims[1] = PyArray_DIMS(%(_z)s)[1]; %(_zout)s = (PyArrayObject*)PyArray_SimpleNew(2, dims, PyArray_TYPE(%(_z)s)); //fprintf(stderr, "Gemm Allocating %%i %%i\\n", dims[0], dims[1]); if(!%(_zout)s) { PyErr_SetString(PyExc_MemoryError, "failed to alloc gemm_no_inplace output"); %(fail)s } } Nz = PyArray_DIMS(%(_zout)s); Sz = PyArray_STRIDES(%(_zout)s); if (PyArray_DESCR(%(_zout)s)->type_num == NPY_FLOAT) { float * zoutdata = (float*)PyArray_DATA(%(_zout)s); int zoi = Sz[0] / sizeof(float); int zoj = Sz[1] / sizeof(float); const float * zdata = (float*)PyArray_DATA(%(_z)s); int zi = PyArray_STRIDES(%(_z)s)[0]/sizeof(float); int zj = PyArray_STRIDES(%(_z)s)[1]/sizeof(float); for (int i = 0; i < Nz[0]; ++i) { for (int j = 0; j < Nz[1]; ++j) { zoutdata[zoi*i + zoj*j] = zdata[zi*i + zj*j]; } } } else if (PyArray_DESCR(%(_zout)s)->type_num == NPY_DOUBLE) { double * zoutdata = (double*) PyArray_DATA(%(_zout)s); int zoi = Sz[0] / sizeof(double); int zoj = Sz[1] / sizeof(double); const double * zdata = (double*)PyArray_DATA(%(_z)s); int zi = PyArray_STRIDES(%(_z)s)[0]/sizeof(double); int zj = PyArray_STRIDES(%(_z)s)[1]/sizeof(double); for (int i = 0; i < Nz[0]; ++i) { for (int j = 0; j < Nz[1]; ++j) { zoutdata[zoi*i + zoj*j] = zdata[zi*i + zj*j]; } } } else { PyErr_SetString(PyExc_AssertionError, "neither float nor double dtype"); %(fail)s } """ case_float_ab_constants = """ #define REAL float float a = (PyArray_DESCR(%(_a)s)->type_num == NPY_FLOAT) ? (REAL)(((float*)PyArray_DATA(%(_a)s))[0]) : (REAL)(((double*)PyArray_DATA(%(_a)s))[0]); float b = (PyArray_DESCR(%(_b)s)->type_num == NPY_FLOAT) ? (REAL)(((float*)PyArray_DATA(%(_b)s))[0]) : (REAL)(((double*)PyArray_DATA(%(_b)s))[0]); #undef REAL """ case_double_ab_constants = """ #define REAL double double a = (PyArray_DESCR(%(_a)s)->type_num == NPY_FLOAT) ? (REAL)(((float*)PyArray_DATA(%(_a)s))[0]) : (REAL)(((double*)PyArray_DATA(%(_a)s))[0]); double b = (PyArray_DESCR(%(_b)s)->type_num == NPY_FLOAT) ? (REAL)(((float*)PyArray_DATA(%(_b)s))[0]) : (REAL)(((double*)PyArray_DATA(%(_b)s))[0]); #undef REAL """ def c_code(self, node, name, inp, out, sub): _z, _a, _x, _y, _b = inp _zout, = out if node.inputs[0].type.dtype.startswith('complex'): raise utils.MethodNotDefined('%s.c_code' % self.__class__.__name__) full_code = self.build_gemm_call() % dict(locals(), **sub) return full_code def c_code_cache_version(self): gv = self.build_gemm_version() if gv: return (6,) + gv else: return gv gemm_inplace = Gemm(inplace=True) gemm_no_inplace = Gemm(inplace=False) # For the user interface. Theano optimization will make them inplace gemm = gemm_no_inplace pprint.assign(gemm_inplace, FunctionPrinter('gemm_inplace')) pprint.assign(gemm_no_inplace, FunctionPrinter('gemm_no_inplace')) def res_is_a(node, op, maxclients=None): if maxclients is not None: retval = (len(node.clients) <= maxclients) else: retval = True return (node.owner and node.owner.op == op and retval) def _as_scalar(res, dtype=None): """Return None or a TensorVariable whose type is in T.float_scalar_types""" if dtype is None: dtype = config.floatX if np.all(res.type.broadcastable): while res.owner and isinstance(res.owner.op, T.DimShuffle): res = res.owner.inputs[0] # may still have some number of True's if res.type.broadcastable: rval = res.dimshuffle() else: rval = res if rval.type.dtype in theano.tensor.integer_dtypes: # We check that the upcast of res and dtype won't change dtype. # If dtype is float64, we will cast int64 to float64. # This is valid when res is a scalar used as input to a dot22 # as the cast of the scalar can be done before or after the dot22 # and this will give the same result. if theano.scalar.upcast(res.dtype, dtype) == dtype: return T.cast(rval, dtype) else: return None return rval def _is_real_matrix(res): return (res.type.dtype in ('float16', 'float32', 'float64') and res.type.ndim == 2 and res.type.broadcastable[0] is False and res.type.broadcastable[1] is False) # cope with tuple vs. list def _is_real_vector(res): return (res.type.dtype in ('float16', 'float32', 'float64') and res.type.ndim == 1 and res.type.broadcastable[0] is False) def _beta_L_plus_alpha_M(beta, L, alpha, M, recurse_flip=True): # print 'BETA L + ALPHA M', beta, L, alpha, M, recurse_flip # EXPRESSION: (beta * L) + (alpha * M) # we've already checked the client counts, now just make the type check. # if res_is_a(M, _dot22, 1): if M.owner and M.owner.op == _dot22: Ml, Mr = M.owner.inputs rval = [gemm_no_inplace(L, alpha, Ml, Mr, beta)] # print 'GEMM 0', rval, beta, L, alpha, M return rval, M # it also might be the case that there is a dimshuffle between the + # and the dot22. local_dot_to_dot22 in particular will put in such things. if (M.owner and isinstance(M.owner.op, T.DimShuffle) and M.owner.inputs[0].owner and isinstance(M.owner.inputs[0].owner.op, Dot22)): MM = M.owner.inputs[0] if M.owner.op.new_order == (0,): # it is making a column MM into a vector MMl, MMr = MM.owner.inputs g = gemm_no_inplace(L.dimshuffle(0, 'x'), alpha, MMl, MMr, beta) rval = [g.dimshuffle(0)] return rval, MM if M.owner.op.new_order == (1,): # it is making a row MM into a vector MMl, MMr = MM.owner.inputs g = gemm_no_inplace(L.dimshuffle('x', 0), alpha, MMl, MMr, beta) rval = [g.dimshuffle(1)] return rval, MM if len(M.owner.op.new_order) == 0: # it is making a row MM into a vector MMl, MMr = MM.owner.inputs g = gemm_no_inplace(L.dimshuffle('x', 'x'), alpha, MMl, MMr, beta) rval = [g.dimshuffle()] return rval, MM # this is False'd out because of inadequate testing. # TODO see ticket #237 if False and res_is_a(M, gemm_no_inplace, 1): # EXPRESSION: (beta * L) + (alpha * (gemm_no_inplace(G, a, u, v, b))) # EXPRESSION: (beta * L) + alpha * (b * G) + alpha * a * dot(u, v) G, a, u, v, b = M.owner.inputs # print 'GEMM', G, L if res_is_a(G, _dot22, 1): # EXPRESSION: (beta * L) + # (alpha * (gemm_no_inplace(dot(x,y), a, u, v, b))) x, y = G.owner.inputs # EXPRESSION: (beta * L) + (alpha * ((b*dot(x,y) + # (a * dot(u, v))))) # EXPRESSION: (beta * L) + (alpha*b*dot(x,y)) + # (alpha * a * dot(u, v)) rval = [gemm_no_inplace(gemm_no_inplace(L, alpha * b, x, y, beta), alpha * a, u, v, 1.0)] return rval if (G is L): # EXPRESSION: (beta * L) + (alpha*b*L) + (alpha * a * dot(u, v)) rval = [gemm_no_inplace(L, alpha * a, u, v, alpha * b + beta)] return rval if (1.0 != alpha): # at the very least, move the alpha inside the gemm_no_inplace rval = [beta * L + gemm_no_inplace(G, alpha * a, u, v, alpha * b)] return rval if recurse_flip: return _beta_L_plus_alpha_M(alpha, M, beta, L, recurse_flip=False) else: return False, False def _gemm_canonicalize(r, scale, rval, maxclients): # Tries to interpret node as a sum of scalars * (vectors or matrices) def scaled(thing): if scale == 1: return thing if scale == -1 and thing.type.dtype != 'bool': return -thing else: return scale * thing try: r.type.broadcastable except Exception: return None if ((r.type.ndim not in (1, 2)) or r.type.dtype not in ('float16', 'float32', 'float64', 'complex64', 'complex128')): rval.append(scaled(r)) return rval if maxclients and len(getattr(r, 'clients', [])) > maxclients: rval.append((scale, r)) return rval if r.owner and r.owner.op == T.sub: _gemm_canonicalize(r.owner.inputs[0], scale, rval, 1) _gemm_canonicalize(r.owner.inputs[1], -scale, rval, 1) elif r.owner and r.owner.op == T.add: for i in r.owner.inputs: _gemm_canonicalize(i, scale, rval, 1) elif r.owner and r.owner.op == T.neg: _gemm_canonicalize(r.owner.inputs[0], -scale, rval, 1) elif r.owner and r.owner.op == T.mul: scalars = [] vectors = [] matrices = [] for i in r.owner.inputs: if np.all(i.type.broadcastable): while i.owner and isinstance(i.owner.op, T.DimShuffle): i = i.owner.inputs[0] if i.type.broadcastable: scalars.append(i.dimshuffle()) else: scalars.append(i) elif _is_real_vector(i): vectors.append(i) elif _is_real_matrix(i): matrices.append(i) else: # just put the original arguments as in the base case rval.append((scale, r)) return rval if len(matrices) == 1: assert len(vectors) == 0 m = matrices[0] if len(scalars) == 0: _gemm_canonicalize(m, scale, rval, 1) elif len(scalars) == 1: _gemm_canonicalize(m, scaled(scalars[0]), rval, 1) else: _gemm_canonicalize(m, T.mul(scaled(scalars[0]), *scalars[1:]), rval, 1) elif len(vectors) == 1: assert len(matrices) == 0 v = vectors[0] if len(scalars) == 0: _gemm_canonicalize(v, scale, rval, 1) elif len(scalars) == 1: _gemm_canonicalize(v, scaled(scalars[0]), rval, 1) else: _gemm_canonicalize(v, T.mul(scaled(scalars[0]), *scalars[1:]), rval, 1) else: # lets not open this up rval.append((scale, r)) else: rval.append((scale, r)) return rval def _factor_canonicalized(lst): # remove duplicates from canonicalized list # we only delete out of the right end of the list, # once i has touched a list element, it is permantent lst = list(lst) # print 'FACTOR', lst # for t in lst: # if not isinstance(t, (list, tuple)): # t = (t,) # for e in t: # try: # theano.printing.debugprint(e) # except TypeError: # print e, type(e) i = 0 while i < len(lst) - 1: try: s_i, M_i = lst[i] except Exception: i += 1 continue j = i + 1 while j < len(lst): try: s_j, M_j = lst[j] except Exception: j += 1 continue if M_i is M_j: s_i = s_i + s_j lst[i] = (s_i, M_i) del lst[j] else: j += 1 i += 1 return lst def _gemm_from_factored_list(lst): """ Returns None, or a list to replace node.outputs. """ lst2 = [] # Remove the tuple that can't be cast correctly. # This can happen when we try to cast a complex to a real for sM in lst: # Make every pair in list have matching dtypes # sM can be a tuple of 2 elements or a theano variable. if isinstance(sM, tuple): sm0, sm1 = sM sm0 = T.as_tensor_variable(sm0) if theano.scalar.upcast(sm0.dtype, sm1.dtype) == sm1.dtype: lst2.append((T.cast(sm0, sm1.dtype), sM[1])) lst = lst2 def item_to_var(t): try: s, M = t except Exception: return t if s == 1: return M if s == -1: return -M return s * M # Try every pair in the sM_list, trying to turn it into a gemm operation for i in xrange(len(lst) - 1): s_i, M_i = lst[i] for j in xrange(i + 1, len(lst)): s_j, M_j = lst[j] if M_i.type != M_j.type: continue # print 'TRYING', (s_i, M_i, s_j, M_j) gemm_of_sM_list, old_dot22 = _beta_L_plus_alpha_M(s_i, M_i, s_j, M_j) # print 'GOT IT', gemm_of_sM_list if gemm_of_sM_list: assert len(gemm_of_sM_list) == 1 add_inputs = [item_to_var(input) for k, input in enumerate(lst) if k not in (i, j)] add_inputs.extend(gemm_of_sM_list) if len(add_inputs) > 1: rval = [T.add(*add_inputs)] else: rval = add_inputs # print "RETURNING GEMM THIGN", rval return rval, old_dot22 def _gemm_from_node2(node): """ :todo: In many expressions, there are many ways to turn it into a gemm. For example dot(a,b) + c + d. This function should return all of them, so that if one version of gemm causes a cycle in the graph, then another application of gemm can be tried. """ lst = [] t0 = time.time() _gemm_canonicalize(node.outputs[0], 1.0, lst, 0) t1 = time.time() # print "GEMM CANON", lst if len(lst) > 1: lst = _factor_canonicalized(lst) t2 = time.time() rval = _gemm_from_factored_list(lst) t3 = time.time() # It can happen that _factor_canonicalized and # _gemm_from_factored_list return a node with an incorrect # type. This happens in particular when one of the scalar # factors forces the upcast of the whole expression. In that # case, we simply skip that candidate for Gemm. This was # discussed in # http://groups.google.com/group/theano-dev/browse_thread/thread/a3096c82856e3ad5, # but never made it into a trac ticket. if rval and (rval[0][0].type == node.outputs[0].type): return rval, t1 - t0, t2 - t1, t3 - t2 return None, t1 - t0, 0, 0 class GemmOptimizer(Optimizer): """Graph optimizer for inserting Gemm operations.""" def __init__(self): Optimizer.__init__(self) self.warned = False def add_requirements(self, fgraph): fgraph.attach_feature(toolbox.ReplaceValidate()) def apply(self, fgraph): did_something = True nb_iter = 0 nb_replacement = 0 nb_replacement_didn_t_remove = 0 nb_inconsistency_make = 0 nb_inconsistency_replace = 0 time_canonicalize = 0 time_factor_can = 0 time_factor_list = 0 time_toposort = 0 if fgraph.profile: validate_before = fgraph.profile.validate_time callbacks_before = fgraph.execute_callbacks_times.copy() callback_before = fgraph.execute_callbacks_time def on_import(new_node): if new_node is not node: nodelist.append(new_node) u = theano.gof.opt.Updater(on_import, None, None, name="GemmOptimizer") fgraph.attach_feature(u) while did_something: nb_iter += 1 t0 = time.time() nodelist = theano.gof.graph.io_toposort(fgraph.inputs, fgraph.outputs) time_toposort += time.time() - t0 did_something = False nodelist.reverse() for node in nodelist: if not (isinstance(node.op, T.Elemwise) and isinstance(node.op.scalar_op, (theano.scalar.Add, theano.scalar.Sub, theano.scalar.Neg, theano.scalar.Mul))): continue if node not in fgraph.apply_nodes: # This mean that we already removed this node from # the graph continue try: new_outputs, time1, time2, time3 = _gemm_from_node2(node) time_canonicalize += time1 time_factor_can += time2 time_factor_list += time3 except InconsistencyError: nb_inconsistency_make += 1 continue if new_outputs: new_outputs, old_dot22 = new_outputs assert len(new_outputs) == len(node.outputs) new_outputs[0].tag.values_eq_approx = values_eq_approx_remove_inf_nan try: fgraph.replace_all_validate_remove( list(zip(node.outputs, new_outputs)), [old_dot22], reason='GemmOptimizer', # For now we disable the warning as we know case # that we need to fix. warn=False, # warn=not self.warned ) did_something = True nb_replacement += 1 except InconsistencyError: # TODO: retry other applications of gemm (see comment # in _gemm_from_node) nb_inconsistency_replace += 1 except ReplacementDidntRemovedError: nb_replacement_didn_t_remove += 1 self.warned = True fgraph.remove_feature(u) if fgraph.profile: validate_time = fgraph.profile.validate_time - validate_before callback_time = fgraph.execute_callbacks_time - callback_before callbacks_time = {} for k, v in iteritems(fgraph.execute_callbacks_times): if k in callbacks_before: callbacks_time[k] = v - callbacks_before[k] else: callbacks_time[k] = v else: validate_time = None callback_time = None callbacks_time = {} return (self, nb_iter, nb_replacement, nb_replacement_didn_t_remove, nb_inconsistency_make, nb_inconsistency_replace, time_canonicalize, time_factor_can, time_factor_list, time_toposort, validate_time, callback_time, callbacks_time,) @staticmethod def print_profile(stream, prof, level=0): blanc = (' ' * level) print(blanc, "GemmOptimizer", file=stream) print(blanc, " nb_iter", prof[1], file=stream) print(blanc, " nb_replacement", prof[2], file=stream) print(blanc, " nb_replacement_didn_t_remove", prof[3], file=stream) print(blanc, " nb_inconsistency_make", prof[4], file=stream) print(blanc, " nb_inconsistency_replace", prof[5], file=stream) print(blanc, " time_canonicalize", prof[6], file=stream) print(blanc, " time_factor_can", prof[7], file=stream) print(blanc, " time_factor_list", prof[8], file=stream) print(blanc, " time_toposort", prof[9], file=stream) print(blanc, " validate_time", prof[10], file=stream) print(blanc, " callback_time", prof[11], file=stream) if prof[11] > 1: print(blanc, " callbacks_time", file=stream) for i in sorted(iteritems(prof[12]), key=lambda a: a[1]): if i[1] > 0: print(i) class Dot22(GemmRelated): """Compute a matrix-matrix product. This is a specialization of the more general Dot(). """ check_input = False def make_node(self, x, y): x = T.as_tensor_variable(x) y = T.as_tensor_variable(y) dtypes = ('float16', 'float32', 'float64', 'complex64', 'complex128') if x.type.ndim != 2 or x.type.dtype not in dtypes: raise TypeError(x) if y.type.ndim != 2 or y.type.dtype not in dtypes: raise TypeError(y) if y.type.dtype != x.type.dtype: raise TypeError('dtype mismatch to Dot22') bz = (x.type.broadcastable[0], y.type.broadcastable[1]) outputs = [T.tensor(x.type.dtype, bz)] return Apply(self, [x, y], outputs) def perform(self, node, inp, out): x, y = inp z, = out try: z[0] = np.asarray(np.dot(x, y)) except ValueError as e: # The error raised by numpy has no shape information, we mean to # add that e.args = e.args + (x.shape, y.shape) raise def infer_shape(self, node, input_shapes): return [[input_shapes[0][0], input_shapes[1][1]]] setup_z_Nz_Sz = """ if ((NULL == %(_zout)s) || (PyArray_DIMS(%(_zout)s)[0] != PyArray_DIMS(%(_x)s)[0]) || (PyArray_DIMS(%(_zout)s)[1] != PyArray_DIMS(%(_y)s)[1])) { if (NULL != %(_zout)s) Py_XDECREF(%(_zout)s); npy_intp dims[2]; dims[0] = PyArray_DIMS(%(_x)s)[0]; dims[1] = PyArray_DIMS(%(_y)s)[1]; %(_zout)s = (PyArrayObject*)PyArray_SimpleNew(2, dims, PyArray_TYPE(%(_x)s)); //fprintf(stderr, "Dot Allocating %%i %%i\\n", dims[0], dims[1]); if(!%(_zout)s) { PyErr_SetString(PyExc_MemoryError, "failed to alloc dot22 output"); %(fail)s } } Nz = PyArray_DIMS(%(_zout)s); Sz = PyArray_STRIDES(%(_zout)s); """ check_ab_double_or_float = "" case_float_ab_constants = """ float a = 1.0; float b = 0.0; """ case_double_ab_constants = """ double a = 1.0; double b = 0.0; """ def c_code(self, node, name, inp, out, sub): # DEBUG _x, _y = inp _zout, = out if node.inputs[0].type.dtype.startswith('complex'): raise utils.MethodNotDefined('%s.c_code' % self.__class__.__name__) if len(self.c_libraries()) <= 0: return super(Dot22, self).c_code(node, name, (_x, _y), (_zout, ), sub) full_code = self.build_gemm_call() % dict(locals(), **sub) return full_code def c_code_cache_version(self): gv = self.build_gemm_version() if gv: return (2,) + gv else: return gv _dot22 = Dot22() @local_optimizer([T.Dot]) def local_dot_to_dot22(node): # This works for tensor.outer too because basic.outer is a macro that # produces a dot(dimshuffle,dimshuffle) of form 4 below if not isinstance(node.op, T.Dot): return x, y = node.inputs if y.type.dtype != x.type.dtype: # TODO: upcast one so the types match _logger.info('Not optimizing dot with inputs %s %s %s %s', x, y, x.type, y.type) return if y.type.dtype in ['float16', 'float32', 'float64', 'complex64', 'complex128']: with inherit_stack_trace(node.outputs): if x.ndim == 2 and y.ndim == 2: return [_dot22(*node.inputs)] if x.ndim == 2 and y.ndim == 1: return [_dot22(x, y.dimshuffle(0, 'x')).dimshuffle(0)] if x.ndim == 1 and y.ndim == 2: return [_dot22(x.dimshuffle('x', 0), y).dimshuffle(1)] if x.ndim == 1 and y.ndim == 1: return [_dot22(x.dimshuffle('x', 0), y.dimshuffle(0, 'x')).dimshuffle()] _logger.info('Not optimizing dot with inputs %s %s %s %s', x, y, x.type, y.type) @local_optimizer([gemm_no_inplace], inplace=True) def local_inplace_gemm(node): if node.op == gemm_no_inplace: with inherit_stack_trace(node.outputs): return [gemm_inplace(*node.inputs)] @local_optimizer([gemv_no_inplace], inplace=True) def local_inplace_gemv(node): if node.op == gemv_no_inplace: with inherit_stack_trace(node.outputs): return [gemv_inplace(*node.inputs)] @local_optimizer([ger], inplace=True) def local_inplace_ger(node): if node.op == ger: with inherit_stack_trace(node.outputs): return [ger_destructive(*node.inputs)] @local_optimizer([gemm_no_inplace]) def local_gemm_to_gemv(node): """GEMM acting on row or column matrices -> GEMV.""" if node.op == gemm_no_inplace: z, a, x, y, b = node.inputs with inherit_stack_trace(node.outputs): if z.broadcastable == x.broadcastable == (True, False): r = gemv_no_inplace(z.dimshuffle(1), a, y.T, x.dimshuffle(1), b) return [r.dimshuffle('x', 0)] if z.broadcastable == y.broadcastable == (False, True): r = gemv_no_inplace(z.dimshuffle(0), a, x, y.dimshuffle(0), b) return [r.dimshuffle(0, 'x')] @local_optimizer([gemm_no_inplace]) def local_gemm_to_ger(node): """GEMM computing an outer-product -> GER.""" if node.op == gemm_no_inplace: z, a, x, y, b = node.inputs if x.broadcastable[1] and y.broadcastable[0]: with inherit_stack_trace(node.outputs): # x and y are both vectors so this might qualifies for a GER xv = x.dimshuffle(0) yv = y.dimshuffle(1) try: bval = T.get_scalar_constant_value(b) except T.NotScalarConstantError: # b isn't a constant, GEMM is doing useful pre-scaling return if bval == 1: # best case a natural GER rval = ger(z, a, xv, yv) return [rval] elif bval == 0: # GER on zeros_like should be faster than GEMM zeros = T.zeros([x.shape[0], y.shape[1]], x.dtype) rval = ger(zeros, a, xv, yv) return [rval] else: # if bval is another constant, then z is being usefully # pre-scaled and GER isn't really the right tool for the job. return # TODO: delete this optimization when we have the proper dot->gemm->ger pipeline # working @local_optimizer([_dot22]) def local_dot22_to_ger_or_gemv(node): """dot22 computing an outer-product -> GER.""" if node.op == _dot22: with inherit_stack_trace(node.outputs): x, y = node.inputs xb = x.broadcastable yb = y.broadcastable one = T.as_tensor_variable(np.asarray(1, dtype=x.dtype)) zero = T.as_tensor_variable(np.asarray(0, dtype=x.dtype)) if xb[1] and yb[0]: # x and y are both vectors so this might qualifies for a GER xv = x.dimshuffle(0) yv = y.dimshuffle(1) zeros = T.zeros([x.shape[0], y.shape[1]], dtype=x.dtype) rval = ger(zeros, one, xv, yv) return [rval] if xb[0] and yb[1]: # x and y are both vectors so this qualifies for a sdot / ddot # TODO: Theano doesn't have a sdot, but gemv is better than _dot22 xv = x.dimshuffle(1) zeros = T.AllocEmpty(x.dtype)(1) rval = gemv_no_inplace(zeros, one, y.T, xv, zero) return [rval.dimshuffle('x', 0)] if xb[0] and not yb[0] and not yb[1]: # x is vector, y is matrix so try gemv xv = x.dimshuffle(1) zeros = T.AllocEmpty(x.dtype)(y.shape[1]) rval = gemv_no_inplace(zeros, one, y.T, xv, zero) return [rval.dimshuffle('x', 0)] if not xb[0] and not xb[1] and yb[1]: # x is matrix, y is vector, try gemv yv = y.dimshuffle(0) zeros = T.AllocEmpty(x.dtype)(x.shape[0]) rval = gemv_no_inplace(zeros, one, x, yv, zero) return [rval.dimshuffle(0, 'x')] ################################# # # Set up the BlasOpt optimizer # ################################# blas_optdb = SequenceDB() # run after numerical stability optimizations (1.5) optdb.register('BlasOpt', blas_optdb, 1.7, 'fast_run', 'fast_compile') # run before specialize (2.0) because specialize is basically a # free-for-all that makes the graph crazy. # fast_compile is needed to have GpuDot22 created. blas_optdb.register('local_dot_to_dot22', in2out(local_dot_to_dot22), 0, 'fast_run', 'fast_compile') blas_optdb.register('gemm_optimizer', GemmOptimizer(), 10, 'fast_run') blas_optdb.register('local_gemm_to_gemv', EquilibriumOptimizer([local_gemm_to_gemv, local_gemm_to_ger, local_dot22_to_ger_or_gemv, local_dimshuffle_lift], max_use_ratio=5, ignore_newtrees=False), 15, 'fast_run') # After destroyhandler(49.5) but before we try to make elemwise things # inplace (75) blas_opt_inplace = in2out(local_inplace_gemm, local_inplace_gemv, local_inplace_ger, name="blas_opt_inplace") optdb.register('InplaceBlasOpt', blas_opt_inplace, 70.0, 'fast_run', 'inplace', 'blas_opt_inplace') class Dot22Scalar(GemmRelated): """Compute a matrix-matrix product. This is a specialization of the more general Dot() Used to call optimized gemm implementation. Also used to generate a gemm later. compute scalar*dot(x,y). """ check_input = False def make_node(self, x, y, a): if a.ndim != 0: raise TypeError(Gemm.E_scalar, a) if x.ndim != 2: raise TypeError(Gemm.E_rank, x) if y.ndim != 2: raise TypeError(Gemm.E_rank, y) if not (a.dtype == x.dtype == y.dtype): raise TypeError('Dot22Scalar requires matching dtypes', (a.dtype, x.dtype, y.dtype)) if (not a.dtype.startswith('float') and not a.dtype.startswith('complex')): raise TypeError('Dot22Scalar requires float or complex args', a.dtype) bz = [x.type.broadcastable[0], y.type.broadcastable[1]] outputs = [T.tensor(x.type.dtype, bz)] return Apply(self, [x, y, a], outputs) def perform(self, node, inp, out): x, y, scalar = inp z, = out try: z[0] = np.asarray(scalar * np.dot(x, y)) except ValueError as e: # The error raised by numpy has no shape information, we # mean to add that e.args = e.args + (x.shape, y.shape) raise def infer_shape(self, node, input_shapes): return [[input_shapes[0][0], input_shapes[1][1]]] setup_z_Nz_Sz = Dot22.setup_z_Nz_Sz check_ab_double_or_float = """ if ((PyArray_DESCR(%(_a)s)->type_num != NPY_DOUBLE) && (PyArray_DESCR(%(_a)s)->type_num != NPY_FLOAT)) {PyErr_SetString(PyExc_NotImplementedError, "type(a) is not double or float"); %(fail)s;} """ case_float_ab_constants = """ #define REAL float float a = (PyArray_DESCR(%(_a)s)->type_num == NPY_FLOAT) ? (REAL)(((float*)PyArray_DATA(%(_a)s))[0]) : (REAL)(((double*)PyArray_DATA(%(_a)s))[0]); #undef REAL float b = 0.0; """ case_double_ab_constants = """ #define REAL double double a = (PyArray_DESCR(%(_a)s)->type_num == NPY_FLOAT) ? (REAL)(((float*)PyArray_DATA(%(_a)s))[0]) : (REAL)(((double*)PyArray_DATA(%(_a)s))[0]); #undef REAL double b = 0.0; """ def c_code(self, node, name, inp, out, sub): _x, _y, _a = inp _zout, = out if node.inputs[0].type.dtype.startswith('complex'): raise utils.MethodNotDefined('%s.c_code' % self.__class__.__name__) if len(self.c_libraries()) <= 0: return super(Dot22Scalar, self).c_code(node, name, (_x, _y), (_zout, ), sub) full_code = self.build_gemm_call() % dict(locals(), **sub) return full_code def c_code_cache_version(self): gv = self.build_gemm_version() if gv: return (2,) + gv else: return gv _dot22scalar = Dot22Scalar() @local_optimizer([T.mul]) def local_dot22_to_dot22scalar(node): """ Notes ----- Previous attempts to alter this optimization to replace dot22 with gemm instead of dot22scalar resulted in some Scan nodes being duplicated and the ScanSaveMem optimization never running on them, resulting in highly increased memory usage. Until this issue is resolved, this optimization should keep using dot22scalar instead of gemm. We upcast the scalar if after the multiplication with the dot this give the same type. We execute this optimizer after the gemm optimizer. This allow to give more priority to gemm that give more speed up then this optimizer, but allow the gemm optimizer to ignore this op. TODO: support when we can reorder the mul to generate a dot22scalar or fix the canonizer to merge them(1 mul with multiple inputs) """ if node.op != T.mul: return False i_dot22 = [x.owner and x.owner.op == _dot22 for x in node.inputs] if not any(i_dot22): return False # no dot22 if i_dot22.count(True) > 1: # TODO: try each of them. pass # return False #TODO fix dot22_idx = i_dot22.index(True) d = node.inputs[dot22_idx] i_scalar = [_as_scalar(x, dtype=d.dtype) for x in node.inputs] if not any(i_scalar): # Check if we can reorder the graph as this mul have a mul in inputs. # We support only 1 additional level of mul. # The canonizer should have merged those mul together. i_mul = [x.owner and x.owner.op == T.mul and any([_as_scalar(x_i, dtype=d.dtype) for x_i in x.owner.inputs]) for x in node.inputs] if not any(i_mul): # no scalar in input and no multiplication # if their was a multiplication we couls reorder the graph # by the associativity of the graph. return False mul_idx = i_mul.index(True) # The first one should always work m = node.inputs[mul_idx] scalar_idx = -1 for i, x in enumerate(m.owner.inputs): if _as_scalar(x, dtype=d.dtype) and (theano.scalar.upcast( x.type.dtype, d.type.dtype) == d.type.dtype): scalar_idx = i break if scalar_idx < 0: _logger.info('Not optimizing dot22 with inputs %s %s, as the' ' type of the scalar cannot be upcasted to the' ' matrix type', node.inputs, [x.type for x in node.inputs]) return False a = T.cast(_as_scalar(m.owner.inputs[scalar_idx], dtype=d.dtype), d.type.dtype) assert not a.type.ndim dot = _dot22scalar(d.owner.inputs[0], d.owner.inputs[1], a) # The other inputs to the original node that were # neither part of the dot22 or this mul should be # factors in the returned "mul" node. assert dot22_idx != mul_idx other_factors = [inpt for i, inpt in enumerate(node.inputs) if i not in (dot22_idx, mul_idx)] other_m_inputs = [inpt for i, inpt in enumerate(m.owner.inputs) if i != scalar_idx] return [T.mul(dot, *(other_factors + other_m_inputs))] scalar_idx = -1 for i, x in enumerate(node.inputs): if (i != dot22_idx and i_scalar[i] is not None and (theano.scalar.upcast(x.type.dtype, d.type.dtype) == d.type.dtype)): scalar_idx = i break if scalar_idx < 0: _logger.info('Not optimizing dot22 with inputs %s %s, as the type ' 'of the scalar cannot be upcasted to the matrix type', node.inputs, [x.type for x in node.inputs]) return False assert scalar_idx < len(node.inputs) s = node.inputs[scalar_idx] o = copy.copy(node.inputs) o.remove(d) o.remove(s) a = T.cast(i_scalar[scalar_idx], d.type.dtype) assert not a.type.ndim if len(o) == 0: return [_dot22scalar(d.owner.inputs[0], d.owner.inputs[1], a)] else: return [T.mul(_dot22scalar(d.owner.inputs[0], d.owner.inputs[1], a), *o)] # must happen after gemm as the gemm optimizer don't understant # dot22scalar and gemm give more speed up then dot22scalar blas_optdb.register('local_dot22_to_dot22scalar', in2out(local_dot22_to_dot22scalar), 11, 'fast_run') class BatchedDot(Op): """ Computes the batched dot product of two variables: batched_dot(a, b)[i] = dot(a[i], b[i]) """ __props__ = () def make_node(self, *inputs): inputs = list(map(T.as_tensor_variable, inputs)) if len(inputs) != 2: raise TypeError("theano.tensor.blas.BatchedDot: 2 arguments" " required, %d given " % len(inputs)) if inputs[0].ndim not in (2, 3): raise TypeError("theano.tensor.blas.BatchedDot: input 0 (0-indexed)" " must have ndim of 2 or 3, %d given. Consider" " calling theano.tensor.batched_dot instead." % inputs[0].ndim) if inputs[1].ndim not in (2, 3): raise TypeError("theano.tensor.blas.BatchedDot: input 1 (0-indexed)" " must have ndim of 2 or 3, %d given. Consider" " calling theano.tensor.batched_dot instead." % inputs[1].ndim) dtype = theano.scalar.upcast(*[input.type.dtype for input in inputs]) # upcast inputs to common dtype if needed upcasted_inputs = [T.cast(input, dtype) for input in inputs] broadcastable = ((inputs[0].type.broadcastable[0] or inputs[1].type.broadcastable[0],) + inputs[0].type.broadcastable[1:-1] + inputs[1].type.broadcastable[2:]) return Apply(self, upcasted_inputs, [T.tensor(dtype, broadcastable)]) def perform(self, node, inp, out): x, y = inp z, = out if x.shape[0] != y.shape[0]: raise TypeError( "theano.tensor.blas.BatchedDot: inputs [%s] must have the" " same size in axis 0, but have sizes [%s]." % (", ".join(map(str, inp)), ", ".join([str(i.shape[0]) for i in inp]))) shape = self.infer_shape(node, [i.shape for i in inp])[0] dtype = node.outputs[0].dtype z0 = z[0] = np.empty(shape, dtype=dtype) for i in xrange(z0.shape[0]): z0[i] = np.dot(x[i], y[i]) def c_support_code(self): batch_gemm_defn = """ template bool batch_gemm(function gemm, int type_size, PyArrayObject* xs, PyArrayObject* ys, PyArrayObject* zs) { npy_intp *Nx = PyArray_DIMS(xs), *Sx = PyArray_STRIDES(xs); npy_intp *Ny = PyArray_DIMS(ys), *Sy = PyArray_STRIDES(ys); npy_intp *Nz = PyArray_DIMS(zs), *Sz = PyArray_STRIDES(zs); if (Nx[0] != Ny[0]) { PyErr_Format(PyExc_ValueError, "Shape mismatch: batch sizes unequal." " x.shape is (%d, %d, %d)," " y.shape is (%d, %d, %d).", Nx[0], Nx[1], Nx[2], Ny[0], Ny[1], Ny[2]); return 1; } if (Nx[2] != Ny[1]) { PyErr_Format(PyExc_ValueError, "Shape mismatch: summation axis sizes unequal." " x.shape is (%d, %d, %d)," " y.shape is (%d, %d, %d).", Nx[0], Nx[1], Nx[2], Ny[0], Ny[1], Ny[2]); return 1; } /* encode the stride structure of _x,_y,_z into a single integer. */ int unit = 0; unit |= ((Sx[2] == type_size || Nx[2] == 1) ? 0x0 : (Sx[1] == type_size || Nx[1]==1) ? 0x1 : 0x2) << 8; unit |= ((Sy[2] == type_size || Ny[2] == 1) ? 0x0 : (Sy[1] == type_size || Ny[1]==1) ? 0x1 : 0x2) << 4; unit |= ((Sz[2] == type_size || Nz[2] == 1) ? 0x0 : (Sz[1] == type_size || Nz[1]==1) ? 0x1 : 0x2) << 0; /* create appropriate strides for malformed matrices that are row or column * vectors, or empty matrices. * In that case, the value of the stride does not really matter, but * some versions of BLAS insist that: * - they are not smaller than the number of elements in the array, * - they are not 0. */ int sx_1 = (Nx[1] > 1) ? Sx[1]/type_size : (Nx[2] + 1); int sx_2 = (Nx[2] > 1) ? Sx[2]/type_size : (Nx[1] + 1); int sy_1 = (Ny[1] > 1) ? Sy[1]/type_size : (Ny[2] + 1); int sy_2 = (Ny[2] > 1) ? Sy[2]/type_size : (Ny[1] + 1); int sz_1 = (Nz[1] > 1) ? Sz[1]/type_size : (Nz[2] + 1); int sz_2 = (Nz[2] > 1) ? Sz[2]/type_size : (Nz[1] + 1); dtype* x = (dtype*)PyArray_DATA(xs); dtype* y = (dtype*)PyArray_DATA(ys); dtype* z = (dtype*)PyArray_DATA(zs); dtype a = 1.0; dtype b = 0.0; char N = 'N'; char T = 'T'; int Nz1 = Nz[1], Nz2 = Nz[2], Nx2 = Nx[2]; // loop over batch axis for (int i = 0; i < Nz[0]; i++) { switch(unit) { case 0x000: gemm(&N, &N, &Nz2, &Nz1, &Nx2, &a, y, &sy_1, x, &sx_1, &b, z, &sz_1); break; case 0x100: gemm(&N, &T, &Nz2, &Nz1, &Nx2, &a, y, &sy_1, x, &sx_2, &b, z, &sz_1); break; case 0x010: gemm(&T, &N, &Nz2, &Nz1, &Nx2, &a, y, &sy_2, x, &sx_1, &b, z, &sz_1); break; case 0x110: gemm(&T, &T, &Nz2, &Nz1, &Nx2, &a, y, &sy_2, x, &sx_2, &b, z, &sz_1); break; case 0x001: gemm(&T, &T, &Nz1, &Nz2, &Nx2, &a, x, &sx_1, y, &sy_1, &b, z, &sz_2); break; case 0x101: gemm(&N, &T, &Nz1, &Nz2, &Nx2, &a, x, &sx_2, y, &sy_1, &b, z, &sz_2); break; case 0x011: gemm(&T, &N, &Nz1, &Nz2, &Nx2, &a, x, &sx_1, y, &sy_2, &b, z, &sz_2); break; case 0x111: gemm(&N, &N, &Nz1, &Nz2, &Nx2, &a, x, &sx_2, y, &sy_2, &b, z, &sz_2); break; default: PyErr_SetString(PyExc_ValueError, "some matrix has no unit stride"); return 1; }; x += Sx[0] / type_size; y += Sy[0] / type_size; z += Sz[0] / type_size; } return 0; } """ return blas_header_text() + batch_gemm_defn def c_libraries(self): return ldflags() def c_compile_args(self): return ldflags(libs=False, flags=True) def c_lib_dirs(self): return ldflags(libs=False, libs_dir=True) def c_header_dirs(self): return ldflags(libs=False, include_dir=True) def c_code_cleanup(self, node, name, inputs, outputs, sub): return """ // clean up views Py_XDECREF(xs); xs = 0; Py_XDECREF(ys); ys = 0; Py_XDECREF(zs); zs = 0; """ def c_code(self, node, name, inp, out, sub): _x, _y = inp _z, = out fail = sub["fail"] # generate contiguity condition def contiguous(var, ndim): strides = "PyArray_STRIDES(%s)" % var if ndim == 1: return "{strides}[0] == type_size".format(strides=strides) return " && ".join([ " && ".join("{strides}[{i}] > 0 && {strides}[{i}] % type_size == 0" .format(strides=strides, i=i) for i in range(1, ndim)), "(%s)" % " || ".join("{strides}[{i}] == type_size" .format(strides=strides, i=i) for i in range(1, ndim)), ]) x_ndim, y_ndim, z_ndim = node.inputs[0].ndim, node.inputs[1].ndim, node.outputs[0].ndim # generate code to allocate output based on runtime input shapes z_dims = ["PyArray_DIMS(%s)[0]" % _x] if x_ndim == 3: z_dims.append("PyArray_DIMS(%s)[1]" % _x) if y_ndim == 3: z_dims.append("PyArray_DIMS(%s)[2]" % _y) assert len(z_dims) == z_ndim z_shape_correct = " && ".join("PyArray_DIMS(%s)[%i] == %s" % (_z, i, dim) for i, dim in enumerate(z_dims)) z_shape = ", ".join(z_dims) z_contiguous = contiguous(_z, z_ndim) allocate = """ if (NULL == %(_z)s || !(%(z_shape_correct)s) || !(%(z_contiguous)s)) { npy_intp dims[%(z_ndim)s] = {%(z_shape)s}; Py_XDECREF(%(_z)s); %(_z)s = (PyArrayObject*)PyArray_SimpleNew( %(z_ndim)s, dims, PyArray_TYPE(%(_x)s)); if(!%(_z)s) { PyErr_SetString(PyExc_MemoryError, "failed to alloc BatchedDot output"); %(fail)s } } """ % locals() # code to reallocate inputs contiguously if necessary contiguate = [] for var, ndim in [(_x, x_ndim), (_y, y_ndim)]: _contiguous = contiguous(var, ndim) contiguate.append(""" if (!(%(_contiguous)s)) { PyArrayObject * _copy = (PyArrayObject *) PyArray_Copy(%(var)s); if (!_copy) %(fail)s Py_XDECREF(%(var)s); %(var)s = _copy; } """ % locals()) contiguate = "\n".join(contiguate) def c_dimshuffle(newname, oldname, shape): _fail = fail _shape = ", ".join("1" if axis is None else "PyArray_DIMS(%s)[%i]" % (oldname, axis) for axis in shape) return """{ npy_intp dims[3] = {%(_shape)s}; PyArray_Dims newshape = {dims, 3}; %(newname)s = (PyArrayObject*)PyArray_Newshape(%(oldname)s, &newshape, NPY_ANYORDER); if (!%(newname)s) %(_fail)s // make sure we didn't accidentally copy assert(PyArray_DATA(%(oldname)s) == PyArray_DATA(%(newname)s)); }""" % locals() # create tensor3 views for any of x, y, z that are not tensor3, so that # we only need to implement the tensor3-tensor3 batched dot product. # xs, ys and zs will point to these views, or to the original array if # it was already tensor3. # in the latter case, we artificially increase the reference count of # the original array so that the c_code_cleanup method can decref them # all indiscriminately. upcast = [] if x_ndim == 3: upcast.append("xs = %(_x)s; Py_XINCREF(xs);") elif x_ndim == 2: upcast.append(c_dimshuffle("xs", _x, (0, None, 1))) if y_ndim == 3: upcast.append("ys = %(_y)s; Py_XINCREF(ys);") elif y_ndim == 2: upcast.append(c_dimshuffle("ys", _y, (0, 1, None))) if z_ndim == 3: upcast.append("zs = %(_z)s; Py_XINCREF(zs);") else: upcast.append(c_dimshuffle( "zs", _z, (0, None if x_ndim == 2 else 1, None if y_ndim == 2 else 1))) upcast = "\n".join(upcast) % locals() return """ int type_num = PyArray_DESCR(%(_x)s)->type_num; int type_size = PyArray_DESCR(%(_x)s)->elsize; // in bytes // xs, ys, zs will point to views onto %(_x)s, %(_y)s, %(_z)s PyArrayObject *xs = 0, *ys = 0, *zs = 0; if (PyArray_NDIM(%(_x)s) != %(x_ndim)s) { PyErr_Format(PyExc_NotImplementedError, "rank(x) != %(x_ndim)s. rank(x) is %%d.", PyArray_NDIM(%(_x)s)); %(fail)s; } if (PyArray_NDIM(%(_y)s) != %(y_ndim)s) { PyErr_Format(PyExc_NotImplementedError, "rank(y) != %(y_ndim)s. rank(y) is %%d.", PyArray_NDIM(%(_y)s)); %(fail)s; } if (%(_z)s && PyArray_NDIM(%(_z)s) != %(z_ndim)s) { PyErr_Format(PyExc_NotImplementedError, "rank(z) != %(z_ndim)s. rank(z) is %%d.", PyArray_NDIM(%(_z)s)); %(fail)s; } // allocate output %(allocate)s // reallocate any noncontiguous arrays or arrays with invalid strides %(contiguate)s // add dims to make sure everything is tensor3 %(upcast)s // from here on, use xs, ys and zs as they are tensor3 and share memory // with the original %(_x)s, %(_y)s and %(_z)s arrays. if ((PyArray_DESCR(xs)->type_num != NPY_DOUBLE) && (PyArray_DESCR(xs)->type_num != NPY_FLOAT)) {PyErr_SetString(PyExc_NotImplementedError, "type(x) is not double or float"); %(fail)s;} if ((PyArray_DESCR(ys)->type_num != NPY_DOUBLE) && (PyArray_DESCR(ys)->type_num != NPY_FLOAT)) {PyErr_SetString(PyExc_NotImplementedError, "type(y) is not double or float"); %(fail)s;} if ((PyArray_DESCR(zs)->type_num != NPY_DOUBLE) && (PyArray_DESCR(zs)->type_num != NPY_FLOAT)) {PyErr_SetString(PyExc_NotImplementedError, "type(z) is not double or float"); %(fail)s;} if ((PyArray_DESCR(xs)->type_num != PyArray_DESCR(ys)->type_num) ||(PyArray_DESCR(xs)->type_num != PyArray_DESCR(zs)->type_num)) { PyErr_SetString(PyExc_NotImplementedError, "type(x), type(y), type(z) are not all the same"); %(fail)s; } switch (type_num) { case NPY_FLOAT: if (batch_gemm(sgemm_, type_size, xs, ys, zs)) { %(fail)s; } break; case NPY_DOUBLE: if (batch_gemm(dgemm_, type_size, xs, ys, zs)) { %(fail)s; } break; } """ % locals() def c_code_cache_version(self): from theano.tensor.blas_headers import blas_header_version return (3, blas_header_version()) def grad(self, inp, grads): x, y = inp gz, = grads xdim, ydim, gdim = x.type.ndim, y.type.ndim, gz.type.ndim # grad is a vector, so x is a matrix and y is a matrix if gdim == 1: xgrad = gz.dimshuffle(0, 'x') * y ygrad = gz.dimshuffle(0, 'x') * x # x is a matrix, y is a tensor3, grad is a matrix elif xdim == 2 and ydim == 3: xgrad = T.batched_dot(gz, y.dimshuffle(0, 2, 1)) ygrad = x.dimshuffle(0, 1, 'x') * gz.dimshuffle(0, 'x', 1) # x is a tensor3, y is a matrix, grad is a matrix elif xdim == 3 and ydim == 2: xgrad = gz.dimshuffle(0, 1, 'x') * y.dimshuffle(0, 'x', 1) ygrad = T.batched_dot(x.dimshuffle(0, 2, 1), gz) # x is a tensor3, y is a tensor3, grad is a tensor3 elif xdim == ydim == 3: xgrad = T.batched_dot(gz, y.dimshuffle(0, 2, 1)) ygrad = T.batched_dot(x.dimshuffle(0, 2, 1), gz) # If x or y contain broadcastable dimensions but only one of # them know that a matching dimensions is broadcastable, the # above code don't always return the right broadcast pattern. # This cause problem down the road. See gh-1461. if xgrad.broadcastable != x.broadcastable: xgrad = T.patternbroadcast(xgrad, x.broadcastable) if ygrad.broadcastable != y.broadcastable: ygrad = T.patternbroadcast(ygrad, y.broadcastable) return xgrad, ygrad def R_op(self, inputs, eval_points): # R_op for batched_dot(a, b) evaluted at c for a and d for b is # simply batched_dot(c, b) + batched_dot(a, d) assert len(inputs) == 2 assert len(eval_points) == 2 if eval_points[0] is None and eval_points[1] is None: return [None] debugger_available = config.compute_test_value != 'off' if debugger_available: try: iv0 = theano.gof.op.get_test_value(inputs[0]) except AttributeError: theano.gof.op.missing_test_message( 'first input passed to BatchedDot.R_op has no test value') debugger_available = False try: iv1 = theano.gof.op.get_test_value(inputs[1]) except AttributeError: theano.gof.op.missing_test_message( 'second input passed to BatchedDot.R_op has no test value') debugger_available = False if eval_points[0]: try: ev0 = theano.gof.op.get_test_value(eval_points[0]) except AttributeError: theano.gof.op.missing_test_message( 'first eval point passed to BatchedDot.R_op ' 'has no test value') debugger_available = False if eval_points[1]: try: ev1 = theano.gof.op.get_test_value(eval_points[1]) except AttributeError: theano.gof.op.missing_test_message( 'second eval point passed to BatchedDot.R_op ' 'has no test value') debugger_available = False if debugger_available: input_values = [iv0, iv1] eval_point_values = [ev0, ev1] for i in xrange(2): if eval_point_values[i] is not None and \ input_values[i].shape != eval_point_values[i].shape: raise ValueError( 'input ' + str(i) + ' and eval_point ' + str(i) + ' to BatchedDot.R_op should have the same shape, but ' 'their shapes are %s and %s, respectively' % ( str(input_values[i].shape), str(eval_point_values[i].shape))) if eval_points[0]: t1 = self(eval_points[0], inputs[1]) if eval_points[1]: t2 = self(inputs[0], eval_points[1]) if eval_points[0] and eval_points[1]: return [t1 + t2] elif eval_points[0]: return [t1] else: return [t2] def infer_shape(self, node, shapes): for shape_ in shapes: if len(shape_) not in (2, 3): raise NotImplementedError() xshp, yshp = shapes return [xshp[:-1] + yshp[2:]] batched_dot = BatchedDot() # from opt import register_specialize, register_canonicalize # @register_specialize @local_optimizer([T.sub, T.add]) def local_print_as_we_go_along(node): if node.op in (T.sub, T.add): debugprint(node) Theano-1.0.0/theano/tensor/blas_headers.py00006000163562€ÿÿÿþ0000020435013203335532022051 0ustar boccoset00000000000000""" Header text for the C and Fortran BLAS interfaces. There is no standard name or location for this header, so we just insert it ourselves into the C code. """ from __future__ import absolute_import, print_function, division import logging import textwrap import sys import os from os.path import dirname from theano import config from theano.gof.cmodule import GCC_compiler _logger = logging.getLogger('theano.tensor.blas') def detect_macos_sdot_bug(): """ Try to detect a bug in the default BLAS in MacOS. The problem in Theano has been reported in gh-1240, the underlying bug has been confirmed in http://www.macresearch.org/lapackblas-fortran-106#comment-17227. This function tries to compile code triggering that bug, and, if necessary, an attempted fix. Three attributes of this function will be set: - detect_macos_sdot_bug.tested will be set to True when this function is called. - detect_macos_sdot_bug.present will be set to True if the bug is detected. Its value is returned by the function - detect_macos_sdot_bug.fix_works will be set to True if the fix was attempted, and succeeded. """ _logger.debug('Starting detection of bug in Mac OS BLAS sdot_ routine') if detect_macos_sdot_bug.tested: return detect_macos_sdot_bug.present if sys.platform != 'darwin' or not config.blas.ldflags: _logger.info('Not Mac OS, no sdot_ bug') detect_macos_sdot_bug.tested = True return False # This code will return -1 if the dot product did not return # the right value (30.). flags = config.blas.ldflags.split() for f in flags: # Library directories should also be added as rpath, # so that they can be loaded even if the environment # variable LD_LIBRARY_PATH does not contain them lib_path = os.environ.get('DYLD_FALLBACK_LIBRARY_PATH', '').split(':') if f.startswith('-L'): flags.append('-Wl,-rpath,' + f[2:]) # also append those paths to DYLD_FALLBACK_LIBRARY_PATH to # support libraries that have the wrong install_name # (such as MKL on canopy installs) if (f[2:] not in lib_path): lib_path.append(f[2:]) # this goes into the python process environment that is # inherited by subprocesses/used by dyld when loading new objects os.environ['DYLD_FALLBACK_LIBRARY_PATH'] = ':'.join(lib_path) test_code = textwrap.dedent("""\ extern "C" float sdot_(int*, float*, int*, float*, int*); int main(int argc, char** argv) { int Nx = 5; int Sx = 1; float x[5] = {0, 1, 2, 3, 4}; float r = sdot_(&Nx, x, &Sx, x, &Sx); if ((r - 30.f) > 1e-6 || (r - 30.f) < -1e-6) { return -1; } return 0; } """) _logger.debug('Trying to compile and run test case.') compilation_ok, run_ok = GCC_compiler.try_compile_tmp( test_code, tmp_prefix='detect_macos_sdot_bug_', flags=flags, try_run=True) detect_macos_sdot_bug.tested = True # If compilation failed, we consider there is a bug, # and the fix does not work if not compilation_ok: _logger.info('Could not compile test case for sdot_.') detect_macos_sdot_bug.present = True return True if run_ok: _logger.info('The sdot_ bug is not present on this system.') detect_macos_sdot_bug.present = False return False # Else, the bug is detected. _logger.info('The sdot_ bug is present on this system.') detect_macos_sdot_bug.present = True # Then, try a simple fix test_fix_code = textwrap.dedent("""\ extern "C" float cblas_sdot(int, float*, int, float*, int); static float sdot_(int* Nx, float* x, int* Sx, float* y, int* Sy) { return cblas_sdot(*Nx, x, *Sx, y, *Sy); } int main(int argc, char** argv) { int Nx = 5; int Sx = 1; float x[5] = {0, 1, 2, 3, 4}; float r = sdot_(&Nx, x, &Sx, x, &Sx); if ((r - 30.f) > 1e-6 || (r - 30.f) < -1e-6) { return -1; } return 0; } """) _logger.debug('Trying to compile and run tentative workaround.') compilation_fix_ok, run_fix_ok = GCC_compiler.try_compile_tmp( test_fix_code, tmp_prefix='detect_macos_sdot_bug_testfix_', flags=flags, try_run=True) _logger.info("Status of tentative fix -- compilation OK: %s, works: %s", compilation_fix_ok, run_fix_ok) detect_macos_sdot_bug.fix_works = run_fix_ok return detect_macos_sdot_bug.present detect_macos_sdot_bug.tested = False detect_macos_sdot_bug.present = False detect_macos_sdot_bug.fix_works = False def cblas_header_text(): """C header for the cblas interface.""" return """ //#include #undef __BEGIN_DECLS #undef __END_DECLS #ifdef __cplusplus #define __BEGIN_DECLS extern "C" { #define __END_DECLS } #else #define __BEGIN_DECLS /* empty */ #define __END_DECLS /* empty */ #endif __BEGIN_DECLS #define MOD % /* * Enumerated and derived types */ #define CBLAS_INDEX size_t /* this may vary between platforms */ enum CBLAS_ORDER {CblasRowMajor=101, CblasColMajor=102}; enum CBLAS_TRANSPOSE {CblasNoTrans=111, CblasTrans=112, CblasConjTrans=113}; enum CBLAS_UPLO {CblasUpper=121, CblasLower=122}; enum CBLAS_DIAG {CblasNonUnit=131, CblasUnit=132}; enum CBLAS_SIDE {CblasLeft=141, CblasRight=142}; float cblas_sdsdot(const int N, const float alpha, const float *X, const int incX, const float *Y, const int incY); double cblas_dsdot(const int N, const float *X, const int incX, const float *Y, const int incY); float cblas_sdot(const int N, const float *X, const int incX, const float *Y, const int incY); double cblas_ddot(const int N, const double *X, const int incX, const double *Y, const int incY); /* * Functions having prefixes Z and C only */ void cblas_cdotu_sub(const int N, const void *X, const int incX, const void *Y, const int incY, void *dotu); void cblas_cdotc_sub(const int N, const void *X, const int incX, const void *Y, const int incY, void *dotc); void cblas_zdotu_sub(const int N, const void *X, const int incX, const void *Y, const int incY, void *dotu); void cblas_zdotc_sub(const int N, const void *X, const int incX, const void *Y, const int incY, void *dotc); /* * Functions having prefixes S D SC DZ */ float cblas_snrm2(const int N, const float *X, const int incX); float cblas_sasum(const int N, const float *X, const int incX); double cblas_dnrm2(const int N, const double *X, const int incX); double cblas_dasum(const int N, const double *X, const int incX); float cblas_scnrm2(const int N, const void *X, const int incX); float cblas_scasum(const int N, const void *X, const int incX); double cblas_dznrm2(const int N, const void *X, const int incX); double cblas_dzasum(const int N, const void *X, const int incX); /* * Functions having standard 4 prefixes (S D C Z) */ CBLAS_INDEX cblas_isamax(const int N, const float *X, const int incX); CBLAS_INDEX cblas_idamax(const int N, const double *X, const int incX); CBLAS_INDEX cblas_icamax(const int N, const void *X, const int incX); CBLAS_INDEX cblas_izamax(const int N, const void *X, const int incX); /* * =========================================================================== * Prototypes for level 1 BLAS routines * =========================================================================== */ /* * Routines with standard 4 prefixes (s, d, c, z) */ void cblas_sswap(const int N, float *X, const int incX, float *Y, const int incY); void cblas_scopy(const int N, const float *X, const int incX, float *Y, const int incY); void cblas_saxpy(const int N, const float alpha, const float *X, const int incX, float *Y, const int incY); void cblas_dswap(const int N, double *X, const int incX, double *Y, const int incY); void cblas_dcopy(const int N, const double *X, const int incX, double *Y, const int incY); void cblas_daxpy(const int N, const double alpha, const double *X, const int incX, double *Y, const int incY); void cblas_cswap(const int N, void *X, const int incX, void *Y, const int incY); void cblas_ccopy(const int N, const void *X, const int incX, void *Y, const int incY); void cblas_caxpy(const int N, const void *alpha, const void *X, const int incX, void *Y, const int incY); void cblas_zswap(const int N, void *X, const int incX, void *Y, const int incY); void cblas_zcopy(const int N, const void *X, const int incX, void *Y, const int incY); void cblas_zaxpy(const int N, const void *alpha, const void *X, const int incX, void *Y, const int incY); /* * Routines with S and D prefix only */ void cblas_srotg(float *a, float *b, float *c, float *s); void cblas_srotmg(float *d1, float *d2, float *b1, const float b2, float *P); void cblas_srot(const int N, float *X, const int incX, float *Y, const int incY, const float c, const float s); void cblas_srotm(const int N, float *X, const int incX, float *Y, const int incY, const float *P); void cblas_drotg(double *a, double *b, double *c, double *s); void cblas_drotmg(double *d1, double *d2, double *b1, const double b2, double *P); void cblas_drot(const int N, double *X, const int incX, double *Y, const int incY, const double c, const double s); void cblas_drotm(const int N, double *X, const int incX, double *Y, const int incY, const double *P); /* * Routines with S D C Z CS and ZD prefixes */ void cblas_sscal(const int N, const float alpha, float *X, const int incX); void cblas_dscal(const int N, const double alpha, double *X, const int incX); void cblas_cscal(const int N, const void *alpha, void *X, const int incX); void cblas_zscal(const int N, const void *alpha, void *X, const int incX); void cblas_csscal(const int N, const float alpha, void *X, const int incX); void cblas_zdscal(const int N, const double alpha, void *X, const int incX); /* * =========================================================================== * Prototypes for level 2 BLAS * =========================================================================== */ /* * Routines with standard 4 prefixes (S, D, C, Z) */ void cblas_sgemv(const enum CBLAS_ORDER order, const enum CBLAS_TRANSPOSE TransA, const int M, const int N, const float alpha, const float *A, const int lda, const float *X, const int incX, const float beta, float *Y, const int incY); void cblas_sgbmv(const enum CBLAS_ORDER order, const enum CBLAS_TRANSPOSE TransA, const int M, const int N, const int KL, const int KU, const float alpha, const float *A, const int lda, const float *X, const int incX, const float beta, float *Y, const int incY); void cblas_strmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const float *A, const int lda, float *X, const int incX); void cblas_stbmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const int K, const float *A, const int lda, float *X, const int incX); void cblas_stpmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const float *Ap, float *X, const int incX); void cblas_strsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const float *A, const int lda, float *X, const int incX); void cblas_stbsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const int K, const float *A, const int lda, float *X, const int incX); void cblas_stpsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const float *Ap, float *X, const int incX); void cblas_dgemv(const enum CBLAS_ORDER order, const enum CBLAS_TRANSPOSE TransA, const int M, const int N, const double alpha, const double *A, const int lda, const double *X, const int incX, const double beta, double *Y, const int incY); void cblas_dgbmv(const enum CBLAS_ORDER order, const enum CBLAS_TRANSPOSE TransA, const int M, const int N, const int KL, const int KU, const double alpha, const double *A, const int lda, const double *X, const int incX, const double beta, double *Y, const int incY); void cblas_dtrmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const double *A, const int lda, double *X, const int incX); void cblas_dtbmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const int K, const double *A, const int lda, double *X, const int incX); void cblas_dtpmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const double *Ap, double *X, const int incX); void cblas_dtrsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const double *A, const int lda, double *X, const int incX); void cblas_dtbsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const int K, const double *A, const int lda, double *X, const int incX); void cblas_dtpsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const double *Ap, double *X, const int incX); void cblas_cgemv(const enum CBLAS_ORDER order, const enum CBLAS_TRANSPOSE TransA, const int M, const int N, const void *alpha, const void *A, const int lda, const void *X, const int incX, const void *beta, void *Y, const int incY); void cblas_cgbmv(const enum CBLAS_ORDER order, const enum CBLAS_TRANSPOSE TransA, const int M, const int N, const int KL, const int KU, const void *alpha, const void *A, const int lda, const void *X, const int incX, const void *beta, void *Y, const int incY); void cblas_ctrmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const void *A, const int lda, void *X, const int incX); void cblas_ctbmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const int K, const void *A, const int lda, void *X, const int incX); void cblas_ctpmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const void *Ap, void *X, const int incX); void cblas_ctrsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const void *A, const int lda, void *X, const int incX); void cblas_ctbsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const int K, const void *A, const int lda, void *X, const int incX); void cblas_ctpsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const void *Ap, void *X, const int incX); void cblas_zgemv(const enum CBLAS_ORDER order, const enum CBLAS_TRANSPOSE TransA, const int M, const int N, const void *alpha, const void *A, const int lda, const void *X, const int incX, const void *beta, void *Y, const int incY); void cblas_zgbmv(const enum CBLAS_ORDER order, const enum CBLAS_TRANSPOSE TransA, const int M, const int N, const int KL, const int KU, const void *alpha, const void *A, const int lda, const void *X, const int incX, const void *beta, void *Y, const int incY); void cblas_ztrmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const void *A, const int lda, void *X, const int incX); void cblas_ztbmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const int K, const void *A, const int lda, void *X, const int incX); void cblas_ztpmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const void *Ap, void *X, const int incX); void cblas_ztrsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const void *A, const int lda, void *X, const int incX); void cblas_ztbsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const int K, const void *A, const int lda, void *X, const int incX); void cblas_ztpsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int N, const void *Ap, void *X, const int incX); /* * Routines with S and D prefixes only */ void cblas_ssymv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const float alpha, const float *A, const int lda, const float *X, const int incX, const float beta, float *Y, const int incY); void cblas_ssbmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const int K, const float alpha, const float *A, const int lda, const float *X, const int incX, const float beta, float *Y, const int incY); void cblas_sspmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const float alpha, const float *Ap, const float *X, const int incX, const float beta, float *Y, const int incY); void cblas_sger(const enum CBLAS_ORDER order, const int M, const int N, const float alpha, const float *X, const int incX, const float *Y, const int incY, float *A, const int lda); void cblas_ssyr(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const float alpha, const float *X, const int incX, float *A, const int lda); void cblas_sspr(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const float alpha, const float *X, const int incX, float *Ap); void cblas_ssyr2(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const float alpha, const float *X, const int incX, const float *Y, const int incY, float *A, const int lda); void cblas_sspr2(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const float alpha, const float *X, const int incX, const float *Y, const int incY, float *A); void cblas_dsymv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const double alpha, const double *A, const int lda, const double *X, const int incX, const double beta, double *Y, const int incY); void cblas_dsbmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const int K, const double alpha, const double *A, const int lda, const double *X, const int incX, const double beta, double *Y, const int incY); void cblas_dspmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const double alpha, const double *Ap, const double *X, const int incX, const double beta, double *Y, const int incY); void cblas_dger(const enum CBLAS_ORDER order, const int M, const int N, const double alpha, const double *X, const int incX, const double *Y, const int incY, double *A, const int lda); void cblas_dsyr(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const double alpha, const double *X, const int incX, double *A, const int lda); void cblas_dspr(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const double alpha, const double *X, const int incX, double *Ap); void cblas_dsyr2(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const double alpha, const double *X, const int incX, const double *Y, const int incY, double *A, const int lda); void cblas_dspr2(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const double alpha, const double *X, const int incX, const double *Y, const int incY, double *A); /* * Routines with C and Z prefixes only */ void cblas_chemv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const void *alpha, const void *A, const int lda, const void *X, const int incX, const void *beta, void *Y, const int incY); void cblas_chbmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const int K, const void *alpha, const void *A, const int lda, const void *X, const int incX, const void *beta, void *Y, const int incY); void cblas_chpmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const void *alpha, const void *Ap, const void *X, const int incX, const void *beta, void *Y, const int incY); void cblas_cgeru(const enum CBLAS_ORDER order, const int M, const int N, const void *alpha, const void *X, const int incX, const void *Y, const int incY, void *A, const int lda); void cblas_cgerc(const enum CBLAS_ORDER order, const int M, const int N, const void *alpha, const void *X, const int incX, const void *Y, const int incY, void *A, const int lda); void cblas_cher(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const float alpha, const void *X, const int incX, void *A, const int lda); void cblas_chpr(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const float alpha, const void *X, const int incX, void *A); void cblas_cher2(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const void *alpha, const void *X, const int incX, const void *Y, const int incY, void *A, const int lda); void cblas_chpr2(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const void *alpha, const void *X, const int incX, const void *Y, const int incY, void *Ap); void cblas_zhemv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const void *alpha, const void *A, const int lda, const void *X, const int incX, const void *beta, void *Y, const int incY); void cblas_zhbmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const int K, const void *alpha, const void *A, const int lda, const void *X, const int incX, const void *beta, void *Y, const int incY); void cblas_zhpmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const void *alpha, const void *Ap, const void *X, const int incX, const void *beta, void *Y, const int incY); void cblas_zgeru(const enum CBLAS_ORDER order, const int M, const int N, const void *alpha, const void *X, const int incX, const void *Y, const int incY, void *A, const int lda); void cblas_zgerc(const enum CBLAS_ORDER order, const int M, const int N, const void *alpha, const void *X, const int incX, const void *Y, const int incY, void *A, const int lda); void cblas_zher(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const double alpha, const void *X, const int incX, void *A, const int lda); void cblas_zhpr(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const double alpha, const void *X, const int incX, void *A); void cblas_zher2(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const void *alpha, const void *X, const int incX, const void *Y, const int incY, void *A, const int lda); void cblas_zhpr2(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, const void *alpha, const void *X, const int incX, const void *Y, const int incY, void *Ap); /* * =========================================================================== * Prototypes for level 3 BLAS * =========================================================================== */ /* * Routines with standard 4 prefixes (S, D, C, Z) */ void cblas_sgemm(const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_TRANSPOSE TransB, const int M, const int N, const int K, const float alpha, const float *A, const int lda, const float *B, const int ldb, const float beta, float *C, const int ldc); void cblas_ssymm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, const enum CBLAS_UPLO Uplo, const int M, const int N, const float alpha, const float *A, const int lda, const float *B, const int ldb, const float beta, float *C, const int ldc); void cblas_ssyrk(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE Trans, const int N, const int K, const float alpha, const float *A, const int lda, const float beta, float *C, const int ldc); void cblas_ssyr2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE Trans, const int N, const int K, const float alpha, const float *A, const int lda, const float *B, const int ldb, const float beta, float *C, const int ldc); void cblas_strmm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int M, const int N, const float alpha, const float *A, const int lda, float *B, const int ldb); void cblas_strsm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int M, const int N, const float alpha, const float *A, const int lda, float *B, const int ldb); void cblas_dgemm(const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_TRANSPOSE TransB, const int M, const int N, const int K, const double alpha, const double *A, const int lda, const double *B, const int ldb, const double beta, double *C, const int ldc); void cblas_dsymm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, const enum CBLAS_UPLO Uplo, const int M, const int N, const double alpha, const double *A, const int lda, const double *B, const int ldb, const double beta, double *C, const int ldc); void cblas_dsyrk(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE Trans, const int N, const int K, const double alpha, const double *A, const int lda, const double beta, double *C, const int ldc); void cblas_dsyr2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE Trans, const int N, const int K, const double alpha, const double *A, const int lda, const double *B, const int ldb, const double beta, double *C, const int ldc); void cblas_dtrmm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int M, const int N, const double alpha, const double *A, const int lda, double *B, const int ldb); void cblas_dtrsm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int M, const int N, const double alpha, const double *A, const int lda, double *B, const int ldb); void cblas_cgemm(const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_TRANSPOSE TransB, const int M, const int N, const int K, const void *alpha, const void *A, const int lda, const void *B, const int ldb, const void *beta, void *C, const int ldc); void cblas_csymm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, const enum CBLAS_UPLO Uplo, const int M, const int N, const void *alpha, const void *A, const int lda, const void *B, const int ldb, const void *beta, void *C, const int ldc); void cblas_csyrk(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE Trans, const int N, const int K, const void *alpha, const void *A, const int lda, const void *beta, void *C, const int ldc); void cblas_csyr2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE Trans, const int N, const int K, const void *alpha, const void *A, const int lda, const void *B, const int ldb, const void *beta, void *C, const int ldc); void cblas_ctrmm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int M, const int N, const void *alpha, const void *A, const int lda, void *B, const int ldb); void cblas_ctrsm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int M, const int N, const void *alpha, const void *A, const int lda, void *B, const int ldb); void cblas_zgemm(const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_TRANSPOSE TransB, const int M, const int N, const int K, const void *alpha, const void *A, const int lda, const void *B, const int ldb, const void *beta, void *C, const int ldc); void cblas_zsymm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, const enum CBLAS_UPLO Uplo, const int M, const int N, const void *alpha, const void *A, const int lda, const void *B, const int ldb, const void *beta, void *C, const int ldc); void cblas_zsyrk(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE Trans, const int N, const int K, const void *alpha, const void *A, const int lda, const void *beta, void *C, const int ldc); void cblas_zsyr2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE Trans, const int N, const int K, const void *alpha, const void *A, const int lda, const void *B, const int ldb, const void *beta, void *C, const int ldc); void cblas_ztrmm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int M, const int N, const void *alpha, const void *A, const int lda, void *B, const int ldb); void cblas_ztrsm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, const int M, const int N, const void *alpha, const void *A, const int lda, void *B, const int ldb); /* * Routines with prefixes C and Z only */ void cblas_chemm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, const enum CBLAS_UPLO Uplo, const int M, const int N, const void *alpha, const void *A, const int lda, const void *B, const int ldb, const void *beta, void *C, const int ldc); void cblas_cherk(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE Trans, const int N, const int K, const float alpha, const void *A, const int lda, const float beta, void *C, const int ldc); void cblas_cher2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE Trans, const int N, const int K, const void *alpha, const void *A, const int lda, const void *B, const int ldb, const float beta, void *C, const int ldc); void cblas_zhemm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, const enum CBLAS_UPLO Uplo, const int M, const int N, const void *alpha, const void *A, const int lda, const void *B, const int ldb, const void *beta, void *C, const int ldc); void cblas_zherk(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE Trans, const int N, const int K, const double alpha, const void *A, const int lda, const double beta, void *C, const int ldc); void cblas_zher2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE Trans, const int N, const int K, const void *alpha, const void *A, const int lda, const void *B, const int ldb, const double beta, void *C, const int ldc); void cblas_xerbla(int p, const char *rout, const char *form, ...); __END_DECLS """ def blas_header_text(): """C header for the fortran blas interface""" blas_code = "" if not config.blas.ldflags: # Include the Numpy version implementation of [sd]gemm_. current_filedir = dirname(__file__) blas_common_filepath = os.path.join(current_filedir, 'c_code', 'alt_blas_common.h') blas_template_filepath = os.path.join(current_filedir, 'c_code', 'alt_blas_template.c') common_code = "" sblas_code = "" dblas_code = "" with open(blas_common_filepath) as code: common_code = code.read() with open(blas_template_filepath) as code: template_code = code.read() sblas_code = template_code % {"float_type": "float", "float_size": 4, "npy_float": "NPY_FLOAT32", "precision": "s"} dblas_code = template_code % {"float_type": "double", "float_size": 8, "npy_float": "NPY_FLOAT64", "precision": "d"} if not common_code or not template_code: raise IOError("Unable to load NumPy implementation of BLAS functions from C source files.") blas_code += common_code blas_code += sblas_code blas_code += dblas_code header = """ extern "C" { void xerbla_(char*, void *); /***********/ /* Level 1 */ /***********/ /* Single Precision */ void srot_(const int*, float *, const int*, float *, const int*, const float *, const float *); void srotg_(float *,float *,float *,float *); void srotm_( const int*, float *, const int*, float *, const int*, const float *); void srotmg_(float *,float *,float *,const float *, float *); void sswap_( const int*, float *, const int*, float *, const int*); void scopy_( const int*, const float *, const int*, float *, const int*); void saxpy_( const int*, const float *, const float *, const int*, float *, const int*); float sdot_(const int*, const float *, const int*, const float *, const int*); void sdot_sub_(const int*, const float *, const int*, const float *, const int*, float *); void sdsdot_sub_( const int*, const float *, const float *, const int*, const float *, const int*, float *); void sscal_( const int*, const float *, float *, const int*); void snrm2_sub_( const int*, const float *, const int*, float *); void sasum_sub_( const int*, const float *, const int*, float *); void isamax_sub_( const int*, const float * , const int*, const int*); /* Double Precision */ void drot_(const int*, double *, const int*, double *, const int*, const double *, const double *); void drotg_(double *,double *,double *,double *); void drotm_( const int*, double *, const int*, double *, const int*, const double *); void drotmg_(double *,double *,double *,const double *, double *); void dswap_( const int*, double *, const int*, double *, const int*); void dcopy_( const int*, const double *, const int*, double *, const int*); void daxpy_( const int*, const double *, const double *, const int*, double *, const int*); void dswap_( const int*, double *, const int*, double *, const int*); double ddot_(const int*, const double *, const int*, const double *, const int*); void dsdot_sub_(const int*, const float *, const int*, const float *, const int*, double *); void ddot_sub_( const int*, const double *, const int*, const double *, const int*, double *); void dscal_( const int*, const double *, double *, const int*); void dnrm2_sub_( const int*, const double *, const int*, double *); void dasum_sub_( const int*, const double *, const int*, double *); void idamax_sub_( const int*, const double * , const int*, const int*); /* Single Complex Precision */ void cswap_( const int*, void *, const int*, void *, const int*); void ccopy_( const int*, const void *, const int*, void *, const int*); void caxpy_( const int*, const void *, const void *, const int*, void *, const int*); void cswap_( const int*, void *, const int*, void *, const int*); void cdotc_sub_( const int*, const void *, const int*, const void *, const int*, void *); void cdotu_sub_( const int*, const void *, const int*, const void *, const int*, void *); void cscal_( const int*, const void *, void *, const int*); void icamax_sub_( const int*, const void *, const int*, const int*); void csscal_( const int*, const float *, void *, const int*); void scnrm2_sub_( const int*, const void *, const int*, float *); void scasum_sub_( const int*, const void *, const int*, float *); /* Double Complex Precision */ void zswap_( const int*, void *, const int*, void *, const int*); void zcopy_( const int*, const void *, const int*, void *, const int*); void zaxpy_( const int*, const void *, const void *, const int*, void *, const int*); void zswap_( const int*, void *, const int*, void *, const int*); void zdotc_sub_( const int*, const void *, const int*, const void *, const int*, void *); void zdotu_sub_( const int*, const void *, const int*, const void *, const int*, void *); void zdscal_( const int*, const double *, void *, const int*); void zscal_( const int*, const void *, void *, const int*); void dznrm2_sub_( const int*, const void *, const int*, double *); void dzasum_sub_( const int*, const void *, const int*, double *); void izamax_sub_( const int*, const void *, const int*, const int*); /***********/ /* Level 2 */ /***********/ /* Single Precision */ void sgemv_(char*, const int*, const int*, const float *, const float *, const int*, const float *, const int*, const float *, float *, const int*); void sgbmv_(char*, const int*, const int*, const int*, const int*, const float *, const float *, const int*, const float *, const int*, const float *, float *, const int*); void ssymv_(char*, const int*, const float *, const float *, const int*, const float *, const int*, const float *, float *, const int*); void ssbmv_(char*, const int*, const int*, const float *, const float *, const int*, const float *, const int*, const float *, float *, const int*); void sspmv_(char*, const int*, const float *, const float *, const float *, const int*, const float *, float *, const int*); void strmv_( char*, char*, char*, const int*, const float *, const int*, float *, const int*); void stbmv_( char*, char*, char*, const int*, const int*, const float *, const int*, float *, const int*); void strsv_( char*, char*, char*, const int*, const float *, const int*, float *, const int*); void stbsv_( char*, char*, char*, const int*, const int*, const float *, const int*, float *, const int*); void stpmv_( char*, char*, char*, const int*, const float *, float *, const int*); void stpsv_( char*, char*, char*, const int*, const float *, float *, const int*); void sger_( const int*, const int*, const float *, const float *, const int*, const float *, const int*, float *, const int*); void ssyr_(char*, const int*, const float *, const float *, const int*, float *, const int*); void sspr_(char*, const int*, const float *, const float *, const int*, float *); void sspr2_(char*, const int*, const float *, const float *, const int*, const float *, const int*, float *); void ssyr2_(char*, const int*, const float *, const float *, const int*, const float *, const int*, float *, const int*); /* Double Precision */ void dgemv_(char*, const int*, const int*, const double *, const double *, const int*, const double *, const int*, const double *, double *, const int*); void dgbmv_(char*, const int*, const int*, const int*, const int*, const double *, const double *, const int*, const double *, const int*, const double *, double *, const int*); void dsymv_(char*, const int*, const double *, const double *, const int*, const double *, const int*, const double *, double *, const int*); void dsbmv_(char*, const int*, const int*, const double *, const double *, const int*, const double *, const int*, const double *, double *, const int*); void dspmv_(char*, const int*, const double *, const double *, const double *, const int*, const double *, double *, const int*); void dtrmv_( char*, char*, char*, const int*, const double *, const int*, double *, const int*); void dtbmv_( char*, char*, char*, const int*, const int*, const double *, const int*, double *, const int*); void dtrsv_( char*, char*, char*, const int*, const double *, const int*, double *, const int*); void dtbsv_( char*, char*, char*, const int*, const int*, const double *, const int*, double *, const int*); void dtpmv_( char*, char*, char*, const int*, const double *, double *, const int*); void dtpsv_( char*, char*, char*, const int*, const double *, double *, const int*); void dger_( const int*, const int*, const double *, const double *, const int*, const double *, const int*, double *, const int*); void dsyr_(char*, const int*, const double *, const double *, const int*, double *, const int*); void dspr_(char*, const int*, const double *, const double *, const int*, double *); void dspr2_(char*, const int*, const double *, const double *, const int*, const double *, const int*, double *); void dsyr2_(char*, const int*, const double *, const double *, const int*, const double *, const int*, double *, const int*); /* Single Complex Precision */ void cgemv_(char*, const int*, const int*, const void *, const void *, const int*, const void *, const int*, const void *, void *, const int*); void cgbmv_(char*, const int*, const int*, const int*, const int*, const void *, const void *, const int*, const void *, const int*, const void *, void *, const int*); void chemv_(char*, const int*, const void *, const void *, const int*, const void *, const int*, const void *, void *, const int*); void chbmv_(char*, const int*, const int*, const void *, const void *, const int*, const void *, const int*, const void *, void *, const int*); void chpmv_(char*, const int*, const void *, const void *, const void *, const int*, const void *, void *, const int*); void ctrmv_( char*, char*, char*, const int*, const void *, const int*, void *, const int*); void ctbmv_( char*, char*, char*, const int*, const int*, const void *, const int*, void *, const int*); void ctpmv_( char*, char*, char*, const int*, const void *, void *, const int*); void ctrsv_( char*, char*, char*, const int*, const void *, const int*, void *, const int*); void ctbsv_( char*, char*, char*, const int*, const int*, const void *, const int*, void *, const int*); void ctpsv_( char*, char*, char*, const int*, const void *, void *,const int*); void cgerc_( const int*, const int*, const void *, const void *, const int*, const void *, const int*, void *, const int*); void cgeru_( const int*, const int*, const void *, const void *, const int*, const void *, const int*, void *, const int*); void cher_(char*, const int*, const float *, const void *, const int*, void *, const int*); void cher2_(char*, const int*, const void *, const void *, const int*, const void *, const int*, void *, const int*); void chpr_(char*, const int*, const float *, const void *, const int*, void *); void chpr2_(char*, const int*, const float *, const void *, const int*, const void *, const int*, void *); /* Double Complex Precision */ void zgemv_(char*, const int*, const int*, const void *, const void *, const int*, const void *, const int*, const void *, void *, const int*); void zgbmv_(char*, const int*, const int*, const int*, const int*, const void *, const void *, const int*, const void *, const int*, const void *, void *, const int*); void zhemv_(char*, const int*, const void *, const void *, const int*, const void *, const int*, const void *, void *, const int*); void zhbmv_(char*, const int*, const int*, const void *, const void *, const int*, const void *, const int*, const void *, void *, const int*); void zhpmv_(char*, const int*, const void *, const void *, const void *, const int*, const void *, void *, const int*); void ztrmv_( char*, char*, char*, const int*, const void *, const int*, void *, const int*); void ztbmv_( char*, char*, char*, const int*, const int*, const void *, const int*, void *, const int*); void ztpmv_( char*, char*, char*, const int*, const void *, void *, const int*); void ztrsv_( char*, char*, char*, const int*, const void *, const int*, void *, const int*); void ztbsv_( char*, char*, char*, const int*, const int*, const void *, const int*, void *, const int*); void ztpsv_( char*, char*, char*, const int*, const void *, void *,const int*); void zgerc_( const int*, const int*, const void *, const void *, const int*, const void *, const int*, void *, const int*); void zgeru_( const int*, const int*, const void *, const void *, const int*, const void *, const int*, void *, const int*); void zher_(char*, const int*, const double *, const void *, const int*, void *, const int*); void zher2_(char*, const int*, const void *, const void *, const int*, const void *, const int*, void *, const int*); void zhpr_(char*, const int*, const double *, const void *, const int*, void *); void zhpr2_(char*, const int*, const double *, const void *, const int*, const void *, const int*, void *); /***********/ /* Level 3 */ /***********/ /* Single Precision */ void sgemm_(char*, char*, const int*, const int*, const int*, const float *, const float *, const int*, const float *, const int*, const float *, float *, const int*); void ssymm_(char*, char*, const int*, const int*, const float *, const float *, const int*, const float *, const int*, const float *, float *, const int*); void ssyrk_(char*, char*, const int*, const int*, const float *, const float *, const int*, const float *, float *, const int*); void ssyr2k_(char*, char*, const int*, const int*, const float *, const float *, const int*, const float *, const int*, const float *, float *, const int*); void strmm_(char*, char*, char*, char*, const int*, const int*, const float *, const float *, const int*, float *, const int*); void strsm_(char*, char*, char*, char*, const int*, const int*, const float *, const float *, const int*, float *, const int*); /* Double Precision */ void dgemm_(char*, char*, const int*, const int*, const int*, const double *, const double *, const int*, const double *, const int*, const double *, double *, const int*); void dsymm_(char*, char*, const int*, const int*, const double *, const double *, const int*, const double *, const int*, const double *, double *, const int*); void dsyrk_(char*, char*, const int*, const int*, const double *, const double *, const int*, const double *, double *, const int*); void dsyr2k_(char*, char*, const int*, const int*, const double *, const double *, const int*, const double *, const int*, const double *, double *, const int*); void dtrmm_(char*, char*, char*, char*, const int*, const int*, const double *, const double *, const int*, double *, const int*); void dtrsm_(char*, char*, char*, char*, const int*, const int*, const double *, const double *, const int*, double *, const int*); /* Single Complex Precision */ void cgemm_(char*, char*, const int*, const int*, const int*, const float *, const float *, const int*, const float *, const int*, const float *, float *, const int*); void csymm_(char*, char*, const int*, const int*, const float *, const float *, const int*, const float *, const int*, const float *, float *, const int*); void chemm_(char*, char*, const int*, const int*, const float *, const float *, const int*, const float *, const int*, const float *, float *, const int*); void csyrk_(char*, char*, const int*, const int*, const float *, const float *, const int*, const float *, float *, const int*); void cherk_(char*, char*, const int*, const int*, const float *, const float *, const int*, const float *, float *, const int*); void csyr2k_(char*, char*, const int*, const int*, const float *, const float *, const int*, const float *, const int*, const float *, float *, const int*); void cher2k_(char*, char*, const int*, const int*, const float *, const float *, const int*, const float *, const int*, const float *, float *, const int*); void ctrmm_(char*, char*, char*, char*, const int*, const int*, const float *, const float *, const int*, float *, const int*); void ctrsm_(char*, char*, char*, char*, const int*, const int*, const float *, const float *, const int*, float *, const int*); /* Double Complex Precision */ void zgemm_(char*, char*, const int*, const int*, const int*, const double *, const double *, const int*, const double *, const int*, const double *, double *, const int*); void zsymm_(char*, char*, const int*, const int*, const double *, const double *, const int*, const double *, const int*, const double *, double *, const int*); void zhemm_(char*, char*, const int*, const int*, const double *, const double *, const int*, const double *, const int*, const double *, double *, const int*); void zsyrk_(char*, char*, const int*, const int*, const double *, const double *, const int*, const double *, double *, const int*); void zherk_(char*, char*, const int*, const int*, const double *, const double *, const int*, const double *, double *, const int*); void zsyr2k_(char*, char*, const int*, const int*, const double *, const double *, const int*, const double *, const int*, const double *, double *, const int*); void zher2k_(char*, char*, const int*, const int*, const double *, const double *, const int*, const double *, const int*, const double *, double *, const int*); void ztrmm_(char*, char*, char*, char*, const int*, const int*, const double *, const double *, const int*, double *, const int*); void ztrsm_(char*, char*, char*, char*, const int*, const int*, const double *, const double *, const int*, double *, const int*); } """ if detect_macos_sdot_bug(): if detect_macos_sdot_bug.fix_works: header += textwrap.dedent("""\ extern "C" float cblas_sdot(int, float*, int, float*, int); static float sdot_(int* Nx, float* x, int* Sx, float* y, int* Sy) { return cblas_sdot(*Nx, x, *Sx, y, *Sy); } """) else: # Make sure the buggy version of sdot_ is never used header += textwrap.dedent("""\ static float sdot_(int* Nx, float* x, int* Sx, float* y, int* Sy) { fprintf(stderr, "FATAL: The implementation of BLAS SDOT " "routine in your system has a bug that " "makes it return wrong results.\\n" "Please contact theano-dev@groups.google.com.\\n" "You can work around this bug by using a " "different BLAS library, or disabling BLAS\\n"); assert(0); } """) return header + blas_code if not config.blas.ldflags: _logger.warning('Using NumPy C-API based implementation for BLAS functions.') def mkl_threads_text(): """C header for MKL threads interface""" header = """ extern "C" { int MKL_Set_Num_Threads_Local(int); #define mkl_set_num_threads_local MKL_Set_Num_Threads_Local void MKL_Set_Num_Threads(int); #define mkl_set_num_threads MKL_Set_Num_Threads int MKL_Get_Max_Threads(void); #define mkl_get_max_threads MKL_Get_Max_Threads int MKL_Domain_Set_Num_Threads(int, int); #define mkl_domain_set_num_threads MKL_Domain_Set_Num_Threads int MKL_Domain_Get_Max_Threads(int); #define mkl_domain_get_max_threads MKL_Domain_Get_Max_Threads void MKL_Set_Dynamic(int); #define mkl_set_dynamic MKL_Set_Dynamic int MKL_Get_Dynamic(void); #define mkl_get_dynamic MKL_Get_Dynamic } """ return header def openblas_threads_text(): """C header for OpenBLAS threads interface""" header = """ extern "C" { void openblas_set_num_threads(int); void goto_set_num_threads(int); int openblas_get_num_threads(void); } """ return header def blas_header_version(): # Version for the base header version = (9,) if detect_macos_sdot_bug(): if detect_macos_sdot_bug.fix_works: # Version with fix version += (1,) else: # Version with error version += (2,) return version def ____gemm_code(check_ab, a_init, b_init): mod = '%' return """ const char * error_string = NULL; int type_num = PyArray_DESCR(_x)->type_num; int type_size = PyArray_DESCR(_x)->elsize; // in bytes npy_intp* Nx = PyArray_DIMS(_x); npy_intp* Ny = PyArray_DIMS(_y); npy_intp* Nz = PyArray_DIMS(_z); npy_intp* Sx = PyArray_STRIDES(_x); npy_intp* Sy = PyArray_STRIDES(_y); npy_intp* Sz = PyArray_STRIDES(_z); size_t sx_0, sx_1, sy_0, sy_1, sz_0, sz_1; int unit = 0; if (PyArray_NDIM(_x) != 2) goto _dot_execute_fallback; if (PyArray_NDIM(_y) != 2) goto _dot_execute_fallback; if (PyArray_NDIM(_z) != 2) goto _dot_execute_fallback; %(check_ab)s if ((PyArray_DESCR(_x)->type_num != NPY_DOUBLE) && (PyArray_DESCR(_x)->type_num != NPY_FLOAT)) goto _dot_execute_fallback; if ((PyArray_DESCR(_y)->type_num != NPY_DOUBLE) && (PyArray_DESCR(_y)->type_num != NPY_FLOAT)) goto _dot_execute_fallback; if ((PyArray_DESCR(_y)->type_num != NPY_DOUBLE) && (PyArray_DESCR(_y)->type_num != NPY_FLOAT)) goto _dot_execute_fallback; if ((PyArray_DESCR(_x)->type_num != PyArray_DESCR(_y)->type_num) ||(PyArray_DESCR(_x)->type_num != PyArray_DESCR(_z)->type_num)) goto _dot_execute_fallback; if ((Nx[0] != Nz[0]) || (Nx[1] != Ny[0]) || (Ny[1] != Nz[1])) { error_string = "Input dimensions do not agree"; goto _dot_execute_fail; } if ((Sx[0] < 1) || (Sx[1] < 1) || (Sx[0] %(mod)s type_size) || (Sx[1] %(mod)s type_size) || (Sy[0] < 1) || (Sy[1] < 1) || (Sy[0] %(mod)s type_size) || (Sy[1] %(mod)s type_size) || (Sz[0] < 1) || (Sz[1] < 1) || (Sz[0] %(mod)s type_size) || (Sz[1] %(mod)s type_size)) { goto _dot_execute_fallback; } /* encode the stride structure of _x,_y,_z into a single integer */ unit |= ((Sx[1] == type_size) ? 0x0 : (Sx[0] == type_size) ? 0x1 : 0x2) << 0; unit |= ((Sy[1] == type_size) ? 0x0 : (Sy[0] == type_size) ? 0x1 : 0x2) << 4; unit |= ((Sz[1] == type_size) ? 0x0 : (Sz[0] == type_size) ? 0x1 : 0x2) << 8; /* create appropriate strides for malformed matrices that are row or column * vectors */ sx_0 = (Nx[0] > 1) ? Sx[0]/type_size : Nx[1]; sx_1 = (Nx[1] > 1) ? Sx[1]/type_size : Nx[0]; sy_0 = (Ny[0] > 1) ? Sy[0]/type_size : Ny[1]; sy_1 = (Ny[1] > 1) ? Sy[1]/type_size : Ny[0]; sz_0 = (Nz[0] > 1) ? Sz[0]/type_size : Nz[1]; sz_1 = (Nz[1] > 1) ? Sz[1]/type_size : Nz[0]; switch (type_num) { case NPY_FLOAT: { #define REAL float float a = %(a_init)s; float b = %(b_init)s; float* x = (float*)PyArray_DATA(_x); float* y = (float*)PyArray_DATA(_y); float* z = (float*)PyArray_DATA(_z); switch(unit) { case 0x000: cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, Nz[0], Nz[1], Nx[1], a, x, sx_0, y, sy_0, b, z, sz_0); break; case 0x001: cblas_sgemm(CblasRowMajor, CblasTrans, CblasNoTrans, Nz[0], Nz[1], Nx[1], a, x, sx_1, y, sy_0, b, z, sz_0); break; case 0x010: cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, Nz[0], Nz[1], Nx[1], a, x, sx_0, y, sy_1, b, z, sz_0); break; case 0x011: cblas_sgemm(CblasRowMajor, CblasTrans, CblasTrans, Nz[0], Nz[1], Nx[1], a, x, sx_1, y, sy_1, b, z, sz_0); break; case 0x100: cblas_sgemm(CblasColMajor, CblasTrans, CblasTrans, Nz[0], Nz[1], Nx[1], a, x, sx_0, y, sy_0, b, z, sz_1); break; case 0x101: cblas_sgemm(CblasColMajor, CblasNoTrans, CblasTrans, Nz[0], Nz[1], Nx[1], a, x, sx_1, y, sy_0, b, z, sz_1); break; case 0x110: cblas_sgemm(CblasColMajor, CblasTrans, CblasNoTrans, Nz[0], Nz[1], Nx[1], a, x, sx_0, y, sy_1, b, z, sz_1); break; case 0x111: cblas_sgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, Nz[0], Nz[1], Nx[1], a, x, sx_1, y, sy_1, b, z, sz_1); break; default: goto _dot_execute_fallback; }; #undef REAL } break; case NPY_DOUBLE: { #define REAL double double a = %(a_init)s; double b = %(b_init)s; double* x = (double*)PyArray_DATA(_x); double* y = (double*)PyArray_DATA(_y); double* z = (double*)PyArray_DATA(_z); switch(unit) { case 0x000: cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, Nz[0], Nz[1], Nx[1], a, x, sx_0, y, sy_0, b, z, sz_0); break; case 0x001: cblas_dgemm(CblasRowMajor, CblasTrans, CblasNoTrans, Nz[0], Nz[1], Nx[1], a, x, sx_1, y, sy_0, b, z, sz_0); break; case 0x010: cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasTrans, Nz[0], Nz[1], Nx[1], a, x, sx_0, y, sy_1, b, z, sz_0); break; case 0x011: cblas_dgemm(CblasRowMajor, CblasTrans, CblasTrans, Nz[0], Nz[1], Nx[1], a, x, sx_1, y, sy_1, b, z, sz_0); break; case 0x100: cblas_dgemm(CblasColMajor, CblasTrans, CblasTrans, Nz[0], Nz[1], Nx[1], a, x, sx_0, y, sy_0, b, z, sz_1); break; case 0x101: cblas_dgemm(CblasColMajor, CblasNoTrans, CblasTrans, Nz[0], Nz[1], Nx[1], a, x, sx_1, y, sy_0, b, z, sz_1); break; case 0x110: cblas_dgemm(CblasColMajor, CblasTrans, CblasNoTrans, Nz[0], Nz[1], Nx[1], a, x, sx_0, y, sy_1, b, z, sz_1); break; case 0x111: cblas_dgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, Nz[0], Nz[1], Nx[1], a, x, sx_1, y, sy_1, b, z, sz_1); break; default: goto _dot_execute_fallback; }; #undef REAL } break; } return 0; //success! _dot_execute_fallback: PyErr_SetString(PyExc_NotImplementedError, "dot->execute() fallback"); return -1; _dot_execute_fail: if (error_string == NULL) PyErr_SetString(PyExc_ValueError, "dot->execute() cant run on these inputs"); return -1; /* v 1 */ """ % locals() Theano-1.0.0/theano/tensor/raw_random.py00006000163562€ÿÿÿþ0000011412413203335532021565 0ustar boccoset00000000000000"""Define random number Type (`RandomStateType`) and Op (`RandomFunction`).""" from __future__ import absolute_import, print_function, division import sys from copy import copy import numpy as np from six import string_types from six.moves import reduce, xrange # local imports import theano from theano import tensor from theano.tensor import opt from theano import gof from theano.compile import optdb __docformat__ = "restructuredtext en" class RandomStateType(gof.Type): """ A Type wrapper for numpy.random.RandomState. The reason this exists (and `Generic` doesn't suffice) is that RandomState objects that would appear to be equal do not compare equal with the '==' operator. This Type exists to provide an equals function that is used by DebugMode. """ def __str__(self): return 'RandomStateType' def filter(self, data, strict=False, allow_downcast=None): if self.is_valid_value(data): return data else: raise TypeError() def is_valid_value(self, a): return type(a) == np.random.RandomState def values_eq(self, a, b): sa = a.get_state() sb = b.get_state() # Should always be the string 'MT19937' if sa[0] != sb[0]: return False # 1-D array of 624 unsigned integer keys if not np.all(sa[1] == sb[1]): return False # integer "pos" representing the position in the array if sa[2] != sb[2]: return False # integer "has_gauss" if sa[3] != sb[3]: return False # float "cached_gaussian". # /!\ It is not initialized if has_gauss == 0 if sa[3] != 0: if sa[4] != sb[4]: return False return True def get_shape_info(self, obj): return None def get_size(self, shape_info): # The size is the data, that have constant size. state = np.random.RandomState().get_state() size = 0 for elem in state: if isinstance(elem, str): size += len(elem) elif isinstance(elem, np.ndarray): size += elem.size * elem.itemsize elif isinstance(elem, int): size += np.dtype("int").itemsize elif isinstance(elem, float): size += np.dtype("float").itemsize else: raise NotImplementedError() return size @staticmethod def may_share_memory(a, b): return a is b # Register RandomStateType's C code for ViewOp. theano.compile.register_view_op_c_code( RandomStateType, """ Py_XDECREF(%(oname)s); %(oname)s = %(iname)s; Py_XINCREF(%(oname)s); """, 1) random_state_type = RandomStateType() class RandomFunction(gof.Op): """ Op that draws random numbers from a numpy.random.RandomState object. Parameters ---------- fn : string or function reference A member function of numpy.random.RandomState. A string will be interpreted as the name of a member function of numpy.random.RandomState. Technically, any function with a signature like the ones in numpy.random.RandomState will do. This function must accept the shape (sometimes called size) of the output as the last positional argument. outtype The theano Type of the output. args A list of default arguments for the function kwargs If the 'inplace' key is there, its value will be used to determine if the op operates inplace or not. If the 'ndim_added' key is there, its value indicates how many more dimensions this op will add to the output, in addition to the shape's dimensions (used in multinomial and permutation). """ __props__ = ("fn", "outtype", "inplace", "ndim_added") def __init__(self, fn, outtype, inplace=False, ndim_added=0): self.__setstate__([fn, outtype, inplace, ndim_added]) def __getstate__(self): d = dict(self.__dict__) del d['exec_fn'] if 'destroy_map' in d: del d['destroy_map'] return d def __setstate__(self, dct): if isinstance(dct, dict): state = [dct['fn'], dct['outtype'], dct['inplace'], dct['ndim_added']] self.__dict__.update(dct) else: state = dct fn, outtype, inplace, ndim_added = state self.fn = fn if isinstance(fn, string_types): self.exec_fn = getattr(np.random.RandomState, fn) else: self.exec_fn = fn self.outtype = outtype self.inplace = inplace if self.inplace: self.destroy_map = {0: [0]} self.ndim_added = ndim_added def __str__(self): return 'RandomFunction{%s}' % self.exec_fn.__name__ def make_node(self, r, shape, *args): """ Parameters ---------- r A numpy.random.RandomState instance, or a Variable of Type RandomStateType that will contain a RandomState instance. shape An lvector with a shape defining how many samples to draw. In the case of scalar distributions, it is the shape of the tensor output by this Op. In that case, at runtime, the value associated with this lvector must have a length equal to the number of dimensions promised by `self.outtype`. In a more general case, the number of output dimensions, len(self.outtype), is equal to len(shape)+self.ndim_added. The special case where len(shape) == 0 means that the smallest shape compatible with the argument's shape will be used. args The values associated with these variables will be passed to the RandomState function during perform as extra "*args"-style arguments. These should be castable to variables of Type TensorType. Returns ------- Apply Apply with two outputs. The first output is a gof.generic Variable from which to draw further random numbers. The second output is the outtype() instance holding the random draw. """ shape_ = tensor.as_tensor_variable(shape, ndim=1) if shape == (): shape = shape_.astype('int64') else: shape = shape_ assert shape.type.ndim == 1 assert (shape.type.dtype == 'int64') or (shape.type.dtype == 'int32') if not isinstance(r.type, RandomStateType): print('WARNING: RandomState instances should be in RandomStateType', file=sys.stderr) # the following doesn't work because we want to ignore the # broadcastable flags in shape.type # assert shape.type == tensor.lvector # convert args to TensorType instances # and append enough None's to match the length of self.args args = list(map(tensor.as_tensor_variable, args)) return gof.Apply(self, [r, shape] + args, [r.type(), self.outtype()]) def infer_shape(self, node, i_shapes): r, shp = node.inputs[0:2] # if shp is a constant array of len 0, then it means 'automatic shape' unknown_shape = len(getattr(shp, 'data', [0, 1, 2])) == 0 # if ndim_added == 0 and shape != () then shape if self.ndim_added == 0 and not unknown_shape: sample_shp = shp else: # if shape == () then it will depend on args # if ndim_added != 0 and shape != () then it will depend on args # Use the default infer_shape implementation. raise tensor.ShapeError() return [None, [sample_shp[i] for i in xrange(node.outputs[1].ndim)]] def perform(self, node, inputs, out_): rout, out = out_ # Use self.fn to draw shape worth of random numbers. # Numbers are drawn from r if self.inplace is True, and from a # copy of r if self.inplace is False r, shape, args = inputs[0], inputs[1], inputs[2:] assert type(r) == np.random.RandomState, (type(r), r) # If shape == [], that means no shape is enforced, and numpy is # trusted to draw the appropriate number of samples, numpy uses # shape "None" to represent that. Else, numpy expects a tuple. # TODO: compute the appropriate shape, and pass it to numpy. if len(shape) == 0: shape = None else: shape = tuple(shape) if (shape is not None and self.outtype.ndim != len(shape) + self.ndim_added): raise ValueError('Shape mismatch: self.outtype.ndim (%i) !=' ' len(shape) (%i) + self.ndim_added (%i)' % (self.outtype.ndim, len(shape), self.ndim_added)) if not self.inplace: r = copy(r) rout[0] = r rval = self.exec_fn(r, *(args + [shape])) if (not isinstance(rval, np.ndarray) or str(rval.dtype) != node.outputs[1].type.dtype): rval = theano._asarray(rval, dtype=node.outputs[1].type.dtype) # When shape is None, numpy has a tendency to unexpectedly # return a scalar instead of a higher-dimension array containing # only one element. This value should be reshaped if shape is None and rval.ndim == 0 and self.outtype.ndim > 0: rval = rval.reshape([1] * self.outtype.ndim) if len(rval.shape) != self.outtype.ndim: raise ValueError('Shape mismatch: "out" should have dimension %i,' ' but the value produced by "perform" has' ' dimension %i' % (self.outtype.ndim, len(rval.shape))) # Check the output has the right shape if shape is not None: if self.ndim_added == 0 and shape != rval.shape: raise ValueError( 'Shape mismatch: "out" should have shape %s, but the' ' value produced by "perform" has shape %s' % (shape, rval.shape)) elif (self.ndim_added > 0 and shape != rval.shape[:-self.ndim_added]): raise ValueError( 'Shape mismatch: "out" should have shape starting with' ' %s (plus %i extra dimensions), but the value produced' ' by "perform" has shape %s' % (shape, self.ndim_added, rval.shape)) out[0] = rval def grad(self, inputs, outputs): return [theano.gradient.grad_undefined(self, k, inp, 'No gradient defined through raw random numbers op') for k, inp in enumerate(inputs)] def R_op(self, inputs, eval_points): return [None for i in eval_points] def _infer_ndim_bcast(ndim, shape, *args): """ Infer the number of dimensions from the shape or the other arguments. Returns ------- (int, variable, tuple) triple, where the variable is an integer vector, and the tuple contains Booleans The first element returned is the inferred number of dimensions. The second element is the shape inferred (combining symbolic and constant informations from shape and args). The third element is a broadcasting pattern corresponding to that shape. """ # Find the minimum value of ndim required by the *args if args: args_ndim = max(arg.ndim for arg in args) else: args_ndim = 0 if isinstance(shape, (tuple, list)): # there is a convention that -1 means the corresponding shape of a # potentially-broadcasted symbolic arg # # This case combines together symbolic and non-symbolic shape # information shape_ndim = len(shape) if ndim is None: ndim = shape_ndim else: if shape_ndim != ndim: raise ValueError('ndim should be equal to len(shape), but\n', 'ndim = %s, len(shape) = %s, shape = %s' % (ndim, shape_ndim, shape)) bcast = [] pre_v_shape = [] for i, s in enumerate(shape): if hasattr(s, 'type'): # s is symbolic bcast.append(False) # todo - introspect further pre_v_shape.append(s) else: if s >= 0: pre_v_shape.append(tensor.as_tensor_variable(s)) bcast.append((s == 1)) elif s == -1: n_a_i = 0 for a in args: # ndim: _ _ _ _ _ _ # ashp: s0 s1 s2 s3 # i if i >= ndim - a.ndim: n_a_i += 1 a_i = i + a.ndim - ndim if not a.broadcastable[a_i]: pre_v_shape.append(a.shape[a_i]) bcast.append(False) break else: if n_a_i == 0: raise ValueError(( 'Auto-shape of -1 must overlap' 'with the shape of one of the broadcastable' 'inputs')) else: pre_v_shape.append(tensor.as_tensor_variable(1)) bcast.append(True) else: ValueError('negative shape', s) # post-condition: shape may still contain both symbolic and # non-symbolic things if len(pre_v_shape) == 0: v_shape = tensor.constant([], dtype='int64') else: v_shape = tensor.stack(pre_v_shape) elif shape is None: # The number of drawn samples will be determined automatically, # but we need to know ndim if not args: raise TypeError(('_infer_ndim_bcast cannot infer shape without' ' either shape or args')) template = reduce(lambda a, b: a + b, args) v_shape = template.shape bcast = template.broadcastable ndim = template.ndim else: v_shape = tensor.as_tensor_variable(shape) if v_shape.ndim != 1: raise TypeError( "shape must be a vector or list of scalar, got '%s'" % v_shape) if ndim is None: ndim = tensor.get_vector_length(v_shape) bcast = [False] * ndim if v_shape.ndim != 1: raise TypeError("shape must be a vector or list of scalar, got '%s'" % v_shape) if v_shape.dtype not in theano.tensor.integer_dtypes: raise TypeError('shape must be an integer vector or list', v_shape.dtype) if args_ndim > ndim: raise ValueError( 'ndim should be at least as big as required by args value', (ndim, args_ndim), args) assert ndim == len(bcast) return ndim, tensor.cast(v_shape, 'int64'), tuple(bcast) def _generate_broadcasting_indices(out_shape, *shapes): """ Return indices over each shape that broadcast them to match out_shape. The first returned list is equivalent to numpy.ndindex(out_shape), the other returned lists are indices corresponding to the other shapes, such that looping over these indices produce tensors of shape out_shape. In particular, the indices over broadcasted dimensions should all be 0. The shapes should have the same length as out_shape. If they are longer, the right-most dimensions are ignored. """ all_shapes = (out_shape,) + shapes # Will contain the return value: a list of indices for each argument ret_indices = [[()] for shape in all_shapes] for dim in xrange(len(out_shape)): # Temporary list to generate the indices _ret_indices = [[] for shape in all_shapes] out_range = list(range(out_shape[dim])) # Verify the shapes are compatible along that dimension # and generate the appropriate range: out_range, or [0, ..., 0] ranges = [out_range] for shape in shapes: if shape[dim] == out_shape[dim]: ranges.append(out_range) elif shape[dim] == 1: # broadcast ranges.append([0] * out_shape[dim]) else: raise ValueError( 'shape[%i] (%i) should be equal to out_shape[%i] (%i) or' ' to 1' % (dim, shape[dim], dim, out_shape[dim]), shape, out_shape, shapes) for prev_index in zip(*ret_indices): for dim_index in zip(*ranges): for i in xrange(len(all_shapes)): _ret_indices[i].append(prev_index[i] + (dim_index[i],)) ret_indices = _ret_indices return ret_indices def uniform(random_state, size=None, low=0.0, high=1.0, ndim=None, dtype=None): """ Sample from a uniform distribution between low and high. If the size argument is ambiguous on the number of dimensions, ndim may be a plain integer to supplement the missing information. If size is None, the output shape will be determined by the shapes of low and high. If dtype is not specified, it will be inferred from the dtype of low and high, but will be at least as precise as floatX. """ low = tensor.as_tensor_variable(low) high = tensor.as_tensor_variable(high) if dtype is None: dtype = tensor.scal.upcast(theano.config.floatX, low.dtype, high.dtype) ndim, size, bcast = _infer_ndim_bcast(ndim, size, low, high) op = RandomFunction('uniform', tensor.TensorType(dtype=dtype, broadcastable=bcast)) return op(random_state, size, low, high) def normal(random_state, size=None, avg=0.0, std=1.0, ndim=None, dtype=None): """ Sample from a normal distribution centered on avg with the specified standard deviation (std). If the size argument is ambiguous on the number of dimensions, ndim may be a plain integer to supplement the missing information. If size is None, the output shape will be determined by the shapes of avg and std. If dtype is not specified, it will be inferred from the dtype of avg and std, but will be at least as precise as floatX. """ avg = tensor.as_tensor_variable(avg) std = tensor.as_tensor_variable(std) if dtype is None: dtype = tensor.scal.upcast(theano.config.floatX, avg.dtype, std.dtype) ndim, size, bcast = _infer_ndim_bcast(ndim, size, avg, std) op = RandomFunction('normal', tensor.TensorType(dtype=dtype, broadcastable=bcast)) return op(random_state, size, avg, std) def binomial(random_state, size=None, n=1, p=0.5, ndim=None, dtype='int64', prob=None): """ Sample n times with probability of success prob for each trial, return the number of successes. If the size argument is ambiguous on the number of dimensions, ndim may be a plain integer to supplement the missing information. If size is None, the output shape will be determined by the shapes of n and prob. """ if prob is not None: p = prob print("DEPRECATION WARNING: the parameter prob to the binomal fct have been renamed to p to have the same name as np.", file=sys.stderr) n = tensor.as_tensor_variable(n) p = tensor.as_tensor_variable(p) ndim, size, bcast = _infer_ndim_bcast(ndim, size, n, p) if n.dtype == 'int64': try: np.random.binomial(n=np.asarray([2, 3, 4], dtype='int64'), p=np.asarray([.1, .2, .3], dtype='float64')) except TypeError: # THIS WORKS AROUND A NUMPY BUG on 32bit machine n = tensor.cast(n, 'int32') op = RandomFunction('binomial', tensor.TensorType(dtype=dtype, broadcastable=(False,) * ndim)) return op(random_state, size, n, p) def random_integers_helper(random_state, low, high, size): """ Helper function to draw random integers. This is a generalization of numpy.random.random_integers to the case where low and high are tensors. Since random_integers is deprecated it calls randint() instead. """ # Figure out the output shape if size is not None: out_ndim = len(size) else: out_ndim = max(low.ndim, high.ndim) # broadcast low and high to out_ndim dimensions if low.ndim > out_ndim: raise ValueError( 'low.ndim (%i) should not be larger than len(size) (%i)' % (low.ndim, out_ndim), low, size) if low.ndim < out_ndim: low = low.reshape((1,) * (out_ndim - low.ndim) + low.shape) if high.ndim > out_ndim: raise ValueError( 'high.ndim (%i) should not be larger than len(size) (%i)' % (high.ndim, out_ndim), high, size) if high.ndim < out_ndim: high = high.reshape((1,) * (out_ndim - high.ndim) + high.shape) if size is not None: out_size = tuple(size) else: out_size = () for dim in xrange(out_ndim): dim_len = max(low.shape[dim], high.shape[dim]) out_size = out_size + (dim_len,) # Build the indices over which to loop out = np.ndarray(out_size) broadcast_ind = _generate_broadcasting_indices(out_size, low.shape, high.shape) # Iterate over these indices, drawing one sample at a time from numpy for oi, li, hi in zip(*broadcast_ind): out[oi] = random_state.randint(low=low[li], high=high[hi] + 1) return out def random_integers(random_state, size=None, low=0, high=1, ndim=None, dtype='int64'): """ Sample a random integer between low and high, both inclusive. If the size argument is ambiguous on the number of dimensions, ndim may be a plain integer to supplement the missing information. If size is None, the output shape will be determined by the shapes of low and high. """ low = tensor.as_tensor_variable(low) high = tensor.as_tensor_variable(high) ndim, size, bcast = _infer_ndim_bcast(ndim, size, low, high) op = RandomFunction(random_integers_helper, tensor.TensorType(dtype=dtype, broadcastable=bcast)) return op(random_state, size, low, high) def choice_helper(random_state, a, replace, p, size): """ Helper function to draw random numbers using numpy's choice function. This is a generalization of numpy.random.choice that coerces `replace` to a bool and replaces `p` with None when p is a vector of 0 elements. """ if a.ndim > 1: raise ValueError('a.ndim (%i) must be 0 or 1' % a.ndim) if p.ndim == 1: if p.size == 0: p = None else: raise ValueError('p.ndim (%i) must be 1' % p.ndim) replace = bool(replace) return random_state.choice(a, size, replace, p) def choice(random_state, size=None, a=2, replace=True, p=None, ndim=None, dtype='int64'): """ Choose values from `a` with or without replacement. `a` can be a 1-D array or a positive scalar. If `a` is a scalar, the samples are drawn from the range 0,...,a-1. If the size argument is ambiguous on the number of dimensions, ndim may be a plain integer to supplement the missing information. If size is None, a scalar will be returned. """ a = tensor.as_tensor_variable(a) if isinstance(replace, bool): replace = tensor.constant(replace, dtype='int8') else: replace = tensor.as_tensor_variable(replace) # encode p=None as an empty vector p = tensor.as_tensor_variable(p or []) ndim, size, bcast = _infer_ndim_bcast(ndim, size) op = RandomFunction(choice_helper, tensor.TensorType(dtype=dtype, broadcastable=bcast)) return op(random_state, size, a, replace, p) def poisson(random_state, size=None, lam=1.0, ndim=None, dtype='int64'): """ Draw samples from a Poisson distribution. The Poisson distribution is the limit of the Binomial distribution for large N. Parameters ---------- lam : float or ndarray-like of the same shape as size parameter Expectation of interval, should be >= 0. size: int or tuple of ints, optional Output shape. If the given shape is, e.g., (m, n, k), then m * n * k samples are drawn. dtype The dtype of the return value (which will represent counts). size or ndim must be given. """ lam = tensor.as_tensor_variable(lam) ndim, size, bcast = _infer_ndim_bcast(ndim, size) op = RandomFunction("poisson", tensor.TensorType(dtype=dtype, broadcastable=bcast)) return op(random_state, size, lam) def permutation_helper(random_state, n, shape): """ Helper function to generate permutations from integers. permutation_helper(random_state, n, (1,)) will generate a permutation of integers 0..n-1. In general, it will generate as many such permutation as required by shape. For instance, if shape=(p,q), p*q permutations will be generated, and the output shape will be (p,q,n), because each permutation is of size n. If you wish to perform a permutation of the elements of an existing vector, see shuffle_row_elements. This is a generalization of numpy.random.permutation to tensors. Otherwise it behaves the same. """ # n should be a 0-dimension array assert n.shape == () # Note that it is important to convert `n` into an integer, because if it # is a long, the numpy permutation function will crash on Windows. n = int(n.item()) if shape is None: # Draw only one permutation, equivalent to shape = () shape = () out_shape = list(shape) out_shape.append(n) out = np.empty(out_shape, int) for i in np.ndindex(*shape): out[i] = random_state.permutation(n) # print 'RETURNING', out.shape return out def permutation(random_state, size=None, n=1, ndim=None, dtype='int64'): """ Return permutations of the integers between 0 and n-1. Returns them as many times as required by size. For instance, if size=(p,q), p*q permutations will be generated, and the output shape will be (p,q,n), because each permutation is of size n. Theano tries to infer the number of dimensions from the length of the size argument and the shape of n, but you may always specify it with the `ndim` parameter. Notes ----- Note that the output will then be of dimension ndim+1. """ if size is None or size == (): if not(ndim is None or ndim == 1): raise TypeError( "You asked for just one permutation but asked for more then 1 dimensions.") ndim = 1 size = () bcast = () else: ndim, size, bcast = _infer_ndim_bcast(ndim, size) # print "NDIM", ndim, size op = RandomFunction(permutation_helper, tensor.TensorType(dtype=dtype, broadcastable=bcast + (False,)), ndim_added=1) return op(random_state, size, n) def multinomial_helper(random_state, n, pvals, size): """ Helper function drawing from multinomial distributions. This is a generalization of numpy.random.multinomial to the case where n and pvals are tensors. """ # Figure out the shape if it's None # Note: the output ndim will be ndim+1, because the multinomial # adds a dimension. The length of that dimension is pvals.shape[-1]. if size is not None: ndim = len(size) else: ndim = max(n.ndim, pvals.ndim - 1) # broadcast n to ndim dimensions and pvals to ndim+1 if n.ndim > ndim: raise ValueError('n.ndim (%i) should not be larger than len(size) (%i)' % (n.ndim, ndim), n, size) if n.ndim < ndim: n = n.reshape((1,) * (ndim - n.ndim) + n.shape) if pvals.ndim - 1 > ndim: raise ValueError( 'pvals.ndim-1 (%i) should not be larger than len(size) (%i)' % (pvals.ndim - 1, ndim), pvals, size) if pvals.ndim - 1 < ndim: pvals = pvals.reshape((1,) * (ndim - pvals.ndim + 1) + pvals.shape) if size is not None: size = tuple(size) else: size = () for dim in xrange(ndim): dim_len = max(n.shape[dim], pvals.shape[dim]) size = size + (dim_len,) out_size = size + (pvals.shape[-1],) # Build the indices over which to loop # Note that here, the rows (inner-most 1D subtensors) of pvals and out # are indexed, not their individual elements out = np.ndarray(out_size) broadcast_ind = _generate_broadcasting_indices(size, n.shape, pvals.shape[:-1]) # Iterate over these indices, drawing from one multinomial at a # time from numpy assert pvals.min() >= 0 for mi, ni, pi in zip(*broadcast_ind): pvi = pvals[pi] # This might someday be fixed upstream # Currently numpy raises an exception in this method if the sum # of probabilities meets or exceeds 1.0. # In perfect arithmetic this would be correct, but in float32 or # float64 it is too strict. pisum = np.sum(pvi) if 1.0 < pisum < 1.0 + 1e-5: # correct if we went a little over # because mtrand.pyx has a ValueError that will trigger if # sum(pvals[:-1]) > 1.0 pvi = pvi * (1.0 - 5e-5) # pvi = pvi * .9 pisum = np.sum(pvi) elif pvi[-1] < 5e-5: # will this even work? pvi = pvi * (1.0 - 5e-5) pisum = np.sum(pvi) assert pisum <= 1.0, pisum out[mi] = random_state.multinomial(n=n[ni], pvals=pvi.astype('float64')) return out def multinomial(random_state, size=None, n=1, pvals=[0.5, 0.5], ndim=None, dtype='int64'): """ Sample from one or more multinomial distributions defined by one-dimensional slices in pvals. Parameters ---------- pvals A tensor of shape "nmulti+(L,)" describing each multinomial distribution. This tensor must have the property that numpy.allclose(pvals.sum(axis=-1), 1) is true. size A vector of shape information for the output; this can also specify the "nmulti" part of pvals' shape. A -1 in the k'th position from the right means to borrow the k'th position from the right in nmulti. (See examples below.) Default ``None`` means size=nmulti. n The number of experiments to simulate for each multinomial. This can be a scalar, or tensor, it will be broadcasted to have shape "nmulti". dtype The dtype of the return value (which will represent counts) Returns ------- tensor Tensor of len(size)+1 dimensions, and shape[-1]==L, with the specified ``dtype``, with the experiment counts. See examples to understand the shape of the return value, which is derived from both size and pvals.shape. In return value rval, "numpy.allclose(rval.sum(axis=-1), n)" will be true. Extended Summary ---------------- For example, to simulate n experiments from each multinomial in a batch of size B: size=None, pvals.shape=(B,L) --> rval.shape=[B,L] rval[i,j] is the count of possibility j in the i'th distribution (row) in pvals. Using size: size=(1,-1), pvals.shape=(A,B,L) --> rval.shape=[1,B,L], and requires that A==1. rval[k,i,j] is the count of possibility j in the distribution specified by pvals[k,i]. Using size for broadcasting of pvals: size=(10, 1, -1), pvals.shape=(A, B, L) --> rval.shape=[10,1,B,L], and requires that A==1. rval[l,k,i,j] is the count of possibility j in the distribution specified by pvals[k,i], in the l'th of 10 draws. """ n = tensor.as_tensor_variable(n) pvals = tensor.as_tensor_variable(pvals) # until ellipsis is implemented (argh) tmp = pvals.T[0].T ndim, size, bcast = _infer_ndim_bcast(ndim, size, n, tmp) bcast = bcast + (pvals.type.broadcastable[-1],) op = RandomFunction(multinomial_helper, tensor.TensorType(dtype=dtype, broadcastable=bcast), ndim_added=1) return op(random_state, size, n, pvals) @gof.local_optimizer([RandomFunction]) def random_make_inplace(node): op = node.op if isinstance(op, RandomFunction) and not op.inplace: # Read op_fn from op.state, not from op.fn, since op.fn # may not be picklable. op_fn, op_outtype, op_inplace, op_ndim_added = op._props() new_op = RandomFunction(op_fn, op_outtype, inplace=True, ndim_added=op_ndim_added) return new_op.make_node(*node.inputs).outputs return False optdb.register('random_make_inplace', opt.in2out(random_make_inplace, ignore_newtrees=True), 99, 'fast_run', 'inplace') class RandomStreamsBase(object): def binomial(self, size=None, n=1, p=0.5, ndim=None, dtype='int64', prob=None): """ Sample n times with probability of success p for each trial and return the number of successes. If the size argument is ambiguous on the number of dimensions, ndim may be a plain integer to supplement the missing information. """ if prob is not None: p = prob print("DEPRECATION WARNING: the parameter prob to the binomal fct have been renamed to p to have the same name as numpy.", file=sys.stderr) return self.gen(binomial, size, n, p, ndim=ndim, dtype=dtype) def uniform(self, size=None, low=0.0, high=1.0, ndim=None, dtype=None): """ Sample a tensor of given size whose element from a uniform distribution between low and high. If the size argument is ambiguous on the number of dimensions, ndim may be a plain integer to supplement the missing information. """ return self.gen(uniform, size, low, high, ndim=ndim, dtype=dtype) def normal(self, size=None, avg=0.0, std=1.0, ndim=None, dtype=None): """ Sample from a normal distribution centered on avg with the specified standard deviation (std). If the size argument is ambiguous on the number of dimensions, ndim may be a plain integer to supplement the missing information. """ return self.gen(normal, size, avg, std, ndim=ndim, dtype=dtype) def random_integers(self, size=None, low=0, high=1, ndim=None, dtype='int64'): """ Sample a random integer between low and high, both inclusive. If the size argument is ambiguous on the number of dimensions, ndim may be a plain integer to supplement the missing information. """ return self.gen(random_integers, size, low, high, ndim=ndim, dtype=dtype) def choice(self, size=None, a=2, replace=True, p=None, ndim=None, dtype='int64'): """ Choose values from `a` with or without replacement. `a` can be a 1-D array or a positive scalar. If `a` is a scalar, the samples are drawn from the range 0,...,a-1. If the size argument is ambiguous on the number of dimensions, ndim may be a plain integer to supplement the missing information. """ return self.gen(choice, size, a, replace, p, ndim=ndim, dtype=dtype) def poisson(self, size=None, lam=None, ndim=None, dtype='int64'): """ Draw samples from a Poisson distribution. The Poisson distribution is the limit of the Binomial distribution for large N. If the size argument is ambiguous on the number of dimensions, ndim may be a plain integer to supplement the missing information. """ return self.gen(poisson, size, lam, ndim=ndim, dtype=dtype) def permutation(self, size=None, n=1, ndim=None, dtype='int64'): """ Return permutations of the integers between 0 and n-1. Returns them as many times as required by size. For instance, if size=(p,q), p*q permutations will be generated, and the output shape will be (p,q,n), because each permutation is of size n. Theano tries to infer the number of dimensions from the length of the size argument and the shape of n, but you may always specify it with the `ndim` parameter. Notes ----- Note that the output will then be of dimension ndim+1. """ return self.gen(permutation, size, n, ndim=ndim, dtype=dtype) def multinomial(self, size=None, n=1, pvals=[0.5, 0.5], ndim=None, dtype='int64'): """ Sample n times from a multinomial distribution defined by probabilities pvals, as many times as required by size. For instance, if size=(p,q), p*q samples will be drawn, and the output shape will be (p,q,len(pvals)). Theano tries to infer the number of dimensions from the length of the size argument and the shapes of n and pvals, but you may always specify it with the `ndim` parameter. Notes ----- Note that the output will then be of dimension ndim+1. """ return self.gen(multinomial, size, n, pvals, ndim=ndim, dtype=dtype) def shuffle_row_elements(self, input): """ Return a variable with every row (rightmost index) shuffled. This uses permutation random variable internally, available via the ``.permutation`` attribute of the return value. """ perm = self.permutation(size=input.shape[:-1], n=input.shape[-1], ndim=input.ndim - 1) shuffled = tensor.permute_row_elements(input, perm) shuffled.permutation = perm return shuffled Theano-1.0.0/theano/__init__.py00006000163562€ÿÿÿþ0000001656313203335531017670 0ustar boccoset00000000000000""" Theano is an optimizing compiler in Python, built to evaluate complicated expressions (especially matrix-valued ones) as quickly as possible. Theano compiles expression graphs (see :doc:`graph` ) that are built by Python code. The expressions in these graphs are called `Apply` nodes and the variables in these graphs are called `Variable` nodes. You compile a graph by calling `function`, which takes a graph, and returns a callable object. One of theano's most important features is that `function` can transform your graph before compiling it. It can replace simple expressions with faster or more numerically stable implementations. To learn more, check out: - Op List (:doc:`oplist`) The markup language used in the docstrings is ReStructured Text, which may be rendered with Sphinx. A rendered version is maintained at http://www.deeplearning.net/software/theano/library/ """ from __future__ import absolute_import, print_function, division __docformat__ = "restructuredtext en" # Set a default logger. It is important to do this before importing some other # theano code, since this code may want to log some messages. import logging import os import sys import warnings def has_handlers(logger): # copied from Logger.hasHandlers() (introduced in Python 3.2) _logger = logger _has_handler = False while _logger: if _logger.handlers: _has_handler = True break if not _logger.propagate: break else: _logger = _logger.parent return _has_handler theano_logger = logging.getLogger("theano") logging_default_handler = logging.StreamHandler() logging_default_formatter = logging.Formatter( fmt='%(levelname)s (%(name)s): %(message)s') logging_default_handler.setFormatter(logging_default_formatter) theano_logger.setLevel(logging.WARNING) if has_handlers(theano_logger) is False: theano_logger.addHandler(logging_default_handler) # Disable default log handler added to theano_logger when the module # is imported. def disable_log_handler(logger=theano_logger, handler=logging_default_handler): if has_handlers(logger): logger.removeHandler(handler) # Version information. from theano.version import version as __version__ # Raise a meaning full warning/error if the theano directory is in the # Python path. from six import PY3 rpath = os.path.realpath(__path__[0]) for p in sys.path: if os.path.realpath(p) != rpath: continue if PY3: raise RuntimeError( "You have the theano directory in your Python path." " This do not work in Python 3.") else: warnings.warn( "You have the theano directory in your Python path." " This is will not work in Python 3.") break from theano.configdefaults import config from theano.configparser import change_flags # This is the api version for ops that generate C code. External ops # might need manual changes if this number goes up. An undefined # __api_version__ can be understood to mean api version 0. # # This number is not tied to the release version and should change # very rarely. __api_version__ = 1 from theano.gof import ( CLinker, OpWiseCLinker, DualLinker, Linker, LocalLinker, PerformLinker, Container, InconsistencyError, FunctionGraph, Apply, Variable, Constant, Op, OpenMPOp, opt, toolbox, Type, Generic, generic, object2, utils) from theano.compile import ( SymbolicInput, In, SymbolicOutput, Out, Mode, predefined_modes, predefined_linkers, predefined_optimizers, FunctionMaker, function, function_dump, OpFromGraph, ProfileStats, Param, shared, as_op) from theano.misc.safe_asarray import _asarray from theano.printing import pprint, pp from theano.scan_module import (scan, map, reduce, foldl, foldr, clone, scan_checkpoints) from theano.updates import OrderedUpdates # scan_module import above initializes tensor and scalar making these imports # redundant # import tensor # import scalar # we don't import by default as we don't want to force having scipy installed. # import sparse from theano.gradient import Rop, Lop, grad, subgraph_grad # This need to be before the init of GPU, as it add config variable # needed during that phase. import theano.tests if hasattr(theano.tests, "TheanoNoseTester"): test = theano.tests.TheanoNoseTester().test else: def test(): raise ImportError("The nose module is not installed." " It is needed for Theano tests.") if (config.device.startswith('cuda') or config.device.startswith('opencl') or config.init_gpu_device.startswith('cuda') or config.init_gpu_device.startswith('opencl') or config.contexts != ''): import theano.gpuarray # Use config.numpy to call numpy.seterr import numpy as np if config.numpy.seterr_all == 'None': _all = None else: _all = config.numpy.seterr_all if config.numpy.seterr_divide == 'None': _divide = None else: _divide = config.numpy.seterr_divide if config.numpy.seterr_over == 'None': _over = None else: _over = config.numpy.seterr_over if config.numpy.seterr_under == 'None': _under = None else: _under = config.numpy.seterr_under if config.numpy.seterr_invalid == 'None': _invalid = None else: _invalid = config.numpy.seterr_invalid np.seterr( all=_all, divide=_divide, over=_over, under=_under, invalid=_invalid) del _all, _divide, _over, _under, _invalid # This is defined here because it is designed to work across symbolic # datatypes (Sparse and Tensor) def dot(l, r): """Return a symbolic matrix/dot product between l and r """ rval = NotImplemented e0, e1 = None, None if rval == NotImplemented and hasattr(l, '__dot__'): try: rval = l.__dot__(r) except Exception as e0: rval = NotImplemented if rval == NotImplemented and hasattr(r, '__rdot__'): try: rval = r.__rdot__(l) except Exception as e1: rval = NotImplemented if rval == NotImplemented: raise NotImplementedError("Dot failed for the following reasons:", (e0, e1)) return rval def get_scalar_constant_value(v): """return the constant scalar(0-D) value underlying variable `v` If v is the output of dimshuffles, fills, allocs, rebroadcasts, cast this function digs through them. If theano.sparse is also there, we will look over CSM op. If `v` is not some view of constant data, then raise a tensor.basic.NotScalarConstantError. """ # Is it necessary to test for presence of theano.sparse at runtime? if 'sparse' in globals() and isinstance(v.type, sparse.SparseType): if v.owner is not None and isinstance(v.owner.op, sparse.CSM): data = v.owner.inputs[0] return tensor.get_scalar_constant_value(data) return tensor.get_scalar_constant_value(v) def sparse_grad(var): """This function return a new variable whose gradient will be stored in a sparse format instead of dense. Currently only variable created by AdvancedSubtensor1 is supported. i.e. a_tensor_var[an_int_vector]. .. versionadded:: 0.6rc4 """ assert isinstance(var.owner.op, tensor.AdvancedSubtensor1) ret = var.owner.op.__class__(sparse_grad=True)(*var.owner.inputs) return ret __import__('theano.tensor.shared_randomstreams') Theano-1.0.0/theano/typed_list/00007500163562€ÿÿÿþ0000000000013203336031017723 5ustar boccoset00000000000000Theano-1.0.0/theano/typed_list/type.py00006000163562€ÿÿÿþ0000000753713203335532021274 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division from theano import gof class TypedListType(gof.Type): """ Parameters ---------- ttype Type of theano variable this list will contains, can be another list. depth Optionnal parameters, any value above 0 will create a nested list of this depth. (0-based) """ def __init__(self, ttype, depth=0): if depth < 0: raise ValueError('Please specify a depth superior or' 'equal to 0') if not isinstance(ttype, gof.Type): raise TypeError('Expected a Theano Type') if depth == 0: self.ttype = ttype else: self.ttype = TypedListType(ttype, depth - 1) def filter(self, x, strict=False, allow_downcast=None): """ Parameters ---------- x Value to filter. strict If true, only native python list will be accepted. allow_downcast Does not have any utility at the moment. """ if strict: if not isinstance(x, list): raise TypeError('Expected a python list') else: x = [self.ttype.filter(y) for y in x] if all(self.ttype.is_valid_value(y) for y in x): return x else: raise TypeError('Expected all elements to' ' be %s' % str(self.ttype)) def __eq__(self, other): """ Two lists are equal if they contain the same type. """ return type(self) == type(other) and self.ttype == other.ttype def __hash__(self): return gof.hashtype(self) ^ hash(self.ttype) def __str__(self): return 'TypedList <' + str(self.ttype) + '>' def get_depth(self): """ Utilitary function to get the 0 based level of the list. """ if isinstance(self.ttype, TypedListType): return self.ttype.get_depth() + 1 else: return 0 def values_eq(self, a, b): if not len(a) == len(b): return False for x in range(len(a)): if not self.ttype.values_eq(a[x], b[x]): return False return True def may_share_memory(self, a, b): if a is b: return True # As a list contain other element, if a or b isn't a list, we # still need to check if that element is contained in the # other list. if not isinstance(a, list): a = [a] if not isinstance(b, list): b = [b] for idx1 in range(len(a)): for idx2 in range(len(b)): if self.ttype.may_share_memory(a[idx1], b[idx2]): return True def c_declare(self, name, sub, check_input=True): return """ PyListObject* %(name)s; """ % dict(name=name) def c_init(self, name, sub): return """ %(name)s = NULL; """ % dict(name=name) def c_extract(self, name, sub, check_input=True): if check_input: pre = """ if (!PyList_Check(py_%(name)s)) { PyErr_SetString(PyExc_TypeError, "expected a list"); %(fail)s }""" % dict(name=name, fail=sub['fail']) else: pre = "" return pre + """ %(name)s = (PyListObject*) (py_%(name)s); """ % dict(name=name, fail=sub['fail']) def c_sync(self, name, sub): return """ Py_XDECREF(py_%(name)s); py_%(name)s = (PyObject*)(%(name)s); Py_INCREF(py_%(name)s); """ % dict(name=name) def c_cleanup(self, name, sub): return "" def c_code_cache_version(self): return (2,) dtype = property(lambda self: self.ttype) ndim = property(lambda self: self.ttype.ndim + 1) Theano-1.0.0/theano/typed_list/opt.py00006000163562€ÿÿÿþ0000000146613203335532021110 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division from theano import gof from theano import compile from theano.gof import TopoOptimizer from theano.typed_list.basic import Reverse, Append, Extend, Insert, Remove @gof.local_optimizer([Append, Extend, Insert, Reverse, Remove], inplace=True) def typed_list_inplace_opt(node): if (isinstance(node.op, (Append, Extend, Insert, Reverse, Remove)) and not node.op.inplace): new_op = node.op.__class__(inplace=True) new_node = new_op(*node.inputs) return [new_node] return False compile.optdb.register('typed_list_inplace_opt', TopoOptimizer(typed_list_inplace_opt, failure_callback=TopoOptimizer.warn_inplace), 60, 'fast_run', 'inplace') Theano-1.0.0/theano/typed_list/tests/00007500163562€ÿÿÿþ0000000000013203336031021065 5ustar boccoset00000000000000Theano-1.0.0/theano/typed_list/tests/test_opt.py00006000163562€ÿÿÿþ0000001211013203335532023275 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import unittest import numpy as np import theano import theano.typed_list from theano import tensor as T from theano.typed_list.type import TypedListType from theano.typed_list.basic import (Insert, Append, Extend, Remove, Reverse) from theano import In # took from tensors/tests/test_basic.py def rand_ranged_matrix(minimum, maximum, shape): return np.asarray(np.random.rand(*shape) * (maximum - minimum) + minimum, dtype=theano.config.floatX) class test_inplace(unittest.TestCase): def test_reverse_inplace(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() z = Reverse()(mySymbolicMatricesList) m = theano.compile.mode.get_default_mode().including("typed_list_inplace_opt") f = theano.function([In(mySymbolicMatricesList, borrow=True, mutable=True)], z, accept_inplace=True, mode=m) self.assertTrue(f.maker.fgraph.toposort()[0].op.inplace) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x, y]), [y, x])) def test_append_inplace(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() mySymbolicMatrix = T.matrix() z = Append()(mySymbolicMatricesList, mySymbolicMatrix) m = theano.compile.mode.get_default_mode().including("typed_list_inplace_opt") f = theano.function([In(mySymbolicMatricesList, borrow=True, mutable=True), In(mySymbolicMatrix, borrow=True, mutable=True)], z, accept_inplace=True, mode=m) self.assertTrue(f.maker.fgraph.toposort()[0].op.inplace) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x], y), [x, y])) def test_extend_inplace(self): mySymbolicMatricesList1 = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() mySymbolicMatricesList2 = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() z = Extend()(mySymbolicMatricesList1, mySymbolicMatricesList2) m = theano.compile.mode.get_default_mode().including("typed_list_inplace_opt") f = theano.function([In(mySymbolicMatricesList1, borrow=True, mutable=True), mySymbolicMatricesList2], z, mode=m) self.assertTrue(f.maker.fgraph.toposort()[0].op.inplace) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x], [y]), [x, y])) def test_insert_inplace(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() mySymbolicIndex = T.scalar(dtype='int64') mySymbolicMatrix = T.matrix() z = Insert()(mySymbolicMatricesList, mySymbolicIndex, mySymbolicMatrix) m = theano.compile.mode.get_default_mode().including("typed_list_inplace_opt") f = theano.function([In(mySymbolicMatricesList, borrow=True, mutable=True), mySymbolicIndex, mySymbolicMatrix], z, accept_inplace=True, mode=m) self.assertTrue(f.maker.fgraph.toposort()[0].op.inplace) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x], np.asarray(1, dtype='int64'), y), [x, y])) def test_remove_inplace(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() mySymbolicMatrix = T.matrix() z = Remove()(mySymbolicMatricesList, mySymbolicMatrix) m = theano.compile.mode.get_default_mode().including("typed_list_inplace_opt") f = theano.function([In(mySymbolicMatricesList, borrow=True, mutable=True), In(mySymbolicMatrix, borrow=True, mutable=True)], z, accept_inplace=True, mode=m) self.assertTrue(f.maker.fgraph.toposort()[0].op.inplace) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x, y], y), [x])) def test_constant_folding(): m = theano.tensor.ones((1,), dtype='int8') l = theano.typed_list.make_list([m, m]) f = theano.function([], l) topo = f.maker.fgraph.toposort() assert len(topo) assert isinstance(topo[0].op, theano.compile.ops.DeepCopyOp) assert f() == [1, 1] Theano-1.0.0/theano/typed_list/tests/__init__.py00006000163562€ÿÿÿþ0000000000013203335532023166 0ustar boccoset00000000000000Theano-1.0.0/theano/typed_list/tests/test_basic.py00006000163562€ÿÿÿþ0000004472413203335532023574 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import unittest from nose.plugins.skip import SkipTest import numpy as np import theano import theano.typed_list from theano import tensor as T from theano.tensor.type_other import SliceType from theano.typed_list.type import TypedListType from theano.typed_list.basic import (GetItem, Insert, Append, Extend, Remove, Reverse, Index, Count, Length, make_list) from theano import sparse from theano.tests import unittest_tools as utt # TODO, handle the case where scipy isn't installed. try: import scipy.sparse as sp scipy_imported = True except ImportError: scipy_imported = False # took from tensors/tests/test_basic.py def rand_ranged_matrix(minimum, maximum, shape): return np.asarray(np.random.rand(*shape) * (maximum - minimum) + minimum, dtype=theano.config.floatX) # took from sparse/tests/test_basic.py def random_lil(shape, dtype, nnz): rval = sp.lil_matrix(shape, dtype=dtype) huge = 2 ** 30 for k in range(nnz): # set non-zeros in random locations (row x, col y) idx = np.random.randint(1, huge + 1, size=2) % shape value = np.random.rand() # if dtype *int*, value will always be zeros! if dtype in theano.tensor.integer_dtypes: value = int(value * 100) # The call to tuple is needed as scipy 0.13.1 do not support # ndarray with lenght 2 as idx tuple. rval.__setitem__( tuple(idx), value) return rval class test_get_item(unittest.TestCase): def setUp(self): utt.seed_rng() def test_sanity_check_slice(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() mySymbolicSlice = SliceType()() z = GetItem()(mySymbolicMatricesList, mySymbolicSlice) self.assertFalse(isinstance(z, T.TensorVariable)) f = theano.function([mySymbolicMatricesList, mySymbolicSlice], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x], slice(0, 1, 1)), [x])) def test_sanity_check_single(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() mySymbolicScalar = T.scalar(dtype='int64') z = GetItem()(mySymbolicMatricesList, mySymbolicScalar) f = theano.function([mySymbolicMatricesList, mySymbolicScalar], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x], np.asarray(0, dtype='int64')), x)) def test_interface(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() mySymbolicScalar = T.scalar(dtype='int64') z = mySymbolicMatricesList[mySymbolicScalar] f = theano.function([mySymbolicMatricesList, mySymbolicScalar], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x], np.asarray(0, dtype='int64')), x)) z = mySymbolicMatricesList[0] f = theano.function([mySymbolicMatricesList], z) self.assertTrue(np.array_equal(f([x]), x)) def test_wrong_input(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() mySymbolicMatrix = T.matrix() self.assertRaises(TypeError, GetItem(), mySymbolicMatricesList, mySymbolicMatrix) def test_constant_input(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() z = GetItem()(mySymbolicMatricesList, 0) f = theano.function([mySymbolicMatricesList], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x]), x)) z = GetItem()(mySymbolicMatricesList, slice(0, 1, 1)) f = theano.function([mySymbolicMatricesList], z) self.assertTrue(np.array_equal(f([x]), [x])) class test_append(unittest.TestCase): def test_inplace(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() myMatrix = T.matrix() z = Append(True)(mySymbolicMatricesList, myMatrix) f = theano.function([mySymbolicMatricesList, myMatrix], z, accept_inplace=True) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x], y), [x, y])) def test_sanity_check(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() myMatrix = T.matrix() z = Append()(mySymbolicMatricesList, myMatrix) f = theano.function([mySymbolicMatricesList, myMatrix], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x], y), [x, y])) def test_interfaces(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() myMatrix = T.matrix() z = mySymbolicMatricesList.append(myMatrix) f = theano.function([mySymbolicMatricesList, myMatrix], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x], y), [x, y])) class test_extend(unittest.TestCase): def test_inplace(self): mySymbolicMatricesList1 = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() mySymbolicMatricesList2 = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() z = Extend(True)(mySymbolicMatricesList1, mySymbolicMatricesList2) f = theano.function([mySymbolicMatricesList1, mySymbolicMatricesList2], z, accept_inplace=True) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x], [y]), [x, y])) def test_sanity_check(self): mySymbolicMatricesList1 = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() mySymbolicMatricesList2 = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() z = Extend()(mySymbolicMatricesList1, mySymbolicMatricesList2) f = theano.function([mySymbolicMatricesList1, mySymbolicMatricesList2], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x], [y]), [x, y])) def test_interface(self): mySymbolicMatricesList1 = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() mySymbolicMatricesList2 = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() z = mySymbolicMatricesList1.extend(mySymbolicMatricesList2) f = theano.function([mySymbolicMatricesList1, mySymbolicMatricesList2], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x], [y]), [x, y])) class test_insert(unittest.TestCase): def test_inplace(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() myMatrix = T.matrix() myScalar = T.scalar(dtype='int64') z = Insert(True)(mySymbolicMatricesList, myScalar, myMatrix) f = theano.function([mySymbolicMatricesList, myScalar, myMatrix], z, accept_inplace=True) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x], np.asarray(1, dtype='int64'), y), [x, y])) def test_sanity_check(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() myMatrix = T.matrix() myScalar = T.scalar(dtype='int64') z = Insert()(mySymbolicMatricesList, myScalar, myMatrix) f = theano.function([mySymbolicMatricesList, myScalar, myMatrix], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x], np.asarray(1, dtype='int64'), y), [x, y])) def test_interface(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() myMatrix = T.matrix() myScalar = T.scalar(dtype='int64') z = mySymbolicMatricesList.insert(myScalar, myMatrix) f = theano.function([mySymbolicMatricesList, myScalar, myMatrix], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x], np.asarray(1, dtype='int64'), y), [x, y])) class test_remove(unittest.TestCase): def test_inplace(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() myMatrix = T.matrix() z = Remove(True)(mySymbolicMatricesList, myMatrix) f = theano.function([mySymbolicMatricesList, myMatrix], z, accept_inplace=True) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x, y], y), [x])) def test_sanity_check(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() myMatrix = T.matrix() z = Remove()(mySymbolicMatricesList, myMatrix) f = theano.function([mySymbolicMatricesList, myMatrix], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x, y], y), [x])) def test_interface(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() myMatrix = T.matrix() z = mySymbolicMatricesList.remove(myMatrix) f = theano.function([mySymbolicMatricesList, myMatrix], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x, y], y), [x])) class test_reverse(unittest.TestCase): def test_inplace(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() z = Reverse(True)(mySymbolicMatricesList) f = theano.function([mySymbolicMatricesList], z, accept_inplace=True) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x, y]), [y, x])) def test_sanity_check(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() z = Reverse()(mySymbolicMatricesList) f = theano.function([mySymbolicMatricesList], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x, y]), [y, x])) def test_interface(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() z = mySymbolicMatricesList.reverse() f = theano.function([mySymbolicMatricesList], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(np.array_equal(f([x, y]), [y, x])) class test_index(unittest.TestCase): def test_sanity_check(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() myMatrix = T.matrix() z = Index()(mySymbolicMatricesList, myMatrix) f = theano.function([mySymbolicMatricesList, myMatrix], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(f([x, y], y) == 1) def test_interface(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() myMatrix = T.matrix() z = mySymbolicMatricesList.ind(myMatrix) f = theano.function([mySymbolicMatricesList, myMatrix], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(f([x, y], y) == 1) def test_non_tensor_type(self): mySymbolicNestedMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)), 1)() mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() z = Index()(mySymbolicNestedMatricesList, mySymbolicMatricesList) f = theano.function([mySymbolicNestedMatricesList, mySymbolicMatricesList], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(f([[x, y], [x, y, y]], [x, y]) == 0) def test_sparse(self): if not scipy_imported: raise SkipTest('Optional package SciPy not installed') mySymbolicSparseList = TypedListType( sparse.SparseType('csr', theano.config.floatX))() mySymbolicSparse = sparse.csr_matrix() z = Index()(mySymbolicSparseList, mySymbolicSparse) f = theano.function([mySymbolicSparseList, mySymbolicSparse], z) x = sp.csr_matrix(random_lil((10, 40), theano.config.floatX, 3)) y = sp.csr_matrix(random_lil((10, 40), theano.config.floatX, 3)) self.assertTrue(f([x, y], y) == 1) class test_count(unittest.TestCase): def test_sanity_check(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() myMatrix = T.matrix() z = Count()(mySymbolicMatricesList, myMatrix) f = theano.function([mySymbolicMatricesList, myMatrix], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(f([y, y, x, y], y) == 3) def test_interface(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() myMatrix = T.matrix() z = mySymbolicMatricesList.count(myMatrix) f = theano.function([mySymbolicMatricesList, myMatrix], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(f([x, y], y) == 1) def test_non_tensor_type(self): mySymbolicNestedMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)), 1)() mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() z = Count()(mySymbolicNestedMatricesList, mySymbolicMatricesList) f = theano.function([mySymbolicNestedMatricesList, mySymbolicMatricesList], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) y = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(f([[x, y], [x, y, y]], [x, y]) == 1) def test_sparse(self): if not scipy_imported: raise SkipTest('Optional package SciPy not installed') mySymbolicSparseList = TypedListType( sparse.SparseType('csr', theano.config.floatX))() mySymbolicSparse = sparse.csr_matrix() z = Count()(mySymbolicSparseList, mySymbolicSparse) f = theano.function([mySymbolicSparseList, mySymbolicSparse], z) x = sp.csr_matrix(random_lil((10, 40), theano.config.floatX, 3)) y = sp.csr_matrix(random_lil((10, 40), theano.config.floatX, 3)) self.assertTrue(f([x, y, y], y) == 2) class test_length(unittest.TestCase): def test_sanity_check(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() z = Length()(mySymbolicMatricesList) f = theano.function([mySymbolicMatricesList], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(f([x, x, x, x]) == 4) def test_interface(self): mySymbolicMatricesList = TypedListType(T.TensorType( theano.config.floatX, (False, False)))() z = mySymbolicMatricesList.__len__() f = theano.function([mySymbolicMatricesList], z) x = rand_ranged_matrix(-1000, 1000, [100, 101]) self.assertTrue(f([x, x]) == 2) class TestMakeList(unittest.TestCase): def test_wrong_shape(self): a = T.vector() b = T.matrix() self.assertRaises(TypeError, make_list, (a, b)) def test_correct_answer(self): a = T.matrix() b = T.matrix() x = T.tensor3() y = T.tensor3() A = np.cast[theano.config.floatX](np.random.rand(5, 3)) B = np.cast[theano.config.floatX](np.random.rand(7, 2)) X = np.cast[theano.config.floatX](np.random.rand(5, 6, 1)) Y = np.cast[theano.config.floatX](np.random.rand(1, 9, 3)) make_list((3., 4.)) c = make_list((a, b)) z = make_list((x, y)) fc = theano.function([a, b], c) fz = theano.function([x, y], z) self.assertTrue((m == n).all() for m, n in zip(fc(A, B), [A, B])) self.assertTrue((m == n).all() for m, n in zip(fz(X, Y), [X, Y])) Theano-1.0.0/theano/typed_list/tests/test_type.py00006000163562€ÿÿÿþ0000001377213203335532023473 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import unittest import numpy as np import theano import theano.typed_list from theano import tensor as T from theano.typed_list.type import TypedListType from theano.tests import unittest_tools as utt # took from tensors/tests/test_basic.py def rand_ranged_matrix(minimum, maximum, shape): return np.asarray(np.random.rand(*shape) * (maximum - minimum) + minimum, dtype=theano.config.floatX) class test_typed_list_type(unittest.TestCase): def setUp(self): utt.seed_rng() def test_wrong_input_on_creation(self): # Typed list type should raises an # error if the argument passed for # type is not a valid theano type self.assertRaises(TypeError, TypedListType, None) def test_wrong_input_on_filter(self): # Typed list type should raises an # error if the argument given to filter # isn't of the same type as the one # specified on creation # list of matrices myType = TypedListType(T.TensorType(theano.config.floatX, (False, False))) self.assertRaises(TypeError, myType.filter, [4]) def test_not_a_list_on_filter(self): # Typed List Value should raises an error # if no iterable variable is given on input # list of matrices myType = TypedListType(T.TensorType(theano.config.floatX, (False, False))) self.assertRaises(TypeError, myType.filter, 4) def test_type_equality(self): # Typed list types should only be equal # when they contains the same theano # variables # list of matrices myType1 = TypedListType(T.TensorType(theano.config.floatX, (False, False))) # list of matrices myType2 = TypedListType(T.TensorType(theano.config.floatX, (False, False))) # list of scalars myType3 = TypedListType(T.TensorType(theano.config.floatX, ())) self.assertTrue(myType2 == myType1) self.assertFalse(myType3 == myType1) def test_filter_sanity_check(self): # Simple test on typed list type filter myType = TypedListType(T.TensorType(theano.config.floatX, (False, False))) x = rand_ranged_matrix(-1000, 1000, [100, 100]) self.assertTrue(np.array_equal(myType.filter([x]), [x])) def test_intern_filter(self): # Test checking if values contained are themselves # filtered. If they weren't this code would raise # an exception. myType = TypedListType(T.TensorType('float64', (False, False))) x = np.asarray([[4, 5], [4, 5]], dtype='float32') self.assertTrue(np.array_equal(myType.filter([x]), [x])) # Will fail for unknown reasons # under search ''' def test_load(self): myType = TypedListType(T.TensorType(theano.config.floatX, (False, False))) x = rand_ranged_matrix(-1000, 1000, [100, 100]) testList = [] for i in range(10000): testList.append(x) self.assertTrue(numpy.array_equal(myType.filter(testList), testList)) ''' def test_basic_nested_list(self): # Testing nested list with one level of depth myNestedType = TypedListType(T.TensorType(theano.config.floatX, (False, False))) myType = TypedListType(myNestedType) x = rand_ranged_matrix(-1000, 1000, [100, 100]) self.assertTrue(np.array_equal(myType.filter([[x]]), [[x]])) def test_comparison_different_depth(self): # Nested list with different depth aren't the same myNestedType = TypedListType(T.TensorType(theano.config.floatX, (False, False))) myNestedType2 = TypedListType(myNestedType) myNestedType3 = TypedListType(myNestedType2) self.assertFalse(myNestedType2 == myNestedType3) def test_nested_list_arg(self): # test for the 'depth' optionnal argument myNestedType = TypedListType(T.TensorType(theano.config.floatX, (False, False)), 3) myType = TypedListType(T.TensorType(theano.config.floatX, (False, False))) myManualNestedType = TypedListType(TypedListType( TypedListType(myType))) self.assertTrue(myNestedType == myManualNestedType) def test_get_depth(self): # test case for get_depth utilitary function myType = TypedListType(T.TensorType(theano.config.floatX, (False, False))) myManualNestedType = TypedListType(TypedListType( TypedListType(myType))) self.assertTrue(myManualNestedType.get_depth() == 3) def test_comparison_uneven_nested(self): # test for comparison between uneven nested list myType = TypedListType(T.TensorType(theano.config.floatX, (False, False))) myManualNestedType1 = TypedListType(TypedListType( TypedListType(myType))) myManualNestedType2 = TypedListType(TypedListType(myType)) self.assertFalse(myManualNestedType1 == myManualNestedType2) self.assertFalse(myManualNestedType2 == myManualNestedType1) def test_variable_is_Typed_List_variable(self): mySymbolicVariable = TypedListType(T.TensorType(theano.config.floatX, (False, False)))() self.assertTrue(isinstance(mySymbolicVariable, theano.typed_list.TypedListVariable)) Theano-1.0.0/theano/typed_list/__init__.py00006000163562€ÿÿÿþ0000000021013203335532022027 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division from .type import TypedListType from .basic import * from . import opt Theano-1.0.0/theano/typed_list/basic.py00006000163562€ÿÿÿþ0000004252313203335532021366 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import numpy as np from .type import TypedListType import theano from theano.gof import Apply, Constant, Op, Variable from theano.tensor.type_other import SliceType from theano import tensor as T from theano.compile.debugmode import _lessbroken_deepcopy class _typed_list_py_operators: def __getitem__(self, index): return getitem(self, index) def __len__(self): return length(self) def append(self, toAppend): return append(self, toAppend) def extend(self, toAppend): return extend(self, toAppend) def insert(self, index, toInsert): return insert(self, index, toInsert) def remove(self, toRemove): return remove(self, toRemove) def reverse(self): return reverse(self) def count(self, elem): return count(self, elem) # name "index" is already used by an attribute def ind(self, elem): return index_(self, elem) ttype = property(lambda self: self.type.ttype) dtype = property(lambda self: self.type.ttype.dtype) ndim = property(lambda self: self.type.ttype.ndim + 1) class TypedListVariable(_typed_list_py_operators, Variable): """ Subclass to add the typed list operators to the basic `Variable` class. """ TypedListType.Variable = TypedListVariable class TypedListConstant(_typed_list_py_operators, Constant): """ Subclass to add the typed list operators to the basic `Variable` class. """ TypedListType.Constant = TypedListConstant class GetItem(Op): # See doc in instance of this Op or function after this class definition. view_map = {0: [0]} __props__ = () def make_node(self, x, index): assert isinstance(x.type, TypedListType) if not isinstance(index, Variable): if isinstance(index, slice): index = Constant(SliceType(), index) return Apply(self, [x, index], [x.type()]) else: index = T.constant(index, ndim=0, dtype='int64') return Apply(self, [x, index], [x.ttype()]) if isinstance(index.type, SliceType): return Apply(self, [x, index], [x.type()]) elif isinstance(index, T.TensorVariable) and index.ndim == 0: assert index.dtype == 'int64' return Apply(self, [x, index], [x.ttype()]) else: raise TypeError('Expected scalar or slice as index.') def perform(self, node, inputs, outputs): (x, index) = inputs (out,) = outputs if not isinstance(index, slice): index = int(index) out[0] = x[index] def __str__(self): return self.__class__.__name__ def c_code(self, node, name, inp, out, sub): x_name, index = inp[0], inp[1] output_name = out[0] fail = sub['fail'] return """ %(output_name)s = (typeof %(output_name)s) PyList_GetItem( (PyObject*) %(x_name)s, *((npy_int64 *) PyArray_DATA(%(index)s))); if(%(output_name)s == NULL){ %(fail)s } Py_INCREF(%(output_name)s); """ % locals() def c_code_cache_version(self): return (1,) getitem = GetItem() """ Get specified slice of a typed list. Parameters ---------- x Typed list. index The index of the value to return from `x`. """ class Append(Op): # See doc in instance of this Op after the class definition. __props__ = ("inplace",) def __init__(self, inplace=False): self.inplace = inplace if self.inplace: self.destroy_map = {0: [0]} # TODO: make destroy_handler support having views and # destroyed version of multiple inputs. # self.view_map = {0: [1]} else: # TODO: make destroy_handler support multiple view # self.view_map = {0: [0, 1]} self.view_map = {0: [0]} def make_node(self, x, toAppend): assert isinstance(x.type, TypedListType) assert x.ttype == toAppend.type, (x.ttype, toAppend.type) return Apply(self, [x, toAppend], [x.type()]) def perform(self, node, inputs, outputs): (x, toAppend) = inputs (out,) = outputs if not self.inplace: out[0] = list(x) else: out[0] = x # need to copy toAppend due to destroy_handler limitation toAppend = _lessbroken_deepcopy(toAppend) out[0].append(toAppend) def __str__(self): return self.__class__.__name__ # DISABLED AS WE NEED TO UPDATE IT TO COPY toAppend(). def _c_code_(self, node, name, inp, out, sub): x_name, toAppend = inp[0], inp[1] output_name = out[0] fail = sub['fail'] if not self.inplace: init = """ %(output_name)s = (PyListObject*) PyList_GetSlice((PyObject*) %(x_name)s, 0, PyList_GET_SIZE((PyObject*) %(x_name)s)) ; """ % locals() else: init = """ %(output_name)s = %(x_name)s; """ % locals() return init + """ if(%(output_name)s==NULL){ %(fail)s }; if(PyList_Append( (PyObject*) %(output_name)s,(PyObject*) %(toAppend)s)){ %(fail)s }; Py_INCREF(%(output_name)s); """ % locals() def c_code_cache_version(self): return (1,) append = Append() """ Append an element at the end of another list. Parameters ---------- x The base typed list. y The element to append to `x`. """ class Extend(Op): # See doc in instance of this Op after the class definition. __props__ = ("inplace",) def __init__(self, inplace=False): self.inplace = inplace if self.inplace: self.destroy_map = {0: [0]} # TODO: make destroy_handler support having views and # destroyed version of multiple inputs. # self.view_map = {0: [1]} else: # TODO: make destroy_handler support multiple view # self.view_map = {0: [0, 1]} self.view_map = {0: [0]} def make_node(self, x, toAppend): assert isinstance(x.type, TypedListType) assert x.type == toAppend.type return Apply(self, [x, toAppend], [x.type()]) def perform(self, node, inputs, outputs): (x, toAppend) = inputs (out,) = outputs if not self.inplace: out[0] = list(x) else: out[0] = x # need to copy toAppend due to destroy_handler limitation if toAppend: o = out[0] for i in toAppend: o.append(_lessbroken_deepcopy(i)) def __str__(self): return self.__class__.__name__ # DISABLED AS WE NEED TO UPDATE IT TO COPY toAppend(). def _c_code_(self, node, name, inp, out, sub): x_name, toAppend = inp[0], inp[1] output_name = out[0] fail = sub['fail'] if not self.inplace: init = """ %(output_name)s = (PyListObject*) PyList_GetSlice((PyObject*) %(x_name)s, 0, PyList_GET_SIZE((PyObject*) %(x_name)s)) ; """ % locals() else: init = """ %(output_name)s = %(x_name)s; """ % locals() return init + """ int i =0; int length = PyList_GET_SIZE((PyObject*) %(toAppend)s); if(%(output_name)s==NULL){ %(fail)s }; for(i; i < length; i++){ if(PyList_Append( (PyObject*) %(output_name)s,(PyObject*) PyList_GetItem((PyObject*) %(toAppend)s,i))==-1){ %(fail)s }; } Py_INCREF(%(output_name)s); """ % locals() def c_code_cache_version_(self): return (1,) extend = Extend() """ Append all elements of a list at the end of another list. Parameters ---------- x The typed list to extend. toAppend The typed list that will be added at the end of `x`. """ class Insert(Op): # See doc in instance of this Op after the class definition. __props__ = ("inplace",) def __init__(self, inplace=False): self.inplace = inplace if self.inplace: self.destroy_map = {0: [0]} # TODO: make destroy_handler support having views and # destroyed version of multiple inputs. # self.view_map = {0: [2]} else: # TODO: make destroy_handler support multiple view # self.view_map = {0: [0, 2]} self.view_map = {0: [0]} def make_node(self, x, index, toInsert): assert isinstance(x.type, TypedListType) assert x.ttype == toInsert.type if not isinstance(index, Variable): index = T.constant(index, ndim=0, dtype='int64') else: assert index.dtype == 'int64' assert isinstance(index, T.TensorVariable) and index.ndim == 0 return Apply(self, [x, index, toInsert], [x.type()]) def perform(self, node, inputs, outputs): (x, index, toInsert) = inputs (out,) = outputs if not self.inplace: out[0] = list(x) else: out[0] = x # need to copy toAppend due to destroy_handler limitation toInsert = _lessbroken_deepcopy(toInsert) out[0].insert(index, toInsert) def __str__(self): return self.__class__.__name__ # DISABLED AS WE NEED TO UPDATE IT TO COPY toAppend(). def _c_code_(self, node, name, inp, out, sub): x_name, index, toInsert = inp[0], inp[1], inp[2] output_name = out[0] fail = sub['fail'] if not self.inplace: init = """ %(output_name)s = (PyListObject*) PyList_GetSlice((PyObject*) %(x_name)s, 0, PyList_GET_SIZE((PyObject*) %(x_name)s)) ; """ % locals() else: init = """ %(output_name)s = %(x_name)s; """ % locals() return init + """ if(%(output_name)s==NULL){ %(fail)s }; if(PyList_Insert((PyObject*) %(output_name)s, *((npy_int64 *) PyArray_DATA(%(index)s)), (PyObject*) %(toInsert)s)==-1){ %(fail)s }; Py_INCREF(%(output_name)s); """ % locals() def c_code_cache_version(self): return (1,) insert = Insert() """ Insert an element at an index in a typed list. Parameters ---------- x The typed list to modify. index The index where to put the new element in `x`. toInsert The new element to insert. """ class Remove(Op): # See doc in instance of this Op after the class definition. __props__ = ("inplace",) def __init__(self, inplace=False): self.inplace = inplace if self.inplace: self.destroy_map = {0: [0]} else: self.view_map = {0: [0]} def make_node(self, x, toRemove): assert isinstance(x.type, TypedListType) assert x.ttype == toRemove.type return Apply(self, [x, toRemove], [x.type()]) def perform(self, node, inputs, outputs): (x, toRemove) = inputs (out,) = outputs if not self.inplace: out[0] = list(x) else: out[0] = x """ Inelegant workaround for ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all() being thrown when trying to remove a matrix from a matrices list. """ for y in range(out[0].__len__()): if node.inputs[0].ttype.values_eq(out[0][y], toRemove): del out[0][y] break def __str__(self): return self.__class__.__name__ remove = Remove() """Remove an element from a typed list. Parameters ---------- x The typed list to be changed. toRemove An element to be removed from the typed list. We only remove the first instance. Notes ----- Python implementation of remove doesn't work when we want to remove an ndarray from a list. This implementation works in that case. """ class Reverse(Op): # See doc in instance of this Op after the class definition. __props__ = ("inplace",) def __init__(self, inplace=False): self.inplace = inplace if self.inplace: self.destroy_map = {0: [0]} else: self.view_map = {0: [0]} def make_node(self, x): assert isinstance(x.type, TypedListType) return Apply(self, [x], [x.type()]) def perform(self, node, inp, outputs): (out,) = outputs if not self.inplace: out[0] = list(inp[0]) else: out[0] = inp[0] out[0].reverse() def __str__(self): return self.__class__.__name__ def c_code(self, node, name, inp, out, sub): x_name = inp[0] output_name = out[0] fail = sub['fail'] if not self.inplace: init = """ %(output_name)s = (PyListObject*) PyList_GetSlice((PyObject*) %(x_name)s, 0, PyList_GET_SIZE((PyObject*) %(x_name)s)) ; """ % locals() else: init = """ %(output_name)s = %(x_name)s; """ % locals() return init + """ if(%(output_name)s==NULL){ %(fail)s }; if(PyList_Reverse((PyObject*) %(output_name)s)==-1){ %(fail)s }; Py_INCREF(%(output_name)s); """ % locals() def c_code_cache_version(self): return (1,) reverse = Reverse() """ Reverse the order of a typed list. Parameters ---------- x The typed list to be reversed. """ class Index(Op): # See doc in instance of this Op after the class definition. __props__ = () def make_node(self, x, elem): assert isinstance(x.type, TypedListType) assert x.ttype == elem.type return Apply(self, [x, elem], [T.scalar()]) def perform(self, node, inputs, outputs): """ Inelegant workaround for ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all() being thrown when trying to remove a matrix from a matrices list """ (x, elem) = inputs (out,) = outputs for y in range(len(x)): if node.inputs[0].ttype.values_eq(x[y], elem): out[0] = np.asarray(y, dtype=theano.config.floatX) break def __str__(self): return self.__class__.__name__ index_ = Index() class Count(Op): # See doc in instance of this Op after the class definition. __props__ = () def make_node(self, x, elem): assert isinstance(x.type, TypedListType) assert x.ttype == elem.type return Apply(self, [x, elem], [T.scalar()]) def perform(self, node, inputs, outputs): """ Inelegant workaround for ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all() being thrown when trying to remove a matrix from a matrices list """ (x, elem) = inputs (out,) = outputs out[0] = 0 for y in range(len(x)): if node.inputs[0].ttype.values_eq(x[y], elem): out[0] += 1 out[0] = np.asarray(out[0], dtype=theano.config.floatX) def __str__(self): return self.__class__.__name__ count = Count() """ Count the number of times an element is in the typed list. Parameters ---------- x The typed list to look into. elem The element we want to count in list. The elements are compared with equals. Notes ----- Python implementation of count doesn't work when we want to count an ndarray from a list. This implementation works in that case. """ class Length(Op): # See doc in instance of this Op after the class definition. __props__ = () def make_node(self, x): assert isinstance(x.type, TypedListType) return Apply(self, [x], [T.scalar(dtype='int64')]) def perform(self, node, x, outputs): (out,) = outputs out[0] = np.asarray(len(x[0]), 'int64') def __str__(self): return self.__class__.__name__ def c_code(self, node, name, inp, out, sub): x_name = inp[0] output_name = out[0] fail = sub['fail'] return """ if(!%(output_name)s) %(output_name)s=(PyArrayObject*)PyArray_EMPTY(0, NULL, NPY_INT64, 0); ((npy_int64*)PyArray_DATA(%(output_name)s))[0]=PyList_Size((PyObject*)%(x_name)s); Py_INCREF(%(output_name)s); """ % locals() def c_code_cache_version(self): return (1,) length = Length() """ Returns the size of a list. Parameters ---------- x Typed list. """ class MakeList(Op): __props__ = () def make_node(self, a): assert isinstance(a, (tuple, list)) a2 = [] for elem in a: if not isinstance(elem, theano.gof.Variable): elem = theano.tensor.as_tensor_variable(elem) a2.append(elem) if not all(a2[0].type == elem.type for elem in a2): raise TypeError( "MakeList need all input variable to be of the same type.") tl = theano.typed_list.TypedListType(a2[0].type)() return Apply(self, a2, [tl]) def perform(self, node, inputs, outputs): (out,) = outputs # We need to make sure that we don't get a view on our inputs out[0] = [_lessbroken_deepcopy(inp) for inp in inputs] make_list = MakeList() """ Build a Python list from those Theano variable. Parameters ---------- a : tuple/list of Theano variable Notes ----- All Theano variables must have the same type. """ Theano-1.0.0/theano/updates.py00006000163562€ÿÿÿþ0000000661413203335532017573 0ustar boccoset00000000000000"""Defines Updates object for storing a (SharedVariable, new_value) mapping. """ from __future__ import absolute_import, print_function, division from collections import OrderedDict from six import iteritems from theano.compile.sharedvalue import SharedVariable import logging import warnings __authors__ = "theano-dev" __copyright__ = "(c) 2010, Universite de Montreal" __license__ = "3-clause BSD License" __contact__ = "theano-dev " __docformat__ = "restructuredtext en" logger = logging.getLogger('theano.updates') # Must be an OrderedDict or updates will be applied in a non-deterministic # order. class OrderedUpdates(OrderedDict): """ Dict-like mapping from SharedVariable keys to their new values. This mapping supports the use of the "+" operator for the union of updates. """ def __init__(self, *key, **kwargs): if (len(key) >= 1 and isinstance(key[0], dict) and len(key[0]) > 1 and not isinstance(key[0], OrderedDict)): # Warn when using as input a non-ordered dictionary. warnings.warn('Initializing an `OrderedUpdates` from a ' 'non-ordered dictionary with 2+ elements could ' 'make your code non-deterministic. You can use ' 'an OrderedDict that is available at ' 'theano.compat.OrderedDict for python 2.6+.') super(OrderedUpdates, self).__init__(*key, **kwargs) for key in self: if not isinstance(key, SharedVariable): raise TypeError( 'OrderedUpdates keys must inherit from SharedVariable', key) def __setitem__(self, key, value): if isinstance(key, SharedVariable): # TODO: consider doing error-checking on value. # insist that it is a Theano variable? Have the right type? # This could have weird consequences - for example a # GPU SharedVariable is customarily associated with a TensorType # value. Should it be cast to a GPU value right away? Should # literals be transformed into constants immediately? return super(OrderedUpdates, self).__setitem__(key, value) else: raise TypeError('OrderedUpdates keys must inherit from ' 'SharedVariable', key) def update(self, other=None): if other is None: return if (isinstance(other, dict) and len(other) > 1 and not isinstance(other, OrderedDict)): # Warn about non-determinism. warnings.warn('Updating an `OrderedUpdates` with a ' 'non-ordered dictionary with 2+ elements could ' 'make your code non-deterministic', stacklevel=2) for key, val in iteritems(OrderedDict(other)): if key in self: if self[key] == val: continue raise KeyError('Collision', key) self[key] = val # __setitem__ does type-checking def __add__(self, other): rval = OrderedUpdates() rval.update(self) rval.update(other) return rval def __radd__(other, self): rval = OrderedUpdates() rval.update(other) rval.update(self) return rval Theano-1.0.0/theano/pathparse.py00006000163562€ÿÿÿþ0000000256413203335531020114 0ustar boccoset00000000000000from __future__ import absolute_import, print_function, division import os import sys class PathParser(object): """ Class that allows to modify system's PATH environment variable at runtime. Currently used in ``theano.gpuarray.dnn`` module on Windows only. **Examples**: ..code-block:: python theano.pathparse.PathParser(pathToAdd1, pathToAdd2, ...) # PATH is then automatically updated for this execution. ..code-block:: python paths = theano.pathparse.PathParser() paths.add(path1) paths.add(path2) # PATH is updated after each call to ``add()``. """ paths = set() def _add(self, path): path = path.strip() if path: if sys.platform == 'win32': # Windows is case-insensitive. path = path.lower() self.paths.add(os.path.abspath(path)) def _update(self): os.environ['PATH'] = os.pathsep.join(sorted(self.paths)) def _parse(self): for path in os.environ['PATH'].split(os.pathsep): self._add(path) def __init__(self, *paths): self._parse() for path in paths: self._add(path) self._update() def add(self, path): self._add(path) self._update() def _debug(self): for path in sorted(self.paths): print(path) Theano-1.0.0/theano/printing.py00006000163562€ÿÿÿþ0000013544613203335531017765 0ustar boccoset00000000000000"""Pretty-printing (pprint()), the 'Print' Op, debugprint() and pydotprint(). They all allow different way to print a graph or the result of an Op in a graph(Print Op) """ from __future__ import absolute_import, print_function, division from copy import copy import logging import os import sys import hashlib import numpy as np from six import string_types, integer_types, iteritems from six.moves import StringIO, reduce import theano from theano import gof from theano import config from theano.gof import Op, Apply from theano.compile import Function, debugmode, SharedVariable pydot_imported = False pydot_imported_msg = "" try: # pydot-ng is a fork of pydot that is better maintained import pydot_ng as pd if pd.find_graphviz(): pydot_imported = True else: pydot_imported_msg = "pydot-ng can't find graphviz. Install graphviz." except ImportError: try: # fall back on pydot if necessary import pydot as pd if hasattr(pd, 'find_graphviz'): if pd.find_graphviz(): pydot_imported = True else: pydot_imported_msg = "pydot can't find graphviz" else: pd.Dot.create(pd.Dot()) pydot_imported = True except ImportError: # tests should not fail on optional dependency pydot_imported_msg = ("Install the python package pydot or pydot-ng." " Install graphviz.") except Exception as e: pydot_imported_msg = "An error happened while importing/trying pydot: " pydot_imported_msg += str(e.args) _logger = logging.getLogger("theano.printing") VALID_ASSOC = set(['left', 'right', 'either']) def debugprint(obj, depth=-1, print_type=False, file=None, ids='CHAR', stop_on_name=False, done=None, print_storage=False, print_clients=False, used_ids=None): """Print a computation graph as text to stdout or a file. :type obj: :class:`~theano.gof.Variable`, Apply, or Function instance :param obj: symbolic thing to print :type depth: integer :param depth: print graph to this depth (-1 for unlimited) :type print_type: boolean :param print_type: whether to print the type of printed objects :type file: None, 'str', or file-like object :param file: print to this file ('str' means to return a string) :type ids: str :param ids: How do we print the identifier of the variable id - print the python id value int - print integer character CHAR - print capital character "" - don't print an identifier :param stop_on_name: When True, if a node in the graph has a name, we don't print anything below it. :type done: None or dict :param done: A dict where we store the ids of printed node. Useful to have multiple call to debugprint share the same ids. :type print_storage: bool :param print_storage: If True, this will print the storage map for Theano functions. Combined with allow_gc=False, after the execution of a Theano function, we see the intermediate result. :type print_clients: bool :param print_clients: If True, this will print for Apply node that have more then 1 clients its clients. This help find who use an Apply node. :type used_ids: dict or None :param used_ids: the id to use for some object, but maybe we only refered to it yet. :returns: string if `file` == 'str', else file arg Each line printed represents a Variable in the graph. The indentation of lines corresponds to its depth in the symbolic graph. The first part of the text identifies whether it is an input (if a name or type is printed) or the output of some Apply (in which case the Op is printed). The second part of the text is an identifier of the Variable. If print_type is True, we add a part containing the type of the Variable If a Variable is encountered multiple times in the depth-first search, it is only printed recursively the first time. Later, just the Variable identifier is printed. If an Apply has multiple outputs, then a '.N' suffix will be appended to the Apply's identifier, to indicate which output a line corresponds to. """ if not isinstance(depth, integer_types): raise Exception("depth parameter must be an int") if file == 'str': _file = StringIO() elif file is None: _file = sys.stdout else: _file = file if done is None: done = dict() if used_ids is None: used_ids = dict() used_ids = dict() results_to_print = [] profile_list = [] order = [] # Toposort smap = [] # storage_map if isinstance(obj, (list, tuple, set)): lobj = obj else: lobj = [obj] for obj in lobj: if isinstance(obj, gof.Variable): results_to_print.append(obj) profile_list.append(None) smap.append(None) order.append(None) elif isinstance(obj, gof.Apply): results_to_print.extend(obj.outputs) profile_list.extend([None for item in obj.outputs]) smap.extend([None for item in obj.outputs]) order.extend([None for item in obj.outputs]) elif isinstance(obj, Function): results_to_print.extend(obj.maker.fgraph.outputs) profile_list.extend( [obj.profile for item in obj.maker.fgraph.outputs]) if print_storage: smap.extend( [obj.fn.storage_map for item in obj.maker.fgraph.outputs]) else: smap.extend( [None for item in obj.maker.fgraph.outputs]) topo = obj.maker.fgraph.toposort() order.extend( [topo for item in obj.maker.fgraph.outputs]) elif isinstance(obj, gof.FunctionGraph): results_to_print.extend(obj.outputs) profile_list.extend([getattr(obj, 'profile', None) for item in obj.outputs]) smap.extend([getattr(obj, 'storage_map', None) for item in obj.outputs]) topo = obj.toposort() order.extend([topo for item in obj.outputs]) elif isinstance(obj, (integer_types, float, np.ndarray)): print(obj) elif isinstance(obj, (theano.In, theano.Out)): results_to_print.append(obj.variable) profile_list.append(None) smap.append(None) order.append(None) else: raise TypeError("debugprint cannot print an object of this type", obj) scan_ops = [] if any([p for p in profile_list if p is not None and p.fct_callcount > 0]): print(""" Timing Info ----------- -->