Theano-1.0.1/ 0000777 0000000 0000000 00000000000 13212236507 012672 5 ustar root root 0000000 0000000 Theano-1.0.1/bin/ 0000777 0000000 0000000 00000000000 13212236507 013442 5 ustar root root 0000000 0000000 Theano-1.0.1/bin/theano-cache 0000777 0000000 0000000 00000000076 13212236347 015714 0 ustar root root 0000000 0000000 #!/usr/bin/env python
import theano_cache
theano_cache.main()
Theano-1.0.1/bin/theano-nose 0000777 0000000 0000000 00000000074 13212236347 015613 0 ustar root root 0000000 0000000 #!/usr/bin/env python
import theano_nose
theano_nose.main()
Theano-1.0.1/bin/theano_cache.py 0000777 0000000 0000000 00000007713 13212236347 016432 0 ustar root root 0000000 0000000 #!/usr/bin/env python
from __future__ import print_function
import logging
import os
import sys
if sys.platform == 'win32':
config_for_theano_cache_script = 'cxx=,device=cpu'
theano_flags = os.environ['THEANO_FLAGS'] if 'THEANO_FLAGS' in os.environ else ''
if theano_flags:
theano_flags += ','
theano_flags += config_for_theano_cache_script
os.environ['THEANO_FLAGS'] = theano_flags
import theano
from theano import config
import theano.gof.compiledir
from theano.gof.cc import get_module_cache
_logger = logging.getLogger('theano.bin.theano-cache')
def print_help(exit_status):
if exit_status:
print('command "%s" not recognized' % (' '.join(sys.argv)))
print('Type "theano-cache" to print the cache location')
print('Type "theano-cache help" to print this help')
print('Type "theano-cache clear" to erase the cache')
print('Type "theano-cache list" to print the cache content')
print('Type "theano-cache unlock" to unlock the cache directory')
print('Type "theano-cache cleanup" to delete keys in the old '
'format/code version')
print('Type "theano-cache purge" to force deletion of the cache directory')
print('Type "theano-cache basecompiledir" '
'to print the parent of the cache directory')
print('Type "theano-cache basecompiledir list" '
'to print the content of the base compile dir')
print('Type "theano-cache basecompiledir purge" '
'to remove everything in the base compile dir, '
'that is, erase ALL cache directories')
sys.exit(exit_status)
def main():
if len(sys.argv) == 1:
print(config.compiledir)
elif len(sys.argv) == 2:
if sys.argv[1] == 'help':
print_help(exit_status=0)
if sys.argv[1] == 'clear':
# We skip the refresh on module cache creation because the refresh will
# be done when calling clear afterwards.
cache = get_module_cache(init_args=dict(do_refresh=False))
cache.clear(unversioned_min_age=-1, clear_base_files=True,
delete_if_problem=True)
# Print a warning if some cached modules were not removed, so that the
# user knows he should manually delete them, or call
# theano-cache purge, # to properly clear the cache.
items = [item for item in sorted(os.listdir(cache.dirname))
if item.startswith('tmp')]
if items:
_logger.warning(
'There remain elements in the cache dir that you may '
'need to erase manually. The cache dir is:\n %s\n'
'You can also call "theano-cache purge" to '
'remove everything from that directory.' %
config.compiledir)
_logger.debug('Remaining elements (%s): %s' %
(len(items), ', '.join(items)))
elif sys.argv[1] == 'list':
theano.gof.compiledir.print_compiledir_content()
elif sys.argv[1] == 'cleanup':
theano.gof.compiledir.cleanup()
cache = get_module_cache(init_args=dict(do_refresh=False))
cache.clear_old()
elif sys.argv[1] == 'unlock':
theano.gof.compilelock.force_unlock()
print('Lock successfully removed!')
elif sys.argv[1] == 'purge':
theano.gof.compiledir.compiledir_purge()
elif sys.argv[1] == 'basecompiledir':
# Simply print the base_compiledir
print(theano.config.base_compiledir)
else:
print_help(exit_status=1)
elif len(sys.argv) == 3 and sys.argv[1] == 'basecompiledir':
if sys.argv[2] == 'list':
theano.gof.compiledir.basecompiledir_ls()
elif sys.argv[2] == 'purge':
theano.gof.compiledir.basecompiledir_purge()
else:
print_help(exit_status=1)
else:
print_help(exit_status=1)
if __name__ == '__main__':
main()
Theano-1.0.1/bin/theano_nose.py 0000777 0000000 0000000 00000025363 13212236347 016334 0 ustar root root 0000000 0000000 #!/usr/bin/env python
"""
This script should behave the same as the `nosetests` command.
The reason for its existence is that on some systems, it may not be obvious to
find where nosetests is installed in order to run it in a different process.
It is also used to load the KnownFailure plugin, in order to hide
KnownFailureTests error messages. Use --without-knownfailure to
disable that plugin.
`run_tests_in_batch.py` will in turn call back this script in another process.
"""
from __future__ import print_function
__authors__ = "Olivier Delalleau, Pascal Lamblin, Eric Larsen"
__contact__ = "delallea@iro"
import logging
_logger = logging.getLogger('theano.bin.theano-nose')
import os
import nose
import textwrap
import sys
from nose.plugins import Plugin
def main_function():
# Handle the --theano arguments
if "--theano" in sys.argv:
i = sys.argv.index("--theano")
import theano
sys.argv[i] = theano.__path__[0]
# Many Theano tests suppose device=cpu, so we need to raise an
# error if device==gpu.
# I don't know how to do this check only if we use theano-nose on
# Theano tests. So I make an try..except in case the script get
# reused elsewhere.
# We should not import theano before call nose.main()
# As this cause import problem with nosetests.
# Should we find a way to don't modify sys.path?
if not os.path.exists('theano/__init__.py'):
try:
from theano import config
if config.device != "cpu":
raise ValueError("Theano tests must be run with device=cpu."
" This will also run GPU tests when possible.\n"
" If you want GPU-related tests to run on a"
" specific GPU device, and not the default one,"
" you should use the init_gpu_device theano flag.")
except ImportError:
pass
# Handle --batch[=n] arguments
batch_args = [arg for arg in sys.argv if arg.startswith('--batch')]
for arg in batch_args:
sys.argv.remove(arg)
batch_size = None
if len(batch_args):
if len(batch_args) > 1:
_logger.warn(
'Multiple --batch arguments detected, using the last one '
'and ignoring the first ones.')
batch_arg = batch_args[-1]
elems = batch_arg.split('=', 1)
if len(elems) == 2:
batch_size = int(elems[1])
# Handle the --debug-batch argument.
display_batch_output = False
if '--debug-batch' in sys.argv:
if not batch_args:
raise AssertionError(
'You can only use the --debug-batch argument with the '
'--batch[=n] option')
while '--debug-batch' in sys.argv:
sys.argv.remove('--debug-batch')
sys.argv += ['--verbose', '--nocapture', '--detailed-errors']
display_batch_output = True
# Handle --time_prof arguments
time_prof_args = [arg for arg in sys.argv if arg=='--time-profile']
for arg in time_prof_args:
sys.argv.remove(arg)
# Time-profiling and batch modes
if time_prof_args or batch_args:
from theano.tests import run_tests_in_batch
return run_tests_in_batch.main(
theano_nose=os.path.realpath(__file__),
batch_size=batch_size,
time_profile=bool(time_prof_args),
display_batch_output=display_batch_output)
# Non-batch mode.
addplugins = []
# We include KnownFailure plugin by default, unless
# it is disabled by the "--without-knownfailure" arg.
if '--without-knownfailure' not in sys.argv:
try:
from numpy.testing.noseclasses import KnownFailure
addplugins.append(KnownFailure())
except ImportError:
_logger.warn(
'KnownFailure plugin from NumPy could not be imported. '
'Use --without-knownfailure to disable this warning.')
else:
sys.argv.remove('--without-knownfailure')
# When 'theano-nose' is called-back under the time-profile option, an
# instance of the custom Nosetests plugin class 'DisabDocString' (see
# below) is loaded. The latter ensures that the test name will not be
# replaced in display by the first line of the documentation string.
if '--disabdocstring' in sys.argv:
addplugins.append(DisabDocString())
try:
if addplugins:
ret = nose.main(addplugins=addplugins)
else:
ret = nose.main()
return ret
except TypeError as e:
if "got an unexpected keyword argument 'addplugins'" in e.message:
# This means nose is too old and does not support plugins
_logger.warn(
'KnownFailure plugin from NumPy can\'t'
' be used as nosetests is too old. '
'Use --without-knownfailure to disable this warning.')
nose.main()
else:
raise
def help():
help_msg = """
This script behaves mostly the same as the `nosetests` command.
The main difference is that it loads automatically the
KnownFailure plugin, in order to hide KnownFailureTests error
messages. It also supports executing tests by batches.
Local options:
--help, -h: Displays this help.
--batch[=n]:
If specified without option '--time-profile', do not run all
the tests in one run, but split the execution in batches of
`n` tests each. Default n is 100.
--time-profile:
Each test will be run and timed separately and the results will
be deposited in the files 'timeprof_sort', 'timeprof_nosort'
and 'timeprof_rawlog' in the current directory. If the
'--batch[=n]' option is also specified, notification of the
progresses will be made to standard output after every group of
n tests. Otherwise, notification will occur after every group
of 100 tests.
The files 'timeprof_sort' and 'timeprof_nosort' both contain one
record for each test and comprise the following fields:
- test running-time
- nosetests sequential test number
- test name
- name of class to which test belongs (if any), otherwise full
information is contained in test name
- test outcome ('OK', 'SKIPPED TEST', 'FAILED TEST' or
'FAILED PARSING')
In 'timeprof_sort', test records are sorted according to
running-time whereas in 'timeprof_nosort' records are reported
according to sequential number. The former classification is the
main information source for time-profiling. Since tests belonging
to same or close classes and files have close sequential, the
latter may be used to identify duration patterns among the tests
numbers. A full log is also saved as 'timeprof_rawlog'.
--without-knownfailure: Do not load the KnownFailure plugin.
--theano: This parameter is replaced with the path to the theano
library. As theano-nose is a wrapper to nosetests, it
expects a path to the tests to run.
If you do not know where theano is installed, use this
option to have it inserted automatically.
--debug-batch:
Use this parameter to run nosetests with options '--verbose',
'--nocapture' and '--detailed-errors' and show the output of
nosetests during batch execution. This can be useful to debug
situations where re-running only the failed tests after batch
execution is not working properly. This option can only be used
in conjunction with the '--batch=[n]' argument.
The other options will be passed to nosetests, see ``nosetests -h``.
"""
print(textwrap.dedent(help_msg))
def main():
if '--help' in sys.argv or '-h' in sys.argv:
help()
else:
result = main_function()
sys.exit(result)
class DisabDocString(Plugin):
"""
When activated, a custom Nosetests plugin created through this class
will preclude automatic replacement in display of the name of the test
by the first line in its documentation string.
Sources:
http://nose.readthedocs.org/en/latest/developing.html
http://nose.readthedocs.org/en/latest/further_reading.html
http://www.siafoo.net/article/54
https://github.com/nose-devs/nose/issues/294
http://python-nose.googlecode.com/svn/trunk/nose/plugins/base.py
Nat Williams:
https://github.com/Merino/nose-description-fixer-plugin/commit/
df94596f29c04fea8001713dd9b04bf3720aebf4
"""
enabled = False # plugin disabled by default
score = 2000 # high score ensures priority over other plugins
def __init__(self):
# 'super.__init__(self):' would have achieved exactly the same
if self.name is None:
self.name = self.__class__.__name__.lower()
if self.enableOpt is None:
self.enableOpt = ("enable_plugin_%s"
% self.name.replace('-', '_'))
def options(self, parser, env):
env_opt = 'NOSE_WITH_%s' % self.name.upper()
# latter expression to be used if plugin called from the command line
parser.add_option("--%s" % self.name,
# will be called with Nosetests 'main' or 'run'
# function's' argument '--disabdocstring'
action="store_true",
dest=self.enableOpt,
# the latter entails that the boolean self.enableOpt
# is set to 'True' when plugin is called through a
# function's argument
default=env.get(env_opt),
# entails that plugin will be enabled when command
# line trigger 'env_opt' will be activated
help="Enable plugin %s: %s [%s]" %
(self.__class__.__name__,
self.help(), env_opt))
def configure(self, options, conf):
self.conf = conf
# plugin will be enabled when called through argument
self.enabled = getattr(options, self.enableOpt)
def describeTest(self, test):
# 'describeTest' is also called when the test result in Nosetests calls
# 'test.shortDescription()' and can thus be used to alter the display.
return False
if __name__ == '__main__':
main()
Theano-1.0.1/bin/__init__.py 0000777 0000000 0000000 00000000000 13212236347 015546 0 ustar root root 0000000 0000000 Theano-1.0.1/DESCRIPTION.txt 0000777 0000000 0000000 00000002162 13212236347 015224 0 ustar root root 0000000 0000000 Theano is a Python library that allows you to define, optimize, and efficiently evaluate mathematical expressions involving multi-dimensional arrays. It is built on top of NumPy_. Theano features:
* **tight integration with NumPy:** a similar interface to NumPy's. numpy.ndarrays are also used internally in Theano-compiled functions.
* **transparent use of a GPU:** perform data-intensive computations up to 140x faster than on a CPU (support for float32 only).
* **efficient symbolic differentiation:** Theano can compute derivatives for functions of one or many inputs.
* **speed and stability optimizations:** avoid nasty bugs when computing expressions such as log(1 + exp(x)) for large values of x.
* **dynamic C code generation:** evaluate expressions faster.
* **extensive unit-testing and self-verification:** includes tools for detecting and diagnosing bugs and/or potential problems.
Theano has been powering large-scale computationally intensive scientific
research since 2007, but it is also approachable enough to be used in the
classroom (IFT6266 at the University of Montreal).
.. _NumPy: http://numpy.scipy.org/
Theano-1.0.1/doc/ 0000777 0000000 0000000 00000000000 13212236507 013437 5 ustar root root 0000000 0000000 Theano-1.0.1/doc/acknowledgement.txt 0000777 0000000 0000000 00000003163 13212236347 017357 0 ustar root root 0000000 0000000 .. _acknowledgement:
Acknowledgements
================
.. note:
This page is in construction. We are missing sources.
* The developers of `NumPy `_.
* All Theano users that have given us feedback.
* The GPU implementation of tensordot is based on code from Tijmen
Tieleman's `gnumpy `_
* The original version of the function ``cpuCount()`` in the file
`theano/misc/cpucount.py` come from the project `pyprocessing
`_, Proceedings of the 2000 Winter Simulation Conference, Dec. 2000, 683--689.
We were authorized by Pierre L'Ecuyer to copy/modify his Java implementation in the `SSJ ` was included in Theano in the previous GPU back-end. It is still in the history, but not in the current version. It has the same license.
Theano-1.0.1/doc/cifarSC2011/ 0000777 0000000 0000000 00000000000 13212236507 015255 5 ustar root root 0000000 0000000 Theano-1.0.1/doc/cifarSC2011/advanced_theano.txt 0000777 0000000 0000000 00000057740 13212236347 021143 0 ustar root root 0000000 0000000
.. _advanced_theano:
***************
Advanced Theano
***************
Conditions
----------
**IfElse**
- Build condition over symbolic variables.
- IfElse Op takes a boolean condition and two variables to compute as input.
- While Switch Op evaluates both 'output' variables, IfElse Op is lazy and only
evaluates one variable respect to the condition.
**IfElse Example: Comparison with Switch**
.. testcode::
from theano import tensor as T
from theano.ifelse import ifelse
import theano, time, numpy
a,b = T.scalars('a','b')
x,y = T.matrices('x','y')
z_switch = T.switch(T.lt(a,b), T.mean(x), T.mean(y))
z_lazy = ifelse(T.lt(a,b), T.mean(x), T.mean(y))
f_switch = theano.function([a,b,x,y], z_switch,
mode=theano.Mode(linker='vm'))
f_lazyifelse = theano.function([a,b,x,y], z_lazy,
mode=theano.Mode(linker='vm'))
val1 = 0.
val2 = 1.
big_mat1 = numpy.ones((10000,1000))
big_mat2 = numpy.ones((10000,1000))
n_times = 10
tic = time.clock()
for i in range(n_times):
f_switch(val1, val2, big_mat1, big_mat2)
print('time spent evaluating both values %f sec' % (time.clock()-tic))
tic = time.clock()
for i in range(n_times):
f_lazyifelse(val1, val2, big_mat1, big_mat2)
print('time spent evaluating one value %f sec' % (time.clock()-tic))
.. testoutput::
:hide:
:options: +ELLIPSIS
time spent evaluating both values ... sec
time spent evaluating one value ... sec
IfElse Op spend less time (about an half) than Switch since it computes only
one variable instead of both.
.. code-block:: none
$ python ifelse_switch.py
time spent evaluating both values 0.6700 sec
time spent evaluating one value 0.3500 sec
Note that IfElse condition is a boolean while Switch condition is a tensor, so
Switch is more general.
It is actually important to use ``linker='vm'`` or ``linker='cvm'``,
otherwise IfElse will compute both variables and take the same computation
time as the Switch Op. The linker is not currently set by default to 'cvm' but
it will be in a near future.
Loops
-----
**Scan**
- General form of **recurrence**, which can be used for looping.
- **Reduction** and **map** (loop over the leading dimensions) are special cases of Scan
- You 'scan' a function along some input sequence, producing an output at each time-step
- The function can see the **previous K time-steps** of your function
- ``sum()`` could be computed by scanning the z + x(i) function over a list, given an initial state of ``z=0``.
- Often a for-loop can be expressed as a ``scan()`` operation, and ``scan`` is the closest that Theano comes to looping.
- The advantage of using ``scan`` over for loops
- The number of iterations to be part of the symbolic graph
- Minimizes GPU transfers if GPU is involved
- Compute gradients through sequential steps
- Slightly faster then using a for loop in Python with a compiled Theano function
- Can lower the overall memory usage by detecting the actual amount of memory needed
**Scan Example: Computing pow(A,k)**
.. code-block:: python
import theano
import theano.tensor as T
k = T.iscalar("k"); A = T.vector("A")
def inner_fct(prior_result, A): return prior_result * A
# Symbolic description of the result
result, updates = theano.scan(fn=inner_fct,
outputs_info=T.ones_like(A),
non_sequences=A, n_steps=k)
# Scan has provided us with A**1 through A**k. Keep only the last
# value. Scan notices this and does not waste memory saving them.
final_result = result[-1]
power = theano.function(inputs=[A,k], outputs=final_result,
updates=updates)
print power(range(10),2)
#[ 0. 1. 4. 9. 16. 25. 36. 49. 64. 81.]
**Scan Example: Calculating a Polynomial**
.. testcode::
import numpy
import theano
import theano.tensor as T
coefficients = theano.tensor.vector("coefficients")
x = T.scalar("x"); max_coefficients_supported = 10000
# Generate the components of the polynomial
full_range=theano.tensor.arange(max_coefficients_supported)
components, updates = theano.scan(fn=lambda coeff, power, free_var:
coeff * (free_var ** power),
outputs_info=None,
sequences=[coefficients, full_range],
non_sequences=x)
polynomial = components.sum()
calculate_polynomial = theano.function(inputs=[coefficients, x],
outputs=polynomial)
test_coeff = numpy.asarray([1, 0, 2], dtype=numpy.float32)
print(calculate_polynomial(test_coeff, 3))
.. testoutput::
19.0
Exercise 4
-----------
- Run both examples
- Modify and execute the polynomial example to have the reduction done by scan
Compilation pipeline
--------------------
.. image:: ../hpcs2011_tutorial/pics/pipeline.png
:width: 400 px
Inplace optimization
--------------------
- 2 type of inplace operations:
- An op that return a view on its inputs (e.g. reshape, inplace transpose)
- An op that write the output on the inputs memory space
- This allows some memory optimization
- The Op must tell Theano if they work inplace
- Inplace Op add constraints to the order of execution
Profiling
---------
- To replace the default mode with this mode, use the Theano flags ``profile=True``
- To enable the memory profiling use the flags ``profile=True,profile_memory=True``
Theano output for running the train function of logistic regression
example from :doc:`here <../tutorial/examples>` for one epoch:
.. code-block:: python
"""
Function profiling
==================
Message: train.py:47
Time in 1 calls to Function.__call__: 5.981922e-03s
Time in Function.fn.__call__: 5.180120e-03s (86.596%)
Time in thunks: 4.213095e-03s (70.430%)
Total compile time: 3.739440e-01s
Number of Apply nodes: 21
Theano Optimizer time: 3.258998e-01s
Theano validate time: 5.632162e-03s
Theano Linker time (includes C, CUDA code generation/compiling): 3.185582e-02s
Import time 3.157377e-03s
Time in all call to theano.grad() 2.997899e-02s
Time since theano import 3.616s
Class
---
<% time> <#call> <#apply>
50.6% 50.6% 0.002s 1.07e-03s Py 2 2 theano.tensor.basic.Dot
27.2% 77.8% 0.001s 5.74e-04s C 2 2 theano.sandbox.cuda.basic_ops.HostFromGpu
18.1% 95.9% 0.001s 3.81e-04s C 2 2 theano.sandbox.cuda.basic_ops.GpuFromHost
2.6% 98.6% 0.000s 1.23e-05s C 9 9 theano.tensor.elemwise.Elemwise
0.8% 99.3% 0.000s 3.29e-05s C 1 1 theano.sandbox.cuda.basic_ops.GpuElemwise
0.3% 99.6% 0.000s 5.60e-06s C 2 2 theano.tensor.elemwise.DimShuffle
0.2% 99.8% 0.000s 6.91e-06s C 1 1 theano.sandbox.cuda.basic_ops.GpuDimShuffle
0.1% 99.9% 0.000s 5.01e-06s C 1 1 theano.compile.ops.Shape_i
0.1% 100.0% 0.000s 5.01e-06s C 1 1 theano.tensor.elemwise.Sum
... (remaining 0 Classes account for 0.00%(0.00s) of the runtime)
Ops
---
<% time> <#call> <#apply>
50.6% 50.6% 0.002s 1.07e-03s Py 2 2 dot
27.2% 77.8% 0.001s 5.74e-04s C 2 2 HostFromGpu
18.1% 95.9% 0.001s 3.81e-04s C 2 2 GpuFromHost
1.0% 97.0% 0.000s 4.39e-05s C 1 1 Elemwise{Composite{((i0 * scalar_softplus(i1)) - (i2 * i3 * scalar_softplus(i4)))}}
0.8% 97.7% 0.000s 3.29e-05s C 1 1 GpuElemwise{Sub}[(0, 1)]
0.4% 98.1% 0.000s 1.50e-05s C 1 1 Elemwise{Composite{(((scalar_sigmoid(i0) * i1 * i2) / i3) - ((i4 * i1 * i5) / i3))}}[(0, 0)]
0.3% 98.4% 0.000s 5.60e-06s C 2 2 InplaceDimShuffle{x}
0.3% 98.6% 0.000s 1.10e-05s C 1 1 Elemwise{ScalarSigmoid}[(0, 0)]
0.2% 98.8% 0.000s 9.06e-06s C 1 1 Elemwise{Composite{(i0 - (i1 * (i2 + (i3 * i0))))}}[(0, 0)]
0.2% 99.0% 0.000s 7.15e-06s C 1 1 Elemwise{gt,no_inplace}
0.2% 99.2% 0.000s 6.91e-06s C 1 1 Elemwise{Composite{(i0 - (i1 * i2))}}[(0, 0)]
0.2% 99.3% 0.000s 6.91e-06s C 1 1 GpuDimShuffle{1,0}
0.2% 99.5% 0.000s 6.91e-06s C 1 1 Elemwise{neg,no_inplace}
0.1% 99.6% 0.000s 5.96e-06s C 1 1 Elemwise{Composite{((-i0) - i1)}}[(0, 0)]
0.1% 99.8% 0.000s 5.01e-06s C 1 1 Elemwise{Cast{float64}}
0.1% 99.9% 0.000s 5.01e-06s C 1 1 Shape_i{0}
0.1% 100.0% 0.000s 5.01e-06s C 1 1 Sum{acc_dtype=float64}
... (remaining 0 Ops account for 0.00%(0.00s) of the runtime)
Apply
------
<% time> <#call>
26.8% 26.8% 0.001s 1.13e-03s 1 1 dot(x, w)
input 0: dtype=float32, shape=(400, 784), strides=c
input 1: dtype=float64, shape=(784,), strides=c
output 0: dtype=float64, shape=(400,), strides=c
26.5% 53.4% 0.001s 1.12e-03s 1 10 HostFromGpu(GpuDimShuffle{1,0}.0)
input 0: dtype=float32, shape=(784, 400), strides=(1, 784)
output 0: dtype=float32, shape=(784, 400), strides=c
23.8% 77.1% 0.001s 1.00e-03s 1 18 dot(x.T, Elemwise{Composite{(((scalar_sigmoid(i0) * i1 * i2) / i3) - ((i4 * i1 * i5) / i3))}}[(0, 0)].0)
input 0: dtype=float32, shape=(784, 400), strides=c
input 1: dtype=float64, shape=(400,), strides=c
output 0: dtype=float64, shape=(784,), strides=c
9.6% 86.7% 0.000s 4.04e-04s 1 3 GpuFromHost(y)
input 0: dtype=float32, shape=(400,), strides=c
output 0: dtype=float32, shape=(400,), strides=(1,)
8.5% 95.2% 0.000s 3.58e-04s 1 2 GpuFromHost(x)
input 0: dtype=float32, shape=(400, 784), strides=c
output 0: dtype=float32, shape=(400, 784), strides=(784, 1)
1.0% 96.3% 0.000s 4.39e-05s 1 13 Elemwise{Composite{((i0 * scalar_softplus(i1)) - (i2 * i3 * scalar_softplus(i4)))}}(y, Elemwise{Composite{((-i0) - i1)}}[(0, 0)].0, TensorConstant{(1,) of -1.0}, HostFromGpu.0, Elemwise{neg,no_inplace}.0)
input 0: dtype=float32, shape=(400,), strides=c
input 1: dtype=float64, shape=(400,), strides=c
input 2: dtype=float64, shape=(1,), strides=c
input 3: dtype=float32, shape=(400,), strides=c
input 4: dtype=float64, shape=(400,), strides=c
output 0: dtype=float64, shape=(400,), strides=c
0.8% 97.1% 0.000s 3.29e-05s 1 7 GpuElemwise{Sub}[(0, 1)](CudaNdarrayConstant{[ 1.]}, GpuFromHost.0)
input 0: dtype=float32, shape=(1,), strides=c
input 1: dtype=float32, shape=(400,), strides=(1,)
output 0: dtype=float32, shape=(400,), strides=c
0.7% 97.7% 0.000s 2.91e-05s 1 11 HostFromGpu(GpuElemwise{Sub}[(0, 1)].0)
input 0: dtype=float32, shape=(400,), strides=c
output 0: dtype=float32, shape=(400,), strides=c
0.4% 98.1% 0.000s 1.50e-05s 1 15 Elemwise{Composite{(((scalar_sigmoid(i0) * i1 * i2) / i3) - ((i4 * i1 * i5) / i3))}}[(0, 0)](Elemwise{Composite{((-i0) - i1)}}[(0, 0)].0, TensorConstant{(1,) of -1.0}, y, Elemwise{Cast{float64}}.0, Elemwise{ScalarSigmoid}[(0, 0)].0, HostFromGpu.0)
input 0: dtype=float64, shape=(400,), strides=c
input 1: dtype=float64, shape=(1,), strides=c
input 2: dtype=float32, shape=(400,), strides=c
input 3: dtype=float64, shape=(1,), strides=c
input 4: dtype=float64, shape=(400,), strides=c
input 5: dtype=float32, shape=(400,), strides=c
output 0: dtype=float64, shape=(400,), strides=c
0.3% 98.4% 0.000s 1.10e-05s 1 14 Elemwise{ScalarSigmoid}[(0, 0)](Elemwise{neg,no_inplace}.0)
input 0: dtype=float64, shape=(400,), strides=c
output 0: dtype=float64, shape=(400,), strides=c
0.2% 98.6% 0.000s 9.06e-06s 1 20 Elemwise{Composite{(i0 - (i1 * (i2 + (i3 * i0))))}}[(0, 0)](w, TensorConstant{(1,) of 0...0000000149}, dot.0, TensorConstant{(1,) of 0...9999999553})
input 0: dtype=float64, shape=(784,), strides=c
input 1: dtype=float64, shape=(1,), strides=c
input 2: dtype=float64, shape=(784,), strides=c
input 3: dtype=float64, shape=(1,), strides=c
output 0: dtype=float64, shape=(784,), strides=c
0.2% 98.7% 0.000s 7.15e-06s 1 16 Elemwise{gt,no_inplace}(Elemwise{ScalarSigmoid}[(0, 0)].0, TensorConstant{(1,) of 0.5})
input 0: dtype=float64, shape=(400,), strides=c
input 1: dtype=float32, shape=(1,), strides=c
output 0: dtype=int8, shape=(400,), strides=c
0.2% 98.9% 0.000s 7.15e-06s 1 0 InplaceDimShuffle{x}(b)
input 0: dtype=float64, shape=(), strides=c
output 0: dtype=float64, shape=(1,), strides=c
0.2% 99.1% 0.000s 6.91e-06s 1 19 Elemwise{Composite{(i0 - (i1 * i2))}}[(0, 0)](b, TensorConstant{0.10000000149}, Sum{acc_dtype=float64}.0)
input 0: dtype=float64, shape=(), strides=c
input 1: dtype=float64, shape=(), strides=c
input 2: dtype=float64, shape=(), strides=c
output 0: dtype=float64, shape=(), strides=c
0.2% 99.2% 0.000s 6.91e-06s 1 9 Elemwise{neg,no_inplace}(Elemwise{Composite{((-i0) - i1)}}[(0, 0)].0)
input 0: dtype=float64, shape=(400,), strides=c
output 0: dtype=float64, shape=(400,), strides=c
0.2% 99.4% 0.000s 6.91e-06s 1 6 GpuDimShuffle{1,0}(GpuFromHost.0)
input 0: dtype=float32, shape=(400, 784), strides=(784, 1)
output 0: dtype=float32, shape=(784, 400), strides=(1, 784)
0.1% 99.5% 0.000s 5.96e-06s 1 5 Elemwise{Composite{((-i0) - i1)}}[(0, 0)](dot.0, InplaceDimShuffle{x}.0)
input 0: dtype=float64, shape=(400,), strides=c
input 1: dtype=float64, shape=(1,), strides=c
output 0: dtype=float64, shape=(400,), strides=c
0.1% 99.7% 0.000s 5.01e-06s 1 17 Sum{acc_dtype=float64}(Elemwise{Composite{(((scalar_sigmoid(i0) * i1 * i2) / i3) - ((i4 * i1 * i5) / i3))}}[(0, 0)].0)
input 0: dtype=float64, shape=(400,), strides=c
output 0: dtype=float64, shape=(), strides=c
0.1% 99.8% 0.000s 5.01e-06s 1 12 Elemwise{Cast{float64}}(InplaceDimShuffle{x}.0)
input 0: dtype=int64, shape=(1,), strides=c
output 0: dtype=float64, shape=(1,), strides=c
0.1% 99.9% 0.000s 5.01e-06s 1 4 Shape_i{0}(y)
input 0: dtype=float32, shape=(400,), strides=c
output 0: dtype=int64, shape=(), strides=c
... (remaining 1 Apply instances account for 0.10%(0.00s) of the runtime)
Memory Profile
(Sparse variables are ignored)
(For values in brackets, it's for linker = c|py
---
Max if no gc (allow_gc=False): 2469KB (2469KB)
CPU: 1242KB (1242KB)
GPU: 1227KB (1227KB)
---
Max if linker=cvm(default): 2466KB (2464KB)
CPU: 1241KB (1238KB)
GPU: 1225KB (1227KB)
---
Memory saved if views are used: 1225KB (1225KB)
Memory saved if inplace ops are used: 17KB (17KB)
Memory saved if gc is enabled: 3KB (4KB)
---
1254400B [(400, 784)] c GpuFromHost(x)
1254400B [(784, 400)] v GpuDimShuffle{1,0}(GpuFromHost.0)
1254400B [(784, 400)] c HostFromGpu(GpuDimShuffle{1,0}.0)
6272B [(784,)] c dot(x.T, Elemwise{Composite{(((scalar_sigmoid(i0) * i1 * i2) / i3) - ((i4 * i1 * i5) / i3))}}[(0, 0)].0)
6272B [(784,)] i Elemwise{Composite{(i0 - (i1 * (i2 + (i3 * i0))))}}[(0, 0)](w, TensorConstant{(1,) of 0...0000000149}, dot.0, TensorConstant{(1,) of 0...9999999553})
3200B [(400,)] c dot(x, w)
3200B [(400,)] i Elemwise{Composite{((-i0) - i1)}}[(0, 0)](dot.0, InplaceDimShuffle{x}.0)
3200B [(400,)] i Elemwise{ScalarSigmoid}[(0, 0)](Elemwise{neg,no_inplace}.0)
3200B [(400,)] c Elemwise{neg,no_inplace}(Elemwise{Composite{((-i0) - i1)}}[(0, 0)].0)
3200B [(400,)] i Elemwise{Composite{(((scalar_sigmoid(i0) * i1 * i2) / i3) - ((i4 * i1 * i5) / i3))}}[(0, 0)](Elemwise{Composite{((-i0) - i1)}}[(0, 0)].0, TensorConstant{(1,) of -1.0}, y, Elemwise{Cast{float64}}.0, Elemwise{ScalarSigmoid}[(0, 0)].0, HostFromGpu.0)
3200B [(400,)] c Elemwise{Composite{((i0 * scalar_softplus(i1)) - (i2 * i3 * scalar_softplus(i4)))}}(y, Elemwise{Composite{((-i0) - i1)}}[(0, 0)].0, TensorConstant{(1,) of -1.0}, HostFromGpu.0, Elemwise{neg,no_inplace}.0)
1600B [(400,)] i GpuElemwise{Sub}[(0, 1)](CudaNdarrayConstant{[ 1.]}, GpuFromHost.0)
1600B [(400,)] c HostFromGpu(GpuElemwise{Sub}[(0, 1)].0)
1600B [(400,)] c GpuFromHost(y)
... (remaining 7 Apply account for 448B/3800192B ((0.01%)) of the Apply with dense outputs sizes)
is taken from the Op's declaration.
Apply nodes marked 'inplace' or 'view' may actually allocate memory, this is not reported here. If you use DebugMode, warnings will be emitted in those cases.
Here are tips to potentially make your code run faster
(if you think of new ones, suggest them on the mailing list).
Test them first, as they are not guaranteed to always provide a speedup.
Sorry, no tip for today.
"""
Exercise 5
-----------
- In the last exercises, do you see a speed up with the GPU?
- Where does it come from? (Use profile=True)
- Is there something we can do to speed up the GPU version?
Printing/Drawing Theano graphs
------------------------------
Consider the following logistic regression model:
>>> import numpy
>>> import theano
>>> import theano.tensor as T
>>> rng = numpy.random
>>> # Training data
>>> N = 400
>>> feats = 784
>>> D = (rng.randn(N, feats).astype(theano.config.floatX), rng.randint(size=N,low=0, high=2).astype(theano.config.floatX))
>>> training_steps = 10000
>>> # Declare Theano symbolic variables
>>> x = T.matrix("x")
>>> y = T.vector("y")
>>> w = theano.shared(rng.randn(feats).astype(theano.config.floatX), name="w")
>>> b = theano.shared(numpy.asarray(0., dtype=theano.config.floatX), name="b")
>>> x.tag.test_value = D[0]
>>> y.tag.test_value = D[1]
>>> # Construct Theano expression graph
>>> p_1 = 1 / (1 + T.exp(-T.dot(x, w)-b)) # Probability of having a one
>>> prediction = p_1 > 0.5 # The prediction that is done: 0 or 1
>>> # Compute gradients
>>> xent = -y*T.log(p_1) - (1-y)*T.log(1-p_1) # Cross-entropy
>>> cost = xent.mean() + 0.01*(w**2).sum() # The cost to optimize
>>> gw,gb = T.grad(cost, [w,b])
>>> # Training and prediction function
>>> train = theano.function(inputs=[x,y], outputs=[prediction, xent], updates=[[w, w-0.01*gw], [b, b-0.01*gb]], name = "train")
>>> predict = theano.function(inputs=[x], outputs=prediction, name = "predict")
We will now make use of Theano's printing features to compare the unoptimized
graph (``prediction``) to the optimized graph (``predict``).
Pretty Printing
~~~~~~~~~~~~~~~
>>> theano.printing.pprint(prediction) # doctest: +NORMALIZE_WHITESPACE
'gt((TensorConstant{1} / (TensorConstant{1} + exp(((-(x \\dot w)) - b)))), TensorConstant{0.5})'
Debug Print
~~~~~~~~~~~
The graph before optimization:
>>> theano.printing.debugprint(prediction) # doctest: +NORMALIZE_WHITESPACE, +SKIP
Elemwise{gt,no_inplace} [@A] ''
|Elemwise{true_div,no_inplace} [@B] ''
| |DimShuffle{x} [@C] ''
| | |TensorConstant{1} [@D]
| |Elemwise{add,no_inplace} [@E] ''
| |DimShuffle{x} [@F] ''
| | |TensorConstant{1} [@D]
| |Elemwise{exp,no_inplace} [@G] ''
| |Elemwise{sub,no_inplace} [@H] ''
| |Elemwise{neg,no_inplace} [@I] ''
| | |dot [@J] ''
| | |x [@K]
| | |w [@L]
| |DimShuffle{x} [@M] ''
| |b [@N]
|DimShuffle{x} [@O] ''
|TensorConstant{0.5} [@P]
The graph after optimization:
>>> theano.printing.debugprint(predict) # doctest: +NORMALIZE_WHITESPACE, +SKIP
Elemwise{Composite{GT(scalar_sigmoid((-((-i0) - i1))), i2)}} [@A] '' 4
|CGemv{inplace} [@B] '' 3
| |Alloc [@C] '' 2
| | |TensorConstant{0.0} [@D]
| | |Shape_i{0} [@E] '' 1
| | |x [@F]
| |TensorConstant{1.0} [@G]
| |x [@F]
| |w [@H]
| |TensorConstant{0.0} [@D]
|InplaceDimShuffle{x} [@I] '' 0
| |b [@J]
|TensorConstant{(1,) of 0.5} [@K]
Picture Printing of Graphs
~~~~~~~~~~~~~~~~~~~~~~~~~~
``pydotprint`` requires graphviz and either pydot or pydot-ng.
The graph before optimization:
>>> theano.printing.pydotprint(prediction, outfile="pics/logreg_pydotprint_prediction.png", var_with_name_simple=True) # doctest: +SKIP
The output file is available at pics/logreg_pydotprint_prediction.png
.. image:: ./pics/logreg_pydotprint_prediction.png
:width: 800 px
The graph after optimization:
>>> theano.printing.pydotprint(predict, outfile="pics/logreg_pydotprint_predict.png", var_with_name_simple=True) # doctest: +SKIP
The output file is available at pics/logreg_pydotprint_predict.png
.. image:: ./pics/logreg_pydotprint_predict.png
:width: 800 px
The optimized training graph:
>>> theano.printing.pydotprint(train, outfile="pics/logreg_pydotprint_train.png", var_with_name_simple=True) # doctest: +SKIP
The output file is available at pics/logreg_pydotprint_train.png
.. image:: ./pics/logreg_pydotprint_train.png
:width: 1500 px
Debugging
---------
- Run with the flag ``mode=DebugMode``
- 100-1000x slower
- Test all optimization steps from the original graph to the final graph
- Checks many things that Op should/shouldn't do
- Executes both the Python and C code versions
- Run with the Theano flag ``compute_test_value = {``off'',``ignore'', ``warn'', ``raise''}``
- Run the code as we create the graph
- Allows you to find the bug earlier (ex: shape mismatch)
- Makes it easier to identify where the problem is in *your* code
- Use the value of constants and shared variables directly
- For pure symbolic variables uses ``x.tag.test_value = numpy.random.rand(5,10)``
- Run with the flag ``mode=FAST_COMPILE``
- Few optimizations
- Run Python code (better error messages and can be debugged interactively in the Python debugger)
Known limitations
-----------------
- Compilation phase distinct from execution phase
- Compilation time can be significant
- Amortize it with functions over big input or reuse functions
- Execution overhead
- Needs a certain number of operations to be useful
- We have started working on this in a branch
- Compilation time superlinear in the size of the graph.
- A few hundreds nodes is fine
- Disabling a few optimizations can speed up compilation
- Usually too many nodes indicates a problem with the graph
Theano-1.0.1/doc/cifarSC2011/boot_camp_overview.txt 0000777 0000000 0000000 00000002244 13212236347 021716 0 ustar root root 0000000 0000000
.. _cifar_summer_school2011_overview:
========
Schedule
========
Theano lab sessions will be in 4 x 90 minute blocks,
on the afternoons of Aug 2, 3, 5, and 6 (but not Aug 4th).
Day 1
-----
* Show of hands - what is your background?
* Python & NumPy in a nutshell
* Theano basics
* Quick tour through Deep Learning Tutorials (think about projects)
.. :
day 1:
I think that I could cover those 2 pages:
* http://deeplearning.net/software/theano/hpcs2011_tutorial/introduction.html
* http://deeplearning.net/software/theano/hpcs2011_tutorial/theano.html
That include:
simple example
linear regression example with shared var
theano flags
grad detail
Symbolic variables
gpu
benchmarck
Day 2
-----
* Loop/Condition in Theano (10-20m)
* Propose/discuss projects
* Form groups and start projects!
Day 3
-----
* Advanced Theano (30 minutes)
* Debugging, profiling, compilation pipeline
* Projects / General hacking / code-sprinting.
Day 4
-----
* *You choose* (we can split the group)
* Extending Theano
* How to write an Op
* How to use pycuda code in Theano
* Projects / General hacking / code-sprinting.
Theano-1.0.1/doc/cifarSC2011/gpundarray.txt 0000777 0000000 0000000 00000002226 13212236347 020201 0 ustar root root 0000000 0000000
.. _cifar2013_gpundarray:
**********
GpuNdArray
**********
Why a common GPU ndarray?
- Currently there are at least 4 different GPU array data structures in use by Python packages
- CudaNdarray (Theano), GPUArray (PyCUDA), CUDAMatrix (cudamat), GPUArray (PyOpenCL), ...
- There are even more if we include other languages
- All of them are a subset of the functionality of ``numpy.ndarray`` on the GPU
- Lots of duplicated effort
- GPU code is harder/slower to do {\bf correctly} and {\bf fast} than on the CPU/Python
- Lack of a common array API makes it harder to port/reuse code
- Also harder to find/distribute code
- Divides development work
Design Goals
- Make it VERY similar to ``numpy.ndarray``
- Be compatible with both CUDA and OpenCL
- Have the base object accessible from C to allow collaboration with more projects, across high-level languages
- We want people from C, C++, Ruby, R, ... all use the same base GPU N-dimensional array
Final GpuNdArray Note
- Under development
- Will be the next GPU array container for Theano (this summer!)
- Probably also for PyCUDA, PyOpenCL
- Mailing list: http://lists.tiker.net/listinfo/gpundarray
Theano-1.0.1/doc/cifarSC2011/index.txt 0000777 0000000 0000000 00000003377 13212236347 017144 0 ustar root root 0000000 0000000 :orphan:
.. _index:
=========================================
TheanoLab @ NCAP CIFAR Summer School 2011
=========================================
Aug 2-6, 2011, Toronto, Canada.
Theano is python software for evaluating complicated array expressions.
What does it do?
* aggressive expression optimizations,
* automatic GPU use, and
* symbolic differentiation.
It complements the Python numeric/scientific software stack (e.g. NumPy, SciPy,
scikits, matplotlib, PIL.)
Design and feature set has been driven by machine learning research
at the University of
Montreal (groups of Yoshua Bengio, Pascal Vincent, Douglas Eck).
The result is a very good library for doing research in deep
learning and neural network training, and a flexible framework for
many other models and algorithms in machine learning more generally.
It has proven to be useful for implementing:
- linear and nonlinear neural network classifiers
- convolutional models
- Energy models: RBM, DBN, GRBM, ssRBM, AIS
- Auto-encoders: DAE, CAE
- GP regression
- sparse coding
- recurrent neural networks, echo state, (HMM?)
- online and batch learning and optimization
As people's needs change this list will grow, but Theano is built around vector,
matrix, and tensor expressions; there is little reason to use it for
calculations on other data structures.
Contents
--------
The structured part of these lab sessions will be a walk-through of the following
material. Interleaved with this structured part will be blocks of time for
individual or group work. The idea is that you can try out Theano and get help
from gurus on hand if you get stuck.
.. toctree::
boot_camp_overview
introduction
theano
advanced_theano
/extending/extending_theano
pyCUDA
gpundarray
Theano-1.0.1/doc/cifarSC2011/introduction.txt 0000777 0000000 0000000 00000023722 13212236347 020552 0 ustar root root 0000000 0000000
.. _cifarSS2011_Introduction:
************
Introduction
************
Background Questionaire
-----------------------
* Who has used Theano before?
* What did you do with it?
* Who has used Python? NumPy? SciPy? matplotlib?
* Who has used iPython?
* Who has used it as a distributed computing engine?
* Who has done C/C++ programming?
* Who has organized computation around a particular physical memory layout?
* Who has used a multidimensional array of >2 dimensions?
* Who has written a Python module in C before?
* Who has written a program to *generate* Python modules in C?
* Who has used a templating engine?
* Who has programmed a GPU before?
* Using OpenGL / shaders ?
* Using CUDA (runtime? / driver?)
* Using PyCUDA ?
* Using OpenCL / PyOpenCL ?
* Using cudamat / gnumpy ?
* Other?
* Who has used Cython?
Python in one slide
-------------------
* General-purpose high-level OO interpreted language
* Emphasizes code readability
* Comprehensive standard library
* Dynamic type and memory management
* Built-in types: int, float, str, list, dict, tuple, object
* Slow execution
* Popular in web-dev and scientific communities
.. code-block:: python
#######################
# PYTHON SYNTAX EXAMPLE
#######################
a = 1 # no type declaration required!
b = (1, 2, 3) # tuple of three int literals
c = [1, 2, 3] # list of three int literals
d = {'a': 5, b: None} # dictionary of two elements
# N.B. string literal, None
print d['a'] # square brackets index
# -> 5
print d[(1, 2, 3)] # new tuple == b, retrieves None
# -> None
print d[6]
# raises KeyError Exception
x, y, z = 10, 100, 100 # multiple assignment from tuple
x, y, z = b # unpacking a sequence
b_squared = [b_i**2 for b_i in b] # list comprehension
def foo(b, c=3): # function w default param c
return a + b + c # note scoping, indentation
foo(5) # calling a function
# -> 1 + 5 + 3 == 9 # N.B. scoping
foo(b=6, c=2) # calling with named args
# -> 1 + 6 + 2 == 9
print b[1:3] # slicing syntax
class Foo(object): # Defining a class
def __init__(self):
self.a = 5
def hello(self):
return self.a
f = Foo() # Creating a class instance
print f.hello() # Calling methods of objects
# -> 5
class Bar(Foo): # Defining a subclass
def __init__(self, a):
self.a = a
print Bar(99).hello() # Creating an instance of Bar
# -> 99
NumPy in one slide
------------------
* Python floats are full-fledged objects on the heap
* Not suitable for high-performance computing!
* NumPy provides a N-dimensional numeric array in Python
* Perfect for high-performance computing.
* NumPy provides
* elementwise computations
* linear algebra, Fourier transforms
* pseudorandom numbers from many distributions
* SciPy provides lots more, including
* more linear algebra
* solvers and optimization algorithms
* matlab-compatible I/O
* I/O and signal processing for images and audio
.. code-block:: python
##############################
# Properties of NumPy arrays
# that you really need to know
##############################
import numpy as np # import can rename
a = np.random.rand(3, 4, 5) # random generators
a32 = a.astype('float32') # arrays are strongly typed
a.ndim # int: 3
a.shape # tuple: (3, 4, 5)
a.size # int: 60
a.dtype # np.dtype object: 'float64'
a32.dtype # np.dtype object: 'float32'
Arrays can be combined with numeric operators, standard mathematical
functions. NumPy has great `documentation >> x = theano.tensor.fmatrix() # doctest: +SKIP
>>> f = theano.function([x], PyCUDADoubleOp()(x)) # doctest: +SKIP
>>> xv=numpy.ones((4,5), dtype="float32") # doctest: +SKIP
>>> assert numpy.allclose(f(xv), xv*2) # doctest: +SKIP
>>> print numpy.asarray(f(xv)) # doctest: +SKIP
Exercises 7
-----------
- Run the above example
- Modify and execute the example to multiple two matrix: x * y
- Modify and execute the example to return 2 outputs: x + y and x - y
- Our current elemwise fusion generate computation with only 1 outputs
- Modify and execute the example to support stride? (Don't force the input to be c contiguous)
Theano-1.0.1/doc/cifarSC2011/theano.txt 0000777 0000000 0000000 00000025163 13212236347 017310 0 ustar root root 0000000 0000000
.. _theano:
******
Theano
******
Pointers
--------
* http://deeplearning.net/software/theano/
* Announcements mailing list: http://groups.google.com/group/theano-announce
* User mailing list: http://groups.google.com/group/theano-users
* Deep Learning Tutorials: http://www.deeplearning.net/tutorial/
* Installation: https://deeplearning.net/software/theano/install.html
Description
-----------
* Mathematical symbolic expression compiler
* Dynamic C/CUDA code generation
* Efficient symbolic differentiation
* Theano computes derivatives of functions with one or many inputs.
* Speed and stability optimizations
* Gives the right answer for ``log(1+x)`` even if x is really tiny.
* Works on Linux, Mac and Windows
* Transparent use of a GPU
* float32 only for now (working on other data types)
* Still in experimental state on Windows
* On GPU data-intensive calculations are typically between 6.5x and 44x faster. We've seen speedups up to 140x
* Extensive unit-testing and self-verification
* Detects and diagnoses many types of errors
* On CPU, common machine learning algorithms are 1.6x to 7.5x faster than competitive alternatives
* including specialized implementations in C/C++, NumPy, SciPy, and Matlab
* Expressions mimic NumPy's syntax & semantics
* Statically typed and purely functional
* Some sparse operations (CPU only)
* The project was started by James Bergstra and Olivier Breuleux
* For the past 1-2 years, I have replaced Olivier as lead contributor
Simple example
--------------
>>> import theano
>>> a = theano.tensor.vector("a") # declare symbolic variable
>>> b = a + a**10 # build symbolic expression
>>> f = theano.function([a], b) # compile function
>>> f([0,1,2])
array([ 0., 2., 1026.])
====================================================== =====================================================
Unoptimized graph Optimized graph
====================================================== =====================================================
.. image:: ../hpcs2011_tutorial/pics/f_unoptimized.png .. image:: ../hpcs2011_tutorial/pics/f_optimized.png
====================================================== =====================================================
Symbolic programming = *Paradigm shift*: people need to use it to understand it.
Exercise 1
-----------
.. code-block:: python
import theano
a = theano.tensor.vector() # declare variable
out = a + a**10 # build symbolic expression
f = theano.function([a], out) # compile function
print f([0,1,2])
# prints `array([0,2,1026])`
theano.printing.pydotprint_variables(b, outfile="f_unoptimized.png", var_with_name_simple=True)
theano.printing.pydotprint(f, outfile="f_optimized.png", var_with_name_simple=True)
Modify and execute the example to do this expression: a**2 + b**2 + 2*a*b
Real example
------------
**Logistic Regression**
* GPU-ready
* Symbolic differentiation
* Speed optimizations
* Stability optimizations
.. code-block:: python
import numpy
import theano
import theano.tensor as T
rng = numpy.random
N = 400
feats = 784
D = (rng.randn(N, feats), rng.randint(size=N,low=0, high=2))
training_steps = 10000
# Declare Theano symbolic variables
x = T.matrix("x")
y = T.vector("y")
w = theano.shared(rng.randn(feats), name="w")
b = theano.shared(0., name="b")
print "Initial model:"
print w.get_value(), b.get_value()
# Construct Theano expression graph
p_1 = 1 / (1 + T.exp(-T.dot(x, w)-b)) # Probability that target = 1
prediction = p_1 > 0.5 # The prediction thresholded
xent = -y*T.log(p_1) - (1-y)*T.log(1-p_1) # Cross-entropy loss function
cost = xent.mean() + 0.01*(w**2).sum() # The cost to minimize
gw,gb = T.grad(cost, [w,b])
# Compile
train = theano.function(
inputs=[x,y],
outputs=[prediction, xent],
updates={w:w-0.1*gw, b:b-0.1*gb})
predict = theano.function(inputs=[x], outputs=prediction)
# Train
for i in range(training_steps):
pred, err = train(D[0], D[1])
print "Final model:"
print w.get_value(), b.get_value()
print "target values for D:", D[1]
print "prediction on D:", predict(D[0])
**Optimizations:**
Where are those optimization applied?
* ``log(1+exp(x))``
* ``1 / (1 + T.exp(var))`` (sigmoid)
* ``log(1-sigmoid(var))`` (softplus, stabilisation)
* GEMV (matrix-vector multiply from BLAS)
* Loop fusion
.. code-block:: python
p_1 = 1 / (1 + T.exp(-T.dot(x, w)-b))
# 1 / (1 + T.exp(var)) -> sigmoid(var)
xent = -y*T.log(p_1) - (1-y)*T.log(1-p_1)
# Log(1-sigmoid(var)) -> -sigmoid(var)
prediction = p_1 > 0.5
cost = xent.mean() + 0.01*(w**2).sum()
gw,gb = T.grad(cost, [w,b])
train = theano.function(
inputs=[x,y],
outputs=[prediction, xent],
# w-0.1*gw: GEMV with the dot in the grad
updates={w:w-0.1*gw, b:b-0.1*gb})
Theano flags
------------
Theano can be configured with flags. They can be defined in two ways
* With an environment variable: ``THEANO_FLAGS="profile=True,profile_memory=True"``
* With a configuration file that defaults to ``~/.theanorc``
Exercise 2
-----------
.. code-block:: python
import numpy
import theano
import theano.tensor as T
rng = numpy.random
N = 400
feats = 784
D = (rng.randn(N, feats).astype(theano.config.floatX),
rng.randint(size=N,low=0, high=2).astype(theano.config.floatX))
training_steps = 10000
# Declare Theano symbolic variables
x = T.matrix("x")
y = T.vector("y")
w = theano.shared(rng.randn(feats).astype(theano.config.floatX), name="w")
b = theano.shared(numpy.asarray(0., dtype=theano.config.floatX), name="b")
x.tag.test_value = D[0]
y.tag.test_value = D[1]
#print "Initial model:"
#print w.get_value(), b.get_value()
# Construct Theano expression graph
p_1 = 1 / (1 + T.exp(-T.dot(x, w)-b)) # Probability of having a one
prediction = p_1 > 0.5 # The prediction that is done: 0 or 1
xent = -y*T.log(p_1) - (1-y)*T.log(1-p_1) # Cross-entropy
cost = xent.mean() + 0.01*(w**2).sum() # The cost to optimize
gw,gb = T.grad(cost, [w,b])
# Compile expressions to functions
train = theano.function(
inputs=[x,y],
outputs=[prediction, xent],
updates={w:w-0.01*gw, b:b-0.01*gb},
name = "train")
predict = theano.function(inputs=[x], outputs=prediction,
name = "predict")
if any( [x.op.__class__.__name__=='Gemv' for x in
train.maker.fgraph.toposort()]):
print 'Used the cpu'
elif any( [x.op.__class__.__name__=='GpuGemm' for x in
train.maker.fgraph.toposort()]):
print 'Used the gpu'
else:
print 'ERROR, not able to tell if theano used the cpu or the gpu'
print train.maker.fgraph.toposort()
for i in range(training_steps):
pred, err = train(D[0], D[1])
#print "Final model:"
#print w.get_value(), b.get_value()
print "target values for D"
print D[1]
print "prediction on D"
print predict(D[0])
# Print the graph used in the slides
theano.printing.pydotprint(predict,
outfile="pics/logreg_pydotprint_predic.png",
var_with_name_simple=True)
theano.printing.pydotprint_variables(prediction,
outfile="pics/logreg_pydotprint_prediction.png",
var_with_name_simple=True)
theano.printing.pydotprint(train,
outfile="pics/logreg_pydotprint_train.png",
var_with_name_simple=True)
Modify and execute the example to run on CPU with floatX=float32
* You will need to use: ``theano.config.floatX`` and ``ndarray.astype("str")``
GPU
---
* Only 32 bit floats are supported (being worked on)
* Only 1 GPU per process
* Use the Theano flag ``device=gpu`` to tell to use the GPU device
* Use ``device=gpu{0, 1, ...}`` to specify which GPU if you have more than one
* Shared variables with float32 dtype are by default moved to the GPU memory space
* Use the Theano flag ``floatX=float32``
* Be sure to use ``floatX`` (``theano.config.floatX``) in your code
* Cast inputs before putting them into a shared variable
* Cast "problem": int32 with float32 to float64
* A new casting mechanism is being developed
* Insert manual cast in your code or use [u]int{8,16}
* Insert manual cast around the mean operator (which involves a division by the length, which is an int64!)
Exercise 3
-----------
* Modify and execute the example of `Exercise 2`_ to run with floatX=float32 on GPU
* Time with: ``time python file.py``
Symbolic variables
------------------
* # Dimensions
* T.scalar, T.vector, T.matrix, T.tensor3, T.tensor4
* Dtype
* T.[fdczbwil]vector (float32, float64, complex64, complex128, int8, int16, int32, int64)
* T.vector to floatX dtype
* floatX: configurable dtype that can be float32 or float64.
* Custom variable
* All are shortcuts to: ``T.tensor(dtype, broadcastable=[False]*nd)``
* Other dtype: uint[8,16,32,64], floatX
Creating symbolic variables: Broadcastability
* Remember what I said about broadcasting?
* How to add a row to all rows of a matrix?
* How to add a column to all columns of a matrix?
Details regarding symbolic broadcasting...
* Broadcastability must be specified when creating the variable
* The only shorcut with broadcastable dimensions are: **T.row** and **T.col**
* For all others: ``T.tensor(dtype, broadcastable=([False or True])*nd)``
Differentiation details
-----------------------
>>> gw,gb = T.grad(cost, [w,b]) # doctest: +SKIP
* T.grad works symbolically: takes and returns a Theano variable
* T.grad can be compared to a macro: it can be applied multiple times
* T.grad takes scalar costs only
* Simple recipe allows to compute efficiently vector x Jacobian and vector x Hessian
* We are working on the missing optimizations to be able to compute efficently the full Jacobian and Hessian and Jacobian x vector
.. _cifar2011_benchmark:
Benchmarks
----------
**Multi-Layer Perceptron**:
60x784 matrix times 784x500 matrix, tanh, times 500x10 matrix, elemwise, then all in reverse for backpropagation
.. image:: ../hpcs2011_tutorial/pics/mlp.png
**Convolutional Network**:
256x256 images convolved with 6 7x7 filters,
downsampled to 6x50x50, tanh, convolution with 16 6x7x7 filter, elementwise
tanh, matrix multiply, softmax elementwise, then in reverse
.. image:: ../hpcs2011_tutorial/pics/conv.png
**Elemwise**
* All on CPU
* Solid blue: Theano
* Dashed Red: numexpr (without MKL)
.. image:: ../hpcs2011_tutorial/pics/multiple_graph.png
Theano-1.0.1/doc/citation.txt 0000777 0000000 0000000 00000003507 13212236347 016024 0 ustar root root 0000000 0000000 :orphan:
.. _citation:
Theano Citation Policy
======================
If you use Theano for academic research, you are highly encouraged (though not
required) to cite the following paper:
* Theano Development Team. `"Theano: A Python framework for fast computation of mathematical expressions"
`_.
(:download:`short BibTeX `, :download:`full BibTeX `)
Theano is primarily developed by academics, and so citations matter a lot to
us. As an added benefit, you increase Theano's exposure and potential user
(and developer) base, which is to the benefit of all users of Theano. Thanks
in advance!
Previously, we asked users of Theano to cite the two papers below.
However, many members of our community have contributed to Theano in the
meantime, and it is inconvenient to cite multiple papers. We intend to
introduce new papers periodically, and only ask for users to cite the
single most recent paper with the most comprehensive author list.
Earlier articles:
* F. Bastien, P. Lamblin, R. Pascanu, J. Bergstra, I. Goodfellow,
A. Bergeron, N. Bouchard, D. Warde-Farley and Y. Bengio.
`"Theano: new features and speed improvements"
`_.
NIPS 2012 deep learning workshop. (`BibTeX
`__)
* J. Bergstra, O. Breuleux, F. Bastien, P. Lamblin, R.
Pascanu, G. Desjardins, J. Turian, D. Warde-Farley and Y.
Bengio. `"Theano: A CPU and GPU Math Expression Compiler"
`_.
*Proceedings of the Python for Scientific Computing Conference (SciPy)
2010. June 30 - July 3, Austin, TX* (`BibTeX
`__)
Theano-1.0.1/doc/core_development_guide.txt 0000777 0000000 0000000 00000002601 13212236347 020713 0 ustar root root 0000000 0000000 :orphan:
Core Development Guide
=======================
The documentation of the core components of Theano is still a work in
progress. For now this is a list of bits and pieces on the subject,
some of them might be outdated though:
* :ref:`theano_type` -- Tutorial for writing a new type in Theano. It
introduces the basics concerning Theano datatypes.
* :ref:`theano_ctype` -- Tutorial on how to make your type C-friendly.
* :ref:`views_and_inplace` -- This is somewhere between extending Theano and
describing how Theano works internally; it talks about views and inplace
operations.
* :ref:`optimization` -- Tutorial on how optimization work in Theano.
* :ref:`pipeline` -- Describes the steps of compiling a Theano Function.
* :ref:`graphstructures` -- Describes the symbolic graphs generated by
:mod:`theano.scan`.
* :ref:`unittest` -- Tutorial on how to use unittest in testing Theano.
* :ref:`sandbox_debugging_step_mode` -- How to step through the execution of
a Theano function and print the inputs and outputs of each op.
* :ref:`sandbox_elemwise` -- Description of element wise operations.
* :ref:`sandbox_maxgotcha` -- Describes the difference between ``numpy.max``
and Python max (something to consider when using max).
* :ref:`sandbox_randnb` -- Description of how Theano deals with random
numbers.
* :ref:`sparse` -- Description of the ``sparse`` type in Theano.
Theano-1.0.1/doc/crei2013/ 0000777 0000000 0000000 00000000000 13212236507 014667 5 ustar root root 0000000 0000000 Theano-1.0.1/doc/crei2013/advanced_theano.txt 0000777 0000000 0000000 00000020652 13212236347 020545 0 ustar root root 0000000 0000000
.. _crei2013_advanced_theano:
***************
Advanced Theano
***************
Profiling
---------
- To replace the default mode with this mode, use the Theano flags ``profile=True``
- To enable the memory profiling use the flags ``profile_memory=True``
Theano output:
.. literalinclude:: logreg_profile.prof
Compilation pipeline
--------------------
.. image:: ../hpcs2011_tutorial/pics/pipeline.png
:width: 400 px
Inplace optimization
--------------------
- 2 type of inplace operations:
- An op that return a view on its inputs (e.g. reshape, inplace transpose)
- An op that write the output on the inputs memory space
- This allows some memory optimization
- The Op must tell Theano if they work inplace
- Inplace Op add constraints to the order of execution
Conditions
----------
**IfElse**
- Build condition over symbolic variables.
- IfElse Op takes a boolean condition and two variables to compute as input.
- While Switch Op evaluates both 'output' variables, IfElse Op is lazy and only
evaluates one variable respect to the condition.
**IfElse Example: Comparison with Switch**
.. literalinclude:: ifelse_switch.py
IfElse Op spend less time (about an half) than Switch since it computes only
one variable instead of both.
>>> python ifelse_switch.py # doctest: +SKIP
time spent evaluating both values 0.230000 sec
time spent evaluating one value 0.120000 sec
Note that IfElse condition is a boolean while Switch condition is a tensor, so
Switch is more general.
It is actually important to use ``linker='vm'`` or ``linker='cvm'``,
otherwise IfElse will compute both variables and take the same computation
time as the Switch Op. The linker is not currently set by default to 'cvm' but
it will be in a near future.
Loops
-----
**Scan**
- General form of **recurrence**, which can be used for looping.
- **Reduction** and **map** (loop over the leading dimensions) are special cases of Scan
- You 'scan' a function along some input sequence, producing an output at each time-step
- The function can see the **previous K time-steps** of your function
- ``sum()`` could be computed by scanning the z + x(i) function over a list, given an initial state of ``z=0``.
- Often a for-loop can be expressed as a ``scan()`` operation, and ``scan`` is the closest that Theano comes to looping.
- The advantage of using ``scan`` over for loops
- The number of iterations to be part of the symbolic graph
- Minimizes GPU transfers if GPU is involved
- Compute gradients through sequential steps
- Slightly faster then using a for loop in Python with a compiled Theano function
- Can lower the overall memory usage by detecting the actual amount of memory needed
**Scan Example: Computing pow(A,k)**
.. literalinclude:: scan_pow.py
**Scan Example: Calculating a Polynomial**
.. literalinclude:: scan_poly.py
Exercise 4
-----------
- Run both examples
- Modify and execute the polynomial example to have the reduction done by scan
Exercise 5
-----------
- In the last exercises, do you see a speed up with the GPU?
- Where does it come from? (Use profile=True)
- Is there something we can do to speed up the GPU version?
Printing/Drawing Theano graphs
------------------------------
Consider the following logistic regression model:
>>> import numpy
>>> import theano
>>> import theano.tensor as T
>>> rng = numpy.random
>>> # Training data
>>> N = 400
>>> feats = 784
>>> D = (rng.randn(N, feats).astype(theano.config.floatX), rng.randint(size=N,low=0, high=2).astype(theano.config.floatX))
>>> training_steps = 10000
>>> # Declare Theano symbolic variables
>>> x = T.matrix("x")
>>> y = T.vector("y")
>>> w = theano.shared(rng.randn(feats).astype(theano.config.floatX), name="w")
>>> b = theano.shared(numpy.asarray(0., dtype=theano.config.floatX), name="b")
>>> x.tag.test_value = D[0]
>>> y.tag.test_value = D[1]
>>> # Construct Theano expression graph
>>> p_1 = 1 / (1 + T.exp(-T.dot(x, w)-b)) # Probability of having a one
>>> prediction = p_1 > 0.5 # The prediction that is done: 0 or 1
>>> # Compute gradients
>>> xent = -y*T.log(p_1) - (1-y)*T.log(1-p_1) # Cross-entropy
>>> cost = xent.mean() + 0.01*(w**2).sum() # The cost to optimize
>>> gw,gb = T.grad(cost, [w,b])
>>> # Training and prediction function
>>> train = theano.function(inputs=[x,y], outputs=[prediction, xent], updates=[[w, w-0.01*gw], [b, b-0.01*gb]], name = "train")
>>> predict = theano.function(inputs=[x], outputs=prediction, name = "predict")
We will now make use of Theano's printing features to compare the unoptimized
graph (``prediction``) to the optimized graph (``predict``).
Pretty Printing
~~~~~~~~~~~~~~~
>>> theano.printing.pprint(prediction) # doctest: +NORMALIZE_WHITESPACE
'gt((TensorConstant{1} / (TensorConstant{1} + exp(((-(x \\dot w)) - b)))), TensorConstant{0.5})'
Debug Print
~~~~~~~~~~~
The graph before optimization:
.. doctest::
:options: +SKIP
>>> theano.printing.debugprint(prediction) # doctest: +NORMALIZE_WHITESPACE
Elemwise{gt,no_inplace} [@A] ''
|Elemwise{true_div,no_inplace} [@B] ''
| |DimShuffle{x} [@C] ''
| | |TensorConstant{1} [@D]
| |Elemwise{add,no_inplace} [@E] ''
| |DimShuffle{x} [@F] ''
| | |TensorConstant{1} [@D]
| |Elemwise{exp,no_inplace} [@G] ''
| |Elemwise{sub,no_inplace} [@H] ''
| |Elemwise{neg,no_inplace} [@I] ''
| | |dot [@J] ''
| | |x [@K]
| | |w [@L]
| |DimShuffle{x} [@M] ''
| |b [@N]
|DimShuffle{x} [@O] ''
|TensorConstant{0.5} [@P]
The graph after optimization:
.. doctest::
:options: +SKIP
>>> theano.printing.debugprint(predict) # doctest: +NORMALIZE_WHITESPACE
Elemwise{Composite{GT(scalar_sigmoid((-((-i0) - i1))), i2)}} [@A] '' 4
|CGemv{inplace} [@B] '' 3
| |Alloc [@C] '' 2
| | |TensorConstant{0.0} [@D]
| | |Shape_i{0} [@E] '' 1
| | |x [@F]
| |TensorConstant{1.0} [@G]
| |x [@F]
| |w [@H]
| |TensorConstant{0.0} [@D]
|InplaceDimShuffle{x} [@I] '' 0
| |b [@J]
|TensorConstant{(1,) of 0.5} [@K]
Picture Printing of Graphs
~~~~~~~~~~~~~~~~~~~~~~~~~~
``pydotprint`` requires graphviz and either pydot or pydot-ng.
The graph before optimization:
.. doctest::
:options: +SKIP
>>> theano.printing.pydotprint(prediction, outfile="pics/logreg_pydotprint_prediction.png", var_with_name_simple=True)
The output file is available at pics/logreg_pydotprint_prediction.png
.. image:: ./pics/logreg_pydotprint_prediction.png
:width: 800 px
The graph after optimization:
.. doctest::
:options: +SKIP
>>> theano.printing.pydotprint(predict, outfile="pics/logreg_pydotprint_predict.png", var_with_name_simple=True)
The output file is available at pics/logreg_pydotprint_predict.png
.. image:: ./pics/logreg_pydotprint_predict.png
:width: 800 px
The optimized training graph:
.. doctest::
:options: +SKIP
>>> theano.printing.pydotprint(train, outfile="pics/logreg_pydotprint_train.png", var_with_name_simple=True)
The output file is available at pics/logreg_pydotprint_train.png
.. image:: ./pics/logreg_pydotprint_train.png
:width: 1500 px
Debugging
---------
- Run with the Theano flag ``compute_test_value = {``off'',``ignore'', ``warn'', ``raise''}``
- Run the code as we create the graph
- Allows you to find the bug earlier (ex: shape mismatch)
- Makes it easier to identify where the problem is in *your* code
- Use the value of constants and shared variables directly
- For pure symbolic variables uses ``x.tag.test_value = numpy.random.rand(5,10)``
- Run with the flag ``mode=FAST_COMPILE``
- Few optimizations
- Run Python code (better error messages and can be debugged interactively in the Python debugger)
- Run with the flag ``mode=DebugMode``
- 100-1000x slower
- Test all optimization steps from the original graph to the final graph
- Checks many things that Op should/shouldn't do
- Executes both the Python and C code versions
Known limitations
-----------------
- Compilation phase distinct from execution phase
- Use ``a_tensor_variable.eval()`` to make this less visible
- Compilation time can be significant
- Amortize it with functions over big input or reuse functions
- Execution overhead
- We have worked on this, but more work needed
- So needs a certain number of operations to be useful
- Compilation time superlinear in the size of the graph.
- Hundreds of nodes is fine
- Disabling a few optimizations can speed up compilation
- Usually too many nodes indicates a problem with the graph
Theano-1.0.1/doc/crei2013/gpundarray.txt 0000777 0000000 0000000 00000002275 13212236347 017617 0 ustar root root 0000000 0000000
.. _crei2013_gpundarray:
**********
GpuNdArray
**********
Why a common GPU ndarray?
-------------------------
- Currently there are at least 4 different GPU array data structures in use by Python packages
- CudaNdarray (Theano), GPUArray (PyCUDA), CUDAMatrix (cudamat), GPUArray (PyOpenCL), ...
- There are even more if we include other languages
- All of them are a subset of the functionality of ``numpy.ndarray`` on the GPU
- Lots of duplicated effort
- GPU code is harder/slower to do {\bf correctly} and {\bf fast} than on the CPU/Python
- Lack of a common array API makes it harder to port/reuse code
- Also harder to find/distribute code
- Divides development work
Design Goals
------------
- Make it VERY similar to ``numpy.ndarray``
- Be compatible with both CUDA and OpenCL
- Have the base object accessible from C to allow collaboration with more projects, across high-level languages
- We want people from C, C++, Ruby, R, ... all use the same base GPU N-dimensional array
Final Note
----------
- Under development
- Will be the next GPU array container for Theano (*this summer!*)
- Probably also for PyCUDA, PyOpenCL
- Mailing list: http://lists.tiker.net/listinfo/gpundarray
Theano-1.0.1/doc/crei2013/index.txt 0000777 0000000 0000000 00000003442 13212236347 016547 0 ustar root root 0000000 0000000 :orphan:
.. _crei2013_index:
===========================
Theano Tutorial @ CREI 2013
===========================
July 19, 2013, Sherbrook, Québec, Canada.
Theano is python software for evaluating complicated array expressions.
What does it do?
* aggressive expression optimizations,
* automatic GPU use,
* symbolic differentiation and R op.
It complements the Python numeric/scientific software stack (e.g. NumPy, SciPy,
scikits, matplotlib, PIL.)
Design and feature set has been driven by machine learning research
at the University of
Montreal (groups of Yoshua Bengio, Pascal Vincent, Aaron Courville and Roland Memisevic)
The result is a very good library for doing research in deep
learning and neural network training, and a flexible framework for
many other models and algorithms in machine learning more generally.
It has proven to be useful for implementing:
- linear and nonlinear neural network classifiers
- convolutional models
- Energy models: RBM, DBN, GRBM, ssRBM, AIS
- Auto-encoders: DAE, CAE
- GP regression
- sparse coding
- recurrent neural networks, echo state, (HMM?)
- online and batch learning and optimization
- Even SVM!
As people's needs change this list will grow, but Theano is built
around vector, matrix, and tensor expressions; there is little reason
to use it for calculations on other data structures except. There is
also some sparse matrix support.
Contents
--------
The structured part of these lab sessions will be a walk-through of the following
material. Interleaved with this structured part will be blocks of time for
individual or group work. The idea is that you can try out Theano and get help
from gurus on hand if you get stuck.
.. toctree::
introduction
theano
advanced_theano
gpundarray
/extending/extending_theano
Theano-1.0.1/doc/crei2013/introduction.txt 0000777 0000000 0000000 00000023112 13212236347 020155 0 ustar root root 0000000 0000000
.. _crei2013_Introduction:
************
Introduction
************
Background Questionaire
-----------------------
* Who has used Theano before?
* What did you do with it?
* Who has used Python? NumPy? SciPy? matplotlib?
* Who has used iPython?
* Who has used it as a distributed computing engine?
* Who has done C/C++ programming?
* Who has organized computation around a particular physical memory layout?
* Who has used a multidimensional array of >2 dimensions?
* Who has written a Python module in C before?
* Who has written a program to *generate* Python modules in C?
* Who has used a templating engine?
* Who has programmed a GPU before?
* Using OpenGL / shaders ?
* Using CUDA (runtime? / driver?)
* Using PyCUDA ?
* Using OpenCL / PyOpenCL ?
* Using cudamat / gnumpy ?
* Other?
* Who has used Cython?
Python in one slide
-------------------
* General-purpose high-level OO interpreted language
* Emphasizes code readability
* Comprehensive standard library
* Dynamic type and memory management
* Built-in types: int, float, str, list, dict, tuple, object
* Slow execution
* Popular in web-dev and scientific communities
.. code-block:: python
#######################
# PYTHON SYNTAX EXAMPLE
#######################
a = 1 # no type declaration required!
b = (1, 2, 3) # tuple of three int literals
c = [1, 2, 3] # list of three int literals
d = {'a': 5, b: None} # dictionary of two elements
# N.B. string literal, None
print d['a'] # square brackets index
# -> 5
print d[(1, 2, 3)] # new tuple == b, retrieves None
# -> None
print d[6]
# raises KeyError Exception
x, y, z = 10, 100, 100 # multiple assignment from tuple
x, y, z = b # unpacking a sequence
b_squared = [b_i**2 for b_i in b] # list comprehension
def foo(b, c=3): # function w default param c
return a + b + c # note scoping, indentation
foo(5) # calling a function
# -> 1 + 5 + 3 == 9 # N.B. scoping
foo(b=6, c=2) # calling with named args
# -> 1 + 6 + 2 == 9
print b[1:3] # slicing syntax
class Foo(object): # Defining a class
def __init__(self):
self.a = 5
def hello(self):
return self.a
f = Foo() # Creating a class instance
print f.hello() # Calling methods of objects
# -> 5
class Bar(Foo): # Defining a subclass
def __init__(self, a):
self.a = a
print Bar(99).hello() # Creating an instance of Bar
# -> 99
NumPy in one slide
------------------
* Python floats are full-fledged objects on the heap
* Not suitable for high-performance computing!
* NumPy provides a N-dimensional numeric array in Python
* Perfect for high-performance computing.
* Slice are return view (no copy)
* NumPy provides
* elementwise computations
* linear algebra, Fourier transforms
* pseudorandom numbers from many distributions
* SciPy provides lots more, including
* more linear algebra
* solvers and optimization algorithms
* matlab-compatible I/O
* I/O and signal processing for images and audio
.. code-block:: python
##############################
# Properties of NumPy arrays
# that you really need to know
##############################
import numpy as np # import can rename
a = np.random.rand(3, 4, 5) # random generators
a32 = a.astype('float32') # arrays are strongly typed
a.ndim # int: 3
a.shape # tuple: (3, 4, 5)
a.size # int: 60
a.dtype # np.dtype object: 'float64'
a32.dtype # np.dtype object: 'float32'
assert a[1, 1, 1] != 10 # a[1, 1, 1] is a view
a[1, 1, 1] = 10 # So affectation to it change the
assert a[1, 1, 1] == 10 # original array
Arrays can be combined with numeric operators, standard mathematical
functions. NumPy has great `documentation `
* Multi-layer perceptron
* Convolutional Neural Networks
* Misc Elemwise operations
Competitors: NumPy + SciPy, MATLAB, EBLearn, Torch5, numexpr
* EBLearn, Torch5: specialized libraries written by practitioners specifically for these tasks
* numexpr: similar to Theano, 'virtual machine' for elemwise expressions
New Benchmarks
--------------
`Example `_ (Page 7 and 9):
* Logistic regression, MLP with 1 and 3 layers
* Recurrent neural networks
Competitors: Torch7, RNNLM
* Torch7, RNNLM: specialized libraries written by practitioners specifically for these tasks
Theano-1.0.1/doc/dev_start_guide.txt 0000777 0000000 0000000 00000050126 13212236347 017361 0 ustar root root 0000000 0000000 .. _dev_start_guide:
=====================
Developer Start Guide
=====================
Contributing
============
You want to contribute to Theano? That is great! This page explain our
workflow and some resource for doing so.
Looking for an idea for a first contribution? Check the `github issues
`_
with a label ``easy fix``. They are good starter. It is recommended
that you write on the issue you want to work on it. This help make
sure it is up to date and see if nobody else is working on it. Also,
we can sometimes provides more information about it. There is also
the label `NeedSomeoneToFinish
`_ that is
interesting to check. The difficulty level is variable.
Resources
=========
See :ref:`theano_community` for a list of Theano resources. The
following groups/mailing-lists are especially useful to Theano
contributors: `theano-dev`_, `theano-buildbot`_, and `theano-github`_.
.. _theano-dev: https://groups.google.com/group/theano-dev
.. _theano-github: https://groups.google.com/group/theano-github
.. _theano-buildbot: https://groups.google.com/group/theano-buildbot
To get up to speed, you'll need to
- Learn some non-basic Python to understand what's going on in some of the
trickier files (like tensor.py).
- Go through the `NumPy documentation`_.
- Learn to write reStructuredText_ for Sphinx_.
- Learn about how unittest_ and nose_ work
.. _Sphinx: http://sphinx.pocoo.org/
.. _reStructuredText: http://docutils.sourceforge.net/rst.html
.. _Allowed docstring sections in Napoleon: https://sphinxcontrib-napoleon.readthedocs.org/en/latest/#docstring-sections
.. _NumPy documentation: http://docs.scipy.org/numpy/
.. _unittest: http://docs.python.org/library/unittest.html
.. _nose: http://nose.readthedocs.org/en/latest/
.. _quality_contributions:
Requirements for Quality Contributions
======================================
* All the code should be properly tested.
* The code should be compatible with Python 2.7 and above, as well as
Python 3.4 and above (using `six` if needed).
* All the code should respect the
`PEP8 Code Style Guide `_.
* The docstrings of all the classes and functions should respect the
`PEP257 `_.
Each point will be referred to more in detail in the following.
Unit tests
----------
When you submit a pull request, your changes will automatically be
tested via Travis-CI. This will post the results of the tests with a
little icon next to your commit. A yellow circle means the tests are
running. A red X means the tests failed and a green circle means the
tests passed.
Just because the tests run automatically does not mean you shouldn't
run them yourself to make sure everything is all right. You can run
only the portion you are modifying to go faster and have Travis to
make sure there are no global impacts.
Also, if you are changing GPU code, Travis doesn't test that, because
there are no GPUs on the test nodes.
To run the test suite with the default options, see
:ref:`test_theano`.
Each night we execute all the unit tests automatically, with several
sets of options. The result is sent by email to the `theano-buildbot`_
mailing list.
For more detail, see :ref:`metadocumentation_nightly_build`.
To run all the tests with the same configuration as the buildbot, run
this script:
.. code-block:: bash
theano/misc/do_nightly_build
This script accepts arguments that it forwards to nosetests. You can
run only some tests or enable pdb by giving the equivalent nosetests
parameters.
Setting up your Editor for PEP8
-------------------------------
Here are instructions for :ref:`Vim ` and :ref:`Emacs
`. If you have similar instructions for other text editors
or IDE, please let us know and we will update this documentation.
.. _vim_pep8:
Vim
~~~
Detection of warnings and errors is done by the `pep8`_ script
(or `flake8`_, that also checks for other things, like syntax
errors). Syntax highlighting and general integration into Vim is done by
the `Syntastic`_ plugin for Vim.
To setup VIM:
#. Install flake8 (if not already installed) with::
pip install "flake8<3"
.. warning:: Starting version 3.0.0, flake8 changed its dependencies and
moved its Python API to a legacy module, breaking Theano's flake8 tests.
We recommend using a version prior to 3.
.. note:: You can use ``easy_install`` instead of ``pip``, and ``pep8``
instead of ``flake8`` if you prefer. The important thing is that the
``flake8`` or ``pep8`` executable ends up in your ``$PATH``.
#. Install vundle with::
git clone https://github.com/VundleVim/Vundle.vim.git ~/.vim/bundle/Vundle.vim
#. Edit ``~/.vimrc`` and add the lines:
.. code-block:: python
set nocompatible " be iMproved, required
filetype off " required
" set the runtime path to include Vundle and initialize
set rtp+=~/.vim/bundle/Vundle.vim
call vundle#begin()
Plugin 'gmarik/Vundle.vim' " let Vundle manage Vundle (required!)
Plugin 'scrooloose/syntastic'
Plugin 'jimf/vim-pep8-text-width'
call vundle#end()
" Syntastic settings
" You can run checkers explicitly by calling :SyntasticCheck :lprevious
map :lnext
You can prefix those by ``autocmd FileType python`` if you want these
bindings to work only on Python files.
.. _pep8: http://pypi.python.org/pypi/pep8
.. _flake8: http://pypi.python.org/pypi/flake8
.. _Syntastic: https://github.com/scrooloose/syntastic/
.. _pathogen.vim: https://github.com/tpope/vim-pathogen
.. _quickfix: http://vimdoc.sourceforge.net/htmldoc/quickfix.html#quickfix
.. _emacs_pep8:
Emacs
~~~~~
There is an **excellent** system to configure emacs for Python:
`emacs-for-python
`_. It gathers many
emacs config into one, and modifies them to behave together nicely. You
can use it to check for pep8 compliance and for Python syntax errors.
To install it on Linux, you can do like this:
.. code-block:: bash
cd
git clone https://github.com/gabrielelanaro/emacs-for-python.git ~/.emacs.d/emacs-for-python
Then in your ``~/.emacs`` file, add this:
.. code-block:: common-lisp
;; Mandatory
(load-file "~/.emacs.d/emacs-for-python/epy-init.el")
(add-to-list 'load-path "~/.emacs.d/emacs-for-python/") ;; tell where to load the various files
;; Each of them enables different parts of the system.
;; Only the first two are needed for pep8, syntax check.
(require 'epy-setup) ;; It will setup other loads, it is required!
(require 'epy-python) ;; If you want the python facilities [optional]
(require 'epy-completion) ;; If you want the autocompletion settings [optional]
(require 'epy-editing) ;; For configurations related to editing [optional]
;; [newer version of emacs-for-python]
(require 'epy-nose) ;; For shortcut to call nosetests [optional]
;; Define f10 to previous error
;; Define f11 to next error
(require 'epy-bindings) ;; For my suggested keybindings [optional]
;; Some shortcut that do not collide with gnome-terminal,
;; otherwise, "epy-bindings" define f10 and f11 for them.
(global-set-key [f2] 'flymake-goto-prev-error)
(global-set-key [f3] 'flymake-goto-next-error)
;; Next two lines are the checks to do. You can add more if you wish.
(epy-setup-checker "pyflakes %f") ;; For python syntax check
(epy-setup-checker "pep8 -r %f") ;; For pep8 check
.. note::
The script highlights problematic lines. This can make part of the
line not readable depending on the background. To replace the line
highlight by an underline, add this to your emacs configuration
file:
;; Make lines readable when there is an warning [optional]
(custom-set-faces
'(flymake-errline ((((class color)) (:underline "red"))))
'(flymake-warnline ((((class color)) (:underline "yellow")))))
Documentation and docstrings
----------------------------
* The documentation and the API documentation are generated using `Sphinx`_.
* The documentation should be written in `reStructuredText`_ and the
docstrings of all the classes and functions should respect the
`PEP257 `_.
* Split the docstrings in sections, according to the `Allowed docstring
sections in Napoleon`_
* To cross-reference other objects (e.g. reference other classes or methods) in
the docstrings, use the
`cross-referencing objects `_
syntax. ``:py`` can be omitted, see e.g. this
`stackoverflow answer `_.
* See :ref:`metadocumentation`, for some information on how to generate the
documentation.
A Docstring Example
~~~~~~~~~~~~~~~~~~~
Here is an example on how to add a docstring to a class.
.. testcode:: python
import theano
class DoubleOp(theano.Op):
"""
Double each element of a tensor.
Parameters
----------
x : tensor
Input tensor
Returns
-------
tensor
a tensor of the same shape and dtype as the input with all
values doubled.
Notes
-----
this is a test note
See Also
--------
:class:`~theano.tensor.elemwise.Elemwise` : You can use this to replace
this example. Just execute `x * 2` with x being a Theano variable.
.. versionadded:: 0.6
"""
This is how it will show up for files that we auto-list in the library
documentation:
.. automodule:: theano.misc.doubleop
:members:
Installation and configuration
==============================
To obtain developer access: register with `GitHub
`_.
This will create your own Theano project on GitHub, referred later
as "YourProfile/Theano", or "origin", from which you will be able to
contribute to the original Theano/Theano, also called "central".
Create a local copy
-------------------
Clone your fork locally with
.. code-block:: bash
git clone git@github.com:YOUR_GITHUB_LOGIN/Theano.git
For this URL to work, you must set your public ssh keys inside your
`github account setting `_.
From your local repository, your own fork on GitHub will be called "origin".
Then, add a reference to the original ("central") Theano repository with
.. code-block:: bash
git remote add central git://github.com/Theano/Theano.git
You can choose another name than "central" to reference Theano/Theano
(for instance, NumPy uses "upstream"), but this documentation will stick
to "central."
You can then test your installation of Theano by following the steps of
:ref:`test_theano`.
Using your local copy
---------------------
To update your library to the latest revision, you should have a local branch
that tracks central/master. You can add one (named "trunk" here) with:
.. code-block:: bash
git fetch central
git branch trunk central/master
Once you have such a branch, in order to update it, do:
.. code-block:: bash
git checkout trunk
git pull
Keep in mind that this branch should be "read-only": if you want to
patch Theano, you should work in another branch, like described in the
:ref:`dev_workflow` section below.
Configure Git
-------------
On your local machine, you need to configure git with basic information:
.. code-block:: bash
git config --global user.email you@yourdomain.example.com
git config --global user.name "Your Name Comes Here"
You can also instruct git to use color in diff. For this, you need to
add those lines in the file ~/.gitconfig
.. code-block:: cfg
[color]
branch = auto
diff = auto
interactive = auto
status = auto
.. _dev_workflow:
Development Workflow
====================
Start a new local branch
------------------------
When working on a new feature in your own fork, start from an up-to-date copy
of the `master` branch (the principal one) of the central repository
(Theano/Theano on GitHub):
.. code-block:: bash
git fetch central
git checkout -b my_shiny_feature central/master
.. note::
This last line is a shortcut for:
.. code-block:: bash
git branch my_shiny_feature central/master
git checkout my_shiny_feature
Submit your changes to the central repository
---------------------------------------------
Once your code is ready for others to review, you need to commit all the changes and then push your
branch to your github fork first:
.. code-block:: bash
git commit -a -m "your message here"
.. code-block:: bash
git push -u origin my_shiny_feature
Then, go to your fork's github page on the github website, select your
feature branch and hit the "Pull Request" button in the top right
corner. This will signal the maintainers that you wish to submit your
changes for inclusion in central/master.
If you don't get any feedback, bug us on the theano-dev mailing list.
Address reviewer comments
-------------------------
Your pull request will be reviewed by members of the core development
team. If your branch is not directly accepted, the reviewers will use
GitHub's system to add "notes", either general (on the entire commit),
or "line notes", relative to a particular line of code.
In order to have the pull request accepted, you may have to answer
the reviewer's questions, you can do that on GitHub.
You may also have to edit your code to address their concerns. Some
of the usual requests include fixing typos in comments, adding or
correcting comments, adding unit tests in the test suite. In order to
do that, you should continue your edits in the same branch you used (in
this example, "my_shiny_feature"). For instance, if you changed your
working branch, you should first:
.. code-block:: bash
git checkout my_shiny_feature
Then, edit your code, and test it appropriately (see
:ref:`quality_contributions` below), and push it again to your GitHub
fork, like the first time (except the ``-u`` option is only needed the
first time):
.. code-block:: bash
git push origin my_shiny_feature
The pull request to the central repository will then be automatically
updated by GitHub. However, the reviewers will not be automatically
notified of your revision, so it is advised to reply to the comments on
GitHub, to let them know that you have submitted a fix.
More Advanced Git Usage
=======================
You can find information and tips in the `numpy development
`_
page. Here are a few.
Cleaning up branches
--------------------
When your pull request has been merged, you can delete the branch from
your GitHub fork's list of branches. This is useful to avoid having too
many branches staying there. Deleting this remote branch is achieved
with:
.. code-block:: bash
git push origin :my_shiny_feature
This lines pushes to the "origin" repository (your fork of Theano on
GitHub), into the branch "my_shiny_feature", an empty content (that's
why there is nothing before the colon), effectively removing it.
The branch will still be present in your local clone of the repository.
If you want to delete it from there, too, you can run:
.. code-block:: bash
git branch -d my_shiny_feature
Amending a submitted pull request
---------------------------------
If you want to fix a commit already submitted within a pull request
(e.g. to fix a small typo), before the pull request is accepted, you can
do it like this to keep history clean:
.. code-block:: bash
git checkout my_shiny_feature
git commit --amend
git push origin my_shiny_feature:my_shiny_feature
Do not abuse that command, and please use it only when there are only
small issues to be taken care of. Otherwise, it becomes difficult to
match the comments made by reviewers with the new modifications.
In the general case, you should stick with the approach described above.
Cleaning up history
-------------------
Sometimes you may have commits in your feature branch that
are not needed in the final pull request. There is a `page
`_ that talks about
this. In summary:
* Commits to the trunk should be a lot cleaner than commits to your
feature branch; not just for ease of reviewing but also
because intermediate commits can break blame (the bisecting tool).
* `git merge --squash` will put all of the commits from your feature branch into one commit.
* There are other tools that are useful if your branch is too big for one squash.
Add another distant repository
------------------------------
To collaborate with another user on some feature he is developing, and
that is not ready for inclusion in central, the easiest way is to use a
branch of their Theano fork (usually on GitHub).
Just like we added Theano/Theano as a remote repository, named
"central", you can add (on your local machine) a reference to their fork
as a new remote repository. REPO_NAME is the name you choose to name
this fork, and GIT_REPO_PATH is the URL of the fork in question.
.. code-block:: bash
git remote add REPO_NAME GIT_REPO_PATH
Then, you can create a new local branch (LOCAL_BRANCH_NAME) based on
a specific branch (REMOTE_BRANCH_NAME) from the remote repository
(REPO_NAME):
.. code-block:: bash
git checkout -b LOCAL_BRANCH_NAME REPO_NAME/REMOTE_BRANCH_NAME
Other tools that can help you
=============================
* `cProfile `_: time profiler that work at function level.
* `Yep `_: A module for profiling compiled extensions.
* `autopep8 `_: A tool that adds github commands to the git command line.
* `git pull-requests `_: Another tool for git/github command line.
Theano-1.0.1/doc/extending/ 0000777 0000000 0000000 00000000000 13212236507 015424 5 ustar root root 0000000 0000000 Theano-1.0.1/doc/extending/cop.txt 0000777 0000000 0000000 00000034236 13212236347 016763 0 ustar root root 0000000 0000000 .. _cop:
====================================
Implementing the arithmetic Ops in C
====================================
Now that we have set up our ``double`` type properly to allow C
implementations for operations that work on it, all we have to do now
is to actually define these operations in C.
How does it work?
=================
Before a C :ref:`op` is executed, the variables related to each of its
inputs will be declared and will be filled appropriately, either from
an input provided by the end user (using c_extract) or it might simply
have been calculated by another operation. For each of the outputs,
the variables associated to them will be declared and initialized.
The operation then has to compute what it needs to using the
input variables and place the variables in the output variables.
What needs to be defined
========================
There are less methods to define for an Op than for a Type:
.. class:: Op
.. method:: c_code(node, name, input_names, output_names, sub)
This must return C code that carries the computation we want to
do.
`sub` is a dictionary of extras parameters to the c_code
method. It contains the following values:
``sub['fail']``
A string of code that you should execute (after ensuring
that a python exception is set) if your C code needs to
raise an exception.
``sub['params']``
(optional) The name of the variable which holds the context
for the node. This will only appear if the op has requested
a context by having a :meth:`get_params()` method that return
something other than None.
.. method:: c_code_cleanup(node, name, input_names, output_names, sub)
This must return C code that cleans up whatever c_code
allocated and that we must free.
*Default:* The default behavior is to do nothing.
.. method:: c_headers([c_compiler])
Returns a list of headers to include in the file. 'Python.h' is
included by default so you don't need to specify it. Also all
of the headers required by the Types involved (inputs and
outputs) will also be included.
The `c_compiler` [#2v]_ parameter is the C compiler that will
be used to compile the code for the node. You may get multiple
calls with different C compilers.
.. method:: c_header_dirs([c_compiler])
Returns a list of directories to search for headers (arguments
to -I).
The `c_compiler` [#2v]_ parameter is the C compiler that will
be used to compile the code for the node. You may get multiple
calls with different C compilers.
.. method:: c_libraries([c_compiler])
Returns a list of library names that your op needs to link to.
All ops are automatically linked with 'python' and the
libraries their types require. (arguments to -l)
The `c_compiler` [#2v]_ parameter is the C compiler that will
be used to compile the code for the node. You may get multiple
calls with different C compilers.
.. method:: c_lib_dirs([c_compiler])
Returns a list of directory to search for libraries (arguments
to -L).
The `c_compiler` [#2v]_ parameter is the C compiler that will
be used to compile the code for the node. You may get multiple
calls with different C compilers.
.. method:: c_compile_args([c_compiler])
Allows to specify additional arbitrary arguments to the C
compiler. This is not usually required.
The `c_compiler` [#2v]_ parameter is the C compiler that will
be used to compile the code for the node. You may get multiple
calls with different C compilers.
.. method:: c_no_compile_args([c_compiler])
Returns a list of C compiler arguments that are forbidden when
compiling this Op.
The `c_compiler` [#2v]_ parameter is the C compiler that will
be used to compile the code for the node. You may get multiple
calls with different C compilers.
.. method:: c_init_code()
Allows you to specify code that will be executed once when the
module is initialized, before anything else is executed. This
is for code that will be executed once per Op.
.. method:: c_init_code_apply(node, name)
Allows you to specify code that will be executed once when the
module is initialized, before anything else is executed and is
specialized for a particular apply of an :ref:`op`.
.. method:: c_init_code_struct(node, name, sub)
Allows you to specify code that will be inserted in the struct
constructor of the Op. This is for code which should be
executed once per thunk (Apply node, more or less).
`sub` is a dictionary of extras parameters to the
c_code_init_code_struct method. It contains the following
values:
``sub['fail']``
A string of code that you should execute (after ensuring
that a python exception is set) if your C code needs to
raise an exception.
``sub['params']``
(optional) The name of the variable which holds the context
for the node. This will only appear if the op has requested
a context by having a :meth:`get_params()` method that return
something other than None.
.. method:: c_support_code()
Allows you to specify helper functions/structs (in a string or a list of string) that the
:ref:`op` needs. That code will be reused for each apply of
this op. It will be inserted at global scope.
.. method:: c_support_code_apply(node, name)
Allows you to specify helper functions/structs specialized for
a particular apply of an :ref:`op`. Use :meth:`c_support_code`
if the code is the same for each apply of an op. It will be
inserted at global scope.
.. method:: c_support_code_struct(node, name)
Allows you to specify helper functions of variables that will
be specific to one particular thunk. These are inserted at
struct scope.
:note:
You cannot specify CUDA kernels in the code returned by this
since that isn't supported by CUDA. You should place your
kernels in :meth:`c_support_code()` or
:meth:`c_support_code_apply()` and call them from this code.
.. method:: c_cleanup_code_struct(node, name)
Allows you to specify code that will be inserted in the struct
destructor of the Op. This is for cleaninp up allocations and
stuff like this when the thunk is released (when you "free" a
compiled function using this op).
.. method:: infer_shape(node, (i0_shapes,i1_shapes,...))
Allow optimizations to lift the Shape op over this op. An
example of why this is good is when we only need the shape of a
variable: we will be able to obtain it without computing the
variable itself.
Must return a list where each element is a tuple representing
the shape of one output.
For example, for the matrix-matrix product ``infer_shape`` will
have as inputs (node, ((x0,x1), (y0,y1))) and should return
[(x0, y1)]. Both the inputs and the return value may be Theano
variables.
.. method:: c_code_cache_version()
Must return a tuple of hashable objects like integers. This
specifies the version of the code. It is used to cache the
compiled code. You MUST change the returned tuple for each
change in the code. If you don't want to cache the compiled
code return an empty tuple or don't implement it.
.. method:: c_code_cache_version_apply(node)
Overrides :meth:`c_code_cache_version` if defined, but
otherwise has the same contract.
.. method:: python_constant_folding(node)
Optional. If present this method will be called before doing
constant folding of a node, with that node as a parameter. If
it return True, we will not generate c code when doing constant
folding of this node. This is useful when the compilation of
the c code will be longer then the computation in python
(e.g. Elemwise of scalars).
In addition, this allow to lower the number of compiled module
and disk access. Particularly useful when the file system load
is high or when theano compilation directory is shared by many
process (like on a network file server on a cluster).
.. method:: get_params(node)
(optional) If defined, should return the runtime params the op
needs. These parameters will be passed to the C code through the
variable named in `sub['params']`. The variable is also
available for use in the code returned by
:meth:`c_init_code_struct`. If it returns `None` this is
considered the same as if the method was not defined.
If this method is defined and does not return `None`, then the
Op *must* have a `params_type` property with the Type to use
for the params variable.
.. attribute:: _f16_ok
(optional) If this attribute is absent or evaluates to `False`,
C code will be disabled for the op if any of its inputs or
outputs contains float16 data. This is added as a check to make
sure we don't compute wrong results since there is no hardware
float16 type so special care must be taken to make sure
operations are done correctly.
If you don't intend to deal with float16 data you can leave
this undefined.
This attribute is internal and may go away at any point during
developpment if a better solution is found.
The ``name`` argument is currently given an invalid value, so steer
away from it. As was the case with Type, ``sub['fail']`` provides
failure code that you *must* use if you want to raise an exception,
after setting the exception message.
The ``node`` argument is an :ref:`apply` node representing an
application of the current Op on a list of inputs, producing a list of
outputs. ``input_names`` and ``output_names`` arguments contain as
many strings as there are inputs and outputs to the application of the
Op and they correspond to the ``name`` that is passed to the type of
each Variable in these lists. For example, if ``node.inputs[0].type ==
double``, then ``input_names[0]`` is the ``name`` argument passed to
``double.c_declare`` etc. when the first input is processed by Theano.
In a nutshell, ``input_names`` and ``output_names`` parameterize the
names of the inputs your operation needs to use and the outputs it
needs to put variables into. But this will be clear with the examples.
.. rubric:: Footnotes
.. [#2v] There are actually two versions of this method one with a
`c_compiler` parameter and one without. The calling code will
try the version with c_compiler and try the version without
if it does not work. Defining both versions is pointless
since the one without `c_compiler` will never get called.
Note that these methods are not specific to a single apply
node so they may get called more than once on the same object
with different values for c_compiler.
Defining the methods
====================
We will be defining C code for the multiplication Op on doubles.
**c_code**
.. testsetup::
from theano import Op
mul = Op()
.. testcode::
def c_code(node, name, input_names, output_names, sub):
x_name, y_name = input_names[0], input_names[1]
output_name = output_names[0]
return """
%(output_name)s = %(x_name)s * %(y_name)s;
""" % locals()
mul.c_code = c_code
And that's it. As we enter the scope of the C code we are defining in
the method above, many variables are defined for us. Namely, the
variables x_name, y_name and output_name are all of the primitive C
``double`` type and they were declared using the C code returned by
``double.c_declare``.
Implementing multiplication is as simple as multiplying the two input
doubles and setting the output double to what comes out of it. If you
had more than one output, you would just set the variable(s) for
each output to what they should be.
.. warning::
Do *NOT* use C's ``return`` statement to return the variable(s) of
the computations. Set the output variables directly as shown
above. Theano will pick them up for you.
**c_code_cleanup**
There is nothing to cleanup after multiplying two doubles. Typically,
you won't need to define this method unless you malloc() some
temporary storage (which you would free() here) or create temporary
Python objects (which you would Py_XDECREF() here).
Final version
=============
As before, I tried to organize the code in order to minimize
repetition. You can check that mul produces the same C code in this
version that it produces in the code I gave above.
.. testcode::
from theano import gof
class BinaryDoubleOp(gof.Op):
__props__ = ("name", "fn", "ccode")
def __init__(self, name, fn, ccode):
self.name = name
self.fn = fn
self.ccode = ccode
def make_node(self, x, y):
if isinstance(x, (int, float)):
x = gof.Constant(double, x)
if isinstance(y, (int, float)):
y = gof.Constant(double, y)
if x.type != double or y.type != double:
raise TypeError('%s only works on doubles' % self.name)
return gof.Apply(self, [x, y], [double()])
def perform(self, node, inp, out):
x, y = inp
z, = out
z[0] = self.fn(x, y)
def __str__(self):
return self.name
def c_code(self, node, name, inp, out, sub):
x, y = inp
z, = out
return self.ccode % locals()
add = BinaryDoubleOp(name='add',
fn=lambda x, y: x + y,
ccode="%(z)s = %(x)s + %(y)s;")
sub = BinaryDoubleOp(name='sub',
fn=lambda x, y: x - y,
ccode="%(z)s = %(x)s - %(y)s;")
mul = BinaryDoubleOp(name='mul',
fn=lambda x, y: x * y,
ccode="%(z)s = %(x)s * %(y)s;")
div = BinaryDoubleOp(name='div',
fn=lambda x, y: x / y,
ccode="%(z)s = %(x)s / %(y)s;")
Theano-1.0.1/doc/extending/ctype.txt 0000777 0000000 0000000 00000053731 13212236347 017327 0 ustar root root 0000000 0000000 .. _theano_ctype:
========================
Implementing double in C
========================
The previous two sections described how to define a double :ref:`type`
and arithmetic operations on that Type, but all of them were
implemented in pure Python. In this section we will see how to define
the double type in such a way that it can be used by operations
implemented in C (which we will define in the section after that).
How does it work?
=================
In order to be C-compatible, a Type must provide a C interface to the
Python data that satisfy the constraints it puts forward. In other
words, it must define C code that can convert a Python reference into
some type suitable for manipulation in C and it must define C code
that can convert some C structure in which the C implementation of an
operation stores its variables into a reference to an object that can be
used from Python and is a valid value for the Type.
For example, in the current example, we have a Type which represents a
Python float. First, we will choose a corresponding C type. The
natural choice would be the primitive ``double`` type. Then, we need
to write code that will take a ``PyObject*``, check that it is a
Python ``float`` and extract its value as a ``double``. Finally, we
need to write code that will take a C ``double`` and will build a
``PyObject*`` of Python type ``float`` that we can work with from
Python. We will be using CPython and thus special care must be given
to making sure reference counts are updated properly!
The C code we will write makes use of CPython's C API which you can
find here_.
.. _here: http://docs.python.org/c-api/index.html
What needs to be defined
========================
In order to be C-compatible, a Type must define several additional
methods, which all start with the ``c_`` prefix. The complete list can
be found in the documentation for :class:`.gof.type.Type`. Here, we'll focus on
the most important ones:
.. class:: CLinkerType
.. method:: c_declare(name, sub, check_input=True)
This must return C code which declares variables. These variables
will be available to operations defined in C. You may also write
typedefs.
.. method:: c_init(name, sub)
This must return C code which initializes the variables declared in
``c_declare``. Either this or ``c_extract`` will be called.
.. method:: c_extract(name, sub, check_input=True)
This must return C code which takes a reference to a Python object
and initializes the variables declared in ``c_declare`` to match the
Python object's data. Either this or ``c_init`` will be called.
.. method:: c_sync(name, sub)
When the computations are done, transfer the variables from the C
structure we put them in to the destination Python object. This will
only be called for the outputs.
.. method:: c_cleanup(name, sub)
When we are done using the data, clean up whatever we allocated and
decrease the appropriate reference counts.
.. method:: c_headers([c_compiler])
c_libraries([c_compiler])
c_header_dirs([c_compiler])
c_lib_dirs([c_compiler])
Allows you to specify headers, libraries and associated directories.
These methods have two versions, one with a `c_compiler`
argument and one without. The version with c_compiler is tried
first and if it doesn't work, the one without is.
The `c_compiler` argument is the C compiler that will be used
to compile the C code for the node that uses this type.
.. method:: c_compile_args([c_compiler])
c_no_compile_args([c_compiler])
Allows to specify special compiler arguments to add/exclude.
These methods have two versions, one with a `c_compiler`
argument and one without. The version with c_compiler is tried
first and if it doesn't work, the one without is.
The `c_compiler` argument is the C compiler that will be used
to compile the C code for the node that uses this type.
.. method:: c_init_code()
Allows you to specify code that will be executed once when the
module is initialized, before anything else is executed.
For instance, if a type depends on NumPy's C API, then
``'import_array();'`` has to be among the snippets returned
by ``c_init_code()``.
.. method:: c_support_code()
Allows to add helper functions/structs (in a string or a list of strings) that the :ref:`type` needs.
.. method:: c_compiler()
Allows to specify a special compiler. This will force this compiler
for the current compilation block (a particular op or the full graph).
This is used for the GPU code.
.. method:: c_code_cache_version()
Should return a tuple of hashable objects like integers. This
specifies the version of the code. It is used to cache the
compiled code. You MUST change the returned tuple for each
change in the code. If you don't want to cache the compiled code
return an empty tuple or don't implement it.
.. method:: c_element_type()
Optional: should return the name of the primitive C type of
items into variables handled by this Theano type. For example,
for a matrix of 32-bit signed NumPy integers, it should return
``"npy_int32"``. If C type may change from an instance to another
(e.g. ``Scalar('int32')`` vs ``Scalar('int64')``), consider
implementing this method. If C type is fixed accross instances,
this method may be useless (as you already know the C type
when you work with the C code).
Each of these functions take two arguments, ``name`` and ``sub`` which
must be used to parameterize the C code they return. ``name`` is a
string which is chosen by the compiler to represent a :ref:`variable` of
the Type in such a way that there are no name conflicts between
different pieces of data. Therefore, all variables declared in
``c_declare`` should have a name which includes ``name``. Furthermore,
the name of the variable containing a pointer to the Python object
associated to the Variable is ``py_``.
``sub``, on the other hand, is a dictionary containing bits of C code
suitable for use in certain situations. For instance, ``sub['fail']``
contains code that should be inserted wherever an error is identified.
``c_declare`` and ``c_extract`` also accept a third ``check_input``
optional argument. If you want your type to validate its inputs, it must
only do it when ``check_input`` is True.
The example code below should help you understand how everything plays
out:
.. warning::
If some error condition occurs and you want to fail and/or raise an
Exception, you must use the ``fail`` code contained in
``sub['fail']`` (there is an example in the definition of ``c_extract``
below). You must *NOT* use the ``return`` statement anywhere, ever,
nor ``break`` outside of your own loops or ``goto`` to strange
places or anything like that. Failure to comply with this
restriction could lead to erratic behavior, segfaults and/or memory
leaks because Theano defines its own cleanup system and assumes
that you are not meddling with it. Furthermore, advanced operations
or types might do code transformations on your code such as
inserting it in a loop -- in that case they can call your
code-generating methods with custom failure code that takes into account
what they are doing!
Defining the methods
====================
.. testsetup::
import theano
double = theano.Type()
**c_declare**
.. testcode::
def c_declare(name, sub):
return """
double %(name)s;
""" % dict(name = name)
double.c_declare = c_declare
Very straightforward. All we need to do is write C code to declare a
double. That double will be named whatever is passed to our function
in the ``name`` argument. That will usually be some mangled name like
"V0", "V2" or "V92" depending on how many nodes there are in the
computation graph and what rank the current node has. This function
will be called for all Variables whose type is ``double``.
You can declare as many variables as you want there and you can also
do typedefs. Make sure that the name of each variable contains the
``name`` argument in order to avoid name collisions (collisions *will*
happen if you don't parameterize the variable names as indicated
here). Also note that you cannot declare a variable called
``py_`` or ``storage_`` because Theano already defines
them.
What you declare there is basically the C interface you are giving to
your Type. If you wish people to develop operations that make use of
it, it's best to publish it somewhere.
**c_init**
.. testcode::
def c_init(name, sub):
return """
%(name)s = 0.0;
""" % dict(name = name)
double.c_init = c_init
This function has to initialize the
double we declared previously to a suitable value. This is useful if
we want to avoid dealing with garbage values, especially if our data
type is a pointer. This is not going to be called for all Variables with
the ``double`` type. Indeed, if a Variable is an input that we pass
from Python, we will want to extract that input from a Python object,
therefore it is the ``c_extract`` method that will be called instead of
``c_init``. You can therefore not assume, when writing ``c_extract``, that the
initialization has been done (in fact you can assume that it *hasn't*
been done).
``c_init`` will typically be called on output Variables, but in general
you should only assume that either ``c_init`` or ``c_extract`` has been
called, without knowing for sure which of the two.
**c_extract**
.. testcode::
def c_extract(name, sub):
return """
if (!PyFloat_Check(py_%(name)s)) {
PyErr_SetString(PyExc_TypeError, "expected a float");
%(fail)s
}
%(name)s = PyFloat_AsDouble(py_%(name)s);
""" % dict(name = name, fail = sub['fail'])
double.c_extract = c_extract
This method is slightly more sophisticated. What happens here is that
we have a reference to a Python object which Theano has placed in
``py_%(name)s`` where ``%(name)s`` must be substituted for the name
given in the inputs. This special variable is declared by Theano as
``PyObject* py_%(name)s`` where ``PyObject*`` is a pointer to a Python
object as defined by CPython's C API. This is the reference that
corresponds, on the Python side of things, to a Variable with the
``double`` type. It is what the end user will give and what he or she
expects to get back.
In this example, the user will give a Python ``float``. The first
thing we should do is verify that what we got is indeed a Python
``float``. The ``PyFloat_Check`` function is provided by CPython's C
API and does this for us. If the check fails, we set an exception and
then we insert code for failure. The code for failure is in
``sub["fail"]`` and it basically does a ``goto`` to cleanup code.
If the check passes then we convert the Python float into a double
using the ``PyFloat_AsDouble`` function (yet again provided by CPython's C
API) and we put it in our double variable that we declared previously.
**c_sync**
.. testcode::
def c_sync(name, sub):
return """
Py_XDECREF(py_%(name)s);
py_%(name)s = PyFloat_FromDouble(%(name)s);
if (!py_%(name)s) {
printf("PyFloat_FromDouble failed on: %%f\\n", %(name)s);
Py_XINCREF(Py_None);
py_%(name)s = Py_None;
}
""" % dict(name = name)
double.c_sync = c_sync
This function is probably the trickiest. What happens here is that we
have computed some operation on doubles and we have put the variable
into the double variable ``%(name)s``. Now, we need to put this data
into a Python object that we can manipulate on the Python side of
things. This Python object must be put into the ``py_%(name)s``
variable which Theano recognizes (this is the same pointer we get in
c_extract).
Now, that pointer is already a pointer to a valid Python object
(unless you or a careless implementer did terribly wrong things with
it). If we want to point to another object, we need to tell Python
that we don't need the old one anymore, meaning that we need to
*decrease the previous object's reference count*. The first line,
``Py_XDECREF(py_%(name)s)`` does exactly this. If it is forgotten,
Python will not be able to reclaim the data even if it is not used
anymore and there will be memory leaks! This is especially important
if the data you work on is large.
Now that we have decreased the reference count, we call
``PyFloat_FromDouble`` on our double variable in order to convert it
to a Python ``float``. This returns a new reference which we assign to
``py_%(name)s``. From there Theano will do the rest and the end user
will happily see a Python ``float`` come out of his computations.
The rest of the code is not absolutely necessary and it is basically
"good practice". ``PyFloat_FromDouble`` can return ``NULL`` on failure.
``NULL`` is a pretty bad reference to have and neither Python nor Theano
like it. If this happens, we change the ``NULL`` pointer (which will
cause us problems) to a pointer to ``None`` (which is *not* a ``NULL``
pointer). Since ``None`` is an object like the others, we need to
increase its reference count before we can set a new pointer to it. This
situation is unlikely to ever happen, but if it ever does, better safe
than sorry.
.. warning::
I said this already but it really needs to be emphasized that if
you are going to change the ``py_%(name)s`` pointer to point to a
new reference, you *must* decrease the reference count of whatever
it was pointing to before you do the change. This is only valid if
you change the pointer, if you are not going to change the pointer,
do *NOT* decrease its reference count!
**c_cleanup**
.. testcode::
def c_cleanup(name, sub):
return ""
double.c_cleanup = c_cleanup
We actually have nothing to do here. We declared a double on the stack
so the C language will reclaim it for us when its scope ends. We
didn't ``malloc()`` anything so there's nothing to ``free()``. Furthermore,
the ``py_%(name)s`` pointer hasn't changed so we don't need to do
anything with it. Therefore, we have nothing to cleanup. Sweet!
There are however two important things to keep in mind:
First, note that ``c_sync`` and ``c_cleanup`` might be called in
sequence, so they need to play nice together. In particular, let's
say that you allocate memory in ``c_init`` or ``c_extract`` for some
reason. You might want to either embed what you allocated to some Python
object in ``c_sync`` or to free it in ``c_cleanup``. If you do the
former, you don't want to free the allocated storage so you should set
the pointer to it to ``NULL`` to avoid that ``c_cleanup`` mistakenly
frees it. Another option is to declare a variable in ``c_declare`` that
you set to true in ``c_sync`` to notify ``c_cleanup`` that ``c_sync``
was called.
Second, whenever you use ``%(fail)s`` in ``c_extract`` or in the code of an
:ref:`operation `, you can count on ``c_cleanup`` being called right
after that. Therefore, it's important to make sure that ``c_cleanup``
doesn't depend on any code placed after a reference to
``%(fail)s``. Furthermore, because of the way Theano blocks code together,
only the variables declared in ``c_declare`` will be visible in ``c_cleanup``!
What the generated C will look like
===================================
``c_init`` and ``c_extract`` will only be called if there is a Python
object on which we want to apply computations using C
code. Conversely, ``c_sync`` will only be called if we want to
communicate the values we have computed to Python, and ``c_cleanup``
will only be called when we don't need to process the data with C
anymore. In other words, the use of these functions for a given Variable
depends on the the relationship between Python and C with respect to
that Variable. For instance, imagine you define the following function
and call it:
.. code-block:: python
x, y, z = double('x'), double('y'), double('z')
a = add(x, y)
b = mul(a, z)
f = function([x, y, z], b)
f(1.0, 2.0, 3.0)
Using the CLinker, the code that will be produced will look roughly
like this:
.. code-block:: c
// BEGIN defined by Theano
PyObject* py_x = ...;
PyObject* py_y = ...;
PyObject* py_z = ...;
PyObject* py_a = ...; // note: this reference won't actually be used for anything
PyObject* py_b = ...;
// END defined by Theano
{
double x; //c_declare for x
x = ...; //c_extract for x
{
double y; //c_declare for y
y = ...; //c_extract for y
{
double z; //c_declare for z
z = ...; //c_extract for z
{
double a; //c_declare for a
a = 0; //c_init for a
{
double b; //c_declare for b
b = 0; //c_init for b
{
a = x + y; //c_code for add
{
b = a * z; //c_code for mul
labelmul:
//c_cleanup for mul
}
labeladd:
//c_cleanup for add
}
labelb:
py_b = ...; //c_sync for b
//c_cleanup for b
}
labela:
//c_cleanup for a
}
labelz:
//c_cleanup for z
}
labely:
//c_cleanup for y
}
labelx:
//c_cleanup for x
}
It's not pretty, but it gives you an idea of how things work (note that
the variable names won't be ``x``, ``y``, ``z``, etc. - they will
get a unique mangled name). The ``fail`` code runs a ``goto`` to the
appropriate label in order to run all cleanup that needs to be
done. Note which variables get extracted (the three inputs ``x``, ``y`` and
``z``), which ones only get initialized (the temporary variable ``a`` and the
output ``b``) and which one is synced (the final output ``b``).
The C code above is a single C block for the whole graph. Depending on
which :term:`linker` is used to process the computation graph, it is
possible that one such block is generated for each operation and that
we transit through Python after each operation. In that situation,
``a`` would be synced by the addition block and extracted by the
multiplication block.
Final version
=============
.. testcode::
from theano import gof
class Double(gof.Type):
def filter(self, x, strict=False, allow_downcast=None):
if strict and not isinstance(x, float):
raise TypeError('Expected a float!')
return float(x)
def values_eq_approx(self, x, y, tolerance=1e-4):
return abs(x - y) / (x + y) < tolerance
def __str__(self):
return "double"
def c_declare(self, name, sub):
return """
double %(name)s;
""" % dict(name = name)
def c_init(self, name, sub):
return """
%(name)s = 0.0;
""" % dict(name = name)
def c_extract(self, name, sub):
return """
if (!PyFloat_Check(py_%(name)s)) {
PyErr_SetString(PyExc_TypeError, "expected a float");
%(fail)s
}
%(name)s = PyFloat_AsDouble(py_%(name)s);
""" % dict(sub, name = name)
def c_sync(self, name, sub):
return """
Py_XDECREF(py_%(name)s);
py_%(name)s = PyFloat_FromDouble(%(name)s);
if (!py_%(name)s) {
printf("PyFloat_FromDouble failed on: %%f\\n", %(name)s);
Py_XINCREF(Py_None);
py_%(name)s = Py_None;
}
""" % dict(name = name)
def c_cleanup(self, name, sub):
return ""
double = Double()
DeepCopyOp
==========
We have an internal Op called DeepCopyOp. It is used to make sure we
respect the user vs Theano memory region as described in the :ref:`tutorial
`. Theano has a Python implementation that calls the object's
``copy()`` or ``deepcopy()`` method for Theano types for which it does not
know how to generate C code.
You can implement c_code for this op. You register it like this:
.. code-block:: python
theano.compile.ops.register_deep_copy_op_c_code(YOUR_TYPE_CLASS, THE_C_CODE, version=())
In your C code, you should use %(iname)s and %(oname)s to represent
the C variable names of the DeepCopyOp input and output
respectively. See an example for the type ``GpuArrayType`` (GPU
array) in the file `theano/gpuarray/type.py`. The version
parameter is what is returned by DeepCopyOp.c_code_cache_version(). By
default, it will recompile the c code for each process.
ViewOp
======
We have an internal Op called ViewOp. It is used for some
verification of inplace/view Ops. Its C implementation increments and
decrements Python reference counts, and thus only works with Python
objects. If your new type represents Python objects, you should tell
ViewOp to generate C code when working with this type, as
otherwise it will use Python code instead. This is achieved by
calling:
.. code-block:: python
theano.compile.ops.register_view_op_c_code(YOUR_TYPE_CLASS, THE_C_CODE, version=())
In your C code, you should use %(iname)s and %(oname)s to represent
the C variable names of the ViewOp input and output
respectively. See an example for the type ``GpuArrayType`` (GPU
array) in the file `thean/gpuarray/type.py`. The version
parameter is what is returned by ViewOp.c_code_cache_version(). By
default, it will recompile the c code for each process.
Shape and Shape_i
=================
We have 2 generic Ops, Shape and Shape_i, that return the shape of any
Theano Variable that has a shape attribute (Shape_i returns only one of
the elements of the shape).
.. code-block:: python
theano.compile.ops.register_shape_c_code(YOUR_TYPE_CLASS, THE_C_CODE, version=())
theano.compile.ops.register_shape_i_c_code(YOUR_TYPE_CLASS, THE_C_CODE, CHECK_INPUT, version=())
The C code works as the ViewOp. Shape_i has the additional ``i`` parameter
that you can use with ``%(i)s``.
In your CHECK_INPUT, you must check that the input has enough dimensions to
be able to access the i-th one.
Theano-1.0.1/doc/extending/extending_faq.txt 0000777 0000000 0000000 00000002170 13212236347 021006 0 ustar root root 0000000 0000000
.. _extend_faq:
=========================================
Extending Theano: FAQ and Troubleshooting
=========================================
I wrote a new Op/Type, and weird stuff is happening...
------------------------------------------------------
First, check the :ref:`op_contract` and the :ref:`type_contract`
and make sure you're following the rules.
Then try running your program in :ref:`using_debugmode`. DebugMode might catch
something that you're not seeing.
I wrote a new optimization, but it's not getting used...
---------------------------------------------------------
Remember that you have to register optimizations with the :ref:`optdb`
for them to get used by the normal modes like FAST_COMPILE, FAST_RUN,
and DebugMode.
I wrote a new optimization, and it changed my results even though I'm pretty sure it is correct.
------------------------------------------------------------------------------------------------
First, check the :ref:`op_contract` and make sure you're following the rules.
Then try running your program in :ref:`using_debugmode`. DebugMode might
catch something that you're not seeing.
Theano-1.0.1/doc/extending/extending_theano.txt 0000777 0000000 0000000 00000102132 13212236347 021514 0 ustar root root 0000000 0000000
.. _extending_theano:
Creating a new Op: Python implementation
========================================
So suppose you have looked through the library documentation and you don't see
a function that does what you want.
If you can implement something in terms of existing Ops, you should do that.
Odds are your function that uses existing Theano expressions is short,
has no bugs, and potentially profits from optimizations that have already been
implemented.
However, if you cannot implement an Op in terms of existing Ops, you have to
write a new one. Don't worry, Theano was designed to make it easy to add new
Ops, Types, and Optimizations.
.. These first few pages will walk you through the definition of a new :ref:`type`,
.. ``double``, and a basic arithmetic :ref:`operations ` on that Type.
As an illustration, this tutorial shows how to write a simple Python-based
:ref:`operations ` which performs operations on
:ref:`type`, ``double``.
.. It also shows how to implement tests that
.. ensure the proper working of an op.
.. note::
This is an introductury tutorial and as such it does not cover how to make
an op that returns a view or modifies the values in its inputs. Thus, all
ops created with the instructions described here MUST return newly
allocated memory or reuse the memory provided in the parameter
``output_storage`` of the :func:`perform` function. See
:ref:`views_and_inplace` for an explanation on how to do this.
If your op returns a view or changes the value of its inputs
without doing as prescribed in that page, Theano will run, but will
return correct results for some graphs and wrong results for others.
It is recommended that you run your tests in DebugMode (Theano *flag*
``mode=DebugMode``) since it verifies if your op behaves correctly in this
regard.
Theano Graphs refresher
-----------------------
.. image:: ../hpcs2011_tutorial/pics/apply_node.png
:width: 500 px
Theano represents symbolic mathematical computations as graphs. Those graphs
are bi-partite graphs (graphs with 2 types of nodes), they are composed of
interconnected :ref:`apply` and :ref:`variable` nodes.
:ref:`variable` nodes represent data in the graph, either inputs, outputs or
intermediary values. As such, Inputs and Outputs of a graph are lists of Theano
:ref:`variable` nodes. :ref:`apply` nodes perform computation on these
variables to produce new variables. Each :ref:`apply` node has a link to an
instance of :ref:`Op` which describes the computation to perform. This tutorial
details how to write such an Op instance. Please refers to
:ref:`graphstructures` for a more detailed explanation about the graph
structure.
Op's basic methods
------------------
An op is any Python object which inherits from :class:`gof.Op`.
This section provides an overview of the basic methods you typically have to
implement to make a new op. It does not provide extensive coverage of all the
possibilities you may encounter or need. For that refer to
:ref:`op_contract`.
.. testcode:: python
import theano
class MyOp(theano.Op):
# Properties attribute
__props__ = ()
#itypes and otypes attributes are
#compulsory if make_node method is not defined.
#They're the type of input and output respectively
itypes = None
otypes = None
#Compulsory if itypes and otypes are not defined
def make_node(self, *inputs):
pass
# Python implementation:
def perform(self, node, inputs_storage, output_storage):
pass
# Other type of implementation
# C implementation: [see theano web site for other functions]
def c_code(self, node, inputs, outputs, sub):
pass
# Other implementations:
def make_thunk(self, node, storage_map, _, _2, impl=None):
pass
# optional:
check_input = True
def __init__(self, *args):
pass
def grad(self, inputs, g):
pass
def R_op(self, inputs, eval_points):
pass
def infer_shape(node, input_shapes):
pass
An op has to implement some methods defined in the the interface of
:class:`gof.Op`. More specifically, it is mandatory for an op to define either
the method :func:`make_node` or :attr:`itypes`, :attr:`otypes` and one of the
implementation methods, either :func:`perform`, :meth:`Op.c_code`
or :func:`make_thunk`.
:func:`make_node` method creates an Apply node representing the application
of the op on the inputs provided. This method is reponsible for three things:
- it first checks that the input Variables types are compatible
with the current op. If the op cannot be applied on the provided
input types, it must raises an exception (such as :class:`TypeError`).
- it operates on the Variables found in
``*inputs`` in Theano's symbolic language to infer the type of
the symbolic output Variables. It creates output Variables of a suitable
symbolic Type to serve as the outputs of this op's
application.
- it creates an Apply instance with the input and output Variable, and
return the Apply instance.
:func:`perform` method defines the Python implementation of an op.
It takes several arguments:
- ``node`` is a reference to an Apply node which was previously
obtained via the ``Op``'s :func:`make_node` method. It is typically not
used in simple ops, but it contains symbolic information that
could be required for complex ops.
- ``inputs`` is a list of references to data which can be operated on using
non-symbolic statements, (i.e., statements in Python, Numpy).
- ``output_storage`` is a list of storage cells where the output
is to be stored. There is one storage cell for each output of the op.
The data put in ``output_storage`` must match the type of the
symbolic output. It is forbidden to change the length of the list(s)
contained in ``output_storage``.
A function Mode may allow ``output_storage`` elements to persist
between evaluations, or it may reset ``output_storage`` cells to
hold a value of ``None``. It can also pre-allocate some memory
for the op to use. This feature can allow ``perform`` to reuse
memory between calls, for example. If there is something
preallocated in the ``output_storage``, it will be of the good
dtype, but can have the wrong shape and have any stride pattern.
:func:`perform` method must be determined by the inputs. That is to say,
when applied to identical inputs the method must return the same outputs.
:class:`gof.Op` allows some other way to define the op implentation.
For instance, it is possible to define :meth:`Op.c_code` to provide a
C-implementation to the op. Please refers to tutorial
:ref:`extending_theano_c` for a description of :meth:`Op.c_code` and other
related c_methods. Note that an op can provide both Python and C
implementation.
:func:`make_thunk` method is another alternative to :func:`perform`.
It returns a thunk. A thunk is defined as a zero-arguments
function which encapsulates the computation to be performed by an
op on the arguments of its corresponding node. It takes several parameters:
- ``node`` is the Apply instance for which a thunk is requested,
- ``storage_map`` is a dict of lists which maps variables to a one-element
lists holding the variable's current value. The one-element list acts as
pointer to the value and allows sharing that "pointer" with other nodes
and instances.
- ``compute_map`` is also a dict of lists.
It maps variables to one-element lists holding booleans. If
the value is 0 then the variable has not been computed and the
value should not be considered valid. If the value is 1 the
variable has been computed and the value is valid. If the value
is 2 the variable has been garbage-collected and is no longer
valid, but shouldn't be required anymore for this call.
The returned function must ensure that it sets the computed
variables as computed in the `compute_map`.
- ``impl`` allow to select between multiple implementation.
It should have a default value of None.
:func:`make_thunk` is useful if you want to generate code and compile
it yourself.
If :func:`make_thunk()` is defined by an op, it will be used by Theano
to obtain the op's implementation.
:func:`perform` and :meth:`Op.c_code` will be ignored.
If :func:`make_node` is not defined, the :attr:`itypes` and :attr:`otypes`
are used by the Op's :func:`make_node` method to implement the functionality
of :func:`make_node` method mentioned above.
Op's auxiliary methods
----------------------
There are other methods that can be optionally defined by the op:
The :func:`__str__` method provides a meaningful string representation of
your op.
:func:`__eq__` and :func:`__hash__` define respectivelly equality
between two ops and the hash of an op instance.
They will be used by the optimization
phase to merge nodes that are doing equivalent computations (same
inputs, same operation).
Two ops that are equal according :func:`__eq__`
should return the same output when they are applied on the same inputs.
The :attr:`__props__` lists the properties
that influence how the computation is performed (Ususally these are those
that you set in :func:`__init__`). It must be a tuple.
If you don't have any properties, then you should set this attribute to the
emtpy tuple `()`.
:attr:`__props__` enables the automatic generation of appropriate
:func:`__eq__` and :func:`__hash__`.
Given the method :func:`__eq__`, automatically generated from
:attr:`__props__`, two ops will be equal if they have the same values for all
the properties listed in :attr:`__props__`.
Given to the method :func:`__hash__` automatically generated from
:attr:`__props__`, two ops will be have the same hash if they have the same
values for all the properties listed in :attr:`__props__`.
:attr:`__props__` will also generate a suitable :func:`__str__` for your op.
This requires development version after September 1st, 2014 or version 0.7.
The :func:`infer_shape` method allows to infer the shape of the op
output variables, without actually computing the outputs.
It takes as input ``node``, a reference to the op Apply node,
and a list of Theano symbolic Varables (``i0_shape``, ``i1_shape``, ...)
which are the shape of the op input Variables.
:func:`infer_shape` returns a list where each element is a tuple representing
the shape of one output.
This could be helpful if one only
needs the shape of the output instead of the actual outputs, which
can be useful, for instance, for optimization procedures.
The :func:`grad` method is required if you want to differentiate some cost
whose expression includes your op. The gradient may be
specified symbolically in this method. It takes two arguments ``inputs`` and
``output_gradients`` which are both lists of symbolic Theano Variables and
those must be operated on using Theano's symbolic language. The grad
method must return a list containing one Variable for each
input. Each returned Variable represents the gradient with respect
to that input computed based on the symbolic gradients with respect
to each output.
If the output is not differentiable with respect to an input then
this method should be defined to return a variable of type NullType
for that input. Likewise, if you have not implemented the grad
computation for some input, you may return a variable of type
NullType for that input. Please refer to :func:`grad` for a more detailed
view.
The :func:`R_op` method is needed if you want ``theano.tensor.Rop`` to
work with your op.
This function implements the application of the R-operator on the
function represented by your op. Let assume that function is :math:`f`,
with input :math:`x`, applying the R-operator means computing the
Jacobian of :math:`f` and right-multiplying it by :math:`v`, the evaluation
point, namely: :math:`\frac{\partial f}{\partial x} v`.
The optional boolean :attr:`check_input` attribute is used to specify
if you want the types used in your op to check their inputs in their
c_code. It can be used to speed up compilation, reduce overhead
(particularly for scalars) and reduce the number of generated C files.
Example: Op definition
----------------------
.. testcode:: example
import theano
#Using make_node
class DoubleOp1(theano.Op):
__props__ = ()
def make_node(self, x):
x = theano.tensor.as_tensor_variable(x)
# Note: using x_.type() is dangerous, as it copies x's broadcasting
# behaviour
return theano.Apply(self, [x], [x.type()])
def perform(self, node, inputs, output_storage):
x = inputs[0]
z = output_storage[0]
z[0] = x * 2
def infer_shape(self, node, i0_shapes):
return i0_shapes
def grad(self, inputs, output_grads):
return [output_grads[0] * 2]
def R_op(self, inputs, eval_points):
# R_op can receive None as eval_points.
# That mean there is no diferientiable path through that input
# If this imply that you cannot compute some outputs,
# return None for those.
if eval_points[0] is None:
return eval_points
return self.grad(inputs, eval_points)
doubleOp1 = DoubleOp1()
#Using itypes and otypes
class DoubleOp2(theano.Op):
__props__ = ()
itypes = [theano.tensor.dmatrix]
otypes = [theano.tensor.dmatrix]
def perform(self, node, inputs, output_storage):
x = inputs[0]
z = output_storage[0]
z[0] = x * 2
def infer_shape(self, node, i0_shapes):
return i0_shapes
def grad(self, inputs, output_grads):
return [output_grads[0] * 2]
def R_op(self, inputs, eval_points):
# R_op can receive None as eval_points.
# That mean there is no diferientiable path through that input
# If this imply that you cannot compute some outputs,
# return None for those.
if eval_points[0] is None:
return eval_points
return self.grad(inputs, eval_points)
doubleOp2 = DoubleOp2()
At a high level, the code fragment declares a class (e.g., ``DoubleOp1``) and then
creates one instance of it (e.g., ``doubleOp1``).
We often gloss over this distinction, but will be precise here:
``doubleOp1`` (the instance) is an Op, not ``DoubleOp1`` (the class which is a
subclass of ``theano.Op``). You can call ``doubleOp1(tensor.vector())`` on a
Variable to build an expression, and in the expression there will be
a ``.op`` attribute that refers to ``doubleOp1``.
.. The first two methods in the Op are relatively boilerplate: ``__eq__``
.. and ``__hash__``.
.. When two Ops are equal, Theano will merge their outputs if they are applied to the same inputs.
.. The base class (Op) says two objects are equal if (and only if)
.. they are the same object.
.. Writing these boilerplate definitions ensures that the logic of the equality comparison is always explicit.
.. It is an essential part of the :ref:`op_contract` that if two Ops compare
.. equal, then they must compute the same result when presented with the same
.. inputs. Here, if we allocated another instance of ``Fibby`` by typing ``fibby2
.. = Fibby()`` then we would have two Ops that behave identically.
..
.. When should the implementation of ``__eq__`` be more complicated?
.. If ``Fibby.__init__`` had parameters, then we could
.. have configured ``fibby2`` differently from ``fibby`` by passing different
.. arguments to the constructor. If we had done that, and if that different
.. configuration made ``fibby2`` compute different results from ``fibby`` (for the
.. same inputs) then we would have to add logic to the ``__eq__`` and ``__hash__``
.. function so that he two ``Fibby`` Ops would *not be equal*. The reason why: Theano's merge
.. optimization looks for Ops comparing equal and merges them. If two Ops compare
.. equal but don't always produce equal results from equal inputs, then you might
.. see wrong calculation.
The ``make_node`` method creates a node to be included in the expression graph.
It runs when we apply our Op (``doubleOp1``) to the Variable (``x``), as
in ``doubleOp1(tensor.vector())``.
When an Op has multiple inputs, their order in the inputs argument to ``Apply``
is important: Theano will call ``make_node(*inputs)`` to copy the graph,
so it is important not to change the semantics of the expression by changing
the argument order.
All the ``inputs`` and ``outputs`` arguments to ``Apply`` must be Variables.
A common and easy way to ensure inputs are variables is to run them through
``as_tensor_variable``. This function leaves TensorType variables alone, raises
an error for non-TensorType variables, and copies any ``numpy.ndarray`` into
the storage for a TensorType Constant. The ``make_node`` method dictates the
appropriate Type for all output variables.
The ``perform`` method implements the Op's mathematical logic in Python.
The inputs (here ``x``) are passed by value, but a single output is returned
indirectly as the first element of single-element lists. If ``doubleOp1`` had
a second output, it would be stored in ``output_storage[1][0]``.
.. jpt: DOn't understand the following
In some execution modes, the output storage might contain the return value of
a previous call. That old value can be reused to avoid memory re-allocation,
but it must not influence the semantics of the Op output.
You can try the new Op as follows:
.. testcode:: example
import theano
x = theano.tensor.matrix()
f = theano.function([x], DoubleOp1()(x))
import numpy
inp = numpy.random.rand(5, 4)
out = f(inp)
assert numpy.allclose(inp * 2, out)
print(inp)
print(out)
.. testoutput:: example
:hide:
:options: +ELLIPSIS, +SKIP
.. code-block:: none
[[ 0.08257206 0.34308357 0.5288043 0.06582951]
[ 0.65977826 0.10040307 0.5402353 0.55472296]
[ 0.82358552 0.29502171 0.97387481 0.0080757 ]
[ 0.77327215 0.65401857 0.76562992 0.94145702]
[ 0.8452076 0.30500101 0.88430501 0.95818655]]
[[ 0.16514411 0.68616713 1.0576086 0.13165902]
[ 1.31955651 0.20080613 1.08047061 1.10944593]
[ 1.64717104 0.59004341 1.94774962 0.0161514 ]
[ 1.5465443 1.30803715 1.53125983 1.88291403]
[ 1.6904152 0.61000201 1.76861002 1.9163731 ]]
.. testcode:: example
import theano
x = theano.tensor.matrix()
f = theano.function([x], DoubleOp2()(x))
import numpy
inp = numpy.random.rand(5, 4)
out = f(inp)
assert numpy.allclose(inp * 2, out)
print(inp)
print(out)
.. testoutput:: example
:hide:
:options: +ELLIPSIS, +SKIP
.. code-block:: none
[[ 0.02443785 0.67833979 0.91954769 0.95444365]
[ 0.60853382 0.7770539 0.78163219 0.92838837]
[ 0.04427765 0.37895602 0.23155797 0.4934699 ]
[ 0.20551517 0.7419955 0.34500905 0.49347629]
[ 0.24082769 0.49321452 0.24566545 0.15351132]]
[[ 0.04887571 1.35667957 1.83909538 1.90888731]
[ 1.21706764 1.55410779 1.56326439 1.85677674]
[ 0.08855531 0.75791203 0.46311594 0.9869398 ]
[ 0.41103034 1.48399101 0.69001811 0.98695258]
[ 0.48165539 0.98642904 0.4913309 0.30702264]]
Example: __props__ definition
-----------------------------
We can modify the previous piece of code in order to demonstrate
the usage of the :attr:`__props__` attribute.
We create an Op that takes a variable ``x`` and returns ``a*x+b``.
We want to say that two such ops are equal when their values of ``a``
and ``b`` are equal.
.. testcode:: properties
import theano
class AXPBOp(theano.Op):
"""
This creates an Op that takes x to a*x+b.
"""
__props__ = ("a", "b")
def __init__(self, a, b):
self.a = a
self.b = b
super(AXPBOp, self).__init__()
def make_node(self, x):
x = theano.tensor.as_tensor_variable(x)
return theano.Apply(self, [x], [x.type()])
def perform(self, node, inputs, output_storage):
x = inputs[0]
z = output_storage[0]
z[0] = self.a * x + self.b
def infer_shape(self, node, i0_shapes):
return i0_shapes
def grad(self, inputs, output_grads):
return [a * output_grads[0] + b]
The use of :attr:`__props__` saves
the user the trouble of implementing :func:`__eq__` and :func:`__hash__`
manually. It also generates a default :func:`__str__` method that prints the
attribute names and their values.
We can test this by running the following segment:
.. testcode:: properties
mult4plus5op = AXPBOp(4, 5)
another_mult4plus5op = AXPBOp(4, 5)
mult2plus3op = AXPBOp(2, 3)
assert mult4plus5op == another_mult4plus5op
assert mult4plus5op != mult2plus3op
x = theano.tensor.matrix()
f = theano.function([x], mult4plus5op(x))
g = theano.function([x], mult2plus3op(x))
import numpy
inp = numpy.random.rand(5, 4).astype(numpy.float32)
assert numpy.allclose(4 * inp + 5, f(inp))
assert numpy.allclose(2 * inp + 3, g(inp))
How To Test it
--------------
Theano has some functionalities to simplify testing. These help test the
``infer_shape``, ``grad`` and ``R_op`` methods. Put the following code
in a file and execute it with the ``theano-nose`` program.
Basic Tests
^^^^^^^^^^^
Basic tests are done by you just by using the op and checking that it
returns the right answer. If you detect an error, you must raise an
*exception*. You can use the ``assert`` keyword to automatically raise an
``AssertionError``.
.. testcode:: tests
import numpy
import theano
from theano.tests import unittest_tools as utt
from theano import config
class test_Double(utt.InferShapeTester):
def setUp(self):
super(test_Double, self).setUp()
self.op_class = DoubleOp
self.op = DoubleOp()
def test_basic(self):
x = theano.tensor.matrix()
f = theano.function([x], self.op(x))
inp = numpy.asarray(numpy.random.rand(5, 4), dtype=config.floatX)
out = f(inp)
# Compare the result computed to the expected value.
utt.assert_allclose(inp * 2, out)
We call ``utt.assert_allclose(expected_value, value)`` to compare
NumPy ndarray.This raise an error message with more information. Also,
the default tolerance can be changed with the Theano flags
``config.tensor.cmp_sloppy`` that take values in 0, 1 and 2. The
defaul value do the most strict comparison, 1 and 2 make less strict
comparison.
Testing the infer_shape
^^^^^^^^^^^^^^^^^^^^^^^
When a class inherits from the ``InferShapeTester`` class, it gets the
``self._compile_and_check`` method that tests the op's ``infer_shape``
method. It tests that the op gets optimized out of the graph if only
the shape of the output is needed and not the output
itself. Additionally, it checks that the optimized graph computes
the correct shape, by comparing it to the actual shape of the computed
output.
``self._compile_and_check`` compiles a Theano function. It takes as
parameters the lists of input and output Theano variables, as would be
provided to ``theano.function``, and a list of real values to pass to the
compiled function. It also takes the op class as a parameter
in order to verify that no instance of it appears in the shape-optimized graph.
If there is an error, the function raises an exception. If you want to
see it fail, you can implement an incorrect ``infer_shape``.
When testing with input values with shapes that take the same value
over different dimensions (for instance, a square matrix, or a tensor3
with shape (n, n, n), or (m, n, m)), it is not possible to detect if
the output shape was computed correctly, or if some shapes with the
same value have been mixed up. For instance, if the infer_shape uses
the width of a matrix instead of its height, then testing with only
square matrices will not detect the problem. This is why the
``self._compile_and_check`` method prints a warning in such a case. If
your op works only with such matrices, you can disable the warning with the
``warn=False`` parameter.
.. testcode:: tests
from theano.tests import unittest_tools as utt
from theano import config
class test_Double(utt.InferShapeTester):
# [...] as previous tests.
def test_infer_shape(self):
x = theano.tensor.matrix()
self._compile_and_check([x], # theano.function inputs
[self.op(x)], # theano.function outputs
# Always use not square matrix!
# inputs data
[numpy.asarray(numpy.random.rand(5, 4),
dtype=config.floatX)],
# Op that should be removed from the graph.
self.op_class)
Testing the gradient
^^^^^^^^^^^^^^^^^^^^
The function :ref:`verify_grad `
verifies the gradient of an op or Theano graph. It compares the
analytic (symbolically computed) gradient and the numeric
gradient (computed through the Finite Difference Method).
If there is an error, the function raises an exception. If you want to
see it fail, you can implement an incorrect gradient (for instance, by removing
the multiplication by 2).
.. testcode:: tests
def test_grad(self):
theano.tests.unittest_tools.verify_grad(self.op,
[numpy.random.rand(5, 7, 2)])
Testing the Rop
^^^^^^^^^^^^^^^
.. TODO: repair defective links in the following paragraph
The class :class:`RopLop_checker` defines the functions
:func:`RopLop_checker.check_mat_rop_lop`, :func:`RopLop_checker.check_rop_lop` and
:func:`RopLop_checker.check_nondiff_rop`. These allow to test the
implementation of the Rop method of a particular op.
For instance, to verify the Rop method of the DoubleOp, you can use this:
.. testcode:: tests
import numpy
import theano.tests
from theano.tests.test_rop import RopLop_checker
class test_DoubleRop(RopLop_checker):
def setUp(self):
super(test_DoubleRop, self).setUp()
def test_double_rop(self):
self.check_rop_lop(DoubleRop()(self.x), self.in_shape)
Running Your Tests
^^^^^^^^^^^^^^^^^^
To perform your tests, you may select either one of the three
following methods:
theano-nose
"""""""""""
The method of choice to conduct tests is to run the file
``theano-nose``. In a regular Theano installation, the latter will be
on the operating system's path and directly accessible from any
folder. Otherwise, it can be accessed in the ``Theano/bin``
folder. The following command lines may be used for the corresponding
purposes:
* ``theano-nose --theano``: Run every test found in Theano's path.
* ``theano-nose folder_name``: Run every test found in the folder *folder_name*.
* ``theano-nose test_file.py``: Run every test found in the file *test_file.py*.
The following are particularly useful for development purposes since
they call for particular classes or even for particular tests:
* ``theano-nose test_file.py:test_DoubleRop``: Run every test found inside the
class *test_DoubleRop*.
* ``theano-nose test_file.py:test_DoubleRop.test_double_op``: Run only the test
*test_double_op* in the class *test_DoubleRop*.
Help with the use and functionalities of ``theano-nose`` may be
obtained by running it with the command line parameter ``--help
(-h)``.
nosetests
"""""""""
The command ``nosetests`` can also be used. Although it lacks the
useful functionalities that ``theano-nose`` provides, ``nosetests``
can be called similarly to ``theano-nose`` from any folder in Python's
path like so:
``nosetests [suffix similar to the above]``.
More documentation on ``nosetests`` is available here:
`nosetests `
as_op
-----
as_op is a python decorator that converts a python function into a
basic Theano op that will call the supplied function during execution.
This isn't the recommended way to build an op, but allows for a quick
implementation.
It takes an optional :func:`infer_shape` parameter that must have this
signature:
.. code-block:: python
def infer_shape(node, input_shapes):
# ...
return output_shapes
- `input_shapes` and `output_shapes` are lists of tuples that
represent the shape of the corresponding inputs/outputs.
.. note::
Not providing the `infer_shape` method prevents shape-related
optimizations from working with this op. For example
`your_op(inputs, ...).shape` will need the op to be executed just
to get the shape.
.. note::
As no grad is defined, this means you won't be able to
differentiate paths that include this op.
.. note::
It converts the Python function to a callable object that takes as
inputs Theano variables that were declared.
.. note::
The python function wrapped by the `as_op` decorator needs to return a new
data allocation, no views or in place modification of the input.
as_op Example
^^^^^^^^^^^^^
.. testcode:: asop
import theano
import numpy
from theano import function
from theano.compile.ops import as_op
def infer_shape_numpy_dot(node, input_shapes):
ashp, bshp = input_shapes
return [ashp[:-1] + bshp[-1:]]
@as_op(itypes=[theano.tensor.fmatrix, theano.tensor.fmatrix],
otypes=[theano.tensor.fmatrix], infer_shape=infer_shape_numpy_dot)
def numpy_dot(a, b):
return numpy.dot(a, b)
You can try it as follows:
.. testcode:: asop
x = theano.tensor.fmatrix()
y = theano.tensor.fmatrix()
f = function([x, y], numpy_dot(x, y))
inp1 = numpy.random.rand(5, 4).astype('float32')
inp2 = numpy.random.rand(4, 7).astype('float32')
out = f(inp1, inp2)
Exercise
^^^^^^^^
Run the code of the *numpy_dot* example above.
Modify and execute to compute: numpy.add and numpy.subtract.
Modify and execute the example to return two outputs: x + y
and x - y.
.. _Documentation:
Documentation and Coding Style
------------------------------
Please always respect the :ref:`quality_contributions` or your contribution
will not be accepted.
NanGuardMode and AllocEmpty
---------------------------
NanGuardMode help users find where in the graph NaN appear. But
sometimes, we want some variables to not be checked. For example, in
the old GPU back-end, we use a float32 CudaNdarray to store the MRG
random number generator state (they are integers). So if NanGuardMode
check it, it will generate false positive. Another case is related to
[Gpu]AllocEmpty or some computation on it (like done by Scan).
You can tell NanGuardMode to do not check a variable with:
``variable.tag.nan_guard_mode_check``. Also, this tag automatically
follow that variable during optimization. This mean if you tag a
variable that get replaced by an inplace version, it will keep that
tag.
Final Note
----------
A more extensive discussion of this section's content may be found in
the advanced tutorial :ref:`Extending Theano`.
The section :ref:`Other ops ` includes more instructions for
the following specific cases:
- :ref:`scalar_ops`
- :ref:`scipy_ops`
- :ref:`sparse_ops`
- :ref:`Random ops `
- :ref:`openmp_ops`
- :ref:`numba_ops`
Theano-1.0.1/doc/extending/extending_theano_c.txt 0000777 0000000 0000000 00000131470 13212236347 022025 0 ustar root root 0000000 0000000
.. _extending_theano_c:
============================
Extending Theano with a C Op
============================
This tutorial covers how to extend Theano with an op that offers a C
implementation. It does not cover ops that run on a GPU but it does introduce
many elements and concepts which are relevant for GPU ops. This tutorial is
aimed at individuals who already know how to extend Theano (see tutorial
:ref:`extending_theano`) by adding a new op with a Python implementation
and will only cover the additional knowledge required to also produce ops
with C implementations.
Providing a Theano op with a C implementation requires to interact with
Python's C-API and Numpy's C-API. Thus, the first step of this tutorial is to
introduce both and highlight their features which are most relevant to the
task of implementing a C op. This tutorial then introduces the most important
methods that the op needs to implement in order to provide a usable C
implementation. Finally, it shows how to combine these elements to write a
simple C op for performing the simple task of multiplying every element in a
vector by a scalar.
Python C-API
============
Python provides a C-API to allows the manipulation of python objects from C
code. In this API, all variables that represent Python objects are of type
``PyObject *``. All objects have a pointer to their type object and a reference
count field (that is shared with the python side). Most python methods have
an equivalent C function that can be called on the ``PyObject *`` pointer.
As such, manipulating a PyObject instance is often straight-forward but it
is important to properly manage its reference count. Failing to do so can
lead to undesired behavior in the C code.
Reference counting
------------------
Reference counting is a mechanism for keeping track, for an object, of
the number of references to it held by other entities. This mechanism is often
used for purposes of garbage collecting because it allows to easily see if
an object is still being used by other entities. When the reference count
for an object drops to 0, it means it is not used by anyone any longer and can
be safely deleted.
PyObjects implement reference counting and the Python C-API defines a number
of macros to help manage those reference counts. The definition of these
macros can be found here : `Python C-API Reference Counting
`_. Listed below are the
two macros most often used in Theano C ops.
.. method:: void Py_XINCREF(PyObject *o)
Increments the reference count of object ``o``. Without effect if the
object is NULL.
.. method:: void Py_XDECREF(PyObject *o)
Decrements the reference count of object ``o``. If the reference count
reaches 0, it will trigger a call of the object's deallocation function.
Without effect if the object is NULL.
The general principle, in the reference counting paradigm, is that the owner
of a reference to an object is responsible for disposing properly of it.
This can be done by decrementing the reference count once the reference is no
longer used or by transfering ownership; passing on the reference to a new
owner which becomes responsible for it.
Some functions return "borrowed references"; this means that they return a
reference to an object **without** transfering ownership of the reference to the
caller of the function. This means that if you call a function which returns a
borrowed reference, you do not have the burden of properly disposing of that
reference. You should **not** call Py_XDECREF() on a borrowed reference.
Correctly managing the reference counts is important as failing to do so can
lead to issues ranging from memory leaks to segmentation faults.
NumPy C-API
===========
The NumPy library provides a C-API to allow users to create, access and
manipulate NumPy arrays from within their own C routines. NumPy's ndarrays
are used extensively inside Theano and so extending Theano with a C op will
require interaction with the NumPy C-API.
This sections covers the API's elements that are often required to write code
for a Theano C op. The full documentation for the API can be found here :
`NumPy C-API `_.
NumPy data types
----------------
To allow portability between platforms, the NumPy C-API defines its own data
types which should be used whenever you are manipulating a NumPy array's
internal data. The data types most commonly used to implement C ops are the
following : ``npy_int{8,16,32,64}``, ``npy_uint{8,16,32,64}`` and
``npy_float{32,64}``.
You should use these data types when manipulating a NumPy array's internal
data instead of C primitives because the size of the memory representation
for C primitives can vary between platforms. For instance, a C ``long`` can be
represented in memory with 4 bytes but it can also be represented with 8.
On the other hand, the in-memory size of NumPy data types remains constant
across platforms. Using them will make your code simpler and more portable.
The full list of defined data types can be found here :
`NumPy C-API data types
`_.
NumPy ndarrays
--------------
In the NumPy C-API, NumPy arrays are represented as instances of the
PyArrayObject class which is a descendant of the PyObject class. This means
that, as for any other Python object that you manipulate from C code, you
need to appropriatedly manage the reference counts of PyArrayObject instances.
Unlike in a standard multidimensionnal C array, a NumPy array's internal data
representation does not have to occupy a continuous region in memory. In fact,
it can be C-contiguous, F-contiguous or non-contiguous. C-contiguous means
that the data is not only contiguous in memory but also that it is organized
such that the index of the latest dimension changes the fastest. If the
following array
.. testcode::
x = [[1, 2, 3],
[4, 5, 6]]
is C-contiguous, it means that, in memory, the six values contained in the
array ``x`` are stored in the order ``[1, 2, 3, 4, 5, 6]`` (the first value is
``x[0,0]``, the second value is ``x[0,1]``, the third value is ``x[0,2]``, the,
fourth value is ``x[1,0]``, etc). F-contiguous (or Fortran Contiguous) also
means that the data is contiguous but that it is organized such that the index
of the latest dimension changes the slowest. If the array ``x`` is
F-contiguous, it means that, in memory, the values appear in the order
``[1, 4, 2, 5, 3, 6]`` (the first value is ``x[0,0]``, the second value is
``x[1,0]``, the third value is ``x[0,1]``, etc).
Finally, the internal data can be non-contiguous. In this case, it occupies
a non-contiguous region in memory but it is still stored in an organized
fashion : the distance between the element ``x[i,j]`` and the element
``x[i+1,j]`` of the array is constant over all valid values of ``i`` and
``j``, just as the distance between the element ``x[i,j]`` and the element
``x[i,j+1]`` of the array is constant over all valid values of ``i`` and ``j``.
This distance between consecutive elements of an array over a given dimension,
is called the stride of that dimension.
Accessing NumPy ndarrays' data and properties
---------------------------------------------
The following macros serve to access various attributes of NumPy ndarrays.
.. method:: void* PyArray_DATA(PyArrayObject* arr)
Returns a pointer to the first element of the array's data. The returned
pointer must be cast to a pointer of the proper Numpy C-API data type
before use.
.. method:: int PyArray_NDIM(PyArrayObject* arr)
Returns the number of dimensions in the the array pointed by ``arr``
.. method:: npy_intp* PyArray_DIMS(PyArrayObject* arr)
Returns a pointer on the first element of ``arr``'s internal array
describing its dimensions. This internal array contains as many elements
as the array ``arr`` has dimensions.
The macro ``PyArray_SHAPE()`` is a synonym of ``PyArray_DIMS()`` : it has
the same effect and is used in an identical way.
.. method:: npy_intp* PyArray_STRIDES(PyArrayObject* arr)
Returns a pointer on the first element of ``arr``'s internal array
describing the stride for each of its dimension. This array has as many
elements as the number of dimensions in ``arr``. In this array, the
strides are expressed in number of bytes.
.. method:: PyArray_Descr* PyArray_DESCR(PyArrayObject* arr)
Returns a reference to the object representing the dtype of the array.
The macro ``PyArray_DTYPE()`` is a synonym of the ``PyArray_DESCR()`` : it
has the same effect and is used in an identical way.
:note:
This is a borrowed reference so you do not need to decrement its
reference count once you are done with it.
.. method:: int PyArray_TYPE(PyArrayObject* arr)
Returns the typenumber for the elements of the array. Like the dtype, the
typenumber is a descriptor for the type of the data in the array. However,
the two are not synonyms and, as such, cannot be used in place of the
other.
.. method:: npy_intp PyArray_SIZE(PyArrayObject* arr)
Returns to total number of elements in the array
.. method:: bool PyArray_CHKFLAGS(PyArrayObject* arr, flags)
Returns true if the array has the specified flags. The variable flag
should either be a NumPy array flag or an integer obtained by applying
bitwise or to an ensemble of flags.
The flags that can be used in with this macro are :
NPY_ARRAY_C_CONTIGUOUS, NPY_ARRAY_F_CONTIGUOUS, NPY_ARRAY_OWNDATA,
NPY_ARRAY_ALIGNED, NPY_ARRAY_WRITEABLE, NPY_ARRAY_UPDATEIFCOPY.
Creating NumPy ndarrays
-----------------------
The following functions allow the creation and copy of NumPy arrays :
.. method:: PyObject* PyArray_EMPTY(int nd, npy_intp* dims, typenum dtype,
int fortran)
Constructs a new ndarray with the number of dimensions specified by
``nd``, shape specified by ``dims`` and data type specified by ``dtype``.
If ``fortran`` is equal to 0, the data is organized in a C-contiguous
layout, otherwise it is organized in a F-contiguous layout. The array
elements are not initialized in any way.
The function ``PyArray_Empty()`` performs the same function as the macro
``PyArray_EMPTY()`` but the data type is given as a pointer to a
``PyArray_Descr`` object instead of a ``typenum``.
.. method:: PyObject* PyArray_ZEROS(int nd, npy_intp* dims, typenum dtype,
int fortran)
Constructs a new ndarray with the number of dimensions specified by
``nd``, shape specified by ``dims`` and data type specified by ``dtype``.
If ``fortran`` is equal to 0, the data is organized in a C-contiguous
layout, otherwise it is organized in a F-contiguous layout. Every element
in the array is initialized to 0.
The function ``PyArray_Zeros()`` performs the same function as the macro
``PyArray_ZEROS()`` but the data type is given as a pointer to a
``PyArray_Descr`` object instead of a ``typenum``.
.. method:: PyArrayObject* PyArray_GETCONTIGUOUS(PyObject* op)
Returns a C-contiguous and well-behaved copy of the array op. If op is
already C-contiguous and well-behaved, this function simply returns a
new reference to op.
Methods the C Op needs to define
================================
There is a key difference between an op defining a Python implementation for
its computation and defining a C implementation. In the case of a Python
implementation, the op defines a function ``perform()`` which executes the
required Python code to realize the op. In the case of a C implementation,
however, the op does **not** define a function that will execute the C code; it
instead defines functions that will **return** the C code to the caller.
This is because calling C code from Python code comes with a significant
overhead. If every op was responsible for executing its own C code, every
time a Theano function was called, this overhead would occur as many times
as the number of ops with C implementations in the function's computational
graph.
To maximize performance, Theano instead requires the C ops to simply return
the code needed for their execution and takes upon itself the task of
organizing, linking and compiling the code from the various ops. Through this,
Theano is able to minimize the number of times C code is called from Python
code.
The following is a very simple example to illustrate how it's possible to
obtain performance gains with this process. Suppose you need to execute,
from Python code, 10 different ops, each one having a C implementation. If
each op was responsible for executing its own C code, the overhead of
calling C code from Python code would occur 10 times. Consider now the case
where the ops instead return the C code for their execution. You could get
the C code from each op and then define your own C module that would call
the C code from each op in succession. In this case, the overhead would only
occur once; when calling your custom module itself.
Moreover, the fact that Theano itself takes care of compiling the C code,
instead of the individual ops, allows Theano to easily cache the compiled C
code. This allows for faster compilation times.
See :ref:`cop` for the full documentation of the various methods of the
class Op that are related to the C implementation. Of particular interest are:
* The methods :meth:`Op.c_libraries` and :meth:`Op.c_lib_dirs` to allow
your op to use external libraries.
* The method :meth:`Op.c_code_cleanup` to specify how the op should
clean up what it has allocated during its execution.
* The methods :meth:`Op.c_init_code` and :meth:`Op.c_init_code_apply`
to specify code that should be executed once when the module is
initialized, before anything else is executed.
* The methods :meth:`Op.c_compile_args` and
:meth:`Op.c_no_compile_args` to specify requirements regarding how
the op's C code should be compiled.
This section describes the methods :meth:`Op.c_code`,
:meth:`Op.c_support_code`, :meth:`Op.c_support_code_apply` and
:meth:`Op.c_code_cache_version` because they are the ones that are most
commonly used.
.. method:: c_code(node, name, input_names, output_names, sub)
This method returns a string containing the C code to perform the
computation required by this op.
The ``node`` argument is an :ref:`apply` node representing an
application of the current Op on a list of inputs, producing a list of
outputs.
``input_names`` is a sequence of strings which contains as many strings
as the op has inputs. Each string contains the name of the C variable
to which the corresponding input has been assigned. For example, the name
of the C variable representing the first input of the op is given by
``input_names[0]``. You should therefore use this name in your
C code to interact with that variable. ``output_names`` is used
identically to ``input_names``, but for the op's outputs.
Finally, ``sub`` is a dictionary of extras parameters to the c_code
method. Among other things, it contains ``sub['fail']`` which is a string
of C code that you should include in your C code (after ensuring that a
Python exception is set) if it needs to raise an exception. Ex:
.. code-block:: c
c_code = """
PyErr_Format(PyExc_ValueError, "X does not have the right value");
%(fail)s;
""" % {'fail' : sub['fail']}
to raise a ValueError Python exception with the specified message.
The function ``PyErr_Format()`` supports string formatting so it is
possible to tailor the error message to the specifics of the error
that occured. If ``PyErr_Format()`` is called with more than two
arguments, the subsequent arguments are used to format the error message
with the same behavior as the function `PyString_FromFormat()
`_. The
``%`` characters in the format characters need to be escaped since the C
code itself is defined in a string which undergoes string formatting.
.. code-block:: c
c_code = """
PyErr_Format(PyExc_ValueError,
"X==%%i but it should be greater than 0", X);
%(fail)s;
""" % {'fail' : sub['fail']}
:note:
Your C code should not return the output of the computation but
rather put the results in the C variables whose names are contained in
the ``output_names``.
.. method:: c_support_code()
Returns a string or a list of strings containing some support C code for this op. This code
will be included at the global scope level and can be used to define
functions and structs that will be used by every apply of this op.
.. method:: c_support_code_apply(node, name)
Returns a string containing some support C code for this op. This code
will be included at the global scope level and can be used to define
functions and structs that will be used by this op. The difference between
this method and ``c_support_code()`` is that the C code specified in
``c_support_code_apply()`` should be specific to each apply of the Op,
while ``c_support_code()`` is for support code that is not specific to
each apply.
Both ``c_support_code()`` and ``c_support_code_apply ()`` are necessary
because a Theano op can be used more than once in a given Theano
function. For example, an op that adds two matrices could be used at some
point in the Theano function to add matrices of integers and, at another
point, to add matrices of doubles. Because the dtype of the inputs and
outputs can change between different applies of the op, any support code
that relies on a certain dtype is specific to a given apply of the op and
should therefore be defined in ``c_support_code_apply()``.
.. method:: c_code_cache_version()
Returns a tuple of integers representing the version of the C code in this
op. Ex : (1, 4, 0) for version 1.4.0
This tuple is used by Theano to cache the compiled C code for this op. As
such, the return value **MUST BE CHANGED** every time the C code is altered
or else Theano will disregard the change in the code and simply load a
previous version of the op from the cache. If you want to avoid caching of
the C code of this op, return an empty tuple or do not implement this
method.
:note:
Theano can handle tuples of any hashable objects as return values
for this function but, for greater readability and easier management,
this function should return a tuple of integers as previously
described.
Important restrictions when implementing an Op
==============================================
There are some important restrictions to remember when implementing an Op.
Unless your Op correctly defines a ``view_map`` attribute, the ``perform`` and ``c_code`` must not
produce outputs whose memory is aliased to any input (technically, if changing the
output could change the input object in some sense, they are aliased).
Unless your Op correctly defines a ``destroy_map`` attribute, ``perform`` and ``c_code`` must
not modify any of the inputs.
TODO: EXPLAIN DESTROYMAP and VIEWMAP BETTER AND GIVE EXAMPLE.
When developing an Op, you should run computations in DebugMode, by using
argument ``mode='DebugMode'`` to ``theano.function``. DebugMode is
slow, but it can catch many common violations of the Op contract.
TODO: Like what? How? Talk about Python vs. C too.
DebugMode is no silver bullet though.
For example, if you modify an Op ``self.*`` during any of
``make_node``, ``perform``, or ``c_code``, you are probably doing something
wrong but DebugMode will not detect this.
TODO: jpt: I don't understand the following sentence.
Ops and Types should usually be considered immutable -- you should
definitely not make a change that would have an impact on ``__eq__``,
``__hash__``, or the mathematical value that would be computed by ``perform``
or ``c_code``.
Simple C Op example
===================
In this section, we put together the concepts that were covered in this
tutorial to generate an op which multiplies every element in a vector
by a scalar and returns the resulting vector. This is intended to be a simple
example so the methods ``c_support_code()`` and ``c_support_code_apply()`` are
not used because they are not required.
In the C code below notice how the reference count on the output variable is
managed. Also take note of how the new variables required for the op's
computation are declared in a new scope to avoid cross-initialization errors.
Also, in the C code, it is very important to properly validate the inputs
and outputs storage. Theano guarantees that the inputs exist and have the
right number of dimensions but it does not guarantee their exact shape. For
instance, if an op computes the sum of two vectors, it needs to validate that
its two inputs have the same shape. In our case, we do not need to validate
the exact shapes of the inputs because we don't have a need that they match
in any way.
For the outputs, things are a little bit more subtle. Theano does not
guarantee that they have been allocated but it does guarantee that, if they
have been allocated, they have the right number of dimension. Again, Theano
offers no guarantee on the exact shapes. This means that, in our example, we
need to validate that the output storage has been allocated and has the same
shape as our vector input. If it is not the case, we allocate a new output
storage with the right shape and number of dimensions.
.. testcode:: examples
import numpy
import theano
from theano import gof
import theano.tensor as T
class VectorTimesScalar(gof.Op):
__props__ = ()
def make_node(self, x, y):
# Validate the inputs' type
if x.type.ndim != 1:
raise TypeError('x must be a 1-d vector')
if y.type.ndim != 0:
raise TypeError('y must be a scalar')
# Create an output variable of the same type as x
output_var = x.type()
return gof.Apply(self, [x, y], [output_var])
def c_code_cache_version(self):
return (1, 0)
def c_code(self, node, name, inp, out, sub):
x, y = inp
z, = out
# Extract the dtypes of the inputs and outputs storage to
# be able to declare pointers for those dtypes in the C
# code.
dtype_x = node.inputs[0].dtype
dtype_y = node.inputs[1].dtype
dtype_z = node.outputs[0].dtype
itemsize_x = numpy.dtype(dtype_x).itemsize
itemsize_z = numpy.dtype(dtype_z).itemsize
fail = sub['fail']
c_code = """
// Validate that the output storage exists and has the same
// dimension as x.
if (NULL == %(z)s ||
PyArray_DIMS(%(x)s)[0] != PyArray_DIMS(%(z)s)[0])
{
/* Reference received to invalid output variable.
Decrease received reference's ref count and allocate new
output variable */
Py_XDECREF(%(z)s);
%(z)s = (PyArrayObject*)PyArray_EMPTY(1,
PyArray_DIMS(%(x)s),
PyArray_TYPE(%(x)s),
0);
if (!%(z)s) {
%(fail)s;
}
}
// Perform the vector multiplication by a scalar
{
/* The declaration of the following variables is done in a new
scope to prevent cross initialization errors */
npy_%(dtype_x)s* x_data_ptr =
(npy_%(dtype_x)s*)PyArray_DATA(%(x)s);
npy_%(dtype_z)s* z_data_ptr =
(npy_%(dtype_z)s*)PyArray_DATA(%(z)s);
npy_%(dtype_y)s y_value =
((npy_%(dtype_y)s*)PyArray_DATA(%(y)s))[0];
int x_stride = PyArray_STRIDES(%(x)s)[0] / %(itemsize_x)s;
int z_stride = PyArray_STRIDES(%(z)s)[0] / %(itemsize_z)s;
int x_dim = PyArray_DIMS(%(x)s)[0];
for(int i=0; i < x_dim; i++)
{
z_data_ptr[i * z_stride] = (x_data_ptr[i * x_stride] *
y_value);
}
}
"""
return c_code % locals()
The ``c_code`` method accepts variable names as arguments (``name``, ``inp``,
``out``, ``sub``) and returns a C code fragment that computes the expression
output. In case of error, the ``%(fail)s`` statement cleans up and returns
properly.
More complex C Op example
=========================
This section introduces a new example, slightly more complex than the previous
one, with an op to perform an element-wise multiplication between the elements
of two vectors. This new example differs from the previous one in its use
of the methods ``c_support_code()`` and ``c_support_code_apply()`` (it does
not `need` to use them but it does so to explain their use) and its capacity
to support inputs of different dtypes.
Recall the method ``c_support_code()`` is meant to produce code that will
be used for every apply of the op. This means that the C code in this
method must be valid in every setting your op supports. If the op is meant
to supports inputs of various dtypes, the C code in this method should be
generic enough to work with every supported dtype. If the op operates on
inputs that can be vectors or matrices, the C code in this method should
be able to accomodate both kinds of inputs.
In our example, the method ``c_support_code()`` is used to declare a C
function to validate that two vectors have the same shape. Because our
op only supports vectors as inputs, this function is allowed to rely
on its inputs being vectors. However, our op should support multiple
dtypes so this function cannot rely on a specific dtype in its inputs.
The method ``c_support_code_apply()``, on the other hand, is allowed
to depend on the inputs to the op because it is apply-specific. Therefore, we
use it to define a function to perform the multiplication between two vectors.
Variables or functions defined in the method ``c_support_code_apply()`` will
be included at the global scale for every apply of the Op. Because of this,
the names of those variables and functions should include the name of the op,
like in the example. Otherwise, using the op twice in the same graph will give
rise to conflicts as some elements will be declared more than once.
The last interesting difference occurs in the ``c_code()`` method. Because the
dtype of the output is variable and not guaranteed to be the same as any of
the inputs (because of the upcast in the method ``make_node()``), the typenum
of the output has to be obtained in the Python code and then included in the
C code.
.. testcode:: examples
class VectorTimesVector(gof.Op):
__props__ = ()
def make_node(self, x, y):
# Validate the inputs' type
if x.type.ndim != 1:
raise TypeError('x must be a 1-d vector')
if y.type.ndim != 1:
raise TypeError('y must be a 1-d vector')
# Create an output variable of the same type as x
output_var = theano.tensor.TensorType(
dtype=theano.scalar.upcast(x.dtype, y.dtype),
broadcastable=[False])()
return gof.Apply(self, [x, y], [output_var])
def c_code_cache_version(self):
return (1, 0, 2)
def c_support_code(self):
c_support_code = """
bool vector_same_shape(PyArrayObject* arr1,
PyArrayObject* arr2)
{
return (PyArray_DIMS(arr1)[0] == PyArray_DIMS(arr2)[0]);
}
"""
return c_support_code
def c_support_code_apply(self, node, name):
dtype_x = node.inputs[0].dtype
dtype_y = node.inputs[1].dtype
dtype_z = node.outputs[0].dtype
c_support_code = """
void vector_elemwise_mult_%(name)s(npy_%(dtype_x)s* x_ptr,
int x_str, npy_%(dtype_y)s* y_ptr, int y_str,
npy_%(dtype_z)s* z_ptr, int z_str, int nbElements)
{
for (int i=0; i < nbElements; i++){
z_ptr[i * z_str] = x_ptr[i * x_str] * y_ptr[i * y_str];
}
}
"""
return c_support_code % locals()
def c_code(self, node, name, inp, out, sub):
x, y = inp
z, = out
dtype_x = node.inputs[0].dtype
dtype_y = node.inputs[1].dtype
dtype_z = node.outputs[0].dtype
itemsize_x = numpy.dtype(dtype_x).itemsize
itemsize_y = numpy.dtype(dtype_y).itemsize
itemsize_z = numpy.dtype(dtype_z).itemsize
typenum_z = numpy.dtype(dtype_z).num
fail = sub['fail']
c_code = """
// Validate that the inputs have the same shape
if ( !vector_same_shape(%(x)s, %(y)s))
{
PyErr_Format(PyExc_ValueError, "Shape mismatch : "
"x.shape[0] and y.shape[0] should match but "
"x.shape[0] == %%i and y.shape[0] == %%i",
PyArray_DIMS(%(x)s)[0], PyArray_DIMS(%(y)s)[0]);
%(fail)s;
}
// Validate that the output storage exists and has the same
// dimension as x.
if (NULL == %(z)s || !(vector_same_shape(%(x)s, %(z)s)))
{
/* Reference received to invalid output variable.
Decrease received reference's ref count and allocate new
output variable */
Py_XDECREF(%(z)s);
%(z)s = (PyArrayObject*)PyArray_EMPTY(1,
PyArray_DIMS(%(x)s),
%(typenum_z)s,
0);
if (!%(z)s) {
%(fail)s;
}
}
// Perform the vector elemwise multiplication
vector_elemwise_mult_%(name)s(
(npy_%(dtype_x)s*)PyArray_DATA(%(x)s),
PyArray_STRIDES(%(x)s)[0] / %(itemsize_x)s,
(npy_%(dtype_y)s*)PyArray_DATA(%(y)s),
PyArray_STRIDES(%(y)s)[0] / %(itemsize_y)s,
(npy_%(dtype_z)s*)PyArray_DATA(%(z)s),
PyArray_STRIDES(%(z)s)[0] / %(itemsize_z)s,
PyArray_DIMS(%(x)s)[0]);
"""
return c_code % locals()
Alternate way of defining C Ops
===============================
The two previous examples have covered the standard way of implementing C Ops
in Theano by inheriting from the class :class:`Op`. This process is mostly
simple but it still involves defining many methods as well as mixing, in the
same file, both Python and C code which tends to make the result less
readable.
To help with this, Theano defines a class, ``COp``, from which new C ops
can inherit. The class ``COp`` aims to simplify the process of implementing
C ops by doing the following :
* It allows you to define the C implementation of your op in a distinct
C code file. This makes it easier to keep your Python and C code
readable and well indented.
* It can automatically handle all the methods that return C code,
in addition to :meth:`Op.c_code_cache_version()` based on the
provided external C implementation.
To illustrate how much simpler the class ``COp`` makes the process of defining
a new op with a C implementation, let's revisit the second example of this
tutorial, the ``VectorTimesVector`` op. In that example, we implemented an op
to perform the task of element-wise vector-vector multiplication. The two
following blocks of code illustrate what the op would look like if it was
implemented using the ``COp`` class.
The new op is defined inside a Python file with the following code :
.. testcode::
import theano
from theano import gof
class VectorTimesVector(gof.COp):
__props__ = ()
func_file = "./vectorTimesVector.c"
func_name = "APPLY_SPECIFIC(vector_times_vector)"
def __init__(self):
super(VectorTimesVector, self).__init__(self.func_file,
self.func_name)
def make_node(self, x, y):
# Validate the inputs' type
if x.type.ndim != 1:
raise TypeError('x must be a 1-d vector')
if y.type.ndim != 1:
raise TypeError('y must be a 1-d vector')
# Create an output variable of the same type as x
output_var = theano.tensor.TensorType(
dtype=theano.scalar.upcast(x.dtype, y.dtype),
broadcastable=[False])()
return gof.Apply(self, [x, y], [output_var])
And the following is the C implementation of the op, defined in an external
C file named vectorTimesVector.c :
.. code-block:: c
#section support_code
// Support code function
bool vector_same_shape(PyArrayObject* arr1, PyArrayObject* arr2)
{
return (PyArray_DIMS(arr1)[0] == PyArray_DIMS(arr2)[0]);
}
#section support_code_apply
// Apply-specific support function
void APPLY_SPECIFIC(vector_elemwise_mult)(
DTYPE_INPUT_0* x_ptr, int x_str,
DTYPE_INPUT_1* y_ptr, int y_str,
DTYPE_OUTPUT_0* z_ptr, int z_str, int nbElements)
{
for (int i=0; i < nbElements; i++){
z_ptr[i * z_str] = x_ptr[i * x_str] * y_ptr[i * y_str];
}
}
// Apply-specific main function
int APPLY_SPECIFIC(vector_times_vector)(PyArrayObject* input0,
PyArrayObject* input1,
PyArrayObject** output0)
{
// Validate that the inputs have the same shape
if ( !vector_same_shape(input0, input1))
{
PyErr_Format(PyExc_ValueError, "Shape mismatch : "
"input0.shape[0] and input1.shape[0] should "
"match but x.shape[0] == %i and "
"y.shape[0] == %i",
PyArray_DIMS(input0)[0], PyArray_DIMS(input1)[0]);
return 1;
}
// Validate that the output storage exists and has the same
// dimension as x.
if (NULL == *output0 || !(vector_same_shape(input0, *output0)))
{
/* Reference received to invalid output variable.
Decrease received reference's ref count and allocate new
output variable */
Py_XDECREF(*output0);
*output0 = (PyArrayObject*)PyArray_EMPTY(1,
PyArray_DIMS(input0),
TYPENUM_OUTPUT_0,
0);
if (!*output0) {
PyErr_Format(PyExc_ValueError,
"Could not allocate output storage");
return 1;
}
}
// Perform the actual vector-vector multiplication
APPLY_SPECIFIC(vector_elemwise_mult)(
(DTYPE_INPUT_0*)PyArray_DATA(input0),
PyArray_STRIDES(input0)[0] / ITEMSIZE_INPUT_0,
(DTYPE_INPUT_1*)PyArray_DATA(input1),
PyArray_STRIDES(input1)[0] / ITEMSIZE_INPUT_1,
(DTYPE_OUTPUT_0*)PyArray_DATA(*output0),
PyArray_STRIDES(*output0)[0] / ITEMSIZE_OUTPUT_0,
PyArray_DIMS(input0)[0]);
return 0;
}
As you can see from this example, the Python and C implementations are nicely
decoupled which makes them much more readable than when they were intertwined
in the same file and the C code contained string formatting markers.
Now that we have motivated the COp class, we can have a more precise look at
what it does for us. For this, we go through the various elements that make up
this new version of the VectorTimesVector op :
* Parent class : instead of inheriting from the class :class:`Op`,
VectorTimesVector inherits from the class ``COp``.
* Constructor : in our new op, the ``__init__()`` method has an
important use; to inform the constructor of the ``COp`` class
of the location, on the filesystem of the C implementation of
this op. To do this, it gives a list of file paths containing
the C code for this op. To auto-generate the c_code method
with a function call you can specify the function name as the
second parameter. The paths should be given as a relative
path from the folder where the descendant of the ``COp`` class
is defined.
* ``make_node()`` : the ``make_node()`` method is absolutely
identical to the one in our old example. Using the ``COp``
class doesn't change anything here.
* External C code : the external C code implements the various
functions associated with the op. Writing this C code
involves a few subtleties which deserve their own respective
sections.
Main function
-------------
If you pass a function name to the ``__init__()`` method of the
``COp`` class, it must respect the following constraints:
* It must return an int. The value of that int indicates whether
the op could perform its task or not. A value of 0 indicates
success while any non-zero value will interrupt the execution
of the Theano function. When returning non-zero the function
must set a python exception indicating the details of the
problem.
* It must receive one argument for each input to the op followed
by one pointer to an argument for each output of the op. The
types for the argument is dependant on the Types (that is
theano Types) of your inputs and outputs.
* You can sepcify the number of inputs and outputs for your op
by setting the ``_cop_num_inputs`` and ``_cop_num_outputs``
attributes on your op. The main function will always be
called with that number of arguments, using NULL to fill in
for missing values at the end. This can be used if your op
has a variable number of inputs or outputs, but with a fixed
maximum.
For example, the main C function of an op that takes two TensorTypes
(which has ``PyArrayObject *`` as its C type) as inputs and returns
both their sum and the difference between them would have four
parameters (two for the op's inputs and two for its outputs) and it's
signature would look something like this :
.. code-block:: c
int sumAndDiffOfScalars(PyArrayObject* in0, PyArrayObject* in1,
PyArrayObject** out0, PyArrayObject** out1)
Macros
------
For certain section tags, your C code can benefit from a number of
pre-defined macros. These section tags have no macros: ``init_code``,
``support_code``. All other tags will have the support macros
discussed below.
* ``APPLY_SPECIFIC(str)`` which will automatically append a name
unique to the :ref:`Apply` node that applies the Op at the end
of the provided ``str``. The use of this macro is discussed
futher below.
For every input which has a :attr:`dtype` attribute (this means
Tensors, and equivalent types on GPU), the following macros will be
defined unless your Op class has an :attr:`Op.check_input` attribute
defined to False. In these descrptions 'i' refers to the position
(indexed from 0) in the input array.
* ``DTYPE_INPUT_{i}`` : NumPy dtype of the data in the array.
This is the variable type corresponding to the NumPy dtype, not the
string representation of the NumPy dtype. For instance, if the op's
first input is a float32 ndarray, then the macro ``DTYPE_INPUT_0``
corresponds to ``npy_float32`` and can directly be used to declare a
new variable of the same dtype as the data in the array :
.. code-block:: c
DTYPE_INPUT_0 myVar = someValue;
* ``TYPENUM_INPUT_{i}`` : Typenum of the data in the array
* ``ITEMSIZE_INPUT_{i}`` : Size, in bytes, of the elements in
the array.
In the same way, the macros ``DTYPE_OUTPUT_{i}``,
``ITEMSIZE_OUTPUT_{i}`` and ``TYPENUM_OUTPUT_{i}`` are defined for
every output 'i' of the op.
In addition to these macros, the ``init_code_struct``, ``code``, and
``code_cleanup`` section tags also have the following macros:
* ``FAIL`` : Code to insert at error points. A python exception
should be set prior to this code. An invocation look like this:
.. code-block:: c
if (error) {
// Set python exception
FAIL
}
You can add a semicolon after the macro if it makes your editor
happy.
* ``PARAMS`` : Name of the params variable for this node. (only
for Ops which have params, which is discussed elsewhere)
Finally the tag ``code`` and ``code_cleanup`` have macros to
pass the inputs and output names. These are name ``INPUT_{i}`` and
``OUTPUT_{i}`` where `i` is the 0-based index position in the input
and output arrays respectively.
Support code
------------
Certain section are limited in what you can place in them due to
semantic and syntactic restrictions of the C++ language. Most of
these restrictions apply to the tags that end in ``_struct``.
When we defined the VectorTimesVector op without using the ``COp``
class, we had to make a distinction between two types of support_code
: the support code that was apply-specific and the support code that
wasn't. The apply-specific code was defined in the
``c_support_code_apply()`` method and the elements defined in that
code (global variables and functions) had to include the name of the
Apply node in their own names to avoid conflicts between the different
versions of the apply-specific code. The code that wasn't
apply-specific was simply defined in the ``c_support_code()`` method.
To make indentifiers that include the :ref:`Apply` node name use the
``APPLY_SPECIFIC(str)`` macro. In the above example, this macro is
used when defining the functions ``vector_elemwise_mult()`` and
``vector_times_vector()`` as well as when calling function
``vector_elemwise_mult()`` from inside ``vector_times_vector()``.
When using the ``COp`` class, we still have to make the distinction
between C code for each of the methods of a C class. These sections of
code are separated by ``#section `` markers. The tag determines
the name of the method this C code applies to with the rule that
```` applies to `c_`. Unknown tags are an error and will be
reported. Duplicate tags will be merged together in the order the
appear in the C files.
The rules for knowing if where a piece of code should be put can be
sometimes tricky. The key thing to remember is that things that can
be shared between instances of the op should be apply-agnostic and go
into a section which does not end in ``_apply`` or ``_struct``. The
distinction of ``_apply`` and ``_struct`` mostly hinghes on how you
want to manange the lifetime of the object. Note that to use an
apply-specific object, you have to be in a apply-specific section, so
some portions of the code that might seem apply-agnostic may still be
apply-specific because of the data they use (this does not include
arguments).
In the above example, the ``function vector_same_shape()`` is
apply-agnostic because it uses none of the macros defined by the class
``COp`` and it doesn't rely on any apply-specific code. The function
``vector_elemwise_mult()`` is apply-specific because it uses the
macros defined by ``COp``. Finally, the function
``vector_times_vector()`` is apply-specific because it uses those same
macros and also because it calls ``vector_elemwise_mult()`` which is
an apply-specific function.
Using GDB to debug Op's C code
==============================
When debugging C code, it can be useful to use GDB for code compiled
by Theano.
For this, you must enable this Theano: `cmodule.remove_gxx_opt=True`.
For the GPU, you must add in this second flag `nvcc.flags=-g` (it slow
down computation on the GPU, but it is enabled by default on the CPU).
Then you must start Python inside GDB and in it start your Python
process (e.g. theano-nose):
.. code-block:: sh
$gdb python
(gdb)r bin/theano-nose theano/
`Quick guide to GDB `.
Theano-1.0.1/doc/extending/extending_theano_gpu.txt 0000777 0000000 0000000 00000026103 13212236347 022372 0 ustar root root 0000000 0000000
.. _extending_theano_gpu:
==============================
Extending Theano with a GPU Op
==============================
.. note::
This covers the :ref:`gpuarray ` back-end for the GPU.
This tutorial covers how to extend Theano with an op that offers a GPU
implementation. It assumes you are familiar with how to write new
Theano ops. If that is not the case you should probably follow the
:ref:`extending_theano` and :ref:`extending_theano_c` sections before
continuing on.
Writing a new GPU op can be done in Python for some simple tasks, but
will usually done in C to access the complete API and avoid paying the
overhead of a Python function call.
Dealing With the Context
========================
One of the major differences with GPU ops is that they require a
context (a.k.a. device) to execute. Most of the time you can infer
the context to run on from your inputs. There is a way for the user
to transfer things between contexts and to tag certain variables for
transfer. It might also be the case that your inputs are not all from
the same context and you would have to choose which one to run on.
In order to support all of those options and have a consistent
interface, :func:`theano.gpuarray.basic_ops.infer_context_name` was
written. An example usage is below::
def make_node(self, a, b, c):
ctx = infer_context_name(a, b, c)
a = as_gpuarray_variable(a, ctx)
b = as_gpuarray_variable(b, ctx)
c = as_gpuarray_variable(c, ctx)
return Apply(self, [a, b, c], [a.type()])
In this example the Op takes three inputs, all on the GPU. In case
one or more of your inputs is not supposed to be on the GPU, you
should not pass it to :func:`infer_context_name` or call
:func:`as_gpuarray_variable` on it.
Also note that :func:`theano.gpuarray.basic_ops.as_gpuarray_variable`
takes ``context_name`` as a mandatory parameter. This is because it's
not enough to know you want the value to be on the GPU, you also want
to know which GPU to put it on. In almost all cases, you can pass in
the return value of :func:`infer_context_name` there.
If you also need the context during runtime (for example to allocate
the output), you can use the context of one of your inputs to know
which one to use. Here is another example::
def perform(self, node, inputs, output_storage):
A, B = inputs
C, = output_storage
C[0] = pygpu.empty([A.shape[0], B.shape[1]], dtype=A.dtype, A.context)
pygpu.blas.gemm(1, A, B, 0, C, overwrite_c=True)
Finally if you require the context before perform, such as during
make_thunk() to initialize kernels and such, you can access the
context of your inputs through the type of the variables::
def make_thunk(self, node, storage_map, compute_map, no_recycling):
ctx = node.inputs[0].type.context
Note that ``GpuArrayType`` objects also have a ``context_name``
attribute which is the symbolic equivalent of ``context``. It can't
be used for calls to pygpu or libgpuarray, but it should be used for
theano operations and variables.
The last place where you might need the context is in the C
initialization code. For that you will have to use the :ref:`params
`. The params type should be
:data:`theano.gpuarray.type.gpu_context_type` and the params object
should be a context object from one of your input variables::
def get_params(self, node):
return node.inputs[0].type.context
If you don't have any input variables on the GPU you can follow the
the example of :class:`GpuFromHost
` or :class:`GpuEye
`. This is not a case that you
should encounter often, so it will not be covered further.
Defining New Kernels
====================
If your op needs to do some transformation on the data, chances are
that you will need to write a new kernel. The best way to do this is
to leverage :class:`GpuKernelBase
` (or :class:`CGpuKernelBase
` if you want to use the
:class:`COp ` functionality).
For plain :class:`GpuKernelBase
`, you have to define a
method called ``gpu_kernels`` which returns a list of :class:`Kernel
` objects. You can define as many
kernels as you want for a single op. An example would look like
this::
def gpu_kernels(self, node, name):
code = """
KERNEL void k(GLOBAL_MEM ga_double *a, ga_size n, ga_size m) {
ga_size nb = n < m ? n : m;
for (ga_size i = LID_0; i < nb; i += LDIM_0) {
a[i*m + i] = 1;
}
}"""
return [Kernel(
code=code, name="k",
params=[gpuarray.GpuArray, gpuarray.SIZE, gpuarray.SIZE],
flags=Kernel.get_flags('float64'))]
If you want to use ``COp``, then you should use ``CGpuKernelBase``
instead. It adds a new section to the parsed files whose tag is
``kernels``. Inside that section you can define some kernels with
``#kernel name:params:flags``.
Here ``name`` is the name of the kernel function in the following
code, ``params`` is a comma-separeted list of numpy typecode names.
There are three exceptions for ``size_t`` which should be noted as
``size``, ``ssize_t`` which should be noted as ``ssize`` and a pointer
which should be noted as ``*``.
``flags`` is a ``|``-separated list of C kernel flag values (can be
empty). The same kernel definition as above would look like this with
``CGpuKernelBase``::
#section kernels
#kernel k : *, size, size : GA_USE_DOUBLE
KERNEL void k(GLOBAL_MEM ga_double *a, ga_size n, ga_size m) {
ga_size nb = n < m ? n : m;
for (ga_size i = LID_0; i < nb; i += LDIM_0) {
a[i*m + i] = 1;
}
}
The second method is to handle the kernel compilation and cache on
your own. This is not recommended because there are lots of details
to pay attention to that can cripple your performance if not done
right, which GpuKernelBase handles for you. But if you really want to
go this way, then you can look up the C API for kernels in
libgpuarray.
In any case you will need to call your compiled kernel with some data,
in most cases in your :meth:`c_code` method. This is done by using
the provided wrapper function. An example calling the above kernel
would be::
size_t dims[2];
size_t n = 256;
// ...
err = k_scall(1, &n, 0, input->ga.data, dims[0], dims[1]);
// ...
If you want explicit control over the scheduling, you can use the
`_call` wrapper instead which works like this::
size_t ls, gs;
// ...
gs = 1;
ls = 256;
err = k_call(1, &gs, &ls, 0, input->ga.data, dims[0], dims[1]);
The name of the wrapper function depends on the name you passed to
``Kernel()`` when you declared it (or the name in your `#kernel`
statement). It defaults to `name + '_call' or '_scall'`.
For other operations in the C code you should refer to the
`libgpuarray documentation
`. Similarly writes
should be wrapped in the function returned by :func:`write_w()
`. Finally working data should
have the type returned by :func:`work_dtype()
`.
Here is a +1 kernel that is not ready to deal with float16 input::
type_x = dtype_to_ctype(x.dtype)
type_y = dtype_to_ctype(y.dtype)
"""
KERNEL void k(const ga_size n, %(type_x)s *x, %(type_y)s *y) {
ga_size i = GID_0 * LDIM_0 + LID_0;
%(type_x) z = x[i];
z += 1;
y[i] = z;
}
""" % dict(dtype_x=dtype_x, dtype_y=dtype_y)
Here is the same kernel, but now ready to handle float16::
type_x = dtype_to_ctype(x.dtype)
type_y = dtype_to_ctype(y.dtype)
work_x = dtype_to_ctype(work_dtype(x.dtype))
load_x = load_w(x.dtype)
write_y = write_w(y.dtype)
"""
KERNEL void k(const ga_size n, %(type_x)s *x, %(type_y)s *y) {
ga_size i = GID_0 * LDIM_0 + LID_0;
%(work_x) z = %(load_x)(x[i]);
z += 1;
y[i] = %(write_y)(z);
}
""" % dict(dtype_x=dtype_x, dtype_y=dtype_y, work_x=work_x, load_x=load_x,
write_y=write_y)
Once you have converted your kernels for float16 support you need to
tag your op with ``_f16_ok = True`` so that the linker will accept to
generate C code on float16 types. This is done by inserting it as a
class property like this::
class SomeOp(Op):
_f16_ok = True
If this attribute is not present or is False, the linker will print a
message saying that it's refusing to use C code for float16 for the
op.
A Complete Example
==================
This is a complete example using both approches for a implementation
of the Eye operation.
GpuKernelBase
-------------
Python File
~~~~~~~~~~~
.. literalinclude:: ../../theano/gpuarray/basic_ops.py
:language: python
:pyobject: GpuEye
CGpuKernelBase
--------------
Python File
~~~~~~~~~~~
.. literalinclude:: ../../theano/gpuarray/tests/test_cgpukernelbase.py
:language: python
:pyobject: GpuEye
``tstgpueye.c``
~~~~~~~~~~~~~~~
.. literalinclude:: ../../theano/gpuarray/tests/c_code/tstgpueye.c
:language: C
Wrapping Exisiting Libraries
============================
PyCUDA
------
For things in PyCUDA (or things wrapped with PyCUDA), we usually need
to create a PyCUDA context. This can be done with the following
code::
with gpuarray_cuda_context:
pycuda_context = pycuda.driver.Context.attach()
If you don't need to create a context, because the library doesn't
require it, you can also just use the pygpu context and a `with`
statement like above for all your code which will make the context the
current context on the cuda stack.
GpuArray objects are compatible with PyCUDA and will expose the
necessary interface so that they can be used in most things. One
notable exception is PyCUDA kernels which require native objects. If
you need to convert a pygpu GpuArray to a PyCUDA GPUArray, this code
should do the trick::
assert pygpu_array.flags['IS_C_CONTIGUOUS']
pycuda_array = pycuda.gpuarray.GPUArray(pygpu_array.shape,
pygpu_array.dtype,
base=pygpu_array,
gpudata=(pygpu_array.gpudata +
pygpu_array.offset))
As long as the computations happen on the NULL stream there are no
special considerations to watch for with regards to synchronization.
Otherwise, you will have to make sure that you synchronize the pygpu
objects by calling the `.sync()` method before scheduling any work and
synchronize with the work that happens in the library after all the
work is scheduled.
Theano-1.0.1/doc/extending/fibby.txt 0000777 0000000 0000000 00000013311 13212236347 017264 0 ustar root root 0000000 0000000
=============================================
Writing an Op to work on an ``ndarray`` in C
=============================================
This section walks through a non-trivial example Op that does something pretty
weird and unrealistic, that is hard to express with existing Ops.
(Technically, we could use ``Scan`` to implement the Op we're about to describe,
but we ignore that possibility for the sake of example.)
The following code works, but important error-checking has been omitted for
clarity. For example, when you write C code that assumes memory is contiguous,
you should check the strides and alignment.
.. testcode::
import theano
class Fibby(theano.Op):
"""
An arbitrarily generalized Fibbonacci sequence
"""
__props__ = ()
def make_node(self, x):
x_ = tensor.as_tensor_variable(x)
assert x_.ndim == 1
return theano.Apply(self,
inputs=[x_],
outputs=[x_.type()])
# using x_.type() is dangerous, it copies x's broadcasting behaviour
def perform(self, node, inputs, output_storage):
x, = inputs
y = output_storage[0][0] = x.copy()
for i in range(2, len(x)):
y[i] = y[i-1] * y[i-2] + x[i]
def c_code(self, node, name, inames, onames, sub):
x, = inames
y, = onames
fail = sub['fail']
return """
Py_XDECREF(%(y)s);
%(y)s = (PyArrayObject*)PyArray_FromArray(
%(x)s, 0, NPY_ARRAY_ENSURECOPY);
if (!%(y)s)
%(fail)s;
{//New scope needed to make compilation work
dtype_%(y)s * y = (dtype_%(y)s*)PyArray_DATA(%(y)s);
dtype_%(x)s * x = (dtype_%(x)s*)PyArray_DATA(%(x)s);
for (int i = 2; i < PyArray_DIMS(%(x)s)[0]; ++i)
y[i] = y[i-1]*y[i-2] + x[i];
}
""" % locals()
def c_code_cache_version(self):
return (1,)
fibby = Fibby()
In the first two lines of the C function, we make y point to a new array with
the correct size for the output. This is essentially simulating the line
``y = x.copy()``.
The variables ``%(x)s`` and ``%(y)s`` are set up by the TensorType to be ``PyArrayObject`` pointers.
TensorType also set up ``dtype_%(x)s`` to be a typdef to the C type for ``x``.
.. code-block:: c
Py_XDECREF(%(y)s);
%(y)s = (PyArrayObject*)PyArray_FromArray(
%(x)s, 0, NPY_ARRAY_ENSURECOPY);
The first line reduces the reference count of the data that y originally
pointed to. The second line allocates the new data and makes y point to it.
In C code for a theano op, numpy arrays are represented as ``PyArrayObject`` C
structs. This is part of the numpy/scipy C API documented at
http://docs.scipy.org/doc/numpy/reference/c-api.types-and-structures.html
TODO: NEEDS MORE EXPLANATION.
.. _op_contract_fibby:
Writing an Optimization
=======================
``fibby`` of a vector of zeros is another vector of zeros of
the same size.
Theano does not attempt to infer this from the code provided via ``Fibby.perform`` or ``Fibby.c_code``.
However, we can write an optimization that makes use of this observation.
This sort of local substitution of special cases is common,
and there is a stage of optimization (specialization) devoted to such optimizations.
The following optimization (``fibby_of_zero``) tests whether the input is
guaranteed to be all zero, and if so it returns the input itself as a replacement
for the old output.
TODO: talk about OPTIMIZATION STAGES
.. If you modify this code, also change :
.. theano/tests/test_tutorial.py:T_fibby.test_fibby_1
.. testcode::
from theano.tensor.opt import get_scalar_constant_value, NotScalarConstantError
# Remove any fibby(zeros(...))
@theano.tensor.opt.register_specialize
@theano.gof.local_optimizer([fibby])
def fibby_of_zero(node):
if node.op == fibby:
x = node.inputs[0]
try:
if numpy.all(0 == get_scalar_constant_value(x)):
return [x]
except NotScalarConstantError:
pass
The ``register_specialize`` decorator is what activates our optimization, and
tells Theano to use it in the specialization stage.
The ``local_optimizer`` decorator builds a class instance around our global
function. The ``[fibby]`` argument is a hint that our optimizer works on nodes
whose ``.op`` attribute equals ``fibby``.
The function here (``fibby_of_zero``) expects an ``Apply`` instance as an
argument for parameter ``node``. It tests using
function ``get_scalar_constant_value``, which determines if a
Variable (``x``) is guaranteed to be a constant, and if so, what constant.
Test the optimization
=====================
Here is some code to test that the optimization is applied only when needed.
.. testcode::
import numpy
import theano.tensor as T
from theano import function
from theano import tensor
# Test it does not apply when not needed
x = T.dvector()
f = function([x], fibby(x))
# We call the function to make sure it runs.
# If you run in DebugMode, it will compare the C and Python outputs.
f(numpy.random.rand(5))
topo = f.maker.fgraph.toposort()
assert len(topo) == 1
assert isinstance(topo[0].op, Fibby)
# Test that the optimization gets applied.
f_zero = function([], fibby(T.zeros([5])))
# If you run in DebugMode, it will compare the output before
# and after the optimization.
f_zero()
# Check that the optimization removes the Fibby Op.
# For security, the Theano memory interface ensures that the output
# of the function is always memory not aliased to the input.
# That is why there is a DeepCopyOp op.
topo = f_zero.maker.fgraph.toposort()
assert len(topo) == 1
assert isinstance(topo[0].op, theano.compile.ops.DeepCopyOp)
Theano-1.0.1/doc/extending/graphstructures.txt 0000777 0000000 0000000 00000037670 13212236347 021454 0 ustar root root 0000000 0000000
.. _graphstructures:
================
Graph Structures
================
Debugging or profiling code written in Theano is not that simple if you
do not know what goes on under the hood. This chapter is meant to
introduce you to a required minimum of the inner workings of Theano.
The first step in writing Theano code is to write down all mathematical
relations using symbolic placeholders (**variables**). When writing down
these expressions you use operations like ``+``, ``-``, ``**``,
``sum()``, ``tanh()``. All these are represented internally as **ops**.
An *op* represents a certain computation on some type of inputs
producing some type of output. You can see it as a *function definition*
in most programming languages.
Theano represents symbolic mathematical computations as graphs. These
graphs are composed of interconnected :ref:`apply`, :ref:`variable` and
:ref:`op` nodes. *Apply* node represents the application of an *op* to some
*variables*. It is important to draw the difference between the
definition of a computation represented by an *op* and its application
to some actual data which is represented by the *apply* node.
Furthermore, data types are represented by :ref:`type` instances. Here is a
piece of code and a diagram showing the structure built by that piece of code.
This should help you understand how these pieces fit together:
**Code**
.. testcode::
import theano.tensor as T
x = T.dmatrix('x')
y = T.dmatrix('y')
z = x + y
**Diagram**
.. _tutorial-graphfigure:
.. image:: apply.png
:align: center
Arrows represent references to the Python objects pointed at. The blue
box is an :ref:`Apply` node. Red boxes are :ref:`Variable` nodes. Green
circles are :ref:`Ops `. Purple boxes are :ref:`Types `.
.. TODO
Clarify the 'acyclic' graph and the 'back' pointers or references that
'don't count'.
When we create :ref:`Variables ` and then :ref:`apply`
:ref:`Ops ` to them to make more Variables, we build a
bi-partite, directed, acyclic graph. Variables point to the Apply nodes
representing the function application producing them via their
``owner`` field. These Apply nodes point in turn to their input and
output Variables via their ``inputs`` and ``outputs`` fields.
(Apply instances also contain a list of references to their ``outputs``, but
those pointers don't count in this graph.)
The ``owner`` field of both ``x`` and ``y`` point to ``None`` because
they are not the result of another computation. If one of them was the
result of another computation, it's ``owner`` field would point to another
blue box like ``z`` does, and so on.
Note that the ``Apply`` instance's outputs points to
``z``, and ``z.owner`` points back to the ``Apply`` instance.
Traversing the graph
====================
The graph can be traversed starting from outputs (the result of some
computation) down to its inputs using the owner field.
Take for example the following code:
>>> import theano
>>> x = theano.tensor.dmatrix('x')
>>> y = x * 2.
If you enter ``type(y.owner)`` you get ````,
which is the apply node that connects the op and the inputs to get this
output. You can now print the name of the op that is applied to get
*y*:
>>> y.owner.op.name
'Elemwise{mul,no_inplace}'
Hence, an elementwise multiplication is used to compute *y*. This
multiplication is done between the inputs:
>>> len(y.owner.inputs)
2
>>> y.owner.inputs[0]
x
>>> y.owner.inputs[1]
InplaceDimShuffle{x,x}.0
Note that the second input is not 2 as we would have expected. This is
because 2 was first :term:`broadcasted ` to a matrix of
same shape as *x*. This is done by using the op ``DimShuffle`` :
>>> type(y.owner.inputs[1])
>>> type(y.owner.inputs[1].owner)
>>> y.owner.inputs[1].owner.op # doctest: +SKIP
>>> y.owner.inputs[1].owner.inputs
[TensorConstant{2.0}]
Starting from this graph structure it is easier to understand how
*automatic differentiation* proceeds and how the symbolic relations
can be *optimized* for performance or stability.
Graph Structures
================
The following section outlines each type of structure that may be used
in a Theano-built computation graph. The following structures are
explained: :ref:`apply`, :ref:`constant`, :ref:`op`, :ref:`variable` and
:ref:`type`.
.. index::
single: Apply
single: graph construct; Apply
.. _apply:
Apply
-----
An *Apply node* is a type of internal node used to represent a
:term:`computation graph ` in Theano. Unlike
:ref:`Variable nodes `, Apply nodes are usually not
manipulated directly by the end user. They may be accessed via
a Variable's ``owner`` field.
An Apply node is typically an instance of the :class:`Apply`
class. It represents the application
of an :ref:`op` on one or more inputs, where each input is a
:ref:`variable`. By convention, each Op is responsible for
knowing how to build an Apply node from a list of
inputs. Therefore, an Apply node may be obtained from an Op
and a list of inputs by calling ``Op.make_node(*inputs)``.
Comparing with the Python language, an :ref:`apply` node is
Theano's version of a function call whereas an :ref:`op` is
Theano's version of a function definition.
An Apply instance has three important fields:
**op**
An :ref:`op` that determines the function/transformation being
applied here.
**inputs**
A list of :ref:`Variables ` that represent the arguments of
the function.
**outputs**
A list of :ref:`Variables ` that represent the return values
of the function.
An Apply instance can be created by calling ``gof.Apply(op, inputs, outputs)``.
.. index::
single: Op
single: graph construct; Op
.. _op:
Op
--
An :ref:`op` in Theano defines a certain computation on some types of
inputs, producing some types of outputs. It is equivalent to a
function definition in most programming languages. From a list of
input :ref:`Variables ` and an Op, you can build an :ref:`apply`
node representing the application of the Op to the inputs.
It is important to understand the distinction between an Op (the
definition of a function) and an Apply node (the application of a
function). If you were to interpret the Python language using Theano's
structures, code going like ``def f(x): ...`` would produce an Op for
``f`` whereas code like ``a = f(x)`` or ``g(f(4), 5)`` would produce an
Apply node involving the ``f`` Op.
.. index::
single: Type
single: graph construct; Type
.. _type:
Type
----
A :ref:`type` in Theano represents a set of constraints on potential
data objects. These constraints allow Theano to tailor C code to handle
them and to statically optimize the computation graph. For instance,
the :ref:`irow ` type in the ``theano.tensor`` package
gives the following constraints on the data the Variables of type ``irow``
may contain:
#. Must be an instance of ``numpy.ndarray``: ``isinstance(x, numpy.ndarray)``
#. Must be an array of 32-bit integers: ``str(x.dtype) == 'int32'``
#. Must have a shape of 1xN: ``len(x.shape) == 2 and x.shape[0] == 1``
Knowing these restrictions, Theano may generate C code for addition, etc.
that declares the right data types and that contains the right number
of loops over the dimensions.
Note that a Theano :ref:`type` is not equivalent to a Python type or
class. Indeed, in Theano, :ref:`irow ` and :ref:`dmatrix
` both use ``numpy.ndarray`` as the underlying type
for doing computations and storing data, yet they are different Theano
Types. Indeed, the constraints set by ``dmatrix`` are:
#. Must be an instance of ``numpy.ndarray``: ``isinstance(x, numpy.ndarray)``
#. Must be an array of 64-bit floating point numbers: ``str(x.dtype) == 'float64'``
#. Must have a shape of MxN, no restriction on M or N: ``len(x.shape) == 2``
These restrictions are different from those of ``irow`` which are listed above.
There are cases in which a Type can fully correspond to a Python type,
such as the ``double`` Type we will define here, which corresponds to
Python's ``float``. But, it's good to know that this is not necessarily
the case. Unless specified otherwise, when we say "Type" we mean a
Theano Type.
.. index::
single: Variable
single: graph construct; Variable
.. _variable:
Variable
--------
A :ref:`variable` is the main data structure you work with when using
Theano. The symbolic inputs that you operate on are Variables and what
you get from applying various Ops to these inputs are also
Variables. For example, when I type
>>> import theano
>>> x = theano.tensor.ivector()
>>> y = -x
``x`` and ``y`` are both Variables, i.e. instances of the :class:`Variable` class. The :ref:`type` of both ``x`` and
``y`` is ``theano.tensor.ivector``.
Unlike ``x``, ``y`` is a Variable produced by a computation (in this
case, it is the negation of ``x``). ``y`` is the Variable corresponding to
the output of the computation, while ``x`` is the Variable
corresponding to its input. The computation itself is represented by
another type of node, an :ref:`apply` node, and may be accessed
through ``y.owner``.
More specifically, a Variable is a basic structure in Theano that
represents a datum at a certain point in computation. It is typically
an instance of the class :class:`Variable` or
one of its subclasses.
A Variable ``r`` contains four important fields:
**type**
a :ref:`type` defining the kind of value this Variable can hold in
computation.
**owner**
this is either None or an :ref:`apply` node of which the Variable is
an output.
**index**
the integer such that ``owner.outputs[index] is r`` (ignored if
``owner`` is None)
**name**
a string to use in pretty-printing and debugging.
Variable has one special subclass: :ref:`Constant `.
.. index::
single: Constant
single: graph construct; Constant
.. _constant:
Constant
^^^^^^^^
A Constant is a :ref:`Variable` with one extra field, *data* (only
settable once). When used in a computation graph as the input of an
:ref:`Op` :ref:`application `, it is assumed that said input
will *always* take the value contained in the constant's data
field. Furthermore, it is assumed that the :ref:`Op` will not under
any circumstances modify the input. This means that a constant is
eligible to participate in numerous optimizations: constant inlining
in C code, constant folding, etc.
A constant does not need to be specified in a :func:`function
`'s list
of inputs. In fact, doing so will raise an exception.
Graph Structures Extension
==========================
When we start the compilation of a Theano function, we compute some
extra information. This section describes a portion of the information
that is made available. Not everything is described, so email
theano-dev if you need something that is missing.
The graph gets cloned at the start of compilation, so modifications done
during compilation won't affect the user graph.
Each variable receives a new field called clients. It is a list with
references to every place in the graph where this variable is used. If
its length is 0, it means the variable isn't used. Each place where it
is used is described by a tuple of 2 elements. There are two types of
pairs:
- The first element is an Apply node.
- The first element is the string "output". It means the
function outputs this variable.
In both types of pairs, the second element of the tuple is an index,
such that: ``var.clients[*][0].inputs[index]`` or
``fgraph.outputs[index]`` is that variable.
>>> import theano
>>> v = theano.tensor.vector()
>>> f = theano.function([v], (v+1).sum())
>>> theano.printing.debugprint(f)
Sum{acc_dtype=float64} [id A] '' 1
|Elemwise{add,no_inplace} [id B] '' 0
|TensorConstant{(1,) of 1.0} [id C]
| [id D]
>>> # Sorted list of all nodes in the compiled graph.
>>> topo = f.maker.fgraph.toposort()
>>> topo[0].outputs[0].clients
[(Sum{acc_dtype=float64}(Elemwise{add,no_inplace}.0), 0)]
>>> topo[1].outputs[0].clients
[('output', 0)]
>>> # An internal variable
>>> var = topo[0].outputs[0]
>>> client = var.clients[0]
>>> client
(Sum{acc_dtype=float64}(Elemwise{add,no_inplace}.0), 0)
>>> type(client[0])
>>> assert client[0].inputs[client[1]] is var
>>> # An output of the graph
>>> var = topo[1].outputs[0]
>>> client = var.clients[0]
>>> client
('output', 0)
>>> assert f.maker.fgraph.outputs[client[1]] is var
Automatic Differentiation
=========================
Having the graph structure, computing automatic differentiation is
simple. The only thing :func:`tensor.grad` has to do is to traverse the
graph from the outputs back towards the inputs through all *apply*
nodes (*apply* nodes are those that define which computations the
graph does). For each such *apply* node, its *op* defines
how to compute the *gradient* of the node's outputs with respect to its
inputs. Note that if an *op* does not provide this information,
it is assumed that the *gradient* is not defined.
Using the
`chain rule `_
these gradients can be composed in order to obtain the expression of the
*gradient* of the graph's output with respect to the graph's inputs.
A following section of this tutorial will examine the topic of :ref:`differentiation`
in greater detail.
Optimizations
=============
When compiling a Theano function, what you give to the
:func:`theano.function ` is actually a graph
(starting from the output variables you can traverse the graph up to
the input variables). While this graph structure shows how to compute
the output from the input, it also offers the possibility to improve the
way this computation is carried out. The way optimizations work in
Theano is by identifying and replacing certain patterns in the graph
with other specialized patterns that produce the same results but are either
faster or more stable. Optimizations can also detect
identical subgraphs and ensure that the same values are not computed
twice or reformulate parts of the graph to a GPU specific version.
For example, one (simple) optimization that Theano uses is to replace
the pattern :math:`\frac{xy}{y}` by *x.*
Further information regarding the optimization
:ref:`process` and the specific :ref:`optimizations` that are applicable
is respectively available in the library and on the entrance page of the documentation.
**Example**
Symbolic programming involves a change of paradigm: it will become clearer
as we apply it. Consider the following example of optimization:
>>> import theano
>>> a = theano.tensor.vector("a") # declare symbolic variable
>>> b = a + a ** 10 # build symbolic expression
>>> f = theano.function([a], b) # compile function
>>> print(f([0, 1, 2])) # prints `array([0,2,1026])`
[ 0. 2. 1026.]
>>> theano.printing.pydotprint(b, outfile="./pics/symbolic_graph_unopt.png", var_with_name_simple=True) # doctest: +SKIP
The output file is available at ./pics/symbolic_graph_unopt.png
>>> theano.printing.pydotprint(f, outfile="./pics/symbolic_graph_opt.png", var_with_name_simple=True) # doctest: +SKIP
The output file is available at ./pics/symbolic_graph_opt.png
We used :func:`theano.printing.pydotprint` to visualize the optimized graph
(right), which is much more compact than the unoptimized graph (left).
.. |g1| image:: ./pics/symbolic_graph_unopt.png
:width: 500 px
.. |g2| image:: ./pics/symbolic_graph_opt.png
:width: 500 px
================================ ====================== ================================
Unoptimized graph Optimized graph
================================ ====================== ================================
|g1| |g2|
================================ ====================== ================================
Theano-1.0.1/doc/extending/index.txt 0000777 0000000 0000000 00000003176 13212236347 017310 0 ustar root root 0000000 0000000
.. _extending:
================
Extending Theano
================
This advanced tutorial is for users who want to extend Theano with new Types,
new Operations (Ops), and new graph optimizations. This first page of the
tutorial mainly focuses on the Python implementation of an Op and then
proposes an overview of the most important methods that define an op.
The second page of the tutorial (:ref:`extending_theano_c`) provides then
information on the C implementation of an Op. The rest of the tutorial
goes more in depth on advanced topics related to Ops, such as how to write
efficient code for an Op and how to write an optimization to speed up the
execution of an Op.
Along the way, this tutorial also introduces many aspects of how Theano works,
so it is also good for you if you are interested in getting more under the hood
with Theano itself.
.. note::
Before tackling this more advanced presentation, it is highly recommended
to read the introductory :ref:`Tutorial`, especially the sections
that introduce the Theano Graphs, as providing a novel Theano op requires a
basic understanting of the Theano Graphs.
See also the :ref:`dev_start_guide` for information regarding the
versioning framework, namely about *git* and *GitHub*, regarding the
development workflow and how to make a quality contribution.
.. toctree::
extending_theano
extending_theano_c
fibby
pipeline
theano_vs_c
graphstructures
type
op
inplace
other_ops
ctype
cop
using_params
extending_theano_gpu
optimization
tips
unittest
scan
extending_faq
Theano-1.0.1/doc/extending/inplace.txt 0000777 0000000 0000000 00000021361 13212236347 017610 0 ustar root root 0000000 0000000
.. _views_and_inplace:
============================
Views and inplace operations
============================
Theano allows the definition of Ops which return a :term:`view` on one
of their inputs or operate :term:`inplace` on one or several
inputs. This allows more efficient operations on numpy's ``ndarray``
data type than would be possible otherwise.
However, in order to work correctly, these Ops need to
implement an additional interface.
Theano recognizes views and inplace operations specially. It ensures
that they are used in a consistent manner and it ensures that
operations will be carried in a compatible order.
An unfortunate fact is that it is impossible to return a view on an
input with the ``double`` type or to operate inplace on it (Python
floats are immutable). Therefore, we can't make examples of these
concepts out of what we've just built. Nonetheless, we will present
the concepts:
.. _views:
Views
=====
A "view" on an object ``x`` is an object ``y`` which shares memory
with ``x`` in some way. In other words, changing ``x`` might also
change ``y`` and vice versa. For example, imagine a ``vector`` structure
which contains two fields: an integer length and a pointer to a memory
buffer. Suppose we have:
::
x = vector {length: 256,
address: 0xDEADBEEF}
y = vector {length: 224,
address: 0xDEADBEEF + 0x10}
z = vector {length: 256,
address: 0xCAFEBABE}
So ``x`` uses the memory range ``0xDEADBEEF - 0xDEADBFEF``, ``y`` the
range ``0xDEADBEFF - 0xDEADBFDF`` and z the range ``0xCAFEBABE -
0xCAFEBBBE``. Since the ranges for ``x`` and ``y`` overlap, ``y`` is
considered to be a view of ``x`` and vice versa.
Suppose you had an Op which took ``x`` as input and returned
``y``. You would need to tell Theano that ``y`` is a view of ``x``. For this
purpose, you would set the ``view_map`` field as follows:
.. testsetup::
from theano import Op
myop = Op()
.. testcode::
myop.view_map = {0: [0]}
What this means is that the first output (position 0) is a view of the
first input (position 0). Even though the interface allows a list of
inputs that are viewed by a given output, this feature is currently
unsupported. Here are more examples:
.. testcode::
myop.view_map = {0: [0]} # first output is a view of first input
myop.view_map = {0: [1]} # first output is a view of second input
myop.view_map = {1: [0]} # second output is a view of first input
myop.view_map = {0: [0], # first output is a view of first input
1: [1]} # *AND* second output is a view of second input
myop.view_map = {0: [0], # first output is a view of first input
1: [0]} # *AND* second output is *ALSO* a view of first input
myop.view_map = {0: [0, 1]} # THIS IS NOT SUPPORTED YET! Only put a single input number in the list!
.. _inplace:
Inplace operations
==================
An inplace operation is one that modifies one or more of its
inputs. For example, the expression ``x += y`` where ``x`` and ``y``
are ``numpy.ndarray`` instances would normally represent an inplace
operation on ``x``.
.. note::
Inplace operations in Theano still work in a functional setting:
they need to return the modified input. Symbolically, Theano
requires one Variable standing for the input *before* being modified
and *another* Variable representing the input *after* being
modified. Therefore, code using inplace operations would look like
this:
.. testcode::
from theano.tensor import dscalars, log
from theano.tensor.inplace import add_inplace
x, y = dscalars('x', 'y')
r1 = log(x)
# r2 is x AFTER the add_inplace - x still represents the value before adding y
r2 = add_inplace(x, y)
# r3 is log(x) using the x from BEFORE the add_inplace
# r3 is the SAME as r1, even if we wrote this line after the add_inplace line
# Theano is actually going to compute r3 BEFORE r2
r3 = log(x)
# this is log(x) using the x from AFTER the add_inplace (so it's like log(x + y))
r4 = log(r2)
Needless to say, this goes for user-defined inplace operations as
well: the modified input must figure in the list of outputs you
give to ``Apply`` in the definition of ``make_node``.
Also, for technical reasons but also because they are slightly
confusing to use as evidenced by the previous code, Theano does not
allow the end user to use inplace operations by default. However,
it does allow *optimizations* to substitute them in in a later
phase. Therefore, typically, if you define an inplace operation,
you will define a pure equivalent and an optimization which
subsitutes one for the other. Theano will automatically verify if
it is possible to do so and will refuse the substitution if it
introduces inconsistencies.
Take the previous definitions of ``x``, ``y`` and ``z`` and suppose an Op which
adds one to every byte of its input. If we give ``x`` as an input to
that Op, it can either allocate a new buffer of the same size as ``x``
(that could be ``z``) and set that new buffer's bytes to the variable of
the addition. That would be a normal, :term:`pure` Op. Alternatively,
it could add one to each byte *in* the buffer ``x``, therefore
changing it. That would be an inplace Op.
Theano needs to be notified of this fact. The syntax is similar to
that of ``view_map``:
.. testcode::
myop.destroy_map = {0: [0]}
What this means is that the first output (position 0) operates inplace on the
first input (position 0).
.. testcode::
myop.destroy_map = {0: [0]} # first output operates inplace on first input
myop.destroy_map = {0: [1]} # first output operates inplace on second input
myop.destroy_map = {1: [0]} # second output operates inplace on first input
myop.destroy_map = {0: [0], # first output operates inplace on first input
1: [1]} # *AND* second output operates inplace on second input
myop.destroy_map = {0: [0], # first output operates inplace on first input
1: [0]} # *AND* second output *ALSO* operates inplace on first input
myop.destroy_map = {0: [0, 1]} # first output operates inplace on both the first and second input
# unlike for views, the previous line is legal and supported
Destructive Operations
======================
While some operations will operate inplace on their inputs, some might
simply destroy or corrupt them. For example, an Op could do temporary
calculations right in its inputs. If that is the case, Theano also
needs to be notified. The way to notify Theano is to assume that some
output operated inplace on whatever inputs are changed or corrupted by
the Op (even if the output does not technically reuse any of the
input(s)'s memory). From there, go to the previous section.
.. warning::
Failure to correctly mark down views and inplace operations using
``view_map`` and ``destroy_map`` can lead to nasty bugs. In the
absence of this information, Theano might assume that it is safe to
execute an inplace operation on some inputs *before* doing other
calculations on the *previous* values of the inputs. For example,
in the code: ``y = log(x); x2 = add_inplace(x, z)`` it is
imperative to do the logarithm before the addition (because after
the addition, the original x that we wanted to take the logarithm
of is gone). If Theano does not know that ``add_inplace`` changes
the value of ``x`` it might invert the order and that will
certainly lead to erroneous computations.
You can often identify an incorrect ``view_map`` or ``destroy_map``
by using :ref:`DebugMode`. *Be sure to use DebugMode when developing
a new Op that uses ``view_map`` and/or ``destroy_map``.*
Inplace optimization and DebugMode
==================================
It is recommended that during the graph construction, all Ops are not inplace.
Then an optimization replaces them with inplace ones. Currently DebugMode checks
all optimizations that were tried even if they got rejected. One reason an inplace
optimization can get rejected is when there is another Op that is already being applied
inplace on the same input. Another reason to reject an inplace optimization is
if it would introduce a cycle into the graph.
The problem with DebugMode is that it will trigger a useless error when
checking a rejected inplace optimization, since it will lead to wrong results.
In order to be able to use DebugMode in more situations, your inplace
optimization can pre-check whether it will get rejected by using the
``theano.gof.destroyhandler.fast_inplace_check()`` function, that will tell
which Ops can be performed inplace. You may then skip the optimization if it is
incompatible with this check. Note however that this check does not cover all
cases where an optimization may be rejected (it will not detect cycles).
Theano-1.0.1/doc/extending/op.txt 0000777 0000000 0000000 00000076377 13212236347 016634 0 ustar root root 0000000 0000000
===============================
Making arithmetic Ops on double
===============================
.. testsetup:: *
from theano import gof
class Double(gof.Type):
def filter(self, x, strict=False, allow_downcast=None):
if strict:
if isinstance(x, float):
return x
else:
raise TypeError('Expected a float!')
elif allow_downcast:
return float(x)
else: # Covers both the False and None cases.
x_float = float(x)
if x_float == x:
return x_float
else:
raise TypeError('The double type cannot accurately represent '
'value %s (of type %s): you must explicitly '
'allow downcasting if you want to do this.'
% (x, type(x)))
def values_eq_approx(self, x, y, tolerance=1e-4):
return abs(x - y) / (abs(x) + abs(y)) < tolerance
def __str__(self):
return "double"
double = Double()
Now that we have a ``double`` type, we have yet to use it to perform
computations. We'll start by defining multiplication.
.. _op_contract:
Op's contract
=============
An Op is any object which inherits from :class:`gof.Op`. It has to
define the following methods.
.. function:: make_node(*inputs)
This method is responsible for creating output Variables of a
suitable symbolic Type to serve as the outputs of this Op's
application. The Variables found in ``*inputs`` must be operated on
using Theano's symbolic language to compute the symbolic output
Variables. This method should put these outputs into an Apply
instance, and return the Apply instance.
This method creates an Apply node representing the application of
the Op on the inputs provided. If the Op cannot be applied to these
inputs, it must raise an appropriate exception.
The inputs of the Apply instance returned by this call must be
ordered correctly: a subsequent ``self.make_node(*apply.inputs)``
must produce something equivalent to the first ``apply``.
.. function:: perform(node, inputs, output_storage)
This method computes the function associated to this Op. ``node`` is
an Apply node created by the Op's ``make_node`` method. ``inputs``
is a list of references to data to operate on using non-symbolic
statements, (i.e., statements in Python, Numpy). ``output_storage``
is a list of storage cells where the variables of the computation
must be put.
More specifically:
- ``node``: This is a reference to an Apply node which was previously
obtained via the ``Op``'s ``make_node`` method. It is typically not
used in simple Ops, but it contains symbolic information that
could be required for complex Ops.
- ``inputs``: This is a list of data from which the values stored in ``output_storage``
are to be computed using non-symbolic language.
- ``output_storage``: This is a list of storage cells where the output is to be stored.
A storage cell is a one-element list. It is forbidden to change
the length of the list(s) contained in ``output_storage``.
There is one storage cell for each output of the Op.
The data put in ``output_storage`` must match the type of the
symbolic output. This is a situation where the ``node`` argument
can come in handy.
A function Mode may allow ``output_storage`` elements to persist
between evaluations, or it may reset ``output_storage`` cells to
hold a value of ``None``. It can also pre-allocate some memory
for the Op to use. This feature can allow ``perform`` to reuse
memory between calls, for example. If there is something
preallocated in the ``output_storage``, it will be of the good
dtype, but can have the wrong shape and have any stride pattern.
This method must be determined by the inputs. That is to say, if
it is evaluated once on inputs A and returned B, then if ever
inputs C, equal to A, are presented again, then outputs equal to
B must be returned again.
You must be careful about aliasing outputs to inputs, and making
modifications to any of the inputs. See :ref:`Views and inplace
operations ` before writing a ``perform``
implementation that does either of these things.
Instead (or in addition to) ``perform()`` You can also provide a
:ref:`C implementation ` of For more details, refer to the
documentation for :ref:`op`.
.. function:: __eq__(other)
``other`` is also an Op.
Returning ``True`` here is a promise to the optimization system
that the other Op will produce exactly the same graph effects
(from perform) as this one, given identical inputs. This means it
will produce the same output values, it will destroy the same
inputs (same destroy_map), and will alias outputs to the same
inputs (same view_map). For more details, see
:ref:`views_and_inplace`.
.. note::
If you set `__props__`, this will be automatically generated.
.. function:: __hash__()
If two Op instances compare equal, then they **must** return the
same hash value.
Equally important, this hash value must not change during the
lifetime of self. Op instances should be immutable in this
sense.
.. note::
If you set `__props__`, this will be automatically generated.
.. op_optional:
Optional methods or attributes
==============================
.. attribute:: __props__
*Default:* Undefined
Must be a tuple. Lists the name of the attributes which influence
the computation performed. This will also enable the automatic
generation of appropriate __eq__, __hash__ and __str__ methods.
Should be set to `()` if you have no attributes that are relevant to
the computation to generate the methods.
.. versionadded:: 0.7
.. attribute:: default_output
*Default:* None
If this member variable is an integer, then the default
implementation of ``__call__`` will return
``node.outputs[self.default_output]``, where ``node`` was returned
by ``make_node``. Otherwise, the entire list of outputs will be
returned, unless it is of length 1, where the single element will be
returned by itself.
.. function:: make_thunk(node, storage_map, compute_map, no_recycling, impl=None)
This function must return a thunk, that is a zero-arguments
function that encapsulates the computation to be performed by this
op on the arguments of the node.
:param node: Apply instance
The node for which a thunk is requested.
:param storage_map: dict of lists
This maps variables to a one-element lists holding the variable's
current value. The one-element list acts as pointer to the value
and allows sharing that "pointer" with other nodes and instances.
:param compute_map: dict of lists
This maps variables to one-element lists holding booleans. If
the value is 0 then the variable has not been computed and the
value should not be considered valid. If the value is 1 the
variable has been computed and the value is valid. If the value
is 2 the variable has been garbage-collected and is no longer
valid, but shouldn't be required anymore for this call.
:param no_recycling: WRITEME
WRITEME
:param impl: None, 'c' or 'py'
Which implementation to use.
The returned function must ensure that is sets the computed
variables as computed in the `compute_map`.
Defining this function removes the requirement for :meth:`perform`
or C code, as you will define the thunk for the computation
yourself.
.. function:: __call__(*inputs, **kwargs)
By default this is a convenience function which calls
:meth:`make_node` with the supplied arguments and returns the
result indexed by `default_output`. This can be overridden by
subclasses to do anything else, but must return either a theano
Variable or a list of Variables.
If you feel the need to override `__call__` to change the graph
based on the arguments, you should instead create a function that
will use your Op and build the graphs that you want and call that
instead of the Op instance directly.
.. function:: infer_shape(node, shapes)
This function is needed for shape optimization. ``shapes`` is a
list with one tuple for each input of the Apply node (which corresponds
to the inputs of the op). Each tuple contains as many elements as the
number of dimensions of the corresponding input. The value of each element
is the shape (number of items) along the corresponding dimension of that
specific input.
While this might sound complicated, it is nothing more than the shape
of each input as symbolic variables (one per dimension).
The function should return a list with one tuple for each output.
Each tuple should contain the corresponding output's computed shape.
Implementing this method will allow Theano to compute the output's
shape without computing the output itself, potentially sparing you
a costly recomputation.
.. function:: flops(inputs, outputs)
It is only used to have more information printed by the memory
profiler. It makes it print the mega flops and giga flops per
second for each apply node. It takes as inputs two lists: one for the
inputs and one for the outputs. They contain tuples that are the
shapes of the corresponding inputs/outputs.
.. function:: __str__()
This allows you to specify a more informative string representation of your
Op. If an Op has parameters, it is highly recommended to have the
``__str__`` method include the name of the op and the Op's parameters'
values.
.. note::
If you set `__props__`, this will be automatically generated.
You can still overide it for custom output.
.. function:: do_constant_folding(node)
*Default:* Return True
By default when optimizations are enabled, we remove during
function compilation Apply nodes whose inputs are all constants.
We replace the Apply node with a Theano constant variable.
This way, the Apply node is not executed at each function
call. If you want to force the execution of an op during the
function call, make do_constant_folding return False.
As done in the Alloc op, you can return False only in some cases by
analyzing the graph from the node parameter.
.. function:: debug_perform(node, inputs, output_storage)
Undefined by default.
If you define this function then it will be used instead of C code
or perform() to do the computation while debugging (currently
DebugMode, but others may also use it in the future). It has the
same signature and contract as :func:`perform`.
This enables ops that cause trouble with DebugMode with their
normal behaviour to adopt a different one when run under that
mode. If your op doesn't have any problems, don't implement this.
If you want your op to work with gradient.grad() you also need to
implement the functions described below.
Gradient
========
These are the function required to work with gradient.grad().
.. function:: grad(inputs, output_gradients)
If the Op being defined is differentiable, its gradient may be
specified symbolically in this method. Both ``inputs`` and
``output_gradients`` are lists of symbolic Theano Variables and
those must be operated on using Theano's symbolic language. The grad
method must return a list containing one Variable for each
input. Each returned Variable represents the gradient with respect
to that input computed based on the symbolic gradients with respect
to each output.
If the output is not differentiable with respect to an input then
this method should be defined to return a variable of type NullType
for that input. Likewise, if you have not implemented the grad
computation for some input, you may return a variable of type
NullType for that input. theano.gradient contains convenience
methods that can construct the variable for you:
:func:`theano.gradient.grad_undefined` and
:func:`theano.gradient.grad_not_implemented`, respectively.
If an element of output_gradient is of type
theano.gradient.DisconnectedType, it means that the cost is not a
function of this output. If any of the op's inputs participate in
the computation of only disconnected outputs, then Op.grad should
return DisconnectedType variables for those inputs.
If the grad method is not defined, then Theano assumes it has been
forgotten. Symbolic differentiation will fail on a graph that
includes this Op.
It must be understood that the Op's grad method is not meant to
return the gradient of the Op's output. theano.tensor.grad computes
gradients; Op.grad is a helper function that computes terms that
appear in gradients.
If an Op has a single vector-valued output y and a single
vector-valued input x, then the grad method will be passed x and a
second vector z. Define J to be the Jacobian of y with respect to
x. The Op's grad method should return dot(J.T,z). When
theano.tensor.grad calls the grad method, it will set z to be the
gradient of the cost C with respect to y. If this op is the only op
that acts on x, then dot(J.T,z) is the gradient of C with respect to
x. If there are other ops that act on x, theano.tensor.grad will
have to add up the terms of x's gradient contributed by the other
op's grad method.
In practice, an op's input and output are rarely implemented as
single vectors. Even if an op's output consists of a list
containing a scalar, a sparse matrix, and a 4D tensor, you can think
of these objects as being formed by rearranging a vector. Likewise
for the input. In this view, the values computed by the grad method
still represent a Jacobian-vector product.
In practice, it is probably not a good idea to explicitly construct
the Jacobian, which might be very large and very sparse. However,
the returned value should be equal to the Jacobian-vector product.
So long as you implement this product correctly, you need not
understand what theano.tensor.grad is doing, but for the curious the
mathematical justification is as follows:
In essence, the grad method must simply implement through symbolic
Variables and operations the chain rule of differential
calculus. The chain rule is the mathematical procedure that allows
one to calculate the total derivative :math:`\frac{d C}{d x}` of the
final scalar symbolic Variable C with respect to a primitive
symbolic Variable x found in the list ``inputs``. The grad method
does this using ``output_gradients`` which provides the total
derivative :math:`\frac{d C}{d f}` of C with respect to a symbolic
Variable that is returned by the Op (this is provided in
``output_gradients``), as well as the knowledge of the total
derivative :math:`\frac{d f}{d x}` of the latter with respect to the
primitive Variable (this has to be computed).
In mathematics, the total derivative of a scalar variable (C) with
respect to a vector of scalar variables (x), i.e. the gradient, is
customarily represented as the row vector of the partial
derivatives, whereas the total derivative of a vector of scalar
variables (f) with respect to another (x), is customarily
represented by the matrix of the partial derivatives, i.e.the
jacobian matrix. In this convenient setting, the chain rule
instructs that the gradient of the final scalar variable C with
respect to the primitive scalar variables in x through those in f is
simply given by the matrix product: :math:`\frac{d C}{d x} = \frac{d
C}{d f} * \frac{d f}{d x}`.
Here, the chain rule must be implemented in a similar but slightly
more complex setting: Theano provides in the list
``output_gradients`` one gradient for each of the Variables returned
by the Op. Where f is one such particular Variable, the
corresponding gradient found in ``output_gradients`` and
representing :math:`\frac{d C}{d f}` is provided with a shape
similar to f and thus not necessarily as a row vector of scalars.
Furthermore, for each Variable x of the Op's list of input variables
``inputs``, the returned gradient representing :math:`\frac{d C}{d
x}` must have a shape similar to that of Variable x.
If the output list of the op is :math:`[f_1, ... f_n]`, then the
list ``output_gradients`` is :math:`[grad_{f_1}(C), grad_{f_2}(C),
... , grad_{f_n}(C)]`. If ``inputs`` consists of the list
:math:`[x_1, ..., x_m]`, then Op.grad should return the list
:math:`[grad_{x_1}(C), grad_{x_2}(C), ..., grad_{x_m}(C)]`, where
:math:`(grad_{y}(Z))_i = \frac{\partial Z}{\partial y_i}` (and
:math:`i` can stand for multiple dimensions).
In other words, :func:`grad` does not return :math:`\frac{d f_i}{d
x_j}`, but instead the appropriate dot product specified by the
chain rule: :math:`\frac{d C}{d x_j} = \frac{d C}{d f_i} \cdot
\frac{d f_i}{d x_j}`. Both the partial differentiation and the
multiplication have to be performed by :func:`grad`.
Theano currently imposes the following constraints on the values
returned by the grad method:
1) They must be Variable instances.
2) When they are types that have dtypes, they must never have an integer dtype.
The output gradients passed *to* Op.grad will also obey these constraints.
Integers are a tricky subject. Integers are the main reason for
having DisconnectedType, NullType or zero gradient. When you have an
integer as an argument to your grad method, recall the definition of
a derivative to help you decide what value to return:
:math:`\frac{d f}{d x} = \lim_{\epsilon \rightarrow 0} (f(x+\epsilon)-f(x))/\epsilon`.
Suppose your function f has an integer-valued output. For most
functions you're likely to implement in theano, this means your
gradient should be zero, because f(x+epsilon) = f(x) for almost all
x. (The only other option is that the gradient could be undefined,
if your function is discontinuous everywhere, like the rational
indicator function)
Suppose your function f has an integer-valued input. This is a
little trickier, because you need to think about what you mean
mathematically when you make a variable integer-valued in
theano. Most of the time in machine learning we mean "f is a
function of a real-valued x, but we are only going to pass in
integer-values of x". In this case, f(x+epsilon) exists, so the
gradient through f should be the same whether x is an integer or a
floating point variable. Sometimes what we mean is "f is a function
of an integer-valued x, and f is only defined where x is an
integer." Since f(x+epsilon) doesn't exist, the gradient is
undefined. Finally, many times in theano, integer valued inputs
don't actually affect the elements of the output, only its shape.
If your function f has both an integer-valued input and an
integer-valued output, then both rules have to be combined:
- If f is defined at (x+epsilon), then the input gradient is
defined. Since f(x+epsilon) would be equal to f(x) almost
everywhere, the gradient should be 0 (first rule).
- If f is only defined where x is an integer, then the gradient
is undefined, regardless of what the gradient with respect to the
output is.
Examples:
1) f(x,y) = dot product between x and y. x and y are integers.
Since the output is also an integer, f is a step function.
Its gradient is zero almost everywhere, so Op.grad should return
zeros in the shape of x and y.
2) f(x,y) = dot product between x and y. x is floating point and y is an integer.
In this case the output is floating point. It doesn't matter
that y is an integer. We consider f to still be defined at
f(x,y+epsilon). The gradient is exactly the same as if y were
floating point.
3) f(x,y) = argmax of x along axis y.
The gradient with respect to y is undefined, because f(x,y) is
not defined for floating point y. How could you take an argmax
along a fraActional axis? The gradient with respect to x is
0, because f(x+epsilon, y) = f(x) almost everywhere.
4) f(x,y) = a vector with y elements, each of which taking on the value x
The grad method should return DisconnectedType()() for y,
because the elements of f don't depend on y. Only the shape of
f depends on y. You probably also want to implement a
connection_pattern method to encode this.
5) f(x) = int(x) converts float x into an int. g(y) = float(y) converts an integer y into a float.
If the final cost C = 0.5 * g(y) = 0.5 g(f(x)), then the
gradient with respect to y will be 0.5, even if y is an
integer. However, the gradient with respect to x will be 0,
because the output of f is integer-valued.
.. function:: connection_pattern(node):
Sometimes needed for proper operation of gradient.grad().
Returns a list of list of bools.
Op.connection_pattern[input_idx][output_idx] is true if the
elements of inputs[input_idx] have an effect on the elements of
outputs[output_idx].
The ``node`` parameter is needed to determine the number of
inputs. Some ops such as Subtensor take a variable number of
inputs.
If no connection_pattern is specified, gradient.grad will
assume that all inputs have some elements connected to some
elements of all outputs.
This method conveys two pieces of information that are otherwise
not part of the theano graph:
1) Which of the op's inputs are truly ancestors of each of the
op's outputs. Suppose an op has two inputs, x and y, and
outputs f(x) and g(y). y is not really an ancestor of f, but
it appears to be so in the theano graph.
2) Whether the actual elements of each input/output are relevant
to a computation.
For example, the shape op does not read its input's elements,
only its shape metadata. d shape(x) / dx should thus raise
a disconnected input exception (if these exceptions are
enabled).
As another example, the elements of the Alloc op's outputs
are not affected by the shape arguments to the Alloc op.
Failing to implement this function for an op that needs it can
result in two types of incorrect behavior:
1) gradient.grad erroneously raising a TypeError reporting that
a gradient is undefined.
2) gradient.grad failing to raise a ValueError reporting that
an input is disconnected.
Even if connection_pattern is not implemented correctly, if
gradient.grad returns an expression, that expression will be
numerically correct.
.. function:: R_op(inputs, eval_points)
Optional, to work with gradient.R_op().
This function implements the application of the R-operator on the
function represented by your op. Let assume that function is :math:`f`,
with input :math:`x`, applying the R-operator means computing the
Jacobian of :math:`f` and right-multiplying it by :math:`v`, the evaluation
point, namely: :math:`\frac{\partial f}{\partial x} v`.
``inputs`` are the symbolic variables corresponding to the value of
the input where you want to evaluate the jacobian, and ``eval_points``
are the symbolic variables corresponding to the value you want to
right multiply the jacobian with.
Same conventions as for the grad method hold. If your op is not
differentiable, you can return None. Note that in contrast to
the method :func:`grad`, for :func:`R_op` you need to return the
same number of outputs as there are ouputs of the op. You can think
of it in the following terms. You have all your inputs concatenated
into a single vector :math:`x`. You do the same with the evaluation
points (which are as many as inputs and of the shame shape) and obtain
another vector :math:`v`. For each output, you reshape it into a vector,
compute the jacobian of that vector with respect to :math:`x` and
multiply it by :math:`v`. As a last step you reshape each of these
vectors you obtained for each outputs (that have the same shape as
the outputs) back to their corresponding shapes and return them as the
output of the :func:`R_op` method.
:ref:`List of op with r op support `.
Defining an Op: ``mul``
=======================
We'll define multiplication as a *binary* operation, even though a
multiplication Op could take an arbitrary number of arguments.
First, we'll instantiate a ``mul`` Op:
.. testcode:: mul
from theano import gof
mul = gof.Op()
**make_node**
This function must take as many arguments as the operation we are
defining is supposed to take as inputs---in this example that would be
two. This function ensures that both inputs have the ``double`` type.
Since multiplying two doubles yields a double, this function makes an
Apply node with an output Variable of type ``double``.
.. testcode:: mul
def make_node(x, y):
if x.type != double or y.type != double:
raise TypeError('mul only works on doubles')
return gof.Apply(mul, [x, y], [double()])
mul.make_node = make_node
The first two lines make sure that both inputs are Variables of the
``double`` type that we created in the previous section. We would not
want to multiply two arbitrary types, it would not make much sense
(and we'd be screwed when we implement this in C!)
The last line is the meat of the definition. There we create an Apply
node representing the application of Op ``mul`` to inputs ``x`` and
``y``, giving a Variable instance of type ``double`` as the output.
.. note::
Theano relies on the fact that if you call the ``make_node`` method
of Apply's first argument on the inputs passed as the Apply's
second argument, the call will not fail and the returned Apply
instance will be equivalent. This is how graphs are copied.
**perform**
This code actually computes the function.
In our example, the data in ``inputs`` will be instances of Python's
built-in type ``float`` because this is the type that ``double.filter()``
will always return, per our own definition. ``output_storage`` will
contain a single storage cell for the multiplication's variable.
.. testcode:: mul
def perform(node, inputs, output_storage):
x, y = inputs[0], inputs[1]
z = output_storage[0]
z[0] = x * y
mul.perform = perform
Here, ``z`` is a list of one element. By default, ``z == [None]``.
.. note::
It is possible that ``z`` does not contain ``None``. If it contains
anything else, Theano guarantees that whatever it contains is what
``perform`` put there the last time it was called with this
particular storage. Furthermore, Theano gives you permission to do
whatever you want with ``z``'s contents, chiefly reusing it or the
memory allocated for it. More information can be found in the
:ref:`op` documentation.
.. warning::
We gave ``z`` the Theano type ``double`` in ``make_node``, which means
that a Python ``float`` must be put there. You should not put, say, an
``int`` in ``z[0]`` because Theano assumes Ops handle typing properly.
Trying out our new Op
=====================
In the following code, we use our new Op:
.. doctest:: mul
>>> import theano
>>> x, y = double('x'), double('y')
>>> z = mul(x, y)
>>> f = theano.function([x, y], z)
>>> f(5, 6)
30.0
>>> f(5.6, 6.7)
37.519999999999996
Note that there is an implicit call to
``double.filter()`` on each argument, so if we give integers as inputs
they are magically cast to the right type. Now, what if we try this?
.. doctest:: mul
>>> x = double('x')
>>> z = mul(x, 2)
Traceback (most recent call last):
File "", line 1, in
File "/u/breuleuo/hg/theano/theano/gof/op.py", line 207, in __call__
File "", line 2, in make_node
AttributeError: 'int' object has no attribute 'type'
Automatic Constant Wrapping
---------------------------
Well, OK. We'd like our Op to be a bit more flexible. This can be done
by modifying ``make_node`` to accept Python ``int`` or ``float`` as
``x`` and/or ``y``:
.. testcode:: mul
def make_node(x, y):
if isinstance(x, (int, float)):
x = gof.Constant(double, x)
if isinstance(y, (int, float)):
y = gof.Constant(double, y)
if x.type != double or y.type != double:
raise TypeError('mul only works on doubles')
return gof.Apply(mul, [x, y], [double()])
mul.make_node = make_node
Whenever we pass a Python int or float instead of a Variable as ``x`` or
``y``, ``make_node`` will convert it to :ref:`constant` for us. ``gof.Constant``
is a :ref:`variable` we statically know the value of.
.. doctest:: mul
>>> import numpy
>>> x = double('x')
>>> z = mul(x, 2)
>>> f = theano.function([x], z)
>>> f(10)
20.0
>>> numpy.allclose(f(3.4), 6.8)
True
Now the code works the way we want it to.
.. note::
Most Theano Ops follow this convention of up-casting literal
make_node arguments to Constants.
This makes typing expressions more natural. If you do
not want a constant somewhere in your graph, you have to pass a Variable
(like ``double('x')`` here).
Final version
=============
The above example is pedagogical. When you define other basic arithmetic
operations ``add``, ``sub`` and ``div``, code for ``make_node`` can be
shared between these Ops. Here is revised implementation of these four
arithmetic operators:
.. testcode::
from theano import gof
class BinaryDoubleOp(gof.Op):
__props__ = ("name", "fn")
def __init__(self, name, fn):
self.name = name
self.fn = fn
def make_node(self, x, y):
if isinstance(x, (int, float)):
x = gof.Constant(double, x)
if isinstance(y, (int, float)):
y = gof.Constant(double, y)
if x.type != double or y.type != double:
raise TypeError('%s only works on doubles' % self.name)
return gof.Apply(self, [x, y], [double()])
def perform(self, node, inp, out):
x, y = inp
z, = out
z[0] = self.fn(x, y)
def __str__(self):
return self.name
add = BinaryDoubleOp(name='add',
fn=lambda x, y: x + y)
sub = BinaryDoubleOp(name='sub',
fn=lambda x, y: x - y)
mul = BinaryDoubleOp(name='mul',
fn=lambda x, y: x * y)
div = BinaryDoubleOp(name='div',
fn=lambda x, y: x / y)
Instead of working directly on an instance of Op, we create a subclass of
Op that we can parametrize. All the operations we define are binary. They
all work on two inputs with type ``double``. They all return a single
Variable of type ``double``. Therefore, ``make_node`` does the same thing
for all these operations, except for the Op reference ``self`` passed
as first argument to Apply. We define ``perform`` using the function
``fn`` passed in the constructor.
This design is a flexible way to define basic operations without
duplicating code. The same way a Type subclass represents a set of
structurally similar types (see previous section), an Op subclass
represents a set of structurally similar operations: operations that
have the same input/output types, operations that only differ in one
small detail, etc. If you see common patterns in several Ops that you
want to define, it can be a good idea to abstract out what you can.
Remember that an Op is just an object which satisfies the contract
described above on this page and that you should use all the tools at
your disposal to create these objects as efficiently as possible.
**Exercise**: Make a generic DoubleOp, where the number of
arguments can also be given as a parameter.
Theano-1.0.1/doc/extending/optimization.txt 0000777 0000000 0000000 00000130354 13212236347 020726 0 ustar root root 0000000 0000000
.. _optimization:
==================
Graph optimization
==================
In this section we will define a couple optimizations on doubles.
.. todo::
This tutorial goes way too far under the hood, for someone who just wants
to add yet another pattern to the libraries in tensor.opt for example.
We need another tutorial that covers the decorator syntax, and explains how
to register your optimization right away. That's what you need to get
going.
Later, the rest is more useful for when that decorator syntax type thing
doesn't work. (There are optimizations that don't fit that model).
.. note::
The optimization tag `cxx_only` is used for optimizations that insert
Ops which have no Python implementation (so they only have C code).
Optimizations with this tag are skipped when there is no C++ compiler
available.
Global and local optimizations
==============================
First, let's lay out the way optimizations work in Theano. There are
two types of optimizations: *global* optimizations and *local*
optimizations. A global optimization takes a ``FunctionGraph`` object (a
FunctionGraph is a wrapper around a whole computation graph, you can see its
:class:`documentation ` for more details) and navigates through it
in a suitable way, replacing some Variables by others in the process. A
local optimization, on the other hand, is defined as a function on a
*single* :ref:`apply` node and must return either ``False`` (to mean that
nothing is to be done) or a list of new Variables that we would like to
replace the node's outputs with. A :ref:`navigator` is a special kind
of global optimization which navigates the computation graph in some
fashion (in topological order, reverse-topological order, random
order, etc.) and applies one or more local optimizations at each step.
Optimizations which are holistic, meaning that they must take into
account dependencies that might be all over the graph, should be
global. Optimizations that can be done with a narrow perspective are
better defined as local optimizations. The majority of optimizations
we want to define are local.
.. optimizer:
Global optimization
-------------------
A global optimization (or optimizer) is an object which defines the following
methods:
.. class:: Optimizer
.. method:: apply(fgraph)
This method takes a FunctionGraph object which contains the computation graph
and does modifications in line with what the optimization is meant
to do. This is one of the main methods of the optimizer.
.. method:: add_requirements(fgraph)
This method takes a FunctionGraph object and adds :ref:`features
` to it. These features are "plugins" that are needed
for the ``apply`` method to do its job properly.
.. method:: optimize(fgraph)
This is the interface function called by Theano.
*Default:* this is defined by Optimizer as ``add_requirement(fgraph);
apply(fgraph)``.
See the section about :class:`FunctionGraph` to understand how to define these
methods.
Local optimization
------------------
A local optimization is an object which defines the following methods:
.. class:: LocalOptimizer
.. method:: transform(node)
This method takes an :ref:`apply` node and returns either ``False`` to
signify that no changes are to be done or a list of Variables which
matches the length of the node's ``outputs`` list. When the
LocalOptimizer is applied by a Navigator, the outputs of the node
passed as argument to the LocalOptimizer will be replaced by the
list returned.
One simplification rule
=======================
For starters, let's define the following simplification:
.. math::
\frac{xy}{y} = x
We will implement it in three ways: using a global optimization, a
local optimization with a Navigator and then using the PatternSub
facility.
Global optimization
-------------------
Here is the code for a global optimization implementing the
simplification described above:
.. testcode::
import theano
from theano import gof
from theano.gof import toolbox
class Simplify(gof.Optimizer):
def add_requirements(self, fgraph):
fgraph.attach_feature(toolbox.ReplaceValidate())
def apply(self, fgraph):
for node in fgraph.toposort():
if node.op == true_div:
x, y = node.inputs
z = node.outputs[0]
if x.owner and x.owner.op == mul:
a, b = x.owner.inputs
if y == a:
fgraph.replace_validate(z, b)
elif y == b:
fgraph.replace_validate(z, a)
simplify = Simplify()
.. todo::
What is add_requirements? Why would we know to do this? Are there other
requirements we might want to know about?
Here's how it works: first, in ``add_requirements``, we add the
``ReplaceValidate`` :ref:`libdoc_gof_fgraphfeature` located in
:ref:`libdoc_gof_toolbox`. This feature adds the ``replace_validate``
method to ``fgraph``, which is an enhanced version of ``replace`` that
does additional checks to ensure that we are not messing up the
computation graph (note: if ``ReplaceValidate`` was already added by
another optimizer, ``extend`` will do nothing). In a nutshell,
``toolbox.ReplaceValidate`` grants access to ``fgraph.replace_validate``,
and ``fgraph.replace_validate`` allows us to replace a Variable with
another while respecting certain validation constraints. You can
browse the list of :ref:`libdoc_gof_fgraphfeaturelist` and see if some of
them might be useful to write optimizations with. For example, as an
exercise, try to rewrite Simplify using :class:`NodeFinder`. (Hint: you
want to use the method it publishes instead of the call to toposort!)
Then, in ``apply`` we do the actual job of simplification. We start by
iterating through the graph in topological order. For each node
encountered, we check if it's a ``div`` node. If not, we have nothing
to do here. If so, we put in ``x``, ``y`` and ``z`` the numerator,
denominator and quotient (output) of the division.
The simplification only occurs when the numerator is a multiplication,
so we check for that. If the numerator is a multiplication we put the
two operands in ``a`` and ``b``, so
we can now say that ``z == (a*b)/y``. If ``y==a`` then ``z==b`` and if
``y==b`` then ``z==a``. When either case happens then we can replace
``z`` by either ``a`` or ``b`` using ``fgraph.replace_validate`` - else we do
nothing. You might want to check the documentation about :ref:`variable`
and :ref:`apply` to get a better understanding of the
pointer-following game you need to get ahold of the nodes of interest
for the simplification (``x``, ``y``, ``z``, ``a``, ``b``, etc.).
Test time:
>>> from theano.scalar import float64, add, mul, true_div
>>> x = float64('x')
>>> y = float64('y')
>>> z = float64('z')
>>> a = add(z, mul(true_div(mul(y, x), y), true_div(z, x)))
>>> e = gof.FunctionGraph([x, y, z], [a])
>>> e
[add(z, mul(true_div(mul(y, x), y), true_div(z, x)))]
>>> simplify.optimize(e)
>>> e
[add(z, mul(x, true_div(z, x)))]
Cool! It seems to work. You can check what happens if you put many
instances of :math:`\frac{xy}{y}` in the graph. Note that it sometimes
won't work for reasons that have nothing to do with the quality of the
optimization you wrote. For example, consider the following:
>>> x = float64('x')
>>> y = float64('y')
>>> z = float64('z')
>>> a = true_div(mul(add(y, z), x), add(y, z))
>>> e = gof.FunctionGraph([x, y, z], [a])
>>> e
[true_div(mul(add(y, z), x), add(y, z))]
>>> simplify.optimize(e)
>>> e
[true_div(mul(add(y, z), x), add(y, z))]
Nothing happened here. The reason is: ``add(y, z) != add(y,
z)``. That is the case for efficiency reasons. To fix this problem we
first need to merge the parts of the graph that represent the same
computation, using the ``MergeOptimizer`` defined in
``theano.gof.opt``.
>>> from theano.gof.opt import MergeOptimizer
>>> MergeOptimizer().optimize(e) # doctest: +ELLIPSIS
(0, ..., None, None, {}, 1, 0)
>>> e
[true_div(mul(*1 -> add(y, z), x), *1)]
>>> simplify.optimize(e)
>>> e
[x]
Once the merge is done, both occurrences of ``add(y, z)`` are
collapsed into a single one and is used as an input in two
places. Note that ``add(x, y)`` and ``add(y, x)`` are still considered
to be different because Theano has no clue that ``add`` is
commutative. You may write your own global optimizer to identify
computations that are identical with full knowledge of the rules of
arithmetics that your Ops implement. Theano might provide facilities
for this somewhere in the future.
.. note::
:class:`FunctionGraph` is a Theano structure intended for the optimization
phase. It is used internally by function and is rarely
exposed to the end user. You can use it to test out optimizations,
etc. if you are comfortable with it, but it is recommended to use
the function frontend and to interface optimizations with
:class:`optdb` (we'll see how to do that soon).
Local optimization
------------------
The local version of the above code would be the following:
.. testcode::
class LocalSimplify(gof.LocalOptimizer):
def transform(self, node):
if node.op == true_div:
x, y = node.inputs
if x.owner and x.owner.op == mul:
a, b = x.owner.inputs
if y == a:
return [b]
elif y == b:
return [a]
return False
def tracks(self):
# This should be needed for the EquilibriumOptimizer
# but it isn't now
# TODO: do this and explain it
return [] # that's not what you should do
local_simplify = LocalSimplify()
.. todo::
Fix up previous example... it's bad and incomplete.
The definition of transform is the inner loop of the global optimizer,
where the node is given as argument. If no changes are to be made,
``False`` must be returned. Else, a list of what to replace the node's
outputs with must be returned. This list must have the same length as
node.ouputs. If one of node.outputs don't have clients(it is not used
in the graph), you can put None in the returned list to remove it.
In order to apply the local optimizer we must use it in conjunction
with a :ref:`navigator`. Basically, a :ref:`navigator` is a global
optimizer that loops through all nodes in the graph (or a well-defined
subset of them) and applies one or several local optimizers on them.
>>> x = float64('x')
>>> y = float64('y')
>>> z = float64('z')
>>> a = add(z, mul(true_div(mul(y, x), y), true_div(z, x)))
>>> e = gof.FunctionGraph([x, y, z], [a])
>>> e
[add(z, mul(true_div(mul(y, x), y), true_div(z, x)))]
>>> simplify = gof.TopoOptimizer(local_simplify)
>>> simplify.optimize(e)
(, 1, 5, 3, ..., ..., ...)
>>> e
[add(z, mul(x, true_div(z, x)))]
OpSub, OpRemove, PatternSub
+++++++++++++++++++++++++++
Theano defines some shortcuts to make LocalOptimizers:
.. function:: OpSub(op1, op2)
Replaces all uses of *op1* by *op2*. In other
words, the outputs of all :ref:`apply` involving *op1* by the outputs
of Apply nodes involving *op2*, where their inputs are the same.
.. function:: OpRemove(op)
Removes all uses of *op* in the following way:
if ``y = op(x)`` then ``y`` is replaced by ``x``. *op* must have as many
outputs as it has inputs. The first output becomes the first input,
the second output becomes the second input, and so on.
.. function:: PatternSub(pattern1, pattern2)
Replaces all occurrences of the first pattern by the second pattern.
See :class:`PatternSub`.
.. testsetup::
from theano.scalar import identity
.. testcode::
from theano.gof.opt import OpSub, OpRemove, PatternSub
# Replacing add by mul (this is not recommended for primarily
# mathematical reasons):
add_to_mul = OpSub(add, mul)
# Removing identity
remove_identity = OpRemove(identity)
# The "simplify" operation we've been defining in the past few
# sections. Note that we need two patterns to account for the
# permutations of the arguments to mul.
local_simplify_1 = PatternSub((true_div, (mul, 'x', 'y'), 'y'),
'x')
local_simplify_2 = PatternSub((true_div, (mul, 'x', 'y'), 'x'),
'y')
.. note::
``OpSub``, ``OpRemove`` and ``PatternSub`` produce local optimizers, which
means that everything we said previously about local optimizers
apply: they need to be wrapped in a Navigator, etc.
.. todo::
wtf is a navigator?
When an optimization can be naturally expressed using ``OpSub``, ``OpRemove``
or ``PatternSub``, it is highly recommended to use them.
WRITEME: more about using PatternSub (syntax for the patterns, how to
use constraints, etc. - there's some decent doc at
:class:`PatternSub` for those interested)
.. _optdb:
The optimization database (optdb)
=================================
Theano exports a symbol called ``optdb`` which acts as a sort of
ordered database of optimizations. When you make a new optimization,
you must insert it at the proper place in the database. Furthermore,
you can give each optimization in the database a set of tags that can
serve as a basis for filtering.
The point of optdb is that you might want to apply many optimizations
to a computation graph in many unique patterns. For example, you might
want to do optimization X, then optimization Y, then optimization
Z. And then maybe optimization Y is an EquilibriumOptimizer containing
LocalOptimizers A, B and C which are applied on every node of the
graph until they all fail to change it. If some optimizations act up,
we want an easy way to turn them off. Ditto if some optimizations are
very CPU-intensive and we don't want to take the time to apply them.
The optdb system allows us to tag each optimization with a unique name
as well as informative tags such as 'stable', 'buggy' or
'cpu_intensive', all this without compromising the structure of the
optimizations.
Definition of optdb
-------------------
optdb is an object which is an instance of
:class:`SequenceDB `,
itself a subclass of :class:`DB `.
There exist (for now) two types of DB, SequenceDB and EquilibriumDB.
When given an appropriate Query, DB objects build an Optimizer matching
the query.
A SequenceDB contains Optimizer or DB objects. Each of them has a
name, an arbitrary number of tags and an integer representing their
order in the sequence. When a Query is applied to a SequenceDB, all
Optimizers whose tags match the query are inserted in proper order in
a SequenceOptimizer, which is returned. If the SequenceDB contains DB
instances, the Query will be passed to them as well and the optimizers
they return will be put in their places.
An EquilibriumDB contains LocalOptimizer or DB objects. Each of them
has a name and an arbitrary number of tags. When a Query is applied to
an EquilibriumDB, all LocalOptimizers that match the query are
inserted into an EquilibriumOptimizer, which is returned. If the
SequenceDB contains DB instances, the Query will be passed to them as
well and the LocalOptimizers they return will be put in their places
(note that as of yet no DB can produce LocalOptimizer objects, so this
is a moot point).
Theano contains one principal DB object, :class:`optdb`, which
contains all of Theano's optimizers with proper tags. It is
recommended to insert new Optimizers in it. As mentioned previously,
optdb is a SequenceDB, so, at the top level, Theano applies a sequence
of global optimizations to the computation graphs.
Query
-----
A Query is built by the following call:
.. code-block:: python
theano.gof.Query(include, require=None, exclude=None, subquery=None)
.. class:: Query
.. attribute:: include
A set of tags (a tag being a string) such that every
optimization obtained through this Query must have **one** of the tags
listed. This field is required and basically acts as a starting point
for the search.
.. attribute:: require
A set of tags such that every optimization obtained
through this Query must have **all** of these tags.
.. attribute:: exclude
A set of tags such that every optimization obtained
through this Query must have **none** of these tags.
.. attribute:: subquery
optdb can contain sub-databases; subquery is a
dictionary mapping the name of a sub-database to a special Query.
If no subquery is given for a sub-database, the original Query will be
used again.
Furthermore, a Query object includes three methods, ``including``,
``requiring`` and ``excluding`` which each produce a new Query object
with include, require and exclude sets refined to contain the new [WRITEME]
Examples
--------
Here are a few examples of how to use a Query on optdb to produce an
Optimizer:
.. testcode::
from theano.gof import Query
from theano.compile import optdb
# This is how the optimizer for the fast_run mode is defined
fast_run = optdb.query(Query(include=['fast_run']))
# This is how the optimizer for the fast_compile mode is defined
fast_compile = optdb.query(Query(include=['fast_compile']))
# This is the same as fast_run but no optimizations will replace
# any operation by an inplace version. This assumes, of course,
# that all inplace operations are tagged as 'inplace' (as they
# should!)
fast_run_no_inplace = optdb.query(Query(include=['fast_run'],
exclude=['inplace']))
Registering an Optimizer
------------------------
Let's say we have a global optimizer called ``simplify``. We can add
it to ``optdb`` as follows:
.. testcode::
# optdb.register(name, optimizer, order, *tags)
optdb.register('simplify', simplify, 0.5, 'fast_run')
Once this is done, the FAST_RUN mode will automatically include your
optimization (since you gave it the 'fast_run' tag). Of course,
already-compiled functions will see no change. The 'order' parameter
(what it means and how to choose it) will be explained in
:ref:`optdb-structure` below.
Registering a LocalOptimizer
----------------------------
LocalOptimizers may be registered in two ways:
* Wrap them in a Navigator and insert them like a global optimizer
(see previous section).
* Put them in an EquilibriumDB.
Theano defines two EquilibriumDBs where you can put local
optimizations:
.. function:: canonicalize
This contains optimizations that aim to *simplify* the graph:
* Replace rare or esoterical operations with their equivalents using
elementary operations.
* Order operations in a canonical way (any sequence of
multiplications and divisions can be rewritten to contain at most
one division, for example; ``x*x`` can be rewritten ``x**2``; etc.)
* Fold constants (``Constant(2)*Constant(2)`` becomes ``Constant(4)``)
.. function:: specialize
This contains optimizations that aim to *specialize* the graph:
* Replace a combination of operations with a special operation that
does the same thing (but better).
For each group, all optimizations of the group that are selected by
the Query will be applied on the graph over and over again until none
of them is applicable, so keep that in mind when designing it: check
carefully that your optimization leads to a fixpoint (a point where it
cannot apply anymore) at which point it returns ``False`` to indicate its
job is done. Also be careful not to undo the work of another local
optimizer in the group, because then the graph will oscillate between
two or more states and nothing will get done.
.. _optdb-structure:
optdb structure
---------------
optdb contains the following Optimizers and sub-DBs, with the given
priorities and tags:
+-------+---------------------+------------------------------+
| Order | Name | Description |
+=======+=====================+==============================+
| 0 | merge1 | First merge operation |
+-------+---------------------+------------------------------+
| 1 | canonicalize | Simplify the graph |
+-------+---------------------+------------------------------+
| 2 | specialize | Add specialized operations |
+-------+---------------------+------------------------------+
| 49 | merge2 | Second merge operation |
+-------+---------------------+------------------------------+
| 49.5 | add_destroy_handler | Enable inplace optimizations |
+-------+---------------------+------------------------------+
| 100 | merge3 | Third merge operation |
+-------+---------------------+------------------------------+
The merge operations are meant to put together parts of the graph that
represent the same computation. Since optimizations can modify the
graph in such a way that two previously different-looking parts of the
graph become similar, we merge at the beginning, in the middle and at
the very end. Technically, we only really need to do it at the end,
but doing it in previous steps reduces the size of the graph and
therefore increases the efficiency of the process.
See previous section for more information about the canonicalize and
specialize steps.
The ``add_destroy_handler`` step is not really an optimization. It is
a marker. Basically:
.. warning::
Any optimization which inserts inplace operations in the
computation graph must appear **after** the ``add_destroy_handler``
"optimizer". In other words, the priority of any such optimization
must be **>= 50**. Failure to comply by this restriction can lead
to the creation of incorrect computation graphs.
The reason the destroy handler is not inserted at the beginning is
that it is costly to run. It is cheaper to run most optimizations
under the assumption there are no inplace operations.
.. _navigator:
Navigator
-------------------
WRITEME
.. _profiling_opt:
Profiling Theano function compilation
=====================================
You find that compiling a Theano function is taking too much time? You
can get profiling information about Theano optimization. The normal
:ref:`Theano profiler ` will provide you with very
high-level information. The indentation shows the included in/subset
relationship between sections. The top of its output look like this:
.. code-block:: none
Function profiling
==================
Message: PATH_TO_A_FILE:23
Time in 0 calls to Function.__call__: 0.000000e+00s
Total compile time: 1.131874e+01s
Number of Apply nodes: 50
Theano Optimizer time: 1.152431e+00s
Theano validate time: 2.790451e-02s
Theano Linker time (includes C, CUDA code generation/compiling): 7.893991e-02s
Import time 1.153541e-02s
Time in all call to theano.grad() 4.732513e-02s
Explanations:
* ``Total compile time: 1.131874e+01s`` gives the total time spent inside `theano.function`.
* ``Number of Apply nodes: 50`` means that after optimization, there are 50 apply node in the graph.
* ``Theano Optimizer time: 1.152431e+00s`` means that we spend 1.15s in the ``theano.function`` phase where we optimize (modify) the graph to make it faster / more stable numerically / work on GPU /...
* ``Theano validate time: 2.790451e-02s`` means that we spent 2.8e-2s in the *validate* subset of the optimization phase.
* ``Theano Linker time (includes C, CUDA code generation/compiling): 7.893991e-02s`` means that we spent 7.9e-2s in *linker* phase of ``theano.function``.
* ``Import time 1.153541e-02s`` is a subset of the linker time where we import the compiled module.
* ``Time in all call to theano.grad() 4.732513e-02s`` tells that we spent a total of 4.7e-2s in all calls to ``theano.grad``. This is outside of the calls to ``theano.function``.
The *linker* phase includes the generation of the C code, the time spent
by g++ to compile and the time needed by Theano to build the object we
return. The C code generation and compilation is cached, so the first
time you compile a function and the following ones could take different
amount of execution time.
Detailed profiling of Theano optimizer
--------------------------------------
You can get more detailed profiling information about the Theano
optimizer phase by setting to `True` the Theano flags
:attr:`config.profile_optimizer` (this require `config.profile` to be `True`
as well).
This will output something like this:
.. code-block:: none
Optimizer Profile
-----------------
SeqOptimizer OPT_FAST_RUN time 1.152s for 123/50 nodes before/after optimization
0.028s for fgraph.validate()
0.131s for callback
time - (name, class, index) - validate time
0.751816s - ('canonicalize', 'EquilibriumOptimizer', 4) - 0.004s
EquilibriumOptimizer canonicalize
time 0.751s for 14 passes
nb nodes (start, end, max) 108 81 117
time io_toposort 0.029s
time in local optimizers 0.687s
time in global optimizers 0.010s
0 - 0.050s 27 (0.000s in global opts, 0.002s io_toposort) - 108 nodes - ('local_dimshuffle_lift', 9) ('local_upcast_elemwise_constant_inputs', 5) ('local_shape_to_shape_i', 3) ('local_fill_sink', 3) ('local_fill_to_alloc', 2) ...
1 - 0.288s 26 (0.002s in global opts, 0.002s io_toposort) - 117 nodes - ('local_dimshuffle_lift', 8) ('local_fill_sink', 4) ('constant_folding', 4) ('local_useless_elemwise', 3) ('local_subtensor_make_vector', 3) ...
2 - 0.044s 13 (0.002s in global opts, 0.003s io_toposort) - 96 nodes - ('constant_folding', 4) ('local_dimshuffle_lift', 3) ('local_fill_sink', 3) ('local_useless_elemwise', 1) ('local_fill_to_alloc', 1) ...
3 - 0.045s 11 (0.000s in global opts, 0.002s io_toposort) - 91 nodes - ('constant_folding', 3) ('local_fill_to_alloc', 2) ('local_dimshuffle_lift', 2) ('local_mul_canonizer', 2) ('MergeOptimizer', 1) ...
4 - 0.035s 8 (0.002s in global opts, 0.002s io_toposort) - 93 nodes - ('local_fill_sink', 3) ('local_dimshuffle_lift', 2) ('local_fill_to_alloc', 1) ('MergeOptimizer', 1) ('constant_folding', 1)
5 - 0.035s 6 (0.000s in global opts, 0.002s io_toposort) - 88 nodes - ('local_fill_sink', 2) ('local_dimshuffle_lift', 2) ('local_fill_to_alloc', 1) ('local_mul_canonizer', 1)
6 - 0.038s 10 (0.001s in global opts, 0.002s io_toposort) - 95 nodes - ('local_fill_sink', 3) ('local_dimshuffle_lift', 3) ('constant_folding', 2) ('local_fill_to_alloc', 1) ('MergeOptimizer', 1)
7 - 0.032s 5 (0.001s in global opts, 0.002s io_toposort) - 91 nodes - ('local_fill_sink', 3) ('MergeOptimizer', 1) ('local_dimshuffle_lift', 1)
8 - 0.034s 5 (0.000s in global opts, 0.002s io_toposort) - 92 nodes - ('local_fill_sink', 3) ('MergeOptimizer', 1) ('local_greedy_distributor', 1)
9 - 0.031s 6 (0.001s in global opts, 0.002s io_toposort) - 90 nodes - ('local_fill_sink', 2) ('local_fill_to_alloc', 1) ('MergeOptimizer', 1) ('local_dimshuffle_lift', 1) ('local_greedy_distributor', 1)
10 - 0.032s 5 (0.000s in global opts, 0.002s io_toposort) - 89 nodes - ('local_dimshuffle_lift', 2) ('local_fill_to_alloc', 1) ('MergeOptimizer', 1) ('local_fill_sink', 1)
11 - 0.030s 5 (0.000s in global opts, 0.002s io_toposort) - 88 nodes - ('local_dimshuffle_lift', 2) ('local_fill_to_alloc', 1) ('MergeOptimizer', 1) ('constant_folding', 1)
12 - 0.026s 1 (0.000s in global opts, 0.003s io_toposort) - 81 nodes - ('MergeOptimizer', 1)
13 - 0.031s 0 (0.000s in global opts, 0.003s io_toposort) - 81 nodes -
times - times applied - nb node created - name:
0.263s - 15 - 0 - constant_folding
0.096s - 2 - 14 - local_greedy_distributor
0.066s - 4 - 19 - local_mul_canonizer
0.046s - 28 - 57 - local_fill_sink
0.042s - 35 - 78 - local_dimshuffle_lift
0.018s - 5 - 15 - local_upcast_elemwise_constant_inputs
0.010s - 11 - 4 - MergeOptimizer
0.009s - 4 - 0 - local_useless_elemwise
0.005s - 11 - 2 - local_fill_to_alloc
0.004s - 3 - 6 - local_neg_to_mul
0.002s - 1 - 3 - local_lift_transpose_through_dot
0.002s - 3 - 4 - local_shape_to_shape_i
0.002s - 2 - 4 - local_subtensor_lift
0.001s - 3 - 0 - local_subtensor_make_vector
0.001s - 1 - 1 - local_sum_all_to_none
0.131s - in 62 optimization that where not used (display only those with a runtime > 0)
0.050s - local_add_canonizer
0.018s - local_mul_zero
0.016s - local_one_minus_erf
0.010s - local_func_inv
0.006s - local_0_dot_x
0.005s - local_track_shape_i
0.004s - local_mul_switch_sink
0.004s - local_fill_cut
0.004s - local_one_minus_erf2
0.003s - local_remove_switch_const_cond
0.003s - local_cast_cast
0.002s - local_IncSubtensor_serialize
0.001s - local_sum_div_dimshuffle
0.001s - local_div_switch_sink
0.001s - local_dimshuffle_no_inplace_at_canonicalize
0.001s - local_cut_useless_reduce
0.001s - local_reduce_join
0.000s - local_sum_sum
0.000s - local_useless_alloc
0.000s - local_reshape_chain
0.000s - local_useless_subtensor
0.000s - local_reshape_lift
0.000s - local_flatten_lift
0.000s - local_useless_slice
0.000s - local_subtensor_of_alloc
0.000s - local_subtensor_of_dot
0.000s - local_subtensor_merge
0.101733s - ('elemwise_fusion', 'SeqOptimizer', 13) - 0.000s
SeqOptimizer elemwise_fusion time 0.102s for 78/50 nodes before/after optimization
0.000s for fgraph.validate()
0.004s for callback
0.095307s - ('composite_elemwise_fusion', 'FusionOptimizer', 1) - 0.000s
FusionOptimizer
nb_iter 3
nb_replacement 10
nb_inconsistency_replace 0
validate_time 0.000249624252319
callback_time 0.00316381454468
time_toposort 0.00375390052795
0.006412s - ('local_add_mul_fusion', 'FusionOptimizer', 0) - 0.000s
FusionOptimizer
nb_iter 2
nb_replacement 3
nb_inconsistency_replace 0
validate_time 6.43730163574e-05
callback_time 0.000783205032349
time_toposort 0.0035240650177
0.090089s - ('inplace_elemwise_optimizer', 'FromFunctionOptimizer', 30) - 0.019s
0.048993s - ('BlasOpt', 'SeqOptimizer', 8) - 0.000s
SeqOptimizer BlasOpt time 0.049s for 81/80 nodes before/after optimization
0.000s for fgraph.validate()
0.003s for callback
0.035997s - ('gemm_optimizer', 'GemmOptimizer', 1) - 0.000s
GemmOptimizer
nb_iter 2
nb_replacement 2
nb_replacement_didn_t_remove 0
nb_inconsistency_make 0
nb_inconsistency_replace 0
time_canonicalize 0.00720071792603
time_factor_can 9.05990600586e-06
time_factor_list 0.00128507614136
time_toposort 0.00311398506165
validate_time 4.60147857666e-05
callback_time 0.00174236297607
0.004569s - ('local_dot_to_dot22', 'TopoOptimizer', 0) - 0.000s
TopoOptimizer
nb_node (start, end, changed) (81, 81, 5)
init io_toposort 0.00139284133911
loop time 0.00312399864197
callback_time 0.00172805786133
0.002283s - ('local_dot22_to_dot22scalar', 'TopoOptimizer', 2) - 0.000s
TopoOptimizer
nb_node (start, end, changed) (80, 80, 0)
init io_toposort 0.00171804428101
loop time 0.000502109527588
callback_time 0.0
0.002257s - ('local_gemm_to_gemv', 'EquilibriumOptimizer', 3) - 0.000s
EquilibriumOptimizer local_gemm_to_gemv
time 0.002s for 1 passes
nb nodes (start, end, max) 80 80 80
time io_toposort 0.001s
time in local optimizers 0.000s
time in global optimizers 0.000s
0 - 0.002s 0 (0.000s in global opts, 0.001s io_toposort) - 80 nodes -
0.002227s - ('use_c_blas', 'TopoOptimizer', 4) - 0.000s
TopoOptimizer
nb_node (start, end, changed) (80, 80, 0)
init io_toposort 0.0014750957489
loop time 0.00068998336792
callback_time 0.0
0.001632s - ('use_scipy_ger', 'TopoOptimizer', 5) - 0.000s
TopoOptimizer
nb_node (start, end, changed) (80, 80, 0)
init io_toposort 0.00138401985168
loop time 0.000202178955078
callback_time 0.0
0.031740s - ('specialize', 'EquilibriumOptimizer', 9) - 0.000s
EquilibriumOptimizer specialize
time 0.031s for 2 passes
nb nodes (start, end, max) 80 78 80
time io_toposort 0.003s
time in local optimizers 0.022s
time in global optimizers 0.004s
0 - 0.017s 6 (0.002s in global opts, 0.001s io_toposort) - 80 nodes - ('constant_folding', 2) ('local_mul_to_sqr', 1) ('local_elemwise_alloc', 1) ('local_div_to_inv', 1) ('local_mul_specialize', 1)
1 - 0.014s 0 (0.002s in global opts, 0.001s io_toposort) - 78 nodes -
times - times applied - nb node created - name:
0.003s - 1 - 1 - local_mul_specialize
0.002s - 1 - 2 - local_elemwise_alloc
0.002s - 2 - 0 - constant_folding
0.001s - 1 - 1 - local_div_to_inv
0.001s - 1 - 1 - local_mul_to_sqr
0.016s - in 69 optimization that where not used (display only those with a runtime > 0)
0.004s - crossentropy_to_crossentropy_with_softmax_with_bias
0.002s - local_one_minus_erf
0.002s - Elemwise{sub,no_inplace}(z, Elemwise{mul,no_inplace}(alpha subject to at 0x7f475e4da050>, SparseDot(x, y))) -> Usmm{no_inplace}(Elemwise{neg,no_inplace}(alpha), x, y, z)
0.002s - local_add_specialize
0.001s - local_func_inv
0.001s - local_useless_elemwise
0.001s - local_abs_merge
0.001s - local_track_shape_i
0.000s - local_one_minus_erf2
0.000s - local_sum_mul_by_scalar
0.000s - local_elemwise_sub_zeros
0.000s - local_cast_cast
0.000s - local_alloc_unary
0.000s - Elemwise{log,no_inplace}(Softmax(x)) -> (x)
0.000s - local_sum_div_dimshuffle
0.000s - local_sum_alloc
0.000s - local_dimshuffle_lift
0.000s - local_reduce_broadcastable
0.000s - local_grad_log_erfc_neg
0.000s - local_advanced_indexing_crossentropy_onehot
0.000s - local_log_erfc
0.000s - local_log1p
0.000s - local_log_add
0.000s - local_useless_alloc
0.000s - local_neg_neg
0.000s - local_neg_div_neg
...
To understand this profile here is some explanation of how optimizations work:
* Optimizations are organized in an hierarchy. At the top level, there
is a ``SeqOptimizer`` (Sequence Optimizer). It contains other optimizers,
and applies them in the order they were specified. Those sub-optimizers can be
of other types, but are all *global* optimizers.
* Each Optimizer in the hierarchy will print some stats about
itself. The information that it prints depends of the type of the
optimizer.
* The SeqOptimizer will print some stats at the start:
.. code-block:: none
Optimizer Profile
-----------------
SeqOptimizer OPT_FAST_RUN time 1.152s for 123/50 nodes before/after optimization
0.028s for fgraph.validate()
0.131s for callback
time - (name, class, index) - validate time
Then it will print, with some additional indentation, each sub-optimizer's profile
information. These sub-profiles are ordered by the time they took to execute,
not by their execution order.
* ``OPT_FAST_RUN`` is the name of the optimizer
* 1.152s is the total time spent in that optimizer
* 123/50 means that before this optimization, there were 123 apply node in the function graph, and after only 50.
* 0.028s means it spent that time calls to ``fgraph.validate()``
* 0.131s means it spent that time for callbacks. This is a mechanism that can trigger other execution when there is a change to the FunctionGraph.
* ``time - (name, class, index) - validate time`` tells how the information for each sub-optimizer get printed.
* All other instances of ``SeqOptimizer`` are described like this. In particular, some sub-optimizer from OPT_FAST_RUN that are also ``SeqOptimizer``.
* The ``SeqOptimizer`` will print some stats at the start:
.. code-block:: none
0.751816s - ('canonicalize', 'EquilibriumOptimizer', 4) - 0.004s
EquilibriumOptimizer canonicalize
time 0.751s for 14 passes
nb nodes (start, end, max) 108 81 117
time io_toposort 0.029s
time in local optimizers 0.687s
time in global optimizers 0.010s
0 - 0.050s 27 (0.000s in global opts, 0.002s io_toposort) - 108 nodes - ('local_dimshuffle_lift', 9) ('local_upcast_elemwise_constant_inputs', 5) ('local_shape_to_shape_i', 3) ('local_fill_sink', 3) ('local_fill_to_alloc', 2) ...
1 - 0.288s 26 (0.002s in global opts, 0.002s io_toposort) - 117 nodes - ('local_dimshuffle_lift', 8) ('local_fill_sink', 4) ('constant_folding', 4) ('local_useless_elemwise', 3) ('local_subtensor_make_vector', 3) ...
2 - 0.044s 13 (0.002s in global opts, 0.003s io_toposort) - 96 nodes - ('constant_folding', 4) ('local_dimshuffle_lift', 3) ('local_fill_sink', 3) ('local_useless_elemwise', 1) ('local_fill_to_alloc', 1) ...
3 - 0.045s 11 (0.000s in global opts, 0.002s io_toposort) - 91 nodes - ('constant_folding', 3) ('local_fill_to_alloc', 2) ('local_dimshuffle_lift', 2) ('local_mul_canonizer', 2) ('MergeOptimizer', 1) ...
4 - 0.035s 8 (0.002s in global opts, 0.002s io_toposort) - 93 nodes - ('local_fill_sink', 3) ('local_dimshuffle_lift', 2) ('local_fill_to_alloc', 1) ('MergeOptimizer', 1) ('constant_folding', 1)
5 - 0.035s 6 (0.000s in global opts, 0.002s io_toposort) - 88 nodes - ('local_fill_sink', 2) ('local_dimshuffle_lift', 2) ('local_fill_to_alloc', 1) ('local_mul_canonizer', 1)
6 - 0.038s 10 (0.001s in global opts, 0.002s io_toposort) - 95 nodes - ('local_fill_sink', 3) ('local_dimshuffle_lift', 3) ('constant_folding', 2) ('local_fill_to_alloc', 1) ('MergeOptimizer', 1)
7 - 0.032s 5 (0.001s in global opts, 0.002s io_toposort) - 91 nodes - ('local_fill_sink', 3) ('MergeOptimizer', 1) ('local_dimshuffle_lift', 1)
8 - 0.034s 5 (0.000s in global opts, 0.002s io_toposort) - 92 nodes - ('local_fill_sink', 3) ('MergeOptimizer', 1) ('local_greedy_distributor', 1)
9 - 0.031s 6 (0.001s in global opts, 0.002s io_toposort) - 90 nodes - ('local_fill_sink', 2) ('local_fill_to_alloc', 1) ('MergeOptimizer', 1) ('local_dimshuffle_lift', 1) ('local_greedy_distributor', 1)
10 - 0.032s 5 (0.000s in global opts, 0.002s io_toposort) - 89 nodes - ('local_dimshuffle_lift', 2) ('local_fill_to_alloc', 1) ('MergeOptimizer', 1) ('local_fill_sink', 1)
11 - 0.030s 5 (0.000s in global opts, 0.002s io_toposort) - 88 nodes - ('local_dimshuffle_lift', 2) ('local_fill_to_alloc', 1) ('MergeOptimizer', 1) ('constant_folding', 1)
12 - 0.026s 1 (0.000s in global opts, 0.003s io_toposort) - 81 nodes - ('MergeOptimizer', 1)
13 - 0.031s 0 (0.000s in global opts, 0.003s io_toposort) - 81 nodes -
times - times applied - nb node created - name:
0.263s - 15 - 0 - constant_folding
0.096s - 2 - 14 - local_greedy_distributor
0.066s - 4 - 19 - local_mul_canonizer
0.046s - 28 - 57 - local_fill_sink
0.042s - 35 - 78 - local_dimshuffle_lift
0.018s - 5 - 15 - local_upcast_elemwise_constant_inputs
0.010s - 11 - 4 - MergeOptimizer
0.009s - 4 - 0 - local_useless_elemwise
0.005s - 11 - 2 - local_fill_to_alloc
0.004s - 3 - 6 - local_neg_to_mul
0.002s - 1 - 3 - local_lift_transpose_through_dot
0.002s - 3 - 4 - local_shape_to_shape_i
0.002s - 2 - 4 - local_subtensor_lift
0.001s - 3 - 0 - local_subtensor_make_vector
0.001s - 1 - 1 - local_sum_all_to_none
0.131s - in 62 optimization that where not used (display only those with a runtime > 0)
0.050s - local_add_canonizer
0.018s - local_mul_zero
0.016s - local_one_minus_erf
0.010s - local_func_inv
0.006s - local_0_dot_x
0.005s - local_track_shape_i
0.004s - local_mul_switch_sink
0.004s - local_fill_cut
0.004s - local_one_minus_erf2
0.003s - local_remove_switch_const_cond
0.003s - local_cast_cast
0.002s - local_IncSubtensor_serialize
0.001s - local_sum_div_dimshuffle
0.001s - local_div_switch_sink
0.001s - local_dimshuffle_no_inplace_at_canonicalize
0.001s - local_cut_useless_reduce
0.001s - local_reduce_join
0.000s - local_sum_sum
0.000s - local_useless_alloc
0.000s - local_reshape_chain
0.000s - local_useless_subtensor
0.000s - local_reshape_lift
0.000s - local_flatten_lift
0.000s - local_useless_slice
0.000s - local_subtensor_of_alloc
0.000s - local_subtensor_of_dot
0.000s - local_subtensor_merge
* ``0.751816s - ('canonicalize', 'EquilibriumOptimizer', 4) - 0.004s``
This line is from ``SeqOptimizer``, and indicates information related
to a sub-optimizer. It means that this sub-optimizer took
a total of .7s. Its name is ``'canonicalize'``. It is an
``EquilibriumOptimizer``. It was executed at index 4 by the
``SeqOptimizer``. It spent 0.004s in the *validate* phase.
* All other lines are from the profiler of the ``EquilibriumOptimizer``.
* An ``EquilibriumOptimizer`` does multiple passes on the Apply nodes from
the graph, trying to apply local and global optimizations.
Conceptually, it tries to execute all global optimizations,
and to apply all local optimizations on all
nodes in the graph. If no optimization got applied during a pass, it
stops. So it tries to find an equilibrium state where none of the
optimizations get applied. This is useful when we do not know a fixed order for
the execution of the optimization.
* ``time 0.751s for 14 passes`` means that it took .7s and did 14 passes over the graph.
* ``nb nodes (start, end, max) 108 81 117`` means that at the start,
the graph had 108 node, at the end, it had 81 and the maximum size
was 117.
* Then it prints some global timing information: it spent 0.029s in
``io_toposort``, all local optimizers took 0.687s together for all
passes, and global optimizers took a total of 0.010s.
* Then we print the timing for each pass, the optimization that
got applied, and the number of time they got applied. For example,
in pass 0, the ``local_dimshuffle_lift`` optimizer changed the graph 9
time.
* Then we print the time spent in each optimizer, the number of times
they changed the graph and the number of nodes they introduced in
the graph.
* Optimizations with that pattern `local_op_lift` means that a node
with that op will be replaced by another node, with the same op,
but will do computation closer to the inputs of the graph.
For instance, ``local_op(f(x))`` getting replaced by ``f(local_op(x))``.
* Optimization with that pattern `local_op_sink` is the opposite of
`lift`. For instance ``f(local_op(x))`` getting replaced by ``local_op(f(x))``.
* Local optimizers can replace any arbitrary node in the graph, not
only the node it received as input. For this, it must return a
dict. The keys being nodes to replace and the
values being the corresponding replacement.
This is useful to replace a client of the node received as
parameter.
Theano-1.0.1/doc/extending/other_ops.txt 0000777 0000000 0000000 00000025251 13212236347 020201 0 ustar root root 0000000 0000000 .. _other_ops:
==============================
Implementing some specific Ops
==============================
This page is a guide on the implementation of some specific types of Ops,
and points to some examples of such implementations.
For the random number generating Ops, it explains different possible
implementation strategies.
.. _scalar_ops:
Scalar/Elemwise/Reduction Ops
=============================
Implementing a Theano scalar Op allows that scalar operation to be reused
by our elemwise operations on tensors. If the scalar operation has C code, the
elemwise implementation will automatically have C code too. This
will enable the fusion of elemwise operations using your new scalar
operation. It can also reuse the GPU elemwise code. It is similar for
reduction operations.
For examples of how to add new scalar operations, you can have a look at
those 2 pull requests, that add `GammaLn and Psi
`_
and `impl() methods related to SciPy
`_.
.. _scipy_ops:
SciPy Ops
=========
We can wrap SciPy functions in Theano. But SciPy is an optional dependency.
Here is some code that allows the Op to be optional:
.. code-block:: python
try:
import scipy.linalg
imported_scipy = True
except ImportError:
# some ops (e.g. Cholesky, Solve, A_Xinv_b) won't work
imported_scipy = False
class SomeOp(Op):
...
def make_node(self, x):
assert imported_scipy, (
"SciPy not available. SciPy is needed for the SomeOp op.")
...
from nose.plugins.skip import SkipTest
class test_SomeOp(utt.InferShapeTester):
...
def test_infer_shape(self):
if not imported_scipy:
raise SkipTest("SciPy needed for the SomeOp op.")
...
.. _sparse_ops:
Sparse Ops
==========
There are a few differences to keep in mind if you want to make an op
that uses :ref:`sparse ` inputs or outputs, rather than the
usual dense tensors. In particular, in the
``make_node()`` function, you have to call
``theano.sparse.as_sparse_variable(x)`` on sparse input variables,
instead of ``as_tensor_variable(x)``.
Another difference is that you need to use ``SparseVariable`` and
``SparseType`` instead of ``TensorVariable`` and ``TensorType``.
Do not forget that we support only sparse matrices (so only 2 dimensions)
and (like in SciPy) they do not support broadcasting operations by default
(although a few Ops do it when called manually). Also, we support only two
formats for sparse type: ``csr`` and ``csc``. So in ``make_mode()``,
you can create output variables like this:
.. code-block:: python
out_format = inputs[0].format # or 'csr' or 'csc' if the output format is fixed
SparseType(dtype=inputs[0].dtype, format=out_format).make_variable()
See the sparse :class:`theano.sparse.basic.Cast` op `code
`_
for a good example of a sparse op with Python code.
.. note::
From the definition of CSR and CSC formats, CSR column indices are
not necessarily sorted. Likewise for CSC row indices. Use
:class:`EnsureSortedIndices
` if your code does not
support it.
Also, there can be explicit zeros in your inputs. Use
:class:`Remove0 ` or ``remove0`` to
make sure they aren't present in your input if you don't support
that.
To remove explicit zeros and make sure indices are sorted, use
:func:`clean `.
Sparse Gradient
---------------
There are 2 types of :ref:`gradients ` for sparse
operations: ``normal``
gradient and ``structured`` gradient. Please document what your op
implements in its docstring. It is important that the user knows it, and
it is not always easy to infer from the code. Also make clear which
inputs/outputs are sparse and which ones are dense.
Sparse C code
-------------
Theano does not have a native C code interface for sparse matrices. The
reason is simple: we use the SciPy sparse matrix objects and they don't
have a C object. So we use a simple trick: a sparse matrix is made of
4 fields that are NumPy vector arrays: ``data``, ``indices``, ``indptr``
and ``shape``. So to make
an op with C code that has sparse variables as inputs, we actually make an op
that takes as input the needed fields of those sparse variables.
You can extract the 4 fields with
:func:`theano.sparse.basic.csm_properties`. You can use
:func:`theano.sparse.basic.csm_data`,
:func:`theano.sparse.basic.csm_indices`,
:func:`theano.sparse.basic.csm_indptr` and
:func:`theano.sparse.basic.csm_shape` to extract the individual
fields.
You can look at the `AddSD
`_
sparse op for an example with C code. It implements the addition of a
sparse matrix with a dense matrix.
Sparse Tests
------------
You can reuse the test system for tensor variables. To generate the
needed sparse variable and data, you can use
:func:`theano.sparse.tests.test_basic.sparse_random_inputs`. It takes
many parameters, including parameters for the format (csr or csc), the shape, the
dtype, whether to have explicit 0 and whether to have unsorted indices.
.. _random_ops:
Random distribution
===================
We have 3 base random number generators. One that wraps NumPy's random
generator, one that implements MRG31k3p and one that wraps CURAND.
The fastest, but less developed, is CURAND. It works only on CUDA-enabled
GPUs. It does not work on the CPU and it has fewer random distributions
implemented.
The recommended and 2nd faster is MRG. It works on the GPU and CPU and
has more implemented distributions.
The slowest is our wrapper on NumPy's random generator.
We explain and provide advice on 3 possibles implementations of new
distributions here:
1. Extend our wrapper around NumPy random functions.
See this `PR `_ as an example.
2. Extend MRG implementation by reusing existing Theano Op. Look into
the ``theano/sandbox/rng_mrg.py`` file and grep for all code about
binomial(). This distribution uses the output of the uniform
distribution and converts it to a binomial distribution with
existing Theano operations. The tests go in
``theano/sandbox/test_rng_mrg.py``
3. Extend MRG implementation with a new Op that takes a uniform sample as
input. Look in the ``theano/sandbox/{rng_mrg,multinomial}.py`` file
and its test in ``theano/sandbox/test_multinomal.py``. This is
recommended when current Theano ops aren't well suited to modify
the uniform to the target distribution. This can happen in
particular if there is a loop or complicated condition.
.. note::
In all cases, you must reuse the same interface as NumPy for compatibility.
.. _openmp_ops:
OpenMP Ops
==========
To allow consistent interface of Ops that support OpenMP, we have some
helper code. Doing this also allows to enable/disable OpenMP globally
or per op for fine-grained control.
Your Op needs to inherit from ``theano.gof.OpenMPOp``. If it overrides
the ``__init__()`` method, it must have an ``openmp=None`` parameter
and must call ``super(MyOpClass, self).__init__(openmp=openmp)``.
The ``OpenMPOp`` class also implements ``c_compile_args`` and
``make_thunk``. This makes it add the correct g++ flags to compile with
OpenMP. It also disables OpenMP and prints a warning if the version of
g++ does not support it.
The Theano flag ``openmp`` is currently False by default as we do not
have code that gets sped up with it. The only current implementation
is ConvOp. It speeds up some cases, but slows down others. That is why
we disable it by default. But we have all the code to have it enabled
by default if there is more than 1 core and the environment
variable OMP_NUM_THREADS is not 1. This allows Theano to respect the
current convention.
.. note:
The OpenMP parameter of an Op should not be used in its __eq__ and
__hash__ methods. Those methods are used to merge equivalent
computation in a Theano graph. If we have 2 Apply nodes with the
same inputs and they execute 2 ConvOp that only differ on the
OpenMP parameter, we want them to be merged.
.. _numba_ops:
Numba Ops
=========
Want C speed without writing C code for your new Op? You can use Numba
to generate the C code for you! Here is an `example
Op `_ doing that.
.. _alternate_theano_types:
Alternate Theano Types
======================
Most ops in Theano are used to manipulate tensors. However, Theano also
supports many other variable types. The supported types are listed below,
along with pointers to the relevant documentation.
* :class:`TensorType ` : Theano type that represents
a multidimensional array containing elements that all have the same
type. Variables of this Theano type are represented in C as objects of
class
`PyArrayObject `_.
* :ref:`TypedList ` : Theano type that represents a
typed list (a list where every element in the list has the same Theano
type). Variables of this Theano type are represented in C as objects
of class `PyListObject `_.
* :ref:`Scalar ` : Theano type that represents a C
primitive type. The C type associated with this Theano type is the
represented C primitive itself.
* :ref:`SparseType ` : Theano type used to represent sparse
tensors. There is no equivalent C type for this Theano Type but you
can split a sparse variable into its parts as TensorVariables. Those
can then be used as inputs to an op with C code.
* :class:`Generic ` : Theano type that
represents a simple Python Object. Variables of this Theano type are
represented in C as objects of class `PyObject
`_.
* :class:`CDataType ` : Theano type that
represents a C data type. The C type associated with this Theano type
depends on the data being represented.
Theano-1.0.1/doc/extending/pipeline.txt 0000777 0000000 0000000 00000012520 13212236347 017777 0 ustar root root 0000000 0000000
.. _pipeline:
====================================
Overview of the compilation pipeline
====================================
The purpose of this page is to explain each step of defining and
compiling a Theano function.
Definition of the computation graph
-----------------------------------
By creating Theano :ref:`Variables ` using
``theano.tensor.lscalar`` or ``theano.tensor.dmatrix`` or by using
Theano functions such as ``theano.tensor.sin`` or
``theano.tensor.log``, the user builds a computation graph. The
structure of that graph and details about its components can be found
in the :ref:`graphstructures` article.
Compilation of the computation graph
------------------------------------
Once the user has built a computation graph, she can use
``theano.function`` in order to make one or more functions that
operate on real data. function takes a list of input :ref:`Variables
` as well as a list of output Variables that define a
precise subgraph corresponding to the function(s) we want to define,
compile that subgraph and produce a callable.
Here is an overview of the various steps that are done with the
computation graph in the compilation phase:
Step 1 - Create a FunctionGraph
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
The subgraph given by the end user is wrapped in a structure called
*FunctionGraph*. That structure defines several hooks on adding and
removing (pruning) nodes as well as on modifying links between nodes
(for example, modifying an input of an :ref:`apply` node) (see the
article about :ref:`libdoc_gof_fgraph` for more information).
FunctionGraph provides a method to change the input of an Apply node from one
Variable to another and a more high-level method to replace a Variable
with another. This is the structure that :ref:`Optimizers
` work on.
Some relevant :ref:`Features ` are typically added to the
FunctionGraph, namely to prevent any optimization from operating inplace on
inputs declared as immutable.
Step 2 - Execute main Optimizer
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Once the FunctionGraph is made, an :term:`optimizer` is produced by
the :term:`mode` passed to ``function`` (the Mode basically has two
important fields, ``linker`` and ``optimizer``). That optimizer is
applied on the FunctionGraph using its optimize() method.
The optimizer is typically obtained through :attr:`optdb`.
Step 3 - Execute linker to obtain a thunk
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Once the computation graph is optimized, the :term:`linker` is
extracted from the Mode. It is then called with the FunctionGraph as
argument to
produce a ``thunk``, which is a function with no arguments that
returns nothing. Along with the thunk, one list of input containers (a
theano.gof.Container is a sort of object that wraps another and does
type casting) and one list of output containers are produced,
corresponding to the input and output Variables as well as the updates
defined for the inputs when applicable. To perform the computations,
the inputs must be placed in the input containers, the thunk must be
called, and the outputs must be retrieved from the output containers
where the thunk put them.
Typically, the linker calls the ``toposort`` method in order to obtain
a linear sequence of operations to perform. How they are linked
together depends on the Linker used. The CLinker produces a single
block of C code for the whole computation, whereas the OpWiseCLinker
produces one thunk for each individual operation and calls them in
sequence.
The linker is where some options take effect: the ``strict`` flag of
an input makes the associated input container do type checking. The
``borrow`` flag of an output, if False, adds the output to a
``no_recycling`` list, meaning that when the thunk is called the
output containers will be cleared (if they stay there, as would be the
case if ``borrow`` was True, the thunk would be allowed to reuse (or
"recycle") the storage).
.. note::
Compiled libraries are stored within a specific compilation directory,
which by default is set to ``$HOME/.theano/compiledir_xxx``, where
``xxx`` identifies the platform (under Windows the default location
is instead ``$LOCALAPPDATA\Theano\compiledir_xxx``). It may be manually set
to a different location either by setting :attr:`config.compiledir` or
:attr:`config.base_compiledir`, either within your Python script or by
using one of the configuration mechanisms described in :mod:`config`.
The compile cache is based upon the C++ code of the graph to be compiled.
So, if you change compilation configuration variables, such as
:attr:`config.blas.ldflags`, you will need to manually remove your compile cache,
using ``Theano/bin/theano-cache clear``
Theano also implements a lock mechanism that prevents
multiple compilations within the same compilation directory (to avoid
crashes with paralell execution of some scripts). This mechanism is
currently enabled by default, but if it causes any problem it may be
disabled using the function
``theano.gof.compilelock.set_lock_status(..)``.
Step 4 - Wrap the thunk in a pretty package
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
The thunk returned by the linker along with input and output
containers is unwieldy. ``function`` hides that complexity away so
that it can be used like a normal function with arguments and return
values.
Theano-1.0.1/doc/extending/scan.txt 0000777 0000000 0000000 00000041330 13212236347 017117 0 ustar root root 0000000 0000000 .. _scan_internals:
Developper documentation for Scan
+++++++++++++++++++++++++++++++++
Context
=======
This document is meant to act as reference material for developers working
on Theano's loop mechanism. This mechanism is called Scan and its internals
are highly complex, hence the need for a centralized repository of knowledge
regarding its inner workings.
The ``theano.scan()`` function is the public-facing interface for looping in
Theano. Under the hood, this function will perform some processing on its
inputs and instantiate the ``Scan`` op class which implements the looping
mechanism. It achieves this by compiling its own Theano function representing
the computation to be done at every iteration of the loop and calling it as
many times as necessary.
The correspondence between the parameters and behaviors of the function and the
op is not always simple since the former is meant for usability and the second
for performance. Since this document is intended to be used by developers
working inside Scan itself, it will mostly discuss things from the point of view
of the ``Scan`` op class. Nonetheless, it will attempt to link those elements to
their corresponding concepts in the scan function as often as is reasonably
practical.
Pre-requisites
==============
The following sections assumes the reader is familiar with the following :
1. Theano's :ref:`graph structure ` (Apply nodes, Variable nodes and Ops)
2. The interface and usage of Theano's :ref:`scan() ` function
Additionally, the :ref:`scan_internals_optimizations` section below assumes
knowledge of:
3. Theano's :ref:`graph optimizations `
Relevant code files
===================
The implementation of Scan is spread over several files in
``theano/scan_module``. The different files, and sections of the code they
deal with, are :
* ``scan.py`` implements the ``scan`` function. The ``scan`` function
arranges the arguments of scan correctly, constructs the scan op and
afterwards calls the constructed scan op on the arguments. This function
takes care of figuring out missing inputs and shared variables.
* ``scan_op.py`` implements the ``Scan`` op class. The ``Scan`` respects
the ``Op`` interface, and contains most of the logic of the scan operator.
* ``scan_utils.py`` contains several helpful functions used throughout out the
other files that are specific of the scan operator.
* ``scan_views.py`` contains different views of the scan op that have
simpler and easier signatures to be used in specific cases.
* ``scan_opt.py`` contains the list of all Theano graph optimizations for the
scan operator.
Notation
========
Scan being a sizeable and complex module, it has its own naming convention for
functions and variables which this section will attempt to introduce.
A scan op contains a Theano function representing the computation
that is done in a single iteration of the loop represented by the scan op (in
other words, the computation given by the function provided as value to
``theano.scan``'s ``fn`` argument ). Whenever we discuss a scan op, the **outer
function** refers to the Theano function that *contains* the scan op whereas the
**inner function** refers to the Theano function that is *contained* inside the
scan op.
In the same spirit, the inputs and outputs of the *Apply node wrapping the scan
op* (or *scan node* for short) are referred to as **outer inputs** and **outer
outputs**, respectively, because these inputs and outputs are variables in the
outer function graph. The inputs and outputs of scan's inner function are
designated **inner inputs** and **inner outputs**, respectively.
Scan variables
==============
The following are the different types of variables that Scan has the
capacity to handle, along with their various caracteristics.
**Sequence** : A sequence is a Theano variable which Scan will iterate
over and give sub-elements to its inner function as input. A sequence
has no associated output. For a sequence variable ``X``, at timestep
``t``, the inner function will receive as input the sequence element
``X[t]``. These variables are used through the argument ``sequences``
of the ``theano.scan()`` function.
**Non-sequences** : A non-sequence is a Theano variable which Scan
*will provide as-is* to its inner function. Like a sequence, a
non-sequence has no associated output. For a non-sequence variable
``X``, at timestep ``t``, the inner function will receive as input
the variable ``X``. These variables are used through the argument
``non_sequences`` of the ``theano.scan()`` function.
**Nitsot (no input tap, single output tap)** : A nitsot is an output
variable of the inner function that is not fed back as an input to the
next iteration of the inner function. Nitsots are typically
encountered in situations where Scan is used to perform a 'map'
operation (every element in a tensor is independently altered using a
given operation to produce a new tensor) such as squaring every number
in a vector.
**Sitsot (single input tap, single output tap)** : A sitsot is an output
variable of the inner function that is fed back as an input to the next
iteration of the inner function. A typical setting where a sitsot might be
encountered is the case where Scan is used to compute the cumulative sum over
the elements of a vector and a sitsot output is employed to act as an
accumulator.
**Mitsot (multiple input taps, single output tap)** : A mitsot is an
output variable of the inner function that is fed back as an input to
future iterations of the inner function (either multiple future
iterations or a single one that isn't the immediate next one). For
example, a mitsot might be used in the case where Scan is used to
compute the Fibonacci sequence, one term of the sequence at every
timestep, since every computed term needs to be reused to compute the
two next terms of the sequence.
**Mitmot (multiple input taps, multiple output taps)** : These outputs exist
but they cannot be directly created by the user. They can appear in a theano
graph as a result of taking the gradient of the output of a Scan with respect
to its inputs: This will result in the creation of a new scan node used to
compute the gradients of the first scan node. If the original Scan had sitsots
or mitsots variables, the new Scan will use mitmots to compute the gradients
through time for these variables.
To synthesize :
=========================================================== ===================================================== ========================================================== =========================================================== ========================================================= ======================================================
Type of scan variables Corresponding outer input Corresponding inner input at timestep `t` (indexed from 0) Corresponding inner output at timestep `t` (indexed from 0) Corresponding outer output `t` Corresponding argument of the `theano.scan()` function
=========================================================== ===================================================== ========================================================== =========================================================== ========================================================= ======================================================
Sequence Sequence of elements X Individual sequence element X[t] *No corresponding inner output* *No corresponding outer output* `sequences`
Non-Sequence Any variable X Variable identical to X *No corresponding inner output* *No corresponding outer output* `non_sequences`
Non-recurring output (nitsot) *No corresponding outer input* *No corresponding inner input* Output value at timestep `t` Concatenation of the values of the output at all timestep `outputs_info`
Singly-recurrent output (sitsot) Initial value (value at timestep `-1`) Output value at previous timestep (`t-1`) Output value at timestep `t` Concatenation of the values of the output at all timestep `outputs_info`
Multiply-recurrent output (mitsot) Initial values for the required timesteps where `t<0` Output value at previous required timesteps Output value at timestep `t` Concatenation of the values of the output at all timestep `outputs_info`
Multiply-recurrent multiple outputs (mitmot) Initial values for the required timesteps where `t<0` Output value at previous required timesteps Output values for current and multiple future timesteps Concatenation of the values of the output at all timestep *No corresponding argument*
=========================================================== ===================================================== ========================================================== =========================================================== ========================================================= ======================================================
.. _scan_internals_optimizations:
Optimizations
=============
remove_constants_and_unused_inputs_scan
---------------------------------------
This optimization serves two purposes, The first is to remove a scan op's
unused inputs. The second is to take a scan op's constant inputs and remove
them, instead injecting the constants directly into the graph or the scan
op's inner function. This will allow constant folding to happen inside the
inner function.
PushOutNonSeqScan
-----------------
This optimizations pushes, out of Scan's inner function and into the outer
function, computation that depends only on non-sequence inputs. Such
computation ends up being done every iteration on the same values so moving
it to the outer function to be executed only once, before the scan op,
reduces the amount of computation that needs to be performed.
PushOutSeqScan
--------------
This optimization resembles PushOutNonSeqScan but it tries to push, out of
the inner function, the computation that only relies on sequence and
non-sequence inputs. The idea behing this optimization is that, when it is
possible to do so, it is generally more computationally efficient to perform
a single operation on a large tensor rather then perform that same operation
many times on many smaller tensors. In many cases, this optimization can
increase memory usage but, in some specific cases, it can also decrease it.
PushOutScanOutput
-----------------
This optimizations attempts to push out some of the computation at the end
of the inner function to the outer function, to be executed after the scan
node. Like PushOutSeqScan, this optimization aims to replace many operations
on small tensors by few operations on large tensors. It can also lead to
increased memory usage.
PushOutDot1
-----------
This is another optimization that attempts to detect certain patterns of
computation in a scan op's inner function and move this computation to the
outer graph.
ScanInplaceOptimizer
--------------------
This optimization attempts to make Scan compute its recurrent outputs inplace
on the input tensors that contain their initial states. This optimization can
improve runtime performance as well as reduce memory usage.
ScanSaveMem
-----------
This optimizations attempts to determine if a scan node, during its execution,
for any of its outputs, can get away with allocating a memory buffer that is
large enough to contain some of the computed timesteps of that output but not
all of them.
By default, during the execution of a scan node, memory buffers will be
allocated to store the values computed for every output at every iteration.
However, in some cases, there are outputs for which there is only really a
need to store the most recent N values, not all of them.
For instance, if a scan node has a sitsot output (last computed value is
fed back as an input at the next iteration) and only the last timestep of
that output is ever used in the outer function, the ScanSaveMem optimization
could determine that there is no need to store all computed timesteps for
that sitsot output. Only the most recently computed timestep ever needs to
be kept in memory.
ScanMerge
---------
This optimization attempts to fuse distinct scan ops into a single scan op
that performs all the computation. The main advantage of merging scan ops
together comes from the possibility of both original ops having some
computation in common. In such a setting, this computation ends up being done
twice. The fused scan op, however, would only need to do it once and could
therefore be more computationally efficient. Also, since every scan node
involves a certain overhead, at runtime, reducing the number of scan nodes in
the graph can improve performance.
scan_merge_inouts
-----------------
This optimization attempts to merge a scan op's identical outer inputs as well
as merge its identical outer outputs (outputs that perform the same
computation on the same inputs). This can reduce the amount of computation as
well as result in a simpler graph for both the inner function and the outer
function.
Helper classes and functions
============================
Because of the complexity involved in dealing with Scan, a large number of
helper classes and functions have been developped over time to implement
operations commonly needed when dealing with the scan op. The scan op
itself defines a large number of them and others can be found in the file
``scan_utils.py``. This sections aims to point out the most useful ones sorted
by usage.
Accessing/manipulating Scan's inputs and outputs by type
--------------------------------------------------------
Declared in ``scan_utils.py``, the class ``scan_args`` handles the
parsing of the inputs and outputs (both inner and outer) to a format
that is easier to analyse and manipulate. Without this class,
analysing Scan's inputs and outputs often required convoluted logic
which make for code that is hard to read and to maintain. Because of
this, you should favor using ``scan_args`` when it is practical and
appropriate to do so.
The scan op also defines a few helper functions for this purpose, such as
``inner_nitsot_outs()`` or ``mitmot_out_taps()``, but they are often poorly
documented and easy to misuse. These should be used with great care.
Navigating between outer inputs/outputs and inner inputs/outputs
----------------------------------------------------------------
Navigation between these four sets of variables can be done in two ways,
depending on the type of navigation that is required.
If the goal is to navigate between variables that are associated with the same
states (ex : going from an outer sequence input to the corresponding inner
sequence input, going from an inner output associated with a recurrent state
to the inner input(s) associated with that same recurrent state, etc.), then
the ``var_mappings`` attribute of the scan op can be used.
This attribute is a dictionnary with 12 {key/value} pairs. The keys are listed
below :
* "outer_inp_from_outer_out"
* "inner_inp_from_outer_out"
* "inner_out_from_outer_out"
* "inner_inp_from_outer_inp"
* "inner_out_from_outer_inp"
* "outer_out_from_outer_inp"
* "outer_inp_from_inner_inp"
* "inner_out_from_inner_inp"
* "outer_out_from_inner_inp"
* "outer_inp_from_inner_out"
* "inner_inp_from_inner_out"
* "outer_out_from_inner_out"
Every corresponding value is a dictionary detailing a mapping from one set of
variables to another. For each of those dictionaries the keys are indices of
variables in one set and the values are the indices of the corresponding
variables in another set. For mappings to outer variables, the values are
individual indices or -1 if there is not corresponding outer variable.
For mappings to inner variables, the values are list of indices because
multiple inner variables may be associated with the same state.
If the goal is to navigate between variables that are *connected*
(meaning that one of them is used to compute the other), the methods
``connection_pattern()`` and ``inner_connection_pattern()`` can be
used. The method ``connection_pattern()`` returns a list of lists
detailing, for every pair of outer input and outer output whether they
are connected or not. The method ``inner_connection_pattern()``
accomplishes the same goal but for every possible pair of inner output
and inner input.
Theano-1.0.1/doc/extending/theano_vs_c.txt 0000777 0000000 0000000 00000002301 13212236347 020456 0 ustar root root 0000000 0000000
.. _theano_vs_c:
============
Theano vs. C
============
We describe some of the patterns in Theano, and present their closest
analogue in a statically typed language such as C:
=============== ===========================================================
Theano C
=============== ===========================================================
Apply function application / function call
Variable local function data / variable
Shared Variable global function data / variable
Op operations carried out in computation / function definition
Type data types
=============== ===========================================================
For example:
.. code-block:: c
int d = 0;
int main(int a) {
int b = 3;
int c = f(b)
d = b + c;
return g(a, c);
}
Based on this code snippet, we can relate ``f`` and ``g`` to Ops, ``a``,
``b`` and ``c`` to Variables, ``d`` to Shared Variable, ``g(a, c)``,
``f(b)`` and ``d = b + c`` (taken as meaning
the action of computing ``f``, ``g`` or ``+`` on their respective inputs) to
Applies. Lastly, ``int`` could be interpreted as the Theano Type of the
Variables ``a``, ``b``, ``c`` and ``d``.
Theano-1.0.1/doc/extending/tips.txt 0000777 0000000 0000000 00000002623 13212236347 017154 0 ustar root root 0000000 0000000
====
Tips
====
Reusing outputs
===============
WRITEME
Don't define new Ops unless you have to
=======================================
It is usually not useful to define Ops that can be easily
implemented using other already existing Ops. For example, instead of
writing a "sum_square_difference" Op, you should probably just write a
simple function:
.. testcode::
from theano import tensor as T
def sum_square_difference(a, b):
return T.sum((a - b)**2)
Even without taking Theano's optimizations into account, it is likely
to work just as well as a custom implementation. It also supports all
data types, tensors of all dimensions as well as broadcasting, whereas
a custom implementation would probably only bother to support
contiguous vectors/matrices of doubles...
Use Theano's high order Ops when applicable
===========================================
Theano provides some generic Op classes which allow you to generate a
lot of Ops at a lesser effort. For instance, Elemwise can be used to
make :term:`elementwise` operations easily whereas DimShuffle can be
used to make transpose-like transformations. These higher order Ops
are mostly Tensor-related, as this is Theano's specialty.
.. _opchecklist:
Op Checklist
============
Use this list to make sure you haven't forgotten anything when
defining a new Op. It might not be exhaustive but it covers a lot of
common mistakes.
WRITEME
Theano-1.0.1/doc/extending/type.txt 0000777 0000000 0000000 00000036127 13212236347 017164 0 ustar root root 0000000 0000000 .. _theano_type:
======================
Making the double type
======================
.. _type_contract:
Type's contract
===============
In Theano's framework, a ``Type`` (:class:`.gof.type.Type`)
is any object which defines the following
methods. To obtain the default methods described below, the Type should
be an instance of ``Type`` or should be an instance of a
subclass of ``Type``. If you will write all methods yourself,
you need not use an instance of ``Type``.
Methods with default arguments must be defined with the same signature,
i.e. the same default argument names and values. If you wish to add
extra arguments to any of these methods, these extra arguments must have
default values.
.. class:: PureType
.. method:: filter(value, strict=False, allow_downcast=None)
This casts a value to match the Type and returns the
cast value. If ``value`` is incompatible with the Type,
the method must raise an exception. If ``strict`` is True, ``filter`` must return a
reference to ``value`` (i.e. casting prohibited).
If ``strict`` is False, then casting may happen, but downcasting should
only be used in two situations:
* if ``allow_downcast`` is True
* if ``allow_downcast`` is ``None`` and the default behavior for this
type allows downcasting for the given ``value`` (this behavior is
type-dependent, you may decide what your own type does by default)
We need to define ``filter`` with three arguments. The second argument
must be called ``strict`` (Theano often calls it by keyword) and must
have a default value of ``False``. The third argument must be called
``allow_downcast`` and must have a default value of ``None``.
.. method:: filter_inplace(value, storage, strict=False, allow_downcast=None)
If filter_inplace is defined, it will be called instead of
filter() This is to allow reusing the old allocated memory. As
of this writing this is used only when we transfer new data to a
shared variable on the gpu.
``storage`` will be the old value. i.e. The old numpy array,
CudaNdarray, ...
.. method:: is_valid_value(value)
Returns True iff the value is compatible with the Type. If
``filter(value, strict = True)`` does not raise an exception, the
value is compatible with the Type.
*Default:* True iff ``filter(value, strict=True)`` does not raise
an exception.
.. method:: values_eq(a, b)
Returns True iff ``a`` and ``b`` are equal.
*Default:* ``a == b``
.. method:: values_eq_approx(a, b)
Returns True iff ``a`` and ``b`` are approximately equal, for a
definition of "approximately" which varies from Type to Type.
*Default:* ``values_eq(a, b)``
.. method:: make_variable(name=None)
Makes a :term:`Variable` of this Type with the specified name, if
``name`` is not ``None``. If ``name`` is ``None``, then the Variable does
not have a name. The Variable will have its ``type`` field set to
the Type object.
*Default:* there is a generic definition of this in Type. The
Variable's ``type`` will be the object that defines this method (in
other words, ``self``).
.. method:: __call__(name=None)
Syntactic shortcut to ``make_variable``.
*Default:* ``make_variable``
.. method:: __eq__(other)
Used to compare Type instances themselves
*Default:* ``object.__eq__``
.. method:: __hash__()
Types should not be mutable, so it should be OK to define a hash
function. Typically this function should hash all of the terms
involved in ``__eq__``.
*Default:* ``id(self)``
.. method:: get_shape_info(obj)
Optional. Only needed to profile the memory of this Type of object.
Return the information needed to compute the memory size of ``obj``.
The memory size is only the data, so this excludes the container.
For an ndarray, this is the data, but not the ndarray object and
other data structures such as shape and strides.
``get_shape_info()`` and ``get_size()`` work in tandem for the memory profiler.
``get_shape_info()`` is called during the execution of the function.
So it is better that it is not too slow.
``get_size()`` will be called on the output of this function
when printing the memory profile.
:param obj: The object that this Type represents during execution
:return: Python object that ``self.get_size()`` understands
.. method:: get_size(shape_info)
Number of bytes taken by the object represented by shape_info.
Optional. Only needed to profile the memory of this Type of object.
:param shape_info: the output of the call to get_shape_info()
:return: the number of bytes taken by the object described by
``shape_info``.
.. method:: clone(dtype=None, broadcastable=None)
Optional, for TensorType-alikes.
Return a copy of the type with a possibly changed value for
dtype and broadcastable (if they aren't `None`).
:param dtype: New dtype for the copy.
:param broadcastable: New broadcastable tuple for the copy.
.. method:: may_share_memory(a, b)
Optional to run, but mandatory for DebugMode. Return True if the Python
objects `a` and `b` could share memory. Return False
otherwise. It is used to debug when Ops did not declare memory
aliasing between variables. Can be a static method.
It is highly recommended to use and is mandatory for Type in Theano
as our buildbot runs in DebugMode.
For each method, the *default* is what ``Type`` defines
for you. So, if you create an instance of ``Type`` or an
instance of a subclass of ``Type``, you
must define ``filter``. You might want to override ``values_eq_approx``,
as well as ``values_eq``. The other defaults generally need not be
overridden.
For more details you can go see the documentation for :ref:`type`.
Additional definitions
----------------------
For certain mechanisms, you can register functions and other such
things to plus your type into theano's mechanisms. These are optional
but will allow people to use you type with familiar interfaces.
`transfer()`
~~~~~~~~~~~~
To plug in additional options for the transfer target, define a
function which takes a theano variable and a target argument and
returns eitehr a new transferred variable (which can be the same as
the input if no transfer is nessecary) or returns None if the transfer
can't be done.
Then register that function by calling :func:`register_transfer()`
with it as argument.
Defining double
===============
We are going to base Type ``double`` on Python's ``float``. We
must define ``filter`` and shall override ``values_eq_approx``.
**filter**
.. testcode::
# Note that we shadow Python's function ``filter`` with this
# definition.
def filter(x, strict=False, allow_downcast=None):
if strict:
if isinstance(x, float):
return x
else:
raise TypeError('Expected a float!')
elif allow_downcast:
return float(x)
else: # Covers both the False and None cases.
x_float = float(x)
if x_float == x:
return x_float
else:
raise TypeError('The double type cannot accurately represent '
'value %s (of type %s): you must explicitly '
'allow downcasting if you want to do this.'
% (x, type(x)))
If ``strict`` is True we need to return ``x``. If ``strict`` is True and ``x`` is not a
``float`` (for example, ``x`` could easily be an ``int``) then it is
incompatible with our Type and we must raise an exception.
If ``strict is False`` then we are allowed to cast ``x`` to a ``float``,
so if ``x`` is an ``int`` it we will return an equivalent ``float``.
However if this cast triggers a precision loss (``x != float(x)``) and
``allow_downcast`` is not True, then we also raise an exception.
Note that here we decided that the default behavior of our type
(when ``allow_downcast`` is set to ``None``) would be the same as
when ``allow_downcast`` is False, i.e. no precision loss is allowed.
**values_eq_approx**
.. testcode::
def values_eq_approx(x, y, tolerance=1e-4):
return abs(x - y) / (abs(x) + abs(y)) < tolerance
The second method we define is ``values_eq_approx``. This method
allows approximate comparison between two values respecting our Type's
constraints. It might happen that an optimization changes the computation
graph in such a way that it produces slightly different variables, for
example because of numerical instability like rounding errors at the
end of the mantissa. For instance, ``a + a + a + a + a + a`` might not
actually produce the exact same output as ``6 * a`` (try with a=0.1),
but with ``values_eq_approx`` we do not necessarily mind.
We added an extra ``tolerance`` argument here. Since this argument is
not part of the API, it must have a default value, which we
chose to be 1e-4.
.. note::
``values_eq`` is never actually used by Theano, but it might be used
internally in the future. Equality testing in
:ref:`DebugMode ` is done using ``values_eq_approx``.
**Putting them together**
What we want is an object that respects the aforementioned
contract. Recall that Type defines default implementations for all
required methods of the interface, except ``filter``. One way to make
the Type is to instantiate a plain Type and set the needed fields:
.. testcode::
from theano import gof
double = gof.Type()
double.filter = filter
double.values_eq_approx = values_eq_approx
Another way to make this Type is to make a subclass of ``gof.Type``
and define ``filter`` and ``values_eq_approx`` in the subclass:
.. code-block:: python
from theano import gof
class Double(gof.Type):
def filter(self, x, strict=False, allow_downcast=None):
# See code above.
...
def values_eq_approx(self, x, y, tolerance=1e-4):
# See code above.
...
double = Double()
``double`` is then an instance of Type ``Double``, which in turn is a
subclass of ``Type``.
There is a small issue with defining ``double`` this way. All
instances of ``Double`` are technically the same Type. However, different
``Double`` Type instances do not compare the same:
.. testsetup::
from theano import gof
class Double(gof.Type):
def filter(self, x, strict=False, allow_downcast=None):
if strict:
if isinstance(x, float):
return x
else:
raise TypeError('Expected a float!')
elif allow_downcast:
return float(x)
else: # Covers both the False and None cases.
x_float = float(x)
if x_float == x:
return x_float
else:
raise TypeError('The double type cannot accurately represent '
'value %s (of type %s): you must explicitly '
'allow downcasting if you want to do this.'
% (x, type(x)))
def values_eq_approx(self, x, y, tolerance=1e-4):
return abs(x - y) / (abs(x) + abs(y)) < tolerance
def __str__(self):
return "double"
double = Double()
>>> double1 = Double()
>>> double2 = Double()
>>> double1 == double2
False
Theano compares Types using ``==`` to see if they are the same.
This happens in DebugMode. Also, Ops can (and should) ensure that their inputs
have the expected Type by checking something like ``if x.type == lvector``.
There are several ways to make sure that equality testing works properly:
#. Define ``Double.__eq__`` so that instances of type Double
are equal. For example:
.. testcode::
def __eq__(self, other):
return type(self) is Double and type(other) is Double
#. Override ``Double.__new__`` to always return the same instance.
#. Hide the Double class and only advertise a single instance of it.
Here we will prefer the final option, because it is the simplest.
Ops in the Theano code often define the ``__eq__`` method though.
Untangling some concepts
========================
Initially, confusion is common on what an instance of Type is versus
a subclass of Type or an instance of Variable. Some of this confusion is
syntactic. A Type is any object which has fields corresponding to the
functions defined above. The Type class provides sensible defaults for
all of them except ``filter``, so when defining new Types it is natural
to subclass Type. Therefore, we often end up with Type subclasses and
it is can be confusing what these represent semantically. Here is an
attempt to clear up the confusion:
* An **instance of Type** (or an instance of a subclass)
is a set of constraints on real data. It is
akin to a primitive type or class in C. It is a *static*
annotation.
* An **instance of Variable** symbolizes data nodes in a data flow
graph. If you were to parse the C expression ``int x;``, ``int``
would be a Type instance and ``x`` would be a Variable instance of
that Type instance. If you were to parse the C expression ``c = a +
b;``, ``a``, ``b`` and ``c`` would all be Variable instances.
* A **subclass of Type** is a way of implementing
a set of Type instances that share
structural similarities. In the ``double`` example that we are doing,
there is actually only one Type in that set, therefore the subclass
does not represent anything that one of its instances does not. In this
case it is a singleton, a set with one element. However, the
:class:`TensorType`
class in Theano (which is a subclass of Type)
represents a set of types of tensors
parametrized by their data type or number of dimensions. We could say
that subclassing Type builds a hierarchy of Types which is based upon
structural similarity rather than compatibility.
Final version
=============
.. testcode::
from theano import gof
class Double(gof.Type):
def filter(self, x, strict=False, allow_downcast=None):
if strict:
if isinstance(x, float):
return x
else:
raise TypeError('Expected a float!')
elif allow_downcast:
return float(x)
else: # Covers both the False and None cases.
x_float = float(x)
if x_float == x:
return x_float
else:
raise TypeError('The double type cannot accurately represent '
'value %s (of type %s): you must explicitly '
'allow downcasting if you want to do this.'
% (x, type(x)))
def values_eq_approx(self, x, y, tolerance=1e-4):
return abs(x - y) / (abs(x) + abs(y)) < tolerance
def __str__(self):
return "double"
double = Double()
We add one utility function, ``__str__``. That way, when we print
``double``, it will print out something intelligible.
Theano-1.0.1/doc/extending/unittest.txt 0000777 0000000 0000000 00000047244 13212236347 020064 0 ustar root root 0000000 0000000 .. _unittest:
============
Unit Testing
============
Theano relies heavily on unit testing. Its importance cannot be
stressed enough!
Unit Testing revolves around the following principles:
* ensuring correctness: making sure that your Op, Type or Optimization
works in the way you intended it to work. It is important for this
testing to be as thorough as possible: test not only the obvious
cases, but more importantly the corner cases which are more likely
to trigger bugs down the line.
* test all possible failure paths. This means testing that your code
fails in the appropriate manner, by raising the correct errors when
in certain situations.
* sanity check: making sure that everything still runs after you've
done your modification. If your changes cause unit tests to start
failing, it could be that you've changed an API on which other users
rely on. It is therefore your responsibility to either a) provide
the fix or b) inform the author of your changes and coordinate with
that person to produce a fix. If this sounds like too much of a
burden... then good! APIs aren't meant to be changed on a whim!
This page is in no way meant to replace tutorials on Python's unittest
module, for this we refer the reader to the `official documentation
`_. We will however
adress certain specificities about how unittests relate to theano.
Unittest Primer
===============
A unittest is a subclass of ``unittest.TestCase``, with member
functions with names that start with the string ``test``. For
example:
.. testcode::
import unittest
class MyTestCase(unittest.TestCase):
def test0(self):
pass
# test passes cleanly
def test1(self):
self.assertTrue(2+2 == 5)
# raises an exception, causes test to fail
def test2(self):
assert 2+2 == 5
# causes error in test (basically a failure, but counted separately)
def test2(self):
assert 2+2 == 4
# this test has the same name as a previous one,
# so this is the one that runs.
How to Run Unit Tests ?
-----------------------
Two options are available:
theano-nose
~~~~~~~~~~~
The easiest by far is to use ``theano-nose`` which is a command line
utility that recurses through a given directory, finds all unittests
matching a specific criteria and executes them. By default, it will
find & execute tests case in test*.py files whose method name starts
with 'test'.
``theano-nose`` is a wrapper around `nosetests
.py
Running a specific unit test ::
theano-nose .py:.
Using unittest module
~~~~~~~~~~~~~~~~~~~~~
To launch tests cases from within python, you can also use the
functionality offered by the ``unittest`` module. The simplest thing
is to run all the tests in a file using ``unittest.main()``. Python's
built-in unittest module uses metaclasses to know about all the
``unittest.TestCase`` classes you have created. This call will run
them all, printing '.' for passed tests, and a stack trace for
exceptions. The standard footer code in theano's test files is:
.. testcode::
if __name__ == '__main__':
unittest.main()
You can also choose to run a subset of the full test suite.
To run all the tests in one or more ``TestCase`` subclasses:
.. code-block:: python
suite = unittest.TestLoader()
suite = suite.loadTestsFromTestCase(MyTestCase0)
suite = suite.loadTestsFromTestCase(MyTestCase1)
...
unittest.TextTestRunner(verbosity=2).run(suite)
To run just a single ``MyTestCase`` member test function called ``test0``:
.. testcode::
MyTestCase('test0').debug()
Folder Layout
-------------
"tests" directories are scattered throughout theano. Each tests
subfolder is meant to contain the unittests which validate the .py
files in the parent folder.
Files containing unittests should be prefixed with the word "test".
Optimally every python module should have a unittest file associated
with it, as shown below. Unittests testing functionality of module
.py should therefore be stored in tests/test_.py::
Theano/theano/tensor/basic.py
Theano/theano/tensor/elemwise.py
Theano/theano/tensor/tests/test_basic.py
Theano/theano/tensor/tests/test_elemwise.py
How to Write a Unittest
=======================
Test Cases and Methods
----------------------
Unittests should be grouped "logically" into test cases, which are
meant to group all unittests operating on the same element and/or
concept. Test cases are implemented as Python classes which inherit
from unittest.TestCase
Test cases contain multiple test methods. These should be prefixed
with the word "test".
Test methods should be as specific as possible and cover a particular
aspect of the problem. For example, when testing the TensorDot Op, one
test method could check for validity, while another could verify that
the proper errors are raised when inputs have invalid dimensions.
Test method names should be as explicit as possible, so that users can
see at first glance, what functionality is being tested and what tests
need to be added.
Example:
.. code-block:: python
import unittest
class TestTensorDot(unittest.TestCase):
def test_validity(self):
# do stuff
...
def test_invalid_dims(self):
# do more stuff
...
Test cases can define a special setUp method, which will get called
before each test method is executed. This is a good place to put
functionality which is shared amongst all test methods in the test
case (i.e initializing data, parameters, seeding random number
generators -- more on this later)
.. testcode:: writeUnitest
import unittest
class TestTensorDot(unittest.TestCase):
def setUp(self):
# data which will be used in various test methods
self.avals = numpy.array([[1,5,3],[2,4,1]])
self.bvals = numpy.array([[2,3,1,8],[4,2,1,1],[1,4,8,5]])
Similarly, test cases can define a tearDown method, which will be
implicitely called at the end of each test method.
Checking for correctness
------------------------
When checking for correctness of mathematical expressions, the user
should preferably compare theano's output to the equivalent numpy
implementation.
Example:
.. code-block:: python
class TestTensorDot(unittest.TestCase):
def setUp(self):
...
def test_validity(self):
a = T.dmatrix('a')
b = T.dmatrix('b')
c = T.dot(a, b)
f = theano.function([a, b], [c])
cmp = f(self.avals, self.bvals) == numpy.dot(self.avals, self.bvals)
self.assertTrue(numpy.all(cmp))
Avoid hard-coding variables, as in the following case:
.. code-block:: python
self.assertTrue(numpy.all(f(self.avals, self.bvals) == numpy.array([[25, 25, 30, 28], [21, 18, 14, 25]])))
This makes the test case less manageable and forces the user to update
the variables each time the input is changed or possibly when the
module being tested changes (after a bug fix for example). It also
constrains the test case to specific input/output data pairs. The
section on random values covers why this might not be such a good
idea.
Here is a list of useful functions, as defined by TestCase:
* checking the state of boolean variables: assert,
assertTrue, assertFalse
* checking for (in)equality constraints: assertEqual,
assertNotEqual
* checking for (in)equality constraints up to a given precision (very
useful in theano): assertAlmostEqual,
assertNotAlmostEqual
Checking for errors
-------------------
On top of verifying that your code provides the correct output, it is
equally important to test that it fails in the appropriate manner,
raising the appropriate exceptions, etc. Silent failures are deadly,
as they can go unnoticed for a long time and a hard to detect
"after-the-fact".
Example:
.. code-block:: python
import unittest
class TestTensorDot(unittest.TestCase):
...
def test_3D_dot_fail(self):
def func():
a = T.TensorType('float64', (False,False,False)) # create 3d tensor
b = T.dmatrix()
c = T.dot(a,b) # we expect this to fail
# above should fail as dot operates on 2D tensors only
self.assertRaises(TypeError, func)
Useful function, as defined by TestCase:
* assertRaises
Test Cases and Theano Modes
---------------------------
When compiling theano functions or modules, a mode parameter can be
given to specify which linker and optimizer to use.
Example:
.. code-block:: python
from theano import function
f = function([a,b],[c],mode='FAST_RUN')
Whenever possible, unit tests should omit this parameter. Leaving
out the mode will ensure that unit tests use the default mode.
This default mode is set to
the configuration variable :attr:`config.mode`, which defaults to
'FAST_RUN', and can be set by various mechanisms (see :mod:`config`).
In particular, the enviromnment variable :envvar:`THEANO_FLAGS`
allows the user to easily switch the mode in which unittests are
run. For example to run all tests in all modes from a BASH script,
type this:
.. code-block:: bash
THEANO_FLAGS='mode=FAST_COMPILE' theano-nose
THEANO_FLAGS='mode=FAST_RUN' theano-nose
THEANO_FLAGS='mode=DebugMode' theano-nose
.. _random_value_in_tests:
Using Random Values in Test Cases
---------------------------------
``numpy.random`` is often used in unit tests to initialize large data
structures, for use as inputs to the function or module being
tested. When doing this, it is imperative that the random number
generator be seeded at the be beginning of each unit test. This will
ensure that unittest behaviour is consistent from one execution to
another (i.e., always pass or always fail).
Instead of using ``numpy.random.seed`` to do this, we encourage users to
do the following:
.. testcode::
from theano.tests import unittest_tools
class TestTensorDot(unittest.TestCase):
def setUp(self):
unittest_tools.seed_rng()
# OR ... call with an explicit seed
unittest_tools.seed_rng(234234) # use only if really necessary!
The behaviour of ``seed_rng`` is as follows:
* If an explicit seed is given, it will be used for seeding numpy's rng.
* If not, it will use ``config.unittests.rseed`` (its default value is ``666``).
* If ``config.unittests.rseed`` is set to ``"random"``, it will seed the rng with
None, which is equivalent to seeding with a random seed.
The main advantage of using ``unittest_tools.seed_rng`` is that it allows
us to change the seed used in the unitests, without having to manually
edit all the files. For example, this allows the nightly build to run
``theano-nose`` repeatedly, changing the seed on every run (hence achieving
a higher confidence that the variables are correct), while still
making sure unittests are deterministic.
Users who prefer their unittests to be random (when run on their local
machine) can simply set ``config.unittests.rseed`` to ``'random'`` (see
:mod:`config`).
Similarly, to provide a seed to ``numpy.random.RandomState``, simply use:
.. testcode::
import numpy
rng = numpy.random.RandomState(unittest_tools.fetch_seed())
# OR providing an explicit seed
rng = numpy.random.RandomState(unittest_tools.fetch_seed(1231)) # again not recommended
Note that the ability to change the seed from one nosetest to another,
is incompatible with the method of hard-coding the baseline variables
(against which we compare the theano outputs). These must then be
determined "algorithmically". Although this represents more work, the
test suite will be better because of it.
To help you check that the boundaries provided to ``numpy.random`` are
correct and your tests will pass those corner cases, you can check
``utt.MockRandomState``. Code using ``utt.MockRandomState`` should not
be committed, it is just a tool to help adjust the sampling range.
Creating an Op UnitTest
=======================
A few tools have been developed to help automate the development of
unitests for Theano Ops.
.. _validating_grad:
Validating the Gradient
-----------------------
The ``verify_grad`` function can be used to validate that the ``grad``
function of your Op is properly implemented. ``verify_grad`` is based
on the Finite Difference Method where the derivative of function ``f``
at point ``x`` is approximated as:
.. math::
\frac{\partial{f}}{\partial{x}} = lim_{\Delta \rightarrow 0} \frac {f(x+\Delta) - f(x-\Delta)} {2\Delta}
``verify_grad`` performs the following steps:
* approximates the gradient numerically using the Finite Difference Method
* calculate the gradient using the symbolic expression provided in the
``grad`` function
* compares the two values. The tests passes if they are equal to
within a certain tolerance.
Here is the prototype for the verify_grad function.
.. code-block:: python
def verify_grad(fun, pt, n_tests=2, rng=None, eps=1.0e-7, abs_tol=0.0001, rel_tol=0.0001):
``verify_grad`` raises an Exception if the difference between the analytic gradient and
numerical gradient (computed through the Finite Difference Method) of a random
projection of the fun's output to a scalar exceeds
both the given absolute and relative tolerances.
The parameters are as follows:
* ``fun``: a Python function that takes Theano variables as inputs,
and returns a Theano variable.
For instance, an Op instance with a single output is such a function.
It can also be a Python function that calls an op with some of its
inputs being fixed to specific values, or that combine multiple ops.
* ``pt``: the list of numpy.ndarrays to use as input values
* ``n_tests``: number of times to run the test
* ``rng``: random number generator used to generate a random vector u,
we check the gradient of sum(u*fn) at pt
* ``eps``: stepsize used in the Finite Difference Method
* ``abs_tol``: absolute tolerance used as threshold for gradient comparison
* ``rel_tol``: relative tolerance used as threshold for gradient comparison
In the general case, you can define ``fun`` as you want, as long as it
takes as inputs Theano symbolic variables and returns a sinble Theano
symbolic variable:
.. testcode::
def test_verify_exprgrad():
def fun(x,y,z):
return (x + tensor.cos(y)) / (4 * z)**2
x_val = numpy.asarray([[1], [1.1], [1.2]])
y_val = numpy.asarray([0.1, 0.2])
z_val = numpy.asarray(2)
rng = numpy.random.RandomState(42)
tensor.verify_grad(fun, [x_val, y_val, z_val], rng=rng)
Here is an example showing how to use ``verify_grad`` on an Op instance:
.. testcode::
def test_flatten_outdimNone():
# Testing gradient w.r.t. all inputs of an op (in this example the op
# being used is Flatten(), which takes a single input).
a_val = numpy.asarray([[0,1,2],[3,4,5]], dtype='float64')
rng = numpy.random.RandomState(42)
tensor.verify_grad(tensor.Flatten(), [a_val], rng=rng)
Here is another example, showing how to verify the gradient w.r.t. a subset of
an Op's inputs. This is useful in particular when the gradient w.r.t. some of
the inputs cannot be computed by finite difference (e.g. for discrete inputs),
which would cause ``verify_grad`` to crash.
.. testcode::
def test_crossentropy_softmax_grad():
op = tensor.nnet.crossentropy_softmax_argmax_1hot_with_bias
def op_with_fixed_y_idx(x, b):
# Input `y_idx` of this Op takes integer values, so we fix them
# to some constant array.
# Although this op has multiple outputs, we can return only one.
# Here, we return the first output only.
return op(x, b, y_idx=numpy.asarray([0, 2]))[0]
x_val = numpy.asarray([[-1, 0, 1], [3, 2, 1]], dtype='float64')
b_val = numpy.asarray([1, 2, 3], dtype='float64')
rng = numpy.random.RandomState(42)
tensor.verify_grad(op_with_fixed_y_idx, [x_val, b_val], rng=rng)
.. note::
Although ``verify_grad`` is defined in ``theano.tensor.basic``, unittests
should use the version of ``verify_grad`` defined in ``theano.tests.unittest_tools``.
This is simply a wrapper function which takes care of seeding the random
number generator appropriately before calling ``theano.tensor.basic.verify_grad``
makeTester and makeBroadcastTester
==================================
Most Op unittests perform the same function. All such tests must
verify that the op generates the proper output, that the gradient is
valid, that the Op fails in known/expected ways. Because so much of
this is common, two helper functions exists to make your lives easier:
``makeTester`` and ``makeBroadcastTester`` (defined in module
``theano.tensor.tests.test_basic``).
Here is an example of ``makeTester`` generating testcases for the Dot
product op:
.. testcode::
from numpy import dot
from numpy.random import rand
from theano.tensor.tests.test_basic import makeTester
DotTester = makeTester(name = 'DotTester',
op = dot,
expected = lambda x, y: numpy.dot(x, y),
checks = {},
good = dict(correct1 = (rand(5, 7), rand(7, 5)),
correct2 = (rand(5, 7), rand(7, 9)),
correct3 = (rand(5, 7), rand(7))),
bad_build = dict(),
bad_runtime = dict(bad1 = (rand(5, 7), rand(5, 7)),
bad2 = (rand(5, 7), rand(8,3))),
grad = dict())
In the above example, we provide a name and a reference to the op we
want to test. We then provide in the ``expected`` field, a function
which ``makeTester`` can use to compute the correct values. The
following five parameters are dictionaries which contain:
* checks: dictionary of validation functions (dictionary key is a
description of what each function does). Each function accepts two
parameters and performs some sort of validation check on each
op-input/op-output value pairs. If the function returns False, an
Exception is raised containing the check's description.
* good: contains valid input values, for which the output should match
the expected output. Unittest will fail if this is not the case.
* bad_build: invalid parameters which should generate an Exception
when attempting to build the graph (call to ``make_node`` should
fail). Fails unless an Exception is raised.
* bad_runtime: invalid parameters which should generate an Exception
at runtime, when trying to compute the actual output values (call to
``perform`` should fail). Fails unless an Exception is raised.
* grad: dictionary containing input values which will be used in the
call to ``verify_grad``
``makeBroadcastTester`` is a wrapper function for makeTester. If an
``inplace=True`` parameter is passed to it, it will take care of
adding an entry to the ``checks`` dictionary. This check will ensure
that inputs and outputs are equal, after the Op's perform function has
been applied.
Theano-1.0.1/doc/extending/using_params.txt 0000777 0000000 0000000 00000016737 13212236347 020700 0 ustar root root 0000000 0000000 .. _extending_op_params:
===============
Using Op params
===============
The Op params is a facility to pass some runtime parameters to the
code of an op without modifying it. It can enable a single instance
of C code to serve different needs and therefore reduce compilation.
The code enables you to pass a single object, but it can be a struct
or python object with multiple values if you have more than one value
to pass.
We will first introduce the parts involved in actually using this
functionality and then present a simple working example.
The params type
----------------
You can either reuse an existing type such as :class:`Generic` or
create your own.
Using a python object for your op parameters (:class:`Generic`) can be
annoying to access from C code since you would have to go through the
Python-C API for all accesses.
Making a purpose-built class may require more upfront work, but can
pay off if you reuse the type for a lot of Ops, by not having to re-do
all of the python manipulation.
The params object
-----------------
The object that you use to store your param values must be hashable
and comparable for equality, because it will be stored in a dictionary
at some point. Apart from those requirements it can be anything that
matches what you have declared as the params type.
Defining a params type
~~~~~~~~~~~~~~~~~~~~~~
.. note::
This section is only relevant if you decide to create your own type.
The first thing you need to do is to define a Theano Type for your
params object. It doesn't have to be complete type because only the
following methods will be used for the type:
- :meth:`filter `
- :meth:`__eq__ `
- :meth:`__hash__ `
- :meth:`values_eq `
Additionaly if you want to use your params with C code, you need the
following methods:
- :meth:`c_declare `
- :meth:`c_init `
- :meth:`c_extract `
- :meth:`c_cleanup `
You can also define other convenience methods such as
:meth:`c_headers ` if you need any special things.
Registering the params with your Op
-----------------------------------
To declare that your Op uses params you have to set the class
attribute :attr:`params_type` to an instance of your params Type.
.. note::
If you want to have multiple parameters, Theano provides the convenient class
:class:`theano.gof.params_type.ParamsType` that allows to bundle many parameters into
one object that will be available in both Python (as a Python object) and C code (as a struct).
See :ref:`ParamsType tutorial and API documentation ` for more infos.
For example if we decide to use an int as the params the following
would be appropriate:
.. code-block:: python
class MyOp(Op):
params_type = Generic()
After that you need to define a :meth:`get_params` method on your
class with the following signature:
.. code-block:: python
def get_params(self, node)
This method must return a valid object for your Type (an object that
passes :meth:`filter`). The `node` parameter is the Apply node for
which we want the params. Therefore the params object can depend on
the inputs and outputs of the node.
.. note::
Due to implementation restrictions, None is not allowed as a
params object and will be taken to mean that the Op doesn't have
parameters.
Since this will change the expected signature of a few methods, it
is strongly discouraged to have your :meth:`get_params` method
return None.
Signature changes from having params
------------------------------------
Having declared a params for your Op will affect the expected
signature of :meth:`perform`. The new expected signature will have an
extra parameter at the end which corresponds to the params object.
.. warning::
If you do not account for this extra parameter, the code will fail
at runtime if it tries to run the python version.
Also, for the C code, the `sub` dictionary will contain an extra entry
`'params'` which will map to the variable name of the params object.
This is true for all methods that recieve a `sub` parameter, so this
means that you can use your params in the :meth:`c_code `
and :meth:`c_init_code_struct ` method.
A simple example
----------------
This is a simple example which uses a params object to pass a value.
This Op will multiply a scalar input by a fixed floating point value.
Since the value in this case is a python float, we chose Generic as
the params type.
.. testcode::
from theano import Op
from theano.gof.type import Generic
from theano.scalar import as_scalar
class MulOp(Op):
params_type = Generic()
__props__ = ('mul',)
def __init__(self, mul):
self.mul = float(mul)
def get_params(self, node):
return self.mul
def make_node(self, inp):
inp = as_scalar(inp)
return Apply(self, [inp], [inp.type()])
def perform(self, node, inputs, output_storage, params):
# Here params is a python float so this is ok
output_storage[0][0] = inputs[0] * params
def c_code(self, node, name, inputs, outputs, sub):
return ("%(z)s = %(x)s * PyFloat_AsDouble(%(p)s);" %
dict(z=outputs[0], x=inputs[0], p=sub['params']))
A more complex example
----------------------
This is a more complex example which actually passes multiple values.
It does a linear combination of two values using floating point
weights.
.. testcode::
from theano import Op
from theano.gof.type import Generic
from theano.scalar import as_scalar
class ab(object):
def __init__(self, alpha, beta):
self.alpha = alpha
self.beta = beta
def __hash__(self):
return hash((type(self), self.alpha, self.beta))
def __eq__(self, other):
return (type(self) == type(other) and
self.alpha == other.alpha and
self.beta == other.beta)
class Mix(Op):
params_type = Generic()
__props__ = ('alpha', 'beta')
def __init__(self, alpha, beta):
self.alpha = alpha
self.beta = beta
def get_params(self, node):
return ab(alpha=self.alpha, beta=self.beta)
def make_node(self, x, y):
x = as_scalar(x)
y = as_scalar(y)
return Apply(self, [x, y], [x.type()])
def c_support_code_struct(self, node, name):
return """
double alpha_%(name)s;
double beta_%(name)s;
""" % dict(name=name)
def c_init_code_struct(self, node, name, sub):
return """{
PyObject *tmp;
tmp = PyObject_GetAttrString(%(p)s, "alpha");
if (tmp == NULL)
%(fail)s
alpha_%(name)s = PyFloat_AsDouble(tmp);
Py_DECREF(%(tmp)s);
if (PyErr_Occurred())
%(fail)s
tmp = PyObject_GetAttrString(%(p)s, "beta");
if (tmp == NULL)
%(fail)s
beta_%(name)s = PyFloat_AsDouble(tmp);
Py_DECREF(tmp);
if (PyErr_Occurred())
%(fail)s
}""" % dict(name=name, p=sub['params'], fail=sub['fail'])
def c_code(self, node, name, inputs, outputs, sub):
return """
%(z)s = alpha_%(name)s * %(x)s + beta_%(name)s * %(y)s;
""" % dict(name=name, z=outputs[0], x=inputs[0], y=inputs[1])
Theano-1.0.1/doc/faq.txt 0000777 0000000 0000000 00000017111 13212236347 014755 0 ustar root root 0000000 0000000 :orphan:
.. _faq:
==========================
Frequently Asked Questions
==========================
Does Theano support Python 3?
------------------------------
We support both Python 2 >= 2.7 and Python 3 >= 3.4.
Output slight numerical difference
----------------------------------
Sometimes when you compare the output of Theano using different
Theano flags, Theano versions, CPU and GPU or with other software like
NumPy, you will see small numerical differences.
This is normal. Floating point numbers are approximations of real
numbers. This is why doing a+(b+c) vs (a+b)+c can give small
differences of value. This is normal. For more details, see: `What
Every Computer Scientist Should Know About Floating-Point Arithmetic
`_.
Faster gcc optimization
-----------------------
You can enable faster gcc optimization with the ``cxxflags`` option.
This list of flags was suggested on the mailing list::
-O3 -ffast-math -ftree-loop-distribution -funroll-loops -ftracer
Use it at your own risk. Some people warned that the ``-ftree-loop-distribution`` optimization resulted in wrong results in the past.
In the past we said that if the ``compiledir`` was not shared by multiple
computers, you could add the ``-march=native`` flag. Now we recommend
to remove this flag as Theano does it automatically and safely,
even if the ``compiledir`` is shared by multiple computers with different
CPUs. In fact, Theano asks g++ what are the equivalent flags it uses, and re-uses
them directly.
.. _faster-theano-function-compilation:
Faster Theano Function Compilation
----------------------------------
Theano function compilation can be time consuming. It can be sped up by setting
the flag ``mode=FAST_COMPILE`` which instructs Theano to skip most
optimizations and disables the generation of any c/cuda code. This is useful
for quickly testing a simple idea.
If c/cuda code is necessary, as when using a GPU, the flag
``optimizer=fast_compile`` can be used instead. It instructs Theano to
skip time consuming optimizations but still generate c/cuda code.
Similarly using the flag ``optimizer_excluding=inplace`` will speed up
compilation by preventing optimizations that replace operations with a
version that reuses memory where it will not negatively impact the
integrity of the operation. Such optimizations can be time
consuming. However using this flag will result in greater memory usage
because space must be allocated for the results which would be
unnecessary otherwise. In short, using this flag will speed up
compilation but it will also use more memory because
``optimizer_excluding=inplace`` excludes inplace optimizations
resulting in a trade off between speed of compilation and memory
usage.
Alternatively, if the graph is big, using the flag ``cycle_detection=fast``
will speedup the computations by removing some of the inplace
optimizations. This would allow theano to skip a time consuming cycle
detection algorithm. If the graph is big enough,we suggest that you use
this flag instead of ``optimizer_excluding=inplace``. It will result in a
computation time that is in between fast compile and fast run.
Theano flag `reoptimize_unpickled_function` controls if an unpickled
theano function should reoptimize its graph or not. Theano users can
use the standard python pickle tools to save a compiled theano
function. When pickling, both graph before and after the optimization
are saved, including shared variables. When set to True, the graph is
reoptimized when being unpickled. Otherwise, skip the graph
optimization and use directly the optimized graph from the pickled
file. The default is False.
Faster Theano function
----------------------
You can set the Theano flag :attr:`allow_gc ` to ``False`` to get a speed-up by using
more memory. By default, Theano frees intermediate results when we don't need
them anymore. Doing so prevents us from reusing this memory. So disabling the
garbage collection will keep all intermediate results' memory space to allow to
reuse them during the next call to the same Theano function, if they are of the
correct shape. The shape could change if the shapes of the inputs change.
.. note::
With :attr:`CNMeM `, this isn't very useful with GPU
anymore.
.. _unsafe_optimization:
Unsafe optimization
===================
Some Theano optimizations make the assumption that the user inputs are
valid. What this means is that if the user provides invalid values (like
incompatible shapes or indexing values that are out of bounds) and
the optimizations are applied, the user error will get lost. Most of the
time, the assumption is that the user inputs are valid. So it is good
to have the optimization being applied, but loosing the error is bad.
The newest optimization in Theano with such assumption will add an
assertion in the graph to keep the user error message. Computing
these assertions could take some time. If you are sure everything is valid
in your graph and want the fastest possible Theano, you can enable an
optimization that will remove those assertions with:
``optimizer_including=local_remove_all_assert``
Faster Small Theano function
----------------------------
.. note::
For Theano 0.6 and up.
For Theano functions that don't do much work, like a regular logistic
regression, the overhead of checking the input can be significant. You
can disable it by setting ``f.trust_input`` to True.
Make sure the types of arguments you provide match those defined when
the function was compiled.
For example, replace the following
.. testcode:: faster
import theano
from theano import function
x = theano.tensor.scalar('x')
f = function([x], x + 1.)
f(10.)
with
.. testcode:: faster
import numpy
import theano
from theano import function
x = theano.tensor.scalar('x')
f = function([x], x + 1.)
f.trust_input = True
f(numpy.array([10.], dtype=theano.config.floatX))
Also, for small Theano functions, you can remove more Python overhead by
making a Theano function that does not take any input. You can use shared
variables to achieve this. Then you can call it like this: ``f.fn()`` or
``f.fn(n_calls=N)`` to speed it up. In the last case, only the last
function output (out of N calls) is returned.
You can also use the ``C`` linker that will put all nodes in the same C
compilation unit. This removes some overhead between node in the graph,
but requires that all nodes in the graph have a C implementation:
.. code-block:: python
x = theano.tensor.scalar('x')
f = function([x], (x + 1.) * 2, mode=theano.Mode(linker='c'))
f(10.)
New GPU backend using libgpuarray
---------------------------------
The new theano GPU backend (:ref:`gpuarray`) uses ``config.gpuarray.preallocate`` for GPU memory allocation.
Likewise, the old back-end uses ``config.lib.cnmem`` for GPU memory allocation.
Related Projects
----------------
We try to list in this `wiki page `_ other Theano related projects.
"What are Theano's Limitations?"
--------------------------------
Theano offers a good amount of flexibility, but has some limitations too.
You must answer for yourself the following question: How can my algorithm be cleverly written
so as to make the most of what Theano can do?
Here is a list of some of the known limitations:
- *While*- or *for*-Loops within an expression graph are supported, but only via
the :func:`theano.scan` op (which puts restrictions on how the loop body can
interact with the rest of the graph).
- Neither *goto* nor *recursion* is supported or planned within expression graphs.
Theano-1.0.1/doc/glossary.txt 0000777 0000000 0000000 00000015437 13212236347 016062 0 ustar root root 0000000 0000000 .. _glossary:
Glossary
========
.. testsetup::
import theano
from theano import tensor
.. glossary::
Apply
Instances of :class:`Apply` represent the application of an :term:`Op`
to some input :term:`Variable` (or variables) to produce some output
:term:`Variable` (or variables). They are like the application of a [symbolic]
mathematical function to some [symbolic] inputs.
Broadcasting
Broadcasting is a mechanism which allows tensors with
different numbers of dimensions to be used in element-by-element
(elementwise) computations. It works by
(virtually) replicating the smaller tensor along
the dimensions that it is lacking.
For more detail, see :ref:`tutbroadcasting`, and also
* `SciPy documentation about numpy's broadcasting `_
* `OnLamp article about numpy's broadcasting `_
Constant
A variable with an immutable value.
For example, when you type
>>> x = tensor.ivector()
>>> y = x + 3
Then a `constant` is created to represent the ``3`` in the graph.
See also: :class:`gof.Constant`
Elementwise
An elementwise operation ``f`` on two tensor variables ``M`` and ``N``
is one such that:
``f(M, N)[i, j] == f(M[i, j], N[i, j])``
In other words, each element of an input matrix is combined
with the corresponding element of the other(s). There are no
dependencies between elements whose ``[i, j]`` coordinates do
not correspond, so an elementwise operation is like a scalar
operation generalized along several dimensions. Elementwise
operations are defined for tensors of different numbers of dimensions by
:term:`broadcasting` the smaller ones.
Expression
See :term:`Apply`
Expression Graph
A directed, acyclic set of connected :term:`Variable` and
:term:`Apply` nodes that express symbolic functional relationship
between variables. You use Theano by defining expression graphs, and
then compiling them with :term:`theano.function`.
See also :term:`Variable`, :term:`Op`, :term:`Apply`, and
:term:`Type`, or read more about :ref:`graphstructures`.
Destructive
An :term:`Op` is destructive (of particular input[s]) if its
computation requires that one or more inputs be overwritten or
otherwise invalidated. For example, :term:`inplace` Ops are
destructive. Destructive Ops can sometimes be faster than
non-destructive alternatives. Theano encourages users not to put
destructive Ops into graphs that are given to :term:`theano.function`,
but instead to trust the optimizations to insert destructive ops
judiciously.
Destructive Ops are indicated via a ``destroy_map`` Op attribute. (See
:class:`gof.Op`.
Graph
see :term:`expression graph`
Inplace
Inplace computations are computations that destroy their inputs as a
side-effect. For example, if you iterate over a matrix and double
every element, this is an inplace operation because when you are done,
the original input has been overwritten. Ops representing inplace
computations are :term:`destructive`, and by default these can only be
inserted by optimizations, not user code.
Linker
Part of a function :term:`Mode` -- an object responsible for 'running'
the compiled function. Among other things, the linker determines whether computations are carried out with C or Python code.
Mode
An object providing an :term:`optimizer` and a :term:`linker` that is
passed to :term:`theano.function`. It parametrizes how an expression
graph is converted to a callable object.
Op
The ``.op`` of an :term:`Apply`, together with its symbolic inputs
fully determines what kind of computation will be carried out for that
``Apply`` at run-time. Mathematical functions such as addition
(``T.add``) and indexing ``x[i]`` are Ops in Theano. Much of the
library documentation is devoted to describing the various Ops that
are provided with Theano, but you can add more.
See also :term:`Variable`, :term:`Type`, and :term:`Apply`,
or read more about :ref:`graphstructures`.
Optimizer
An instance of :class:`Optimizer`, which has the capacity to provide
an :term:`optimization` (or optimizations).
Optimization
A :term:`graph` transformation applied by an :term:`optimizer` during
the compilation of a :term:`graph` by :term:`theano.function`.
Pure
An :term:`Op` is *pure* if it has no :term:`destructive` side-effects.
Storage
The memory that is used to store the value of a Variable. In most
cases storage is internal to a compiled function, but in some cases
(such as :term:`constant` and :term:`shared variable ` the storage is not internal.
Shared Variable
A :term:`Variable` whose value may be shared between multiple functions. See :func:`shared ` and :func:`theano.function `.
theano.function
The interface for Theano's compilation from symbolic expression graphs
to callable objects. See :func:`function.function`.
Type
The ``.type`` of a
:term:`Variable` indicates what kinds of values might be computed for it in a
compiled graph.
An instance that inherits from :class:`Type`, and is used as the
``.type`` attribute of a :term:`Variable`.
See also :term:`Variable`, :term:`Op`, and :term:`Apply`,
or read more about :ref:`graphstructures`.
Variable
The the main data structure you work with when using Theano.
For example,
>>> x = theano.tensor.ivector()
>>> y = -x**2
``x`` and ``y`` are both `Variables`, i.e. instances of the :class:`Variable` class.
See also :term:`Type`, :term:`Op`, and :term:`Apply`,
or read more about :ref:`graphstructures`.
View
Some Tensor Ops (such as Subtensor and Transpose) can be computed in
constant time by simply re-indexing their inputs. The outputs from
[the Apply instances from] such Ops are called `Views` because their
storage might be aliased to the storage of other variables (the inputs
of the Apply). It is important for Theano to know which Variables are
views of which other ones in order to introduce :term:`Destructive`
Ops correctly.
View Ops are indicated via a ``view_map`` Op attribute. (See
:class:`gof.Op`.
Theano-1.0.1/doc/index.txt 0000777 0000000 0000000 00000031372 13212236347 015322 0 ustar root root 0000000 0000000
Welcome
=======
Theano is a Python library that allows you to define, optimize, and
evaluate mathematical expressions involving multi-dimensional
arrays efficiently. Theano features:
* **tight integration with NumPy** -- Use `numpy.ndarray` in Theano-compiled functions.
* **transparent use of a GPU** -- Perform data-intensive computations much faster than on a CPU.
* **efficient symbolic differentiation** -- Theano does your derivatives for functions with one or many inputs.
* **speed and stability optimizations** -- Get the right answer for ``log(1+x)`` even when ``x`` is really tiny.
* **dynamic C code generation** -- Evaluate expressions faster.
* **extensive unit-testing and self-verification** -- Detect and diagnose many types of errors.
Theano has been powering large-scale computationally intensive
scientific investigations since 2007. But it is also approachable
enough to be used in the classroom (University of Montreal's `deep
learning/machine learning `_ and the next release (renamed to 1.0) will be the last main release.
* 2017/09/20: Release of Theano 0.10.0beta3, new features and many bugfixes, release candidate to coming.
* 2017/09/07: Release of Theano 0.10.0beta2, new features and many bugfixes, release candidate to coming.
* 2017/08/09: Release of Theano 0.10.0beta1, many improvements and bugfixes, release candidate to coming.
* Removed support for the old (device=gpu) backend. Use the new
backend (device=cuda) for gpu computing. See `Converting to the new
gpu back end(gpuarray)
`_
for help with conversion.
* 2017/03/20: Release of Theano 0.9.0. Everybody is encouraged to update.
* 2017/03/13: Release of Theano 0.9.0rc4, with crash fixes and bug fixes.
* 2017/03/06: Release of Theano 0.9.0rc3, with crash fixes, bug fixes and improvements.
* 2017/02/27: Release of Theano 0.9.0rc2, with crash fixes, bug fixes and improvements.
* 2017/02/20: Release of Theano 0.9.0rc1, many improvements and bugfixes, final release to coming.
* 2017/01/24: Release of Theano 0.9.0beta1, many improvements and bugfixes, release candidate to coming.
* 2016/05/09: New technical report on Theano:
`Theano: A Python framework for fast computation of mathematical expressions `_.
This is the new preferred reference.
* 2016/04/21: Release of Theano 0.8.2, adding support for CuDNN v5.
* 2016/03/29: Release of Theano 0.8.1, fixing a compilation issue on MacOS X with XCode 7.3.
* 2016/03/21: Release of Theano 0.8. Everybody is encouraged to update.
* Multi-GPU.
* We added support for CNMeM to speed up the GPU memory allocation.
* Theano 0.7 was released 26th March 2015. Everybody is encouraged to update.
* We support cuDNN if it is installed by the user.
* Open Machine Learning Workshop 2014 :download:`presentation `.
* Colin Raffel `tutorial on Theano `_.
* Ian Goodfellow did a `12h class with exercises on Theano `_.
* New technical report on Theano: `Theano: new features and speed improvements `_.
* `HPCS 2011 Tutorial `_.
We included a few fixes discovered while doing the Tutorial.
.. image:: images/talk2010.png
:scale: 75%
:align: left
You can watch a quick (20 minute) introduction to Theano given as a talk at `SciPy 2010 `.
.. only:: html
Status
======
.. raw:: html
`_.
(:download:`short BibTeX `, :download:`full BibTeX `)
Theano is primarily developed by academics, and so citations matter a lot to
us. As an added benefit, you increase Theano's exposure and potential user
(and developer) base, which is to the benefit of all users of Theano. Thanks
in advance!
See our :ref:`citation` for details.
Documentation
=============
Roughly in order of what you'll want to check out:
* :ref:`install` -- How to install Theano.
* :ref:`introduction` -- What is Theano?
* :ref:`tutorial` -- Learn the basics.
* :ref:`troubleshooting` -- Tips and tricks for common debugging.
* :ref:`libdoc` -- Theano's functionality, module by module.
* :ref:`faq` -- A set of commonly asked questions.
* :ref:`optimizations` -- Guide to Theano's graph optimizations.
* :ref:`extending` -- Learn to add a Type, Op, or graph optimization.
* :ref:`dev_start_guide` -- How to contribute code to Theano.
* :ref:`internal` -- How to maintain Theano and more...
* :ref:`release` -- How our release should work.
* :ref:`acknowledgement` -- What we took from other projects.
* `Related Projects`_ -- link to other projects that implement new functionalities on top of Theano
You can download the latest `PDF documentation `_, rather than reading it online.
Check out how Theano can be used for Machine Learning: `Deep Learning Tutorials `_.
Theano was featured at `SciPy 2010 `__.
.. _theano_community:
Community
=========
"Thank YOU for correcting it so quickly. I wish all packages I worked
with would have such an active maintenance - this is as good as it
gets :-)"
(theano-users, Aug 2, 2010)
* Register to `theano-announce`_ if you want to be kept informed on important change on theano(low volume).
* Register and post to `theano-users`_ if you want to talk to all Theano users.
* Register and post to `theano-dev`_ if you want to talk to the developers.
* Register to `theano-github`_ if you want to receive an email for all changes to the GitHub repository.
* Register to `theano-buildbot`_ if you want to receive our daily buildbot email.
* Ask/view questions/answers at `StackOverflow`_
* We use `Github tickets `__ to keep track of issues
(however, some old tickets can still be found on
`Assembla `__).
* Come visit us in Montreal! Most developers are students in the LISA_ group at the `University of Montreal`_.
.. _support:
Help!
=====
How to Seek Help
----------------
The appropriate venue for seeking help depends on the kind of question you have.
* How do I? -- `theano-users`_ mailing list or `StackOverflow`_
* I got this error, why? -- `theano-users`_ mailing list or `StackOverflow`_ (please include the *full* error message, even if it's long)
* I got this error and I'm sure it's a bug -- `Github ticket `__
* I have an idea/request -- post the suggestion to `theano-dev`_ or, even better, implement the idea and submit a `GitHub `__ pull request!
* Why do you? -- `theano-users`_ mailing list (not appropriate for StackOverflow)
* When will you? -- `theano-dev`_ mailing list (not appropriate for StackOverflow)
Please do take some time to search for similar questions that were asked and answered in the past. If you find something
similar that doesn't fully answer your question, it can be helpful to say something like "I found X but it doesn't
address facet Y" and link to the previous discussion.
When asking questions on StackOverflow, please use the *theano* tag, so your question can be found, and follow
StackOverflow's guidance on `asking questions `__. Consider also using the
*python* and *numpy* tags, especially if you are unsure which library your problem relates to.
It's often helpful to include the following details with your question:
* If you have an error, the *full* error message, even if it's long
* Which versions of Python and Theano you're using
* Whether you're using a CPU or GPU device
* Details of your Theano configuration settings (you can print this in Python via `print theano.config `__)
Spending the time to create a minimal specific example of a problem is likely to get you to an answer quicker than
posting something quickly that has too much irrelevant detail or is too vague. A minimal example may take you a bit more
time to create but the first response is more likely to be the answer you need than, rather than a frustrated request
for clarification.
How to provide help
-------------------
If you see a question on the `theano-users`_ mailing list, or on `StackOverflow`_, that you
feel reasonably confident you know an answer to, please do support the community by helping others.
We were all newbies to Theano once and, as the community expands, there is a constant stream of new Theano users looking
for help. Perhaps you asked a question when you were first starting out? Now you can pay it forward by helping others.
It's also a good way to reinforce your own Theano knowledge.
Often it's easiest to answer a question directly but sometimes it may be better to refer people to a good answer that
was provided in the past. Pointing people to relevant sections in the documentation, or to a Theano tutorial, can also
be helpful.
When answering questions please `be nice `__ (as always!) and, on
StackOverflow, follow their guidance for `answering questions `__.
.. toctree::
:maxdepth: 1
:hidden:
NEWS
introduction
requirements
install
updating
tutorial/index
extending/index
dev_start_guide
optimizations
library/index
troubleshooting
glossary
links
internal/index
acknowledgement
LICENSE
.. _theano-dev: http://groups.google.com/group/theano-dev
.. _theano-users: http://groups.google.com/group/theano-users
.. _theano-announce: http://groups.google.com/group/theano-announce
.. _theano-github: http://groups.google.com/group/theano-github
.. _theano-buildbot: http://groups.google.com/group/theano-buildbot
.. _tickets: http://pylearn.org/theano/trac/query?status=accepted&status=assigned&status=new&status=reopened&group=milestone&max=200&col=id&col=summary&col=status&col=owner&col=type&col=priority&col=component&col=time&report=9&order=priority
.. _StackOverflow: http://stackoverflow.com/questions/tagged/theano
.. _LISA: http://www.iro.umontreal.ca/~lisa
.. _University of Montreal: http://www.umontreal.ca
Theano-1.0.1/doc/install.txt 0000777 0000000 0000000 00000000621 13212236347 015652 0 ustar root root 0000000 0000000 .. _install:
Installing Theano
=================
Supported platforms:
.. toctree::
:maxdepth: 1
install_ubuntu
install_macos
install_windows
install_centos6
install_others
Once your setup is complete and if you installed the GPU libraries, head to :ref:`testing_the_gpu` to find how to verify
everything is working properly.
To update your current installation see :ref:`updating`.
Theano-1.0.1/doc/install_centos6.txt 0000777 0000000 0000000 00000001741 13212236347 017317 0 ustar root root 0000000 0000000 .. include:: css.inc
.. _install_centos6:
CentOS 6 Installation Instructions
##################################
.. warning::
If you want to install the bleeding-edge or development version of Theano
from GitHub, please make sure you are reading `the latest version of this
page `_.
.. |PythonDistRecommended| replace:: The development package (python-dev or python-devel on most Linux distributions) is recommended (see just below)
.. |PlatformCompiler| replace:: ``python-dev``, ``g++`` >= 4.2
.. |CompilerName| replace:: ``g++``
.. include:: requirements.inc
.. include:: install_generic.inc
:start-line: 5
Requirements through System Packages (not recommended)
------------------------------------------------------
.. code-block:: bash
sudo yum install python-devel python-nose python-setuptools gcc gcc-gfortran gcc-c++ blas-devel lapack-devel atlas-devel
sudo easy_install pip
Theano-1.0.1/doc/install_macos.txt 0000777 0000000 0000000 00000014513 13212236347 017041 0 ustar root root 0000000 0000000 .. include:: css.inc
.. _install_macos:
Mac OS Installation Instructions
################################
.. warning::
If you want to install the bleeding-edge or development version of Theano
from GitHub, please make sure you are reading `the latest version of this
page `_.
There are various ways to install Theano dependencies on a Mac. Here
we describe the process in detail with Anaconda, Homebrew or MacPorts
but if you did it differently and it worked, please let us know the
details on the `theano-users`_ mailing-list, so that we can add
alternative instructions here.
.. _theano-users: http://groups.google.com/group/theano-users?pli=1
.. |PythonDistRecommended| replace:: The conda distribution is highly recommended
.. |PlatformCompiler| replace:: ``clang`` (the system version)
.. |CompilerName| replace:: ``Clang``
.. include:: requirements.inc
.. _gpu_macos:
.. attention::
For MacOS you should be able to follow the above instructions to
setup CUDA, but be aware of the following caveats:
* If you want to compile the CUDA SDK code, you may need to temporarily
revert back to Apple's gcc (``sudo port select gcc``) as their Makefiles
are not compatible with MacPort's gcc.
* If CUDA seems unable to find a CUDA-capable GPU, you may need to manually
toggle your GPU on, which can be done with
`gfxCardStatus `__.
.. attention::
Theano officially supports only clang on OS X. This can be installed
by getting XCode from the App Store and running it once to install the
command-line tools.
.. include:: install_generic.inc
:start-line: 5
Requirements through Homebrew (not recommended)
-----------------------------------------------
Install python with homebrew:
.. code-block:: bash
$ brew install python # or python3 if you prefer
This will install pip. Then use pip to install numpy, scipy:
.. code-block:: bash
$ pip install numpy scipy
If you want to use openblas instead of Accelerate, you have to install
numpy and scipy with hombrew:
.. code-block:: bash
$ brew tap homebrew/python
$ brew install numpy --with-openblas
$ brew install scipy --with-openblas
Requirements through MacPorts (not recommended)
-----------------------------------------------
Using `MacPorts `_,
focusing on how to install the appropriate dependencies.
Nicolas Pinto provides `ebuild scripts `_.
AWS Marketplace with Bitfusion AMI
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
AWS EC2 AMI pre-installed with Nvidia drivers, CUDA, cuDNN, Theano, Keras, Lasagne, Python 2, Python 3, PyCuda, Scikit-Learn, Pandas, Enum34, iPython, and Jupyter. Note, as always there is no charge for Theano and other open software, however there is a charge for AWS hosting + Bitfusion.
`Launch `_ an instance from the AWS Marketplace.
Docker
~~~~~~
Builds of Theano are available as `Docker `_
images: `Theano Docker (CPU) `_.
CUDA support requires `NVIDIA Docker `_.
For more details on how to use the Theano Docker images,
consult the `source project `_.
Theano-1.0.1/doc/install_ubuntu.txt 0000777 0000000 0000000 00000006237 13212236347 017265 0 ustar root root 0000000 0000000 .. include:: css.inc
.. _install_ubuntu:
Ubuntu Installation Instructions
################################
.. warning::
If you want to install the bleeding-edge or development version of Theano
from GitHub, please make sure you are reading `the latest version of this
page `_.
.. _gpu_linux:
.. |PythonDistRecommended| replace:: The development package (python-dev or python-devel on most Linux distributions) is recommended (see just below)
.. |PlatformCompiler| replace:: ``python-dev``, ``g++`` >= 4.2
.. |CompilerName| replace:: ``g++``
.. include:: requirements.inc
.. include:: install_generic.inc
:start-line: 5
Prerequisites through System Packages (not recommended)
-------------------------------------------------------
If you want to acquire the requirements through your system packages
and install them system wide follow these instructions:
For Ubuntu 16.04 with cuda 7.5
.. code-block:: bash
sudo apt-get install python-numpy python-scipy python-dev python-pip python-nose g++ libopenblas-dev git graphviz
sudo pip install Theano
# cuda 7.5 don't support the default g++ version. Install an supported version and make it the default.
sudo apt-get install g++-4.9
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.9 20
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 10
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.9 20
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 10
sudo update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30
sudo update-alternatives --set cc /usr/bin/gcc
sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30
sudo update-alternatives --set c++ /usr/bin/g++
For Ubuntu 11.10 through 14.04:
.. code-block:: bash
sudo apt-get install python-numpy python-scipy python-dev python-pip python-nose g++ libopenblas-dev git
On 14.04, this will install Python 2 by default. If you want to use Python 3:
.. code-block:: bash
sudo apt-get install python3-numpy python3-scipy python3-dev python3-pip python3-nose g++ libopenblas-dev git
sudo pip3 install Theano
For Ubuntu 11.04:
.. code-block:: bash
sudo apt-get install python-numpy python-scipy python-dev python-pip python-nose g++ git libatlas3gf-base libatlas-dev
Manual Openblas installation (deprecated)
-----------------------------------------
The openblas included in some older Ubuntu version is limited to 2
threads. Ubuntu 14.04 do not have this limit. If you want to use more
cores at the same time, you will need to compile it yourself. Here is
some code that will help you.
.. code-block:: bash
# remove openblas if you installed it
sudo apt-get remove libopenblas-base
# Download the development version of OpenBLAS
git clone git://github.com/xianyi/OpenBLAS
cd OpenBLAS
make FC=gfortran
sudo make PREFIX=/usr/local/ install
# Tell Theano to use OpenBLAS.
# This works only for the current user.
# Each Theano user on that computer should run that line.
echo -e "\n[blas]\nldflags = -lopenblas\n" >> ~/.theanorc
Theano-1.0.1/doc/install_windows.txt 0000777 0000000 0000000 00000006635 13212236347 017437 0 ustar root root 0000000 0000000 .. include:: css.inc
.. _install_windows:
Windows Installation Instructions
#################################
.. warning::
If you want to install the bleeding-edge or development version of Theano
from GitHub, please make sure you are reading `the latest version of this
page `_.
.. |PythonDistRecommended| replace:: The conda distribution is highly recommended
.. |PlatformCompiler| replace:: GCC compiler with ``g++`` (version >= ``4.2.*``), and Python development files
.. |CompilerName| replace:: ``g++``
.. List of requirements, optional requirements, and installation of miniconda.
.. include:: requirements.inc
:end-before: .. install_requirements_and_optional_packages
Install requirements and optional packages
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. code-block:: bash
conda install numpy scipy mkl-service libpython
.. note::
* Arguments between <...> are optional.
* ``m2w64-toolchain`` package provides a fully-compatible version of GCC and is then highly recommended.
* ``git`` package installs git source control through conda, which is required for the development versions of Theano and libgpuarray
Package ``parameterized`` is also optional but may be required for unit testing. It is available via ``pip``.
.. code-block:: bash
pip install parameterized
.. _gpu_windows:
Install and configure the GPU drivers (recommended)
---------------------------------------------------
.. warning::
OpenCL support is still minimal for now.
Install CUDA drivers
^^^^^^^^^^^^^^^^^^^^
Follow `this link `__
to install the CUDA driver and the CUDA Toolkit.
You must reboot the computer after the driver installation.
.. Installation of Theano and libgpuarray.
.. include:: install_generic.inc
:start-after: .. _install_generic:
Instructions for other Python distributions (not recommended)
=============================================================
If you plan to use Theano with other Python distributions, these are
generic guidelines to get a working environment:
* Look for the mandatory requirements in the package manager's repositories of your distribution. Many
distributions come with ``pip`` package manager which use `PyPI repository `__.
The required modules are Python (of course), NumPy, SciPy and a BLAS implementation (MKL or OpenBLAS).
Use the versions recommended at the top of this documentation.
* If the package manager provide a GCC compiler with the recommended version (see at top), install it. If not,
you could use the build `TDM GCC `__.
Theano-1.0.1/doc/internal/ 0000777 0000000 0000000 00000000000 13212236507 015253 5 ustar root root 0000000 0000000 Theano-1.0.1/doc/internal/dev_start_guide.txt 0000777 0000000 0000000 00000000255 13212236347 021173 0 ustar root root 0000000 0000000 .. _old_dev_start_guide:
============================
Developer Start Guide MOVED!
============================
The developer start guide :ref:`moved `.
Theano-1.0.1/doc/internal/how_to_release.txt 0000777 0000000 0000000 00000012046 13212236347 021023 0 ustar root root 0000000 0000000 .. _how_to_release:
==================================================
How to make a release
==================================================
Update files
============
Update the NEWS.txt and move the old stuff in the HISTORY.txt file.
To update the NEWS.txt file, check all ticket closed for this release
and all commit log messages. Update the ``Theano/doc/index.txt`` *News* section.
Update the "Vision"/"Vision State" in the file
``Theano/doc/introduction.txt``.
Update the file ``.mailmap`` to clean up the list of contributor.
Get a fresh copy of the repository
==================================
Clone the code::
git clone git@github.com:Theano/Theano.git Theano-0.X
It does not have to be in your PYTHONPATH.
Update the version number
=========================
``Theano/doc/conf.py`` should be updated in the following ways:
* Change the upper copyright year to the current year if necessary.
Update the year in the ``Theano/LICENSE.txt`` file too, if necessary.
``NEWS.txt`` usually contains the name and date of the release, change them
too.
Update the fallback version in ``theano/version.py``.
Update the code and the documentation for the theano flags
``warn.ignore_bug_before`` to accept the new version. You must modify the
file ``theano/configdefaults.py`` and ``doc/library/config.txt``.
Tag the release
===============
You will need to commit the previous changes, tag the resulting version, and
push that into the original repository. The syntax is something like the
following::
git commit -m "Modifications for 0.X.Y release" setup.py doc/conf.py NEWS.txt HISTORY.txt theano/configdefaults.py doc/library/config.txt
git tag -a rel-0.X.Y
git push
git push --tags
This will trigger and build and upload of the conda package to the
mila-udem channel.
The documentation will be automatically regenerated in the next few hours.
Generate and upload the package
===============================
On PyPI
-------
Set your umask to ``0022`` to ensure that the package file will be readable from other people.
To check your umask::
umask
To set your umask::
umask 0022
Now change ``ISRELEASED`` in ``setup.py`` to ``True``.
Finally, use setuptools to register and upload the release::
python setup.py register sdist --formats=gztar,zip upload
This command register and uploads the package on pypi.python.org. To be able
to do that, you must register on PyPI (you can create an new account, or use
OpenID), and be listed among the "Package Index Owners" of Theano.
There is a bug in some versions of distutils that raises a
UnicodeDecodeError if there are non-ASCII characters in NEWS.txt. You
would need to change NEWS.txt so it contains only ASCII characters (the
problem usually comes from diacritics in people's names).
On mloss.org (for final releases only)
--------------------------------------
Project page is at http://mloss.org/software/view/241/.
Account jaberg is listed as submitter.
1. log in as jaberg to mloss
2. search for theano and click the logo
3. press 'update this project' on the left and change
- the version number
- the download link
- the description of what has changed
4. press save
Make sure the "what's changed" text isn't too long because it will show up on
the front page of mloss. You have to indent bullet lines by 4 spaces I think in
the description.
You can "update this project" and save lots of times to get the revision text
right. Just do not change the version number.
Finally
-------
Change ``ISRELEASED`` back to ``False``.
Update documentation server scripts
===================================
The documentation server runs the auto-generation script regularly. It
compiles the latest development version and puts it in
``$webroot/theano_versions/dev/``. It then checks if the release branch
has been updated and if it has, the release documentation is updated and
put into ``$webroot/theano/``. Finally, it checks for archived versions in
``$webroot/theano_versions/`` and generates a ``versions.json`` file
on the server that is used to populate the version switcher.
If the release branch has changed, you must update the web server script.
Login to the ``deeplearning.net`` server as the user in charge of
document generation. In the shell script ``~/bin/updatedocs``, update the
variable ``release`` to the branch name for the current release.
You can also add previous releases to the versions documentation archive.
In the script ``~/bin/updatedocs_versions``, change the variable
``Versions`` to the git tag of the documentation version to generate,
then run the script.
Announce the release
====================
Generate an e-mail from the template in ``EMAIL.txt``, including content
from ``NEWS.txt``.
For final releases, send the e-mail to the following mailing lists:
* theano-users
* theano-announce
* numpy-discussion@scipy.org
* scipy-user@python.org
* G+, Scientific Python: https://plus.google.com/communities/108773711053400791849
For release candidates, only e-mail:
* theano-announce
* theano-dev
* theano-users
For alpha and beta releases, only e-mail:
* theano-dev
* theano-users
Theano-1.0.1/doc/internal/index.txt 0000777 0000000 0000000 00000000303 13212236347 017124 0 ustar root root 0000000 0000000
.. _internal:
======================
Internal Documentation
======================
.. toctree::
:maxdepth: 2
release
dev_start_guide
metadocumentation
python
how_to_release
Theano-1.0.1/doc/internal/metadocumentation.txt 0000777 0000000 0000000 00000010543 13212236347 021544 0 ustar root root 0000000 0000000 .. _metadocumentation:
==================================================
Documentation Documentation AKA Meta-Documentation
==================================================
How to build documentation
--------------------------
Let's say you are writing documentation, and want to see the `sphinx
: output the html files in the specified dir
--rst: only compile the doc (requires sphinx)
--nopdf: do not produce a PDF file from the doc, only HTML
--help: this help
Use ReST for documentation
--------------------------
* `ReST `__ is standardized.
trac wiki-markup is not.
This means that ReST can be cut-and-pasted between code, other
docs, and TRAC. This is a huge win!
* ReST is extensible: we can write our own roles and directives to automatically link to WIKI, for example.
* ReST has figure and table directives, and can be converted (using a standard tool) to latex documents.
* No text documentation has good support for math rendering, but ReST is closest: it has three renderer-specific solutions (render latex, use latex to build images for html, use itex2mml to generate MathML)
How to link to class/function documentations
--------------------------------------------
Link to the generated doc of a function this way::
:func:`perform`
For example::
of the :func:`perform` function.
Link to the generated doc of a class this way::
:class:`RopLop_checker`
For example::
The class :class:`RopLop_checker`, give the functions
However, if the link target is ambiguous, Sphinx will generate warning or errors.
How to add TODO comments in Sphinx documentation
-------------------------------------------------
To include a TODO comment in Sphinx documentation, use an indented block as
follows::
.. TODO: This is a comment.
.. You have to put .. at the beginning of every line :(
.. These lines should all be indented.
It will not appear in the output generated.
.. TODO: Check it out, this won't appear.
.. Nor will this.
How documentation is built on deeplearning.net
----------------------------------------------
The server that hosts the theano documentation runs a cron job roughly every
2 hours that fetches a fresh Theano install (clone, not just pull) and
executes the docgen.py script. It then over-writes the previous docs with the
newly generated ones.
Note that the server will most definitely use a different version of sphinx
than yours so formatting could be slightly off, or even wrong. If you're
getting unxpected results and/or the auto-build of the documentation seems
broken, please contact theano-dev@.
In the future, we might go back to the system of auto-refresh on push (though
that might increase the load of the server quite significantly).
pylint
---------------------------------------
pylint output is not autogenerated anymore.
Pylint documentation is generated using pylintrc file: ``Theano/doc/pylintrc``
.. _metadocumentation_nightly_build:
The nightly build/tests process
---------------------------------------
We use the Jenkins software to run daily buildbots for Theano, libgpuarray and
the Deep Learning Tutorials. Jenkins downloads/updates the repos and then runs their test
scripts. Those scripts test the projects under various condition.
Jenkins also run some tests in 32 bit Python 2.7 and Python 3.4 for Theano.
The output is emailed automatically to the `theano-buildbot`_ mailing list. The
jenkins log and test reports are published online:
* `Theano buildbot `_ will
give you a warm feeling in your stomach.
Non-Basic Python features
-------------------------
Theano doesn't use your grandfather's python.
* properties
a specific attribute that has get and set methods which python automatically invokes.
See [http://www.python.org/doc/newstyle/ New style classes].
* static methods vs. class methods vs. instance methods
* Decorators:
.. code-block:: python
@f
def g():
...
runs function ``f`` before each invocation of ``g``.
See `PEP 0318 .py directly, then during the test
cases, all .py classes instantiated by unit tests would have type
``__main__.``, instead of type ``.``. This should never
happen under normal usage, and can cause problems (like the one you are/were
experiencing).
Theano-1.0.1/doc/internal/release.txt 0000777 0000000 0000000 00000005730 13212236347 017446 0 ustar root root 0000000 0000000 .. _release:
=======
Release
=======
Having a release system has many benefits. First and foremost, it makes trying
out Theano easy. You can install a stable version of Theano, without having to
worry about the current state of the repository. While we usually try NOT to
break the trunk, mistakes can happen. This also greatly simplifies the
installation process: mercurial is no longer required and certain python
dependencies can be handled automatically (numpy for now, cython later).
The Theano release plan is detailed below. Comments and/or suggestions are
welcome on the mailing list.
1) We aim to update Theano several times a year. These releases will be made
as new features are implemented.
2) Urgent releases will only be made when a bug generating incorrect
output is discovered and fixed.
3) Each release must satisfy the following criteria. Non-compliance will
result in us delaying or skipping the release in question.
1) No regression errors.
2) No known, silent errors.
3) No errors giving incorrect results.
4) No test errors/failures, except for known errors.
1) Known errors should not be used to encode "feature wish lists", as
is currently the case.
2) Incorrect results should raise errors and not known errors (this
has always been the case)
3) All known errors should have a ticket and a reference to that
ticket in the error message.
5) All commits should have been reviewed, to ensure none of the above
problems are introduced.
4) The release numbers will follow the X.Y.Z scheme:
1) We update Z for small urgent bugs or support for new versions of dependencies.
2) We update Y for interface changes and/or significant features
we wish to publicize.
3) The Theano v1.0.0 release will be made when the interface is deemed
stable enough and covers most of numpy's interface.
5) The trunk will be tagged on each release.
6) Each release will be uploaded to pypi.python.org, mloss.org and freshmeat.net
7) Release emails will be sent to theano-users, theano-announce, numpy-discussion@scipy.org and scipy-user@scipy.org .
Optional:
8) A 1-week scrum might take place before a release, in order to fix bugs
which would otherwise prevent a release.
1) Occasional deadlines might cause us to skip a release.
2) Everybody can (and should) participate, even people on the mailing
list.
3) The scrum should encourage people to finish what they have already
started (missing documentation, missing test, ...). This should help
push out new features and keep the documentation up to date.
4) If possible, aim for the inclusion of one new interesting feature.
5) Participating in the scrum should benefit all those involved, as you
will learn more about our tools and help develop them in the process. A
good indication that you should participate is if you have a need for a
feature which is not yet implemented.
Theano-1.0.1/doc/introduction.txt 0000777 0000000 0000000 00000022056 13212236347 016733 0 ustar root root 0000000 0000000
.. _introduction:
==================
Theano at a Glance
==================
Theano is a Python library that lets you to define, optimize, and evaluate
mathematical expressions, especially ones with multi-dimensional arrays
(numpy.ndarray). Using Theano it is
possible to attain speeds rivaling hand-crafted C implementations for problems
involving large amounts of data. It can also surpass C on a CPU by many orders
of magnitude by taking advantage of recent GPUs.
Theano combines aspects of a computer algebra system (CAS) with aspects of an
optimizing compiler. It can also generate customized C code for many
mathematical operations. This combination of CAS with optimizing compilation
is particularly useful for tasks in which complicated mathematical expressions
are evaluated repeatedly and evaluation speed is critical. For situations
where many different expressions are each evaluated once Theano can minimize
the amount of compilation/analysis overhead, but still provide symbolic
features such as automatic differentiation.
Theano's compiler applies many optimizations of varying complexity to
these symbolic expressions. These optimizations include, but are not
limited to:
* use of GPU for computations
* constant folding
* merging of similar subgraphs, to avoid redundant calculation
* arithmetic simplification (e.g. ``x*y/x -> y``, ``--x -> x``)
* inserting efficient BLAS_ operations (e.g. ``GEMM``) in a variety of
contexts
* using memory aliasing to avoid calculation
* using inplace operations wherever it does not interfere with aliasing
* loop fusion for elementwise sub-expressions
* improvements to numerical stability (e.g. :math:`\log(1+\exp(x))` and :math:`\log(\sum_i \exp(x[i]))`)
* for a complete list, see :ref:`optimizations`
Theano was written at the LISA_ lab to support rapid development of
efficient machine learning algorithms. Theano is
named after the `Greek mathematician`_, who may have been Pythagoras'
wife. Theano is released under a BSD license (:ref:`link `).
Sneak peek
==========
Here is an example of how to use Theano. It doesn't show off many of
Theano's features, but it illustrates concretely what Theano is.
.. If you modify this code, also change :
.. theano/tests/test_tutorial.py:T_introduction.test_introduction_1
.. code-block:: python
import theano
from theano import tensor
# declare two symbolic floating-point scalars
a = tensor.dscalar()
b = tensor.dscalar()
# create a simple expression
c = a + b
# convert the expression into a callable object that takes (a,b)
# values as input and computes a value for c
f = theano.function([a,b], c)
# bind 1.5 to 'a', 2.5 to 'b', and evaluate 'c'
assert 4.0 == f(1.5, 2.5)
Theano is not a programming language in the normal sense because you
write a program in Python that builds expressions for Theano. Still it
is like a programming language in the sense that you have to
- declare variables (``a,b``) and give their types
- build expressions for how to put those variables together
- compile expression graphs to functions in order to use them for computation.
It is good to think of ``theano.function`` as the interface to a
compiler which builds a callable object from a purely symbolic graph.
One of Theano's most important features is that ``theano.function``
can optimize a graph and even compile some or all of it into native
machine instructions.
What does it do that they don't?
================================
Theano is a Python library and optimizing compiler for manipulating
and evaluating expressions, especially matrix-valued
ones. Manipulation of matrices is typically done using the numpy
package, so what does Theano do that Python and numpy do not?
- *execution speed optimizations*: Theano can use `g++` or `nvcc` to compile
parts your expression graph into CPU or GPU instructions, which run
much faster than pure Python.
- *symbolic differentiation*: Theano can automatically build symbolic graphs
for computing gradients.
- *stability optimizations*: Theano can recognize [some] numerically unstable
expressions and compute them with more stable algorithms.
The closest Python package to Theano is sympy_.
Theano focuses more on tensor expressions than Sympy, and has more machinery
for compilation. Sympy has more sophisticated algebra rules and can
handle a wider variety of mathematical operations (such as series, limits, and integrals).
If numpy_ is to be compared to MATLAB_ and sympy_ to Mathematica_,
Theano is a sort of hybrid of the two which tries to combine the best of
both worlds.
Getting started
===============
:ref:`install`
Instructions to download and install Theano on your system.
:ref:`tutorial`
Getting started with Theano's basic features. Go here if you are
new!
:ref:`libdoc`
Details of what Theano provides. It is recommended to go through
the :ref:`tutorial` first though.
A PDF version of the online documentation may be found `here
`_.
Theano Vision
=============
This is the vision we have for Theano. This is give people an idea of what to
expect in the future of Theano, but we can't promise to implement all
of it. This should also help you to understand where Theano fits in relation
to other computational tools.
* Support tensor and sparse operations
* Support linear algebra operations
* Graph Transformations
* Differentiation/higher order differentiation
* 'R' and 'L' differential operators
* Speed/memory optimizations
* Numerical stability optimizations
* Can use many compiled languages, instructions sets: C/C++, CUDA, OpenCL, PTX, CAL, AVX, ...
* Lazy evaluation
* Loop
* Parallel execution (SIMD, multi-core, multi-node on cluster,
multi-node distributed)
* Support all NumPy/basic SciPy functionality
* Easy wrapping of library functions in Theano
Note: There is no short term plan to support multi-node computation.
Theano Vision State
===================
Here is the state of that vision as of November 15th, 2017 (after Theano 1.0.0):
* `MILA will stop developing Theano. `_.
We will provide support for one year, starting from ``1.0`` release (November 15th, 2017 to November 15th, 2018).
* We support tensors using the `numpy.ndarray` object and we support many operations on them.
* We support sparse types by using the `scipy.{csc,csr,bsr}_matrix` object and support some operations on them.
* We have implementing/wrapping more advanced linear algebra operations. Still more possible.
* We have basic support for the creation of new operations from graphs at runtime. It supports well gradient overload
for every input and inlining at the start of compilation. We don't cover well the case when it is not inlined.
* We have many graph transformations that cover the 4 categories listed above.
* We can improve the graph transformation with better storage optimization
and instruction selection.
* Similar to auto-tuning during the optimization phase, but this
doesn't apply to only 1 op.
* Example of use: Determine if we should move computation to the
GPU or not depending on the input size.
* We support Python 2 and Python 3.
* We have a new CUDA backend for tensors with many dtype support.
* Loops work, but not all related optimizations are currently done.
* The cvm linker allows lazy evaluation. It is the current default linker.
* How to have `DebugMode` check it? Right now, DebugMode checks the computation non-lazily.
* SIMD parallelism on the CPU comes from the compiler.
* Multi-core parallelism support limited.
If the external BLAS implementation supports it,
many dot are parallelized via gemm, gemv and ger.
Also, element-wise operation are supported. See :ref:`tut_multi_cores`.
* No multi-node support.
* Most, but not all NumPy functions/aliases are implemented.
* https://github.com/Theano/Theano/issues/1080
* Wrapping an existing Python function in easy and documented.
* We know how to separate the shared variable memory
storage location from its object type (tensor, sparse, dtype, broadcast
flags), but we need to do it.
Contact us
==========
Discussion about Theano takes place in the theano-dev_ and
theano-users_ mailing lists. People interested in development of
Theano should check the former, while the latter is reserved for
issues that concern the end users.
Questions, comments, praise, criticism as well as bug reports should
be submitted to these mailing lists.
We welcome all kinds of contributions. If you have any questions
regarding how to extend Theano, please feel free to ask on the
theano-dev_ mailing list.
.. _LISA: https://mila.umontreal.ca/
.. _Greek mathematician: http://en.wikipedia.org/wiki/Theano_(mathematician)
.. _numpy: http://numpy.scipy.org/
.. _BLAS: http://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms
.. _sympy: http://www.sympy.org/
.. _MATLAB: http://www.mathworks.com/products/matlab/
.. _Mathematica: http://www.wolfram.com/mathematica/
.. _theano-users: http://groups.google.com/forum/#!forum/theano-users
.. _theano-dev: http://groups.google.com/forum/#!forum/theano-dev
Theano-1.0.1/doc/library/ 0000777 0000000 0000000 00000000000 13212236507 015103 5 ustar root root 0000000 0000000 Theano-1.0.1/doc/library/compile/ 0000777 0000000 0000000 00000000000 13212236507 016533 5 ustar root root 0000000 0000000 Theano-1.0.1/doc/library/compile/debugmode.txt 0000777 0000000 0000000 00000017725 13212236347 021250 0 ustar root root 0000000 0000000
.. _debugmode:
=================
:mod:`debugmode`
=================
.. module:: theano.compile.debugmode
:platform: Unix, Windows
:synopsis: defines DebugMode
.. moduleauthor:: LISA
Guide
=====
The DebugMode evaluation mode includes a number of self-checks and assertions
that can help to diagnose several kinds of programmer errors that can lead to
incorrect output.
It is much slower to evaluate a function or method with DebugMode than
it would be in ``'FAST_RUN'`` or even ``'FAST_COMPILE'``. We recommended you use
DebugMode during development, but not when you launch 1000 processes on
a cluster.
DebugMode can be used as follows:
.. testcode::
import theano
from theano import tensor
from theano.compile.debugmode import DebugMode
x = tensor.dscalar('x')
f = theano.function([x], 10*x, mode='DebugMode')
f(5)
f(0)
f(7)
It can also be used by setting the configuration variable :attr:`config.mode`.
It can also be used by passing a DebugMode instance as the mode, as in
>>> f = theano.function([x], 10*x, mode=DebugMode(check_c_code=False))
If any problem is detected, DebugMode will raise an exception according to
what went wrong, either at call time (``f(5)``) or compile time (
``f = theano.function(x, 10*x, mode='DebugMode')``). These exceptions
should *not* be ignored; talk to your local Theano guru or email the
users list if you cannot make the exception go away.
Some kinds of errors can only be detected for certain input value combinations.
In the example above, there is no way to guarantee that a future call to say,
``f(-1)`` won't cause a problem. DebugMode is not a silver bullet.
If you instantiate DebugMode using the constructor ``compile.DebugMode``
rather than the keyword ``DebugMode`` you can configure its behaviour via
constructor arguments.
Reference
==========
.. class:: DebugMode(Mode)
Evaluation Mode that detects internal theano errors.
This mode catches several kinds of internal error:
- inconsistent outputs when calling the same Op twice with the same
inputs, for instance if c_code and perform implementations, are
inconsistent, or in case of incorrect handling of output memory
(see `BadThunkOutput`)
- a variable replacing another when their runtime values don't match. This is a symptom of
an incorrect optimization step, or faulty Op implementation (raises `BadOptimization`)
- stochastic optimization ordering (raises `StochasticOrder`)
- incomplete `destroy_map` specification (raises `BadDestroyMap`)
- an op that returns an illegal value not matching the output Variable Type (raises
InvalidValueError)
Each of these exceptions inherits from the more generic `DebugModeError`.
If there are no internal errors, this mode behaves like FAST_RUN or FAST_COMPILE, but takes
a little longer and uses more memory.
If there are internal errors, this mode will raise an `DebugModeError` exception.
.. attribute:: stability_patience = config.DebugMode.patience
When checking for the stability of optimization, recompile the graph this many times.
Default 10.
.. attribute:: check_c_code = config.DebugMode.check_c
Should we evaluate (and check) the `c_code` implementations?
``True`` -> yes, ``False`` -> no.
Default yes.
.. attribute:: check_py_code = config.DebugMode.check_py
Should we evaluate (and check) the `perform` implementations?
``True`` -> yes, ``False`` -> no.
Default yes.
.. attribute:: check_isfinite = config.DebugMode.check_finite
Should we check for (and complain about) ``NaN``/``Inf`` ndarray elements?
``True`` -> yes, ``False`` -> no.
Default yes.
.. attribute:: require_matching_strides = config.DebugMode.check_strides
Check for (and complain about) Ops whose python and C
outputs are ndarrays with different strides. (This can catch bugs, but
is generally overly strict.)
0 -> no check, 1 -> warn, 2 -> err.
Default warn.
.. method:: __init__(self, optimizer='fast_run', stability_patience=None, check_c_code=None, check_py_code=None, check_isfinite=None, require_matching_strides=None, linker=None)
Initialize member variables.
If any of these arguments (except optimizer) is not None, it overrides the class default.
The linker arguments is not used. It is set there to allow
Mode.requiring() and some other functions to work with DebugMode too.
The keyword version of DebugMode (which you get by using ``mode='DebugMode``)
is quite strict, and can raise several different Exception types.
There following are DebugMode exceptions you might encounter:
.. class:: DebugModeError(Exception)
This is a generic error. All the other exceptions inherit from this one.
This error is typically not raised directly.
However, you can use ``except DebugModeError: ...`` to catch any of the more
specific types of Exception.
.. class:: BadThunkOutput(DebugModeError)
This exception means that different calls to the same Op with the same
inputs did not compute the same thing like they were supposed to.
For instance, it can happen if the python (``perform``) and c (``c_code``)
implementations of the Op are inconsistent (the problem might be a bug in
either ``perform`` or ``c_code`` (or both)). It can also happen if
``perform`` or ``c_code`` does not handle correctly output memory that
has been preallocated (for instance, if it did not clear the memory before
accumulating into it, or if it assumed the memory layout was C-contiguous
even if it is not).
.. class:: BadOptimization(DebugModeError)
This exception indicates that an Optimization replaced one variable (say V1)
with another one (say V2) but at runtime, the values for V1 and V2 were
different. This is something that optimizations are not supposed to do.
It can be tricky to identify the one-true-cause of an optimization error, but
this exception provides a lot of guidance. Most of the time, the
exception object will indicate which optimization was at fault.
The exception object also contains information such as a snapshot of the
before/after graph where the optimization introduced the error.
.. class:: BadDestroyMap(DebugModeError)
This happens when an Op's ``perform()`` or ``c_code()`` modifies an input that it wasn't
supposed to. If either the ``perform`` or ``c_code`` implementation of an Op
might modify any input, it has to advertise that fact via the ``destroy_map``
attribute.
For detailed documentation on the ``destroy_map`` attribute, see :ref:`inplace`.
.. class:: BadViewMap(DebugModeError)
This happens when an Op's perform() or c_code() creates an alias or alias-like
dependency between an input and an output... and it didn't warn the
optimization system via the ``view_map`` attribute.
For detailed documentation on the ``view_map`` attribute, see :ref:`views`.
.. class:: StochasticOrder(DebugModeError)
This happens when an optimization does not perform the same graph operations
in the same order when run several times in a row. This can happen if any
steps are ordered by ``id(object)`` somehow, such as via the default object
hash function. A Stochastic optimization invalidates the pattern of work
whereby we debug in DebugMode and then run the full-size jobs in FAST_RUN.
.. class:: InvalidValueError(DebugModeError)
This happens when some Op's ``perform`` or ``c_code`` implementation computes
an output that is invalid with respect to the type of the corresponding output
variable. Like if it returned a complex-valued ndarray for a ``dscalar``
Type.
This can also be triggered when floating-point values such as NaN and Inf are
introduced into the computations. It indicates which Op created the first
NaN. These floating-point values can be allowed by passing the
``check_isfinite=False`` argument to DebugMode.
Theano-1.0.1/doc/library/compile/function.txt 0000777 0000000 0000000 00000021207 13212236347 021130 0 ustar root root 0000000 0000000
.. _usingfunction:
===========================================
:mod:`function` - defines theano.function
===========================================
.. module:: theano.compile.function
:platform: Unix, Windows
:synopsis: defines theano.function and related classes
.. moduleauthor:: LISA
Guide
=====
This module provides :func:`function`, commonly accessed as `theano.function`,
the interface for compiling graphs into callable objects.
You've already seen example usage in the basic tutorial... something like this:
>>> import theano
>>> x = theano.tensor.dscalar()
>>> f = theano.function([x], 2*x)
>>> f(4)
array(8.0)
The idea here is that we've compiled the symbolic graph (``2*x``) into a function that can be called on a number and will do some computations.
The behaviour of function can be controlled in several ways, such as
:class:`In`, :class:`Out`, ``mode``, ``updates``, and ``givens``. These are covered
in the :ref:`tutorial examples ` and :ref:`tutorial on modes `.
Reference
=========
.. class:: In
A class for attaching information to function inputs.
.. attribute:: variable
A variable in an expression graph to use as a compiled-function parameter
.. attribute:: name
A string to identify an argument for this parameter in keyword arguments.
.. attribute:: value
The default value to use at call-time (can also be a Container where
the function will find a value at call-time.)
.. attribute:: update
An expression which indicates updates to the Value after each function call.
.. attribute:: mutable
``True`` means the compiled-function is allowed to modify this
argument. ``False`` means it is not allowed.
.. attribute:: borrow
``True`` indicates that a reference to internal storage may be returned, and that the caller is aware that subsequent function evaluations might overwrite this memory.
.. attribute:: strict
If ``False``, a function argument may be copied or cast to match the type
required by the parameter `variable`. If ``True``, a function argument
must exactly match the type required by `variable`.
.. attribute:: allow_downcast
``True`` indicates that the value you pass for this input can be silently downcasted to fit the right type, which may lose precision. (Only applies when `strict` is ``False``.)
.. attribute:: autoname
``True`` means that the `name` is set to variable.name.
.. attribute:: implicit
``True`` means that the input is implicit in the sense that the user is not allowed to provide a value for it. Requires 'value' to be set.
``False`` means that the user can provide a value for this input.
.. method:: __init__(self, variable, name=None, value=None, update=None, mutable=None, strict=False, allow_downcast=None, autoname=True, implicit=None, borrow=None, shared=False)
Initialize attributes from arguments.
.. class:: Out
A class for attaching information to function outputs
.. attribute:: variable
A variable in an expression graph to use as a compiled-function
output
.. attribute:: borrow
``True`` indicates that a reference to internal storage may be returned, and that the caller is aware that subsequent function evaluations might overwrite this memory.
.. method:: __init__(variable, borrow=False)
Initialize attributes from arguments.
.. function:: function(inputs, outputs, mode=None, updates=None, givens=None, no_default_updates=False, accept_inplace=False, name=None, rebuild_strict=True, allow_input_downcast=None, profile=None, on_unused_input='raise')
Return a :class:`callable object ` that will calculate `outputs` from `inputs`.
:type params: list of either Variable or In instances, but not shared
variables.
:param params: the returned :class:`Function` instance will have
parameters for these variables.
:type outputs: list of Variables or Out instances
:param outputs: expressions to compute.
:type mode: None, string or :class:`Mode` instance.
:param mode: compilation mode
:type updates: iterable over pairs (shared_variable, new_expression).
List, tuple or dict.
:param updates: expressions for new :class:`SharedVariable` values
:type givens: iterable over pairs (Var1, Var2) of Variables.
List, tuple or dict. The Var1
and Var2 in each pair must have the same Type.
:param givens: specific substitutions to make in the
computation graph (Var2 replaces Var1).
:type no_default_updates: either bool or list of Variables
:param no_default_updates:
if True, do not perform any automatic update on Variables.
If False (default), perform them all.
Else, perform automatic updates on all Variables that are
neither in ``updates`` nor in ``no_default_updates``.
:param name: an optional name for this function.
The profile mode will print the time spent in this function.
:param rebuild_strict: True (Default) is the safer and better
tested setting, in which case `givens` must substitute new
variables with the same Type as the variables they replace.
False is a you-better-know-what-you-are-doing setting, that
permits `givens` to replace variables with new variables of
any Type. The consequence of changing a Type is that all
results depending on that variable may have a different Type
too (the graph is rebuilt from inputs to outputs). If one of
the new types does not make sense for one of the Ops in the
graph, an Exception will be raised.
:type allow_input_downcast: Boolean or None
:param allow_input_downcast: True means that the values passed as
inputs when calling the function can be silently downcasted to
fit the dtype of the corresponding Variable, which may lose
precision. False means that it will only be cast to a more
general, or precise, type. None (default) is almost like
False, but allows downcasting of Python float scalars to
floatX.
:type profile: None, True, or ProfileStats instance
:param profile: accumulate profiling information into a given
ProfileStats instance. If argument is `True` then a new
ProfileStats instance will be used. This profiling object
will be available via self.profile.
:param on_unused_input: What to do if a variable in the 'inputs'
list is not used in the graph. Possible values are 'raise',
'warn', and 'ignore'.
:rtype: :class:`Function `
instance
:returns: a callable object that will compute the outputs (given the inputs)
and update the implicit function arguments according to the `updates`.
Inputs can be given as variables or In instances.
:class:`In` instances also have a variable, but they attach some extra
information about how call-time arguments corresponding to that variable
should be used. Similarly, :class:`Out` instances can attach information
about how output variables should be returned.
The default is typically 'FAST_RUN' but this can be changed in
:doc:`theano.config <../config>`. The mode
argument controls the sort of optimizations that will be applied to the
graph, and the way the optimized graph will be evaluated.
After each function evaluation, the `updates` mechanism can replace the
value of any SharedVariable [implicit] inputs with new values computed
from the expressions in the `updates` list. An exception will be raised
if you give two update expressions for the same SharedVariable input (that
doesn't make sense).
If a SharedVariable is not given an update expression, but has a
``default_update`` member containing an expression, this expression
will be used as the update expression for this variable. Passing
``no_default_updates=True`` to ``function`` disables this behavior
entirely, passing ``no_default_updates=[sharedvar1, sharedvar2]``
disables it for the mentioned variables.
Regarding givens: Be careful to make sure that these substitutions are
independent, because behaviour when Var1 of one pair appears in the graph leading
to Var2 in another expression is undefined (e.g. with ``{a: x, b: a + 1}``).
Replacements specified with
givens are different from optimizations in that Var2 is not expected to be
equivalent to Var1.
.. autofunction:: theano.compile.function.function_dump
.. autoclass:: theano.compile.function_module.Function
:members: free, copy, __call__
Theano-1.0.1/doc/library/compile/index.txt 0000777 0000000 0000000 00000000706 13212236347 020413 0 ustar root root 0000000 0000000
.. _libdoc_compile:
==============================================================
:mod:`compile` -- Transforming Expression Graphs to Functions
==============================================================
.. module:: compile
:platform: Unix, Windows
:synopsis: transforming expression graphs to functions
.. moduleauthor:: LISA
.. toctree::
:maxdepth: 1
shared
function
io
ops
mode
debugmode
nanguardmode
Theano-1.0.1/doc/library/compile/io.txt 0000777 0000000 0000000 00000027037 13212236347 017721 0 ustar root root 0000000 0000000
.. note::
***TODO*** Freshen up this old documentation
.. _function_inputs:
===========================================
:mod:`io` - defines theano.function [TODO]
===========================================
.. module:: theano.compile.io
:platform: Unix, Windows
:synopsis: defines In and Out
.. moduleauthor:: LISA
Inputs
======
The ``inputs`` argument to ``theano.function`` is a list, containing the ``Variable`` instances for which values will be specified at the time of the function call. But inputs can be more than just Variables.
``In`` instances let us attach properties to ``Variables`` to tell function more about how to use them.
.. class:: In(object)
.. method:: __init__(variable, name=None, value=None, update=None, mutable=False, strict=False, autoname=True, implicit=None)
``variable``: a Variable instance. This will be assigned a value
before running the function, not computed from its owner.
``name``: Any type. (If ``autoname_input==True``, defaults to
``variable.name``). If ``name`` is a valid Python identifier, this input
can be set by ``kwarg``, and its value can be accessed by
``self.``. The default value is ``None``.
``value``: literal or ``Container``. The initial/default value for this
input. If update is`` None``, this input acts just like
an argument with a default value in Python. If update is not ``None``,
changes to this
value will "stick around", whether due to an update or a user's
explicit action.
``update``: Variable instance. This expression Variable will
replace ``value`` after each function call. The default value is
``None``, indicating that no update is to be done.
``mutable``: Bool (requires value). If ``True``, permit the
compiled function to modify the Python object being used as the
default value. The default value is ``False``.
``strict``: Bool (default: ``False`` ). ``True`` means that the value
you pass for this input must have exactly the right type. Otherwise, it
may be cast automatically to the proper type.
``autoname``: Bool. If set to ``True``, if ``name`` is ``None`` and
the Variable has a name, it will be taken as the input's
name. If autoname is set to ``False``, the name is the exact
value passed as the name parameter (possibly ``None``).
``implicit``: Bool or ``None`` (default: ``None``)
``True``: This input is implicit in the sense that the user is not allowed
to provide a value for it. Requires ``value`` to be set.
``False``: The user can provide a value for this input. Be careful
when ``value`` is a container, because providing an input value will
overwrite the content of this container.
``None``: Automatically choose between ``True`` or ``False`` depending on the
situation. It will be set to ``False`` in all cases except if
``value`` is a container (so that there is less risk of accidentally
overwriting its content without being aware of it).
Value: initial and default values
---------------------------------
A non-None `value` argument makes an In() instance an optional parameter
of the compiled function. For example, in the following code we are
defining an arity-2 function ``inc``.
>>> import theano.tensor as T
>>> from theano import function
>>> from theano.compile.io import In
>>> u, x, s = T.scalars('u', 'x', 's')
>>> inc = function([u, In(x, value=3), In(s, update=(s+x*u), value=10.0)], [])
Since we provided a ``value`` for ``s`` and ``x``, we can call it with just a value for ``u`` like this:
>>> inc(5) # update s with 10+3*5
[]
>>> print(inc[s])
25.0
The effect of this call is to increment the storage associated to ``s`` in ``inc`` by 15.
If we pass two arguments to ``inc``, then we override the value associated to
``x``, but only for this one function call.
>>> inc(3, 4) # update s with 25 + 3*4
[]
>>> print(inc[s])
37.0
>>> print(inc[x]) # the override value of 4 was only temporary
3.0
If we pass three arguments to ``inc``, then we override the value associated
with ``x`` and ``u`` and ``s``.
Since ``s``'s value is updated on every call, the old value of ``s`` will be ignored and then replaced.
>>> inc(3, 4, 7) # update s with 7 + 3*4
[]
>>> print(inc[s])
19.0
We can also assign to ``inc[s]`` directly:
>>> inc[s] = 10
>>> inc[s]
array(10.0)
Input Argument Restrictions
---------------------------
The following restrictions apply to the inputs to ``theano.function``:
- Every input list element must be a valid ``In`` instance, or must be
upgradable to a valid ``In`` instance. See the shortcut rules below.
- The same restrictions apply as in Python function definitions:
default arguments and keyword arguments must come at the end of
the list. Un-named mandatory arguments must come at the beginning of
the list.
- Names have to be unique within an input list. If multiple inputs
have the same name, then the function will raise an exception. [***Which
exception?**]
- Two ``In`` instances may not name the same Variable. I.e. you cannot
give the same parameter multiple times.
If no name is specified explicitly for an In instance, then its name
will be taken from the Variable's name. Note that this feature can cause
harmless-looking input lists to not satisfy the two conditions above.
In such cases, Inputs should be named explicitly to avoid problems
such as duplicate names, and named arguments preceding unnamed ones.
This automatic naming feature can be disabled by instantiating an In
instance explicitly with the ``autoname`` flag set to False.
Access to function values and containers
----------------------------------------
For each input, ``theano.function`` will create a ``Container`` if
``value`` was not already a ``Container`` (or if ``implicit`` was ``False``). At the time of a function call,
each of these containers must be filled with a value. Each input (but
especially ones with a default value or an update expression) may have a
value between calls. The function interface defines a way to get at
both the current value associated with an input, as well as the container
which will contain all future values:
- The ``value`` property accesses the current values. It is both readable
and writable, but assignments (writes) may be implemented by an internal
copy and/or casts.
- The ``container`` property accesses the corresponding container.
This property accesses is a read-only dictionary-like interface. It is
useful for fetching the container associated with a particular input to
share containers between functions, or to have a sort of pointer to an
always up-to-date value.
Both ``value`` and ``container`` properties provide dictionary-like access based on three types of keys:
- integer keys: you can look up a value/container by its position in the input list;
- name keys: you can look up a value/container by its name;
- Variable keys: you can look up a value/container by the Variable it corresponds to.
In addition to these access mechanisms, there is an even more convenient
method to access values by indexing a Function directly by typing
``fn[]``, as in the examples above.
To show some examples of these access methods...
>>> from theano import tensor as T, function
>>> a, b, c = T.scalars('xys') # set the internal names of graph nodes
>>> # Note that the name of c is 's', not 'c'!
>>> fn = function([a, b, ((c, c+a+b), 10.0)], [])
>>> # the value associated with c is accessible in 3 ways
>>> fn['s'] is fn.value[c]
True
>>> fn['s'] is fn.container[c].value
True
>>> fn['s']
array(10.0)
>>> fn(1, 2)
[]
>>> fn['s']
array(13.0)
>>> fn['s'] = 99.0
>>> fn(1, 0)
[]
>>> fn['s']
array(100.0)
>>> fn.value[c] = 99.0
>>> fn(1,0)
[]
>>> fn['s']
array(100.0)
>>> fn['s'] == fn.value[c]
True
>>> fn['s'] == fn.container[c].value
True
Input Shortcuts
---------------
Every element of the inputs list will be upgraded to an In instance if necessary.
- a Variable instance ``r`` will be upgraded like ``In(r)``
- a tuple ``(name, r)`` will be ``In(r, name=name)``
- a tuple ``(r, val)`` will be ``In(r, value=value, autoname=True)``
- a tuple ``((r,up), val)`` will be ``In(r, value=value, update=up, autoname=True)``
- a tuple ``(name, r, val)`` will be ``In(r, name=name, value=value)``
- a tuple ``(name, (r,up), val)`` will be ``In(r, name=name, value=val, update=up, autoname=True)``
Example:
>>> import theano
>>> from theano import tensor as T
>>> from theano.compile.io import In
>>> x = T.scalar()
>>> y = T.scalar('y')
>>> z = T.scalar('z')
>>> w = T.scalar('w')
>>> fn = theano.function(inputs=[x, y, In(z, value=42), ((w, w+x), 0)],
... outputs=x + y + z)
>>> # the first two arguments are required and the last two are
>>> # optional and initialized to 42 and 0, respectively.
>>> # The last argument, w, is updated with w + x each time the
>>> # function is called.
>>> fn(1) # illegal because there are two required arguments # doctest: +ELLIPSIS
Traceback (most recent call last):
...
TypeError: Missing required input: y
>>> fn(1, 2) # legal, z is 42, w goes 0 -> 1 (because w <- w + x)
array(45.0)
>>> fn(1, y=2) # legal, z is 42, w goes 1 -> 2
array(45.0)
>>> fn(x=1, y=2) # illegal because x was not named # doctest: +ELLIPSIS
Traceback (most recent call last):
...
TypeError: Unknown input or state: x. The function has 3 named inputs (y, z, w), and 1 unnamed input which thus cannot be accessed through keyword argument (use 'name=...' in a variable's constructor to give it a name).
>>> fn(1, 2, 3) # legal, z is 3, w goes 2 -> 3
array(6.0)
>>> fn(1, z=3, y=2) # legal, z is 3, w goes 3 -> 4
array(6.0)
>>> fn(1, 2, w=400) # legal, z is 42 again, w goes 400 -> 401
array(45.0)
>>> fn(1, 2) # legal, z is 42, w goes 401 -> 402
array(45.0)
In the example above, ``z`` has value 42 when no value is explicitly given.
This default value is potentially used at every function invocation, because
``z`` has no ``update`` or storage associated with it.
.. _function_outputs:
Outputs
=======
The ``outputs`` argument to function can be one of
- ``None``, or
- a Variable or ``Out`` instance, or
- a list of Variables or ``Out`` instances.
An ``Out`` instance is a structure that lets us attach options to individual output ``Variable`` instances,
similarly to how ``In`` lets us attach options to individual input ``Variable`` instances.
**Out(variable, borrow=False)** returns an ``Out`` instance:
* ``borrow``
If ``True``, a reference to function's internal storage
is OK. A value returned for this output might be clobbered by running
the function again, but the function might be faster.
Default: ``False``
If a single ``Variable`` or ``Out`` instance is given as argument, then the compiled function will return a single value.
If a list of ``Variable`` or ``Out`` instances is given as argument, then the compiled function will return a list of their values.
>>> import numpy
>>> from theano.compile.io import Out
>>> x, y, s = T.matrices('xys')
>>> # print a list of 2 ndarrays
>>> fn1 = theano.function([x], [x+x, Out((x+x).T, borrow=True)])
>>> fn1(numpy.asarray([[1,0],[0,1]]))
[array([[ 2., 0.],
[ 0., 2.]]), array([[ 2., 0.],
[ 0., 2.]])]
>>> # print a list of 1 ndarray
>>> fn2 = theano.function([x], [x+x])
>>> fn2(numpy.asarray([[1,0],[0,1]]))
[array([[ 2., 0.],
[ 0., 2.]])]
>>> # print an ndarray
>>> fn3 = theano.function([x], outputs=x+x)
>>> fn3(numpy.asarray([[1,0],[0,1]]))
array([[ 2., 0.],
[ 0., 2.]])
Theano-1.0.1/doc/library/compile/mode.txt 0000777 0000000 0000000 00000004067 13212236347 020234 0 ustar root root 0000000 0000000
.. _libdoc_compile_mode:
======================================
:mod:`mode` -- controlling compilation
======================================
.. module:: theano.compile.mode
:platform: Unix, Windows
:synopsis: controlling compilation
.. moduleauthor:: LISA
Guide
=====
The ``mode`` parameter to :func:`theano.function` controls how the
inputs-to-outputs graph is transformed into a callable object.
Theano defines the following modes by name:
- ``'FAST_COMPILE'``: Apply just a few graph optimizations and only use Python implementations.
- ``'FAST_RUN'``: Apply all optimizations, and use C implementations where possible.
- ``'DebugMode'``: A mode for debugging. See :ref:`DebugMode ` for details.
- ``'NanGuardMode``: :ref:`Nan detector