bdtl-1.0.3/0000755000175000017500000000000012330444352012232 5ustar melamedmelamedbdtl-1.0.3/flags.mk0000644000175000017500000000234612122405072013657 0ustar melamedmelamed #-------------------------------------------------------------------------- # setting variables for compilation #-------------------------------------------------------------------------- ifdef CONDOR_COMPILE CONDOR_CMP = condor_compile USECONDOR = -DUSE_CONDOR else CONDOR_CMP = USECONDOR = endif ifdef 32BIT ARCH_SIZE = 32 ARCH_LIB_SUFX = else ARCH_SIZE = 64 ARCH_LIB_SUFX = 64 endif ARCH_FLAG = -m$(ARCH_SIZE) ifdef DYNAMIC STATIC_FLAG = else STATIC_FLAG = -static -Bstatic endif ifdef VALID VALIDATION_FLAG = -DVALIDATION else VALIDATION_FLAG = endif ifdef USE_BOOST_MEM_POOL MEM_POOL_FLAG = -DBOOST_MEM_POOL -DBOOST_NO_PROPER_STL_DEALLOCATE else MEM_POOL_FLAG = endif ifdef DBG OPT_DEBUG_FLAG = -ggdb -O0 -fno-inline SUFFIX=dbg LIBSUFFIX = _dbg endif ifdef OPT OPT_DEBUG_FLAG = -O3 -DNDEBUG SUFFIX = opt LIBSUFFIX = else OPT_DEBUG_FLAG = -O0 -ggdb -fno-inline SUFFIX = dbg LIBSUFFIX = _dbg endif ifdef OPTDBG OPT_DEBUG_FLAG = -O3 -ggdb SUFFIX = optdbg LIBSUFFIX = _optdbg endif ifdef GOOG GOOG_FLAG = -L/usr/local/lib -lprofiler -ltcmalloc else GOOG_FLAG = endif ifdef GPROF USE_GPROF = -pg SUFFIX := $(SUFFIX)_gprof LIBSUFFIX := $(LIBSUFFIX)_gprof else USE_GRPOF = endif bdtl-1.0.3/shared.mk0000644000175000017500000000071412151020420014016 0ustar melamedmelamed######################################################################################## # # BDTL configuration variables that are unlikely to change, # but are needed by multiple parts of the installation # ######################################################################################## include $(BASE_DIR)/flags.mk # determine the machine architecture ARCH := $(shell uname -s) BIN_BASE = $(BASE_DIR)/bin BIN = $(BIN_BASE)/$(ARCH)/$(SUFFIX) bdtl-1.0.3/WHAT.this.is0000644000175000017500000000261712151020420014271 0ustar melamedmelamedBoosted Decision Trees and Lists (BDTL) The BDTL software package implements two main boosting algorithms with many variations. The first one is Turian and Melamed's extension of confidence-rated boosting (Schapire & Singer, 1999). See Joseph Turian's thesis (NYU, 2007) for details. The second algorithm is Galron and Melamed's extension of the first one, to boost decision lists rather than decision trees. (Every decision tree ensemble is equivalent to some decision list ensemble, and vice versa.) Major features of the software include: * Classification and regression * tested on 3M+ examples with 1M+ features, but limited only by your computer's RAM. * Easily customizable loss functions and regularization methods. Currently implemented ones are logistic and exponential loss for classification, and squared loss for regression, each with L1 or L2 regularization. * Selection of weak learners to directly optimize the regularized training objective. * One-shot training following an entire regularization path, which can save a lot of time during hyperparameter optimization. * Continuous checkpointing, so that if a long learning cycle crashes, you can continue training where it left off. * Warm-start training. * Both binary and scalar feature types. To get started, read the file README.1st in the top level directory. Questions, suggestions, and offers of collaboration are most welcome. bdtl-1.0.3/ToDo0000644000175000017500000000363012171052546013027 0ustar melamedmelamed* make DecTree::m_frontier a container of BSP instead of NodeID's, to better encapsulate the DecTree representation * when using heavy examples to split scalar features, adjust decision boundaries post-hoc (or on the fly?) to be equidistant between all +/- supports * use only active set of features during feature selection * speed up decoding by counting satisfied featureTests for each CFT * make example vectors keep track of the total weight, so it doesn't need to be recomputed * split Ensemble::m_CFTIDWeights into separate structures for the bool and the double. Make the former static in Ensemble::confidenceCompute(), so that that method can be const. * Speed up confidence computation over scalar feature tests by taking advantage of the number line of their possible values. E.g., if an example has a scalar feature with value X, and X already satisfied the test that it is more than 5, then we don't need to check whether it is more than 4. * Instead of using separate data structures, m_iFeatOrder and m_fFeatOrder, for integral and non-integral feature values, it might make for shorter code and use less memory if we used just one structure over features values of type IFUnion_t. * implement special case of optimizing exponential loss with L2 reg'n, which can be done analytically instead of using line search ========================= splitting trainer methods based on weak learner type: methods that currently check for parameter::addInternalNodesToEnsemble() = * trainLoop, called by trainLoopWrapper * attemptSplit (alas, in the middle), called by buildTree * newNode, called by buildTree and attemptSplit * updateSampleAndTrainConfs, called by newNode (alas, in the middle AND the end), only if parameter::addInternalNodesToEnsemble() == true and revertSplit, which can be called only if parameter::addInternalNodesToEnsemble() == true * updateProvisionalExampleConfs, called by newNode asdfsfgfdsdf bdtl-1.0.3/scripts/0000755000175000017500000000000012330444352013721 5ustar melamedmelamedbdtl-1.0.3/scripts/sort_scoresets.pl0000755000175000017500000000261212122405072017336 0ustar melamedmelamed#!/usr/bin/perl ######################################################################## # Author: I. Dan Melamed # Computes: sorts lines delimited by and # Loads: nothing # Streams: grammar, etc. # N.B.: sort is numeric on first column and alphabetic on the rest ####################################################################### $TRUE = 1; $FALSE = 0; # 2 statuses -- either within a scoreset or not $inScoreSet = $FALSE; while (<>) { if ($inScoreSet) { if (/^/) { die "nested SCORESET?"; } elsif (/^<\/SCORESET>/) { print "\n"; # sort the indexes from 0 to $#score first by the score and then by the rest @indexes = sort {$score[$a] <=> $score[$b] || $rest[$a] cmp $rest[$b]} (0 .. $#score); # dump them in order foreach $ind (@indexes) { print "$score[$ind]\t$rest[$ind]"; }; print "\n"; # clean up $inScoreSet = $FALSE; undef @score; undef @rest; } else { # store it ($score, $rest) = split(' ', $_, 2); $score[$.] = $score; $rest[$.] = $rest; } } else { # not in scoreset if (/^<\/SCORESET>/) { die "SCORESET ended before starting?"; } elsif (/^/) { $inScoreSet = $TRUE; $. = -1; # reset line counter } else { print; } } } if ($inScoreSet) { die "scoreset not finished!"; }; bdtl-1.0.3/scripts/SVMLi2f.withDict.pl0000755000175000017500000000164212215740062017223 0ustar melamedmelamed#!/usr/bin/perl ######################################################################## # Author: I. Dan Melamed # Computes: converts integer feature values to reals, for some feature types # Loads: list of feature types to convert # Streams: example set in SVMLight format # N.B.: conversion is done merely by adding .0 ####################################################################### $TRUE = 1; $FALSE = 0; $INFINITY = 9999999999; $TINY = 1 - $INFINITY / ($INFINITY + 1); $| = 1; #check for correct usage if ($#ARGV < 0) { print "usage: $0 []\n"; exit; }; print STDERR "$0: Reading feature list... from $ARGV[0]"; open(F, $ARGV[0]) || die "\nCouldn't open $ARGV[0]: $!\n"; while () { chomp; push(@float, $_); }; close(F); shift; print STDERR " done.\n"; while (<>) { foreach $feat (@float) { s/( $feat:\-?\d+)([^\.])/${1}.0$2/; } print; } bdtl-1.0.3/scripts/SVMLi2f.pl0000755000175000017500000000236512215740062015450 0ustar melamedmelamed#!/usr/bin/perl ######################################################################## # Author: I. Dan Melamed # Purpose: convert integer feature values to floats, if that feature's # value ever appears as a float, so that BDT::parameter # doesn't get confused # Streams: examples in SMLight format, twice ####################################################################### #check for correct usage if ($#ARGV < 0) { print "usage: $0 \n"; exit; }; $inFile = shift; open(F, $inFile) || die "\nCouldn't open $inFile: $!\n"; while () { chomp; @tok = split; $objective = shift @tok; if ($objective ne "1" && $objective ne "0") { $scalarObjective = 1; }; foreach $tok (@tok) { ($name, $val) = split(/:/, $tok); if ($val =~ /\./) { $float{$name} = $TRUE; } } } close F; open(F, $inFile) || die "\nCouldn't open $inFile: $!\n"; while () { chomp; @tok = split; $objective = shift @tok; if ($scalarObjective && $objective !~ /\./) { $objective .= "."; }; $out = $objective; foreach $tok (@tok) { ($name, $val) = split(/:/, $tok); if ($val !~ /\./ && exists $float{$name}) { $val .= ".0"; } $out .= " $name:$val"; } print "$out\n"; } close F; bdtl-1.0.3/scripts/SVML2BDT.sh0000755000175000017500000000067412122405072015457 0ustar melamedmelamed#!/bin/tcsh # Original Author: Dan Melamed # Created on: June 4, 2010 # Purpose: convert examples in SVMLight format into a format more suitable for BDT # N.B.: use the faster SVM2BDT.withDict.sh instead if you have a file of float feature types if ($#argv < 1) then echo "usage: $0 " exit 0 endif SVMLi2f.pl $1 > stage1.$$ SVMLInt2Bool.pl stage1.$$ \ | perl -pe 's/^\+1/1/; s/^\-1/0/;' # to stdout rm *.$$ bdtl-1.0.3/scripts/freqThreshold.pl0000755000175000017500000000170112330443263017072 0ustar melamedmelamed#!/usr/bin/perl ######################################################################## # Author: I. Dan Melamed # Purpose: remove boolean features that occur less than given number of times # Loads: entire example set in BDT format ####################################################################### use autodie qw(:all); $TRUE = 1; $FALSE = 0; # duck compiler warning $TRUE = 1; $FALSE = 0; #check for correct usage if ($#ARGV < 0) { print "usage: $0 []\n"; exit; }; $thresh = shift; print STDERR "$0: Reading examples... "; while (<>) { chomp; push(@ex, $_); ($objective, @feat) = split; foreach $feat (@feat) { if ($feat =~ /:/) { # scalar next; } ++$count{$feat}; } }; print STDERR " done.\n"; foreach $ex (@ex) { ($objective, @feat) = split(' ', $ex); $out = $objective; foreach $feat (@feat) { if ($feat =~ /:/ || ($count{$feat} >= $thresh)) { $out .= " $feat"; } } print "$out\n"; }; bdtl-1.0.3/scripts/prettyBDT.pl0000755000175000017500000000455312122405072016144 0ustar melamedmelamed#!/usr/bin/perl ######################################################################## # Author: I. Dan Melamed # Computes: pretty-prints BDT tree sequences # Loads: feature vocab # Streams: tree sequence ####################################################################### $TRUE = 1; $FALSE = 0; $INFINITY = 9999999999; $TINY = 1 - $INFINITY / ($INFINITY + 1); $| = 1; #check for correct usage if ($#ARGV < 0) { print "usage: $0 [tree sequence file]\n"; exit; }; print STDERR "$0: Reading feature vocab ..."; open(F, $ARGV[0]) || die "Couldn't open $ARGV[0]: $!\n"; $delim = ; chomp($delim); die "malformed vocab" if ($delim ne ""); my $ind = 1; while () { if (/VOCAB>/) { last; }; chomp; $fv{$ind++} = $_; }; close(F); shift; print STDERR " done.\n"; $tree = 0; while (<>) { if (! /\S/) { print; next; }; if (/BeginTree/) { undef %leaf, %pos, %neg, %feat, %sign, %thresh; ++$tree; } elsif (/EndTree/) { print "\n"; if (exists $leaf{0}) { printTree(0, 0); } } elsif (/Node# (\d+) \S+ \[ \+ (\d+) , \- (\d+) \] \+child= (\d+) \-child= (\d+) splitOn \( (\d+) (.+) (.+) \) depth= (\d+)/) { $nn = $1; $leaf{$nn} = $FALSE; $poseg{$nn} = $2; $negeg{$nn} = $3; $pos{$nn} = $4; $neg{$nn} = $5; $feat{$nn} = $6; $sign{$nn} = $7; $thresh{$nn} = $8; $dep = $9; } elsif (/Node# (\d+) confidence= ([\d\.\+\-e]+) \S+ \[ \+ (\d+) , \- (\d+) \] \+child= (\d+) \-child= (\d+) splitOn \( (\d+) (.+) (.+) \) depth= (\d+)/) { $nn = $1; $leaf{$nn} = $FALSE; $conf{$nn} = $2; $poseg{$nn} = $3; $negeg{$nn} = $4; $pos{$nn} = $5; $neg{$nn} = $6; $feat{$nn} = $7; $sign{$nn} = $8; $thresh{$nn} = $9; $dep = $10; } elsif (/Leaf# (\d+) confidence= ([\d\.\+\-e]+) \S+ \[ \+ (\d+) , \- (\d+) \] weightBiases \[.*\] depth= (\d+)/) { $nn = $1; $leaf{$nn} = $TRUE; $conf{$nn} = $2; $poseg{$nn} = $3; $negeg{$nn} = $4; $dep = $5; } else { die "malformed line: $_"; }; }; sub printTree { my($nn, $depth) = @_; print " " x $depth; if ($leaf{$nn}) { print "+ $poseg{$nn} / - $negeg{$nn} # $conf{$nn}\n"; } else { print "+ $poseg{$nn} / - $negeg{$nn} # $fv{$feat{$nn}} $sign{$nn} $thresh{$nn} $conf{$nn}\n"; printTree($pos{$nn}, $depth +1); printTree($neg{$nn}, $depth +1); }; }; bdtl-1.0.3/scripts/standardize.pl0000755000175000017500000000416412330443263016576 0ustar melamedmelamed#!/usr/bin/perl ######################################################################## # Author: I. Dan Melamed # Purpose: standardize features to 0 mean, 1 variance # Loads: entire example set in BDT format # N.B.: representation becomes dense, because all booleans become scalars # N.B.2: all missing values, including scalars, treated as zeros ####################################################################### use autodie qw(:all); $TRUE = 1; $FALSE = 0; # duck compiler warning $TRUE = 1; $FALSE = 0; while (<>) { chomp; push(@ex, $_); ($objective, @feat) = split; foreach $feat (@feat) { if ($feat =~ /(\d+):([\-\d\.]+)/) { # scalar $featNum = $1; $featVal = $2; $isScalar{$featNum} = $TRUE; $sum{$featNum} += $featVal; $sumsq{$featNum} += $featVal ** 2; ++$count{$featNum}; } else { # boolean ++$count{$feat}; } } }; $numEx = scalar @ex; @sortedFeats = sort {$a <=> $b} keys %count; for ($i = 0; $i <= $#sortedFeats; ++$i) { $featNum = $sortedFeats[$i]; $count = $count{$featNum}; if (exists $isScalar{$featNum}) { # this treats missing values as zero $mean[$i] = $sum{$featNum} / $numEx; $std[$i] = sqrt($sumsq{$featNum} / $numEx - ($sum{$featNum} / $numEx) ** 2); } else { $mean[$i] = $count / $numEx; $standardizedOne = 1 - $mean[$i]; $standardizedZero = 0 - $mean[$i]; $numZeros = $numEx - $count; # $sum = $count; # $sumsq = $count * 1 ** 2; $std[$i] = sqrt($count / $numEx - ($count / $numEx) ** 2); } } foreach $ex (@ex) { ($objective, @feat) = split(' ', $ex); $out = $objective; %featVal = (); foreach $feat (@feat) { if ($feat =~ /(\d+):([\-\d\.]+)/) { # scalar $featNum = $1; $featVal{$featNum} = $2; } else { # boolean $featVal{$feat} = $TRUE; } } for ($i = 0; $i <= $#sortedFeats; ++$i) { $featNum = $sortedFeats[$i]; if (not exists $featVal{$featNum}) { $featVal{$featNum} = 0.0; # works for both scalars and booleans } if ($std[$i] <= 0.0) { # can be less due to rounding error $normVal = 0.0; } else { $normVal = ($featVal{$featNum} - $mean[$i]) / $std[$i]; } $out .= " $featNum:$normVal"; } print "$out\n"; }; bdtl-1.0.3/scripts/literalizeRegexp.pl0000755000175000017500000000005712122405072017575 0ustar melamedmelamed#!/usr/bin/perl -p s/(\W)/\\$1/g; s/\\\n/\n/; bdtl-1.0.3/scripts/rms.pl0000755000175000017500000000055612171623303015066 0ustar melamedmelamed#!/usr/bin/perl ######################################################################## # Author: I. Dan Melamed # Computes: root mean squared of a list of numbers ####################################################################### $counter = $ssq = 0; while (<>) { chomp; $ssq += $_ * $_; ++$counter; } printf "%.15f\n", sqrt($ssq / $counter); bdtl-1.0.3/scripts/SVMLInt2Bool.pl0000755000175000017500000000256612122405072016417 0ustar melamedmelamed#!/usr/bin/perl ######################################################################## # Author: I. Dan Melamed # Purpose: convert integer feature values to bools, if that feature's # value appears only as a 0 or 1, where a bool feature name # either appears or not, but never has a colon or value # Streams: examples in SMLight format, twice # N.B.: the result is not strictly in SVMLight format ####################################################################### #check for correct usage if ($#ARGV < 0) { print "usage: $0 \n"; exit; }; $TRUE = 1; $inFile = shift; open(F, $inFile) || die "\nCouldn't open $inFile: $!\n"; while () { chomp; @tok = split; $objective = shift @tok; foreach $tok (@tok) { ($name, $val) = split(/:/, $tok); if ($val ne "1" && $val ne "0") { $nonbool{$name} = $TRUE; delete $bool{$name}; } else { if (! exists $nonbool{$name}) { $bool{$name} = $TRUE; } } } } close F; open(F, $inFile) || die "\nCouldn't open $inFile: $!\n"; while () { chomp; @tok = split; $objective = shift @tok; $out = $objective; foreach $tok (@tok) { ($name, $val) = split(/:/, $tok); if (exists $bool{$name}) { if ($val eq "1") { $out .= " $name"; } else { # skip it } } else { $out .= " $name:$val"; } } print "$out\n"; } close F; bdtl-1.0.3/scripts/SVML2BDT.withDict.sh0000755000175000017500000000100112122405072017216 0ustar melamedmelamed#!/bin/tcsh # Original Author: Dan Melamed # Created on: Jan 6, 2011 # Purpose: convert examples in SVMLight format into a format more suitable for BDT # use the slower SVML2BDT.sh instead if you have no float feature dictionary if ($#argv < 2) then echo "usage: $0 [float feat dict] " exit 0 endif set floatDict=$1 shift set inFile=$1 shift SVMLi2f.withDict.pl $floatDict $inFile > stage1.$$ SVMLInt2Bool.pl stage1.$$ \ | perl -pe 's/^\+1/1/; s/^\-1/0/;' # to stdout rm *.$$ bdtl-1.0.3/Makefile0000644000175000017500000000153212122405072013666 0ustar melamedmelamed# top-level Makefile # we do not want these to be done in parallel .NOTPARALLEL: SUBDIRS = src sandbox # doc/code .PHONY: $(SUBDIRS) # default rule valid: $(SUBDIRS) @echo OK $(SUBDIRS): $(MAKE) -rC $@ clean: for dir in $(SUBDIRS); do \ $(MAKE) -rC $$dir clean || exit; \ done spotless: for dir in $(SUBDIRS); do \ $(MAKE) -rC $$dir spotless || exit; \ done depend: $(MAKE) -rC src depend # use this target to run without validation run: for dir in $(SUBDIRS); do \ echo && echo "--> $$dir start at " `date`; \ $(MAKE) -rC $$dir run || exit; \ done @ echo && echo "--> done at " `date`; \ check: for dir in sandbox; do \ $(MAKE) -rC $$dir check || exit; \ done expected: for dir in sandbox; do \ $(MAKE) -rC $$dir expected || exit; \ done pretend: for dir in sandbox; do \ $(MAKE) -rC $$dir pretend || exit; \ done bdtl-1.0.3/doc/0000755000175000017500000000000012330444352012777 5ustar melamedmelamedbdtl-1.0.3/doc/background/0000755000175000017500000000000012330444352015116 5ustar melamedmelamedbdtl-1.0.3/doc/background/math.tex0000644000175000017500000001342612151020420016562 0ustar melamedmelamed\documentclass{article} \usepackage{fullpage} %\usepackage{hltnaacl03} \usepackage{latexsym,amsmath,array,natbib} % \usepackage{psfig,here} \usepackage{bibtex-cite-macros} \usepackage{boxedminipage} \usepackage{amsmath} \usepackage{algorithmicx} \newcommand{\ignore}[1]{} \newcommand{\bi}{\begin{itemize}} \newcommand{\ei}{\end{itemize}} \newcommand{\be}{\begin{enumerate}} \newcommand{\ee}{\end{enumerate}} \hyphenation{Figure Table Equation} \hyphenation{Section Chapter} \widowpenalty = 10000 \clubpenalty = 10000 \title{Mathematical Background for the {\tt BDTL} software package} \author{ I. Dan Melamed \\ AT\&T Shannon Laboratory \\ Florham Park, NJ, U.S.A., 07932 \\ {\tt \{lastname\}@research.att.com} } \date{} \begin{document} \maketitle \begin{abstract} Sample abstract. \end{abstract} \section{Introduction} Let the atomic feature set be represented by $A$. Let the training set be represented by $I = \{(y_i, X_i)\}$, where $y_i$ are the labels and $X_i$ are the feature value vectors $(x_{i,1}, x_{i,2}, \ldots, x_{i,|A|})$ . Each feature $x_{i,f}$ can be Boolean (0/1) or scalar (integer or real). Every node in a decision tree contains a Boolean atomic feature test $f$. If $f$ involves a Boolean atomic feature $a$, then it will be of the form $a = true$. If it involves a scalar atomic feature, then it will be of the form $a \diamond c$ for some constant $c$ and relation $\diamond$. The possible relations in that case are $<, >, \leq,$ and $\geq$. Let $F$ be the set of possible atomic feature tests. The path from the root to a non-leaf node in a decision tree represents a Boolean conjunction of atomic feature tests, which we call a compound feature test. Elsewhere in the literature, such Boolean expressions are called decision lists. We denote them by $q$ and their 0/1 value on a particular example $i$ is $q(i)$. Since the path from the root to a node in a tree is unique, every non-leaf decision tree node has a unique compound feature test associated with it, in addition to a unique atomic feature test. However, since conjunctions are commutative, a given compound feature test can have its elements in any order, so multiple nodes can be associated with a given compound feature test. Let $Q$ be the set of possible compound feature tests. If all the atomic features in $A$ are Boolean, then $|Q| = 3^{|A|}$ because each atomic feature test can be positive, negative, or absent in a compound feature test. However, when we allow atomic features to be scalar, the number of possible compound feature tests becomes much larger, because every scalar feature value can be compared to many possible values of $c$. Let the model be represented by $\Theta = (\theta_1, \theta_2, \ldots, \theta_{|Q|})$ . There is one parameter for every possible compound feature test, though usually most of their values will be zero. Let the hypothesis function be represented by \begin{equation} h(i) = \Theta \cdot Q(i) = \sum_{k = 1}^{|Q|} \theta_k q_k(i) \end{equation} We grow decision trees by splitting their leaves to minimize risk over some training set, following \cite{TurianThesis}. For each leaf node $n$, we choose a new feature test $\hat{f}$ to maximize a gain function $G$ over the examples $I$ in $n$. Let $q$ be the compound feature test associated with the parent node of $n$. Then \begin{equation} \hat{f} = {\arg \max}_{f \in F} [ G(I ; q \wedge f) + G(I ; q \wedge \neg f) ] \end{equation} where $\wedge$ is conjunction and $\neg$ is negation. $G(I ; p)$ involves the gradient of the loss function over $I$ with respect to the model parameter $\theta_p$. \ignore{ For now we assume a loss function that treats each example independently: \begin{equation} L(I) = \sum_{i \in I} l(i) \end{equation} } \section{Regression} We use the squared loss \begin{equation} L(I) = \sum_{i \in I} l(i) = \sum_{i \in I} \frac{1}{2} (y_i - h(i))^2 \end{equation} Let $\mu(i) = y_i - h(i)$ so that $l(i) = \frac{1}{2} \mu(i)^2$ . To find the gradient, we have: \begin{eqnarray} \frac{\partial L(I)}{\partial \theta_k} & = & \sum_{i \in I} \frac{\partial l(i)}{\partial \theta_k} \\ & = & \sum_{i \in I} \frac{\partial l(i)}{\partial \mu(i)} \cdot \frac{\partial \mu(i)}{\partial \theta_k} \\ & = & \sum_{i \in I} \mu(i) \cdot -q_k(i) \\ & = & \sum_{i \in I} (h(i) - y_i) q_k(i) \\ \label{eqn:feas} & = & \sum_{i \in I : q_k(i) = 1} h(i) - y_i \\ & = & \left(\sum_{i \in I : q_k(i) = 1, h(i) \geq y_i} h(i) - y_i \right) - \left(\sum_{i \in I : q_k(i) = 1, h(i) < y_i} y_i - h(i) \right) \end{eqnarray} Equation~\ref{eqn:feas} works because $q$ can be only 0 or 1, and it is what makes this optimization feasible. \ignore{ \subsection{Training algorithm} \begin{Listing}[thb] \caption{Training algorithm.} \label{alg:training} \begin{algorithmic}[1] \Procedure{Find best split point on scalar feature}{f} \State find $G_\Theta(f)$ \label{alg:training:conf-update-begin} \State choose \alpha_f to minimize \objective \label{alg:training:objective-minimization} \State add \alpha_f to $h(i)$ for all i in this leaf \label{alg:training:confidence-update} \EndFor \label{alg:training:conf-update-end} \EndFor \EndProcedure \end{algorithmic} \end{Listing} } \ignore{ \begin{figure} \centerline{ % (a) %\raisebox{.2in}{\psfig{figure=parsematrix.eps,width=1.9in}} \hfill (b) %\psfig{figure=venn.eps,width=1.1in} } \caption{Generalizations of ordinary parsing. % $D = $ dimensionality of grammar; $I =$ dimensionality of input. % (a) Classification with respect to dimensionality; (b) Venn diagram. \label{venn} \label{parsematrix}} \end{figure} \section{Conclusion} } \section*{Acknowledgments} Thanks to Pannaga Shivaswamy for helpful discussions. %%%%%%%%%%%%%%%%%% BIBLIOGRAPHY \bibliographystyle{bibtex-cite-macros} %\bibliography{mt} \end{document} bdtl-1.0.3/doc/background/bibtex-cite-macros.sty0000644000175000017500000002266612122405072021347 0ustar melamedmelamed% File acl04.sty % January 07, 2004 % Contact: rambow@cs.columbia.edu % This is the LaTeX style file for ACL 2004. It is identical to the % style files for ACL 2003, ACL 2002, ACL 2001, ACL 2000, EACL 95 and EACL % 99. % % -- Roberto Zamparelli, March 26, 2001 % -- Dekang Lin, October 1, 2001 % This is the LaTeX style file for ACL 2000. It is nearly identical to the % style files for EACL 95 and EACL 99. Minor changes include editing the % instructions to reflect use of \documentclass rather than \documentstyle % and removing the white space before the title on the first page % -- John Chen, June 29, 2000 % To convert from submissions prepared using the style file aclsub.sty % prepared for the ACL 2000 conference, proceed as follows: % 1) Remove submission-specific information: \whichsession, \id, % \wordcount, \otherconferences, \area, \keywords % 2) \summary should be removed. The summary material should come % after \maketitle and should be in the ``abstract'' environment % 3) Check all citations. This style should handle citations correctly % and also allows multiple citations separated by semicolons. % 4) Check figures and examples. Because the final format is double- % column, some adjustments may have to be made to fit text in the column % or to choose full-width (\figure*} figures. % 5) Change the style reference from aclsub to acl2000, and be sure % this style file is in your TeX search path % This is the LaTeX style file for EACL-95. It is identical to the % style file for ANLP '94 except that the margins are adjusted for A4 % paper. -- abney 13 Dec 94 % The ANLP '94 style file is a slightly modified % version of the style used for AAAI and IJCAI, using some changes % prepared by Fernando Pereira and others and some minor changes % by Paul Jacobs. % Papers prepared using the aclsub.sty file and acl.bst bibtex style % should be easily converted to final format using this style. % (1) Submission information (\wordcount, \subject, and \makeidpage) % should be removed. % (2) \summary should be removed. The summary material should come % after \maketitle and should be in the ``abstract'' environment % (between \begin{abstract} and \end{abstract}). % (3) Check all citations. This style should handle citations correctly % and also allows multiple citations separated by semicolons. % (4) Check figures and examples. Because the final format is double- % column, some adjustments may have to be made to fit text in the column % or to choose full-width (\figure*} figures. % Place this in a file called aclap.sty in the TeX search path. % (Placing it in the same directory as the paper should also work.) % Prepared by Peter F. Patel-Schneider, liberally using the ideas of % other style hackers, including Barbara Beeton. % This style is NOT guaranteed to work. It is provided in the hope % that it will make the preparation of papers easier. % % There are undoubtably bugs in this style. If you make bug fixes, % improvements, etc. please let me know. My e-mail address is: % pfps@research.att.com % Papers are to be prepared using the ``acl'' bibliography style, % as follows: % \documentclass[11pt]{article} % \usepackage{acl2000} % \title{Title} % \author{Author 1 \and Author 2 \\ Address line \\ Address line \And % Author 3 \\ Address line \\ Address line} % \begin{document} % ... % \bibliography{bibliography-file} % \bibliographystyle{acl} % \end{document} % Author information can be set in various styles: % For several authors from the same institution: % \author{Author 1 \and ... \and Author n \\ % Address line \\ ... \\ Address line} % if the names do not fit well on one line use % Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\ % For authors from different institutions: % \author{Author 1 \\ Address line \\ ... \\ Address line % \And ... \And % Author n \\ Address line \\ ... \\ Address line} % To start a seperate ``row'' of authors use \AND, as in % \author{Author 1 \\ Address line \\ ... \\ Address line % \AND % Author 2 \\ Address line \\ ... \\ Address line \And % Author 3 \\ Address line \\ ... \\ Address line} % If the title and author information does not fit in the area allocated, % place \setlength\titlebox{} right after % \usepackage{acl2000} % where can be something larger than 2.25in % \typeout{Conference Style for ACL 2000 -- released June 20, 2000} \typeout{Conference Style for ACL 2002 -- released April 8, 2002} % NOTE: Some laser printers have a serious problem printing TeX output. % These printing devices, commonly known as ``write-white'' laser % printers, tend to make characters too light. To get around this % problem, a darker set of fonts must be created for these devices. % % % Physical page layout - slightly modified from IJCAI by pj % \setlength\topmargin{0.0in} \setlength\oddsidemargin{-0.0in} % \setlength\textheight{9.0in} \setlength\textwidth{6.5in} % \setlength\columnsep{0.2in} % \newlength\titlebox % \setlength\titlebox{2.25in} % \setlength\headheight{0pt} \setlength\headsep{0pt} % %\setlength\footheight{0pt} % \setlength\footskip{0pt} % \thispagestyle{empty} \pagestyle{empty} % \flushbottom \twocolumn \sloppy % %% A4 version of page layout % \setlength\topmargin{-0.45cm} % changed by Rz -1.4 % \setlength\oddsidemargin{.8mm} % was -0cm, changed by Rz % \setlength\textheight{23.5cm} % \setlength\textwidth{15.8cm} % \setlength\columnsep{0.6cm} % \newlength\titlebox % \setlength\titlebox{2.00in} % \setlength\headheight{5pt} % \setlength\headsep{0pt} % %\setlength\footheight{0pt} % \setlength\footskip{0pt} % \thispagestyle{empty} % \pagestyle{empty} % most of cite format is from aclsub.sty by SMS % don't box citations, separate with ; and a space % also, make the penalty between citations negative: a good place to break % changed comma back to semicolon pj 2/1/90 % \def\@citex[#1]#2{\if@filesw\immediate\write\@auxout{\string\citation{#2}}\fi % \def\@citea{}\@cite{\@for\@citeb:=#2\do % {\@citea\def\@citea{;\penalty\@citeseppen\ }\@ifundefined % {b@\@citeb}{{\bf ?}\@warning % {Citation `\@citeb' on page \thepage \space undefined}}% % {\csname b@\@citeb\endcsname}}}{#1}} % don't box citations, separate with ; and a space % Replaced for multiple citations (pj) % don't box citations and also add space, semicolon between multiple citations \def\@citex[#1]#2{\if@filesw\immediate\write\@auxout{\string\citation{#2}}\fi \def\@citea{}\@cite{\@for\@citeb:=#2\do {\@citea\def\@citea{; }\@ifundefined {b@\@citeb}{{\bf ?}\@warning {Citation `\@citeb' on page \thepage \space undefined}}% {\csname b@\@citeb\endcsname}}}{#1}} % Allow short (name-less) citations, when used in % conjunction with a bibliography style that creates labels like % \citename{, } % \let\@internalcite\cite \def\cite{\def\citename##1{##1, }\@internalcite} \def\shortcite{\def\citename##1{}\@internalcite} \def\newcite{\leavevmode\def\citename##1{{##1} (}\@internalciteb} % Macros for \newcite, which leaves name in running text, and is % otherwise like \shortcite. \def\@citexb[#1]#2{\if@filesw\immediate\write\@auxout{\string\citation{#2}}\fi \def\@citea{}\@newcite{\@for\@citeb:=#2\do {\@citea\def\@citea{;\penalty\@m\ }\@ifundefined {b@\@citeb}{{\bf ?}\@warning {Citation `\@citeb' on page \thepage \space undefined}}% \hbox{\csname b@\@citeb\endcsname}}}{#1}} \def\@internalciteb{\@ifnextchar [{\@tempswatrue\@citexb}{\@tempswafalse\@citexb[]}} \def\@newcite#1#2{{#1\if@tempswa, #2\fi)}} \def\@biblabel#1{\def\citename##1{##1}[#1]\hfill} %%% More changes made by SMS (originals in latex.tex) % Use parentheses instead of square brackets in the text. \def\@cite#1#2{({#1\if@tempswa , #2\fi})} % Don't put a label in the bibliography at all. Just use the unlabeled format % instead. \def\thebibliography#1{\vskip\parskip% \vskip\baselineskip% \def\baselinestretch{1}% \ifx\@currsize\normalsize\@normalsize\else\@currsize\fi% \vskip-\parskip% \vskip-\baselineskip% \section*{References\@mkboth {References}{References}}\list {}{\setlength{\labelwidth}{0pt}\setlength{\leftmargin}{\parindent} \setlength{\itemindent}{-\parindent}} \def\newblock{\hskip .11em plus .33em minus -.07em} \sloppy\clubpenalty4000\widowpenalty4000 \sfcode`\.=1000\relax} \let\endthebibliography=\endlist % Allow for a bibliography of sources of attested examples \def\thesourcebibliography#1{\vskip\parskip% \vskip\baselineskip% \def\baselinestretch{1}% \ifx\@currsize\normalsize\@normalsize\else\@currsize\fi% \vskip-\parskip% \vskip-\baselineskip% \section*{Sources of Attested Examples\@mkboth {Sources of Attested Examples}{Sources of Attested Examples}}\list {}{\setlength{\labelwidth}{0pt}\setlength{\leftmargin}{\parindent} \setlength{\itemindent}{-\parindent}} \def\newblock{\hskip .11em plus .33em minus -.07em} \sloppy\clubpenalty4000\widowpenalty4000 \sfcode`\.=1000\relax} \let\endthesourcebibliography=\endlist \def\@lbibitem[#1]#2{\item[]\if@filesw { \def\protect##1{\string ##1\space}\immediate \write\@auxout{\string\bibcite{#2}{#1}}\fi\ignorespaces}} \def\@bibitem#1{\item\if@filesw \immediate\write\@auxout {\string\bibcite{#1}{\the\c@enumi}}\fi\ignorespaces} bdtl-1.0.3/doc/background/Makefile0000644000175000017500000000467012122405072016560 0ustar melamedmelamed# $Id: Makefile,v 1.1 2004/09/14 00:30:12 melamed Exp $ # Usage: "make help" or simply "make" ################################################################## SHELL = /bin/bash ## Disable the standard LaTeX rule. %.dvi: %.tex #%.pdf: %.ps %.pdf: %.tex SUFF = *.bak *.out *.toc *.log *.aux *.idx *.ind *.ilg \ *.lot *.loc *.lof *.bbl *.blg *.tar *.gz SUFF1 := $(patsubst *.%,\%.%,$(SUFF)) FIGS := $(patsubst %.fig,%.eps,$(wildcard *.fig)) DIAS := $(patsubst %.dia,%.eps,$(wildcard *.dia)) HTMLDIR = .html # these targets will not be treated as files. .PHONY: help clean # The "make" program will not automatically delete intermediate files with # the following suffixes. .PRECIOUS: $(SUFF1) %.eps %.dvi %.ps %.pdf .IGNORE: %.bbl ## programs LATEX = latex BIBTEX = -bibtex --min-crossrefs=100 LATEX2HTML = latex2html PS2PDF = ps2pdf PDFLATEX = pdflatex PDF2PS = pdf2ps EPS2PDF = epstopdf FIG2DEV = fig2dev DVI2PS = dvips ## TARGETS help: @echo "LaTeX Makefile v2.0" @echo "----------------------" @echo "Targets:" @echo " help (or nothing) Show this message" @echo " Make the .dvi file" @echo " .dvi Make the .dvi file" @echo " .ps Make the .ps file" @echo " .pdf Make the .pdf file" @echo " .html Make the html files in the .html directory" @echo " .tar Tar the directory" @echo " .tar.gz Tar and gzip the directory" @echo " clean Remove all generated files" @echo %.tar: @echo "Generating $@..." @tar cvf - . > $@ @echo "done" %.gz: % @gzip < $< > $@ clean: @echo "cleaning files..." rm -rf $(SUFF) $(FIGS) $(DIAS) *.ps *.pdf *.dvi @if [ -d $(HTMLDIR) ]; then rm -rf $(HTMLDIR); fi @echo "done." %.bbl: %.tex $(FIGS) $(DIAS) echo the figures are $(FIGS) $(DIAS) $(LATEX) $< $(BIBTEX) -min-crossrefs=100 $* %.dvi: %.tex *.tex %.bbl $(FIGS) $(DIAS) $(LATEX) $< $(LATEX) $< %.ps: %.dvi $(DVI2PS) $< -o $@ #%.ps: %.pdf # $(PDF2PS) $< $@ %.eps: %.fig @echo "Generating $@..." @$(FIG2DEV) -L eps -p portrait $< > $@ %.eps: %.dia echo "Generating $@..." dia -e $@ $< #%.pdf: %.eps # $(EPS2PDF) $< > $@ #%.pdf: %.ps # $(PS2PDF) $< $@ %.pdf: %.tex $(PDFLATEX) $< %.html: %.dvi @echo -n "Creating the html files..." @if [ ! -d "$(HTMLDIR)" ]; then mkdir $(HTMLDIR); fi @$(LATEX2HTML) -dir $(HTMLDIR) $* >& /dev/null; @echo "done" %: %.dvi bdtl-1.0.3/doc/background/bibtex-cite-macros.bst0000644000175000017500000005610312122405072021311 0ustar melamedmelamed% BibTeX `acl' style file for BibTeX version 0.99c, LaTeX version 2.09 % This version was made by modifying `aaai-named' format based on the master % file by Oren Patashnik (PATASHNIK@SCORE.STANFORD.EDU) % Copyright (C) 1985, all rights reserved. % Modifications Copyright 1988, Peter F. Patel-Schneider % Further modifictions by Stuart Shieber, 1991, and Fernando Pereira, 1992. % Copying of this file is authorized only if either % (1) you make absolutely no changes to your copy, including name, or % (2) if you do make changes, you name it something other than % btxbst.doc, plain.bst, unsrt.bst, alpha.bst, and abbrv.bst. % This restriction helps ensure that all standard styles are identical. % There are undoubtably bugs in this style. If you make bug fixes, % improvements, etc. please let me know. My e-mail address is: % pfps@spar.slb.com % Citation format: [author-last-name, year] % [author-last-name and author-last-name, year] % [author-last-name {\em et al.}, year] % % Reference list ordering: alphabetical by author or whatever passes % for author in the absence of one. % % This BibTeX style has support for short (year only) citations. This % is done by having the citations actually look like % \citename{name-info, }year % The LaTeX style has to have the following % \let\@internalcite\cite % \def\cite{\def\citename##1{##1}\@internalcite} % \def\shortcite{\def\citename##1{}\@internalcite} % \def\@biblabel#1{\def\citename##1{##1}[#1]\hfill} % which makes \shortcite the macro for short citations. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % Changes made by SMS for thesis style % no emphasis on "et al." % "Ph.D." includes periods (not "PhD") % moved year to immediately after author's name %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% ENTRY { address author booktitle chapter edition editor howpublished institution journal key month note number organization pages publisher school series title type volume year } {} { label extra.label sort.label } INTEGERS { output.state before.all mid.sentence after.sentence after.block } FUNCTION {init.state.consts} { #0 'before.all := #1 'mid.sentence := #2 'after.sentence := #3 'after.block := } STRINGS { s t } FUNCTION {output.nonnull} { 's := output.state mid.sentence = { ", " * write$ } { output.state after.block = { add.period$ write$ newline$ "\newblock " write$ } { output.state before.all = 'write$ { add.period$ " " * write$ } if$ } if$ mid.sentence 'output.state := } if$ s } FUNCTION {output} { duplicate$ empty$ 'pop$ 'output.nonnull if$ } FUNCTION {output.check} { 't := duplicate$ empty$ { pop$ "empty " t * " in " * cite$ * warning$ } 'output.nonnull if$ } FUNCTION {output.bibitem} { newline$ "\bibitem[" write$ label write$ "]{" write$ cite$ write$ "}" write$ newline$ "" before.all 'output.state := } FUNCTION {fin.entry} { add.period$ write$ newline$ } FUNCTION {new.block} { output.state before.all = 'skip$ { after.block 'output.state := } if$ } FUNCTION {new.sentence} { output.state after.block = 'skip$ { output.state before.all = 'skip$ { after.sentence 'output.state := } if$ } if$ } FUNCTION {not} { { #0 } { #1 } if$ } FUNCTION {and} { 'skip$ { pop$ #0 } if$ } FUNCTION {or} { { pop$ #1 } 'skip$ if$ } FUNCTION {new.block.checka} { empty$ 'skip$ 'new.block if$ } FUNCTION {new.block.checkb} { empty$ swap$ empty$ and 'skip$ 'new.block if$ } FUNCTION {new.sentence.checka} { empty$ 'skip$ 'new.sentence if$ } FUNCTION {new.sentence.checkb} { empty$ swap$ empty$ and 'skip$ 'new.sentence if$ } FUNCTION {field.or.null} { duplicate$ empty$ { pop$ "" } 'skip$ if$ } FUNCTION {emphasize} { duplicate$ empty$ { pop$ "" } { "{\em " swap$ * "}" * } if$ } INTEGERS { nameptr namesleft numnames } FUNCTION {format.names} { 's := #1 'nameptr := s num.names$ 'numnames := numnames 'namesleft := { namesleft #0 > } { s nameptr "{ff~}{vv~}{ll}{, jj}" format.name$ 't := nameptr #1 > { namesleft #1 > { ", " * t * } { numnames #2 > { "," * } 'skip$ if$ t "others" = { " et~al." * } { " and " * t * } if$ } if$ } 't if$ nameptr #1 + 'nameptr := namesleft #1 - 'namesleft := } while$ } FUNCTION {format.authors} { author empty$ { "" } { author format.names } if$ } FUNCTION {format.editors} { editor empty$ { "" } { editor format.names editor num.names$ #1 > { ", editors" * } { ", editor" * } if$ } if$ } FUNCTION {format.title} { title empty$ { "" } { title "t" change.case$ } if$ } FUNCTION {n.dashify} { 't := "" { t empty$ not } { t #1 #1 substring$ "-" = { t #1 #2 substring$ "--" = not { "--" * t #2 global.max$ substring$ 't := } { { t #1 #1 substring$ "-" = } { "-" * t #2 global.max$ substring$ 't := } while$ } if$ } { t #1 #1 substring$ * t #2 global.max$ substring$ 't := } if$ } while$ } FUNCTION {format.date} { year empty$ { month empty$ { "" } { "there's a month but no year in " cite$ * warning$ month } if$ } { month empty$ { "" } { month } if$ } if$ } FUNCTION {format.btitle} { title emphasize } FUNCTION {tie.or.space.connect} { duplicate$ text.length$ #3 < { "~" } { " " } if$ swap$ * * } FUNCTION {either.or.check} { empty$ 'pop$ { "can't use both " swap$ * " fields in " * cite$ * warning$ } if$ } FUNCTION {format.bvolume} { volume empty$ { "" } { "volume" volume tie.or.space.connect series empty$ 'skip$ { " of " * series emphasize * } if$ "volume and number" number either.or.check } if$ } FUNCTION {format.number.series} { volume empty$ { number empty$ { series field.or.null } { output.state mid.sentence = { "number" } { "Number" } if$ number tie.or.space.connect series empty$ { "there's a number but no series in " cite$ * warning$ } { " in " * series * } if$ } if$ } { "" } if$ } FUNCTION {format.edition} { edition empty$ { "" } { output.state mid.sentence = { edition "l" change.case$ " edition" * } { edition "t" change.case$ " edition" * } if$ } if$ } INTEGERS { multiresult } FUNCTION {multi.page.check} { 't := #0 'multiresult := { multiresult not t empty$ not and } { t #1 #1 substring$ duplicate$ "-" = swap$ duplicate$ "," = swap$ "+" = or or { #1 'multiresult := } { t #2 global.max$ substring$ 't := } if$ } while$ multiresult } FUNCTION {format.pages} { pages empty$ { "" } { pages multi.page.check { "pages" pages n.dashify tie.or.space.connect } { "page" pages tie.or.space.connect } if$ } if$ } FUNCTION {format.year.label} { year extra.label * } FUNCTION {format.vol.num.pages} { volume field.or.null number empty$ 'skip$ { "(" number * ")" * * volume empty$ { "there's a number but no volume in " cite$ * warning$ } 'skip$ if$ } if$ pages empty$ 'skip$ { duplicate$ empty$ { pop$ format.pages } { ":" * pages n.dashify * } if$ } if$ } FUNCTION {format.chapter.pages} { chapter empty$ 'format.pages { type empty$ { "chapter" } { type "l" change.case$ } if$ chapter tie.or.space.connect pages empty$ 'skip$ { ", " * format.pages * } if$ } if$ } FUNCTION {format.in.ed.booktitle} { booktitle empty$ { "" } { editor empty$ { "In " booktitle emphasize * } { "In " format.editors * ", " * booktitle emphasize * } if$ } if$ } FUNCTION {empty.misc.check} { author empty$ title empty$ howpublished empty$ month empty$ year empty$ note empty$ and and and and and key empty$ not and { "all relevant fields are empty in " cite$ * warning$ } 'skip$ if$ } FUNCTION {format.thesis.type} { type empty$ 'skip$ { pop$ type "t" change.case$ } if$ } FUNCTION {format.tr.number} { type empty$ { "Technical Report" } 'type if$ number empty$ { "t" change.case$ } { number tie.or.space.connect } if$ } FUNCTION {format.article.crossref} { key empty$ { journal empty$ { "need key or journal for " cite$ * " to crossref " * crossref * warning$ "" } { "In {\em " journal * "\/}" * } if$ } { "In " key * } if$ " \cite{" * crossref * "}" * } FUNCTION {format.crossref.editor} { editor #1 "{vv~}{ll}" format.name$ editor num.names$ duplicate$ #2 > { pop$ " et~al." * } { #2 < 'skip$ { editor #2 "{ff }{vv }{ll}{ jj}" format.name$ "others" = { " et~al." * } { " and " * editor #2 "{vv~}{ll}" format.name$ * } if$ } if$ } if$ } FUNCTION {format.book.crossref} { volume empty$ { "empty volume in " cite$ * "'s crossref of " * crossref * warning$ "In " } { "Volume" volume tie.or.space.connect " of " * } if$ editor empty$ editor field.or.null author field.or.null = or { key empty$ { series empty$ { "need editor, key, or series for " cite$ * " to crossref " * crossref * warning$ "" * } { "{\em " * series * "\/}" * } if$ } { key * } if$ } { format.crossref.editor * } if$ " \cite{" * crossref * "}" * } FUNCTION {format.incoll.inproc.crossref} { editor empty$ editor field.or.null author field.or.null = or { key empty$ { booktitle empty$ { "need editor, key, or booktitle for " cite$ * " to crossref " * crossref * warning$ "" } { "In {\em " booktitle * "\/}" * } if$ } { "In " key * } if$ } { "In " format.crossref.editor * } if$ " \cite{" * crossref * "}" * } FUNCTION {article} { output.bibitem format.authors "author" output.check new.block format.year.label "year" output.check new.block format.title "title" output.check new.block crossref missing$ { journal emphasize "journal" output.check format.vol.num.pages output format.date output } { format.article.crossref output.nonnull format.pages output } if$ new.block note output fin.entry } FUNCTION {book} { output.bibitem author empty$ { format.editors "author and editor" output.check } { format.authors output.nonnull crossref missing$ { "author and editor" editor either.or.check } 'skip$ if$ } if$ new.block format.year.label "year" output.check new.block format.btitle "title" output.check crossref missing$ { format.bvolume output new.block format.number.series output new.sentence publisher "publisher" output.check address output } { new.block format.book.crossref output.nonnull } if$ format.edition output format.date output new.block note output fin.entry } FUNCTION {booklet} { output.bibitem format.authors output new.block format.year.label "year" output.check new.block format.title "title" output.check howpublished address new.block.checkb howpublished output address output format.date output new.block note output fin.entry } FUNCTION {inbook} { output.bibitem author empty$ { format.editors "author and editor" output.check } { format.authors output.nonnull crossref missing$ { "author and editor" editor either.or.check } 'skip$ if$ } if$ format.year.label "year" output.check new.block new.block format.btitle "title" output.check crossref missing$ { format.bvolume output format.chapter.pages "chapter and pages" output.check new.block format.number.series output new.sentence publisher "publisher" output.check address output } { format.chapter.pages "chapter and pages" output.check new.block format.book.crossref output.nonnull } if$ format.edition output format.date output new.block note output fin.entry } FUNCTION {incollection} { output.bibitem format.authors "author" output.check new.block format.year.label "year" output.check new.block format.title "title" output.check new.block crossref missing$ { format.in.ed.booktitle "booktitle" output.check format.bvolume output format.number.series output format.chapter.pages output new.sentence publisher "publisher" output.check address output format.edition output format.date output } { format.incoll.inproc.crossref output.nonnull format.chapter.pages output } if$ new.block note output fin.entry } FUNCTION {inproceedings} { output.bibitem format.authors "author" output.check new.block format.year.label "year" output.check new.block format.title "title" output.check new.block crossref missing$ { format.in.ed.booktitle "booktitle" output.check format.bvolume output format.number.series output format.pages output address empty$ { organization publisher new.sentence.checkb organization output publisher output format.date output } { address output.nonnull format.date output new.sentence organization output publisher output } if$ } { format.incoll.inproc.crossref output.nonnull format.pages output } if$ new.block note output fin.entry } FUNCTION {conference} { inproceedings } FUNCTION {manual} { output.bibitem author empty$ { organization empty$ 'skip$ { organization output.nonnull address output } if$ } { format.authors output.nonnull } if$ format.year.label "year" output.check new.block new.block format.btitle "title" output.check author empty$ { organization empty$ { address new.block.checka address output } 'skip$ if$ } { organization address new.block.checkb organization output address output } if$ format.edition output format.date output new.block note output fin.entry } FUNCTION {mastersthesis} { output.bibitem format.authors "author" output.check new.block format.year.label "year" output.check new.block format.title "title" output.check new.block "Master's thesis" format.thesis.type output.nonnull school "school" output.check address output format.date output new.block note output fin.entry } FUNCTION {misc} { output.bibitem format.authors output new.block format.year.label output new.block title howpublished new.block.checkb format.title output howpublished new.block.checka howpublished output format.date output new.block note output fin.entry empty.misc.check } FUNCTION {phdthesis} { output.bibitem format.authors "author" output.check new.block format.year.label "year" output.check new.block format.btitle "title" output.check new.block "{Ph.D.} thesis" format.thesis.type output.nonnull school "school" output.check address output format.date output new.block note output fin.entry } FUNCTION {proceedings} { output.bibitem editor empty$ { organization output } { format.editors output.nonnull } if$ new.block format.year.label "year" output.check new.block format.btitle "title" output.check format.bvolume output format.number.series output address empty$ { editor empty$ { publisher new.sentence.checka } { organization publisher new.sentence.checkb organization output } if$ publisher output format.date output } { address output.nonnull format.date output new.sentence editor empty$ 'skip$ { organization output } if$ publisher output } if$ new.block note output fin.entry } FUNCTION {techreport} { output.bibitem format.authors "author" output.check new.block format.year.label "year" output.check new.block format.title "title" output.check new.block format.tr.number output.nonnull institution "institution" output.check address output format.date output new.block note output fin.entry } FUNCTION {unpublished} { output.bibitem format.authors "author" output.check new.block format.year.label "year" output.check new.block format.title "title" output.check new.block note "note" output.check format.date output fin.entry } FUNCTION {default.type} { misc } MACRO {jan} {"January"} MACRO {feb} {"February"} MACRO {mar} {"March"} MACRO {apr} {"April"} MACRO {may} {"May"} MACRO {jun} {"June"} MACRO {jul} {"July"} MACRO {aug} {"August"} MACRO {sep} {"September"} MACRO {oct} {"October"} MACRO {nov} {"November"} MACRO {dec} {"December"} MACRO {acmcs} {"ACM Computing Surveys"} MACRO {acta} {"Acta Informatica"} MACRO {cacm} {"Communications of the ACM"} MACRO {ibmjrd} {"IBM Journal of Research and Development"} MACRO {ibmsj} {"IBM Systems Journal"} MACRO {ieeese} {"IEEE Transactions on Software Engineering"} MACRO {ieeetc} {"IEEE Transactions on Computers"} MACRO {ieeetcad} {"IEEE Transactions on Computer-Aided Design of Integrated Circuits"} MACRO {ipl} {"Information Processing Letters"} MACRO {jacm} {"Journal of the ACM"} MACRO {jcss} {"Journal of Computer and System Sciences"} MACRO {scp} {"Science of Computer Programming"} MACRO {sicomp} {"SIAM Journal on Computing"} MACRO {tocs} {"ACM Transactions on Computer Systems"} MACRO {tods} {"ACM Transactions on Database Systems"} MACRO {tog} {"ACM Transactions on Graphics"} MACRO {toms} {"ACM Transactions on Mathematical Software"} MACRO {toois} {"ACM Transactions on Office Information Systems"} MACRO {toplas} {"ACM Transactions on Programming Languages and Systems"} MACRO {tcs} {"Theoretical Computer Science"} READ FUNCTION {sortify} { purify$ "l" change.case$ } INTEGERS { len } FUNCTION {chop.word} { 's := 'len := s #1 len substring$ = { s len #1 + global.max$ substring$ } 's if$ } INTEGERS { et.al.char.used } FUNCTION {initialize.et.al.char.used} { #0 'et.al.char.used := } EXECUTE {initialize.et.al.char.used} FUNCTION {format.lab.names} { 's := s num.names$ 'numnames := numnames #1 = { s #1 "{vv }{ll}" format.name$ } { numnames #2 = { s #1 "{vv }{ll }and " format.name$ s #2 "{vv }{ll}" format.name$ * } { s #1 "{vv }{ll }\bgroup et al.\egroup " format.name$ } if$ } if$ } FUNCTION {author.key.label} { author empty$ { key empty$ { cite$ #1 #3 substring$ } { key #3 text.prefix$ } if$ } { author format.lab.names } if$ } FUNCTION {author.editor.key.label} { author empty$ { editor empty$ { key empty$ { cite$ #1 #3 substring$ } { key #3 text.prefix$ } if$ } { editor format.lab.names } if$ } { author format.lab.names } if$ } FUNCTION {author.key.organization.label} { author empty$ { key empty$ { organization empty$ { cite$ #1 #3 substring$ } { "The " #4 organization chop.word #3 text.prefix$ } if$ } { key #3 text.prefix$ } if$ } { author format.lab.names } if$ } FUNCTION {editor.key.organization.label} { editor empty$ { key empty$ { organization empty$ { cite$ #1 #3 substring$ } { "The " #4 organization chop.word #3 text.prefix$ } if$ } { key #3 text.prefix$ } if$ } { editor format.lab.names } if$ } FUNCTION {calc.label} { type$ "book" = type$ "inbook" = or 'author.editor.key.label { type$ "proceedings" = 'editor.key.organization.label { type$ "manual" = 'author.key.organization.label 'author.key.label if$ } if$ } if$ duplicate$ "\protect\citename{" swap$ * "}" * year field.or.null purify$ * 'label := year field.or.null purify$ * sortify 'sort.label := } FUNCTION {sort.format.names} { 's := #1 'nameptr := "" s num.names$ 'numnames := numnames 'namesleft := { namesleft #0 > } { nameptr #1 > { " " * } 'skip$ if$ s nameptr "{vv{ } }{ll{ }}{ ff{ }}{ jj{ }}" format.name$ 't := nameptr numnames = t "others" = and { "et al" * } { t sortify * } if$ nameptr #1 + 'nameptr := namesleft #1 - 'namesleft := } while$ } FUNCTION {sort.format.title} { 't := "A " #2 "An " #3 "The " #4 t chop.word chop.word chop.word sortify #1 global.max$ substring$ } FUNCTION {author.sort} { author empty$ { key empty$ { "to sort, need author or key in " cite$ * warning$ "" } { key sortify } if$ } { author sort.format.names } if$ } FUNCTION {author.editor.sort} { author empty$ { editor empty$ { key empty$ { "to sort, need author, editor, or key in " cite$ * warning$ "" } { key sortify } if$ } { editor sort.format.names } if$ } { author sort.format.names } if$ } FUNCTION {author.organization.sort} { author empty$ { organization empty$ { key empty$ { "to sort, need author, organization, or key in " cite$ * warning$ "" } { key sortify } if$ } { "The " #4 organization chop.word sortify } if$ } { author sort.format.names } if$ } FUNCTION {editor.organization.sort} { editor empty$ { organization empty$ { key empty$ { "to sort, need editor, organization, or key in " cite$ * warning$ "" } { key sortify } if$ } { "The " #4 organization chop.word sortify } if$ } { editor sort.format.names } if$ } FUNCTION {presort} { calc.label sort.label " " * type$ "book" = type$ "inbook" = or 'author.editor.sort { type$ "proceedings" = 'editor.organization.sort { type$ "manual" = 'author.organization.sort 'author.sort if$ } if$ } if$ * " " * year field.or.null sortify * " " * title field.or.null sort.format.title * #1 entry.max$ substring$ 'sort.key$ := } ITERATE {presort} SORT STRINGS { longest.label last.sort.label next.extra } INTEGERS { longest.label.width last.extra.num } FUNCTION {initialize.longest.label} { "" 'longest.label := #0 int.to.chr$ 'last.sort.label := "" 'next.extra := #0 'longest.label.width := #0 'last.extra.num := } FUNCTION {forward.pass} { last.sort.label sort.label = { last.extra.num #1 + 'last.extra.num := last.extra.num int.to.chr$ 'extra.label := } { "a" chr.to.int$ 'last.extra.num := "" 'extra.label := sort.label 'last.sort.label := } if$ } FUNCTION {reverse.pass} { next.extra "b" = { "a" 'extra.label := } 'skip$ if$ label extra.label * 'label := label width$ longest.label.width > { label 'longest.label := label width$ 'longest.label.width := } 'skip$ if$ extra.label 'next.extra := } EXECUTE {initialize.longest.label} ITERATE {forward.pass} REVERSE {reverse.pass} FUNCTION {begin.bib} { et.al.char.used { "\newcommand{\etalchar}[1]{$^{#1}$}" write$ newline$ } 'skip$ if$ preamble$ empty$ 'skip$ { preamble$ write$ newline$ } if$ "\begin{thebibliography}{" "}" * write$ newline$ } EXECUTE {begin.bib} EXECUTE {init.state.consts} ITERATE {call.type$} FUNCTION {end.bib} { newline$ "\end{thebibliography}" write$ newline$ } EXECUTE {end.bib} bdtl-1.0.3/doc/background/algorithmicx.sty0000644000175000017500000006417612122405072020362 0ustar melamedmelamed% ALGORITHMIC STYLE -- Released 27 APR 2005 % for LaTeX version 2e % % Copyright Szasz Janos % E-mail szaszjanos@users.sourceforge.net % % % *** INITIALISING *** % % \NeedsTeXFormat{LaTeX2e} \ProvidesPackage{algorithmicx}[2005/04/27 v1.2 Algorithmicx] \RequirePackage{ifthen} \typeout{Document Style algorithmicx 1.2 - a greatly improved `algorithmic' style} % \newcounter{ALG@line} \newcounter{ALG@rem} \newcounter{ALG@nested} \newlength{\ALG@tlm} \newlength{\ALG@thistlm} \newcounter{ALG@Lnr}% the number of defined languages \setcounter{ALG@Lnr}{0} \newcounter{ALG@blocknr}% the number of defined blocks \setcounter{ALG@blocknr}{0} \newcounter{ALG@storecount}% number of stored but not restored algorithmic environments \setcounter{ALG@storecount}{0} \newcounter{ALG@tmpcounter}% only to decrement things \newlength\ALG@tmplength% %\def\algorithmicnoindent{-\ALG@tlm} % \def\algbackskipbegin{\hskip\ALG@ctlm} %\def\algbackskip{\hskip-\ALG@thistlm} %\def\algbackskipend{\hskip-\ALG@tlm} \def\ALG@defaultindent{\algorithmicindent} % % conditional states % \def\ALG@newcondstate#1% {% \expandafter\edef\csname ALG@x@#1\endcsname% {\expandafter\noexpand\csname @@ALG@x@#1\endcsname}% }% \ALG@newcondstate{notext}% \ALG@newcondstate{default}% % % % *** ALGORITHMIC *** % % \newcommand\ALG@beginblock[1]% #1 - indentation {% \ALG@thistlm\ALG@tlm% \addtolength\ALG@tlm{#1}% \addtocounter{ALG@nested}{1}% \setlength\ALG@tmplength{#1}% \expandafter\edef\csname ALG@ind@\theALG@nested\endcsname{\the\ALG@tmplength}% }% \newcommand\ALG@endblock% {% \addtolength\ALG@tlm{-\csname ALG@ind@\theALG@nested\endcsname}% \addtocounter{ALG@nested}{-1}% \ALG@thistlm\ALG@tlm% }% % % algorithmic environment % \def\ALG@step% {% \addtocounter{ALG@line}{1}% \addtocounter{ALG@rem}{1}% \ifthenelse{\equal{\arabic{ALG@rem}}{\ALG@numberfreq}}% {\setcounter{ALG@rem}{0}\alglinenumber{\arabic{ALG@line}}}% {}% }% \newenvironment{algorithmic}[1][0]% {% \edef\ALG@numberfreq{#1}% \def\@currentlabel{\theALG@line}% % \setcounter{ALG@line}{0}% \setcounter{ALG@rem}{0}% % \let\\\algbreak% % \expandafter\edef\csname ALG@currentblock@\theALG@nested\endcsname{0}% \expandafter\let\csname ALG@currentlifetime@\theALG@nested\endcsname\relax% % \begin{list}% {\ALG@step}% {% \rightmargin\z@% \itemsep\z@ \itemindent\z@ \listparindent2em% \partopsep\z@ \parskip\z@ \parsep\z@% \labelsep 0.5em \topsep 0.2em%\skip 1.2em \ifthenelse{\equal{#1}{0}}% {\labelwidth 0.5em}% {\labelwidth 1.2em}% \leftmargin\labelwidth \addtolength{\leftmargin}{\labelsep}% Ok. the perfect leftmargin :-)) \ALG@tlm\z@% }% \setcounter{ALG@nested}{0}% \ALG@beginalgorithmic% }% {% end{algorithmic} % check if all blocks are closed \ALG@closeloops% \expandafter\ifnum\csname ALG@currentblock@\theALG@nested\endcsname=0\relax% \else% \PackageError{algorithmicx}{Some blocks are not closed!!!}{}% \fi% \ALG@endalgorithmic% \end{list}% }% % % % *** Functional core *** % % \def\ALG@makeentity#1% execute the entity (#1) {% \def\ALG@thisentity{#1}% \expandafter\ifx\csname ALG@b@\ALG@L @#1@0\endcsname\relax% \let\ALG@makenobeginrepeat\ALG@makenobegin\ALG@makenobeginrepeat% this entitie ends or continues blocks \else% \let\ALG@makebeginrepeat\ALG@makebegin\ALG@makebeginrepeat% this entitie can open blocks \fi% \ALG@entitiecommand% }% % \def\ALG@makebegin% executes an entitie that can open blocks {% \expandafter\let\expandafter\ALG@thislifetime\csname ALG@currentlifetime@\theALG@nested\endcsname% \ifx\ALG@thislifetime\relax% \let\ALG@makebeginrepeat\ALG@doentity% in infinite block I can open my block \else% \ifnum\ALG@thislifetime>0\relax% \ifnum\ALG@thislifetime>65534\else% \setcounter{ALG@tmpcounter}{\ALG@thislifetime}% the block has 'space' for another included block \addtocounter{ALG@tmpcounter}{-1}% \expandafter\edef\csname ALG@currentlifetime@\theALG@nested\endcsname{\arabic{ALG@tmpcounter}}% \fi% \let\ALG@makebeginrepeat\ALG@doentity% \else% the block needs to be closed \expandafter\ifx\csname ALG@b@\ALG@L @\ALG@thisentity @\csname ALG@currentblock@\theALG@nested\endcsname\endcsname\relax% \ALG@closebyforce% I can not close this block, continue after it is closed by force % \ALG@makebegin% \else% % the block would be closed automatically, but this entitie can close it, so let's do it with the entity \let\ALG@makebeginrepeat\ALG@doentity% \fi% \fi% \fi% \ALG@makebeginrepeat% }% % \def\ALG@makenobegin% executes an entitie that can not open blocks {% \expandafter\ifx\csname ALG@currentlifetime@\theALG@nested\endcsname\relax% \let\ALG@makenobeginrepeat\ALG@doentity% an infinite block must be broken \else% \expandafter\ifx\csname ALG@b@\ALG@L @\ALG@thisentity @\csname ALG@currentblock@\theALG@nested\endcsname\endcsname\relax% \ALG@closebyforce% the block must be ended by force, \else% \let\ALG@makenobeginrepeat\ALG@doentity% I can continue / end this block, let's do it \fi% \fi% \ALG@makenobeginrepeat% }% % \def\ALG@dobegin% {% \ALG@beginblock{\csname ALG@i@\ALG@L @\ALG@thisentity @\ALG@thisblock\endcsname}% \expandafter\edef\csname ALG@currentblock@\theALG@nested\endcsname{\csname ALG@b@\ALG@L @\ALG@thisentity @\ALG@thisblock\endcsname}% \expandafter\ifx\csname ALG@c@\ALG@L @\ALG@thisentity @\ALG@thisblock\endcsname\relax% \expandafter\let\csname ALG@currentlifetime@\theALG@nested\endcsname\relax% \else% \expandafter\edef\csname ALG@currentlifetime@\theALG@nested\endcsname{\csname ALG@c@\ALG@L @\ALG@thisentity @\ALG@thisblock\endcsname}% \fi% }% % \def\ALG@doend% {% \ALG@endblock% }% % \def\ALG@doentity% the number of the closed block, the entitie {% \edef\ALG@thisblock{\csname ALG@currentblock@\theALG@nested\endcsname}% \expandafter\ifx\csname ALG@b@\ALG@L @\ALG@thisentity @\ALG@thisblock\endcsname\relax% \def\ALG@thisblock{0}% \fi% \ALG@getentitytext% \ifnum\ALG@thisblock=0\else\ALG@doend\fi% \ifx\ALG@text\ALG@x@notext% \item[]\nointerlineskip%\vskip-\prevdepth\nointerlineskip% bug: if there are no text and no lines, then this is wrong \else% \item% \fi% \noindent\hskip\ALG@tlm% \expandafter\ifnum0=\csname ALG@b@\ALG@L @\ALG@thisentity @\ALG@thisblock\endcsname\else% \ALG@dobegin% \fi% \def\ALG@entitiecommand{\ALG@displayentity}% }% % \def\ALG@getentitytext% {% \expandafter\let\expandafter\ALG@text\csname ALG@t@\ALG@L @\ALG@thisentity @\ALG@thisblock\endcsname% \ifx\ALG@text\ALG@x@default% % block specific - default \expandafter\let\expandafter\ALG@text\csname ALG@t@\ALG@L @\ALG@thisentity\endcsname% \ifx\ALG@text\ALG@x@default% % block specific - default, language specific - default \def\ALG@text{\ALG@deftext{\ALG@thisentity}}% \fi% \fi% }% % \def\ALG@deftext{\csname ALG@deftext@\ALG@L\endcsname}% % \def\ALG@displayentity% {% \ifx\ALG@text\ALG@x@notext% \let\ALG@text\relax% \fi \ALG@text% }% % \def\ALG@closebyforce% {% \ALG@endblock% }% % \def\ALG@closeloops% closes all finite blocks {% \expandafter\ifx\csname ALG@currentlifetime@\theALG@nested\endcsname\relax% \else% only if it is finite \ALG@closebyforce% the block must be ended by force, \ALG@closeloops% the command still runs \fi% }% % % % *** Low level block/entitie defining commands *** % % \def\ALG@bl@{0}% the BIG block \let\ALG@bl@@\ALG@bl@% the BIG block % % Create a block % \def\ALG@createblock#1% create the block #1, if it does not exists {% \@ifundefined{ALG@bl@\ALG@Ld @#1}% needs to be created? {% \addtocounter{ALG@blocknr}{1}% increment the block counter \expandafter\edef\csname ALG@bl@\ALG@Ld @#1\endcsname{\arabic{ALG@blocknr}}% set the block number }% {}% }% % % Get the block number % \def\ALG@getblocknumber#1{\csname ALG@bl@\ALG@Ld @#1\endcsname}% % % Create an entitie % \def\ALG@createentitie#1% create the entitie #1, if it does not exists {% \expandafter\ALG@edefcmd\csname #1\endcsname{\noexpand\ALG@makeentity{#1}}% \@ifundefined{ALG@t@\ALG@Ld @#1}% the entity text is defined in this language? {% \expandafter\let\csname ALG@t@\ALG@Ld @#1\endcsname\ALG@x@default% }% {}% }% % \def\ALG@createtext#1#2% #1 = closed block; #2 = entitie; creates \ALG@t@#2@#1 {% \expandafter\let\csname ALG@t@\ALG@Ld @#2@#1\endcsname\ALG@x@default% }% % % End and Continue block % \def\ALG@endandcontinueblock#1#2#3#4#5% #1 = new block; #2 = old block; #3 = entitie; #4 = credits; #5 = indent {% \ifthenelse{\equal{#3}{}}{}% execute only if the entity is not empty {% \ALG@createentitie{#3}% create the entitie \ALG@createblock{#2}% create the old block, if needed \ifthenelse{\equal{#1}{}}% whe need to open a new block? {\expandafter\edef\csname ALG@b@\ALG@Ld @#3@\ALG@getblocknumber{#2}\endcsname{0}}% no, just close the old one {% yes, \ALG@createblock{#1}% create the block \expandafter\edef\csname ALG@b@\ALG@Ld @#3@\ALG@getblocknumber{#2}\endcsname{\ALG@getblocknumber{#1}}% ending the old block opens a new one \ifthenelse{\equal{#4}{}}% infinite or finite credits? {\expandafter\let\csname ALG@c@\ALG@Ld @#3@\ALG@getblocknumber{#2}\endcsname\relax}% infinite credits {\expandafter\edef\csname ALG@c@\ALG@Ld @#3@\ALG@getblocknumber{#2}\endcsname{#4}}% finite credits \ifthenelse{\equal{#5}{}}% default or specified indentation {\expandafter\let\csname ALG@i@\ALG@Ld @#3@\ALG@getblocknumber{#2}\endcsname\ALG@defaultindent}% default indentation {\expandafter\edef\csname ALG@i@\ALG@Ld @#3@\ALG@getblocknumber{#2}\endcsname{#5}}% indentation is specified }% \ALG@createtext{\ALG@getblocknumber{#2}}{#3}% }% }% % % macros used in declarations % \def\ALG@p@endtext@E{\algrenewtext{\ALG@v@end}}% \def\ALG@p@endtext@xE{\algrenewtext[\ALG@v@newblock]{\ALG@v@end}}% \def\ALG@p@endtext@nE{\algnotext{\ALG@v@end}}% \def\ALG@p@endtext@xnE{\algnotext[\ALG@v@newblock]{\ALG@v@end}}% \def\ALG@p@endtext@{}% % starttext defines are more compex -- care must be taken for the optional parameters \def\ALG@p@starttext@S{\ALG@p@s@process{\algrenewtext}}% \def\ALG@p@starttext@C{\ALG@p@s@process{\algrenewtext}}% \def\ALG@p@starttext@xC{\ALG@p@s@process{\algrenewtext[\ALG@v@oldblock]}}% \def\ALG@p@s@process#1% {% \ifthenelse{\equal{\ALG@v@start}{}}% {\ALG@p@endtext}% {\@ifnextchar{[}{\ALG@p@s@getparamcount{#1}}{\ALG@p@s@simple{#1}}}% }% \def\ALG@p@s@getparamcount#1[#2]% {% \@ifnextchar{[}{\ALG@p@s@getdefparam{#1}{#2}}{\ALG@p@s@param{#1}{#2}}% }% \def\ALG@p@s@getdefparam#1#2[#3]% {% \ALG@p@s@defparam{#1}{#2}{#3}% }% \def\ALG@p@s@simple#1#2{#1{\ALG@v@start}{#2}\ALG@p@endtext}% \def\ALG@p@s@param#1#2#3{#1{\ALG@v@start}[#2]{#3}\ALG@p@endtext}% \def\ALG@p@s@defparam#1#2#3#4{#1{\ALG@v@start}[#2][#3]{#4}\ALG@p@endtext}% % the rest of the crew \def\ALG@p@starttext@nS{\algnotext{\ALG@v@start}\ALG@p@endtext}% \def\ALG@p@starttext@nC{\algnotext{\ALG@v@start}\ALG@p@endtext}% \def\ALG@p@starttext@xnC{\algnotext[\ALG@v@oldblock]{\ALG@v@start}\ALG@p@endtext}% \def\ALG@p@starttext@{\ALG@p@endtext}% \def\ALG@p@indent@def#1{\def\ALG@v@indent{#1}\ALG@p@setup}% \def\ALG@p@indent@{\def\ALG@v@indent{}\ALG@p@setup}% \def\ALG@p@credits@def#1{\def\ALG@v@credits{#1}\ALG@p@indent}% \def\ALG@p@credits@{\ALG@p@indent}% \def\ALG@p@end@def#1{\def\ALG@v@end{#1}\ALG@p@credits}% \def\ALG@p@end@{\def\ALG@v@end{}\ALG@p@credits}% \def\ALG@p@start@def#1{\def\ALG@v@start{#1}\ALG@p@end}% \def\ALG@p@start@{\def\ALG@v@start{}\ALG@p@end}% \def\ALG@p@oldblock@def#1{\def\ALG@v@oldblock{#1}\ALG@p@start}% \def\ALG@p@oldblock@{\def\ALG@v@oldblock{}\ALG@p@start}% \newcommand\ALG@p@newblock[1][]{\def\ALG@v@newblock{#1}\ALG@p@oldblock}% \def\ALG@p@setup% {% \ifthenelse{\equal{\ALG@v@newblock}{}}% {% \ifthenelse{\equal{\ALG@v@start}{}}% {% \PackageError{algorithmicx}{Block or starting entitie must be specified!!!}{}% }% {% \let\ALG@v@newblock\ALG@v@start% }% }% {% }% \ALG@endandcontinueblock% {\ALG@v@newblock}{\ALG@v@oldblock}{\ALG@v@start}% {\ALG@v@credits}{\ALG@v@indent}% \ALG@endandcontinueblock% {}{\ALG@v@newblock}{\ALG@v@end}% {}{}% \ALG@p@starttext% }% % % param handling % \newcommand\ALG@p@def[2][def]% {% \expandafter\let\csname ALG@p@#2\expandafter\endcsname\csname ALG@p@#2@#1\endcsname% }% \def\ALG@p@undef{\ALG@p@def[]}% % \def\ALG@p@ons{\ALG@p@def{start}}% \def\ALG@p@onS{\ALG@p@def{start}\ALG@p@def[S]{starttext}}% \def\ALG@p@onc{\ALG@p@def{oldblock}\ALG@p@def{start}}% \def\ALG@p@onC{\ALG@p@def{oldblock}\ALG@p@def{start}\ALG@p@def[C]{starttext}}% \def\ALG@p@one{\ALG@p@def{end}}% \def\ALG@p@onE{\ALG@p@def{end}\ALG@p@def[E]{endtext}}% \def\ALG@p@onxC{\ALG@p@def{oldblock}\ALG@p@def{start}\ALG@p@def[xC]{starttext}}% \def\ALG@p@onxE{\ALG@p@def{end}\ALG@p@def[xE]{endtext}}% \def\ALG@p@onnS{\ALG@p@def{start}\ALG@p@def[nS]{starttext}}% \def\ALG@p@onnC{\ALG@p@def{oldblock}\ALG@p@def{start}\ALG@p@def[nC]{starttext}}% \def\ALG@p@onnE{\ALG@p@def{end}\ALG@p@def[nE]{endtext}}% \def\ALG@p@onxnC{\ALG@p@def{oldblock}\ALG@p@def{start}\ALG@p@def[xnC]{starttext}}% \def\ALG@p@onxnE{\ALG@p@def{end}\ALG@p@def[xnE]{endtext}}% \def\ALG@p@onb{\def\ALG@v@credits{}}% \def\ALG@p@onl{\def\ALG@v@credits{1}}% \def\ALG@p@onL{\ALG@p@def{credits}}% \def\ALG@p@oni{\ALG@p@def{indent}}% % \def\ALG@p@main#1% {% \@ifundefined{ALG@ps@\ALG@p@state @#1}% {% \csname ALG@ps@\ALG@p@state @other\endcsname{#1}% }% {% \csname ALG@ps@\ALG@p@state @#1\endcsname% }% \ALG@p@rec% }% % STATE : <> \expandafter\def\csname ALG@ps@@]\endcsname{\let\ALG@p@rec\relax}% \def\ALG@ps@@s{\ALG@p@ons}% \def\ALG@ps@@S{\ALG@p@onS}% \def\ALG@ps@@c{\ALG@p@onc}% \def\ALG@ps@@C{\ALG@p@onC}% \def\ALG@ps@@e{\ALG@p@one}% \def\ALG@ps@@E{\ALG@p@onE}% \def\ALG@ps@@N{\typeout{algdef: 'N' obsoloted, use 'nE'.}\ALG@p@onnE}% \def\ALG@ps@@b{\ALG@p@onb}% \def\ALG@ps@@l{\ALG@p@onl}% \def\ALG@ps@@L{\ALG@p@onL}% \def\ALG@ps@@i{\ALG@p@oni}% \def\ALG@ps@@x{\def\ALG@p@state{x}}% \def\ALG@ps@@n{\def\ALG@p@state{n}}% \def\ALG@ps@@other#1{\typeout{algdef: Ignoring unknown token #1}}% % STATE : x \def\ALG@ps@x@C{\def\ALG@p@state{}\ALG@p@onxC}% \def\ALG@ps@x@E{\def\ALG@p@state{}\ALG@p@onxE}% \def\ALG@ps@x@N{\def\ALG@p@state{}\typeout{algdef: 'xN' obsoloted, use 'xnE'.}\ALG@p@onxnE}% \def\ALG@ps@x@n{\def\ALG@p@state{xn}}% \def\ALG@ps@x@other#1% {% \typeout{algdef: Ignoring 'x' before '#1'.}% \def\ALG@p@state{}% \def\ALG@p@rec{\let\ALG@p@rec\ALG@p@main\ALG@p@rec#1}% }% % STATE : n \def\ALG@ps@n@S{\def\ALG@p@state{}\ALG@p@onnS}% \def\ALG@ps@n@C{\def\ALG@p@state{}\ALG@p@onnC}% \def\ALG@ps@n@E{\def\ALG@p@state{}\ALG@p@onnE}% \def\ALG@ps@n@x{\def\ALG@p@state{nx}}% \def\ALG@ps@n@other#1% {% \typeout{algdef: Ignoring 'n' before '#1'.}% \def\ALG@p@state{}% \def\ALG@p@rec{\let\ALG@p@rec\ALG@p@main\ALG@p@rec#1}% }% % STATE : xn \def\ALG@ps@xn@C{\def\ALG@p@state{}\ALG@p@onxnC}% \def\ALG@ps@xn@E{\def\ALG@p@state{}\ALG@p@onxnE}% \def\ALG@ps@xn@x{\typeout{algdef: Ignoring 'x' after 'xn'.}}% \def\ALG@ps@xn@n{\typeout{algdef: Ignoring 'n' after 'xn'.}}% \def\ALG@ps@xn@other#1% {% \typeout{algdef: Ignoring 'xn' before '#1'.}% \def\ALG@p@state{}% \def\ALG@p@rec{\let\ALG@p@rec\ALG@p@main\ALG@p@rec#1}% }% % STATE : nx \def\ALG@ps@nx@C{\def\ALG@p@state{}\ALG@p@onxnC}% \def\ALG@ps@nx@E{\def\ALG@p@state{}\ALG@p@onxnE}% \def\ALG@ps@nx@x{\typeout{algdef: Ignoring 'x' after 'nx'.}}% \def\ALG@ps@nx@n{\typeout{algdef: Ignoring 'n' after 'nx'.}}% \def\ALG@ps@nx@other#1% {% \typeout{algdef: Ignoring 'nx' before '#1'.}% \def\ALG@p@state{}% \def\ALG@p@rec{\let\ALG@p@rec\ALG@p@main\ALG@p@rec#1}% }% % % % *** User level block/entitie commands *** % % % % algdef{switches}... -- the king of all definitions in the algorithmicx package % \newcommand\algdef[1]% {% \ALG@p@undef{oldblock}% \ALG@p@undef{start}% \ALG@p@undef{end}% \def\ALG@v@credits{}% \ALG@p@undef{credits}% \ALG@p@undef{indent}% \ALG@p@undef{starttext}% \ALG@p@undef{endtext}% \def\ALG@p@state{}% \let\ALG@p@rec\ALG@p@main% \ALG@p@rec#1]% \ALG@p@newblock% }% % % a lot of other macros are provided for convenience % \def\algblock{\algdef{se}}% \def\algcblock{\algdef{ce}}% \def\algloop{\algdef{sl}}% \def\algcloop{\algdef{cl}}% \def\algsetblock{\algdef{seLi}}% \def\algsetcblock{\algdef{ceLi}}% \def\algblockx{\algdef{SxE}}% \def\algblockdefx{\algdef{SE}}% \def\algcblockx{\algdef{CxE}}% \def\algcblockdefx{\algdef{CE}}% \def\algsetblockx{\algdef{SxELi}}% \def\algsetblockdefx{\algdef{SELi}}% \def\algsetcblockx{\algdef{CxELi}}% \def\algsetcblockdefx{\algdef{CELi}}% \def\algloopdefx{\algdef{Sl}}% \def\algcloopx{\algdef{xCl}}% \def\algcloopdefx{\algdef{Cl}}% % algloopx is not correct, use algloopdefx % % Text output commands % \newcommand\algrenewtext[2][]% [block]{entity} {% \ifthenelse{\equal{#2}{}}{}% {% \ifthenelse{\equal{#1}{}}% {% \expandafter\let\csname ALG@t@\ALG@Ld @#2\endcsname\relax% \expandafter\newcommand\csname ALG@t@\ALG@Ld @#2\endcsname% }% {% \expandafter\let\csname ALG@t@\ALG@Ld @#2@\ALG@getblocknumber{#1}\endcsname\relax% \expandafter\newcommand\csname ALG@t@\ALG@Ld @#2@\ALG@getblocknumber{#1}\endcsname% }% }% }% % \def\ALG@letentitytext#1#2% [block]{entity} {% \ifthenelse{\equal{#2}{}}{}% {% \ifthenelse{\equal{#1}{}}% {% \expandafter\let\csname ALG@t@\ALG@Ld @#2\endcsname% }% {% \expandafter\let\csname ALG@t@\ALG@Ld @#2@\ALG@getblocknumber{#1}\endcsname% }% }% }% % \newcommand\algnotext[2][]% [block]{entity} {% \ALG@letentitytext{#1}{#2}\ALG@x@notext% }% % \newcommand\algdefaulttext[2][]% [block]{entity} {% \ALG@letentitytext{#1}{#2}\ALG@x@default% }% % \def\ALG@notext*{\algnotext}% \def\algtext{\@ifnextchar{*}{\ALG@notext}{\algrenewtext}}% % % % *** LANGUAGE SWITCHING *** % % % \newcommand\algnewlanguage[1]% {% \@ifundefined{ALG@L@#1}% needs to be created? {}% {% \PackageError{algorithmicx}{Language '#1' already defined!}{}% }% \addtocounter{ALG@Lnr}{1}% increment the language counter \expandafter\edef\csname ALG@L@#1\endcsname{\arabic{ALG@Lnr}}% set the language number \edef\ALG@Ld{\csname ALG@L@#1\endcsname}% \expandafter\let\csname ALG@bl@\ALG@Ld @\endcsname\ALG@bl@% the BIG block \expandafter\let\csname ALG@bl@\ALG@Ld @@\endcsname\ALG@bl@% the BIG block \algdef{SL}[STATE]{State}{0}{}% \expandafter\def\csname ALG@deftext@\ALG@Ld\endcsname{\textbf}% \algnewcommand\algorithmiccomment[1]{\hfill\(\triangleright\) ##1}% \algnewcommand\algorithmicindent{1.5em}% \algnewcommand\alglinenumber[1]{\footnotesize ##1:}% \algnewcommand\ALG@beginalgorithmic\relax% for user overrides \algnewcommand\ALG@endalgorithmic\relax% for user overrides }% % \newcommand\algsetlanguage[1]% {% \@ifundefined{ALG@L@#1}% needs to be created? {% \PackageError{algorithmicx}{Language '#1' is not yet defined!}{}% }{}% \edef\ALG@L{\csname ALG@L@#1\endcsname}% }% % \newcommand\algdeflanguage[1]% {% \@ifundefined{ALG@L@#1}% needs to be created? {% \PackageError{algorithmicx}{Language '#1' is not yet defined!}{}% }{}% \edef\ALG@Ld{\csname ALG@L@#1\endcsname}% }% % \newcommand\alglanguage[1]% {% \algdeflanguage{#1}% \algsetlanguage{#1}% }% % % % *** Defining language dependent stuff *** % % \def\ALG@eatoneparam#1{}% \def\ALG@defbasecmd#1#2% {% \edef\ALG@tmp{\expandafter\ALG@eatoneparam\string #2}% \@ifundefined\ALG@tmp{\edef #2{\noexpand\csname ALG@cmd@\noexpand\ALG@L @\ALG@tmp\endcsname}}{}% \expandafter#1\csname ALG@cmd@\ALG@Ld @\ALG@tmp\endcsname% }% \newcommand\algnewcommand{\ALG@defbasecmd\newcommand}% \newcommand\algrenewcommand{\ALG@defbasecmd\renewcommand}% \def\ALG@letcmd{\ALG@defbasecmd\let}% \def\ALG@defcmd{\ALG@defbasecmd\def}% \def\ALG@edefcmd{\ALG@defbasecmd\edef}% % % % *** OTHERS *** % % \def\BState{\State \algbackskip}% \def\Statex{\item[]}% an empty line \newcommand\algrenewcomment{\algrenewcommand\algorithmiccomment}% \def\Comment{\algorithmiccomment}% \def\algref#1#2{\ref{#1}.\ref{#2}}% \algnewlanguage{default}% \algsetlanguage{default}% % % % *** Line breaks *** % % \newcommand\algbreak% for multiline parameters !!! needs fix {% \item% % \hskip\ALG@parindent%!!! not yet implemented % \hskip-\algorithmicindent% }% % \def\ALG@noputindents% {% \hskip\ALG@tlm% }% % % % *** algorithm store / restore *** % % % store % \ALG@newcondstate{mustrestore}% \def\algstore% {% \renewcommand\ALG@beginblock% {% \PackageError{algorithmicx}{The environment must be closed after store!}{}% }% \@ifstar{\ALG@starstore}{\ALG@nostarstore}% }% \def\ALG@nostarstore#1% save all infos into #1 and terminate the algorithmic block {% \addtocounter{ALG@storecount}{1}% \expandafter\global\expandafter\let\csname ALG@save@mustrestore@#1\endcsname\ALG@x@mustrestore% \ALG@starstore{#1}% }% \def\ALG@starstore#1% {% \@ifundefined{ALG@save@line@#1}{}% {\PackageError{algorithmicx}{This save name '#1' is already used!}{}}% \def\ALG@savename{#1}% \expandafter\xdef\csname ALG@save@totalnr@\ALG@savename\endcsname{\theALG@nested}% \expandafter\xdef\csname ALG@save@line@\ALG@savename\endcsname{\theALG@line}% \expandafter\xdef\csname ALG@save@numberfreq@\ALG@savename\endcsname{\ALG@numberfreq}% \expandafter\xdef\csname ALG@save@rem@\ALG@savename\endcsname{\theALG@rem}% \let\ALG@storerepeat\ALG@store% \ALG@storerepeat% }% \def\ALG@store% simply terminate all open blocks {% \ifnum\theALG@nested=0\let\ALG@storerepeat\relax% \else% \expandafter\xdef\csname ALG@save@currentblock@\ALG@savename @\theALG@nested\endcsname% {\csname ALG@currentblock@\theALG@nested\endcsname}% \expandafter\ifx\csname ALG@currentlifetime@\theALG@nested\endcsname\relax% \else% \expandafter\xdef\csname ALG@save@currentlifetime@\ALG@savename @\theALG@nested\endcsname% {\csname ALG@currentlifetime@\theALG@nested\endcsname}% \fi% \expandafter\xdef\csname ALG@save@ind@\ALG@savename @\theALG@nested\endcsname% {\csname ALG@ind@\theALG@nested\endcsname}% \ALG@closebyforce% \fi% \ALG@storerepeat% }% % % restore % \def\algrestore% {% \@ifstar{\ALG@starrestore}{\ALG@nostarrestore}% }% \def\ALG@starrestore% {% \let\ALG@restorerem\relax% \let\ALG@restorereprem\relax% \ALG@restoremain% }% \def\ALG@nostarrestore% {% \let\ALG@restorerem\ALG@restoreremovesave% \let\ALG@restorereprem\ALG@restorerepremovesave% \ALG@restoremain% }% \def\ALG@restoreremovesave% {% \expandafter\global\expandafter\let\csname ALG@save@totalnr@\ALG@savename\endcsname\relax% \expandafter\global\expandafter\let\csname ALG@save@line@\ALG@savename\endcsname\relax% \expandafter\global\expandafter\let\csname ALG@save@rem@\ALG@savename\endcsname\relax% \expandafter\global\expandafter\let\csname ALG@save@totalnr@\ALG@savename\endcsname\relax% \expandafter\global\expandafter\let\csname ALG@save@numberfreq@\ALG@savename\endcsname\relax% }% \def\ALG@restorerepremovesave% {% \expandafter\global\expandafter\let\csname ALG@save@currentblock@\ALG@savename @\theALG@tmpcounter\endcsname\relax% \expandafter\global\expandafter\let\csname ALG@save@currentlifetime@\ALG@savename @\theALG@tmpcounter\endcsname\relax% \expandafter\global\expandafter\let\csname ALG@save@currentlifetime@\ALG@savename @\theALG@tmpcounter\endcsname\relax% \expandafter\global\expandafter\let\csname ALG@save@ind@\ALG@savename @\theALG@tmpcounter\endcsname\relax% }% \def\ALG@restoremain#1% restore all infos from #1 in an open algorithmic block {% \ifnum\theALG@line=0% \else\PackageError{algorithmicx}{Restore might be used only at the beginning of the environment!}{}% \fi% \def\ALG@savename{#1}% \expandafter\ifx\csname ALG@save@totalnr@\ALG@savename\endcsname\relax% \PackageError{algorithmicx}{Save '\ALG@savename'\space not defined!!!}{}% \fi% \@ifundefined{ALG@save@mustrestore@\ALG@savename}{}% {% \addtocounter{ALG@storecount}{-1}% \expandafter\global\expandafter\let\csname ALG@save@mustrestore@\ALG@savename\endcsname\relax% }% \setcounter{ALG@line}{\csname ALG@save@line@\ALG@savename\endcsname}% \edef\ALG@numberfreq{\csname ALG@save@numberfreq@\ALG@savename\endcsname}% \setcounter{ALG@rem}{\csname ALG@save@rem@\ALG@savename\endcsname}% \setcounter{ALG@tmpcounter}{\csname ALG@save@totalnr@\ALG@savename\endcsname}% \setcounter{ALG@nested}{0}% \ALG@restorerem% \let\ALG@restorerepeat\ALG@restore% \ALG@restorerepeat% }% \def\ALG@restore% {% \ifnum\theALG@tmpcounter>0% \expandafter\edef\csname ALG@currentblock@\theALG@tmpcounter\endcsname% {\csname ALG@save@currentblock@\ALG@savename @\theALG@tmpcounter\endcsname}% \expandafter\ifx\csname ALG@save@currentlifetime@\ALG@savename @\theALG@tmpcounter\endcsname\relax% \expandafter\let\csname ALG@currentlifetime@\theALG@tmpcounter\endcsname\relax% \else% \expandafter\edef\csname ALG@currentlifetime@\theALG@tmpcounter\endcsname% {\csname ALG@save@currentlifetime@\ALG@savename @\theALG@tmpcounter\endcsname}% \fi% % \ALG@beginblock{\csname ALG@save@ind@\ALG@savename @\theALG@tmpcounter\endcsname}% \ALG@restorereprem% \addtocounter{ALG@tmpcounter}{-1}% \else\let\ALG@restorerepeat\relax% \fi% \ALG@restorerepeat% }% \AtEndDocument% {% \ifnum\theALG@storecount>0\relax% \PackageError{algorithmicx}{Some stored algorithms are not restored!}{}% \fi% }% bdtl-1.0.3/doc/HOWTO.build0000777000175000017500000000000012330444352017640 2../src/HOWTO.buildustar melamedmelamedbdtl-1.0.3/doc/HOWTO.use0000777000175000017500000000000012330444352017134 2usage/HOWTO.useustar melamedmelamedbdtl-1.0.3/doc/usage/0000755000175000017500000000000012330444352014103 5ustar melamedmelamedbdtl-1.0.3/doc/usage/HOWTO.use0000644000175000017500000003470012151020420015510 0ustar melamedmelamedHow to Use the Boosted Decision Trees (BDTL) Software (after it's built) BDTL currently provides 3 binary executables and several utility scripts. The binary executables are trainOne, testOne, and compileEnsemble. trainOne and testOne do training and testing. compileEnsemble is a utility for converting BDTL's models into a more compact format. trainOne and testOne can work with models in either format. The default output format of trainOne is less compact to facilitate continuous checkpointing. The utility scripts include a pretty-printer for BDTL's model format and scripts for converting examples from SVMLight format to BDTL's more informative and more compact format. BDTL comes with a "sandbox", which serves multiple purposes. One of its purposes is to show typical usage to new users. Users who want a quick start can proceed as follows: 1. Set the environment variable BDTL_BASE to the top directory of your installation. OTHERWISE NOTHING WILL WORK! 2. Follow the instructions in src/HOWTO.build to build the executables. 3. Change to the sandbox/ directory and type `make`. The sub-Makefiles in the sandbox can serve as templates that users can modify for various purposes. For other uses of the sandbox, refer to sandbox/HOWTO.play . BDTL has more configuration parameters than most people expect in this kind of software. However, users can ignore them, and just use the defaults in the configuration files provided in the sandbox. More info about the configuration parameters is provided below, for those who want more control. BDTL reads examples one per line, in the format "){ cerr<<"Error: expected , got "<> ws; return in; } } return in; // _super::_curIndex = _super::_startIndex; // X x; // while(in>>x) { // _super::_byWord[x] = _super::_curIndex; // _super::_byIndex[_super::_curIndex] = (_super::_byWord.find(x)->first); // _super::_curIndex++; // } return in; } ostream& writeImpl(ostream& out) const { out<<""<second)<first<second)<"< double Tiebreak::_stepper = STEPPER_INCR; namespace HashNS { //! Template specialization of STL's hash object template<> template struct hash > { size_t operator()(const Tiebreak& tb) const{ return tb.hash(); } }; template<> template struct hash > { size_t operator()(const Tiebreak& tb) const{ return tb.hash(); } }; } bdtl-1.0.3/src/util/containers/File.cpp0000644000175000017500000000246712122405072017532 0ustar melamedmelamed#include "File.hpp" using namespace std; string oFile::s_suffix=""; std::_Ios_Openmode oFile::s_writeFlag = ios::out; void iFile::rewind(iFile& i){ i.clear(); i.seekg(0); } /** * @brief Get a line from the file. Comment lines are ignored. * @param line The line read. * @param sep The symbol seperating lines. * @return The stream. */ iFile& iFile::getline(string& line, char sep) { static const char trimChars[] = {' ', '\t'}; // Note: sizeof(trimchars) is the size of the whole array, so // dividing by sizeof(char) gives the number of elements in the // array. static const char* trimCharsSentinel = trimChars + (sizeof(trimChars) / sizeof(char)); size_t i; bool isComment; do { std::getline(*this, line, sep); // check for blank line, in which case get another i = line.length(); while(i > 0 && (line[i-1] == '\r' || line[i-1] == '\n' || line[i-1] == ' ' || line[i-1] == '\t')){ --i; } if(i==0) {continue;} // allow comments to be preceded by whitespace i = 0; while(i < line.length() && (std::find(trimChars, trimCharsSentinel, line[i]) != trimCharsSentinel)) { ++i; } // break unless it's a comment line and we're not at eof() isComment = ( line.find(m_comment, i) == i); } while (! eof() && isComment); return *this; } bdtl-1.0.3/src/util/containers/Tree.cpp0000644000175000017500000000230512122405072017541 0ustar melamedmelamed#include "Tree.hpp" #include "HeadMTreeNode.hpp" #include "PositionTreeNode.hpp" #include "CharniakTreeNode.hpp" // Many of the methods of the tree base class should have been static // but aren't. They take an iterator and a tree, but don't actually // use the tree. Instead of passing around an extra tree param, we use // a static dummy tree where necessary. Unfortunately, C++ doesn't // allow generic definition of static data members of template // classes, so this has to be specialized for each needed type. template<> tree Tree::sm_dummyTree = tree(); template<> tree Tree::sm_dummyTree = tree(); template<> tree Tree::sm_dummyTree = tree(); template<> const tree::sibling_iterator Tree::sm_endIter = tree::sibling_iterator(); template<> const tree::sibling_iterator Tree::sm_endIter = tree::sibling_iterator(); template<> const tree::sibling_iterator Tree::sm_endIter = tree::sibling_iterator(); bdtl-1.0.3/src/util/containers/VocabBase.hpp0000644000175000017500000002164512122405072020504 0ustar melamedmelamed#ifndef _VocabBase_H_ #define _VocabBase_H_ //#include "COMPILER-DEPENDENT_HEADERS.hpp" #include #include "MoreStrings.hpp" #include "Dbug.hpp" #include "UniqueID.hpp" // //#include // //#include // //#include // //#include #include "google/sparse_hash_map" using namespace std; #define VOCAB_SEP "\t" typedef unsigned int VocabularyIndex; typedef VocabularyIndex VI; static const VocabularyIndex VocabularyNone = (unsigned int) -1; template class VocabBase; template ostream& operator<<(ostream& out, const VocabBase& vcb); template istream& operator>>(istream& in, VocabBase& vcb); //! A generic VocabBase class with template parameterized //! by the type of its content. The main functionality of //! this class is to assign an 'index' to a 'word', and provide //! interfaces for the word<->index mapping. template class VocabBase : public UniqueID, public Dbug { public: typedef X word_t; typedef typename google::sparse_hash_map > map_t; typedef typename google::sparse_hash_map > rev_map_t; typedef typename map_t::const_iterator const_iterator; typedef typename rev_map_t::const_iterator const_iterator_rev; const_iterator begin() const { return _byWord.begin();} const_iterator end() const { return _byWord.end(); } const_iterator_rev beginIndex() const { return _byIndex.begin();} const_iterator_rev endIndex() const { return _byIndex.end(); } //! Constructor. Specifying the start index at the same time. //! deletion should not be necessary for Vocabularies, but sometimes //their indexes are stored in ScoreSets, which use 0 as the deleted //key of google::sparse_hash_map. So we default the startInx to 1. VocabBase(size_t startInx = 1) : _startIndex(startInx), _curIndex(startInx) { // the following are unnecessary because that's the default // _byIndex.clear_deleted_key(); // _byWord.clear_deleted_key(); } //! Destructor. ~VocabBase() {} //! Returns the base VocabBase. const VocabBase& baseVocabulary() const { return *this; } VocabBase& baseVocabulary() { return *this; } //! Returns the index of the word. Returns VocabularyNone if not found. const VocabularyIndex getIndex(const X& word) const ; //! Whether the vocabulary contains the word bool contains(const X& word) const ; //! Returns the word of the index. //! If not found, exist is set to be false. const X* getWord(const VocabularyIndex& index, bool& exist ) const; const X* getWord(const VocabularyIndex& index ) const { // ignore the bool; test for null instead bool flag; return getWord(index, flag); }; //! Adds a word into VocabBase, if it's not already there, and returns its index. const VocabularyIndex addWord(const X& word); //! Adds a word into VocabBase, even if it's already there, and returns its index. const VocabularyIndex addDuplicateWord(const X& word); //! Returns the number of entries in the VocabBase. size_t size() const { return _byIndex.size(); } //! Tells whether the vocab is empty bool empty() const { return _byIndex.empty(); } //! Clear the content. void clear(); //! Get the last index; size_t getLastIndex(); //! Output the vocabulary from a certain index ostream& writeFromIndex(ostream& out, size_t index); //! Loads VocabBase from istream. istream& read(istream& in); //! Dumps VocabBase to ostream. ostream& write(ostream& out) const; //! Dumps VocabBase with Indexes to ostream. ostream& writeWithIndex(ostream& out) const; //! Assignment. VocabBase& operator=(const VocabBase& other); // output into stream. friend ostream& operator<<<>(ostream& out, const VocabBase& vcb); // read vocab from stream. friend istream& operator>><>(istream& in, VocabBase& vcb); template void save(Archive & ar, const unsigned int version) const; template void load(Archive & ar, const unsigned int version); // BOOST_SERIALIZATION_SPLIT_MEMBER() protected: //! Start index. size_t _startIndex; //! (One past the ) current highest index. size_t _curIndex; //! Stored by word. map_t _byWord; //! Stored by index. rev_map_t _byIndex; }; ///////////////////////////////////////////////////////////////////////////////// template istream& operator>>(istream& in, VocabBase& vcb) { vcb.read(in); return in; } template istream& VocabBase::read(istream& in) { // curiously recurring template pattern (CRTP) static_cast(this)->readImpl(in); return in; } template ostream& operator<<(ostream& out, const VocabBase& vcb) { return vcb.write(out); } template ostream& VocabBase::write(ostream& out) const { // curiously recurring template pattern (CRTP) static_cast(this)->writeImpl(out); return out; } template ostream& VocabBase::writeFromIndex(ostream& out, size_t index) { size_t i = index; size_t counter = index; while(i < _curIndex) { const_iterator_rev iter; if((iter = _byIndex.find(i)) != _byIndex.end()) { out << (iter->second) << endl; counter++; } // if(counter == _byIndex.size()) { // cerr << "counter == _byIndex.size(): breaking" << endl; // break; // } i++; } return out; } template ostream& VocabBase::writeWithIndex(ostream& out) const { out<<""<first<second)<"< size_t VocabBase::getLastIndex() { return _curIndex-1; } template template void VocabBase::save(Archive & ar, const unsigned int version) const { ar & _byWord; ar & _startIndex; ar & _curIndex; } template template void VocabBase::load(Archive & ar, const unsigned int version) { ar & _byWord; ar & _startIndex; ar & _curIndex; const_iterator iter = _byWord.begin(); size_t curIndex = _startIndex; while (true) { X word = iter->first; _byIndex[_byWord.find(word)->second] = (_byWord.find(word)->first); iter++; curIndex++; if (curIndex == _byWord.size()) break; } } template const X* VocabBase::getWord(const VocabularyIndex& key, bool& exist) const { const_iterator_rev iter; iter = _byIndex.find(key); if(iter != _byIndex.end()) { exist = true; return &(iter->second); } else { exist = false; return NULL; } } // If a word is not already in the vocab, then add it and give it an // index that's one higher than the current highest, regardless of // whether there are any "missing" indexes. template const VocabularyIndex VocabBase::addWord(const X& word) { const VocabularyIndex index = getIndex(word); if(index == VocabularyNone){ _byWord.insert(pair(word,_curIndex)); // find first instance of the word, in case there's more than one X firstIndex = _byWord.find(word)->first; _byIndex.insert(pair(_curIndex, firstIndex)); _curIndex++; if (debug(6)) cerr << "Vocab " << m_uniqueID << " inserting " << word << " at index " << _curIndex << endl; return _curIndex - 1; } else { return index; } } // Add word to vocab - even if it's already there - and give it an // index that's one higher than the current highest, regardless of // whether there are any "missing" indexes. template const VocabularyIndex VocabBase::addDuplicateWord(const X& word) { _byWord[word] = _curIndex; _byIndex[_curIndex] = (_byWord.find(word)->first); _curIndex++; return _curIndex - 1; } template bool VocabBase::contains(const X& key) const { const_iterator iter; iter = _byWord.find(key); return iter != _byWord.end(); } template const VocabularyIndex VocabBase::getIndex(const X& key) const { const_iterator iter = _byWord.find(key); if(iter != _byWord.end()) return iter->second; else return VocabularyNone; } template VocabBase& VocabBase::operator=(const VocabBase& other) { clear(); _startIndex = other._startIndex; _curIndex = other._curIndex; const_iterator iter; for(iter = other.begin(); iter != other.end(); iter++) { _byWord[iter->first] = iter->second; _byIndex[iter->second] = (_byWord.find(iter->first)->first); } return *this; } template void VocabBase::clear() { _byIndex.clear(); _byWord.clear(); _curIndex = _startIndex; } #endif bdtl-1.0.3/src/util/containers/NBest.hpp0000644000175000017500000000726512122405072017674 0ustar melamedmelamed/*! * \file NBest.hpp * \class NBest * \brief An n-best list implementation. * * This implementation is based on a sorted container, the multiset. * \warning Duplicates are *not* checked for. * * The type T of elements in the NBest list must * define 'operator>' for priority comparison. */ #ifndef _NBestH__ #define _NBestH__ #include #include #ifndef DOXYGEN using namespace std; #endif template class NBest { protected: /// Maximum size of the priority queue. size_t n; public: /// Create an n-best list. /// \param N The size of the n-best list. NBest(size_t N) : n(N) { sanityCheck(); } /// Clear the n-best list. void clear(); /// Is the n-best list empty? bool empty() const; /// Is the n-best list full? bool full() const; /// How many elements are there? size_t size() const; /// Retrieve the highest priority element. const T& top() const; /// Retrieve the lowest priority element. const T& bottom() const; /// Push an element onto the list. /// Iff the list isn't full, just insert the new element. /// Otherwise, the list is full: /// Iff the pushed element has lower priority than the /// worst element in the list, then don't insert the element. /// Otherwise, evict the worst element and insert the /// new element. /// /// \warning We do not detect duplicates: Iff there is already an /// element on the list with the same object (i.e. object equality, /// not necessarily priority equality), we won't detect that. /// /// \bug We don't do proper tiebreaking if two or more items /// (including or excluding the pushed item) have the same worst /// priority. void push(const T& t); /// Remove the highest priority item. void pop(); friend ostream& operator<<(ostream &o, const NBest& q) { typename multiset >::const_iterator qci = q.Items.begin(); const typename multiset >::const_iterator sentinel = q.Items.end(); for(; qci != sentinel; ++qci) { o << (*qci) << endl; } return o; } private: void sanityCheck(); /// Storage of the elements multiset > Items; }; template void NBest::sanityCheck() { assert(n > 0); assert(Items.size() <= n); } template void NBest::push(const T& t) { sanityCheck(); if (full()) { if (t < bottom()) { //DEBUG(10, "NBest::push ignoring " << t << endl); return; } // Otherwise, erase the worst element to prepare for inserting t. typename multiset::iterator last = Items.end(); --last; //DEBUG(10, "NBest::push erasing " << *last << endl); Items.erase(last); } //DEBUG(10, "NBest::push inserting " << t << endl); Items.insert(t); sanityCheck(); } /// Clear the n-best list. template void NBest::clear() { sanityCheck(); Items.clear(); sanityCheck(); } /// Is the n-best list empty? template bool NBest::empty() const { return Items.empty(); } /// Is the n-best list full? template bool NBest::full() const { return Items.size() == n; } /// How many elements are there? template size_t NBest::size() const { return Items.size(); } /// Retrieve the highest priority element. template const T& NBest::top() const { return *(Items.begin()); } /// Retrieve the lowest priority element. template const T& NBest::bottom() const { typename multiset::iterator last = Items.end(); return *(--last); } /// Remove the highest priority item. template void NBest::pop() { sanityCheck(); Items.erase(Items.begin()); sanityCheck(); } #endif /* _NBestH_ */ bdtl-1.0.3/src/HOWTO.build0000644000175000017500000000325612151020420014733 0ustar melamedmelamedHow to build the BDTL software BDTL uses several Boost libraries. If Boost is not already installed on your system, get the tarball from boost.org, and build the binaries. Name the top level directory of your boost installation (or a symlink to that directory) 'boost-$(ARCH_SIZE)bit', where $(ARCH_SIZE) is either 32 or 64, depending on your system's bus size. Copy local.mk.EXAMPLE to local.mk, and edit it to set the BOOST_PARENT parameter to point to the PARENT of that direcory. BDTL also uses the Google Sparsehash library. Download it from http://code.google.com/p/sparsehash/ . Set the GOOG_INCLUDES parameter in local.mk to point to that library (which is all headers, so there's nothing to build). Check the other variables in local.mk to ensure that they point to the correct executables, especially your compiler. *** VERY IMPORTANT *** Set the environment variable BDTL_BASE to point to the top directory of your BDTL installation. Nothing will work without that, not even the sandbox. Decide which build flags you want to use. Currently, the available flags are: 32BIT You must use this flag if you want to run BDTL on a 32bit machine. The default is 64bit. DYNAMIC If you want dynamic libraries. The default is static. OPT If you want to compile optimized binaries without debug info. The default is unoptimized with debug info. OPTDBG If you want to compile optimized binaries with debug info. By default is unoptimized with debug info. GOOG If you want to compile with the Google profiling tools. GPROF If you want to compile with the gprof profiler. Change to the src/ directory and type `make`. The syntax for using the build flags is like this: `make 32BIT=y OPT=y` bdtl-1.0.3/src/local.mk0000777000175000017500000000000012330444352016277 2../local.mkustar melamedmelamedbdtl-1.0.3/src/Makefile0000644000175000017500000001405312301202623014453 0ustar melamedmelamed# Makefile for Boosted Decision Trees (BDTL) executables and libraries # used for compilation within emacs, where we can't see ENV variables # BDTL_BASE = /home/melamed/dev/bdtl-onForge-trunk include $(BDTL_BASE)/local.mk # according to the gcc manual, some linker switches require copies of # compiler switches that were used, but including compiler switches # never hurts the link process VERSION = 1.0.1 SRC = $(BASE_DIR)/src SRC_BDTL = $(SRC)/bdtl MISC_UTIL_SRC = $(UTIL_BASE)/misc CONTAINERS_SRC = $(UTIL_BASE)/containers BDTL_DEPEND_FILE = $(SRC)/bdtl.depend.mk.$(ARCH).$(SUFFIX) UTIL_BASE = $(SRC)/util OBJ_BASE = $(BASE_DIR)/obj BDTL_LIBS_DIR = $(OBJ_BASE)/$(ARCH)/$(SUFFIX) BDTL_OBJ = $(BDTL_LIBS_DIR)/bdtl UTIL_OBJ = $(BDTL_LIBS_DIR)/util MISC_UTIL_OBJ = $(UTIL_OBJ)/misc CONTAINERS_OBJ = $(UTIL_OBJ)/containers BOOST_BASE = $(BOOST_PARENT)/boost-$(ARCH_SIZE)bit # BOOST_BASE = $(BOOST_PARENT)/boost_1_49_0.64bit BOOST_INCLUDES = -I $(BOOST_BASE) BOOST_LIB_DIR = $(BOOST_BASE)/lib # -lboost_system is required to compile 64 bit code with boost-1.43 or higher (and possibly some lower versions too) BOOST_LIBS = -lboost_filesystem$(BLSX) -lboost_program_options$(BLSX) -lboost_date_time$(BLSX) -lboost_system$(BLSX) # -lboost_serialization$(BLSX) BDTL_INCLUDES = -I$(SRC) -I$(CONTAINERS_SRC) -I$(MISC_UTIL_SRC) # collect all the non-bdtl includes in one variable EXTERNAL_INCLUDES = $(BOOST_INCLUDES) $(GOOG_INCLUDES) INCLUDE = $(EXTERNAL_INCLUDES) $(BDTL_INCLUDES) -I $(CMP)/include # collect all the flags in one variable # the -fno and -Wno flags suppresses some useless warnings from g++4.1 # LOCAL_CC_ARGS = --std=c++0x -pedantic -Wall -fno-strict-aliasing -Wno-non-template-friend -Wno-long-long -Wno-deprecated $(ARCH_FLAG) LOCAL_CC_ARGS = --std=gnu++0x -pedantic -Wall -Wno-non-template-friend $(ARCH_FLAG) # LOCAL_CC_ARGS = -pedantic -Wall $(ARCH_FLAG) # Note: the FORCED_CMP_VERSION, if any, must come first CC_ARGS = $(FORCED_CMP_VERSION) $(OPT_DEBUG_FLAG) $(VALIDATION_FLAG) $(LOCAL_CC_ARGS) $(INCLUDE) \ $(MEM_POOL_FLAG) $(USE_GPROF) $(USECONDOR) $(GOOG_FLAG) ## $(STATIC_FLAG) $(OPT_DEBUG_FLAG) $(VALIDATION_FLAG) #CC_ARGS += -Wno-long-long -ggdb -O0 #CC_ARGS += -Wall -pedantic -g -pipe -O3 #CC_ARGS += -pg #CC_ARGS += -O3 #CC_ARGS += -O3 -DNDEBUG #OPT=y ifdef DYNAMIC CC_ARGS += -fPIC -DPIC endif # collect all the external libs in one variable BDTL_LIBS = EXTERNAL_LIBS = $(BOOST_LIBS) -lm -lstdc++ LIBS = $(BDTL_LIBS) $(EXTERNAL_LIBS) BDTL_LIB_PATHS = -L $(BDTL_LIBS_DIR) BOOST_LIB_PATHS = -L $(BOOST_LIB_DIR) # these should include the '-L' switches LIB_PATHS = $(BDTL_LIB_PATHS) $(BOOST_LIB_PATHS) $(EXTRA_LIB_PATHS) ifdef DYNAMIC LIBSPEC = $(LIB_PATHS) -L$(CMP)/lib$(ARCH_LIB_SUFX) -Wl,-R$(BDTL_OBJ) $(LIBS) else LIBSPEC = $(LIB_PATHS) -L$(CMP)/lib$(ARCH_LIB_SUFX) -Wl,-Bstatic -static-libgcc $(LIBS) endif LOCAL_LCC_ARGS = $(ARCH_FLAG) LCC_ARGS = $(LOCAL_LCC_ARGS) $(OPT_DEBUG_FLAG) $(VALIDATION_FLAG) # Library target for this build TARGETLIB_OBJS = $(UTILLIB_OBJS) \ $(BDTL_OBJ)/parameter.o \ $(BDTL_OBJ)/Feature.o \ $(BDTL_OBJ)/FeatureTest.o \ $(BDTL_OBJ)/CompoundFeatureTest.o \ $(BDTL_OBJ)/Weights.o \ $(BDTL_OBJ)/Example.o \ $(BDTL_OBJ)/ExampleWithY.o \ $(BDTL_OBJ)/SetExample.o \ $(BDTL_OBJ)/CompoundExample.o \ $(BDTL_OBJ)/ExVec.o \ $(BDTL_OBJ)/ExComparator.o \ $(BDTL_OBJ)/ExBuilder.o \ $(BDTL_OBJ)/LossFunctionBase.o \ $(BDTL_OBJ)/LossFunction.o \ $(BDTL_OBJ)/DecTree.o \ $(BDTL_OBJ)/DecNode.o \ $(BDTL_OBJ)/Ensemble.o \ $(BDTL_OBJ)/Trainer.o \ $(BDTL_OBJ)/IOHandler.o UTILLIB_OBJS = \ $(MISC_UTIL_OBJ)/Dbug.o \ $(MISC_UTIL_OBJ)/Time.o \ $(MISC_UTIL_OBJ)/ResourceUsage.o \ $(MISC_UTIL_OBJ)/OptionsPrinter.o \ $(MISC_UTIL_OBJ)/MoreStrings.o \ $(MISC_UTIL_OBJ)/MoreMath.o \ $(MISC_UTIL_OBJ)/MoreAlgorithms.o \ $(MISC_UTIL_OBJ)/UniqueID.o \ $(MISC_UTIL_OBJ)/RandomID.o \ $(MISC_UTIL_OBJ)/RNGWrapper.o \ $(MISC_UTIL_OBJ)/Path.o \ $(CONTAINERS_OBJ)/File.o \ $(CONTAINERS_OBJ)/BTNode.o TARGETLIBNAME = bdtl$(LIBSUFFIX) ifdef DYNAMIC TARGETLIBREF = lib$(TARGETLIBNAME).so TLIB_CMD = $(LCC) $(LCC_ARGS) -shared -Wl,-soname,$(TARGETLIBREF).$(VERSION) -o else TARGETLIBREF = lib$(TARGETLIBNAME).a TLIB_CMD = $(AR) rcs endif TARGETLIB = $(BDTL_LIBS_DIR)/$(TARGETLIBREF).$(VERSION) # targets start here all ALL: dir trainOne testOne compileEnsemble # machine-independent .so object for shipping pack: \ $(TARGETLIB_OBJS) $(LCC) $(LCC_ARGS) -shared -Wl,-soname,$(TARGETLIBREF).$(VERSION) -o $(TARGETLIBREF).$(VERSION) $^ $(TARGETLIB): \ $(TARGETLIB_OBJS) $(TLIB_CMD) $@ $^ cd $(BDTL_LIBS_DIR); ln -sf $(TARGETLIBREF).$(VERSION) $(TARGETLIBREF) trainOne: \ $(BDTL_OBJ)/trainOne.o \ $(TARGETLIB) @echo $(LCC) $^ $(LCC_ARGS) $(LIBSPEC) -l $(TARGETLIBNAME) -o $(BIN)/$@ testOne: \ $(BDTL_OBJ)/testOne.o \ $(TARGETLIB) @echo $(LCC) $(LCC_ARGS) $^ $(LIBSPEC) -l $(TARGETLIBNAME) -o $(BIN)/$@ compileEnsemble: \ $(BDTL_OBJ)/compileEnsemble.o \ $(TARGETLIB) @echo $(LCC) $(LCC_ARGS) $^ $(LIBSPEC) -l $(TARGETLIBNAME) -o $(BIN)/$@ myTest: \ $(BDTL_OBJ)/myTest.o \ $(TARGETLIB) @echo $(LCC) $(LCC_ARGS) $^ $(LIBSPEC) -l $(TARGETLIBNAME) -o $(BIN)/$@ dir: $(BIN) $(MISC_UTIL_OBJ) $(CONTAINERS_OBJ) $(BDTL_OBJ) @echo $(BDTL_OBJ): -mkdir -p $(BDTL_OBJ) $(BIN): -mkdir -p $(BIN) $(MISC_UTIL_OBJ): $(UTIL_OBJ_BASE) -mkdir -p $(MISC_UTIL_OBJ) $(CONTAINERS_OBJ): $(UTIL_OBJ_BASE) -mkdir -p $(CONTAINERS_OBJ) # object files $(BDTL_OBJ)/%.o: $(SRC_BDTL)/%.cpp # $(BDTL_OBJ) $(CC) $(CC_ARGS) -c -o $@ $< $(MISC_UTIL_OBJ)/%.o: $(MISC_UTIL_SRC)/%.cpp # $(MISC_UTIL_OBJ) $(CC) $(CC_ARGS) -c $< -o $@ $(CONTAINERS_OBJ)/%.o: $(CONTAINERS_SRC)/%.cpp # $(CONTAINERS_OBJ) $(CC) $(CC_ARGS) -c $< -o $@ depend: $(CC) -MM -MP $(CC_ARGS) $(SRC_BDTL)/*.cpp \ | sed -e 's|.*\.o:|$(BDTL_OBJ)/&|' \ > $(BDTL_DEPEND_FILE).new -cp $(BDTL_DEPEND_FILE).new $(BDTL_DEPEND_FILE) clean: -rm $(BIN)/* $(BDTL_OBJ)/* # -rm -f $(BDTL_DEPEND_FILE) spotless: clean -rm $(MISC_UTIL_OBJ)/* $(CONTAINERS_OBJ)/* # the dependencies are included below, to match the standard format -include $(BDTL_DEPEND_FILE) # DO NOT DELETE bdtl-1.0.3/src/bdtl/0000755000175000017500000000000012330444352013746 5ustar melamedmelamedbdtl-1.0.3/src/bdtl/IOHandler.hpp0000644000175000017500000000505012151020420016247 0ustar melamedmelamed#include "Vector.hpp" #include "File.hpp" #include "bdtl/DecTree.hpp" #ifndef _BDTIohandlerH__ #define _BDTIohandlerH__ namespace bdt { class Ensemble; /// class to handle all the files that BDT's training uses class IOHandler { public: IOHandler(string key, IOHandler* pNextHandler = NULL); ~IOHandler(); void setKey(const string& key) {m_key = key;} void write(unsigned iteration, const DecTree& tree, double regnFactor); /// read and store all the relevant sequences void read(); /// \param ret the Ensemble built from the stored tree sequence /// \param iteration current iteration, which will updated by the number of trees read in /// \param newMinRegr use trees only up to this new minimum regularization penalty /// \param updateRet update the ret Ensemble iff this flag is true; /// otherwise just read in the previous ensemble and dump it to the /// m_pNextHandler. void getEnsemble(Ensemble*& ret, unsigned& iteration, double newMinRegr = 0.0, bool updateRet = true); void open(ios_base::openmode mode = std::ios_base::out); bool is_open() { return m_is_open;} void flush(); void close(); /// Make sure that the checkpoint vector has exactly len values. /// If it has more, then chop the vector to this length and rewrite the checkpoint file. /// \pre v.size() >= len /// \post v.size() == len /// \todo Template the container type too template void ensureLength(unsigned len, Vector& v, const string& filename); /// advance sequence marker past the given reg'r void advanceSeqMarker(double regr); const Vector& getTreeVec() { return treeVec;} const Vector& getRegrVec() { return regrVec;} static bool readDoneTraining(const string& key); /// write a file called 'doneTraining' /// \param why String describing why we stopped. /// \post IOHandler::readDoneTraining(key) == true void writeDone(const string& why); /// write the key to a file called 'key' void writeKey(); static const string makeFileName(const string& prefix, const string& key); private: string m_key; bool m_is_open; // if this is the handler for the "previous" ensemble, then the // nextHandler is used to dump parts of the previous ensemble into // the current one IOHandler* m_pNextHandler; oFile* treeSequence; oFile* regrSequence; string treeSeqFileName; string regrSeqFileName; Vector treeVec; Vector regrVec; // says how far down the stored sequences we have looked unsigned m_seqMarker; }; } #include "bdtl/IOHandler.ipp" #endif bdtl-1.0.3/src/bdtl/SetExample.cpp0000644000175000017500000002210012151020420016477 0ustar melamedmelamed/*! * \file SetExample.cpp */ #include "bdtl/SetExample.hpp" #include "bdtl/Feature.hpp" #include "Dbug.hpp" #include "bdtl/parameter.hpp" #include "MoreExceptions.hpp" namespace bdt { template template SetExample::SetExample(const fHashT2& fset, unsigned partition, predict_t yValue, fpVal_t weight) : ExampleWithY(partition, yValue, weight), m_fSet(new fHash_t(fset.size())) { constructHelp(); m_fSet->insert(fset.begin(), fset.end()); // An empty feature set is perfectly valid. -IDM // if (m_fSet->empty()) throw std::underflowError("SetExample given empty feature set"); assert(m_fSet->size() == fset.size()); } // template<> template<> // DenseSetExample::SetExample(const SparseSetExample& e) { // Dbug::warning("Why are you doing this conversion?"); // THROW(""); // } // template<> template<> // SparseSetExample::SetExample(const DenseSetExample& e) { // Dbug::warning("Why are you doing this conversion?"); // THROW(""); // } // template<> void DenseSetExample::constructHelp() { m_fSet->set_empty_key(EMPTYFEATURE); } template<> void SparseSetExample::constructHelp() { } template<> void SparseMapExample::constructHelp() { } template<> bool SparseSetExample:: satisfies(const FeatureTest& ft) const { return satisfies(ft.m_feature, ft.m_rel, ft.m_val); } template<> bool SparseMapExample:: satisfies(const FeatureTest& ft) const { const Feature theFeat(ft.m_feature); rel_t theRel(ft.m_rel); double testVal(ft.m_val); return satisfies(theFeat, theRel, testVal); } template<> bool SparseSetExample:: satisfies(const Feature& theFeat, const rel_t& theRel, const double& testVal) const { bool hasIt = hasFeature(theFeat); bool isSame = (hasIt && testVal == 1.0) || (! hasIt && testVal == 0.0); switch(theRel) { case eq: return isSame; break; case ne: return (! isSame); break; default: THROW("inequalities not allowed in SparseSetExample::satisfies()"); } return false; // duck compiler warning } template<> bool SparseMapExample:: satisfies(const Feature& theFeat, const rel_t& theRel, const double& testVal) const { const hash_t::const_iterator hci = m_fSet->find(theFeat); const hash_t::const_iterator sentinel = m_fSet->end(); if (hci == sentinel) { // default feature value is 0.0f, which also represents 'false' for binary features static const double egVal = DEFAULT_FEATURE_VALUE; switch(theRel) { case eq: THROW("equality comparison on floats!"); break; case ne: THROW("equality comparison on floats!"); break; case lt: return (egVal < testVal); break; case gt: return (egVal > testVal); break; case le: return (egVal <= testVal); break; case ge: return (egVal >= testVal); break; default: THROW("unknown rel_t"); } } return satisfies(hci, theRel, testVal); } // // this one is static // template<> bool SparseSetExample:: // satisfies(const SparseFeatureSet::const_iterator& fci, const rel_t& theRel, const double& testVal) { // // here we assume that whoever called this has the feature, so it's value is true, i.e. 1. // switch(theRel) { // case eq: THROW("equality comparison on floats!"); break; // case ne: THROW("equality comparison on floats!"); break; // case lt: return (1.0 < testVal); break; // case gt: return (1.0 > testVal); break; // case le: return (1.0 <= testVal); break; // case ge: return (1.0 >= testVal); break; // default: THROW("unknown rel_t"); // } // } // this one is static template<> bool SparseMapExample:: satisfies(const SparseFeatureMap::const_iterator& fci, const rel_t& theRel, const double& testVal) { double egVal; if (parameter::isFloat(fci->first)) egVal = fci->second.f; else egVal = (double)(fci->second.i); switch(theRel) { case eq: THROW("equality comparison on floats!"); break; case ne: THROW("equality comparison on floats!"); break; case lt: return (egVal < testVal); break; case gt: return (egVal > testVal); break; case le: return (egVal <= testVal); break; case ge: return (egVal >= testVal); break; default: THROW("unknown rel_t"); } THROW(""); // should never get to here return false; // duck compiler warning } template<> bool SparseSetExample:: hasFeature(const Feature& f) const { return m_fSet->find(f) != m_fSet->end(); } template<> bool SparseMapExample:: hasFeature(const Feature& f) const { return m_fSet->find(f) != m_fSet->end(); } template<> IFUnion_t SparseMapExample:: lookup(const unsigned& featID) const { const Feature& theFeat(Feature::get(featID)); if (parameter::isFloat(featID)) { return lookup(theFeat, true); } else { return lookup(theFeat, false); } } template<> IFUnion_t SparseMapExample:: lookup(const Feature& theFeat, const bool isFloat) const { IFUnion_t ret; const hash_t::const_iterator iter = m_fSet->find(theFeat); if (iter == m_fSet->end()) { if (isFloat) ret.f = DEFAULT_FEATURE_VALUE; else ret.i = (intVal_t)DEFAULT_FEATURE_VALUE; } else { ret = iter->second; } return ret; } template<> string SparseMapExample::name() { return "SparseMapExample"; } template<> string SparseSetExample::name() { return "SparseSetExample"; } // template<> string DenseSetExample::name() { return "DenseSetExample"; } template<> template bool SparseSetExample:: setFeatures(const fHashT2& fset) { m_fSet->insert(fset.begin(), fset.end()); assert(m_fSet->size() == fset.size()); return true; } template<> bool SparseSetExample:: setFeatures(const featStringCont_t& strs, const char delim) { m_fSet->clear(); featStringCont_t::const_iterator i = strs.begin(); const featStringCont_t::const_iterator sentinel = strs.end(); for (; i != sentinel; ++i) { const string& stringFeatName = *i; const FeatureBase_t& featName = lexical_cast(stringFeatName); const Feature theFeat(featName); m_fSet->insert(theFeat); parameter::setFloat(theFeat, false); } return true; } template<> bool SparseMapExample:: setFeatures(const featStringCont_t& strs, const char delim) { featStringCont_t::const_iterator i = strs.begin(); featStringCont_t::const_iterator sentinel = strs.end(); IFUnion_t ifu; for (; i != sentinel; ++i) { string::size_type spacePos = i->find(delim); if (spacePos == string::npos) { const string& stringFeatName = *i; const FeatureBase_t& featName = lexical_cast(stringFeatName); const Feature theFeat(featName); ifu.i = 1; // TRUE parameter::setInteger(theFeat, true); m_fSet->insert(make_pair(theFeat,ifu)); } else { const string& stringFeatName = i->substr(0, spacePos); const FeatureBase_t& featName = lexical_cast(stringFeatName); const Feature theFeat(featName); const string featValS = i->substr(spacePos + 1); // till the end by default if (parameter::isFloat(theFeat) || featValS.find('.') != string::npos) { // it's floating-point ifu.f = lexical_cast(featValS); parameter::setFloat(theFeat, true); } else { ifu.i = lexical_cast(featValS); parameter::setInteger(theFeat, true); } m_fSet->insert(make_pair(theFeat,ifu)); } } return true; } ostream& operator<<(ostream& out, const SparseSetExample& fe) { out << fe.name() << " "; fe.print(out); out << "; fset ="; SparseFeatureSet::const_iterator hashci = fe.m_fSet->begin(); const SparseFeatureSet::const_iterator sentinel = fe.m_fSet->end(); for(; hashci != sentinel; ++hashci) { out << " " << *hashci; } return out; } ostream& operator<<(ostream& out, const SparseMapExample& fe) { out << fe.name() << " "; fe.print(out); out << "; fset ="; SparseFeatureMap::const_iterator hashci = fe.m_fSet->begin(); const SparseFeatureMap::const_iterator sentinel = fe.m_fSet->end(); for(; hashci != sentinel; ++hashci) { const Feature& theFeat = hashci->first; out << " " << theFeat << " "; const IFUnion_t& theVal = hashci->second; if (parameter::isFloat(theFeat)) out << theVal.f; else out << theVal.i; } return out; } // TEMPLATE INSTANTIATIONS //template SparseSetExample::SetExample(const Example& e); //template SparseSetExample::SetExample(const IntermediateExample& e); //template bool SparseSetExample::hasFeature(const Feature* f) const; //template DenseSetExample::SetExample(const Example& e); //template DenseSetExample::SetExample(const IntermediateExample& e); //template bool DenseSetExample::hasFeature(const Feature* f) const; template bool SparseMapExample::hasFeature(const Feature& f) const; template bool SparseSetExample::hasFeature(const Feature& f) const; // template bool DenseSetExample::hasFeature(const Feature& f) const; // template SparseSetExample::SetExample(const SparseFeatureSet& fset, unsigned partition, bool yValue, fpVal_t weight); // template SparseMapExample::SetExample(const SparseFeatureMap& fset, unsigned partition, bool yValue, fpVal_t weight); template bool SparseSetExample::setFeatures(const featStringCont_t& strs, const char delim); template bool SparseMapExample::setFeatures(const featStringCont_t& strs, const char delim); } // namespace bdt bdtl-1.0.3/src/bdtl/CompoundExample.ipp0000644000175000017500000004170212151020420017547 0ustar melamedmelamed/*! * \file CompoundExample.ipp */ #include "bdtl/CompoundExample.hpp" #include "bdtl/Feature.hpp" #include "Dbug.hpp" #include "bdtl/parameter.hpp" #include "MoreExceptions.hpp" namespace bdt { template CompoundExample:: CompoundExample(unsigned partition, predict_t yValue, fpVal_t weightBias) : ExampleWithY(partition, yValue, weightBias), m_featSet(new fHashSet_t()), m_featMap(new fHashMap_t()) { constructHelp(); } template template CompoundExample:: CompoundExample(const fHashSet2T& featSet, const fHashMap2T& featMap, unsigned partition, predict_t yValue, fpVal_t weightBias) : ExampleWithY(partition, yValue, weightBias), m_featSet(new fHashSet_t(featSet.size())), m_featMap(new fHashMap_t(featMap.size())) { constructHelp(); m_featSet->insert(featSet.begin(), featSet.end()); m_featMap->insert(featMap.begin(), featMap.end()); // An empty feature set is perfectly valid. -IDM // if (M_FeatSet->empty()) throw std::underflowError("Example ctor given empty feature set"); assert(m_featSet->size() == featSet.size()); assert(m_featMap->size() == featMap.size()); } template void CompoundExample:: constructHelp() {} // template<> void boolDenseCompoundExample::constructHelp() { // m_featSet->set_empty_key(EMPTYFEATURE); // m_featMap->set_empty_key(EMPTYFEATURE); // } template bool CompoundExample:: satisfies(const FeatureTest& ft) const { const Feature& theFeat(ft.m_feature); const rel_t& theRel(ft.m_rel); const double& testVal(ft.m_val); return satisfies(theFeat, theRel, testVal); } template bool CompoundExample:: satisfies(const Feature& theFeat, const rel_t& theRel, const double& testVal) const { if(theRel != eq && theRel != ne) return satisfiesMap(theFeat, theRel, testVal); else return satisfiesSet(theFeat, theRel, testVal); } template bool CompoundExample:: satisfiesSet(const Feature& theFeat, const rel_t& theRel, const double& testVal) const { bool hasIt = hasFeature(theFeat); DEBUG(NODE_DBG_LEVEL + 1, "example has feature " << theFeat << ": " << (hasIt ? "yes" : "no") << endl); bool isSame = (hasIt && testVal == 1.0) || (! hasIt && testVal == 0.0); DEBUG(NODE_DBG_LEVEL + 1, "example isSame for " << theFeat << ": " << (isSame ? "yes" : "no") << endl); switch(theRel) { case eq: DEBUG(NODE_DBG_LEVEL + 1, "theRel == eq" << endl); return isSame; break; case ne: DEBUG(NODE_DBG_LEVEL + 1, "theRel == ne" << endl); return (! isSame); break; default: THROW("inequalities not allowed here"); } return false; // duck compiler warning } template bool CompoundExample:: satisfiesMap(const Feature& theFeat, const rel_t& theRel, const double& testVal) const { const typename hashT_B::const_iterator hci = m_featMap->find(theFeat); const typename hashT_B::const_iterator sentinel = m_featMap->end(); if (hci == sentinel) { // default feature value is 0.0f, which also represents 'false' for binary features static const double egVal = DEFAULT_FEATURE_VALUE; bool ret; switch(theRel) { case eq: THROW("equality comparison on floats!"); break; case ne: THROW("equality comparison on floats!"); break; case lt: ret = (egVal < testVal); DEBUG(FEAT_DBG_LEVEL, "Testing " << egVal << " < " << testVal << " for feature " << theFeat << ": " << (ret ? "true" : "false") << endl); break; case gt: ret = (egVal > testVal); DEBUG(FEAT_DBG_LEVEL, "Testing " << egVal << " > " << testVal << " for feature " << theFeat << ": " << (ret ? "true" : "false") << endl); break; case le: ret = (egVal <= testVal); DEBUG(FEAT_DBG_LEVEL, "Testing " << egVal << " <= " << testVal << " for feature " << theFeat << ": " << (ret ? "true" : "false") << endl); break; case ge: ret = (egVal >= testVal); DEBUG(FEAT_DBG_LEVEL, "Testing " << egVal << " >= " << testVal << " for feature " << theFeat << ": " << (ret ? "true" : "false") << endl); break; default: THROW("unknown rel_t"); } return ret; } const typename hashT_A::const_iterator sentinel2(m_featSet->end()); return satisfies(sentinel2, hci, sentinel2, theRel, testVal); } // this one is static template bool CompoundExample:: satisfies(const typename hashT_A::const_iterator& fci_a, const typename hashT_B::const_iterator& fci_b, const typename hashT_A::const_iterator& sentinel_a, const rel_t& theRel, const double& testVal) { if(fci_a == sentinel_a) return satisfiesMap(fci_b, theRel, testVal); else return satisfiesSet(theRel, testVal); } // this one is static template bool CompoundExample:: satisfiesSet(const rel_t& theRel, const double& testVal) { // here we assume that whoever called this has the feature, so it's value is true, i.e. 1. switch(theRel) { case eq: THROW("equality comparison on floats!"); break; case ne: THROW("equality comparison on floats!"); break; case lt: return (1.0 < testVal); break; case gt: return (1.0 > testVal); break; case le: return (1.0 <= testVal); break; case ge: return (1.0 >= testVal); break; default: THROW("unknown rel_t"); } } // this one is static template bool CompoundExample:: satisfiesMap(const typename fHashMap_t::const_iterator& fci, const rel_t& theRel, const double& testVal) { fpVal_t egVal; if (parameter::isFloat(fci->first)) egVal = fci->second.f; else egVal = (fpVal_t)(fci->second.i); bool ret; switch(theRel) { case eq: THROW("equality comparison on floats!"); break; case ne: THROW("equality comparison on floats!"); break; case lt: ret = (egVal < testVal); DEBUG(FEAT_DBG_LEVEL, "Testing " << egVal << " < " << testVal << " for feature " << fci->first << ": " << (ret ? "true" : "false") << endl); break; case gt: ret = (egVal > testVal); DEBUG(FEAT_DBG_LEVEL, "Testing " << egVal << " > " << testVal << " for feature " << fci->first << ": " << (ret ? "true" : "false") << endl); break; case le: ret = (egVal <= testVal); DEBUG(FEAT_DBG_LEVEL, "Testing " << egVal << " <= " << testVal << " for feature " << fci->first << ": " << (ret ? "true" : "false") << endl); break; case ge: ret = (egVal >= testVal); DEBUG(FEAT_DBG_LEVEL, "Testing " << egVal << " >= " << testVal << " for feature " << fci->first << ": " << (ret ? "true" : "false") << endl); break; default: THROW("unknown rel_t"); } return ret; } // template<> bool boolDenseCompoundExample:: // // satisfies(const ScalarFeatureTest& ft) const { // satisfies(const FeatureTest& ft) const { // const Feature theFeat(ft.m_feature); // rel_t theRel(ft.m_rel); // double testVal(ft.m_val); // return satisfies(theFeat, theRel, testVal); // } // template<> bool boolDenseCompoundExample:: // satisfies(const Feature& theFeat, const rel_t& theRel, const double& testVal) const { // if(setBhasFeature(theFeat)) // return satisfiesMap(theFeat, theRel, testVal); // else // return satisfiesSet(theFeat, theRel, testVal); // } // template<> bool boolDenseCompoundExample:: // satisfiesSet(const Feature& theFeat, const rel_t& theRel, const double& testVal) const { // bool hasIt = hasFeature(theFeat); // bool isSame = (hasIt && testVal == 1.0) || (! hasIt && testVal == 0.0); // switch(theRel) { // case eq: return isSame; break; // case ne: return (! isSame); break; // default: THROW("inequalities not allowed in boolDenseCompoundExample::satisfies()"); // } // return false; // duck compiler warning // } // // examine here // template<> bool boolDenseCompoundExample:: // satisfiesMap(const Feature& theFeat, const rel_t& theRel, const double& testVal) const { // const hashT_B::const_iterator hci = m_featMap->find(theFeat); // const hashT_B::const_iterator sentinel = m_featMap->end(); // if (hci == sentinel) { // // default feature value is 0.0f, which also represents 'false' for binary features // static const fpVal_t egVal = DEFAULT_FEATURE_VALUE; // // upper bound for float comparisons // switch(theRel) { // case eq: THROW("equality comparison on floats!"); break; // case ne: THROW("equality comparison on floats!"); break; // case lt: return (egVal < testVal); break; // case gt: return (egVal > testVal); break; // case le: return (egVal <= testVal); break; // case ge: return (egVal >= testVal); break; // default: THROW("unknown rel_t"); // } // } // const hashT_A::const_iterator sentinel2(m_featSet->end()); // return satisfies(sentinel2, hci, sentinel2, theRel, testVal); // } // // this one is static // template<> bool boolDenseCompoundExample:: // satisfies(const hashT_A::const_iterator& fci_a, const hashT_B::const_iterator& fci_b, // const hashT_A::const_iterator& sentinel_a, // const rel_t& theRel, const double& testVal) { // if(fci_a == sentinel_a) // return satisfiesMap(fci_b, theRel, testVal); // else // return satisfiesSet(fci_a, theRel, testVal); // } // // this one is static // template<> bool boolDenseCompoundExample:: // satisfiesSet(const DenseFeatureSet::const_iterator& fci, const rel_t& theRel, const double& testVal) { // // here we assume that whoever called this has the feature, so it's value is true, i.e. 1. // switch(theRel) { // case eq: THROW("equality comparison on floats!"); break; // case ne: THROW("equality comparison on floats!"); break; // case lt: return (1.0 < testVal); break; // case gt: return (1.0 > testVal); break; // case le: return (1.0 <= testVal); break; // case ge: return (1.0 >= testVal); break; // default: THROW("unknown rel_t"); // } // } // // this one is static // template<> bool boolDenseCompoundExample:: // satisfiesMap(const DenseFeatureMap::const_iterator& fci, const rel_t& theRel, const double& testVal) { // fpVal_t egVal; // if (parameter::isFloat(fci->first)) // egVal = fci->second.f; // else // egVal = (fpVal_t)(fci->second.i); // switch(theRel) { // case eq: THROW("equality comparison on floats!"); break; // case ne: THROW("equality comparison on floats!"); break; // case lt: return (egVal < testVal); break; // case gt: return (egVal > testVal); break; // case le: return (egVal <= testVal); break; // case ge: return (egVal >= testVal); break; // default: THROW("unknown rel_t"); // } // THROW(""); // should never get to here // return false; // duck compiler warning // } template bool CompoundExample:: hasFeature(const Feature& f) const { return setAhasFeature(f) || setBhasFeature(f); } template bool CompoundExample:: setAhasFeature(const Feature& f) const { return m_featSet->find(f) != m_featSet->end(); } template bool CompoundExample:: setBhasFeature(const Feature& f) const { // find is expensive, let's try to avoid doing it if we can if(m_featMap->size() == 0) return false; else return m_featMap->find(f) != m_featMap->end(); } template IFUnion_t CompoundExample:: lookup(const unsigned& featID) const { const Feature& theFeat(Feature::get(featID)); if (parameter::isFloat(theFeat)) return lookup(theFeat, true); return lookup(theFeat, false); } // template<> IFUnion_t boolDenseCompoundExample:: // lookup(const unsigned& featID) const { // const Feature& theFeat(Feature::get(featID)); // if (parameter::isFloat(theFeat)) // return lookup(theFeat, true); // return lookup(theFeat, false); // } // This only needs to look at the map, since it doesn't make sense to do a lookup // for a binary feature template IFUnion_t CompoundExample:: lookup(const Feature& theFeat, const bool isFloat) const { IFUnion_t ret; // if(parameter::isBinary(theFeat)) { // const typename hashT_A::const_iterator iter = m_featSet->find(theFeat); // if(iter == m_featSet->end()) ret.i = 0; // else ret.i = 1; // } // else { const typename hashT_B::const_iterator iter = m_featMap->find(theFeat); if (iter == m_featMap->end()) { if (isFloat) ret.f = DEFAULT_FEATURE_VALUE; else ret.i = (intVal_t)DEFAULT_FEATURE_VALUE; } else { ret = iter->second; } // } return ret; } // // This only needs to look at the map, since it doesn't make sense to do a lookup // // for a binary feature // template<> IFUnion_t boolDenseCompoundExample:: // lookup(const Feature& theFeat, const bool isFloat) const { // IFUnion_t ret; // if(parameter::isBinary(theFeat)) { // const hashT_A::const_iterator iter = m_featSet->find(theFeat); // if(iter == m_featSet->end()) ret.i = 0; // else ret.i = 1; // } // else { // const hashT_B::const_iterator iter = m_featMap->find(theFeat); // if (iter == m_featMap->end()) { // if (isFloat) // ret.f = DEFAULT_FEATURE_VALUE; // else // ret.i = (intVal_t)DEFAULT_FEATURE_VALUE; // } else { // ret = iter->second; // } // } // return ret; // } template template bool CompoundExample:: setFeatures(const fHashSet2T& featSet, const fHashMap2T& featMap) { m_featSet->insert(featSet.begin(), featSet.end()); m_featMap->insert(featMap.begin(), featMap.end()); // An empty feature set is perfectly valid. -IDM // if (M_FeatSet->empty()) throw std::underflowError("Example given empty feature set"); assert(m_featSet->size() == featSet.size()); assert(m_featMap->size() == featMap.size()); return true; } template bool CompoundExample:: setFeatures(const featStringCont_t& strs, const char delim) { m_featSet->clear(); m_featMap->clear(); typename featStringCont_t::const_iterator i = strs.begin(); typename featStringCont_t::const_iterator sentinel = strs.end(); IFUnion_t ifu; for(; i != sentinel; ++i) { DEBUG(FEAT_DBG_LEVEL+1, "CompEx::setFeats working on feature:value " << *i << endl); string::size_type delimPos = i->find(delim); if(delimPos == string::npos) { const string& stringFeatName = *i; const FeatureBase_t& featName = lexical_cast(stringFeatName); const Feature theFeat(featName); m_featSet->insert(theFeat); parameter::setBinary(theFeat, true); DEBUG(FEAT_DBG_LEVEL+1, "Adding binary feature " << theFeat << endl); } else { const string& stringFeatName = i->substr(0, delimPos); const FeatureBase_t& featName = lexical_cast(stringFeatName); const Feature theFeat(featName); const string featValS = i->substr(delimPos + 1); if(parameter::isFloat(theFeat) || featValS.find('.') != string::npos) { ifu.f = lexical_cast(featValS); parameter::setFloat(theFeat, true); DEBUG(FEAT_DBG_LEVEL+1, "Adding float-point feature " << theFeat << " with value " << ifu.f << endl); } else { ifu.i = lexical_cast(featValS); parameter::setInteger(theFeat, true); DEBUG(FEAT_DBG_LEVEL+1, "Adding integer feature " << theFeat << " with value " << ifu.i << endl); } m_featMap->insert(make_pair(theFeat, ifu)); } } return true; } } // namespace bdt bdtl-1.0.3/src/bdtl/compileEnsemble.cpp0000644000175000017500000000275712151020420017553 0ustar melamedmelamed#include "BoostMacros.hpp" #include "OptionsPrinter.hpp" #include "Vocabulary.hpp" #include "StringWithSpaces.hpp" #include "bdtl/parameter.hpp" #include "bdtl/Ensemble.hpp" using namespace bdt; /// Compile an Ensemble from a file of decision trees, down to a given regularization coefficient. int main(int argc, char** argv) { // BEGIN configuration string progName = "compileEnsemble"; string progDescr = "compile an Ensemble from a file of decision trees"; string penaltySeqFileName, treeSeqFileName; double minGlobalRegnFactor; bool useCFTCounts = false; po::options_description local("LocalParameters"); local.add_options() ("penaltySeqFileName,l", po::value(&penaltySeqFileName)) ("treeSeqFileName,c", po::value(&treeSeqFileName)) ("minGlobalRegnFactor,r", po::value(&minGlobalRegnFactor)) ("useCFTCounts,u", po::value(&useCFTCounts)) ; parameter::read(argc, argv, local); // END configuration // BEGIN VALIDATION if (penaltySeqFileName.empty()) { cerr << "missing penalty sequence file name" << endl; exit(1); } if (treeSeqFileName.empty()) { cerr << "missing tree sequence file name" << endl; exit(1); } // END VALIDATION // THE MAIN EVENT Ensemble ens("", useCFTCounts); const double minRegn = minGlobalRegnFactor; const double maxRegn = 0.0; // don't use oFile* pDumpFile = NULL; ens.regularizedLoad(penaltySeqFileName, treeSeqFileName, minRegn, maxRegn, pDumpFile); ens.print(cout); } bdtl-1.0.3/src/bdtl/ExBuilder.hpp0000644000175000017500000000275112151020420016332 0ustar melamedmelamed/*! * \file ExBuilder.hpp * \class bdt::ExBuilder * \brief A class for building Example%s. */ #ifndef _BDTExBuilderH__ #define _BDTExBuilderH__ #include #include #include #include "Dbug.hpp" #include "bdtl/globals.hpp" #ifndef DOXYGEN using namespace std; #endif /* DOXYGEN */ namespace bdt { class Example; template class ExBuilder : public Dbug { public: typedef string key_t; typedef set keySet_t; typedef list featStringCont_t; /// default ctors suffice template static void buildExContainer(exCont_t&, const string& filename, ExampleFormat exFormat = GenPar); static Example* constructorHelp2(string& key, const string& exString, const ExampleFormat exFormat, const unsigned exCount, const keySet_t& ks = makeEmptySet()); static void constructorHelp3(unsigned exCount); static const keySet_t& makeEmptySet() { static const keySet_t emptyKeySet = keySet_t(); return emptyKeySet; } protected: // methods /// GP is for GenPar format; SVML is for SVM Light format static void readYValueGP(const string& yString, typename ex_t::m_predT& yValue); static string readMetaGP(const string& metaData, typename ex_t::m_predT& yValue, unsigned& partition, double& bias); static void readMetaSVML(const string& metaData, typename ex_t::m_predT& yValue); }; } // namespace bdt #include "bdtl/ExBuilder.ipp" #endif /* _BDTExBuilderH_ */ bdtl-1.0.3/src/bdtl/Ensemble.hpp0000644000175000017500000002126112151020420016176 0ustar melamedmelamed/*! * \file Ensemble.hpp * \class bdt::Ensemble * \brief An Ensemble of DecTree%s. */ #ifndef _BDTEnsembleH__ #define _BDTEnsembleH__ #include #include #include "bdtl/CompoundFeatureTest.hpp" // #include "UniqueID.hpp" #include "Vocabulary.hpp" #include "StringWithSpaces.hpp" #include "bdtl/ExVec.hpp" #include "bdtl/IOHandler.hpp" #ifndef DOXYGEN using namespace std; #endif /* DOXYGEN */ namespace bdt { class DecTree; class Ensemble: public Dbug // , public UniqueID // , boost::addable, boost::addable > { { //typedef hash_map map_t; //typedef google::dense_hash_map map_t; typedef google::sparse_hash_map > map_t; public: /// construct an empty Ensemble Ensemble(); Ensemble(const string key, const bool useCFTCounts = false); ~Ensemble() { } /// read compiled ensemble void loadCompiled(const std::string& key); /// Load an Ensemble from a file, one which is done training. /// \note We read the best Ensemble prefix size, and that many /// trees from the treeSequence file. /// \param key The key for the Ensemble /// \pre size() == 0 /// \pre IOHandler::readDoneTraining(key) void loadDone(const std::string& key); /// Load an Ensemble from a file, from a particular maximum l1 down to a particular minimum l1. /// \note We find the relevant starting and ending trees, and read them from the treeSequence file. /// \param key The key for the Ensemble /// \param maxRegn The maximum l1 at which to start reading trees. /// \param minRegn The minimum l1 to read the Ensemble through. /// \param pDumpFile dump whatever is read into this file, to initialize the checkpointing for the current run /// \pre size() == 0 void regularizedLoad(const std::string& key, const double minRegn, const double maxRegn = 0.0, oFile* pDumpFile = NULL); void regularizedLoad(const string& penaltySeqFileName, const string& treeSeqFileName, const double minRegn, const double maxRegn = 0.0, oFile* pDumpFile = NULL); /// Load an Ensemble from a file. /// \param filename The filename from which to read the Ensemble /// \param loadFrom starting reading at this tree /// \param loadTo finish reading at this tree /// \param pDumpFile dump the trees to this file as they're read /// \pre size() == 0 /// \post size() == prefixSize.get() || !prefixSize void load(const std::string& filename, const unsigned loadFrom = 0, const unsigned loadTo = 0, oFile* pDumpFile = NULL); /// use ensemble to predict confidences over example set void initializePredictor(const string& modelDirName, const string& configFileName = ""); static void predict(int argc, char **argv); template double predictOne(const string& exString, size_t exNum = 0) const; /// \note We use m_trees iff /// parameter::useDenseEnsembleOptimizationsDuringConfidenceComputation(), /// otherwise we use m_CFTWeights template double confidenceCompute(const ex_t& e) const; /// add a tree void add(const DecTree& tree, const double multiplier = 1.0); /// subtract a tree void subtract(const DecTree& tree); /// helper for add(Tree, multiplier) and for +=(Ensemble) void add(const CompoundFeatureTest& f, const double confDelta); /// Add this Tree to the Ensemble void operator+=(const DecTree& tree); void operator+=(const vector& trees); void operator+=(const Ensemble& other); /// Set Example confidences. /// \param exmpls The Example%s whose confidences should be set. template void updateConfidences(BSP exmpls) const; template void updateDPSNorms(BSP exmpls); fpVal_t getTreeCount() const { return m_treeCount; } inline const map_t& getCFTWeights() const {return m_CFTWeights;} unsigned getUseCFTCounts() const { return m_useCFTCounts; } void setUseCFTCounts(const bool val) { m_useCFTCounts = val; } inline const map_t& getDPSNorms() const {return m_DPSNorms;} inline map_t& getDPSNorms() {return m_DPSNorms;} bool empty() const { if(parameter::useOptimizedConfidenceComputation()) { return m_CFTIds.empty(); } else { return m_CFTWeights.empty(); } } /// The weight of a particular CompoundFeatureTest. /// 0 if the CompoundFeatureTest is not present. double weight(const CompoundFeatureTest& f) const; double unpenWeight(const CompoundFeatureTest& f) const; /// Compute the regularization penalty of this Ensemble. double penalty(double globalRegr) const; /// Compute the regularization penalty of this CFT. double penalty(const CompoundFeatureTest& theCFT, double globalRegr) const; // This is an additional regularization term in the risk function, // defined as k(Theta_f) * lambda_2 / (b(f) ^ lambda_3) where k() is // the Kroenecker delta function, and b(f) = sum_{i \in I_f} 1. // I.e. b(f) is the count of training examples that satisfy feature // f, which is constant for a given set of examples. Since we want // to penalize each CFT only once, we must check its current value // and its new value. If that value is going from zero to non-zero, // then we add this penalty term. If it's going from non-zero to // zero, then we subtract the penalty term. Otherwise, the penalty // is zero, cuz this feature's penalty has already been accounted // for. Note that we'll never need to compute this quantity when // b(f) < 1. double brittlePenalty(const CompoundFeatureTest& theCFT, const double theCount, double newConf) const; // the derivative (w.r.t. Theta_f) of the brittlePenalty is // lambda_2 / (b(f) ^ lambda_3) if the CFT is about to change its // confidence away from zero, and zero otherwise. We ignore the // possibility of it changing back to zero, since it happens rarely // and is expensive to predict at feature selection time. double brittleGradient(const CompoundFeatureTest& theCFT, const double theCount) const; void clear(); /// Output statistics string statistics() const; // I/O in compiled form ostream& print(ostream& out, const featVocab_t* featVocab = NULL) const; friend ostream& operator<< (ostream& out, const Ensemble& ens); friend istream& operator>>(istream& in, Ensemble& ens); const string& getKey() const { return m_key; } protected: // methods double getDPSNorm(const CompoundFeatureTest& cf) const; void setDPSNorm(const CompoundFeatureTest& cf, const Weights& wei); unsigned getNextCFTID() { return m_nextCFTID++; } static void getPredictParams(int argc, char **argv); /// helper method, which assumes that all relevant info is in parameter object template static void predict(); private: // methods void ctorHelper(); // reduce storage space and access time void optimizeStorage(); protected: // data /// tag of the example set currently being processed string m_key; //given that this is an index for CFTs, it should be set on a per-Ensemble basis, rather than be static unsigned int m_nextCFTID; fpVal_t m_treeCount; // NOTE: the key for a sparse_hash_map cannot be const if it is // ever erased, cuz erasing is done by replacing with the // deleted_key map_t m_CFTWeights; map_t m_unpenCFTWeights; // used in optimized example scoring typedef google::sparse_hash_map > cft2ui_map_t; cft2ui_map_t m_CFTIds; typedef google::sparse_hash_map, HashNS::hash > ft2ids_map_t; ft2ids_map_t m_featureTests; // bool marks whether or not to add the confidence of the CFT during computation typedef Vector>> confs_t; confs_t m_CFTIDWeights; bool m_useCFTCounts; // used to count how many times each CFT appears in the set-of-trees representation typedef google::sparse_hash_map > countMap_t; countMap_t m_CFTCounts; // these are used only during training, and only when using the DPS normalizer map_t m_DPSNorms; /// The Ensemble as it was created, a list of Tree%s. /// \note We store this iff /// parameter::useDenseEnsembleOptimizationsDuringConfidenceComputation() boost::optional > m_trees; }; } // namespace bdt #include "bdtl/Ensemble.ipp" #endif /* _BDTEnsembleH_ */ bdtl-1.0.3/src/bdtl/parameter.hpp0000644000175000017500000004026512171623344016451 0ustar melamedmelamed/*! * \file parameter.hpp * \class parameter * \brief Parameters global to the system. * * We can also read in the parameters at runtime. * See ./parameters.example for an example parameters file, which also contains * a description of the parameters file format. * * \warning All functions herein (except the constructor and helpers) * should start with "parameter::initialize()" to call the constructor. * * \note This class is based upon the singleton design pattern. * * \todo Have helper/wrapper methods for common local parameters * \todo Allow positional option for parameters-file */ #ifndef _BDTParameterH__ #define _BDTParameterH__ #include #include "Vector.hpp" #ifndef DOXYGEN using namespace std; #endif /* DOXYGEN */ #include #include #include #include namespace po = boost::program_options; #include "Vocabulary.hpp" #include "StringWithSpaces.hpp" #include "bdtl/globals.hpp" #include "bdtl/Feature.hpp" namespace bdt { enum parameterTy {STRINGTY, BOOLTY, UNSIGNEDTY, INTTY, DOUBLETY}; class LossFunctionBase; class parameter { public: // methods /// Read in the parameters. Parameters are read in from the /// command-line and, possibly, from a parameters-file specified on /// the command-line. /// \param fromCaller Parameters inherited from the calling function /// \param argc Command-line argument count. /// \param argv Command-line argument vector. /// \return The variables map into which we read the parameters. /// \todo Check that all (required) local parameters are set herein. /// \todo Make argv a const! /// \todo Does it work to look at vm.count("parametersFile") /// before calling notify(vm)? static const po::variables_map& read(int argc, char *argv[], const po::options_description& fromCaller = po::options_description()); /// Read in the parameters from a parameters-file. /// \todo Check that all (required) local parameters are set herein. /// \todo Does it work to look at vm.count("parametersFile") /// before calling notify(vm)? static void readConfigFile(string& parametersFile); /// Dump all parameters to a string. /// \todo Assert that the strings don't contain quotation marks. /// \todo Set proper floating point precision. static const string& str(string s=""); /// \todo All this feature stuff should really live in the Feature class; /// The feature class should be subclassed from Vocab static bool isBinary(const Feature& theFeat) { return sm_featType[theFeat.uid()] == B; } static bool isBinary(const size_t& featID) { return sm_featType[featID] == B; } static bool isInteger(const Feature& theFeat) { return sm_featType[theFeat.uid()] == I; } static bool isInteger(const size_t& featID) { return sm_featType[featID] == I; } static bool isFloat(const Feature& theFeat) { return sm_featType[theFeat.uid()] == F; } static bool isFloat(const size_t& featID) { return sm_featType[featID] == F; } static void setBinary(const Feature& theFeat, const bool val); static void setFloat(const Feature& theFeat, const bool val); static void setInteger(const Feature& theFeat, const bool val); static unsigned int getScalarFeatureSize() { parameter::initialize(); return sm_scalarFeatures.size(); } static unsigned int getBinaryFeatureSize() { parameter::initialize(); return sm_binaryFeatures.size(); } static const Vector& getScalarFeatureList() { parameter::initialize(); return sm_scalarFeatures; } static const Vector& getBinaryFeatureList() { parameter::initialize(); return sm_binaryFeatures; } static ExampleFormat& getExFormat() {parameter::initialize(); return ExFormat; } static bool putKeysInFilenames() { parameter::initialize(); return PutKeysInFilenames; } static bool useCompiledIO() { parameter::initialize(); return UseCompiledIO; } static const string& exampleFileName() { parameter::initialize(); return ExampleFileName; } static const BSP treeSeqFileName(const string key = "") { parameter::initialize(); return makeFileName(TreeSeqFileName, key); } static const BSP lastRegSeqFileName(const string key = "") { parameter::initialize(); return makeFileName(LastRegSeqFileName, key); } static const BSP compiledEnsembleName(const string key = "") { parameter::initialize(); return makeFileName(CompiledEnsembleName, key); } static const BSP doneTrainingFileName(const string key = "") { parameter::initialize(); return makeFileName(DoneTrainingFileName, key); } static const BSP makeFileName(const string& prefix, const string& key); static string& key2dirMapFileName() { parameter::initialize(); return Key2dirMapFileName; } static string& externalFeatVocabFileName() { parameter::initialize(); return ExternalFeatVocabFileName; } static featVocab_t*& externalFeatVocab() { parameter::initialize(); return ExternalFeatVocab; } static string& hypothesisDirectory() { parameter::initialize(); return HypothesisDirectory; } static bool dumpVolatileDiagnostics() { parameter::initialize(); return DumpVolatileDiagnostics; } static double maxTimePerKey() { parameter::initialize(); return MaxTimePerKey; } static const string& logLocation() { parameter::initialize(); return LogLocation; } static bool& resumeTraining() { parameter::initialize(); return ResumeTraining; } static bool shuffleExamplesInExBuilder() { parameter::initialize(); return ShuffleExamplesInExBuilder; } static label_t& labelType() { parameter::initialize(); return Label_type; } static const loss_t& lossFn() { parameter::initialize(); return Loss_type; } static regn_t& regnFn() { parameter::initialize(); return Regn_type; } static double minimumReductionInPenalizedTrainLossPerTree() { parameter::initialize(); return MinimumReductionInPenalizedTrainLossPerTree; } static double minimumReductionInPenalizedTrainLossPerSplit() { parameter::initialize(); return MinimumReductionInPenalizedTrainLossPerSplit; } static double minGainFactorForSplit() { parameter::initialize(); return MinGainFactorForSplit; } static unsigned maxAttemptsPerNode() { parameter::initialize(); return MaxAttemptsPerNode; } static bool useUnpenalizedGain() { parameter::initialize(); return UseUnpenalizedGain; } static double initialGlobalRegnFactor() { parameter::initialize(); return InitialGlobalRegnFactor; } static double minGlobalRegnFactor() { parameter::initialize(); return MinGlobalRegnFactor; } static double globalRegnFactor() { parameter::initialize(); return GlobalRegnFactor; } static double globalRegnFactorOverscale() { parameter::initialize(); return GlobalRegnFactorOverscale; } static double complexityCoeff() { parameter::initialize(); return ComplexityCoeff; } static double complexityExponent() { parameter::initialize(); return ComplexityExponent; } static const unsigned short& minExPerNode() { parameter::initialize(); return MinExPerNode; } static bool shareExampleWeightPerState() { parameter::initialize(); return ShareExampleWeightPerState; } static bool& subsamplingUsed() { parameter::initialize(); return SubsamplingUsed; } static unsigned maxSubsampleSize() { parameter::initialize(); return MaxSubsampleSize; } static double subsamplingRate() { parameter::initialize(); return SubsamplingRate; } static double learningRate() { parameter::initialize(); return LearningRate; } static unsigned heavyExampleSampleSize() { parameter::initialize(); return HeavyExampleSampleSize; } static bool normalizeWeightBiases() { parameter::initialize(); return NormalizeWeightBiases; } static bool normalizeFeatureCounts() { parameter::initialize(); return NormalizeFeatureCounts; } static bool useDPSNormalizer() { parameter::initialize(); return UseDPSNormalizer; } static bool useDenseEnsembleOptimizationsDuringConfidenceComputation() { parameter::initialize(); return UseDenseEnsembleOptimizationsDuringConfidenceComputation; } static bool useOptimizedConfidenceComputation() { parameter::initialize(); return parameter::UseOptimizedConfidenceComputation; } static double featValEpsilon() { parameter::initialize(); return FeatValEpsilon; } static unsigned outputFloatPrecision() { parameter::initialize(); return OutputFloatPrecision; } static unsigned maxIterations() { parameter::initialize(); return MaxIterations; } static unsigned getDbugLevel() { parameter::initialize(); return debugLevel; } static double confidenceEpsilon() { parameter::initialize(); return ConfidenceEpsilon; } static unsigned maxDepth() { parameter::initialize(); return MaxDepth; } static unsigned seed() { return Seed; } // don't initialize here to avoid infinite loop static bool bufferDebug() { parameter::initialize(); return BufferDebug; } static bool traceConfidencePaths() { parameter::initialize(); return TraceConfidencePaths; } static bool randomizedTieBreaking() { parameter::initialize(); return RandomizedTieBreaking; } static bool splitIfGreaterGainThanNewTreeRoot() { parameter::initialize(); return SplitNodeIfGreaterGainThanNewTreeRootSplit; } static bool penalizeRoots() { parameter::initialize(); return PenalizeRoots; } static bool addInternalNodesToEnsemble() { parameter::initialize(); return AddInternalNodesToEnsemble; } static bool useUnregularizedConfidencesForFeaturePrior() { parameter::initialize(); return UseUnregularizedConfidencesForFeaturePrior; } static bool useL1PenaltyForECR() { parameter::initialize(); return UseRegForECR; } static bool applyFeatPriorOnlyWhenNotZero() { parameter::initialize(); return false; } static bool preSortedFeatures() { parameter::initialize(); return PreSortedFeatures;} // Only a few variables are mutable after locking. static void setDbugLevel(unsigned u) { parameter::initialize(); debugLevel = u; parameter::sanityCheck(); } static void setGlobalRegnFactor(double d); static void setTraceConfidencePaths(bool b) { parameter::initialize(); TraceConfidencePaths = b; parameter::sanityCheck(); } static void setUseDenseEnsembleOptimizationsDuringConfidenceComputation(bool b) { parameter::initialize(); UseDenseEnsembleOptimizationsDuringConfidenceComputation = b; parameter::sanityCheck(); } static ofstream* k2dmOutFile() { parameter::initialize(); return sm_k2dmOutFile; } static ofstream* featFamilyOutFile() { parameter::initialize(); return sm_featFamilyOutFile; } static ostringstream* featFamilyOutBuffer() { parameter::initialize(); return sm_featFamilyOutBuffer; } static short getFeatFamily(unsigned int f) { assert(f < sm_featFamily.size()); return sm_featFamily[f]; } static bool useFeatFamilyPriors() { return (sm_featFamilyStats.size() > 0); } static const pair& getFeatFamilyStats(const unsigned short family, const short polarity, const double regn, const string& classKey = "def"); /// Sanity check all variable values that are relevant to training. /// \todo Add more sanity checks. static void trainingSanityCheck(); /// Sanity check all variable values. /// \todo Add more sanity checks. static void sanityCheck(); public: // data /// pointer to a (singleton) instance of LossFunction static LossFunctionBase* pLossFunc; private: // methods /// Private initialization. /// Add all parameters to our parameter list. static void initialize(); /// Add a parameter to the list of valid parameters. /// \param name The name of the new parameter. /// \param ptr A pointer to the storage of the parameter. /// \param ty The type of value the parameter takes. static void addHidden(string name, void* ptr, parameterTy ty); protected: // data /// The variables map into which we read parameters. static po::variables_map vm; private: // data /// list of known binary features static Vector sm_binaryFeatures; /// list of known scalar features static Vector sm_scalarFeatures; /// indicates, for each feature, whether its value is to be treated as integral or floating point /// N.B.: std::vector is badly broken -- in particular, size() doesn't work static Vector sm_featType; /// the feature family of each feature static Vector sm_featFamily; /// the sample statistics of each feature family (per regn factor) /// maps feature family --> floor(log(regn) / log(2)) --> //static hash_map, hash_map>> sm_featFamilyStats; typedef unordered_map > ffStats_t; typedef unordered_map, ffStats_t, boost::hash>> ffStatsMap_t; static ffStatsMap_t sm_featFamilyStats; /// the file to which to write the feature family values static ofstream* sm_featFamilyOutFile; static ostringstream* sm_featFamilyOutBuffer; static string Key2DirMapOut; static ofstream* sm_k2dmOutFile; /// Once a single parameter value has been retrieved, we lock /// all the parameters from any further changes. static bool didReadConfigFile; /// False until init() is called. static bool initialized; /// map from variable names to their storage and type typedef unordered_map > varTypeMap_t; static varTypeMap_t varTypeMap; /// Names of the parameters, in order of insertion. static Vector names; /// Options that will be allowed both on command line and /// in config file. static po::options_description localShared; /// Hidden options, will be allowed both on command line and /// in config file, but will not be shown to the user in 'help'. static po::options_description hidden; static ExampleFormat ExFormat; static string ef_str; // the string version of ExFormat static bool SplitNodeIfGreaterGainThanNewTreeRootSplit; static bool PenalizeRoots; static bool AddInternalNodesToEnsemble; static bool PutKeysInFilenames; static bool UseCompiledIO; static string ExampleFileName; static string TreeSeqFileName; static string LastRegSeqFileName; static string DoneTrainingFileName; static string CompiledEnsembleName; static string Key2dirMapFileName; static string ExternalFeatVocabFileName; static featVocab_t* ExternalFeatVocab; static string HypothesisDirectory; static bool DumpVolatileDiagnostics; static double MaxTimePerKey; static string LogLocation; static bool ResumeTraining; static bool ShuffleExamplesInExBuilder; static label_t Label_type; static string label_t_str; // the string version of label_t static loss_t Loss_type; static string loss_t_str; // the string version of loss_t static regn_t Regn_type; static string regn_t_str; // the string version of regn_t static double MinimumReductionInPenalizedTrainLossPerTree; static double MinimumReductionInPenalizedTrainLossPerSplit; static unsigned MaxAttemptsPerNode; static double MinGainFactorForSplit; static bool UseUnpenalizedGain; static double InitialGlobalRegnFactor; static double MinGlobalRegnFactor; static double GlobalRegnFactorOverscale; static double GlobalRegnFactor; static double ComplexityCoeff; static double ComplexityExponent; static unsigned short MinExPerNode; static bool ShareExampleWeightPerState; static bool SubsamplingUsed; static unsigned MaxSubsampleSize; static double SubsamplingRate; static double LearningRate; static unsigned HeavyExampleSampleSize; static bool NormalizeWeightBiases; static bool NormalizeFeatureCounts; static bool UseDPSNormalizer; static bool UseDenseEnsembleOptimizationsDuringConfidenceComputation; static bool UseOptimizedConfidenceComputation; static double FeatValEpsilon; static unsigned short OutputFloatPrecision; static unsigned MaxIterations; static unsigned debugLevel; static double ConfidenceEpsilon; static unsigned MaxDepth; static unsigned Seed; static bool BufferDebug; static bool TraceConfidencePaths; static bool RandomizedTieBreaking; static string FeatureFamilyFileName; static string FeatureFamilyStatsFileName; static string FeatureFamilyValueOutputFileName; static bool UseUnregularizedConfidencesForFeaturePrior; static bool UseRegForECR; static bool PreSortedFeatures; }; } // namespace bdt #endif /* _BDTParameterH_ */ bdtl-1.0.3/src/bdtl/Trainer.ipp0000644000175000017500000017014612151020420016060 0ustar melamedmelamed/*! * \file Trainer.ipp */ // #include "boost/bind.hpp" #include "Dbug.hpp" #include "Time.hpp" #include "Cursor.hpp" #include "bdtl/Trainer.hpp" #include "bdtl/Ensemble.hpp" #include "bdtl/ExComparator.hpp" #include "bdtl/DecTree.hpp" #include "bdtl/DecNode.hpp" #include "bdtl/LossFunction.hpp" #include "bdtl/Sampler.hpp" #include "bdtl/parameter.hpp" namespace bdt { using boost::numeric_cast; template void Trainer::prepareTrainingData() { static const size_t MIN_EX_SET_SIZE = 2000; static const ExBuilder theBuilder; // read the data ExampleFormat eF = parameter::getExFormat(); string exFileName = parameter::exampleFileName(); m_spTrainingExmpls->reserve(MIN_EX_SET_SIZE); theBuilder.buildExContainer(*m_spTrainingExmpls, exFileName, eF); unsigned int inputSize = m_spTrainingExmpls->size(); DEBUG(TREE_DBG_LEVEL + 1, "Read " << inputSize << " examples" << " from " << parameter::exampleFileName << endl); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL + 1, rUsage::resourceUsage() << endl); } // short-cut for when we don't need to keep the Ensemble around template void Trainer::trainLoopWrapper(string key) { m_pEnsemble = new Ensemble; m_pPartialEnsemble = new Ensemble; prepareTrainingData(); trainLoop(); } template IOHandler* Trainer::initializeTraining(const string& prevDir, unsigned& iteration) { if(m_spTrainingExmpls->empty()) THROW("No training examples!"); assert(m_pEnsemble->getTreeCount() == 0); THROWUNLESS(parameter::initialGlobalRegnFactor() >= parameter::minGlobalRegnFactor(), "initial reg'r < min reg'r!"); bool startFromScratch = true; const string tsfn(*parameter::treeSeqFileName(m_pEnsemble->getKey())); if(parameter::resumeTraining() && boost::filesystem::exists(tsfn)) { // !!! and its size is more than zero startFromScratch = false; } if (! prevDir.empty()) { // remember hypo dir string currHypoDir = parameter::hypothesisDirectory(); // reset hypo dir DEBUG(TREE_DBG_LEVEL, "Building on top of previous ensemble in " << prevDir << endl); parameter::hypothesisDirectory() = prevDir; m_pPrevIOHandler = new IOHandler(m_pEnsemble->getKey(), m_pCurrIOHandler); m_pPrevIOHandler->read(); // restore hypo dir parameter::hypothesisDirectory() = currHypoDir; } if(! startFromScratch) { m_pCurrIOHandler->read(); if(m_pCurrIOHandler->getTreeVec().empty()) { // previous files were empty startFromScratch = true; } else { double newPenalty = m_pCurrIOHandler->getRegrVec().back(); parameter::setGlobalRegnFactor(newPenalty); m_pCurrIOHandler->getEnsemble(m_pEnsemble, iteration); m_pEnsemble->updateConfidences(m_spTrainingExmpls); m_pEnsemble->updateDPSNorms(m_spTrainingExmpls); // advance sequence marker past the trees that have already // been added to the current ensemble if (m_pPrevIOHandler != NULL) { m_pPrevIOHandler->advanceSeqMarker(newPenalty); } // get ready to append to previous files m_pCurrIOHandler->open(ios_base::app); } // \bug We should check all stopping conditions here } // end reading in checkpoint files // test it again, in case it was reset above if(startFromScratch) { m_pCurrIOHandler->writeKey(); parameter::setGlobalRegnFactor(parameter::initialGlobalRegnFactor()); // erase any previous files m_pCurrIOHandler->open(ios_base::out); } if (m_pPrevIOHandler != NULL) { // pull in the relevant trees from previous ensemble m_pPartialEnsemble->clear(); m_pPrevIOHandler->getEnsemble(m_pPartialEnsemble, iteration, parameter::globalRegnFactor()); if(! m_pPartialEnsemble->empty()) { m_pPartialEnsemble->updateConfidences(m_spTrainingExmpls); m_pPartialEnsemble->updateDPSNorms(m_spTrainingExmpls); (*m_pEnsemble) += (*m_pPartialEnsemble); } } return m_pPrevIOHandler; } template void Trainer::trainLoop(const string prevDir) { DEBUG(TREE_DBG_LEVEL + 1, "Starting training loop for key " << m_pEnsemble->getKey() << endl); // See if we are done training const string dtfn(*parameter::doneTrainingFileName(m_pEnsemble->getKey())); if(parameter::resumeTraining() && boost::filesystem::exists(dtfn)) { DEBUG(TREE_DBG_LEVEL, "Done training sentinel exists in " << *(parameter::doneTrainingFileName(m_pEnsemble->getKey())) << endl); DEBUG(TREE_DBG_LEVEL, "Exiting..." << endl); return; } if (parameter::normalizeWeightBiases()) { m_spTrainingExmpls->normalize(); } if (parameter::preSortedFeatures()) { presortOnScalarFeatures(m_spTrainingExmpls); } DEBUG(TREE_DBG_LEVEL + 1, "Training on <" << ex_t::name() << ">..." << endl); DEBUG(TREE_DBG_LEVEL + 1, m_spTrainingExmpls->size() << " training examples" << endl); if (parameter::dumpVolatileDiagnostics()) // this differs on some 32bit and 64bit systems DEBUG(TREE_DBG_LEVEL + 1, "\tFYI, sizeof(" << ex_t::name() << ") = " << sizeof(ex_t) << " bytes" << endl); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL + 1, rUsage::resourceUsage() << endl); DEBUG(EX_DBG_LEVEL, Feature::dumpAll); // Number of iterations of training unsigned iteration = 0; // If the min reg'r is zero, then decreaseRegr() won't change it, // so we need a special way to keep track of it. bool doneWithZeroRegn = false; static const DecTree emptyTree; // for efficient use of m_pPrevIOHandler m_pPartialEnsemble->clear(); m_pCurrIOHandler = new IOHandler(m_pEnsemble->getKey()); m_pPrevIOHandler = initializeTraining(prevDir, iteration); BSP sampleEptrs = BSP(new ExamplePtrs); // initialize diagnostics double globalRegr = parameter::globalRegnFactor(); m_unpenalizedTrainLoss = parameter::pLossFunc->unpenalizedLoss(*(m_spTrainingExmpls.get())); m_ensemblePenalty = m_pEnsemble->penalty(globalRegr); m_penalizedTrainLoss = m_unpenalizedTrainLoss + m_ensemblePenalty; time_t startTime; time(&startTime); double elapsed = 0.0; double maxTime = parameter::maxTimePerKey(); // Each iteration of the while() loop corresponds to one iteration of boosting. while (iteration < parameter::maxIterations()) { if (doneWithZeroRegn) { m_pCurrIOHandler->writeDone("Can't train further even with no regularization."); break; } // check time limit, if using if( maxTime > 0 && elapsed > maxTime ) { DEBUG(TREE_DBG_LEVEL, "training time exceeded limit of " << maxTime << endl); // read and dump the rest of any previously induced ensemble parameter::setGlobalRegnFactor(0.0); if (m_pPrevIOHandler != NULL) { DEBUG(TREE_DBG_LEVEL, "Dumping the rest of previously induced ensemble..."); bool updateIt = false; m_pPrevIOHandler->getEnsemble(m_pPartialEnsemble, iteration, parameter::globalRegnFactor(), updateIt); m_pCurrIOHandler->flush(); DEBUG(TREE_DBG_LEVEL, " done."); m_pCurrIOHandler->writeDone("training timed out"); break; } } // check reg'r double globalRegr = parameter::globalRegnFactor(); if (globalRegr < parameter::minGlobalRegnFactor()) { ostringstream o; o << "l1 penalty factor=" << globalRegr << " is below minimum of " << parameter::minGlobalRegnFactor(); m_pCurrIOHandler->writeDone(o.str()); break; } // start timer time_t loopStartTime, loopEndTime; time(&loopStartTime); iteration++; DEBUG(TREE_DBG_LEVEL, endl << "Beginning outer loop iteration #" << iteration); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL, " at " << nsTime::currentTime()); DEBUG(TREE_DBG_LEVEL, "..." << endl); DEBUG(TREE_DBG_LEVEL + 1, "Current unpenalized train loss = " << m_unpenalizedTrainLoss); DEBUG(TREE_DBG_LEVEL + 1, endl); DEBUG(TREE_DBG_LEVEL + 1, "Ensemble penalty = " << m_ensemblePenalty << endl); DEBUG(TREE_DBG_LEVEL, "(globalRegnFactor currently " << globalRegr << ")" << endl); getSample(sampleEptrs, m_spTrainingExmpls); // ********* HERE IS THE MAIN EVENT *************** // Build a decision tree from the examples. BSP pTree = buildTree(m_spTrainingExmpls, sampleEptrs); // if we're evaluating each split instead of whole trees if (parameter::addInternalNodesToEnsemble()) { doneWithZeroRegn = handleNewDecLists(pTree, iteration); } else { doneWithZeroRegn = handleNewDecTree(pTree, iteration); } time(&loopEndTime); elapsed += difftime(loopEndTime, loopStartTime); DEBUG(TREE_DBG_LEVEL, "...finished iteration #" << iteration << endl << endl); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL + 1, rUsage::resourceUsage() << endl); } // while (iteration <= ... // print compiled version if desired if (parameter::useCompiledIO()) { oFile ce(*parameter::compiledEnsembleName(m_pEnsemble->getKey()), ios_base::out, parameter::outputFloatPrecision()); m_pEnsemble->print(ce); } // clean up delete m_pPrevIOHandler; } template bool Trainer::handleNewDecLists(const BSP pTree, unsigned& iteration) { bool doneWithZeroRegn = false; double globalRegr = parameter::globalRegnFactor(); // write out the latest update m_pCurrIOHandler->write(iteration, *pTree, globalRegr); if(parameter::featFamilyOutFile()) { (*parameter::featFamilyOutFile()) << parameter::featFamilyOutBuffer()->str(); DEBUG(TREE_DBG_LEVEL + 1, "a DUMPING CONFS TO FILE: " << parameter::featFamilyOutBuffer()->str() << endl); parameter::featFamilyOutFile()->flush(); parameter::featFamilyOutBuffer()->str(""); parameter::featFamilyOutBuffer()->clear(); } if (pTree->isJustRoot()) { if (MA::doubleEpsilonAdditiveEq(0., parameter::globalRegnFactor())) { if (parameter::labelType() == BOOLEAN) { doneWithZeroRegn = true; } else { // parameter::labelType() == SCALAR double rootConf = pTree->rootNode()->getConfidence(); DEBUG(TREE_DBG_LEVEL + 1, "Got degenerate tree with root conf = " << rootConf << endl); if (fabs(rootConf) <= parameter::confidenceEpsilon()) { doneWithZeroRegn = true; } } } else { // reg'n not at zero double sulg = pTree->sulg(); globalRegr = decreaseRegr(sulg); updateFromPrevEnsemble(iteration, globalRegr); } } return doneWithZeroRegn; } // evaluating whole tree at a time template bool Trainer::handleNewDecTree(const BSP pTree, unsigned& iteration) { bool doneWithZeroRegn = false; double globalRegr = parameter::globalRegnFactor(); static const DecTree emptyTree; // choose the leaf confidences, since we haven't done so already chooseLeafConfidences(*pTree); bool keepTree = true; // pTree->postulatedTrainingSetLoss() iterates over all its leaves, which together iterate over all the examples double unpenalizedTrainLossWithTree = pTree->postulatedTrainingSetLoss(); DEBUG(TREE_DBG_LEVEL + 1, "unpenalized train loss with tree = " << unpenalizedTrainLossWithTree << endl); // provisionally add the tree to the ensemble m_pEnsemble->add(*pTree); m_ensemblePenalty = m_pEnsemble->penalty(globalRegr); DEBUG(TREE_DBG_LEVEL + 1, "penalty with tree = " << m_ensemblePenalty << endl); double penalizedTrainLossWithTree = unpenalizedTrainLossWithTree + m_ensemblePenalty ; DEBUG(TREE_DBG_LEVEL + 1, "penalized train loss with tree = " << penalizedTrainLossWithTree << endl); DEBUG(TREE_DBG_LEVEL + 1, "old unpenalized train loss = " << m_unpenalizedTrainLoss << endl); double dtrainLoss; if (pTree->isJustRoot() && !parameter::penalizeRoots()) { // then we're dealing with the unregularized bias term dtrainLoss = m_unpenalizedTrainLoss - unpenalizedTrainLossWithTree; DEBUG(TREE_DBG_LEVEL + 1, "Tree reduces training bias loss by " << dtrainLoss); } else { dtrainLoss = m_penalizedTrainLoss - penalizedTrainLossWithTree; DEBUG(TREE_DBG_LEVEL + 1, "Tree reduces training set objective (penalized train loss) by " << dtrainLoss); DEBUG(TREE_DBG_LEVEL + 1, " from " << m_penalizedTrainLoss << " to " << penalizedTrainLossWithTree); } DEBUG(TREE_DBG_LEVEL + 1, endl); // does this tree reduce penalized loss by the required minimum? if (dtrainLoss < parameter::minimumReductionInPenalizedTrainLossPerTree()) { DEBUG(TREE_DBG_LEVEL + 1, "Not keeping tree because dtrainLoss " << dtrainLoss << " < minimumReductionInPenalizedTrainLossPerTree " << parameter::minimumReductionInPenalizedTrainLossPerTree() << endl); keepTree = false; } if (keepTree) { // yes, add the tree // Update all example confidences, since we we're adding whole trees, so we haven't done that yet. updateTreeExampleConfs(*pTree); double unpenLoss = parameter::pLossFunc->unpenalizedLoss(*(m_spTrainingExmpls.get())); DEBUG(TREE_DBG_LEVEL + 1, "Current penalized loss " << penalizedTrainLossWithTree << " = " << unpenLoss << " (unpenalized loss) + " << m_ensemblePenalty << " (penalty)" << endl); // this assertion fails at high precision // assert(MA::doubleEpsilonMultiplicativeEq(penalizedTrainLossWithTree, unpenLoss + penaltyWithTree, // parameter::confidenceEpsilon())); // write out the latest update m_pCurrIOHandler->write(iteration, *pTree, globalRegr); if(parameter::featFamilyOutFile()) { // This assertion will always fail here, so if we want to // use feature families, we need to figure out why they // don't work with // parameter::addInternalNodesToEnsemble(). assert(parameter::addInternalNodesToEnsemble()); (*parameter::featFamilyOutFile()) << parameter::featFamilyOutBuffer()->str(); DEBUG(TREE_DBG_LEVEL + 1, "b DUMPING CONFS TO FILE: " << parameter::featFamilyOutBuffer()->str() << endl); parameter::featFamilyOutFile()->flush(); parameter::featFamilyOutBuffer()->str(""); parameter::featFamilyOutBuffer()->clear(); } m_penalizedTrainLoss = penalizedTrainLossWithTree; m_unpenalizedTrainLoss = unpenalizedTrainLossWithTree; } else { // discard the tree // remove the tree from the ensemble m_pEnsemble->subtract(*pTree); // output blank Tree m_pCurrIOHandler->write(iteration, emptyTree, globalRegr); if (MA::doubleEpsilonAdditiveEq(0., parameter::globalRegnFactor())) { if (parameter::labelType() == BOOLEAN) { doneWithZeroRegn = true; } else { // parameter::labelType() == SCALAR double rootConf = pTree->rootNode()->getConfidence(); DEBUG(TREE_DBG_LEVEL + 1, "Got degenerate tree with root conf = " << rootConf << endl); if (fabs(rootConf) <= parameter::confidenceEpsilon()) { doneWithZeroRegn = true; } } } else { // reg'n not at zero // Decay the reg'n penalty. If the tree is degenerate, // then decay by whatever amount it would take to get a // non-degenerate tree, based on the // steepestUnpenalizedLossGradient of the current tree. // Negative sulg means don't use it cuz it's not a // degenerate tree. double sulg = -1.0; if (pTree->isJustRoot()) { sulg = pTree->sulg(); } globalRegr = decreaseRegr(sulg); // read in more of a pre-existing ensemble, if any updateFromPrevEnsemble(iteration, globalRegr); } } return doneWithZeroRegn; } template void Trainer::getSample(BSP& sampleEptrs, BSP& trainingEptrs) { const size_t trainingSize = trainingEptrs->size(); unsigned sampleSize = trainingSize * parameter::subsamplingRate(); const unsigned maxSubsampleSize = parameter::maxSubsampleSize(); if (maxSubsampleSize > 0 && sampleSize > maxSubsampleSize) { sampleSize = maxSubsampleSize; }; if (sampleSize < trainingSize) { parameter::subsamplingUsed() = true; sampleEptrs->deleteAll(); vector sampledIndex(trainingSize, 0); Sampler::getPrioritySample(sampleEptrs, trainingEptrs, sampleSize, sampledIndex); if (parameter::preSortedFeatures()){ const vector& scalarFeats = parameter::getScalarFeatureList(); //If there is sampling, sampleIndex gives which examples were selected to be in sampleEptrs; //get the ordering on the selected examples in m_iFeatOrder and m_fFeatOrder for(vector::const_iterator it = scalarFeats.begin(); it != scalarFeats.end(); ++it) { unsigned featID = *it; if (parameter::isFloat(featID)) { m_fFeatOrder[featID].clear(); vector& temp = m_fFeatOrderBkp[featID]; for(vector::const_iterator it=temp.begin(); it < temp.end(); ++it){ if (sampledIndex[it->second] == 0){ continue; } fuPair temp1; temp1.second = sampledIndex[it->second] - 1; temp1.first = it->first; m_fFeatOrder[featID].push_back(temp1); } } else { m_iFeatOrder[featID].clear(); vector& temp = m_iFeatOrderBkp[featID]; for(vector::const_iterator it=temp.begin(); it < temp.end(); ++it){ if (sampledIndex[it->second] == 0){ continue; } iuPair temp1; temp1.second = sampledIndex[it->second] - 1; temp1.first = it->first; m_iFeatOrder[featID].push_back(temp1); } } } } } else { // NOT sampleSize < trainingSize sampleEptrs = m_spTrainingExmpls; if (parameter::preSortedFeatures()){ // If there is no sampling, simply copy the feature ordering from the backup version. m_fFeatOrder = m_fFeatOrderBkp; m_iFeatOrder = m_iFeatOrderBkp; } } } template void Trainer::updateFromPrevEnsemble(unsigned& iteration, double regn) { m_pPartialEnsemble->clear(); if (m_pPrevIOHandler != NULL) { // pull in trees from previous ensemble; put them in a new // temporary ensemble, write them out, updating all the // checkpointing files as we go; finally, return the temporary // ensemble to the caller, so it can be added to the current // ensemble. m_pPrevIOHandler->getEnsemble(m_pPartialEnsemble, iteration, regn); } if (! m_pPartialEnsemble->empty()) { m_pPartialEnsemble->updateConfidences(m_spTrainingExmpls); m_pPartialEnsemble->updateDPSNorms(m_spTrainingExmpls); (*m_pEnsemble) += (*m_pPartialEnsemble); // iterationsBelowMrl = 0; } else { // iterationsBelowMrl++; // if (iterationsBelowMrl >= parameter::maxIterationsWithoutTreeAdd()) { // m_pCurrIOHandler->writeDone("Too many iterations below training MRL"); // \todo Output count // break; // } } } // Build a tree from scratch template BSP Trainer::buildTree(BSP m_spTrainingExmpls, BSP buildEptrs, bool isProvisional) { DEBUG(TREE_DBG_LEVEL, "starting Trainer::buildTree"); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL, " at " << nsTime::currentTime()); DEBUG(TREE_DBG_LEVEL, "..." << endl); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL + 1, rUsage::resourceUsage() << endl); BSP pTree(new DecTree); if (isProvisional) { DEBUG(TREE_DBG_LEVEL + 1, "* Creating new provisional root node. "); } else { DEBUG(TREE_DBG_LEVEL + 1, "* Creating new actual root node. "); } DEBUG(TREE_DBG_LEVEL + 1, "Tree potential: " << parameter::pLossFunc->currentPotential(*(m_spTrainingExmpls.get()), isProvisional, false) << endl); NodeID rootID = newNode(*pTree, NONODE, 0, CompoundFeatureTest(), buildEptrs, m_spTrainingExmpls, isProvisional); // newNode() automatically adds the node to the frontier pTree->setRootID(rootID); DEBUG(TREE_DBG_LEVEL + 1, "Root node (#" << pTree->getRootID() << ") created" << endl); assert(pTree->getRootID() != NONODE); assert(pTree->getRootID() == 0); if(parameter::preSortedFeatures()){ //Set the two indices in the root node. DecNode* pNode = static_cast(pTree->rootNode()); pNode->setFeatOrderBegin(0); pNode->setFeatOrderEnd(buildEptrs->size()); } // these cannot be static if this method calls itself typename DecTree::frontier_t frontierCopy; typename DecTree::frontier_t::iterator fIter; unsigned pass = 0; double provisionalTreeGain = 0.0; bool hadSplitOnPreviousPass = true; do { ++pass; DEBUG(TREE_DBG_LEVEL + 1, "Pass #" << pass << " started"); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL + 1, " at " << nsTime::currentTime()); DEBUG(TREE_DBG_LEVEL + 1, "." << endl); hadSplitOnPreviousPass = false; // We iterate through a copy of the frontier, rather than the // frontier itself, because deleting elements from the frontier // can invalidate the iterator. frontierCopy.clear(); frontierCopy.insert(pTree->getFrontier().begin(), pTree->getFrontier().end()); for (fIter = frontierCopy.begin(); fIter != frontierCopy.end(); ++fIter) { const NodeID& theID = *fIter; DecNode* pNode = static_cast(pTree->getNode(theID)); DEBUG(NODE_DBG_LEVEL + 1, "\t attempting to split node " << theID << " ..." << endl); // The best splitting feature tests for this node, weighted by // the goodness of their split. static bestSplitV_t bestSplitV(parameter::maxAttemptsPerNode()); bestSplitV.clear(); bool didSplit = false; double gain = getBestSplitV(*pNode, bestSplitV, isProvisional); if (isProvisional) { pTree->setRootGain(gain); // if we're just testing a provisional root split, then stop here return pTree; } // If the container is empty, then there are no feasible splits. if (bestSplitV.empty()) { DEBUG(NODE_DBG_LEVEL + 1, "\t bestSplitVector is empty " << endl); } else { size_t origSize = bestSplitV.size(); while (! didSplit && ! bestSplitV.empty()) { DEBUG(NODE_DBG_LEVEL + 1, "\t bestSplitVector size is " << origSize << endl); DEBUG(NODE_DBG_LEVEL + 1, "\t bestSplitVector is " << endl << bestSplitV); gain = bestSplitV.top().first.get(); const FeatureTest& splitFT(bestSplitV.top().second); DEBUG(NODE_DBG_LEVEL, "Gain = " << gain << " from splitting node " << *pNode << endl); if(parameter::splitIfGreaterGainThanNewTreeRoot() && (!pNode->isRoot())) { DEBUG(NODE_DBG_LEVEL, "comparing provisionalTreeGain " << provisionalTreeGain << " against gain from splitting " << gain << endl); if(provisionalTreeGain > gain) { DEBUG(NODE_DBG_LEVEL, "NOT SPLITTING" << endl); break; // no point testing the other possible splits, cuz they'll have less gain } } size_t attemptNum = 1 + origSize - bestSplitV.size(); DEBUG(NODE_DBG_LEVEL, "Starting split attempt #" << attemptNum << endl); // DecNode* pNode = static_cast(pTree->getNode(theID)); // typename ExamplePtrs::const_iterator exIter1 = pNode->getBuildExamples().begin(); // const typename ExamplePtrs::const_iterator sentinel1 = pNode->getBuildExamples().end(); // const vector& scalarFeats1 = parameter::getScalarFeatureList(); // for (; exIter1 != sentinel1; ++exIter1) { // const ex_t* exPtr(*exIter1); // for(vector::const_iterator it = scalarFeats1.begin(); it != scalarFeats1.end(); ++it) { // unsigned featID = *it; // Feature theFeat = Feature::get(featID); // IFUnion_t foo = exPtr->lookup(theFeat,parameter::isFloat(featID)); // cout << foo.f << "\t"; // } // cout << endl; // } didSplit = attemptSplit(*pTree, theID, splitFT); if (didSplit) { hadSplitOnPreviousPass = true; DEBUG(NODE_DBG_LEVEL + 1, "SPLIT SUCCEEDED on attempt # " << attemptNum << endl); break; } else { DEBUG(NODE_DBG_LEVEL + 1, "SPLIT FAILED on attempt # " << attemptNum << endl); } bestSplitV.pop(); } } if (! didSplit) { DEBUG(NODE_DBG_LEVEL, "Cannot find a split for node #" << theID << ". " << endl); } pTree->removeFromFrontier(theID); } if (pTree->isJustRoot()) { pTree->setSULGFromRoot(); } // if we did not expand the frontier, then there's no point inducing the provisional tree // note: hadSplitOnPreviousPass implies ! pTree->getFrontier().empty() if(hadSplitOnPreviousPass && parameter::splitIfGreaterGainThanNewTreeRoot() ) { DEBUG(TREE_DBG_LEVEL + 1, "====== Creating provisional tree ======" << endl); // Create new provisional tree given the nodes just split static const bool isProvisional = true; BSP pProvisionalTree = buildTree(m_spTrainingExmpls, buildEptrs, isProvisional); provisionalTreeGain = pProvisionalTree->getRootGain(); DEBUG(TREE_DBG_LEVEL + 1, "====== Got provisionalTreeGain " << provisionalTreeGain << " ======" << endl); } else { provisionalTreeGain = 0.0; } DEBUG(TREE_DBG_LEVEL + 1, "Pass #" << pass << " done"); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL + 1, " at " << nsTime::currentTime()); DEBUG(TREE_DBG_LEVEL + 1, "." << endl); } while(hadSplitOnPreviousPass); DEBUG(TREE_DBG_LEVEL, "...Trainer::buildTree finished"); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL, " at " << nsTime::currentTime()); DEBUG(TREE_DBG_LEVEL, endl); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL + 1, rUsage::resourceUsage() << endl); return pTree; } // Choose confidences for all leaf nodes. template void Trainer::chooseLeafConfidences(DecTree& theTree) { DecNode* dNode; DecTree::iterator dIter; for (dIter = theTree.begin(); dIter != theTree.end(); ++dIter) { if ((*dIter)->isLeaf()) { dNode = static_cast(*dIter); if (!dNode->hasConfidence()) { setConfidence(*dNode); } } } } // Perform a node split. // \param n The NodeID of node we are going to split. // \param ft The featuretest with which to split the node. // \todo Check that none of n's parents have used ft // \todo Dbug output l1, old conf, new confs, frontier size // \ret true if the split succeeded template bool Trainer::attemptSplit(DecTree& theTree, const NodeID n, const FeatureTest& ft) { bool succeeded = true; DEBUG(TREE_DBG_LEVEL + 1, "Starting Trainer::attemptSplit()" << endl); DecNode* pParentNode = static_cast(theTree.getNode(n)); assert(pParentNode->id() == n); assert(pParentNode->isLeaf()); pair, BSP> splitBuildEptrs = pParentNode->getBuildExamples().split(ft); pair, BSP> splitTrainEptrs = pParentNode->getTrainExamples().split(ft); const CompoundFeatureTest& parentCFT = pParentNode->getCFT(); const CompoundFeatureTest trueCFT(parentCFT + ft); const CompoundFeatureTest falseCFT(parentCFT + ft.negate()); const unsigned childDepth = pParentNode->depth() + 1; NodeID trueID = newNode(theTree, n, childDepth, trueCFT, splitBuildEptrs.first, splitTrainEptrs.first); NodeID falseID = newNode(theTree, n, childDepth, falseCFT, splitBuildEptrs.second, splitTrainEptrs.second); DecNode* trueChild = static_cast(theTree.getNode(trueID)); DecNode* falseChild = static_cast(theTree.getNode(falseID)); assert(childDepth == trueChild->depth()); assert(childDepth == falseChild->depth()); if(parameter::addInternalNodesToEnsemble()) { // check that the split reduced loss by at least the minimum amount const double minRed = parameter::minimumReductionInPenalizedTrainLossPerSplit(); assert(minRed > 0); const double parPL = pParentNode->getPenalizedLoss(); const double truePL = trueChild->getPenalizedLoss(); const double falsePL = falseChild->getPenalizedLoss(); const double red = parPL - truePL - falsePL; DEBUG(NODE_DBG_LEVEL, "loss reduction = " << red << "); parent = " << parPL << "; trueChild = " << truePL << "; falseChild = " << falsePL << endl); if (red < minRed) { revertSplit(trueID, falseID, trueCFT, falseCFT, theTree, pParentNode, trueChild, falseChild); succeeded = false; return succeeded; } if (parameter::featFamilyOutFile()) { if(!trueCFT.empty()) { assert(!falseCFT.empty()); const pair& trueFF = trueCFT.getLastFF(); const pair& falseFF = falseCFT.getLastFF(); if(parameter::useUnregularizedConfidencesForFeaturePrior()) { (*parameter::featFamilyOutBuffer()) << m_pEnsemble->getKey() << "\t" << trueFF.first << "\t" << trueFF.second << "\t" << "0" << "\t" << trueCFT << "\t" << trueChild->getUnpenDConf() << endl; (*parameter::featFamilyOutBuffer()) << m_pEnsemble->getKey() << "\t" << falseFF.first << "\t" << falseFF.second << "\t" << "0" << "\t" << falseCFT << "\t" << falseChild->getUnpenDConf() << endl; } else { (*parameter::featFamilyOutBuffer()) << m_pEnsemble->getKey() << "\t" << trueFF.first << "\t" << trueFF.second << "\t" << parameter::globalRegnFactor() << "\t" << trueCFT << "\t" << trueChild->getConfidence() << endl; (*parameter::featFamilyOutBuffer()) << m_pEnsemble->getKey() << "\t" << falseFF.first << "\t" << falseFF.second << "\t" << parameter::globalRegnFactor() << "\t" << falseCFT << "\t" << falseChild->getConfidence() << endl; } } } } if(parameter::preSortedFeatures() && succeeded){ updateFeatureSorts(ft, pParentNode, trueChild, falseChild); } pParentNode->setFTandChildren(ft, trueID, falseID); pParentNode->freeMemory(); DEBUG(EX_DBG_LEVEL, "Split node:" << endl << pParentNode->str("\t")); DEBUG(TREE_DBG_LEVEL + 1, "Finished Trainer::attemptSplit()" << endl); return succeeded; } template void Trainer::revertSplit(const NodeID& trueID, const NodeID& falseID, const CompoundFeatureTest& trueCFT, const CompoundFeatureTest& falseCFT, DecTree& theTree, DecNode* pParentNode, DecNode* trueChild, DecNode* falseChild) { DEBUG(TREE_DBG_LEVEL + 1, "DOH. This split does not reduce loss." << endl); static const bool inverse = true; updateSampleAndTrainConfs(*trueChild, inverse); updateSampleAndTrainConfs(*falseChild, inverse); // revert CFT confidences by adding the negative m_pEnsemble->add(trueCFT, - trueChild->getConfidence()); m_pEnsemble->add(falseCFT, - falseChild->getConfidence()); // and prune the children theTree.pruneNode(falseID); theTree.pruneNode(trueID); pParentNode->resetFTandChildren(); } template NodeID Trainer::newNode(DecTree& theTree, const NodeID parent, const unsigned depth, const CompoundFeatureTest& currCFT, const BSP buildEptrs, const BSP trainingEptrs, bool isProvisional) { DecNode* pNode(new DecNode(theTree.size(), parent, depth, currCFT, buildEptrs, trainingEptrs)); const NodeID theID = pNode->id(); theTree.insert(pNode); // frontier is automatically updated by the tree whenever a node is inserted // theTree.getFrontier().insert(theID); assert(theTree.size() - 1 == theID); // do this only if the node itself is not provisional if(parameter::splitIfGreaterGainThanNewTreeRoot() && ! isProvisional) { // set the provisional confidence of the new node and the examples in it setProvisionalConfidence(*pNode); updateProvisionalExampleConfs(*pNode); } if(parameter::addInternalNodesToEnsemble()) { // update the confidence of the new node and the examples in it // this will short-circuit to use the provisional confidence, when available // MAIN CONFIDENCE SETTING EVENT setConfidence(*pNode); static const bool inverse = false; updateSampleAndTrainConfs(*pNode, inverse); m_pEnsemble->add(currCFT, pNode->getConfidence()); } pNode->setPotential(parameter::pLossFunc->currentPotential(pNode->getBuildExamples(), isProvisional)); DEBUG(NODE_DBG_LEVEL, "newNode potential " << pNode->getPotential() << endl); if(parameter::addInternalNodesToEnsemble()) { // set m_penalizedLoss // At first we thought, "This quantity is used only to validate a // split in attemptSplit(), so we might as well do that over all // the training examples, not just the build examples." But then // we realized that (a) that's slower, (b) it takes the teeth out // of the complexity penalty and, most importantly, (c) on large // training sets, noise will validate far more splits than // warranted. const double uLoss = parameter::pLossFunc->unpenalizedLoss(*(buildEptrs.get())); const double ePenalty = m_pEnsemble->penalty(currCFT, parameter::globalRegnFactor()); double penLoss = uLoss + ePenalty; if (depth != 0) DEBUG(TREE_DBG_LEVEL + 1, "\t"); DEBUG(TREE_DBG_LEVEL + 1, "unpenalized loss = " << uLoss << " ; ensemble penalty = " << ePenalty); if (parameter::complexityCoeff() != 0) { const double bPenalty = m_pEnsemble->brittlePenalty(currCFT, buildEptrs->size(), pNode->getConfidence()); penLoss += bPenalty; DEBUG(TREE_DBG_LEVEL + 1, " ; brittleness penalty = " << bPenalty); } pNode->setPenalizedLoss(penLoss); DEBUG(TREE_DBG_LEVEL + 1, endl); } return theID; } template void Trainer::setConfidence(DecNode& theNode) { assert(!theNode.hasConfidence()); DEBUG(NODE_DBG_LEVEL + 1, "Choosing confidence for Node..." << theNode << endl); pot_t potential; double bestDConf; const CompoundFeatureTest& currCFT(theNode.getCFT()); unsigned short featureFamily = 0; short polarity = 0; const string& ensKey = m_pEnsemble->getKey(); if(parameter::useFeatFamilyPriors()) { featureFamily = currCFT.getLastFF().first; polarity = currCFT.getLastFF().second; } bool penalized = true; // \todo the following is a horrible hack involving programming by side-effects const regn_t usualRegnFunc = parameter::regnFn(); if (theNode.isRoot()) { // don't penalize root loss if this parameter is not set if (parameter::penalizeRoots()) { parameter::regnFn() = L2; } else { penalized = false; } } if(theNode.hasProvisionalConfidence()) { DEBUG(NODE_DBG_LEVEL, "Setting node confidence to provisional confidence " << theNode.getProvisionalConfidence() << endl); theNode.setConfidence(theNode.getProvisionalConfidence()); } else { DEBUG(NODE_DBG_LEVEL + 1, "Computing bestDConf on CFT " << currCFT << " that is " << (penalized ? "" : "NOT ")); if(parameter::useFeatFamilyPriors()) { DEBUG(NODE_DBG_LEVEL + 1, "penalized, with FF " << featureFamily << " and label " << polarity << endl); } else { DEBUG(NODE_DBG_LEVEL + 1, "penalized" << endl); } potential = parameter::pLossFunc->currentPotential(theNode.getTrainExamples(), false, penalized); bestDConf = parameter::pLossFunc->bestDConf(theNode.getTrainExamples(), potential, m_pEnsemble->weight(currCFT), penalized, featureFamily, polarity, ensKey); theNode.setConfidence(parameter::learningRate() * bestDConf); } if(parameter::useUnregularizedConfidencesForFeaturePrior()) { potential = parameter::pLossFunc->currentPotential(theNode.getTrainExamples(), false, false); bestDConf = parameter::pLossFunc->bestDConf(theNode.getTrainExamples(), potential, m_pEnsemble->weight(currCFT), false, featureFamily, polarity, ensKey); theNode.setUnpenDConf(parameter::learningRate() * bestDConf); } assert(theNode.hasConfidence()); DEBUG(NODE_DBG_LEVEL + 1, "Chose confidence " << theNode.getConfidence() << " for Node #" << theNode.id() << " (potential was " << theNode.getPotential() << ")" << endl); // reset regnFn to whatever it was, in any event parameter::regnFn() = usualRegnFunc; } template void Trainer::setProvisionalConfidence(DecNode& theNode) { const CompoundFeatureTest& currCFT(theNode.getCFT()); unsigned short featureFamily = 0; short polarity = 0; const string& ensKey = m_pEnsemble->getKey(); if(parameter::useFeatFamilyPriors()) { featureFamily = currCFT.getLastFF().first; polarity = currCFT.getLastFF().second; } DEBUG(NODE_DBG_LEVEL + 1, "Choosing provisonal confidence for Node..." << theNode << endl); bool penalized = true; if(theNode.isRoot() && ! parameter::penalizeRoots()) penalized = false; pot_t potential = parameter::pLossFunc->currentPotential(theNode.getTrainExamples(), true, penalized); double bestDConf = parameter::pLossFunc->bestDConf(theNode.getTrainExamples(), potential, m_pEnsemble->weight(currCFT), penalized, featureFamily, polarity, ensKey); theNode.setProvisionalConfidence(parameter::learningRate() * bestDConf); assert(theNode.hasProvisionalConfidence()); // DEBUG(NODE_DBG_LEVEL + 1, "The weight bias was: " << theNode.getTrainExamples().getWeightBias() << endl); DEBUG(NODE_DBG_LEVEL + 1, "Chose provisional confidence " << theNode.getProvisionalConfidence() << " for Node #" << theNode.id() << " (potential was " << theNode.getPotential() << ")" << endl); } template void Trainer::updateTreeExampleConfs(DecTree& theTree) { static const bool inverse = false; typename DecTree::nodeContainer_t& theNodes = theTree.getNodes(); typename DecTree::nodeContainer_t::iterator iter = theNodes.begin(); for (; iter != theNodes.end(); ++iter) { DecNode* pNode = static_cast(*iter); if (pNode->isLeaf()) updateTrainConfs(*pNode, inverse); } } template void Trainer::updateSampleAndTrainConfs(DecNode& theNode, const bool inverse) { double delta = updateTrainConfs(theNode, inverse); if (parameter::addInternalNodesToEnsemble() && parameter::subsamplingUsed()) { // then the build examples are copies of some train examples and // should be updated separately. As an optimization, we do this // only if addInternalNodesToEnsemble(), since otherwise the // conf updates happen only when the build examples are about to // be discarded. theNode.getBuildExamples().addConfidence(delta); } } template double Trainer::updateTrainConfs(DecNode& theNode, const bool inverse) { assert(theNode.hasConfidence()); if(parameter::useUnregularizedConfidencesForFeaturePrior()) { double delta = theNode.getUnpenDConf(); if (MA::doubleEpsilonAdditiveEq(delta, 0.0)) // nothing to do return delta; if(inverse) delta *= -1; theNode.getTrainExamples().addUnpenConfidence(delta); if(parameter::addInternalNodesToEnsemble() && parameter::subsamplingUsed()) // then the build examples are copies of some train examples and // should be updated separately. As an optimization, we do this // only if addInternalNodesToEnsemble(), since otherwise the // conf updates happen only when the build examples are about to // be discarded. theNode.getBuildExamples().addUnpenConfidence(delta); return delta; } // else double delta = theNode.getConfidence(); if (MA::doubleEpsilonAdditiveEq(delta, 0.0)) // nothing to do return delta; if (inverse) delta = - delta; double origTrainUnpenalizedLoss = 0.0; // initialize to duck compiler warning if (Dbug::isDebugAll(NODE_DBG_LEVEL + 1)) { origTrainUnpenalizedLoss = parameter::pLossFunc->unpenalizedLoss(theNode.getTrainExamples()); DEBUG(NODE_DBG_LEVEL + 1, "Updating confidences of examples at node # " << theNode.id() << endl << "Potential before update: " << parameter::pLossFunc->currentPotential(theNode.getTrainExamples()) << endl << "updateSampleAndTrainConfs adding confidence " << delta << endl); } theNode.getTrainExamples().addConfidence(delta);\ if (Dbug::isDebugAll(NODE_DBG_LEVEL + 1)) { double newTrainUnpenalizedLoss = parameter::pLossFunc->unpenalizedLoss(theNode.getTrainExamples()); double thePenalty = parameter::pLossFunc->penalize(theNode.getConfidence()); DEBUG(NODE_DBG_LEVEL + 1, theNode.str()); if (delta != 0) DEBUG(NODE_DBG_LEVEL + 1, "\tleaf feature = " << theNode.getCFT() << endl); DEBUG(NODE_DBG_LEVEL + 1, "\t" << theNode.getTrainExamples().size() << " training examples" << endl); DEBUG(NODE_DBG_LEVEL + 1, "\ttrain loss = " << newTrainUnpenalizedLoss + thePenalty << " = " << newTrainUnpenalizedLoss << " (unpenalized train loss) + " << thePenalty << " (penalty)"); if (delta != 0) DEBUG(NODE_DBG_LEVEL + 1, " (conf=0 train loss was " << origTrainUnpenalizedLoss << ")"); DEBUG(NODE_DBG_LEVEL + 1, endl); DEBUG(NODE_DBG_LEVEL + 1, "\tPotential after update: " << parameter::pLossFunc->currentPotential(theNode.getTrainExamples()) << endl); } return delta; } template void Trainer::getDecisionPointsForScalarFeatures(const DecNode& theNode) { ExamplePtrs* pSampleVec; if (parameter::preSortedFeatures()){ pSampleVec = &theNode.getBuildExamples(); } else { // pointer is NOT to a temporary, because getHeavyExamples() keeps its result static pSampleVec = &Sampler::getHeavyExamples(theNode.getBuildExamples()); } ExamplePtrs& sampleVec = *pSampleVec; short sampleSize = sampleVec.size(); DEBUG(EX_DBG_LEVEL, "sample of " << sampleSize << " in getDecisionPointsForScalarFeatures is " << endl << sampleVec); // precompute iterators that we'll use a lot typename ExamplePtrs::iterator sampleStart = sampleVec.begin(); typename ExamplePtrs::iterator sampleEnd = sampleVec.end(); // For each feature, first create a list of candidate decPoints. // Then, traverse the sample again, keeping track of which // yValue(s) appear on either side of each decPoint. If a // decPoint has a different yValue on one side than on the other, // then add it to the final decPointV. In particular, if a // decPoint has more than one yValue on either side of it, then // add it to the final decPointV. const vector& scalarFeats = parameter::getScalarFeatureList(); unsigned featID; static ExFeatComparator fComp(0, true); static ExFeatComparator iComp(0, false); Feature theFeat; typename ExamplePtrs::iterator exIter; ex_t* exPtr; fpVal_t prevFeatVal, currFeatVal; double nextDecPoint; typedef typename ex_t::m_predT pred_t; pred_t currYVal; bool isFloat, inFinalSegment; IFUnion_t featVal; unsigned short candCounter, candidateCount, exCounter; for(vector::const_iterator it = scalarFeats.begin(); it != scalarFeats.end(); ++it) { featID = *it; theFeat = Feature::get(featID); Vector& decPointV = m_decPointVV[featID]; isFloat = parameter::isFloat(featID); Vector candidate; candidate.reserve(sampleSize); if (!parameter::preSortedFeatures()) { if (isFloat) { fComp.setFeat(featID); sort(sampleStart, sampleEnd, fComp); } else { iComp.setFeat(featID); sort(sampleStart, sampleEnd, iComp); } featVal = (*sampleStart)->lookup(theFeat, isFloat); prevFeatVal = isFloat ? featVal.f : numeric_cast(featVal.i); for(exIter = sampleStart + 1; exIter != sampleEnd; ++exIter) { featVal = (*exIter)->lookup(theFeat, isFloat); currFeatVal = isFloat ? featVal.f : numeric_cast(featVal.i); if (! MA::doubleEpsilonAdditiveEq(currFeatVal, prevFeatVal)) { nextDecPoint = (currFeatVal + prevFeatVal) / 2.; candidate.push_back(nextDecPoint); prevFeatVal = currFeatVal; } } } else { if (isFloat) { //No need to sort when we have presorted features, just lookup and form the //decision points. //TODO: lookup from the training examples which will enable us to remove the feature value field. //exPtr = *(sampleStart + m_fFeatOrder[featID][theNode.getFeatOrderBegin()].second ); //featVal = exPtr->lookup(theFeat, isFloat); //prevFeatVal = featVal.f; prevFeatVal = m_fFeatOrder[featID][theNode.getFeatOrderBegin()].first; for (unsigned int itr = theNode.getFeatOrderBegin() + 1; itr < theNode.getFeatOrderEnd(); ++itr){ //exPtr = *(sampleStart + m_fFeatOrder[featID][itr].second); //featVal = exPtr->lookup(theFeat, isFloat); //currFeatVal = featVal.f; currFeatVal = m_fFeatOrder[featID][itr].first; if (!MA::doubleEpsilonAdditiveEq(currFeatVal, prevFeatVal)) { candidate.push_back( (prevFeatVal + currFeatVal)/2.0 ); prevFeatVal = currFeatVal; } } } else { //exPtr = *(sampleStart + m_iFeatOrder[featID][theNode.getFeatOrderBegin()].second ); //featVal = exPtr->lookup(theFeat, isFloat); //prevFeatVal = numeric_cast(featVal.i); prevFeatVal = m_iFeatOrder[featID][theNode.getFeatOrderBegin()].first; for (unsigned int itr = theNode.getFeatOrderBegin() + 1; itr < theNode.getFeatOrderEnd(); ++itr){ //exPtr = *(sampleStart + m_iFeatOrder[featID][itr].second); //featVal = exPtr->lookup(theFeat, isFloat); //currFeatVal = numeric_cast(featVal.i); currFeatVal = m_iFeatOrder[featID][itr].first; if (!MA::doubleEpsilonAdditiveEq(currFeatVal, prevFeatVal)) { candidate.push_back( (prevFeatVal + currFeatVal)/2.0 ); prevFeatVal = currFeatVal; } } } } candidateCount = candidate.size(); // Now, traverse the sample again, keeping track of which // yValues appear on either side of each decPoint, and whether // there is more than one on each side. // // vector is broken in some compilers/libraries, so we use vector instead. Vector diverseSegment(candidateCount + 1, false); // true if a segment has multiple yValues in it // Vector doesn't compile when pred_t = bool vector firstYVal; // the first (and perhaps the only) yValue in each segment firstYVal.reserve(candidateCount + 1); inFinalSegment = false; candCounter = 0; nextDecPoint = candidate[candCounter]; if (!parameter::preSortedFeatures()){ exPtr = static_cast(sampleVec[0]); } else { if (isFloat) { exPtr = static_cast(sampleVec[m_fFeatOrder[featID][theNode.getFeatOrderBegin()].second]); } else { exPtr = static_cast(sampleVec[m_iFeatOrder[featID][theNode.getFeatOrderBegin()].second]); } } firstYVal[0] = exPtr -> getYValue(); // first featVal cannot be more than first decPoint, so we start at 1 for(exCounter = 1; exCounter < sampleSize; ++exCounter) { if (!parameter::preSortedFeatures()){ exPtr = static_cast(sampleVec[exCounter]); } else { if (isFloat) { exPtr = static_cast(sampleVec[m_fFeatOrder[featID][theNode.getFeatOrderBegin() + exCounter].second]); } else { exPtr = static_cast(sampleVec[m_iFeatOrder[featID][theNode.getFeatOrderBegin() + exCounter].second]); } } currYVal = exPtr->getYValue(); featVal = exPtr->lookup(theFeat, isFloat); currFeatVal = isFloat ? featVal.f : numeric_cast(featVal.i); if (! inFinalSegment && currFeatVal > nextDecPoint) { ++candCounter; firstYVal[candCounter] = currYVal; if (candCounter < candidateCount) { nextDecPoint = candidate[candCounter]; } else { inFinalSegment = true; } } else { if (! diverseSegment[candCounter]) { if (! MA::doubleEpsilonAdditiveEq(currYVal, firstYVal[candCounter])) { diverseSegment[candCounter] = true; } } else { // diverseSegment[candCounter] == true if (inFinalSegment) break; // cuz then we have all the info we need } } } // now the firstYVal and diverseSegment vectors are populated // Finally, check to see which candidate decPoints should be kept. for(candCounter = 0; candCounter < candidateCount; ++candCounter) { // Accept a decPoint if it either has more than one yValue on // one or both sides of it, or if the yValue on one side is // different from the yValue on the other side. if (diverseSegment[candCounter] || diverseSegment[candCounter + 1] || ! MA::doubleEpsilonAdditiveEq(firstYVal[candCounter],firstYVal[candCounter + 1])) { decPointV.push_back(candidate[candCounter]); } } DEBUG(FEAT_DBG_LEVEL, "decPoints for feature " << Feature::get(featID) << " (featID " << featID << ") are " << decPointV << endl); bool OK = MA::sortedUnique::iterator>(decPointV.begin(), decPointV.end()); assert(OK || ((cerr << "sortedUnique failed on " << decPointV << endl) && 0)); } } template void Trainer::updateProvisionalExampleConfs(DecNode& theNode) { static const bool isProvisional = true; double delta = theNode.getProvisionalConfidence(); DEBUG(NODE_DBG_LEVEL + 1, "Updating provisional confidences of examples at node # " << theNode.id() << endl); DEBUG(NODE_DBG_LEVEL + 1, "(irrelevant) Provisional potential before update: " << parameter::pLossFunc->currentPotential(theNode.getTrainExamples(), isProvisional) << endl); DEBUG(NODE_DBG_LEVEL + 1, "Actual potential before update: " << parameter::pLossFunc->currentPotential(theNode.getTrainExamples()) << endl); DEBUG(NODE_DBG_LEVEL + 1, "updateProvisionalExampleConfs adding confidence " << delta << endl); theNode.getTrainExamples().addProvisionalConfidence(delta); DEBUG(NODE_DBG_LEVEL + 1, "Provisional potential after update: " << parameter::pLossFunc->currentPotential(theNode.getTrainExamples(), isProvisional) << endl); if (parameter::addInternalNodesToEnsemble() && parameter::subsamplingUsed()) { // then the build examples are copies of some train examples and // should be updated separately. As an optimization, we do this // only if addInternalNodesToEnsemble(), since otherwise the // conf updates happen only when the build examples are about to // be discarded. theNode.getBuildExamples().addProvisionalConfidence(delta); } } template void Trainer::accumulatePotential(const DecNode& theNode, bool provisional) { ExamplePtrs& buildExs = theNode.getBuildExamples(); const unsigned numExs = buildExs.size(); // if (Dbug::isDebugAll(EX_DBG_LEVEL)) { // ExWeightComparator wcomp; // sort(buildExs.begin(), buildExs.end(), wcomp); // } DEBUG(NODE_DBG_LEVEL, "About to accumulate build weight from " << numExs << " examples..." << endl); if (parameter::dumpVolatileDiagnostics()) DEBUG(NODE_DBG_LEVEL, rUsage::resourceUsage() << endl); for (unsigned egIdx = 0; egIdx < numExs; ++egIdx) { const ex_t* exPtr = static_cast(buildExs.at(egIdx)); const ex_t& theEx = *exPtr; const double theWeight = provisional ? parameter::pLossFunc->provisionalWeight(exPtr) : parameter::pLossFunc->weight(exPtr); const pot_t exPotential = theEx.potential(theWeight); DEBUG(EX_DBG_LEVEL, "Got weight " << theWeight << " for example #" << egIdx << endl); accumulatePotentialOnScalarFeatures(exPtr, exPotential); accumulatePotentialOnBinaryFeatures(exPtr, exPotential); } // now add in the unaccounted-for examples for each scalar feature static Vector::const_iterator decPointIter; const vector& scalarFeats = parameter::getScalarFeatureList(); for(vector::const_iterator it = scalarFeats.begin(); it != scalarFeats.end(); ++it) { const unsigned featID = *it; const pot_t missingPotential = theNode.getPotential() - m_includedPotential[featID]; const size_t missingCount = buildExs.size() - m_includedCount[featID]; const Vector& decPointVec = m_decPointVV[featID]; if (decPointVec.empty()) continue; decPointIter = lower_bound(decPointVec.begin(), decPointVec.end(), DEFAULT_FEATURE_VALUE); size_t theBin = std::distance(decPointVec.begin(), decPointIter); DEBUG(FEAT_DBG_LEVEL, "feat " << featID << " bin " << theBin << ": missing " << missingPotential << " = " << theNode.getPotential() << " - " << m_includedPotential[featID] << endl); m_potVV[featID][theBin] += missingPotential; m_countVV[featID][theBin] += missingCount; } if (Dbug::isDebugAll(EX_DBG_LEVEL)) { DEBUG(EX_DBG_LEVEL, "Current Potential Vector for Binary Features: " << endl); for(unsigned i = 0; i < m_binaryFPotentialVector.size(); i++) { DEBUG(EX_DBG_LEVEL, i << ": " << m_binaryFPotentialVector[i] << endl); } DEBUG(EX_DBG_LEVEL, "Current Potential Vector for Scalar Features: " << endl); for(VPH_t::const_iterator it = m_potVV.begin(); it != m_potVV.end(); ++it) { DEBUG(EX_DBG_LEVEL, it->first << ": " << it->second << endl); } } } // Compute steepestUnpenalizedLossGradient and bestSplitV, unless they already exist. // \note We might decay the regularization penalty after computing the root's steepestUnpenalizedLossGradient template double Trainer::getBestSplitV(DecNode& theNode, bestSplitV_t& bsV, bool provisional) { const ExamplePtrs& exVec = theNode.getBuildExamples(); if (exVec.empty()) { DEBUG(WARNING_DBG_LEVEL, "getBestSplitV should never deal with an empty node!" << endl); exit(13); } double ret = 0.; // If the number of nodes in the parent is less than twice the // minExPerNode, then any split will result in at least one child // having too few examples. This test also prevents trying to // split a node with only 1 example, since minExPerNode is at // least 1. const unsigned short minEPN = parameter::minExPerNode(); if (minEPN * 2u > exVec.size()) { DEBUG(NODE_DBG_LEVEL, "aborting due to min node size limit" << endl); return ret; } // check if we are dealing with a pure node typename ExamplePtrs::const_iterator iter = exVec.begin(); const typename ExamplePtrs::const_iterator sentinel = exVec.end(); ex_t* exPtr = static_cast(*iter); typename ex_t::m_predT firstYValue = exPtr->getYValue(); // at this point, we know the node has at least 2 examples, so one ++iter will not invalidate the iterator ++iter; typename ex_t::m_predT yValue = firstYValue; while (iter != sentinel && yValue == firstYValue) { exPtr = static_cast(*iter); yValue = exPtr->getYValue(); ++iter; } if (yValue != firstYValue) { DEBUG(NODE_DBG_LEVEL, "Node " << theNode.id() << " has examples with more than one yValue." << endl); } else { DEBUG(NODE_DBG_LEVEL, "Trainer::setBestSplitV aborting due to pure node " << theNode.id() << endl); return ret; } if (theNode.depth() == parameter::maxDepth()) { DEBUG(NODE_DBG_LEVEL, "aborting due to max depth limit" << endl); return ret; } const pot_t potential = theNode.getPotential(); DEBUG(NODE_DBG_LEVEL, "getBestSplitV: potential for node " << theNode.id() << " is " << potential << endl); // \todo Write feature count normalization code. assert(!parameter::normalizeFeatureCounts()); const CompoundFeatureTest& currCFT(theNode.getCFT()); // if we're using feature priors, we need to include the feature family of the CFT unsigned short featureFamily = 0; short polarity = 0; if(parameter::useFeatFamilyPriors()) { featureFamily = currCFT.getLastFF().first; polarity = currCFT.getLastFF().second; } // Initialize maxGain to the minimum value that it needs to reach // in order for a split to happen. Note that if we are adding // internal nodes to the ensemble, then // parameter::minGainFactorForSplit() has been set to 0, so it has // no effect. const double cftWeight = m_pEnsemble->weight(currCFT); Tiebreak origGain(parameter::pLossFunc->gain(potential, cftWeight, featureFamily, polarity), parameter::randomizedTieBreaking()); Tiebreak maxGain(0.0, parameter::randomizedTieBreaking()); maxGain = origGain() * parameter::minGainFactorForSplit(); // get ready to split // First build the decision point vectors for each scalar feature's number line // allocate vectors of the same size as the atomic feature vector size_t numScalarFeats = parameter::getScalarFeatureSize(); if (numScalarFeats > 0) { m_decPointVV.clear(); m_decPointVV.resize(numScalarFeats); getDecisionPointsForScalarFeatures(theNode); } // reset() must follow getDecisionPointsForScalarFeatures() cuz it // uses the vector sizes in m_decPointVV to initialize m_potVV // and m_countVV reset(); accumulatePotential(theNode, provisional); DEBUG(NODE_DBG_LEVEL, "...done accumulating build weight from " << theNode.getBuildExamples().size() << " examples into " << (m_potVV.size() + m_binaryFPotentialVector.size()) << " feature weights" << endl); if (parameter::dumpVolatileDiagnostics()) DEBUG(NODE_DBG_LEVEL, rUsage::resourceUsage() << endl); ///////////// MAIN EVENT /////////////// double maxUnpGain = 0.0; pot_t maxSatPotential; size_t bestCount; getBestSplitOnBinaryFeature(theNode, bsV, maxSatPotential, bestCount, maxGain, maxUnpGain); getBestSplitOnScalarFeature(theNode, bsV, maxSatPotential, bestCount, maxGain, maxUnpGain); if (theNode.isRoot()) { theNode.setSULG(maxUnpGain); } if (! bsV.empty()) { // not clear if the ret value is ever used in this case ret = maxGain(); if (Dbug::debug(NODE_DBG_LEVEL + 1)) { // weightVec is indexed by Feature.uid(), not FeatureTest.uid(), // because we should be able to figure out the optimal decision // boundary in advance for each Feature, even if it isn't binary const FeatureTest& ft = bsV.top().second; static CompoundFeatureTest trueCFT, falseCFT; DEBUG(NODE_DBG_LEVEL + 1, "Node #" << theNode.id() << ":" << endl); DEBUG(NODE_DBG_LEVEL + 1, "\tpre-accumulated potential = " << potential << endl); DEBUG(NODE_DBG_LEVEL + 1, "\tsplit feature: " << ft << endl); // for examples that are true on ft trueCFT = currCFT + ft; if(parameter::useFeatFamilyPriors()) { featureFamily = trueCFT.getLastFF().first; polarity = trueCFT.getLastFF().second; assert(polarity == 1); } double maxSatPotentialPenalizedGain = parameter::pLossFunc->gain(maxSatPotential, m_pEnsemble->weight(trueCFT), featureFamily, polarity); // for examples that are false on ft falseCFT = currCFT + ft.negate(); if(parameter::useFeatFamilyPriors()) { featureFamily = falseCFT.getLastFF().first; polarity = falseCFT.getLastFF().second; assert(polarity == -1); } const pot_t maxNSatPotential = potential - maxSatPotential; double maxNSatPotentialPenalizedGain = parameter::pLossFunc->gain(maxNSatPotential, m_pEnsemble->weight(falseCFT), featureFamily, polarity); DEBUG(NODE_DBG_LEVEL + 1, "\t+child penalized gain " << maxSatPotentialPenalizedGain << " (unclipped=" << parameter::pLossFunc->unpenGain(maxSatPotential) << ") with weights " << maxSatPotential << endl); DEBUG(NODE_DBG_LEVEL + 1, "\t-child penalized gain " << maxNSatPotentialPenalizedGain << " (unclipped=" << parameter::pLossFunc->unpenGain(maxNSatPotential) << ") with weights " << maxNSatPotential << endl); DEBUG(NODE_DBG_LEVEL + 1, "\t-child example count: " << bestCount << endl); DEBUG(NODE_DBG_LEVEL + 1, "\tbest gain = " << maxGain() << endl); DEBUG(NODE_DBG_LEVEL + 1, "\t+child gain " << parameter::pLossFunc->unpenGain(maxSatPotential) << " with weights " << maxSatPotential << endl); DEBUG(NODE_DBG_LEVEL + 1, "\t-child gain " << parameter::pLossFunc->unpenGain(maxNSatPotential) << " with weights " << maxNSatPotential << endl); DEBUG(NODE_DBG_LEVEL + 1, "\tvs. unsplit gain " << origGain() << " with potential " << potential << endl); } // if (Dbug:: ... } else { // no bestft found if (theNode.isRoot()) { DEBUG(TREE_DBG_LEVEL, "Root cannot be split!\n"); DEBUG(NODE_DBG_LEVEL + 1, "\troot gain = " << origGain() << endl); ret = origGain(); } DEBUG(NODE_DBG_LEVEL, "No split for Node #" << theNode.id() << " is possible with global reg'n factor of " << parameter::globalRegnFactor() << endl); } DEBUG(NODE_DBG_LEVEL + 1, "...bestSplitV()" << endl); return ret; } } // namespace bdt bdtl-1.0.3/src/bdtl/ExampleWithY.cpp0000644000175000017500000000161112151020420017014 0ustar melamedmelamed/*! * \file ExampleWithY.cpp */ #include "bdtl/ExampleWithY.hpp" #include "bdtl/globals.hpp" namespace bdt { // for classification template<> double ExampleWithY::potential(const double weight) const { // derivable from Turian thesis equation 4.34 return m_yValue ? -weight : weight; } // for regression // Q: Why do we use the Confidence for ExampleWithY but not for ExampleWithY? // A: In classification, Confidence is taken into account in the // margin. However, in regression, in order to maintain the usual // semantics of the sign of the margin, and to keep all weights // positive for priority sampling, we ignore the sign of the // margin. So here we must re-introduce it. template<> double ExampleWithY::potential(const double weight) const { return (m_yValue > Confidence) ? -weight : weight; } } // namespace bdt bdtl-1.0.3/src/bdtl/LossFunctionBase.ipp0000644000175000017500000002452512151020420017674 0ustar melamedmelamed#ifndef _BDTLossFunctionBaseI__ #define _BDTLossFunctionBaseI__ #include "bdtl/LossFunctionBase.hpp" #include "bdtl/parameter.hpp" #include "Vector.hpp" #include "bdtl/Weights.hpp" #include "bdtl/Example.hpp" #include "MoreMath.hpp" namespace bdt { // \pre hasConfidence() template double LossFunctionBase::margin(const ex_t* exPtr) const { assert(exPtr->hasConfidence()); return marginOf(exPtr->getConfidence(), exPtr->getYValue()); } template double LossFunctionBase::provisionalMargin(const ex_t* exPtr) const { return marginOf(exPtr->getProvisionalConfidence(), exPtr->getYValue()); } template double LossFunctionBase::unpenMargin(const ex_t* exPtr) const { return marginOf(exPtr->unpenConfidence(), exPtr->getYValue()); } template double LossFunctionBase::weight(const ex_t* exPtr) const { assert(exPtr->hasConfidence()); double ret = exPtr->getWeightBias() * weightOf(margin(exPtr)); if (! parameter::randomizedTieBreaking()) ret = MM::round(ret, parameter::confidenceEpsilon()); DEBUG(PEDANTIC_DBG_LEVEL, "got weight of " << ret << " for example " << *exPtr << endl); return ret; } template double LossFunctionBase::provisionalWeight(const ex_t* exPtr) const { double ret = exPtr->getWeightBias() * weightOf(provisionalMargin(exPtr)); DEBUG(PEDANTIC_DBG_LEVEL, "got prov-weight of " << ret << " for example " << *exPtr << endl); return ret; } template double LossFunctionBase::unpenWeight(const ex_t* exPtr) const { double ret = exPtr->getWeightBias() * weightOf(unpenMargin(exPtr)); DEBUG(PEDANTIC_DBG_LEVEL, "got unpen-weight of " << ret << " for example " << *exPtr << endl); return ret; } // Set the weight of this Example. // If HasConfidence, then set WeightBias such that weight() == weight. // Otherwise, set WeightBias to weight. template void LossFunctionBase::scaleWeightBias(ex_t* exPtr, const double& newWeight) const { if (exPtr->hasConfidence()) { double curWgt = weight(exPtr); exPtr->setWeightBias(exPtr->getWeightBias() * newWeight / curWgt); } else { exPtr->setWeightBias(newWeight); // (i.e. assume curWgt = 1.0) } } template double LossFunctionBase::currentPotential(const ExamplePtrs& exVec, bool provisional, bool penalized) const { // The provisional confidence of each example is penalized or not // depending on whether parameter::penalizeRoots() is true. // Therefore, the 'penalized' parameter to this method has no effect // when the 'provisional' parameter is true. double wgt, increment, ret; const ex_t* exPtr; typename ExamplePtrs::const_iterator e = exVec.begin(); for (; e != exVec.end(); ++e) { exPtr = static_cast(*e); if(provisional) { wgt = provisionalWeight(exPtr); } else if(penalized) { wgt = weight(exPtr); } else { // not penalized wgt = unpenWeight(exPtr); } increment = exPtr->potential(wgt); ret += increment; } if (! parameter::randomizedTieBreaking()) ret = MM::round(ret, parameter::confidenceEpsilon()); return ret; } // Find the loss of the Example, possibly given a confidence change dconf // \pre exPtr->hasConfidence() template double LossFunctionBase::loss(const ex_t* exPtr, double dconf) const { assert(exPtr->hasConfidence()); double marg = marginOf(exPtr->getConfidence() + dconf, exPtr->getYValue()); double theLoss = lossOf(marg); DEBUG(EX_DBG_LEVEL, "Ex " << exPtr->getUniqueID() << ": "); DEBUG(EX_DBG_LEVEL, "marginOf(" << exPtr->getConfidence() << " + " << dconf << ", " << exPtr->getYValue() << ") = " << marg << "; "); DEBUG(EX_DBG_LEVEL, "lossOf(margin) = " << theLoss << endl); return theLoss * exPtr->getWeightBias(); } template double LossFunctionBase::unpenalizedLoss(const ExamplePtrs& exVec, double dconf) const { double ret = 0; double acc; typename ExamplePtrs::const_iterator e = exVec.begin(); const typename ExamplePtrs::const_iterator sentinel = exVec.end(); for (; e != sentinel; ++e) { const ex_t* exPtr(static_cast(*e)); acc = loss(exPtr, dconf); DEBUG(EX_DBG_LEVEL, "LossFunctionBase::unpenalizedLoss for eg " << exPtr->getUniqueID() << " is " << acc << endl); ret += acc; } DEBUG(NODE_DBG_LEVEL, "LossFunctionBase::unpenalizedLoss total is " << ret << endl); return ret; } // Compute the penalized loss over a set of examples for the given change in confidence template double LossFunctionBase::postulatedLoss(const ExamplePtrs& exs, double currentConf, double dconf, bool penalized, unsigned short featFamily, short polarity, string classKey) const { double newConf = currentConf + dconf; double unpenalizedLoss = 0; for (typename ExamplePtrs::const_iterator e = exs.begin(); e != exs.end(); ++e) { ex_t* exPtr = static_cast(*e); unpenalizedLoss += loss(exPtr, dconf); } double totalLoss; if (penalized) { double penalty = penalize(newConf); totalLoss = unpenalizedLoss + penalty; if(parameter::useFeatFamilyPriors() && polarity != 0 && (totalLoss >= parameter::confidenceEpsilon() || !parameter::applyFeatPriorOnlyWhenNotZero())) { DEBUG(EX_DBG_LEVEL+1, "penalty = " << penalty << "; totalLoss = " << totalLoss << endl); double fp = featprior(newConf, classKey, featFamily, polarity); totalLoss += fp; DEBUG(EX_DBG_LEVEL+1, "total loss is " << totalLoss << " after adding feature prior: " << fp << " on newConf " << newConf << endl); } // DEBUG(EX_DBG_LEVEL) << "postulatedLoss(" << dconf << ") = " << totalLoss << " = " // << unpenalizedLoss << " (unpenalized loss) + " << penalty << " (penalty)" << endl; } else { totalLoss = unpenalizedLoss; // DEBUG(EX_DBG_LEVEL, "unpenalizedLoss(" << dconf << ") = " << totalLoss << endl); } return totalLoss; } // Choose the confidence update that minimizes penalized loss over this set of examples. // We do this with a line search. // \param currentConfidence The current parameter value to be adjusted (and delta'd with) template double LossFunctionBase::bestDConf(const ExamplePtrs& exs, pot_t potential, double currentConfidence, bool penalized, unsigned short featFamily, short polarity, const string& classKey) const { // If the gain is 0, then we won't be able to decrease it, so the best dconfidence = 0. // \todo Sanity-check that we get confidence 0 if we do the work. if (penalized) { if (gain(potential, currentConfidence, featFamily, polarity) < parameter::confidenceEpsilon()) return 0.; } else { if (unpenGain(potential) < parameter::confidenceEpsilon()) return 0.; } double ret = lineSearch(exs, potential, currentConfidence, penalized, featFamily, polarity, classKey); return ret; } template double LossFunctionBase::lineSearch(const ExamplePtrs& exs, pot_t potential, double currentConfidence, bool penalized, unsigned short featFamily, short polarity, const string& classKey) const { static const double INITIAL_STEP_SIZE = 1.; // We use a map rather than a hash_map because we want debug output in sorted order // \todo Switch to hash_map for speed in production. static map searchCache; searchCache.clear(); map::const_iterator cacheIter; double dconf = 0.; // delta confidence double bestDconf = 0.; double newLoss, dloss; // As a small optimization, initialize step direction to match // likely sign of confidence. This also removes the systematic // bias for +ive and -ive potentials to have different confidence // magnitudes. double step = INITIAL_STEP_SIZE; if (potential > 0.) step *= -1.; double currLoss = postulatedLoss(exs, currentConfidence, dconf, penalized, featFamily, polarity, classKey); searchCache[dconf] = currLoss; DEBUG(EX_DBG_LEVEL, "Initial loss: " << currLoss << " on confidence (" << currentConfidence << ") and dconf (" << dconf << ")" << endl); double bestLoss = currLoss; do { dconf += step; cacheIter = searchCache.find(dconf); if (cacheIter != searchCache.end()) { // DEBUG(PEDANTIC_DBG_LEVEL, "CACHE HIT" << endl); newLoss = cacheIter->second; } else { newLoss = postulatedLoss(exs, currentConfidence, dconf, penalized, featFamily, polarity, classKey); searchCache[dconf] = newLoss; } dloss = newLoss - currLoss; DEBUG(EX_DBG_LEVEL, "New loss: " << newLoss << " on confidence (" << currentConfidence << ") and dconf (" << dconf << ") dloss = " << dloss << endl); if (newLoss < currLoss) { // making progress if (bestLoss > newLoss) { bestLoss = newLoss; bestDconf = dconf; } } else { // halve the step size and switch direction step *= -0.5; } currLoss = newLoss; } while(fabs(step) > parameter::confidenceEpsilon()); THROWIF(searchCache.empty(), "Loss cache is empty"); if (Dbug::isDebugAll(NODE_DBG_LEVEL)) { double penalty = 0.; if (penalized) penalty = penalize(currentConfidence + bestDconf); double unpenalizedLoss = bestLoss - penalty; THROWUNLESS(unpenalizedLoss >= 0., "unpenalized loss = " << unpenalizedLoss); DEBUG(NODE_DBG_LEVEL, "Best dconf " << bestDconf << " (for confidence=" << currentConfidence << ") has penalized loss " << bestLoss << " = " << unpenalizedLoss << " (unpenalized loss) + " << penalty << " (penalty)" << endl); DEBUG(NODE_DBG_LEVEL, searchCache.size() << " cache misses in line search" << endl); if (MA::doubleEpsilonAdditiveEq(bestDconf, 0.0)) { ostringstream o; o << "bestDconf = 0!" << endl; o << "penalized gain = " << gain(potential, currentConfidence) << " at current confidence " << currentConfidence << ", regn = " << parameter::globalRegnFactor() << endl; if (Dbug::isDebugAll(EX_DBG_LEVEL)) { for (map::const_iterator z = searchCache.begin(); z != searchCache.end(); z++) o << z->first << " dconfidence has loss " << z->second << endl; o << endl; } DEBUG(NODE_DBG_LEVEL, o.str() << endl); } } return bestDconf; } } // namespace bdt #endif bdtl-1.0.3/src/bdtl/Ensemble.ipp0000644000175000017500000001711612151020420016203 0ustar melamedmelamed/*! * \file Ensemble.ipp */ #include "Dbug.hpp" #include "Time.hpp" #include "bdtl/LossFunction.hpp" #include "bdtl/DecTree.hpp" #include "bdtl/ExBuilder.hpp" #include "bdtl/parameter.hpp" namespace bdt { template void Ensemble::predict() { Ensemble Ens; DEBUG(TREE_DBG_LEVEL, "Reading ensemble..."); if (parameter::useCompiledIO()) Ens.loadCompiled(""); else Ens.regularizedLoad("", parameter::minGlobalRegnFactor()); DEBUG(TREE_DBG_LEVEL, " done" << endl); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL + 1, rUsage::resourceUsage() << endl); DEBUG(NODE_DBG_LEVEL, "Dumping loaded Ensemble..." << endl << Ens << endl); // ExampleFormat eF = parameter::getExFormat(); istream* inPtr; const string& exFile = parameter::exampleFileName(); // if exampleFileName not specified then read examples from STDIN if (exFile == "") { inPtr = &cin; } else { inPtr = new iFile(exFile); } string line; size_t counter = 0; while(getline(*inPtr, line)) { double conf = Ens.predictOne(line, ++counter); cout << conf << endl; } if (exFile != "") { delete inPtr; } DEBUG(TREE_DBG_LEVEL, "\n\nDONE TESTING." << endl); } template double Ensemble::predictOne(const string& exString, size_t exNum) const { static const ExampleFormat eF = parameter::getExFormat(); static const ExBuilder theBuilder; static string key = ""; const ex_t* pEx = static_cast(theBuilder.constructorHelp2(key, exString, eF, exNum)); if (! pEx) { THROW("The following example could not be parsed: " << exString) } double conf = confidenceCompute(*pEx); return conf; } template double Ensemble::confidenceCompute(const ex_t& e) const { DEBUG(EX_DBG_LEVEL, "Scoring " << e); double totalConf = 0; if (parameter::useDenseEnsembleOptimizationsDuringConfidenceComputation()) { const vector& theTrees = m_trees.get(); DEBUG(EX_DBG_LEVEL, "Ensemble::confidenceCompute: Using Dense Ensemble Optimization" << std::endl); for (vector::const_iterator t = theTrees.begin(); t != theTrees.end(); t++) { DEBUG(EX_DBG_LEVEL, "Ensemble::confidenceCompute on tree" << std::endl); totalConf += t->confidence(e); } } else if(parameter::useOptimizedConfidenceComputation()) { // Iterate over all the FTs and see if the example satisfies the FT. // Set a bit indicating whether we need to add the confidence. // Note that the bits are set (to true) by default when the Pair is added to m_CFTIDWeights. for(ft2ids_map_t::const_iterator ftestIt = m_featureTests.begin(); ftestIt != m_featureTests.end(); ++ftestIt) { if(!e.satisfies(ftestIt->first)) { const Vector& cfts = ftestIt->second; for(Vector::const_iterator cftIdIt = cfts.begin(); cftIdIt != cfts.end(); ++cftIdIt) { m_CFTIDWeights[*cftIdIt]->first &= false; } } } // Iterate over the CFTs, and add the confs of those whose use-it bit is set. unsigned satCount = 0; // how many CFTs were satisfied for(unsigned cftid = 0; cftid < m_CFTIDWeights.size(); ++cftid) { if(m_CFTIDWeights[cftid]->first) { const double currConf = m_CFTIDWeights[cftid]->second; if (! MA::doubleEpsilonAdditiveEq(currConf, 0.0)) { totalConf += currConf; ++satCount; DEBUG(FEAT_DBG_LEVEL, "fired CFT with ID " << cftid << endl); } } else { // set the bit back to the default (true) to prepare for next test m_CFTIDWeights[cftid]->first = true; } } DEBUG(EX_DBG_LEVEL, "; used " << satCount << " CFTs out of " << m_CFTIDWeights.size()); } else { unsigned satCount = 0; // how many CFTs were satisfied map_t::const_iterator f = m_CFTWeights.begin(); const map_t::const_iterator sentinel = m_CFTWeights.end(); for(; f != sentinel; ++f) { const CompoundFeatureTest& theCFT = f->first; DEBUG(EX_DBG_LEVEL + 1, "Ensemble::confidenceCompute testing compound feature " << theCFT << endl); if (theCFT(e)) { const double currConf = f->second; if (parameter::externalFeatVocabFileName() == "") { DEBUG(EX_DBG_LEVEL + 1, "SAT: " << theCFT << " " << currConf); DEBUG(EX_DBG_LEVEL + 1, endl); } else { DEBUG(EX_DBG_LEVEL + 1, "SAT: "); if (debug(EX_DBG_LEVEL + 1)) { theCFT.print(getDOut(), parameter::externalFeatVocab()); } DEBUG(EX_DBG_LEVEL + 1, " " << currConf << endl); //DEBUG(FEAT_DBG_LEVEL, endl << theCFT << endl); } if (! MA::doubleEpsilonAdditiveEq(currConf, 0.0)) { totalConf += currConf; ++satCount; DEBUG(FEAT_DBG_LEVEL, "fired CFT " << theCFT << endl); } } } // note here we count m_CFTWeights not m_CFTIDWeights. DEBUG(EX_DBG_LEVEL, "; used " << satCount << " CFTs out of " << m_CFTWeights.size()); } DEBUG(EX_DBG_LEVEL, " to get conf = " << totalConf << endl); return totalConf; } template void Ensemble::updateConfidences(BSP exmpls) const { DEBUG(TREE_DBG_LEVEL + 1, "Updating confidences for " << exmpls->size() << " examples..." << endl); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL + 1, rUsage::resourceUsage() << endl); unsigned cnt = 0; typename ExamplePtrs::iterator e = exmpls->begin(); for(; e != exmpls->end(); e++) { ex_t* exPtr = static_cast(*e); ex_t& theEx = *exPtr; double conf = confidenceCompute(theEx); exPtr->setConfidence(exPtr->getConfidence() + conf); exPtr->setProvisionalConfidence(exPtr->getProvisionalConfidence() + conf); cnt++; if (cnt % 100000 == 0) { DEBUG(TREE_DBG_LEVEL + 1, "\t" << cnt << " / " << exmpls->size() << " (" << cnt*100./exmpls->size() << "%) done..." << endl); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL + 1, rUsage::resourceUsage() << endl); } else if (cnt % 10000 == 0) { DEBUG(NODE_DBG_LEVEL, "\t" << cnt << " / " << exmpls->size() << " (" << cnt*100./exmpls->size() << "%) done..." << endl); if (parameter::dumpVolatileDiagnostics()) DEBUG(NODE_DBG_LEVEL, rUsage::resourceUsage() << endl); } else if (cnt % 1000 == 0) { DEBUG(NODE_DBG_LEVEL + 1, "\t" << cnt << " / " << exmpls->size() << " (" << cnt*100./exmpls->size() << "%) done..." << endl); if (parameter::dumpVolatileDiagnostics()) DEBUG(NODE_DBG_LEVEL + 1, rUsage::resourceUsage() << endl); } } DEBUG(TREE_DBG_LEVEL + 1, "...done updating confidences for " << exmpls->size() << " examples" << endl); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL + 1, rUsage::resourceUsage() << endl); } template void Ensemble::updateDPSNorms(BSP exmpls) { if (! parameter::useDPSNormalizer()) return; assert(!parameter::useOptimizedConfidenceComputation()); for (map_t::const_iterator fiter = getCFTWeights().begin(); fiter != getCFTWeights().end(); ++fiter) { const CompoundFeatureTest& theCF = fiter->first; // if there's no value for this feature in the DPS norm map yet, // then compute it map_t::const_iterator DPSiter = getDPSNorms().find(theCF); if (DPSiter == getDPSNorms().end()) { Weights wei; typename ExamplePtrs::const_iterator e = exmpls->begin(); for (; e != exmpls->end(); e++) { ex_t* exPtr = static_cast(*e); wei.add(parameter::pLossFunc->weight(exPtr), exPtr->getYValue()); } const double theDPSNorm = parameter::pLossFunc->DPSNormalizer(wei); getDPSNorms()[theCF] = theDPSNorm; } } } } // namespace bdt bdtl-1.0.3/src/bdtl/LossFunction.cpp0000644000175000017500000000143012151020420017061 0ustar melamedmelamed/*! * \file LossFunction.cpp */ #include #include "bdtl/LossFunction.hpp" namespace bdt { template<> double LossFunction::lossOf(double margin) const { return log(1+exp(-margin)); } template<> double LossFunction::weightOf(double margin) const { return 1./(1. + exp(margin)); } template<> double LossFunction::lossOf(double margin) const { return exp(-margin); } template<> double LossFunction::weightOf(double margin) const { return exp(-margin); } template<> double LossFunction::lossOf(double margin) const { return margin * margin / 2.; } template<> double LossFunction::weightOf(double margin) const { return -margin; } } // namespace bdt bdtl-1.0.3/src/bdtl/ExComparator.ipp0000644000175000017500000000332012151020420017045 0ustar melamedmelamed#include "bdtl/ExComparator.hpp" #include "bdtl/parameter.hpp" #include "bdtl/LossFunction.hpp" namespace bdt { template bool ExWeightComparator::operator()(const Example* abstractExPtr1, const Example* abstractExPtr2) { bool ret; const double eps = parameter::confidenceEpsilon(); const ex_t* exPtr1 = static_cast(abstractExPtr1); const ex_t* exPtr2 = static_cast(abstractExPtr2); const double w1 = MM::round(parameter::pLossFunc->weight(exPtr1), eps); const double w2 = MM::round(parameter::pLossFunc->weight(exPtr2), eps); DEBUG(PEDANTIC_DBG_LEVEL, "ExWeightComparator comparing examples " << *exPtr1 << " ( wgt = " << w1 << " ) and " << *exPtr2 << " ( wgt = " << w2 << " )"); // It doesn't work to use doubleEpsilonAdditiveEq(w1,w2,eps) here, // because if the comparator is used for sorting, then this // approach violates the transitivity of equality. if (w1 == w2) ret = exPtr1->getRandomID() > exPtr2->getRandomID(); else ret = w1 > w2; assert(exPtr1->getUniqueID() != exPtr2->getUniqueID() && exPtr1->getRandomID() != exPtr2->getRandomID()); // "Weird: exPtr1->getRandomID() == exPtr2->getRandomID()"); return ret; } template bool ExLabelWeightComparator::operator()(const ex_t* exPtr1, const ex_t* exPtr2) { const typename ex_t::m_predT exPtr1val = exPtr1->getYValue(); const typename ex_t::m_predT exPtr2val = exPtr2->getYValue(); if (exPtr1val != exPtr2val) { if (exPtr1val == m_greaterLabel) { return true; } else { return false; } } // else tied, so consider weights bool ret = m_wcomp(exPtr1, exPtr2); return ret; } } bdtl-1.0.3/src/bdtl/LossFunctionBase.cpp0000644000175000017500000001767012151020420017671 0ustar melamedmelamed/*! * \file LossFunctionBase.cpp */ #include #include "bdtl/Weights.hpp" #include "bdtl/parameter.hpp" #include "bdtl/LossFunctionBase.hpp" #include "Dbug.hpp" #include "ScoreDefs.hpp" #include "MoreAlgorithms.hpp" namespace bdt { // Find the margin of a prediction. // \param confidence The confidence of the prediction. // \param yValue the correct prediction // \return The margin of this prediction. // for classification, we use Turian thesis equation 4.16 double LossFunctionBase::marginOf(const double confidence, const bool yValue) const { if (yValue) { return confidence; } else { return -1.0 * confidence; } } // For regression, we use the negative absolute difference. Using // the the squared or (positive) absolute difference would be wrong // because then the margin would be proportional to the loss. double LossFunctionBase::marginOf(const double confidence, const double yValue) const { const double diff = yValue - confidence; const double ret = - fabs(diff); return ret; // return diff; } double LossFunctionBase::featPriorGain(const string& classKey, double parameterValue, unsigned short featFamily, short polarity) const { double penalty = 0.0; // todo: see if we can still apply the DPSNormalizer double regn = parameter::useUnregularizedConfidencesForFeaturePrior() ? 0. : parameter::globalRegnFactor(); const pair stats = parameter::getFeatFamilyStats(featFamily, polarity, regn, classKey); DEBUG(EX_DBG_LEVEL, "Got stats " << stats.first << ", " << stats.second << " for key " << classKey << " FF " << featFamily << " " << polarity << endl); // if the parameter value is stats.first, if(MA::doubleEpsilonAdditiveEq((parameterValue - stats.first), 0.0)) { DEBUG(EX_DBG_LEVEL, "parameterValue = 0; returning" << endl); // verify that this is the right thing to do! \todo //return 0.0; // c.f. Turian thesis equation 4.38 if(stats.second == 0 && !parameter::useL1PenaltyForECR()) { penalty = -POSINF; } else { if (parameter::useL1PenaltyForECR()) penalty = -parameter::globalRegnFactor(); else penalty = -(1.0 / stats.second); } DEBUG(EX_DBG_LEVEL, "got feature prior penalty (eq 0) " << penalty << endl); } else { if(stats.second == 0) { penalty = (parameterValue > stats.first) ? POSINF : NEGINF; } else { if (parameter::useL1PenaltyForECR()) { penalty = (parameterValue > stats.first) ? parameter::globalRegnFactor() : -parameter::globalRegnFactor(); } else { penalty = (parameterValue > stats.first) ? (1.0 / stats.second) : (-1.0 / stats.second); } } DEBUG(EX_DBG_LEVEL, "got feature prior penalty (neq 0) " << penalty << endl); } return penalty; } double LossFunctionBase::gain(const double unpenGradient, double parameterValue, unsigned short featFamily, short polarity, string classKey) const { if (parameter::useUnpenalizedGain()) return unpenGain(unpenGradient); else { double ret = penalizedGain(unpenGradient, parameterValue); if(parameter::useFeatFamilyPriors() && polarity != 0) { double f = featPriorGain(classKey, parameterValue, featFamily, polarity); DEBUG(EX_DBG_LEVEL, "Got Penalized Gain " << ret << " and feat prior penalty " << f << endl); ret += f; } return fabs(ret); } } /// Absolute value of unpenalized loss gradient double LossFunctionBase::unpenGain(const double unpenGradient) const { return fabs(unpenGradient); } // Decrease in loss as we adjust the current parameter value. double LossFunctionBase::penalizedGain(const double unpenGradient, double parameterValue) const { double ret = 0.0; if (parameter::regnFn() == L1) { const double regn = parameter::globalRegnFactor(); // todo: see if we can still apply the DPSNormalizer // DEBUG(EX_DBG_LEVEL, "regn term is " << regn << endl); if (MA::doubleEpsilonAdditiveEq(parameterValue, 0.0)) { const double unrGain = unpenGain(unpenGradient); assert(unrGain >= 0.0); const double regGain = unrGain - regn; // Turian thesis equation 4.38 ret = max(0., regGain); } else { const double unpenalizedGradient = unpenGradient; const double penalty = (parameterValue > 0) ? regn : -regn; // Turian thesis equation 4.40 // FABS gets applied in gain() ret = unpenalizedGradient + penalty; } } else if (parameter::regnFn() == L2) { const double unpenalizedGradient = unpenGradient; const double penalty = parameterValue * 2 * parameter::globalRegnFactor(); // Turian thesis equation 4.42 // fabs() gets applied in gain() ret = unpenalizedGradient + penalty; } else THROW("Unknown regularization function"); return ret; } double LossFunctionBase::featprior(double confidence, const string& classKey, unsigned short featFamily, short polarity) const { double regn = parameter::useUnregularizedConfidencesForFeaturePrior() ? 0. : parameter::globalRegnFactor(); const pair& stats = parameter::getFeatFamilyStats(featFamily, polarity, regn, classKey); if(stats.second == 0 && !parameter::useL1PenaltyForECR()) return POSINF; DEBUG(EX_DBG_LEVEL + 1, " adding (1.0 / " << (stats.second) << ") * fabs(" << confidence << " - " << stats.first << ")" << endl); if (parameter::useL1PenaltyForECR()) return parameter::globalRegnFactor() * fabs(confidence - stats.first); else return (1.0 / (stats.second)) * fabs(confidence - stats.first); } // Determine the regularization penalty associated with // a model confidence and its DPS normalizer double LossFunctionBase::penalize(double confidence, double DPSNorm) const { if (parameter::regnFn() == L2) return confidence * confidence * parameter::globalRegnFactor(); else if (parameter::regnFn() == L1) return fabs(confidence) * DPSNorm * parameter::globalRegnFactor(); else THROW("Unknown regularization function"); } // method for normalizing the L1 Regularization term from Dudik, Phillips & Schapire (COLT'04). // \beta = beta0 * stddev(featureValue) / sqrt(numberOfExamples) // where beta0 is the tied parameter optimized by cross-validation (e.g.) // and stddev(featureValue) / sqrt(numberOfExamples) is the quantity returned by this method. // Patrick Haffner (p.c., 2007) additionally recommended to lower-bound this quantity by // beta0 / numberOfExamples ([sic] *NOT* sqrt(numberOfExamples)) double LossFunctionBase::DPSNormalizer(const Weights& wei) const { if (! parameter::useDPSNormalizer()) return 1.0; const double totalWeight = wei.sum(); const double pos = wei.pos(); const double neg = wei.neg(); if (totalWeight <= 1 || pos == 0 || neg == 0) { // then stddev = 0, and we return Haffner's lower bound const double ret = 1 / totalWeight; DEBUG(EX_DBG_LEVEL, "DPS normalizer cutting out for pos weight " << pos << " and neg weight " << neg << " is " << ret << endl); return ret; } const double mean = pos / totalWeight; const double oneMinusMean = 1 - mean; const double sumSquaredError = neg * mean * mean + (pos * oneMinusMean * oneMinusMean); const double biasCorrectedVariance = sumSquaredError / (totalWeight - 1); const double stddev = sqrt(biasCorrectedVariance); const double ret = stddev / sqrt(totalWeight); DEBUG(EX_DBG_LEVEL, "DPS normalizer for pos weight " << pos << " and neg weight " << neg << " is " << ret << endl); assert(ret); // avoid 0 assert(ret == ret); // avoid NaN assert(ret < POSINF); return ret; } double LossFunctionBase::lossGradient(const Weights& wei) const { // derivable from Turian thesis equation 4.34 return wei.neg() - wei.pos(); } double LossFunctionBase::regnTerm(const Weights& wei) const { const double theDPSNormalizer = DPSNormalizer(wei); return theDPSNormalizer * parameter::globalRegnFactor(); } } // namespace bdt bdtl-1.0.3/src/bdtl/parameter.cpp0000644000175000017500000010531412171623344016441 0ustar melamedmelamed/*! * \file parameter.cpp * \brief Implementation of parameter methods. */ #include "Dbug.hpp" #include "File.hpp" #include "OptionsPrinter.hpp" #include "Vector.hpp" //#include "RNGWrapper.hpp" #include "bdtl/parameter.hpp" #include "bdtl/LossFunction.hpp" namespace bdt { bool parameter::initialized = false; bool parameter::didReadConfigFile = false; parameter::varTypeMap_t parameter::varTypeMap = parameter::varTypeMap_t(); Vector parameter::names = Vector(); po::options_description parameter::hidden("Hidden options"); po::options_description parameter::localShared("Shared options"); po::variables_map parameter::vm; Vector parameter::sm_scalarFeatures(0, 0); Vector parameter::sm_binaryFeatures(0, 0); Vector parameter::sm_featType(4000000,U); Vector parameter::sm_featFamily(4000000,0); parameter::ffStatsMap_t parameter::sm_featFamilyStats = parameter::ffStatsMap_t(); ofstream* parameter::sm_featFamilyOutFile = NULL; ostringstream* parameter::sm_featFamilyOutBuffer = NULL; ofstream* parameter::sm_k2dmOutFile = NULL; ///////////////////// params used for both training and decoding ////////////////////// // Global debug level. The choices are roughly as follows: // WARNING_DBG_LEVEL = 1; // TREE_DBG_LEVEL = 3; // NODE_DBG_LEVEL = 5; // EX_DBG_LEVEL = 7; // FEAT_DBG_LEVEL = 9; // PEDANTIC_DBG_LEVEL = 11; unsigned parameter::debugLevel = 1; // Should output be buffered? bool parameter::BufferDebug = true; // Output info that can change with each run, such as time and CPU // usage, which can get in the way of diffs against expected outputs bool parameter::DumpVolatileDiagnostics = true; // Either "cerr", "file", "both", or "none" // \note Case-sensitive string parameter::LogLocation = "cerr"; // default name of examples file string parameter::ExampleFileName = ""; // Should ensembles be automatically compiled into a "compiledEnsemble" file at the end of training? // And should they be read in from such files, instead of the tree and lastReg sequences? bool parameter::UseCompiledIO = false; // default names of ensemble files string parameter::TreeSeqFileName = "treeSequence"; string parameter::LastRegSeqFileName = "lastRegSequence"; string parameter::DoneTrainingFileName = "doneTraining"; string parameter::CompiledEnsembleName = "compiledEnsemble"; // path to a file that contains a mapping of classifier keys to hypothesis files // if this file name is non-empty, then it is used during ensemble // loading. Otherwise, a constant default is used. string parameter::Key2dirMapFileName = ""; // path to a file to dump the key2dir map string parameter::Key2DirMapOut = ""; // Directory containing the training hypotheses string parameter::HypothesisDirectory = "."; // (optional) vocabulary file used to decode the BDT vocabulary string parameter::ExternalFeatVocabFileName = ""; featVocab_t* parameter::ExternalFeatVocab = NULL; // Constant epsilon, used for determine negligible differences between feature values // For multiplicative differences, values must be a factor of // 1+featValEpsilon from each other. // \note must be >= 0.0 double parameter::FeatValEpsilon = MINfpVAL; ///////////////////// params used for training only ////////////////////// // Random seed // \todo Make this a long int? unsigned parameter::Seed = 13; // default precision of floating point outputs unsigned short parameter::OutputFloatPrecision = std::numeric_limits::digits10; // Should the filenames generated by a training run contain the classifier key? bool parameter::PutKeysInFilenames = false; // maximum training time (in seconds) for each key, after loading // any pre-existing ensembles; 0 means don't use double parameter::MaxTimePerKey = 0.0; // If the training process might be stopped (intentionally or not) // before convergence, then setting this flag will resume training // from where it stopped. However, this flag should not be set // during validation testing, because it prevents removal of old runs // before a new test. bool parameter::ResumeTraining = true; // Should we randomly shuffle examples in ExBuilder? // \todo Make sure that shuffling is deterministic! bool parameter::ShuffleExamplesInExBuilder = true; // type of label in examples (BOOLEAN for classification; SCALAR for regression) label_t parameter::Label_type = BOOLEAN; string parameter::label_t_str = ""; // used to read param loss_t parameter::Loss_type = LOGISTIC; string parameter::loss_t_str = ""; // used to read param /// pointer to a (singleton) instance of LossFunction LossFunctionBase* parameter::pLossFunc = NULL; regn_t parameter::Regn_type = L1; string parameter::regn_t_str = ""; // used to read param // If adding the Tree to the Ensemble does not reduce penalized training // set loss by at least this much, then don't add the tree to the ensemble, // but just decay the regularization coefficient. double parameter::MinimumReductionInPenalizedTrainLossPerTree = 0.0; //double parameter::MinimumReductionInPenalizedTrainLossPerTree = 0; // If adding the children of the split to the Ensemble does not reduce // penalized training set loss by at least this much, then don't make // the split // double parameter::MinimumReductionInPenalizedTrainLossPerSplit = 1e-3; double parameter::MinimumReductionInPenalizedTrainLossPerSplit = 1e-2; // Allow a node to be split only if the sum of the gains of the // children is at least minGainFactor * the gain of the node. double parameter::MinGainFactorForSplit = 1.01; //double parameter::MinGainFactorForSplit = 1.00; // The split with the highest postulated gain might turn out to have // insufficient actual gain. In that case, it might be worthwhile // to try some other splits at the same node before deciding that // the node can't be split. unsigned parameter::MaxAttemptsPerNode = 1; // When computing the gain (gradient of the objective), // use the unpenalized version? bool parameter::UseUnpenalizedGain = false; // Initial regularization coefficient // intuition: we start by accepting any split that gets at least this many more examples right double parameter::InitialGlobalRegnFactor = 4.0; // current regularization level // \warning This is not a parameter that can be loaded from a parameters // file. double parameter::GlobalRegnFactor = parameter::InitialGlobalRegnFactor; // Minimum regularization coefficient // \note Stopping condition double parameter::MinGlobalRegnFactor = 1e-8; //double parameter::MinGlobalRegnFactor = 0; // When we reduce the regularization level, we reduce it to the // maximum useful threshold, downweighted by globalRegnFactorOverscale double parameter::GlobalRegnFactorOverscale = 0.5; //double parameter::GlobalRegnFactorOverscale = 0.9; //double parameter::GlobalRegnFactorOverscale = 1; // complexity penalty is computed as X / (min(frq1, frq2) ^ Y), // where frq1 and frq2 are the example counts of the new child // nodes, and all variables are >= 0. double parameter::ComplexityCoeff = 5.0; // X double parameter::ComplexityExponent = 2.0; // Y // maximum number of training iterations unsigned parameter::MaxIterations = 999999; // Find the best leaf confidence, within this epsilon double parameter::ConfidenceEpsilon = MINfpVAL; //double parameter::ConfidenceEpsilon = 0.0001; // Maximum depth of any decision tree node unsigned parameter::MaxDepth = 10000; // Minimum number of examples per node unsigned short parameter::MinExPerNode = 2; // Maximum sample size to use when downsampling examples for // feature selection (decision-tree building, but not weighting?). // The sampling method is based upon "Sampling to estimate arbitrary // subset sums" (Duffield et al.). A value of zero indicates no maximum. unsigned parameter::MaxSubsampleSize = 0; // Sampling rate to use when downsampling examples for feature // selection. The resulting number of examples can be limited by the // parameter::MaxSubsampleSize. double parameter::SubsamplingRate = 1.0; // Learning rate. Every gradient step size is multiplied by this fraction. double parameter::LearningRate = 0.1; // Tells whether we're using subsampling. This parameter is derived // after loading the training examples from the values of // MaxSubsampleSize, SubsamplingRate, and the number of training // examples. bool parameter::SubsamplingUsed = false; // Treat each state as having weight 1, and give half this weight // to the positive examples therein and half this weight to the negative // examples therein. // If false, all examples are equally weighted. bool parameter::ShareExampleWeightPerState = false; //bool parameter::ShareExampleWeightPerState = true; /// number of heaviest examples of each sign used to find decision points for scalar feature tests unsigned parameter::HeavyExampleSampleSize = 10; // Normalize the weight biases of all examples to sum to 1. bool parameter::NormalizeWeightBiases = false; // Normalize feature counts to mean 0, variance 1 before applying // the regularization penalty (from Riezler, ACL). // We do this only during the feature selection (tree building) // stage, not the confidence-rating stage. // If true, the gain of some node is scaled by sqrt(p * (1-p)), // where p, 0<=p<=1, is the weighted probability of an example // ending up in this node. bool parameter::NormalizeFeatureCounts = false; /// Use the method for normalizing the L1 regularization term from Dudik, Phillips & Schapire (COLT'04). /// beta = beta0 * stddev(featureValue) / sqrt(numberOfExamples) /// where beta0 is the tied parameter optimized by cross-validation (e.g.) /// and stddev(featureValue) / sqrt(numberOfExamples) is the quantity returned by this method. /// Patrick Haffner (p.c., 2007) additionally recommended to lower-bound this quantity by /// beta0 / numberOfExamples ([sic] *NOT* sqrt(numberOfExamples)) bool parameter::UseDPSNormalizer = false; // Assume that the Ensemble(s) is dense, and try to optimize // confidence computation for this case bool parameter::UseDenseEnsembleOptimizationsDuringConfidenceComputation = false; // Galron's optimization of confidence computation bool parameter::UseOptimizedConfidenceComputation = true; // Should we trace the paths taken by an example down a tree, // in computing its confidence? // \note Toggled internally. bool parameter::TraceConfidencePaths = false; // Should tie-breaking be randomized, or deterministic? The latter // is useful for validation testing. bool parameter::RandomizedTieBreaking = true; // should we not split a node if the gain made by the split // is less than the gain made by creating a new tree? bool parameter::SplitNodeIfGreaterGainThanNewTreeRootSplit = false; // Should we apply the regularization penalty to the root of each // tree? Tibshirani et al. recommend not, but that might not be // optimal when combining classifiers. bool parameter::PenalizeRoots = false; // when adding a tree to the Ensemble, should we add only the CFTs // represented by the leaves, or by all the nodes in the tree? // The implications of adding all the nodes are: // 1. The examples falling into a new node are reweighted as soon as the node is created. // 2. When adding CFTs to Ensemble, we iterate over all the nodes rather than just the leaves. // 3. When deciding whether to keep a node, we do not consider whether its gain is greater than that of its parent. bool parameter::AddInternalNodesToEnsemble = true; // whether to presort features or not. bool parameter::PreSortedFeatures = true; // Format of examples in example files string parameter::ef_str = "GenPar"; ExampleFormat parameter::ExFormat = GenPar; // map of features to feature families, one feature per line string parameter::FeatureFamilyFileName = ""; // map of feature families/regularizer multipliers to sample statistics string parameter::FeatureFamilyStatsFileName = ""; // name of file to which to dump feature values string parameter::FeatureFamilyValueOutputFileName = ""; bool parameter::UseUnregularizedConfidencesForFeaturePrior = false; bool parameter::UseRegForECR = false; //Vector parameter::sm_featFamilyRegnFactor; void parameter::setBinary(const Feature& theFeat, const bool val) { const unsigned idx(theFeat.uid()); if(idx >= sm_featType.size()) sm_featType.resize(idx + 1, U); if(sm_featType[idx] == U) { if (val == true) { sm_featType[idx] = B; sm_binaryFeatures.push_back(idx); } } else if(sm_featType[idx] == B && val == true) { return; } else if(sm_featType[idx] != B && val == false) { return; } else { if(sm_featType[idx] == B) cerr << "featType = B" << endl; else if(sm_featType[idx] == F) cerr << "featType = F" << endl; else if(sm_featType[idx] == M) cerr << "featType = M" << endl; else if(sm_featType[idx] == I) cerr << "featType = I" << endl; else if(sm_featType[idx] == U) cerr << "featType = U" << endl; else cerr << "featType = huh?" << endl; THROW("Trying to change type of feature " << theFeat << " from " << sm_featType[idx] << " to " << "B"); } } void parameter::setInteger(const Feature& theFeat, const bool val) { const unsigned idx(theFeat.uid()); if(idx >= sm_featType.size()) sm_featType.resize(idx + 1, U); if(sm_featType[idx] == U) { if(val == true) { sm_featType[idx] = I; sm_scalarFeatures.push_back(idx); } } else if(sm_featType[idx] == I && val == true) { return; } else if(sm_featType[idx] != I && val == false) { return; } else THROW("Trying to change type of feature " << theFeat << " from " << sm_featType[idx] << " to " << "I"); } void parameter::setFloat(const Feature& theFeat, const bool val) { const unsigned idx(theFeat.uid()); if(idx >= sm_featType.size()) sm_featType.resize(idx + 1, U); // never change it once it's set if (sm_featType[idx] == U) { if (val == true) sm_featType[idx] = F; else sm_featType[idx] = I; sm_scalarFeatures.push_back(idx); } else if (sm_featType[idx] == F && val == true) { return; } else if (sm_featType[idx] == I && val == false) { return; } else THROW("Trying to change type of feature " << theFeat << "(" << idx << ")" << " from " << sm_featType[idx] << " to " << (val ? "F" : "I")); } const pair& parameter::getFeatFamilyStats(const unsigned short family, const short polarity, const double regn, const string& classKey) { static const pair defRet(make_pair(0.0,0.0)); pair k = make_pair(family, polarity); //tuple k(classKey,family,polarity); if(sm_featFamilyStats.find(k) == sm_featFamilyStats.end()) { // back off pair bo = make_pair(0, polarity); //tuple bo(classKey,0,polarity); if(sm_featFamilyStats.find(bo) == sm_featFamilyStats.end()) return defRet; k = bo; } int bucket = (int)floor(log(regn) / log(2)); if(parameter::useUnregularizedConfidencesForFeaturePrior()) bucket = 0; if(sm_featFamilyStats[k].find(bucket) == sm_featFamilyStats[k].end()) return defRet; return sm_featFamilyStats[k][bucket]; } // Dump all globals to a string. // \todo Assert that the strings don't contain quotation marks. // \todo Set proper floating point precision. const string& parameter::str(string s) { static string ret; static ostringstream o; parameter::sanityCheck(); Vector::iterator n; for (n = parameter::names.begin(); n != parameter::names.end(); n++) { assert(parameter::varTypeMap.find(*n) != parameter::varTypeMap.end()); varTypeMap_t::iterator i = parameter::varTypeMap.find(*n); assert(*n == i->first); o << s << *n << " = "; switch (i->second.second) { case STRINGTY: // \todo Assert that the strings don't contain quotation marks. o << "'" << *((string*)i->second.first) << "'"; break; case BOOLTY: if (*((bool*)i->second.first)) { o << "true"; } else { o << "false"; } break; case INTTY: o << *((int*)i->second.first); break; case UNSIGNEDTY: o << *((unsigned*)i->second.first); break; case DOUBLETY: o << *((double*)i->second.first); break; default: throw; } if (!parameter::vm.count(*n)) o << " [default]"; o << endl; } ret = o.str(); return ret; } // Read in the parameters. Parameters are read in from the // command-line and, possibly, from a parameters-file specified on // the command-line. // \param fromCaller Parameters passed in from calling program. // \param argc Command-line argument count. // \param argv Command-line argument vector. // \return The variables map into which we read the parameters. // \todo Make argv a const! // \todo Does it work to look at vm.count("parametersFile") // before calling notify(vm)? const po::variables_map& parameter::read(int argc, char *argv[], const po::options_description& fromCaller) { if (parameter::didReadConfigFile) { // already read return parameter::vm; } parameter::initialize(); // parametersFile can appear in another config file if it's nested string parametersFile; localShared.add_options() ("debugLevel,d", po::value(&debugLevel)) ("minimumRegFactor,t", po::value(&MinGlobalRegnFactor)) ("parametersFile,p", po::value(¶metersFile), "file from which to read parameters [optional]") ; // Declare a group of options that will be // allowed only on command line po::options_description cmdLineOnly("CmdLineOnly options"); cmdLineOnly.add_options() ("version,v", "print version string") ("help,h", "produce help message") ; po::options_description cmdlineOptions; cmdlineOptions.add(cmdLineOnly).add(localShared).add(hidden).add(fromCaller); po::options_description configFileOptions; configFileOptions.add(localShared).add(hidden).add(fromCaller); po::options_description visible; visible.add(cmdLineOnly).add(localShared).add(fromCaller); // read in command-line params, overriding config file and environment store(po::command_line_parser(argc, argv).options(cmdlineOptions).run(), parameter::vm); // read in relevant environment variables, overriding config file // The following line started segfaulting for unknown reasons. Fortunately, we don't use it currently. // store(po::parse_environment(configFileOptions, "BDT"), parameter::vm); if (parameter::vm.count("parametersFile")) { string parametersFile = parameter::vm["parametersFile"].as(); // We can't check parameter::dumpVolatileDiagnostics() until we call notify(parameter::vm); :( /* // We haven't opened the debug stream yet if (parameter::dumpVolatileDiagnostics()) cerr << "Reading parameters file: " << parametersFile << endl; */ iFile ifs(parametersFile); store(parse_config_file(ifs, configFileOptions), parameter::vm); } notify(parameter::vm); if (parameter::vm.count("help")) { cerr << visible << endl; cerr.flush(); exit(0); } if (parameter::vm.count("version")) { throw; } // Limit precision in sandbox mode, so that it doesn't vary on // different architectures, unless it is explicitly specified in // the config params. if (! parameter::RandomizedTieBreaking) { if (! parameter::vm.count("outputFloatPrecision")) parameter::OutputFloatPrecision = min(14, std::numeric_limits::digits10); if (! parameter::vm.count("featValEpsilon")) parameter::FeatValEpsilon = max((fpVal_t)1e-8, MINfpVAL); if (! parameter::vm.count("confidenceEpsilon")) parameter::ConfidenceEpsilon = max((fpVal_t)1e-8, MINfpVAL); } // If we are not adding internal nodes to the ensemble, then the // gain of a new split must exceed that of the node being split. // Otherwise, we don't need a minGainFactor, so set it to 0 to // make it have no effect. if (parameter::addInternalNodesToEnsemble()) MinGainFactorForSplit = 0.; // set up output stream(s) cout.precision(parameter::outputFloatPrecision()); cerr.precision(parameter::outputFloatPrecision()); static const string key; string loggingTo = ""; string inLoc(parameter::logLocation()); string fName = "log"; if (inLoc == "cerr") { loggingTo = "cerr"; Dbug::setGlobalDOut(cerr); } else if (inLoc == "file") { loggingTo = fName; oFile outFile(fName); outFile.precision(parameter::outputFloatPrecision()); // if (parameter::resumeTraining()) // outFile = fopen(logfile.c_str(), "at"); // else // outFile = fopen(logfile.c_str(), "w"); // assert(outFile); // Out.push(boost::iostreams::bzip2::bzip2Compressor()); // Out.push(boost::iostreams::file_descriptor(fileno(outFile))); Dbug::setGlobalDOut(outFile); } else if (inLoc == "both") { loggingTo = fName + "+ cerr"; } else if (inLoc == "none") { // do nothing } else { THROW("Unknown logging option"); } if (inLoc == "both" || inLoc == "none") { // not implemented THROW("Not Implemented"); } if (!parameter::bufferDebug()) Dbug::getGlobalDOut().setf(std::ios_base::unitbuf); // Disable buffering for Out Dbug::getGlobalDOut().precision(parameter::outputFloatPrecision()); DEBUG(TREE_DBG_LEVEL, "\n" << endl); DEBUG(TREE_DBG_LEVEL, "#####################################################################" << endl); DEBUG(TREE_DBG_LEVEL, "#####################################################################" << endl); DEBUG(TREE_DBG_LEVEL, "Logging BDT to " << loggingTo << endl); if (parameter::vm.count("exampleFormat")) { ef_str = vm["exampleFormat"].as(); if (ef_str == "GenPar") { ExFormat = GenPar; } else if (ef_str == "SVMLight") { ExFormat = SVMLight; } else { THROW("Unknown example type" << endl); } } else { ExFormat = GenPar; } if (parameter::vm.count("labelType")) { label_t_str = vm["labelType"].as(); if (label_t_str == "boolean") { Label_type = BOOLEAN; } else if (label_t_str == "scalar") { Label_type = SCALAR; } else { THROW("Unknown label type" << endl); } } else { // default Label_type = BOOLEAN; } if (parameter::vm.count("lossFn")) { loss_t_str = vm["lossFn"].as(); if (loss_t_str == "logistic") { Loss_type = LOGISTIC; pLossFunc = new LossFunction; } else if (loss_t_str == "exponential") { Loss_type = EXPONENTIAL; pLossFunc = new LossFunction; } else if (loss_t_str == "squared") { Loss_type = SQUARED; pLossFunc = new LossFunction; } else { THROW("Unknown loss type" << endl); } } else { // default Loss_type = LOGISTIC; pLossFunc = new LossFunction; } if (parameter::vm.count("regnFn")) { regn_t_str = vm["regnFn"].as(); if (regn_t_str == "L1") { Regn_type = L1; } else if (regn_t_str == "L2") { Regn_type = L2; } else { THROW("Unknown regularization type" << endl); } } else { // default Regn_type = L1; } // \Todo BDT debug level shouldn't affect everything else if(!Dbug::isDebugAll(debugLevel)) Dbug::debugall(debugLevel); DEBUG(WARNING_DBG_LEVEL, endl); DEBUG(WARNING_DBG_LEVEL, "####################################" << endl); DEBUG(WARNING_DBG_LEVEL, "# Global parameters:" << endl); DEBUG(WARNING_DBG_LEVEL, "Option values of \"" << argv[0] << "\"" << endl); DEBUG(WARNING_DBG_LEVEL, OptionsPrinter(visible, vm) << endl); DEBUG(WARNING_DBG_LEVEL, OptionsPrinter(hidden, vm) << endl); // for efficiency sm_binaryFeatures.reserve(100000); // read in feature vocabulary, if using if (parameter::vm.count("externalFeatVocabFileName")) { const string featVocabFileName = vm["externalFeatVocabFileName"].as(); iFile fv(featVocabFileName); externalFeatVocab() = new featVocab_t; fv >> *externalFeatVocab(); fv.close(); // copy into the Turian-style Feature vocab, so that they match in order featVocab_t::const_iterator_rev fvci = externalFeatVocab()->beginIndex(); const featVocab_t::const_iterator_rev sentinel = externalFeatVocab()->endIndex(); for(; fvci != sentinel; ++fvci) { const StringWithSpaces& sws = fvci->second; // constructing it should be enough to store it in the static set Feature foo(sws); } } // read in feature families, if using if (parameter::vm.count("featureFamilyFileName")) { string filename = vm["featureFamilyFileName"].as(); iFile ffFile(filename); unsigned short aFamily; unsigned int count = 1; while(ffFile >> aFamily) { sm_featFamily[count] = aFamily; ++count; } ffFile.close(); // Read in feature family sample statistics, if using // format: classKey FF sign Regn mu tau if(parameter::vm.count("featureFamilyStatsFileName")) { filename = vm["featureFamilyStatsFileName"].as(); iFile ffsFile(filename); double aMu, aTau; int bucket; short n; string classKey; pair k; while(!ffsFile.eof()) { ffsFile >> count >> n >> bucket >> aMu >> aTau; k = make_pair(count, n); if(sm_featFamilyStats.find(k) == sm_featFamilyStats.end()) { sm_featFamilyStats.insert(make_pair(k, ffStats_t())); } sm_featFamilyStats[k].insert(make_pair(bucket, make_pair(aMu, aTau))); } ffsFile.close(); } if(parameter::vm.count("featureFamilyValueOutputFileName")) { filename = vm["featureFamilyValueOutputFileName"].as(); sm_featFamilyOutFile = new oFile(filename, ios_base::app, parameter::outputFloatPrecision()); sm_featFamilyOutBuffer = new ostringstream(); } } if(parameter::vm.count("key2dirMapOutput")) { sm_k2dmOutFile = new oFile(vm["key2dirMapOutput"].as(), ios_base::app); } parameter::sanityCheck(); parameter::didReadConfigFile = true; return parameter::vm; } void parameter::readConfigFile(string& parametersFile) { if (parameter::didReadConfigFile) { // already read return; } parameter::initialize(); cerr << "Reading BDT configuration parameters from: " << parametersFile << endl; iFile ifs(parametersFile); assert(ifs.is_open() && ifs.good()); store(parse_config_file(ifs, parameter::hidden), parameter::vm); notify(parameter::vm); parameter::sanityCheck(); parameter::didReadConfigFile = true; } void parameter::addHidden(string name, void* location, parameterTy ty) { assert (!initialized); assert(varTypeMap.find(name) == varTypeMap.end()); varTypeMap.insert(make_pair(name, make_pair(location, ty))); names.push_back(name); switch (ty) { case STRINGTY: hidden.add_options()(name.c_str(), po::value((string*)location)); break; case BOOLTY: hidden.add_options()(name.c_str(), po::value((bool*)location)); break; case INTTY: hidden.add_options()(name.c_str(), po::value((int*)location)); break; case UNSIGNEDTY: hidden.add_options()(name.c_str(), po::value((unsigned*)location)); break; case DOUBLETY: hidden.add_options()(name.c_str(), po::value((double*)location)); break; default: throw; } } // Add all parameters to our parameter list. void parameter::initialize() { // Don't do this more than once. if (initialized) return; assert(!didReadConfigFile); // RNGWrapper::init(); srand48(parameter::seed()); localShared.add_options()("exampleFileName,e", po::value(&ExampleFileName)); localShared.add_options()("treeSequence", po::value(&TreeSeqFileName)); localShared.add_options()("lastRegSequence", po::value(&LastRegSeqFileName)); localShared.add_options()("compiledEnsemble", po::value(&CompiledEnsembleName)); addHidden("key2dirMapFileName", &Key2dirMapFileName, STRINGTY); addHidden("key2dirMapOutput", &Key2DirMapOut, STRINGTY); addHidden("externalFeatVocabFileName,f", &ExternalFeatVocabFileName, STRINGTY); addHidden("hypothesisDirectory,y", &HypothesisDirectory, STRINGTY); addHidden("putKeysInFilenames", &PutKeysInFilenames, BOOLTY); addHidden("dumpVolatileDiagnostics", &DumpVolatileDiagnostics, BOOLTY); addHidden("maxTimePerKey", &MaxTimePerKey, DOUBLETY); addHidden("logLocation", &LogLocation, STRINGTY); addHidden("useCompiledIO", &UseCompiledIO, BOOLTY); addHidden("resumeTraining", &ResumeTraining, BOOLTY); addHidden("shuffleExamplesInExBuilder", &ShuffleExamplesInExBuilder, BOOLTY); addHidden("labelType", &label_t_str, STRINGTY); addHidden("lossFn", &loss_t_str, STRINGTY); addHidden("regnFn", ®n_t_str, STRINGTY); addHidden("minimumReductionInPenalizedTrainLossPerTree", &MinimumReductionInPenalizedTrainLossPerTree, DOUBLETY); addHidden("minimumReductionInPenalizedTrainLossPerSplit", &MinimumReductionInPenalizedTrainLossPerSplit, DOUBLETY); addHidden("maxAttemptsPerNode", &MaxAttemptsPerNode, UNSIGNEDTY); addHidden("splitNodeIfGreaterGainThanNewTreeRootSplit", &SplitNodeIfGreaterGainThanNewTreeRootSplit, BOOLTY); addHidden("penalizeRoots", &PenalizeRoots, BOOLTY); addHidden("addInternalNodesToEnsemble", &AddInternalNodesToEnsemble, BOOLTY); addHidden("minGainFactorForSplit", &MinGainFactorForSplit, DOUBLETY); addHidden("useUnpenalizedGain", &UseUnpenalizedGain, BOOLTY); addHidden("initialGlobalRegnFactor", &InitialGlobalRegnFactor, DOUBLETY); // addHidden("minimumRegFactor", &MinGlobalRegnFactor, DOUBLETY); addHidden("globalRegnFactorOverscale", &GlobalRegnFactorOverscale, DOUBLETY); addHidden("complexityCoeff", &ComplexityCoeff, DOUBLETY); addHidden("complexityExponent", &ComplexityExponent, DOUBLETY); addHidden("shareExampleWeightPerState", &ShareExampleWeightPerState, BOOLTY); addHidden("maxSubsampleSize", &MaxSubsampleSize, UNSIGNEDTY); addHidden("subsamplingRate", &SubsamplingRate, DOUBLETY); addHidden("learningRate", &LearningRate, DOUBLETY); addHidden("heavyExampleSampleSize", &HeavyExampleSampleSize, UNSIGNEDTY); addHidden("normalizeWeightBiases", &NormalizeWeightBiases, BOOLTY); addHidden("normalizeFeatureCounts", &NormalizeFeatureCounts, BOOLTY); addHidden("useDPSNormalizer", &UseDPSNormalizer, BOOLTY); addHidden("useDenseEnsembleOptimizationsDuringConfidenceComputation", &UseDenseEnsembleOptimizationsDuringConfidenceComputation, BOOLTY); addHidden("useOptimizedConfidenceComputation", &UseOptimizedConfidenceComputation, BOOLTY); addHidden("featValEpsilon", &FeatValEpsilon, DOUBLETY); addHidden("outputFloatPrecision", &OutputFloatPrecision, UNSIGNEDTY); addHidden("maxIterations", &MaxIterations, UNSIGNEDTY); addHidden("preSortedFeatures", &PreSortedFeatures, BOOLTY); addHidden("confidenceEpsilon", &ConfidenceEpsilon, DOUBLETY); addHidden("maxDepth", &MaxDepth, UNSIGNEDTY); addHidden("minExPerNode", &MinExPerNode, UNSIGNEDTY); addHidden("seed", &Seed, UNSIGNEDTY); addHidden("bufferDebug", &BufferDebug, BOOLTY); addHidden("traceConfidencePaths", &TraceConfidencePaths, BOOLTY); addHidden("randomizedTieBreaking", &RandomizedTieBreaking, BOOLTY); addHidden("exampleFormat", &ef_str, STRINGTY); addHidden("featureFamilyFileName", &FeatureFamilyFileName, STRINGTY); addHidden("featureFamilyStatsFileName", &FeatureFamilyStatsFileName, STRINGTY); addHidden("featureFamilyValueOutputFileName", &FeatureFamilyValueOutputFileName, STRINGTY); addHidden("useUnregularizedConfidencesForFeaturePrior", &UseUnregularizedConfidencesForFeaturePrior, BOOLTY); addHidden("useL1PenaltyForECR", &UseRegForECR, BOOLTY); initialized = true; } // Sanity check all variable values. // \todo Add more sanity checks. inline void parameter::sanityCheck() {} void parameter::trainingSanityCheck() { THROWIF(ExampleFileName == "", "Example file not specified!"); THROWUNLESS(MinimumReductionInPenalizedTrainLossPerTree >= 0., ""); THROWUNLESS(MinimumReductionInPenalizedTrainLossPerSplit >= 0., ""); // at least one of them must be active, i.e. strictly more than 0 THROWUNLESS((MinimumReductionInPenalizedTrainLossPerTree > 0. && ! AddInternalNodesToEnsemble) || (MinimumReductionInPenalizedTrainLossPerSplit > 0. && AddInternalNodesToEnsemble), ""); THROWUNLESS((MaxAttemptsPerNode >= 1), ""); THROWUNLESS((MinExPerNode >= 1), ""); THROWUNLESS((AddInternalNodesToEnsemble || MinGainFactorForSplit >= 1.), ""); THROWUNLESS(GlobalRegnFactorOverscale <= 1. && GlobalRegnFactorOverscale > 0., ""); THROWUNLESS(NormalizeFeatureCounts == false, ""); THROWUNLESS(MinGlobalRegnFactor >= 0., ""); THROWUNLESS((LogLocation == "cerr" || LogLocation == "file" || LogLocation == "both" || LogLocation == "none"), ""); THROWUNLESS(FeatValEpsilon >= MINfpVAL, "Epsilon of " << FeatValEpsilon << " is less than minimum value to do math with, which is " << MINfpVAL << endl); THROWUNLESS(ConfidenceEpsilon >= MINfpVAL, "Epsilon of " << ConfidenceEpsilon << " is less than minimum value to do math with, which is " << MINfpVAL << endl); // internal nodes incompatible with per-tree thresholding THROWUNLESS(! AddInternalNodesToEnsemble || MinimumReductionInPenalizedTrainLossPerTree == 0.0, "incompatible parameters with MinimumReductionInPenalizedTrainLossPerTree"); THROWUNLESS( // no way to handle both yet (MinimumReductionInPenalizedTrainLossPerSplit == 0.0 || MinimumReductionInPenalizedTrainLossPerTree == 0.0 ) , "incompatible parameters with MinimumReductionInPenalizedTrainLossPerSplit"); } void parameter::setGlobalRegnFactor(double d) { parameter::initialize(); GlobalRegnFactor = d; parameter::sanityCheck(); } const BSP parameter::makeFileName(const string& prefix, const string& key) { const boost::filesystem::path basePath(parameter::hypothesisDirectory()); const boost::filesystem::path fullPath = basePath / prefix; string pathString(fullPath.string()); if (parameter::putKeysInFilenames() && key != "") pathString.append(".key-" + key); DEBUG(NODE_DBG_LEVEL, "parameter::makeFileName made " << pathString << endl); const BSP pOut(new string(pathString)); return pOut; } } // namespace bdt bdtl-1.0.3/src/bdtl/Sampler.hpp0000644000175000017500000000442312151020420016050 0ustar melamedmelamed/*! * \file Sampler.hpp */ /*! * \class bdt::Sampler * \brief * * The sampling method is based upon "Sampling to estimate arbitrary * subset sums" (Duffield et al.): * "we consider a [stream] of items i=0,...,n-1 with weights wi. For each * item i, we generate a random number ri in (0,1) and create a priority * qi=wi/ri. The sample S consists of the k highest priority items. Let * t be the (k+1)th highest priority. Each sampled item i in S gets * a weight estimate Wi=max{wi,t}, while non-sampled items get weight * estimate Wi=0." * * \todo We may even be able to get away with keeping the Example * Feature vectors around, uncompressed. (HeavyExample) * \todo Design an Sampler iterator (perhaps using * the Boost Iterators library). * \todo Use a Random type for random numbers. * */ #ifndef _BDTSamplerH__ #define _BDTSamplerH__ #include "bdtl/ExVec.hpp" #include #ifndef DOXYGEN using namespace std; #endif /* DOXYGEN */ namespace bdt { namespace Sampler { /// The sampling method is based upon "Sampling to estimate /// arbitrary subset sums" (Duffield et al.): "we consider a /// [stream] of items i=0,...,n-1 with weights wi. For each item /// i, we generate a random number ri in (0,1) and create a /// priority qi=wi/ri. The sample S consists of the k highest /// priority items. Let t be the (k+1)th highest priority. Each /// sampled item i in S gets a weight estimate Wi=max{wi,t}, while /// non-sampled items get weight estimate Wi=0." /// \arg sampleEPtrs the result, returned by reference /// \arg inPtrs the input set to sample from /// \arg filterByYValue should only examples with the given yValue be included? /// \arg yvalue Required y-value for each example sampled /// \arg maxn The maximum number of examples to sample /// \post the sample will contain at most maxn yvalue'd examples template void getPrioritySample(BSP sampleEptrs, const BSP inPtrs, unsigned maxn, vector& sampledIndex, bool filterByYValue = false, bool yValue = false); template ExamplePtrs& getHeavyExamples(ExamplePtrs& exVec); // const; } } // namespace bdt #include #endif /* _BDTSamplerH_ */ bdtl-1.0.3/src/bdtl/ExComparator.cpp0000644000175000017500000000133112151020420017037 0ustar melamedmelamed#include "bdtl/ExComparator.hpp" #include "bdtl/parameter.hpp" #include "bdtl/Feature.hpp" namespace bdt { bool ExFeatComparator::featComp(const IFUnion_t& val1, const IFUnion_t& val2, unsigned id1, unsigned id2) { const double eps = parameter::featValEpsilon(); if (m_isFloat) { const fpVal_t v1 = val1.f; const fpVal_t v2 = val2.f; if (v2 - v1 > eps) return true; else if (v1 - v2 > eps) return false; else return (id2 < id1); } else { // not a float const intVal_t v1 = val1.i; const intVal_t v2 = val2.i; if (v1 < v2) return true; else if (v1 > v2) return false; else return (id2 < id1); } } } bdtl-1.0.3/src/bdtl/DecTree.cpp0000644000175000017500000001116312151020420015752 0ustar melamedmelamed/*! * \file DecTree.ipp */ #include #include "bdtl/DecTree.hpp" #include "bdtl/Ensemble.hpp" #include "Dbug.hpp" #include "bdtl/parameter.hpp" namespace bdt { // ctor DecTree::DecTree() : m_rootNode(NONODE), m_steepestUnpenalizedLossGradient(-1.0) {} // True iff this DecTree is an unsplit root. bool DecTree::isJustRoot() const { assert(!m_nodes.empty()); assert(m_nodes.at(m_rootNode)->isRoot()); bool ret = !m_nodes.at(m_rootNode)->hasChildren(); return ret; } const DecTree::nodeContainer_t& DecTree::leaves() const { static nodeContainer_t leaves; leaves.clear(); nodeContainer_t::const_iterator n; for (n = m_nodes.begin(); n != m_nodes.end(); ++n) { const nodePtr_t pNode(*n); if (pNode->isLeaf()) leaves.push_back(pNode); } return leaves; } // Clear the m_nodes and m_frontier void DecTree::clear() { DEBUG(TREE_DBG_LEVEL + 1, "DecTree::clear()..." << endl); iterator iter = getNodes().begin(); const const_iterator sentinel = getNodes().end(); for(; iter != sentinel; ++iter) delete(*iter); m_nodes.clear(); m_frontier.clear(); DEBUG(TREE_DBG_LEVEL + 1, "...DecTree::clear()" << endl); } // Remove NodeID n from DecTree. // \note This only works if n is the LAST node in the DecTree. void DecTree::pruneNode(const NodeID n) { DEBUG(TREE_DBG_LEVEL + 1, "Pruning node #" << n << " from tree" << endl); assert(m_nodes.size() - 1 == n); assert(m_nodes.back()->id() == n); removeFromFrontier(n); delete(m_nodes.back()); m_nodes.pop_back(); } void DecTree::removeFromFrontier(const NodeID n) { m_frontier.erase(n); DEBUG(NODE_DBG_LEVEL, "DecTree::removeFromFrontier() removed node #" << n << " from frontier" << endl); DEBUG(NODE_DBG_LEVEL, "Frontier now contains " << m_frontier.size() << " nodes." << endl); } // Build an empty tree. // DecTree::DecTree() // : m_pEnsemble(NULL), m_steepestUnpenalizedLossGradient(-1.0) // {} istream& DecTree::read(istream &theStream) { clear(); string ignore; getline(theStream, ignore); THROWUNLESS(ignore == "BeginTree", "last line is: " << ignore); nodeContainer_t& nodeVec = m_nodes; char c = theStream.peek(); while (c == '\t') { // Each node is on a tab-indented line DecNode* newNodePtr(new DecNode); theStream >> (*newNodePtr); nodeVec.push_back(newNodePtr); // We don't worry about the frontier, because we never read in trees that we might grow further. c = theStream.peek(); } getline(theStream, ignore); THROWUNLESS(ignore == "EndTree", "last line is: " << ignore); // eat up trailing newlines c = theStream.peek(); while (c == '\n') { getline(theStream, ignore); c = theStream.peek(); } // Add parents and features for all nodes nodeContainer_t::iterator nIter = nodeVec.begin(); const nodeContainer_t::iterator sentinel = nodeVec.end(); unsigned i; for (i=0; nIter != sentinel; ++nIter, ++i) { const DecNode* pNode = *nIter; const DecNode& theNode = *pNode; assert(theNode.id() == i); assert(nodeVec.at(i)->id() == i); if (! theNode.isLeaf()) { const FeatureTest posFT(theNode.getFT()); NodeID trueChild = theNode.getTrueChildID(); assert(nodeVec[trueChild]->parentID() == NONODE); // make sure it's not set yet CompoundFeatureTest cfpos(theNode.getCFT() + posFT); DecNode* tNode = nodeVec[trueChild]; tNode->setParentAndCFT(i, cfpos); const FeatureTest negFT = posFT.negate(); NodeID falseChild = theNode.getFalseChildID(); assert(nodeVec[falseChild]->parentID() == NONODE); // make sure it's not set yet CompoundFeatureTest cfneg(theNode.getCFT() + negFT); DecNode* fNode = nodeVec[falseChild]; fNode->setParentAndCFT(i, cfneg); } } return theStream; } // Convert a Tree to a string. string DecTree::detailedStr(const string s) const { string o; o += "BeginDecTree\n"; nodeContainer_t::const_iterator n = m_nodes.begin(); const nodeContainer_t::const_iterator sentinel = m_nodes.end(); for (; n != sentinel; ++n) { const DecNode* pNode = *n; o += pNode->detailedStr(s + "\t"); } o += "EndDecTree\n"; return o; } // Convert a Tree to a string. string DecTree::str(const string s) const { string o; o += "BeginTree\n"; nodeContainer_t::const_iterator n = m_nodes.begin(); const nodeContainer_t::const_iterator sentinel = m_nodes.end(); for (; n != sentinel; ++n) { const DecNode* pNode = *n; o += pNode->str(s + "\t"); } o += "EndTree\n"; return o; } } // namespace bdt bdtl-1.0.3/src/bdtl/Sampler.ipp0000644000175000017500000002076112151020420016054 0ustar melamedmelamed/*! * \file Sampler.ipp */ //#include "bdtl/Sampler.hpp" #include "ResourceUsage.hpp" #include "Time.hpp" #include "NBest.hpp" #include "bdtl/PrioritizedExPtr.hpp" #include "bdtl/LossFunction.hpp" #include "bdtl/parameter.hpp" namespace bdt { template void Sampler::getPrioritySample(BSP sampleEptrs, const BSP inPtrs, unsigned maxn, vector& sampledIndex, bool filterByYValue, bool yValue) { assert(maxn > 0); assert(! inPtrs->empty()); if (filterByYValue) { DEBUG(TREE_DBG_LEVEL + 1, "Sampling only " << (yValue ? "+ive" : "-ive") << " examples" << endl); } DEBUG(TREE_DBG_LEVEL + 1, "Sampling at most " << maxn << " examples"); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL + 1, " at " << nsTime::currentTime()); DEBUG(TREE_DBG_LEVEL + 1, "..." << endl); // allow for the sample already having something in it const size_t origSize = sampleEptrs->size(); // get maxn+1 examples because we use the last one to determine the minWeight for reweighting // for preSorted features, we also need to know the indices of the training examples that were // selected to be in build pointers. NBest, int> > bestExSeq(maxn + 1); // create sample using priority sampling // examples are added by pointer, to avoid redundant re-construction unsigned tot = 0; double totWeight = 0.; double minWeight; typename ExamplePtrs::const_iterator exIter = inPtrs->begin(); const typename ExamplePtrs::const_iterator sentinel = inPtrs->end(); for (; exIter != sentinel; ++exIter) { const ex_t* pEx = static_cast(*exIter); if (filterByYValue && pEx->getYValue() != yValue) continue; pair, int> ex_tIndexPair(pEx, exIter - inPtrs->begin()); // pushing pointers, so no construction bestExSeq.push(ex_tIndexPair); if (tot % 100000 == 0 && !bestExSeq.empty()) { minWeight = bestExSeq.bottom().first.priority(); DEBUG(NODE_DBG_LEVEL, "\tsourced " << tot + 1 << " examples (minWeight in sample = " << minWeight << ")"); if (parameter::dumpVolatileDiagnostics()) DEBUG(NODE_DBG_LEVEL, " at " << nsTime::currentTime()); DEBUG(NODE_DBG_LEVEL, "..." << endl); } tot++; totWeight += parameter::pLossFunc->weight(pEx); if (!bestExSeq.empty()) DEBUG(PEDANTIC_DBG_LEVEL, "Sampler totWeight = " << totWeight << " minWeight = " << bestExSeq.bottom().first.priority() << " after adding " << *pEx << endl); } const unsigned sizeDelta = bestExSeq.size(); DEBUG(TREE_DBG_LEVEL + 1, "Sampled " << sizeDelta - 1 << " examples"); unsigned howManyToUse = maxn; // If the sample isn't full then keep all examples if (sizeDelta < maxn + 1) { howManyToUse = sizeDelta; DEBUG(TREE_DBG_LEVEL + 1, " (less than maxn=" << maxn << ")"); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL + 1, " at " << nsTime::currentTime()); DEBUG(TREE_DBG_LEVEL + 1, endl); } const unsigned newSize = origSize + howManyToUse; sampleEptrs->reserve(newSize); minWeight = bestExSeq.bottom().first.priority(); DEBUG(TREE_DBG_LEVEL + 1, " (minWeight in sample = " << minWeight << ")"); for(unsigned i = 0; i < howManyToUse; ++i) { assert(!bestExSeq.empty()); pair, int> ex_tIndexPair = bestExSeq.top(); const ex_t* exPtr = static_cast(ex_tIndexPair.first.examplePtr()); sampledIndex[ex_tIndexPair.second] = i + 1; // here we insert by copy construction ex_t* exCopyPtr = new ex_t(*exPtr); sampleEptrs->push_back(exCopyPtr); bestExSeq.pop(); } unsigned replaced = 0; double trueWeight = 0.; double keepWeight = 0.; double replacedWeight = 0.; double currWeight; // if the sample uses all the examples, then there's no point reweighting // if the sample uses all the examples, then bestExSeq will not be full, // so sizeDelta will be < maxn + 1 and so will howManyToUse if (howManyToUse < maxn + 1) { for(unsigned i = origSize; i < newSize; ++i) { ex_t* currExPtr = static_cast(sampleEptrs->at(i)); currWeight = parameter::pLossFunc->weight(currExPtr); trueWeight += currWeight; if(currWeight < minWeight) { parameter::pLossFunc->scaleWeightBias(currExPtr, minWeight); replaced++; replacedWeight += minWeight; } keepWeight += parameter::pLossFunc->weight(currExPtr); } } assert(sampleEptrs->size() == newSize); assert(sampleEptrs->size() == sampleEptrs->capacity()); DEBUG(TREE_DBG_LEVEL + 1, "\tSampling lost " << (1. - trueWeight/totWeight) * 100. << "% of " << "example weight" << endl); DEBUG(TREE_DBG_LEVEL + 1, "\tReplaced " << replaced << " " << "examples (" << 100. * replaced / maxn << "%) with minWeight " << minWeight << endl); DEBUG(TREE_DBG_LEVEL + 1, "\tTotal example weight, prior to sampling = " << totWeight << endl); DEBUG(TREE_DBG_LEVEL + 1, "\tReplaced example weight = " << replacedWeight << endl); DEBUG(TREE_DBG_LEVEL + 1, "\tSample example weight = " << keepWeight << endl); DEBUG(TREE_DBG_LEVEL + 1, "\tOriginal weight of examples in sample = " << trueWeight << endl); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL + 1, rUsage::resourceUsage() << endl); // shuffle the examples, unless we are in sandbox mode // Do not shuffle in the preSortedFeatures mode, since this doesn't make a difference. if (parameter::randomizedTieBreaking() && !parameter::preSortedFeatures()) std::random_shuffle(sampleEptrs->begin(), sampleEptrs->end()); return; } template ExamplePtrs& Sampler::getHeavyExamples(ExamplePtrs& exVec) { // const { // For M = min(N, nodeSize), find M heaviest examples, with no // more than N/2 having the same yValue, and the set having at // least 2 different yValues. typedef typename ex_t::m_predT pred_t; typedef typename google::sparse_hash_map > yHash_t; static yHash_t labelCounts; labelCounts.clear(); const unsigned nodeSize = exVec.size(); const unsigned idealSampleSize = parameter::heavyExampleSampleSize(); const unsigned sameYLimit = (idealSampleSize + 1) / 2; // Initially set sampleSize to half of what we really want, // because we will double its size again inside the loop. unsigned sampleSize = min(sameYLimit, nodeSize); static ExamplePtrs sampleVec; sampleVec.clear(); static ExWeightComparator weightComp; // partial_sort a progressively larger subsample of exVec, until // it satisfies the above criteria. const typename ExamplePtrs::iterator exVecStart = exVec.begin(); const typename ExamplePtrs::iterator exVecEnd = exVec.end(); typename ExamplePtrs::iterator sortStart = exVecStart; typename ExamplePtrs::iterator exIter; ex_t* exPtr; do { // Double the sample size on every try. sampleSize *= 2; sampleVec.reserve(sampleSize); unsigned iteratorIncrement = min(sampleSize, nodeSize); DEBUG(EX_DBG_LEVEL + 1, "getHeavyExamples() increasing sampleSize to " << iteratorIncrement << endl); const typename ExamplePtrs::iterator sentinel = exVecStart + iteratorIncrement; // Find the sampleSize set of next highest elements in the range // (sortStart, exVecEnd) and sort them into the range // (sortStart, sentinel). exIter = sortStart; // save it before sorting, just in case std::partial_sort(sortStart, sentinel, exVecEnd, weightComp); // Add the next highest elements to the sample, as long as they // don't violate the limit on examples with the same yValue. for(; exIter != sentinel; ++exIter) { exPtr = static_cast(*exIter); pred_t yValue = exPtr->getYValue(); typename yHash_t::iterator yIter = labelCounts.find(yValue); if (yIter == labelCounts.end()) { labelCounts.insert(make_pair(yValue, 0)); } else { unsigned count = yIter->second; if(count >= sameYLimit) continue; } ++labelCounts[yValue]; sampleVec.push_back(exPtr); DEBUG(EX_DBG_LEVEL + 1, "Heavy sample appending " << *exPtr << " with weight " << parameter::pLossFunc->weight(exPtr) << endl); } // Start the next segment to be sorted where the previous one ended. sortStart = sentinel; } while (sampleSize <= nodeSize && (sampleVec.size() < idealSampleSize || labelCounts.size() < 2)); return sampleVec; } } // namespace bdt bdtl-1.0.3/src/bdtl/LossFunction.hpp0000644000175000017500000000161612151020420017074 0ustar melamedmelamed/*! * \file LossFunction.hpp * \brief Class template parameterized by various loss functions as policies. * \todo Inline some of these functions? */ #ifndef _BDTLossFuncH_ #define _BDTLossFuncH_ #ifndef DOXYGEN using namespace std; #endif /* DOXYGEN */ #include "bdtl/LossFunctionBase.hpp" namespace bdt { template class LossFunction : public LossFunctionBase { public: /// Convert a margin to a loss. /// \param margin The margin of the prediction. /// \return The loss of this example. double lossOf(double margin) const; /// Convert an example's margin to a weight, i.e., the rate at /// which the loss decreases as the margin increases. /// \param margin The margin of the example. /// \return The weight of this example. double weightOf(double margin) const; }; // class LossFunction } // namespace bdt #endif /* _BDTLossFuncH_ */ bdtl-1.0.3/src/bdtl/IOHandler.ipp0000644000175000017500000000125012151020420016246 0ustar melamedmelamed/*! * \file IOHandler.ipp */ #include "Vector.hpp" #include "File.hpp" #include "bdtl/DecTree.hpp" #include "bdtl/Ensemble.hpp" namespace bdt { template void IOHandler::ensureLength(unsigned len, Vector& vec, const string& filename) { THROWIF(vec.size() < len, filename << " checkpoint vector length " << vec.size() << " < desired length " << len); if (vec.size() > len) { vec.resize(len); } // needed because if a tree was only partially read in, v.size() could equal len. // we would still want to rewrite it to clear the partial tree. vec.put(filename, "\n"); assert(vec.size() == len); } } // namespace bdt bdtl-1.0.3/src/bdtl/Weights.hpp0000644000175000017500000000375112135440030016067 0ustar melamedmelamed/*! * \file Weights.hpp * \class bdt::Weights * \brief A pair of weights. * * Typically, cumulative (total) Example weight that falls into some DecNode, partitioned by y-value. * The first value is negative y-value weight. * The second value is positive y-value weight. * \note The \b weight of an example is the gradient of the (unpenalized) loss of an Example with respect to its margin. * \note negative y-value weight - positive y-value weight = gradient of (unpenalized) loss over all Examples accumulated * with respect to increasing confidence. * */ #ifndef _BDTWeightsH_ #define _BDTWeightsH_ #include #include "Pair.hpp" #ifndef DOXYGEN using namespace std; #endif /* DOXYGEN */ namespace bdt { class Weights : public Pair { public: typedef Pair _super; Weights(); Weights(const Weights& w); Weights(const _super& w); Weights(const double& neg, const double& pos); Weights operator+(const Weights& w) const; void operator+=(const Weights& w); Weights operator-(const Weights& w) const; /// reset to zeros void clear(); /// Positive weight stored. const double& pos() const; /// Negative weight stored. const double& neg() const; /// Add weight. /// \param w The weight to be added. /// \param yValue Whether we should add positive or negative example weight. void add(const double& w, bool yValue); /// The sum of the positive and negative weight herein. double sum() const; friend ostream& operator<<(ostream &o, const Weights &w) { o << "[ + " << w.pos() << " , - " << w.neg() << " ]"; return o; } friend istream& operator>>(istream &theStream, Weights &w) { string ignore; double pos, neg; theStream >> ignore >> ws >> ignore >> ws // [ + >> pos >> ignore >> ws >> ignore >> neg >> ignore; w = Weights(neg, pos); return theStream; } }; } // namespace bdt #endif /* _BDTWeightsH_ */ bdtl-1.0.3/src/bdtl/Ensemble.cpp0000644000175000017500000006413512151020420016200 0ustar melamedmelamed/*! * \file Ensemble.cpp */ #include "bdtl/Ensemble.hpp" #include "bdtl/DecTree.hpp" #include "bdtl/DecNode.hpp" #include "bdtl/LossFunction.hpp" #include "bdtl/parameter.hpp" #include "File.hpp" #include #include const unsigned int MAXUINT = (unsigned int )-1; #ifndef DOXYGEN using namespace std; #endif /* DOXYGEN */ namespace bdt { Ensemble::Ensemble() : m_key(""), m_nextCFTID(0), m_treeCount(0.), m_useCFTCounts(false) { ctorHelper(); } Ensemble::Ensemble(const string key, const bool useCFTCounts) : m_key(key), m_nextCFTID(0), m_treeCount(0.), m_useCFTCounts(useCFTCounts) { ctorHelper(); } void Ensemble::ctorHelper() { // // An empty CompoundFeatureTest should not be used as the deletedKey, // // because that CompoundFeatureTest appears as the path from the root // // to a leaf in degenerate trees where the root is a leaf. // Feature EMPTYFEAT; const Feature delFeat; CompoundFeatureTest neverUsedCF; neverUsedCF.insert(make_pair(delFeat, true)); // CompoundFeatureTest emptyCF; // emptyCF.insert(make_pair(EMPTYFEATURE, true)); // m_CFTWeights.set_empty_key(emptyCF); m_CFTWeights.set_deleted_key(neverUsedCF); // m_DPSNorms.set_empty_key(emptyCF); m_DPSNorms.set_deleted_key(neverUsedCF); // in order to avoid tripping an assertion in google::dense_hash_map, // we need to set these even if not useOptimizedConfidenceComputation() CompoundFeatureTest emptyCF; emptyCF.insert(make_pair(EMPTYFEATURE, true)); // m_CFTIds.set_empty_key(emptyCF); m_CFTIds.set_deleted_key(neverUsedCF); const int val = 1; const rel_t rt = eq; FeatureTest delFT(delFeat, rt, val); FeatureTest emptyFT(EMPTYFEATURE, rt, val); // m_featureTests.set_empty_key(emptyFT); m_featureTests.set_deleted_key(delFT); if (parameter::useDenseEnsembleOptimizationsDuringConfidenceComputation()) m_trees = vector(); } // this version is for a simple API void Ensemble::initializePredictor(const string& modelDirName, const string& configFileName) { // read params static const int argc = 1 + 2 * 1; // executable name + switch name and argument for each of 1 predefined params char *argv[argc]; Vector args(argc); args[0] = "foo"; if (configFileName == "") { args[1] = "", args[2] = ""; } else { args[1] = "-p", args[2] = configFileName; } for(int i = 0; i < argc; ++i) argv[i] = const_cast(args[i].c_str()); //const BPO::variables_map& vm = parameter::read(argc, argv); parameter::read(argc, argv); // override other sources of this info parameter::hypothesisDirectory() = modelDirName; parameter::getExFormat() = SVMLight; // load ensemble DEBUG(TREE_DBG_LEVEL, "Reading ensemble..."); regularizedLoad("", parameter::minGlobalRegnFactor()); DEBUG(TREE_DBG_LEVEL, " done" << endl); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL + 1, rUsage::resourceUsage() << endl); } void Ensemble::predict(int argc, char **argv) { getPredictParams(argc, argv); if (parameter::labelType() == SCALAR) { predict(); } else { predict(); } } void Ensemble::getPredictParams(int argc, char **argv) { // const BPO::variables_map& vm = parameter::read(argc, argv); parameter::read(argc, argv); // missing exampleFileName means use STDIN // validate // const unsigned vmc = vm.count("exampleFileName"); // assert(vmc); string fn = parameter::externalFeatVocabFileName(); if (fn != "") { iFile fv(fn); // note reference to pointer featVocab_t*& pFV = parameter::externalFeatVocab(); pFV = new featVocab_t; fv >> *parameter::externalFeatVocab(); fv.close(); } } double Ensemble::unpenWeight(const CompoundFeatureTest& f) const { double ret = 0.0; const map_t::const_iterator cfIter = m_unpenCFTWeights.find(f); if(cfIter != m_unpenCFTWeights.end()) { ret = cfIter->second; } return ret; } double Ensemble::weight(const CompoundFeatureTest& f) const { double ret = 0.0; if(parameter::useOptimizedConfidenceComputation()) { const cft2ui_map_t::const_iterator cfIter = m_CFTIds.find(f); if(cfIter == m_CFTIds.end()) { // DEBUG(NODE_DBG_LEVEL + 1, "1 Ensemble::weight returning 0 for " << f << endl); } else { ret = m_CFTIDWeights[cfIter->second]->second; // DEBUG(NODE_DBG_LEVEL + 1, "2 Ensemble::weight returning " << ret << " for " << f << endl); } } else { const map_t::const_iterator cfIter = m_CFTWeights.find(f); if (cfIter == m_CFTWeights.end()) { // DEBUG(NODE_DBG_LEVEL + 1, "1 Ensemble::weight returning 0 for " << f << endl); } else { ret = cfIter->second; // DEBUG(NODE_DBG_LEVEL + 1, "2 Ensemble::weight returning " << ret << " for " << f << endl); } } return ret; } void Ensemble::loadCompiled(const std::string& key) { iFile ci(*parameter::compiledEnsembleName(key)); ci >> *this; // optimizeStorage(); // should be done automatically during the add()'s } // void Ensemble::loadDone(const std::string& key) { // assert(getTreeCount() == 0); // THROWUNLESS(IOHandler::readDoneTraining(key), "Ensemble was not done training! Key: " << key // << ", filename: " << parameter::treeSeqFileName(key).get()); // Vector lastRegSequence; // lastRegSequence.get(*parameter::lastRegSeqFileName(key)); // load(*parameter::treeSeqFileName(key), 0, lastRegSequence.size()); // } void Ensemble::regularizedLoad(const std::string& key, const double minRegn, const double maxRegn, oFile* pDumpFile) { const BSP pLastRegSeqFileName = parameter::lastRegSeqFileName(key); const BSP pTreeSeqFileName = parameter::treeSeqFileName(key); regularizedLoad(*pLastRegSeqFileName, *pTreeSeqFileName, minRegn, maxRegn, pDumpFile); } void Ensemble::regularizedLoad(const string& penaltySeqFileName, const string& treeSeqFileName, const double minRegn, const double maxRegn, oFile* pDumpFile) { assert(getTreeCount() == 0); Vector lastRegSequence; lastRegSequence.get(penaltySeqFileName); unsigned loadFrom = 0; if (maxRegn > 0) { // 0 means don't use it // Find the index of the iteration where we drop below the maxRegn for (; loadFrom < lastRegSequence.size(); ++loadFrom) { if (maxRegn - lastRegSequence[loadFrom] > parameter::confidenceEpsilon()) break; } } // Find the index of the iteration where we drop below the minRegn unsigned loadTo = loadFrom; for (; loadTo < lastRegSequence.size(); ++loadTo) { if (minRegn - lastRegSequence[loadTo] > parameter::confidenceEpsilon()) break; } load(treeSeqFileName, loadFrom, loadTo, pDumpFile); } // Read in the Ensemble from ::filename() \note We duplicate a lot // of code from Tree.ipp::readSequence(const // string& file, vector& v, boost::optional size) // because we save memory keeping only one Tree at a time. void Ensemble::load(const std::string& filename, const unsigned loadFrom, const unsigned loadTo, oFile* pDumpFile) { assert(getTreeCount() == 0); assert(m_CFTWeights.empty()); if (loadTo == 0 || loadTo == loadFrom) return; iFile treeFile(filename); DecTree aTree; unsigned cnt = 0; unsigned prevIter = 0; unsigned currIter; DEBUG(TREE_DBG_LEVEL, "Reading trees from: " << filename << "..." << endl); for(; (cnt < loadFrom) && (! treeFile.eof()); ++cnt) { try { if (NEW_TREE_FILE_FORMAT == true) { treeFile >> currIter >> ws; aTree.read(treeFile); // THROWIF(i.fail(), "Fail bit set on " << file); THROWUNLESS(currIter > prevIter, "Iterations garbled in tree sequence: " << currIter << " follows " << prevIter << endl); prevIter = currIter; } else { aTree.read(treeFile); } } catch (exception& e) { THROW("After reading " << cnt << " trees from " << filename << ", it threw " << e.what()); } } THROWUNLESS(cnt == loadFrom, "Did not read " << loadFrom << " items from " << filename << "; read only " << cnt); for(; (cnt < loadTo) && (! treeFile.eof()); ++cnt) { DEBUG(TREE_DBG_LEVEL + 1, "Reading tree # " << cnt << endl); try { if (NEW_TREE_FILE_FORMAT == true) { treeFile >> currIter >> ws; aTree.read(treeFile); // THROWIF(i.fail(), "Fail bit set on " << file); THROWUNLESS(currIter > prevIter, "Iterations garbled in tree sequence: " << currIter << " follows " << prevIter << endl); prevIter = currIter; } else { aTree.read(treeFile); } if (pDumpFile) *pDumpFile << aTree.str(); } catch (exception& e) { THROW("After reading " << cnt << " trees from " << filename << ", it threw " << e.what()); } // MAIN EVENT add(aTree); } THROWUNLESS(cnt == loadTo, "Did not read " << loadTo << " items from " << filename << "; read only " << cnt); DEBUG(WARNING_DBG_LEVEL + 1, "Read " << cnt << " trees from: " << filename << "\n" << endl); treeFile.close(); // at this point we compress the CFTs into their optimized forms // \todo directly read the CFTs into the optimized structures optimizeStorage(); } void Ensemble::optimizeStorage() { if(parameter::useOptimizedConfidenceComputation()) { // iterate over all the CFTs in m_CFTWeights for(map_t::iterator featIter = m_CFTWeights.begin(); featIter != m_CFTWeights.end(); ++featIter) { CompoundFeatureTest cft = featIter->first; double score = featIter->second; unsigned cftID = getNextCFTID(); DEBUG(NODE_DBG_LEVEL, "Assigned ID " << cftID << " to CFT " << cft << endl); for(CompoundFeatureTest::iterator it = cft.begin(); it != cft.end(); ++it) { FeatureTest ft = *it; if(m_featureTests.find(ft) == m_featureTests.end()) { Vector cfts; cfts.push_back(cftID); m_featureTests.insert(make_pair(ft, cfts)); } else { Vector& cfts = m_featureTests[ft]; cfts.push_back(cftID); } } BSP > sc(new Pair(true, score)); m_CFTIDWeights.push_back(sc); m_CFTIds.insert(make_pair(cft, cftID)); } assert(m_CFTIDWeights.size() == m_CFTIds.size()); m_CFTWeights.clear(); // release the space } } void Ensemble::subtract(const DecTree& tree) { // subtracting is the same as adding the negatives add(tree, -1.0); } void Ensemble::add(const DecTree& tree, double multiplier) { DecTree::nodeContainer_t::const_iterator ni = tree.begin(); const DecTree::nodeContainer_t::const_iterator sentinel = tree.end(); for (; ni != sentinel; ++ni) { DecTree::nodeContainer_t::value_type pNode(*ni); // if this is an internal node and we're not adding internal // nodes, then the conf should not be set. if (! pNode->hasConfidence()) continue; double confDelta = multiplier * pNode->getConfidence(); if (fabs(confDelta) < parameter::confidenceEpsilon()) continue; const CompoundFeatureTest& theCFT = pNode->getCFT(); add(theCFT, confDelta); // not sure how to useDPSNormalizer without Weights in the nodes // if (parameter::useDPSNormalizer()) { // const Weights& wei = pNode->getWeights(); // m_DPSNorms[theCFT] = parameter::pLossFunc->DPSNormalizer(wei); // } if (m_useCFTCounts) { countMap_t::iterator cmci = m_CFTCounts.find(theCFT); if (cmci == m_CFTCounts.end()) m_CFTCounts[theCFT] = 1; else cmci->second += 1; } } m_treeCount += multiplier; DEBUG(TREE_DBG_LEVEL, "in Ensemble::add, CFT count is " << m_CFTIDWeights.size() << endl); } void Ensemble::add(const CompoundFeatureTest& f, double confDelta) { // removed this, to increase the chances that changes will cancel each other out, reducing ensemble size // if (fabs(confDelta) < parameter::confidenceEpsilon()) { // DEBUG(NODE_DBG_LEVEL, "Ensemble::add " << confDelta << " to CFT " << f << "; (short-circuiting)" << endl); // return; // } else { DEBUG(NODE_DBG_LEVEL, "Ensemble::add " << confDelta << " to CFT " << f << "; old = "); // } // if we already have a partial ensemble in optimized form, then // directly increment the scores of the CFT in the optimized data structures if(parameter::useOptimizedConfidenceComputation()) { cft2ui_map_t::iterator cft2idIt = m_CFTIds.find(f); if(cft2idIt != m_CFTIds.end()) { // get the cft id const unsigned cftID = cft2idIt->second; const double storedConf = m_CFTIDWeights[cftID]->second; DEBUG(NODE_DBG_LEVEL, storedConf); double newConf = storedConf + confDelta; if (fabs(newConf) < parameter::confidenceEpsilon()) { // When the CFT's confidence is close to zero, set it to // exactly zero, for consistency with deletion in the // non-optimized confidence computation. It would be more // space-efficient to delete it, but that would throw off // the ID correspondence between m_CFTIds and // m_CFTIDWeights. newConf = 0.; } m_CFTIDWeights[cftID]->second = newConf; m_CFTIDWeights[cftID]->first = true; // this step is redundant, because true is the steady state DEBUG(NODE_DBG_LEVEL, "; new = " << newConf << endl); } else { if (fabs(confDelta) >= parameter::confidenceEpsilon()) { const unsigned cftID = getNextCFTID(); m_CFTIds.insert(make_pair(f, cftID)); BSP > sc(new Pair(true, confDelta)); m_CFTIDWeights.push_back(sc); assert(m_CFTIDWeights.size() == cftID + 1); for(CompoundFeatureTest::const_iterator it = f.begin(); it != f.end(); ++it) { ft2ids_map_t::iterator fIt2 = m_featureTests.find(*it); if(fIt2 == m_featureTests.end()) { Vector cftids; cftids.push_back(cftID); m_featureTests.insert(make_pair(*it, cftids)); } else { Vector& cftids = fIt2->second; cftids.push_back(cftID); } } } else { confDelta = 0.; } DEBUG(NODE_DBG_LEVEL, "0.0; new = " << confDelta << endl); } } else { map_t::iterator featIter = m_CFTWeights.find(f); if (featIter == m_CFTWeights.end()) { if (fabs(confDelta) >= parameter::confidenceEpsilon()) m_CFTWeights.insert(make_pair(f, confDelta)); else confDelta = 0.; DEBUG(NODE_DBG_LEVEL, "0.0; new = " << confDelta << endl); } else { // storedConf is a reference, so that changing it will change the value in the map double& storedConf = featIter->second; DEBUG(NODE_DBG_LEVEL, storedConf); double newConf = storedConf + confDelta; // If adding this confDelta makes the CFT's confidence close // to zero then delete the CFT. if (fabs(newConf) < parameter::confidenceEpsilon()) { m_CFTWeights.erase(featIter); if (parameter::useDPSNormalizer()) { featIter = m_DPSNorms.find(f); if (featIter != m_DPSNorms.end()) m_DPSNorms.erase(featIter); } } else { storedConf = newConf; } DEBUG(NODE_DBG_LEVEL, "; new = " << newConf << endl); } } } void Ensemble::operator+=(const Ensemble& other) { if(parameter::useOptimizedConfidenceComputation()) { // add CFT<-->ID mapping const cft2ui_map_t::const_iterator sentinel = other.m_CFTIds.end(); cft2ui_map_t::const_iterator iter = other.m_CFTIds.begin(); Vector idMap(other.m_CFTIds.size(), 0); // maps other.index to this.index cft2ui_map_t::iterator thisIdIt; for(; iter != sentinel; ++iter) { if((thisIdIt = m_CFTIds.find(iter->first)) != m_CFTIds.end()) { idMap[iter->second] = thisIdIt->second; m_CFTIDWeights[thisIdIt->second]->second += other.m_CFTIDWeights[iter->second]->second; m_CFTIDWeights[thisIdIt->second]->first = true; } else { idMap[iter->second] = getNextCFTID(); m_CFTIds.insert(make_pair(iter->first, idMap[iter->second])); m_CFTIDWeights.push_back(other.m_CFTIDWeights[iter->second]); } } const ft2ids_map_t::const_iterator sentinel2 = other.m_featureTests.end(); ft2ids_map_t::const_iterator iter2 = other.m_featureTests.begin(); // iterate over the other featureTest<-->id mapping for(; iter2 != sentinel2; ++iter2) { const FeatureTest& ft = iter2->first; const Vector& otherCftIds = iter2->second; ft2ids_map_t::iterator thisFtEntry; if((thisFtEntry = m_featureTests.find(ft)) != m_featureTests.end()) { Vector& thisCftIds = thisFtEntry->second; for(Vector::const_iterator it = otherCftIds.begin(); it != otherCftIds.end(); ++it) { thisCftIds.push_back(idMap[*it]); } } else { Vector thisCftIds; for(Vector::const_iterator it = otherCftIds.begin(); it != otherCftIds.end(); ++it) { thisCftIds.push_back(idMap[*it]); } m_featureTests.insert(make_pair(ft, thisCftIds)); } } } else { const map_t::const_iterator sentinel = other.m_CFTWeights.end(); map_t::const_iterator iter = other.m_CFTWeights.begin(); // assert(m_CFTWeights.find(iter->first) == m_CFTWeights.end()); for(; iter != sentinel; ++iter) { if(m_CFTWeights.find(iter->first) == m_CFTWeights.end()) { m_CFTWeights.insert(*iter); } else { m_CFTWeights[iter->first] += iter->second; } } } const map_t::const_iterator sentinel2 = other.m_DPSNorms.end(); map_t::const_iterator iter = other.m_DPSNorms.begin(); for (; iter != sentinel2; ++iter) { m_DPSNorms.insert(*iter); } m_treeCount += other.m_treeCount; } void Ensemble::operator+=(const DecTree& tree) { add(tree); } void Ensemble::operator+=(const vector& trees) { for (vector::const_iterator t = trees.begin(); t != trees.end(); t++) add(*t); } // Return the regularization penalty of this Ensemble, given the // global regularization coefficient double Ensemble::penalty(double globalRegr) const { double p = 0; if(parameter::useOptimizedConfidenceComputation()) { cft2ui_map_t::const_iterator iter = m_CFTIds.begin(); const cft2ui_map_t::const_iterator sentinel = m_CFTIds.end(); for(; iter != sentinel; ++iter) { const CompoundFeatureTest& theCF = iter->first; const unsigned& cftID = iter->second; double confidence = m_CFTIDWeights[cftID]->second; double DPSNorm = getDPSNorm(theCF); p += parameter::pLossFunc->penalize(confidence, DPSNorm); } } else { map_t::const_iterator f = m_CFTWeights.begin(); const map_t::const_iterator sentinel = m_CFTWeights.end(); for (; f != sentinel; ++f) { const CompoundFeatureTest& theCF = f->first; double confidence = f->second; double DPSNorm = getDPSNorm(theCF); p += parameter::pLossFunc->penalize(confidence, DPSNorm); } } return p; } // Return the regularization penalty of this CFT, given the // global regularization coefficient double Ensemble::penalty(const CompoundFeatureTest& theCFT, double globalRegr) const { double ret; double DPSNorm = getDPSNorm(theCFT); if (theCFT.empty()) { ret = parameter::pLossFunc->penalize(0.0, DPSNorm); } else { if(parameter::useOptimizedConfidenceComputation()) { cft2ui_map_t::const_iterator iter = m_CFTIds.find(theCFT); if (iter == m_CFTIds.end()) { // this can happen if the CFT had confidence < confidenceEpsilon, and so got deleted ret = parameter::pLossFunc->penalize(0.0, DPSNorm); } else { const unsigned& cftID = iter->second; double confidence = m_CFTIDWeights[cftID]->second; ret = parameter::pLossFunc->penalize(confidence, DPSNorm); } } else { map_t::const_iterator iter = m_CFTWeights.find(theCFT); if (iter == m_CFTWeights.end()) { // this can happen if the CFT had confidence < confidenceEpsilon, and so got deleted ret = parameter::pLossFunc->penalize(0.0, DPSNorm); } else { double confidence = iter->second; ret = parameter::pLossFunc->penalize(confidence, DPSNorm); } } } return ret; } // This is an additional regularization term in the risk function, // defined as k(Theta_f) * lambda_2 / (b(f) ^ lambda_3) where k() is // the Kroenecker delta function, and b(f) = sum_{i \in I_f} 1. // I.e. b(f) is the count of training examples that satisfy feature // f, which is constant for a given set of examples. Since we want // to penalize each CFT only once, we must check its current value // and its new value. If that value is going from zero to non-zero, // then we add this penalty term. If it's going from non-zero to // zero, then we subtract the penalty term. Otherwise, the penalty // is zero, cuz this feature's penalty has already been accounted // for. Note that we'll never need to compute this quantity when // b(f) < 1. double Ensemble::brittlePenalty(const CompoundFeatureTest& theCFT, const double theCount, double newConf) const { double ret = 0.0; const double currConf = weight(theCFT); // from zero to non-zero if (MA::doubleEpsilonAdditiveEq(currConf, 0.0) && ! MA::doubleEpsilonAdditiveEq(newConf, 0.0) ) { ret = parameter::complexityCoeff() / pow(theCount, parameter::complexityExponent()); // from non-zero to zero } else if (MA::doubleEpsilonAdditiveEq(currConf + newConf, 0.0) && ! MA::doubleEpsilonAdditiveEq(currConf, 0.0)) { ret = - parameter::complexityCoeff() / pow(theCount, parameter::complexityExponent()); } return ret; } // the derivative (w.r.t. Theta_f) of the brittlePenalty is // lambda_2 / (b(f) ^ lambda_3) if the CFT is about to change its // confidence away from zero, and zero otherwise. We ignore the // possibility of it changing back to zero, since it happens rarely // and is expensive to predict at feature selection time. double Ensemble::brittleGradient(const CompoundFeatureTest& theCFT, const double theCount) const { assert(theCount > 0); double ret = 0.0; const double currConf = weight(theCFT); if (MA::doubleEpsilonAdditiveEq(currConf, 0.0)) { ret = parameter::complexityCoeff() / pow(theCount, parameter::complexityExponent()); } return ret; } double Ensemble::getDPSNorm(const CompoundFeatureTest& cf) const { if (! parameter::useDPSNormalizer()) return 1.0; const map_t::const_iterator iter = m_DPSNorms.find(cf); assert(iter != m_DPSNorms.end()); double ret = iter->second; return ret; } void Ensemble::setDPSNorm(const CompoundFeatureTest& cf, const Weights& wei) { double DPSNorm = parameter::pLossFunc->DPSNormalizer(wei); m_DPSNorms.insert(make_pair(cf, DPSNorm)); } void Ensemble::clear() { m_treeCount = 0.; m_CFTWeights.clear(); m_CFTIds.clear(); m_featureTests.clear(); m_CFTIDWeights.clear(); m_nextCFTID = 0; if (m_trees) m_trees.get().clear(); } ostream& operator<<(ostream& out, const Ensemble& theEns) { return theEns.print(out); } ostream& Ensemble::print(ostream& out, const featVocab_t* featVocab) const { // private helper class class dumpCFT_ { public: void operator()(const unsigned theID_, const double theConf_, const CompoundFeatureTest& theCFT_, const featVocab_t* featVocab_, ostream& out_, const bool useCFTCounts_) const { out_ << theConf_ << " "; DEBUG(NODE_DBG_LEVEL, "#" << theID_ << " "); if (useCFTCounts_) { out_ << "( " << theID_ << " ) "; } theCFT_.print(out_, featVocab_); out_ << endl; }; } dumpCFT; DEBUG(TREE_DBG_LEVEL, "Ensemble::print " << m_CFTIDWeights.size() << " CFTs" << endl); out << "" << endl; if(parameter::useOptimizedConfidenceComputation()) { Ensemble::cft2ui_map_t::const_iterator mapci = m_CFTIds.begin(); const Ensemble::cft2ui_map_t::const_iterator sentinel = m_CFTIds.end(); for(; mapci != sentinel; ++mapci) { const CompoundFeatureTest& theCFT = mapci->first; const unsigned theID = mapci->second; const double theConf = m_CFTIDWeights[theID]->second; if (MA::doubleEpsilonAdditiveEq(theConf, 0.0)) { DEBUG(NODE_DBG_LEVEL, "Ensemble::print skipping 0-conf CFT " << theCFT << endl); continue; } dumpCFT(theID, theConf, theCFT, featVocab, out, m_useCFTCounts); } } else { unsigned counter = 0; Ensemble::map_t::const_iterator mapci = m_CFTWeights.begin(); const Ensemble::map_t::const_iterator sentinel = m_CFTWeights.end(); for(; mapci != sentinel; ++mapci) { const CompoundFeatureTest& theCFT = mapci->first; const double theConf = mapci->second; if (MA::doubleEpsilonAdditiveEq(theConf, 0.0)) { DEBUG(NODE_DBG_LEVEL, "Ensemble::print skipping 0-conf CFT " << theCFT << endl); continue; } ++counter; dumpCFT(counter, theConf, theCFT, featVocab, out, m_useCFTCounts); } } if (parameter::useDenseEnsembleOptimizationsDuringConfidenceComputation()) { const vector& theTrees = m_trees.get(); vector::const_iterator tIter = theTrees.begin(); for(; tIter != theTrees.end(); ++tIter) { const DecTree& aTree(*tIter); out << aTree.str() << endl; } } out << "" << endl; return out; } istream& operator>>(istream& inStream, Ensemble& ens) { string delimString, ignore; inStream >> delimString >> ws; assert(delimString == ""); CompoundFeatureTest aCF; double conf; unsigned count = 0; while (inStream.peek() != '<'){ THROWIF(inStream.eof(), "Ensemble end marker missing!" << endl); // compiled ensemble format is like // 0.676758 ( 3 ) 5 5 > 1220 AND 6 6 > 1039.5 inStream >> conf >> ws; // if not m_useCFTCounts, then the ( 3 ) would be absent if (ens.m_useCFTCounts) inStream >> ignore >> ws >> count >> ws >> ignore >> ws; inStream >> aCF; // check for duplicates if(parameter::useOptimizedConfidenceComputation()) { assert(ens.m_CFTIds.find(aCF) == ens.m_CFTIds.end()); } else { assert(ens.m_CFTWeights.find(aCF) == ens.m_CFTWeights.end()); } if (ens.m_useCFTCounts) ens.m_CFTCounts[aCF] = count; ens.add(aCF, conf); } if(parameter::useOptimizedConfidenceComputation()) { count = ens.m_CFTIds.size(); } else { count = ens.m_CFTWeights.size(); } DEBUG(WARNING_DBG_LEVEL + 1, "Ensemble::>> resulted in " << count << " CFTs in Ensemble." << endl); inStream >> delimString; assert(delimString == ""); return inStream; } } // namespace bdt bdtl-1.0.3/src/bdtl/Example.hpp0000644000175000017500000000624312171623344016062 0ustar melamedmelamed/*! * \file Example.hpp * \class bdt::Example * \brief An Abstract training example. */ #ifndef _BDTExampleH__ #define _BDTExampleH__ #include #include #include #include #include #include "Dbug.hpp" #include "UniqueID.hpp" #include "RandomID.hpp" #include "Cursor.hpp" #include "bdtl/globals.hpp" #include "bdtl/Feature.hpp" #include "bdtl/FeatureTest.hpp" #ifndef DOXYGEN using namespace std; #endif /* DOXYGEN */ namespace bdt { class Example : public UniqueID, public RandomID, public Dbug { public: Example(); Example(unsigned partition, fpVal_t weightBias=1.); virtual ~Example() {}; /// GETTERS /// From which partition is this Example derived? unsigned partition() const { return m_partition;} double getConfidence() const { assert(HasConfidence); return Confidence;} double getProvisionalConfidence() const { return m_provisionalConfidence;} double unpenConfidence() const { return m_unpenConfidence;} fpVal_t getWeightBias() const { return m_weightBias; } bool hasConfidence() const { return HasConfidence; } /// SETTERS void setConfidence(const double& conf) {Confidence = conf; HasConfidence = true;} void setProvisionalConfidence(const double& conf) {m_provisionalConfidence = conf; } void setUnpenConfidence(const double& conf) {m_unpenConfidence = conf; } void setWeightBias(const double& weightBias) { m_weightBias = weightBias; } void setPartition(unsigned pa) { m_partition = pa; } typedef list featStringCont_t; virtual bool setFeatures(const featStringCont_t& strs, const char delim = ' ') = 0; virtual bool hasFeature(const Feature& f) const = 0; virtual bool satisfies(const FeatureTest& ft) const = 0; virtual bool satisfies(const Feature& feat, const rel_t& rel, const double& val) const = 0; virtual IFUnion_t lookup(const unsigned& featID) const = 0; virtual IFUnion_t lookup(const Feature& theFeat, const bool isFloat) const = 0; virtual double potential(const double weight) const = 0; virtual BSP> getBinaryFeatCursor() const = 0; virtual BSP>> getScalarFeatCursor() const = 0; virtual void print(ostream&) const = 0; friend ostream& operator<<(ostream& theStream, const Example& ae) { ae.print(theStream); return theStream; } protected: void sanityCheck() const; /// Relative importance of this Example. /// An Example with importance X is treated the same as X Examples of importance 1. fpVal_t m_weightBias; /// The prediction that the model currently makes on this example (or its equiv class). /// \note Unused iff !HasConfidence /// \todo? make this a boost::optional double Confidence; /// Do we actually have this Example's confidence? bool HasConfidence; /// The provisional confidence of this Example //boost::optional m_provisionalConfidence; double m_provisionalConfidence; double m_unpenConfidence; /// m_partition indicates what part of the data the example came from /// E.g., in NLP this might be the sentence (tuple) number. unsigned m_partition; }; } // namespace bdt #endif /* _BDTExampleH_ */ bdtl-1.0.3/src/bdtl/ExVec.cpp0000644000175000017500000000644012151020420015453 0ustar melamedmelamed/*! * \file ExVec.cpp */ #include "bdtl/ExVec.hpp" #include "bdtl/Example.hpp" #include "bdtl/ExBuilder.hpp" #include "bdtl/Weights.hpp" #include "Dbug.hpp" #include "bdtl/parameter.hpp" namespace bdt { ////////////// template implementations for ExamplePtrs void ExamplePtrs::normalize() { assert(parameter::normalizeWeightBiases()); double tot = 0; for (ExamplePtrs::const_iterator e = this->begin(); e != this->end(); e++) tot += (*e)->getWeightBias(); DEBUG(TREE_DBG_LEVEL + 1, "Normalizing " << this->size() << " examples with (unnormalized) total weight bias = " << tot << endl); for (ExamplePtrs::iterator e = this->begin(); e != this->end(); e++) (*e)->setWeightBias((*e)->getWeightBias()/tot); } /* template ExamplePtrs::~ExamplePtrs() { for(ExamplePtrs::iterator egIt = this->begin(); egIt != this->end(); egIt++) { ex_t* example = *egIt; delete example; } }*/ pair, BSP> ExamplePtrs::split(const Feature& f) const { BSP posEptrs(new ExamplePtrs); BSP negEptrs(new ExamplePtrs); for(ExamplePtrs::const_iterator e = this->begin(); e != this->end(); e++) if ((*e)->hasFeature(f)) posEptrs->push_back(*e); else negEptrs->push_back(*e); assert(posEptrs->size() + negEptrs->size() == this->size()); return make_pair(posEptrs, negEptrs); } pair, BSP> ExamplePtrs::split(const FeatureTest& ft) const { BSP posEptrs(new ExamplePtrs); BSP negEptrs(new ExamplePtrs); for(ExamplePtrs::const_iterator e = this->begin(); e != this->end(); e++) { Example* exPtr = *e; if (exPtr->satisfies(ft)) posEptrs->push_back(exPtr); else negEptrs->push_back(exPtr); } assert(posEptrs->size() + negEptrs->size() == this->size()); return make_pair(posEptrs, negEptrs); } void ExamplePtrs::deleteAll() { ExamplePtrs::iterator e = this->begin(); for (; e != this->end(); ++e) delete(*e); _super::clear(); } void ExamplePtrs::addUnpenConfidence(double conf) { for(ExamplePtrs::iterator e = this->begin(); e != this->end(); e++) { double current = (*e)->unpenConfidence(); double newConf = current + conf; (*e)->setUnpenConfidence(newConf); } } void ExamplePtrs::addConfidence(double conf) { for (ExamplePtrs::iterator e = this->begin(); e != this->end(); e++) { double current = (*e)->getConfidence(); double newConf = current + conf; (*e)->setConfidence(newConf); DEBUG(EX_DBG_LEVEL + 1, "Changed conf from " << current << " to " << newConf << " for " << **e << endl); } } void ExamplePtrs::addProvisionalConfidence(double provConf) { for(ExamplePtrs::iterator e = this->begin(); e != this->end(); e++) { Example& theEx(**e); if(theEx.hasConfidence()) { double current = theEx.getConfidence(); double newConf = current + provConf; theEx.setProvisionalConfidence(newConf); DEBUG(EX_DBG_LEVEL + 1, "Changed provConf from " << current << " to " << newConf << " for " << theEx << endl); } else { DEBUG(EX_DBG_LEVEL + 1, "Changed provConf from 0.0 to " << provConf << " for " << theEx << endl); theEx.setProvisionalConfidence(provConf); } } } } // namespace bdt bdtl-1.0.3/src/bdtl/Trainer.cpp0000644000175000017500000006030312151020420016043 0ustar melamedmelamed/*! * \file Trainer.cpp */ #include "bdtl/Trainer.hpp" #include "bdtl/DecTree.hpp" #include "bdtl/DecNode.hpp" #include "bdtl/LossFunction.hpp" #include "bdtl/CompoundExample.hpp" #include "File.hpp" #include "bdtl/parameter.hpp" #ifndef DOXYGEN using namespace std; #endif /* DOXYGEN */ namespace bdt { Trainer::Trainer() : m_pEnsemble(NULL), m_pPartialEnsemble(NULL), m_pPrevIOHandler(NULL), m_pCurrIOHandler(NULL), m_spTrainingExmpls(new ExamplePtrs()) { m_potVV.set_empty_key(-1); m_countVV.set_empty_key(-1); m_decPointVV.set_empty_key(-1); m_includedPotential.set_empty_key(-1); m_includedCount.set_empty_key(-1); } Trainer::~Trainer() { delete m_pEnsemble; delete m_pPartialEnsemble; delete m_pPrevIOHandler; delete m_pCurrIOHandler; } void Trainer::reset() { // We're assuming here that most features are binary, so we don't bother to reduce the vector sizes unsigned featureSetSize = Feature::size(); m_binaryFPotentialVector.clear(); m_binaryFSatCountVector.clear(); m_binaryFPotentialVector.assign(featureSetSize, 0.); m_binaryFSatCountVector.assign(featureSetSize, 0); assert(m_binaryFPotentialVector.size() > 0); size_t theSize = parameter::getScalarFeatureSize(); if (theSize > 0) { // weights of examples that fall in each interval between decision points m_potVV.clear(); // counts of positive/negative examples that satisfy each feature test m_countVV.clear(); const Vector& scalarFeats = parameter::getScalarFeatureList(); for(Vector::const_iterator it = scalarFeats.begin(); it != scalarFeats.end(); ++it) { const unsigned featID = *it; // number of bins will be one more than the number of decPoints theSize = m_decPointVV[featID].size() + 1; pair potVInit(featID, potV_t(theSize, 0.)); m_potVV.insert(potVInit); pair countVInit(featID, countV_t(theSize, 0.)); m_countVV.insert(countVInit); } // marginals of the weights and counts m_includedPotential.clear(); m_includedCount.clear(); m_includedPotential.resize(parameter::getScalarFeatureSize()); m_includedCount.resize(parameter::getScalarFeatureSize()); for(Vector::const_iterator it = scalarFeats.begin(); it != scalarFeats.end(); ++it) { const unsigned featID = *it; pair initPairP = make_pair(featID, 0.); m_includedPotential.insert(initPairP); pair initPairC = make_pair(featID, 0); m_includedCount.insert(initPairC); } } } const po::variables_map& Trainer::getTrainingParams(int argc, char *argv[]) { po::options_description local; const po::variables_map& vm = parameter::read(argc, argv, local); parameter::trainingSanityCheck(); return vm; } void Trainer::train(int argc, char **argv) { getTrainingParams(argc, argv); DEBUG(WARNING_DBG_LEVEL, "\nTraining ensemble." << endl); if (parameter::labelType() == SCALAR) { // HERE'S THE MAIN EVENT trainLoopWrapper(); } else { // HERE'S THE MAIN EVENT trainLoopWrapper(); } DEBUG(WARNING_DBG_LEVEL, "\n\nDONE TRAINING." << endl); } void Trainer::presortOnScalarFeatures(const BSP& trainingEptrs) { //Intialize the datastructures for presorted features. const vector& scalarFeats = parameter::getScalarFeatureList(); unsigned int numFloatFeats = 0, numIntFeats = 0; for(vector::const_iterator it = scalarFeats.begin(); it != scalarFeats.end(); ++it) { unsigned featID = *it; if (parameter::isFloat(featID)) { numFloatFeats++; } else { numIntFeats++; } } if (numIntFeats > 0 ) { m_iFeatOrder.set_empty_key(-1); m_iFeatOrder.clear(); m_iFeatOrder.resize(numIntFeats); m_iFeatOrderBkp.set_empty_key(-1); m_iFeatOrderBkp.clear(); m_iFeatOrderBkp.resize(numIntFeats); } if (numFloatFeats > 0 ) { m_fFeatOrder.set_empty_key(-1); m_fFeatOrder.clear(); m_fFeatOrder.resize(numFloatFeats); m_fFeatOrderBkp.set_empty_key(-1); m_fFeatOrderBkp.clear(); m_fFeatOrderBkp.resize(numFloatFeats); } vector iFeatIndex; vector fFeatIndex; for(vector::const_iterator it = scalarFeats.begin(); it != scalarFeats.end(); ++it) { unsigned featID = *it; //For every scalar feature, sort the feature values and keep track of where the example would end up // in the sorted order. if (parameter::isFloat(featID)) { ExamplePtrs::const_iterator begin = trainingEptrs->begin(); const ExamplePtrs::const_iterator sentinel = trainingEptrs->end(); fFeatIndex.clear(); for (ExamplePtrs::const_iterator exIter =begin; exIter != sentinel; ++exIter) { fuPair tempPair; tempPair.first = (*exIter)->lookup(featID).f; tempPair.second = std::distance(begin, exIter); fFeatIndex.push_back(tempPair); } // iterate through the examples std::sort(fFeatIndex.begin(), fFeatIndex.end()); for(unsigned int itr = 0; itr < fFeatIndex.size(); ++itr) { m_fFeatOrderBkp[featID].push_back(fFeatIndex[itr]); } } else { iFeatIndex.clear(); ExamplePtrs::const_iterator begin = trainingEptrs->begin(); const ExamplePtrs::const_iterator sentinel = trainingEptrs->end(); for (ExamplePtrs::const_iterator exIter = begin; exIter != sentinel; ++exIter) { iuPair tempPair; tempPair.first = (*exIter)->lookup(featID).i; tempPair.second = std::distance(begin, exIter); iFeatIndex.push_back(tempPair); } // iterate through the examples std::sort(iFeatIndex.begin(), iFeatIndex.end()); for(unsigned int itr = 0; itr < iFeatIndex.size(); ++itr) { m_iFeatOrderBkp[featID].push_back(iFeatIndex[itr]); } } } //end iterating through the scalar features } void Trainer::updateFeatureSorts(const FeatureTest& ft, DecNode* pParentNode, DecNode* trueChild, DecNode* falseChild) { //When a decision tree is split, some examples end up on the falseCHild and the rest end up on the true Child. //Here we reorder the data structures m_iFeatOrder and m_fFeatOrder such that the examples in each child node //forms one contiguous block and still maintains the order. unsigned int splitPoint = falseChild->getBuildExamples().size(); //Set both the indices in both the children. trueChild->setFeatOrderEnd(pParentNode->getFeatOrderEnd()); falseChild->setFeatOrderBegin(pParentNode->getFeatOrderBegin()); falseChild->setFeatOrderEnd(pParentNode->getFeatOrderBegin() + splitPoint); trueChild->setFeatOrderBegin(pParentNode->getFeatOrderBegin() + splitPoint); //First get the inverse permutation of the feature on which we did the split. // This gives where an example would end up after the split. ExamplePtrs::const_iterator exIter = pParentNode->getBuildExamples().begin(); ExamplePtrs::const_iterator start = exIter; const ExamplePtrs::const_iterator sentinel = pParentNode->getBuildExamples().end(); vector inversePermutation; inversePermutation.resize(pParentNode->getFeatOrderEnd() - pParentNode->getFeatOrderBegin()); unsigned int lower = 0, higher = splitPoint; for (; exIter != sentinel; ++exIter) { if ((*exIter)->satisfies(ft)) { inversePermutation[ exIter - start ] = higher; ++higher; } else { inversePermutation[ exIter - start ] = lower; ++lower; } } const vector& scalarFeats = parameter::getScalarFeatureList(); //cout << "---Before sorting features---\n" << "split feat " << splitFeatID << "split point " << splitPoint << "\n"; //for (unsigned int itr = pParentNode->getFeatOrderBegin(); itr < pParentNode->getFeatOrderEnd(); ++itr){ // for(vector::const_iterator it = scalarFeats.begin(); it != scalarFeats.end(); ++it) { // unsigned featID = *it; // bool isFloat = parameter::isFloat(featID); // if(isFloat) { // cout << m_fFeatOrder[featID][itr].second << "----" << m_fFeatOrder[featID][itr].first << " "; // } else { // cout << m_iFeatOrder[featID][itr].second << "----" << m_iFeatOrder[featID][itr].first << " "; // } // } // cout << "\n"; // } //} //From the inversePermutation, reorder m_fFeatOrder and m_iFeatOrder for(vector::const_iterator it = scalarFeats.begin(); it != scalarFeats.end(); ++it) { unsigned featID = *it; unsigned int lowerIt=0, higherIt=splitPoint; bool isFloat = parameter::isFloat(featID); if (isFloat) { vector fReOrder; fReOrder.resize(pParentNode->getFeatOrderEnd() - pParentNode->getFeatOrderBegin()); for (unsigned int itr = pParentNode->getFeatOrderBegin(); itr < pParentNode->getFeatOrderEnd(); ++itr){ if (inversePermutation[m_fFeatOrder[featID][itr].second] >= splitPoint) { fReOrder[higherIt].second = inversePermutation[m_fFeatOrder[featID][itr].second] - splitPoint; fReOrder[higherIt].first = m_fFeatOrder[featID][itr].first; higherIt++; } else { fReOrder[lowerIt].second = inversePermutation[m_fFeatOrder[featID][itr].second]; fReOrder[lowerIt].first = m_fFeatOrder[featID][itr].first; lowerIt++; } } for (unsigned int itr = pParentNode->getFeatOrderBegin(); itr < pParentNode->getFeatOrderEnd(); ++itr){ m_fFeatOrder[featID][itr] = fReOrder[itr - pParentNode->getFeatOrderBegin()]; } } else { vector iReOrder; iReOrder.resize(pParentNode->getFeatOrderEnd() - pParentNode->getFeatOrderBegin()); for (unsigned int itr = pParentNode->getFeatOrderBegin(); itr < pParentNode->getFeatOrderEnd(); ++itr){ if (inversePermutation[m_iFeatOrder[featID][itr].second] >= splitPoint) { iReOrder[higherIt].second = inversePermutation[m_iFeatOrder[featID][itr].second] - splitPoint; iReOrder[higherIt].first = m_iFeatOrder[featID][itr].first; higherIt++; } else { iReOrder[lowerIt].second = inversePermutation[m_iFeatOrder[featID][itr].second]; iReOrder[lowerIt].first = m_iFeatOrder[featID][itr].first; lowerIt++; } } for (unsigned int itr = pParentNode->getFeatOrderBegin(); itr < pParentNode->getFeatOrderEnd(); ++itr){ m_iFeatOrder[featID][itr] = iReOrder[itr - pParentNode->getFeatOrderBegin()]; } } } //cout << "\n---After sorting features---\n"; //for (unsigned int itr = pParentNode->getFeatOrderBegin(); itr < pParentNode->getFeatOrderEnd(); ++itr){ // for(vector::const_iterator it = scalarFeats.begin(); it != scalarFeats.end(); ++it) { // unsigned featID = *it; // bool isFloat = parameter::isFloat(featID); // if(isFloat) { // cout << m_fFeatOrder[featID][itr].second << "----" << m_fFeatOrder[featID][itr].first << " "; // } else { // cout << m_iFeatOrder[featID][itr].second << "----" << m_iFeatOrder[featID][itr].first << " "; // } // } // cout << "\n"; //} } double Trainer::featurePenalizedGain(const DecNode& theNode, const FeatureTest& ft, const pot_t truePotential, const pot_t falsePotential, const size_t satCount) const { DEBUG(FEAT_DBG_LEVEL, "Trainer::fPG based on " << truePotential << " and " << falsePotential << endl); unsigned short featureFamily = 0; short polarity = 0; const CompoundFeatureTest& currCFT(theNode.getCFT()); static CompoundFeatureTest trueCFT, falseCFT; if (currCFT.contains(ft)) return 0.0; trueCFT = currCFT + ft; if(parameter::useFeatFamilyPriors()) { featureFamily = trueCFT.getLastFF().first; polarity = trueCFT.getLastFF().second; } const double truePenalizedGain = parameter::pLossFunc->gain(truePotential, m_pEnsemble->weight(trueCFT), featureFamily, polarity); falseCFT = currCFT + ft.negate(); if(parameter::useFeatFamilyPriors()) { featureFamily = falseCFT.getLastFF().first; polarity = falseCFT.getLastFF().second; } const double falsePenalizedGain = parameter::pLossFunc->gain(falsePotential, m_pEnsemble->weight(falseCFT), featureFamily, polarity); // Turian thesis Equation 4.53 (without the max function) double penalizedGain = truePenalizedGain + falsePenalizedGain; const size_t nsatCount = theNode.getBuildExamples().size() - satCount; DEBUG(FEAT_DBG_LEVEL, "Penalized Gain for splitting on " << ft << " is " << penalizedGain << "( " << truePenalizedGain << " + " << falsePenalizedGain << " )" << endl); DEBUG(FEAT_DBG_LEVEL + 1, "+ive child: weights " << truePotential << "; penalized gain " << truePenalizedGain << endl); DEBUG(FEAT_DBG_LEVEL + 1, "+ive child example count: " << nsatCount << endl); DEBUG(FEAT_DBG_LEVEL + 1, "-ive child: weights " << falsePotential << "; penalized gain " << falsePenalizedGain << endl); DEBUG(FEAT_DBG_LEVEL + 1, "-ive child example count: " << satCount << endl); if (parameter::complexityCoeff() != 0.0) { const double count1 = satCount; const double bPenalty1 = m_pEnsemble->brittleGradient(trueCFT, count1); const double count2 = nsatCount; const double bPenalty2 = m_pEnsemble->brittleGradient(falseCFT, count2); penalizedGain -= (bPenalty1 + bPenalty2); DEBUG(FEAT_DBG_LEVEL, "Brittleness penalties are " << bPenalty1 << " and " << bPenalty2 << " ; twice-penalized-gain is " << penalizedGain << endl); } // if randomizedTieBreaking() then we're not in the sandbox, so // don't limit the precision if (parameter::randomizedTieBreaking()) { return penalizedGain; } double roundPenalizedGain = MM::round(penalizedGain, parameter::confidenceEpsilon()); return roundPenalizedGain; } void Trainer::accumulatePotentialOnScalarFeatures(const Example* exPtr, const pot_t& exPotential) { DEBUG(EX_DBG_LEVEL, "::accPotOnSF working on example " << *exPtr << endl); double featVal; static Vector::const_iterator decPointIter; BSP>> spC = exPtr->getScalarFeatCursor(); while(!spC->isDone()) { const pair& thePair = spC->getNext(); const Feature& theFeat = thePair.first; const unsigned featID = theFeat.uid(); const IFUnion_t featValU = thePair.second; const Vector& decPointVec = m_decPointVV[featID]; if (decPointVec.empty()) continue; if (parameter::isFloat(featID)) featVal = featValU.f; else featVal = (double)(featValU.i); // find the bin for this example decPointIter = std::lower_bound(decPointVec.begin(), decPointVec.end(), featVal); // this should return the correct bin even when the value is greater than all the decPoints size_t theBin = std::distance(decPointVec.begin(), decPointIter); // theBin can be zero, so size must be at least one, so comparator is strict < assert(m_potVV[featID].size() > theBin); DEBUG(FEAT_DBG_LEVEL, "feat " << featID << " = " << featVal << " : adding " << exPotential << " to bin " << theBin << " (prev val = " << m_potVV[featID][theBin] << ")" << endl); m_potVV[featID][theBin] += exPotential; m_includedPotential[featID] += exPotential; assert(m_countVV[featID].size() > theBin); ++m_countVV[featID][theBin]; ++m_includedCount[featID]; } } void Trainer::accumulatePotentialOnBinaryFeatures(const Example* exPtr, const pot_t& exPotential) { DEBUG(EX_DBG_LEVEL, "::accPotOnBF working on example " << *exPtr << endl); BSP> spC = exPtr->getBinaryFeatCursor(); while(!spC->isDone()) { const Feature& theFeat = spC->getNext(); const unsigned featID = theFeat.uid(); DEBUG(FEAT_DBG_LEVEL, "feat " << featID << " : adding " << exPotential << " to current potential of " << m_binaryFPotentialVector[featID] << endl); m_binaryFPotentialVector[featID] += exPotential; ++m_binaryFSatCountVector[featID]; } } void Trainer::getBestSplitOnScalarFeature(const DecNode& theNode, bestSplitV_t& bestFTV, pot_t& maxSatPotential, size_t& bestCount, Tiebreak& maxGain, double& maxUnpGain) { pot_t satPotential, nsatPotential; double unpGain; static const rel_t DEFAULT_RELATION = gt; // ">" const unsigned short minEPN = parameter::minExPerNode(); const size_t nodeSize = theNode.size(); const pot_t nodePotential = theNode.getPotential(); size_t satCount, nsatCount; // Compute gain from splitting on scalar valued features, since this splits on each decision point. const vector& scalarFeats = parameter::getScalarFeatureList(); for(vector::const_iterator it = scalarFeats.begin(); it != scalarFeats.end(); ++it) { const unsigned featID = *it; const potV_t& potVec = m_potVV[featID]; //const Vector& countVec = m_countVV[featID]; const countV_t& countVec = m_countVV[featID]; const size_t numDecPoints = m_decPointVV[featID].size(); if (numDecPoints < 1) continue; satPotential = 0.; satCount = 0; for(unsigned cnt = 0; cnt < numDecPoints; ++cnt) { //satCount += countVec[cnt]; satCount = satCount + countVec[cnt]; assert(satCount <= nodeSize); // check that this split wouldn't violate the minExPerNode constraint if (satCount < minEPN) { DEBUG(FEAT_DBG_LEVEL + 1, "Continuing because thisSide = " << satCount << " < minEPN" << endl); continue; } nsatCount = nodeSize - satCount; assert(nsatCount <= nodeSize); if (nsatCount < minEPN) { DEBUG(FEAT_DBG_LEVEL + 1, "Continuing because nsatCount = " << nsatCount << " < minEPN" << endl); continue; } satPotential = satPotential + potVec[cnt]; unpGain = parameter::pLossFunc->unpenGain(satPotential); if (maxUnpGain < unpGain) maxUnpGain = unpGain; // if (MA::doubleEpsilonAdditiveEq(satPotential, 0., parameter::confidenceEpsilon()) { // continue; // } // The example potential that falls in the negative child nsatPotential = nodePotential - satPotential; unpGain = parameter::pLossFunc->unpenGain(nsatPotential); if (maxUnpGain < unpGain) maxUnpGain = unpGain; // if (MA::doubleEpsilonAdditiveEq(nsatPotential, 0., parameter::confidenceEpsilon()) { // continue; // } // construct FeatureTest const Feature& aFeat = Feature::get(featID); const FeatureTest ft = FeatureTest(aFeat, DEFAULT_RELATION, m_decPointVV[featID][cnt]); // find the gain DEBUG(FEAT_DBG_LEVEL + 1, "Computing penalized gain on scalar " << ft << ", " << satPotential << ", " << nsatPotential << ", " << satCount << nsatCount << endl); const double penalizedGain = featurePenalizedGain(theNode, ft, satPotential, nsatPotential, satCount); if (penalizedGain <= 0) { DEBUG(FEAT_DBG_LEVEL + 1, "penalized Gain == " << penalizedGain << "; continuing..." << endl); continue; } Tiebreak twicePenalizedGain(penalizedGain, parameter::randomizedTieBreaking()); DEBUG(FEAT_DBG_LEVEL + 1, "Got twicePenalized gain " << twicePenalizedGain << endl); // Keep this as the best feature if the split's gain beats the // previous best gain. Note that the first maxGain value // includes a small increment to reflect the min additional gain // required for splitting a node, unless this node is the root // node. if (twicePenalizedGain > maxGain && twicePenalizedGain > 0.0) { maxGain = twicePenalizedGain; const Pair, FeatureTest> wft(make_pair(maxGain, ft)); bestFTV.push(wft); DEBUG(FEAT_DBG_LEVEL, "bestFTV is now " << endl << bestFTV); maxSatPotential = satPotential; bestCount = satCount; } else { DEBUG(FEAT_DBG_LEVEL, "Did not use this featureTest because: " << twicePenalizedGain << " <= " << maxGain() << " or " << twicePenalizedGain() << " <= 0.0" << endl); } } } // done iterating thru features } void Trainer::getBestSplitOnBinaryFeature(const DecNode& theNode, bestSplitV_t& bestFTV, double& maxSatPotential, size_t& bestCount, Tiebreak& maxGain, double& maxUnpGain) { const unsigned short minEPN = parameter::minExPerNode(); const size_t nodeSize = theNode.size(); const pot_t nodePotential = theNode.getPotential(); size_t satCount, nsatCount; double satPotential, nsatPotential; double unpGain; for(unsigned featID = 0; featID < m_binaryFPotentialVector.size(); ++featID) { DEBUG(EX_DBG_LEVEL + 1, "Deciding whether to split on " << featID << endl); satCount = m_binaryFSatCountVector[featID]; if(satCount == 0) { DEBUG(FEAT_DBG_LEVEL + 1, "Continuing because satCount == 0" << endl); continue; } // check that this split wouldn't violate the minExPerNode constraint if (satCount < minEPN) { DEBUG(FEAT_DBG_LEVEL + 1, "Continuing because satCount = " << satCount << " < minEPN" << endl); continue; } nsatCount = nodeSize - satCount; if (nsatCount < minEPN) { DEBUG(FEAT_DBG_LEVEL + 1, "Continuing because nsatCount = " << nsatCount << " < minEPN" << endl); continue; } satPotential = m_binaryFPotentialVector[featID]; // if (MA::doubleEpsilonAdditiveEq(satPotential, 0., parameter::confidenceEpsilon()) { // continue; // } unpGain = parameter::pLossFunc->unpenGain(satPotential); DEBUG(FEAT_DBG_LEVEL, "Feature " << featID << " got satCount = " << satCount << " and unpGain " << unpGain << endl); if(maxUnpGain < unpGain) maxUnpGain = unpGain; nsatPotential = nodePotential - satPotential; // if (MA::doubleEpsilonAdditiveEq(nsatPotential, 0., parameter::confidenceEpsilon()) { // continue; // } DEBUG(FEAT_DBG_LEVEL, "Feature " << featID << " got nsatPotential = " << nsatPotential << " = " << nodePotential << " - " << satPotential << endl); unpGain = parameter::pLossFunc->unpenGain(nsatPotential); if(maxUnpGain < unpGain) maxUnpGain = unpGain; const Feature& aFeat = Feature::get(featID); const FeatureTest ft = FeatureTest(aFeat, eq, 1.0); /// MAIN EVENT DEBUG(FEAT_DBG_LEVEL, "Computing penalized gain on binary " << ft << ", " << satPotential << ", " << satCount << endl); const double penalizedGain = featurePenalizedGain(theNode, ft, satPotential, nsatPotential, satCount); if(penalizedGain <= 0) { DEBUG(FEAT_DBG_LEVEL + 1, "penalized Gain == " << penalizedGain << "; continuing..." << endl); continue; } Tiebreak twicePenalizedGain(penalizedGain, parameter::randomizedTieBreaking()); DEBUG(FEAT_DBG_LEVEL + 1, "Got twicePenalized gain " << twicePenalizedGain << endl); // Keep this as the best feature if the split's gain beats the // previous best gain. Note that the first maxGain value // includes a small increment to reflect the min additional gain // required for splitting a node, unless this node is the root // node. if (twicePenalizedGain > maxGain && twicePenalizedGain > 0.0) { maxGain = twicePenalizedGain; const Pair, FeatureTest> wft(make_pair(maxGain, ft)); bestFTV.push(wft); DEBUG(FEAT_DBG_LEVEL, "bestFTV is now " << endl << bestFTV); maxSatPotential = satPotential; bestCount = satCount; } else { DEBUG(FEAT_DBG_LEVEL, "Did not use this featureTest because: " << twicePenalizedGain << " <= " << maxGain() << " or " << twicePenalizedGain() << " <= 0.0" << endl); } } } double Trainer::decreaseRegr(double sulg) const { double oldPenalty = parameter::globalRegnFactor(); double newPenalty; // If we are not penalizing roots and the sulg is non-negative, // then we just induced a degenerate tree that has no other useful // info. So decay the penalty, such that the root can be split // next time. if (sulg >= 0.0 && ! parameter::penalizeRoots()) { DEBUG(TREE_DBG_LEVEL + 1, "SULG > 0" << endl); double newBase = min(oldPenalty, sulg); DEBUG(NODE_DBG_LEVEL, "min(" << oldPenalty << "," << sulg << ") * " << parameter::globalRegnFactorOverscale() << endl); newPenalty = newBase * parameter::globalRegnFactorOverscale(); } else { // ordinary decay newPenalty = oldPenalty * parameter::globalRegnFactorOverscale(); } DEBUG(TREE_DBG_LEVEL + 1, "Decaying penalty from " << oldPenalty << " to " << newPenalty << endl); assert(newPenalty < oldPenalty); parameter::setGlobalRegnFactor(newPenalty); return newPenalty; } } // namespace bdt bdtl-1.0.3/src/bdtl/DecNode.cpp0000644000175000017500000001123312151020420015736 0ustar melamedmelamed/*! * \file DecNode.cpp */ #include // also necessary for boost/numeric/conversion #include #include #include "Dbug.hpp" #include "ResourceUsage.hpp" #include "bdtl/DecNode.hpp" #include "bdtl/parameter.hpp" namespace bdt { using boost::numeric_cast; /// Create a node with no parent. DecNode::DecNode() : _super(), m_CFT(CompoundFeatureTest()) { } /// Create a DecNode with a parent. /// \param newid The NodeID to assign to this new node. /// \param parent The NodeID of the parent of the node about to be constructed. /// \param depth The depth of the node about to be constructed. /// NULL iff this is a root node. /// \param buildEptrs Example%s that fall in this DecNode, used for building. DecNode::DecNode(const NodeID newID, const NodeID parent, const unsigned depth, const CompoundFeatureTest& theCFT, const BSP& buildEptrs, const BSP& trainEptrs) : _super(newID, parent, depth), m_CFT(theCFT), m_steepestUnpenalizedLossGradient(0.0), m_potential(0.), m_buildEptrs(buildEptrs), m_trainEptrs(trainEptrs) { assert(m_buildEptrs); assert(m_trainEptrs); // the first few of these diagnostics are now output in the _super's ctor if (depth != 0) DEBUG(TREE_DBG_LEVEL + 1, "\t"); DEBUG(TREE_DBG_LEVEL + 1, "created DecNode #" << newID << " at depth " << depth); if (parent != NONODE) DEBUG(TREE_DBG_LEVEL + 1, " with parent #" << parent); DEBUG(TREE_DBG_LEVEL + 1, " containing " << getBuildExamples().size() << " build examples, " << getTrainExamples().size() << " training examples" << endl); DEBUG(NODE_DBG_LEVEL, str("\t")); if (parameter::dumpVolatileDiagnostics()) DEBUG(NODE_DBG_LEVEL, rUsage::resourceUsage()); DEBUG(NODE_DBG_LEVEL, endl); } void DecNode::sanityCheck() const { _super::sanityCheck(); if (m_depth == 0) { assert(m_CFT.empty()); } else { assert(!m_CFT.empty()); } } bool DecNode::hasConfidenceAndWeightBiases() const { return hasConfidence(); } // template // double DecNode::computeSteepestUnpenalizedLossGradient(const vector& weightVec) const { // assert(isRoot()); // double maxGain = parameter::pLossFunc->unpenGain(getWeights()); // for (vector::const_iterator i = weightVec.begin(); i != weightVec.end(); i++) { // const Weights& posWeight = *i; // // The example weight that falls in the negative leaf child // Weights negWeight = getWeights() - posWeight; // const double posUnpenGain = parameter::pLossFunc->unpenGain(posWeight); // if (maxGain < posUnpenGain) // maxGain = posUnpenGain; // const double negUnpenGain = parameter::pLossFunc->unpenGain(negWeight); // if (maxGain < negUnpenGain) // maxGain = negUnpenGain; // } // return maxGain; // } void DecNode::freeMemory() { m_buildEptrs.reset(); m_trainEptrs.reset(); DEBUG(NODE_DBG_LEVEL, "Freeing memory for Node #" << id() << endl); if (parameter::dumpVolatileDiagnostics()) DEBUG(NODE_DBG_LEVEL, rUsage::resourceUsage() << endl); } string DecNode::detailedStr(const string s) const { ostringstream o; o << s << "DecNode #" << id() << " ("; if (isRoot()) o << "root, "; else o << "parent #" << parentID() << ", "; if (isLeaf()) { o << "leaf)" << endl; } else { o << "+child #" << getTrueChildID() << ", "; o << "-child #" << getFalseChildID() << ")" << endl; } o << s << "depth = " << depth() << endl; if (isLeaf()) { if (hasConfidence()) { o << s << "confidence=" << getConfidence() << endl; } } else { //assert(m_featureTest != ""); //o << s << "splitFeature = " << thi->getFT()->str() << endl; o << s << "splitFeature = " << getFT() << endl; } return o.str(); } string DecNode::str(const string sep) const { ostringstream o; o.precision(parameter::outputFloatPrecision()); o << sep; if (isLeaf()) { o << "Leaf# " << id(); if (m_confidence) o << " confidence= " << getConfidence(); else o << " confidence= 0.00"; // o << " e.g.-count " << m_buildEptrs->size(); } else { // non-leaf o << "Node# " << id(); if (m_confidence) o << " confidence= " << getConfidence(); else o << " confidence= 0.00"; // o << " ex. count " << m_buildEptrs->size(); o << " +child= " << getTrueChildID(); o << " -child= " << getFalseChildID(); o << " splitOn ( " << getFT() << " )"; } o << " depth= " << depth(); o << endl; return o.str(); } } // namespace bdt bdtl-1.0.3/src/bdtl/ExampleWithY.hpp0000644000175000017500000000303112151020420017017 0ustar melamedmelamed/*! * \file ExampleWithY.hpp * \class bdt::ExampleWithY * \brief A training example with a y-value */ #ifndef _BDTExampleWithYH__ #define _BDTExampleWithYH__ #include "bdtl/globals.hpp" #include "bdtl/Example.hpp" #ifndef DOXYGEN using namespace std; #endif /* DOXYGEN */ namespace bdt { template class ExampleWithY : public Example { public: typedef predict_t m_predT; typedef Example _super; ExampleWithY(); ExampleWithY(unsigned partition, predict_t yValue, fpVal_t weightBias = 1.); /// GETTERS predict_t getYValue() const { assert(m_hasYValue); return m_yValue;} /// \todo REMOVEME bool hasYValue() const { return m_hasYValue; } /// SETTERS void setYValue(predict_t yValue) {m_yValue = yValue; m_hasYValue = true;} double potential(const double weight) const; void print(ostream&) const; friend ostream& operator<<(ostream& theStream, const ExampleWithY& ae) { ae.print(theStream); return theStream; } protected: /// y-value predict_t m_yValue; /// True iff this example has a y-value (yValue = true or false). /// False iff this example has an undefined y-value. (This is used, e.g., when /// we are making new predictions.) bool m_hasYValue; }; template<> double ExampleWithY::potential(const double weight) const; template<> double ExampleWithY::potential(const double weight) const; } // namespace bdt #include "ExampleWithY.ipp" #endif /* _BDTExampleWithYH_ */ bdtl-1.0.3/src/bdtl/CompoundFeatureTest.hpp0000644000175000017500000000742512151020420020412 0ustar melamedmelamed/*! * \file CompoundFeatureTest.hpp * \brief A conjunction (possibly with negations) of atomic Feature%s. * * * \todo Sanity-check that we do not have competing (opposite valued * atomic Feature already in the CompoundFeatureTest). * * */ #ifndef _BDTCompoundFeatureTestH__ #define _BDTCompoundFeatureTestH__ #include "bdtl/Feature.hpp" #include "bdtl/FeatureTest.hpp" #include "SortedVector.hpp" #include #include #include "Vocabulary.hpp" #include "StringWithSpaces.hpp" #include "Pair.hpp" namespace bdt { class CompoundFeatureTest // : public UniqueID { typedef SortedVector map_t; // MUST BE 0-INDEXED FOR IT TO WORK AS A VECTOR INDEX static unsigned int nextCFTID; public: typedef map_t::iterator iterator; typedef map_t::const_iterator const_iterator; bool empty() const { return m_conjuncts.empty(); } void clear() {m_conjuncts.clear(); } /// default constructor CompoundFeatureTest() : m_lastFF(std::make_pair(0, 0)) {} /// Create a new CompoundFeatureTest by adding a conjunct to this. /// We make sure that we are not adding a contradictory or /// duplicatory conjunct. /// Note: expensive cuz it involves copying *this CompoundFeatureTest operator+(const pair& newConjunct) const; CompoundFeatureTest operator+(const FeatureTest& newConjunct) const; static unsigned int getNextID() { return nextCFTID++; } //! Reset CFT ID. Needed because we want the CFT IDs to be reset //! for each ensemble static void resetNextID() { nextCFTID = 0; } /// Apply this CompoundFeatureTest to an Example, and return true iff it holds. template bool operator()(const Ex_t& e) const { map_t::const_iterator i = m_conjuncts.begin(); const map_t::const_iterator sentinel = m_conjuncts.end(); for (; i != sentinel; ++i) { if (! e.satisfies(*i)) return false; } return true; } iterator begin() { return m_conjuncts.begin(); } const_iterator begin() const { return m_conjuncts.begin(); } iterator end() { return m_conjuncts.end(); } const_iterator end() const { return m_conjuncts.end(); } size_t hashValue() const; bool operator==(const CompoundFeatureTest& f) const; ostream& print(ostream& o, const featVocab_t* featVocab = NULL) const; friend ostream& operator<<(ostream &o, const CompoundFeatureTest& f); friend istream& operator>>(istream &theStream, CompoundFeatureTest& theCFT); /// Insert a new conjunct into this CompoundFeatureTest. /// We make sure that we are not adding a contradictory or /// duplicatory conjunct. void insert(const pair& newConjunct); void insert(const FeatureTest& newConjunct); bool contains(const FeatureTest& ft) const; inline const pair& getLastFF() const { return m_lastFF; } private: // data /// Conjunctive elements of the CompoundFeatureTest. /// A map from atomic Feature%s to a pair of (relation, value) /// This is essentially an indexed form of FeatureTest /// If the atomic Feature is not present, its corresponding test is unspecified. map_t m_conjuncts; /// the last family of the feature added to this CFT pair m_lastFF; }; } // namespace bdt #ifndef DOXYGEN namespace HashNS { //! Template specialization of STL's hash object. template<> class hash { public: size_t operator() (const bdt::CompoundFeatureTest& f) const { return f.hashValue(); } }; template<> class hash { public: size_t operator() (const bdt::CompoundFeatureTest& f) const { return f.hashValue(); } }; } #endif #endif /* _BDTCompoundFeatureTestH_ */ bdtl-1.0.3/src/bdtl/ExComparator.hpp0000644000175000017500000000362012151020420017047 0ustar melamedmelamed#ifndef _BDTExCOMPH_ #define _BDTExCOMPH_ #include "bdtl/parameter.hpp" #include "Dbug.hpp" #include "bdtl/Feature.hpp" #include "bdtl/Example.hpp" namespace bdt { /// compares example *pointers* on the value of a given feature class ExFeatComparator { protected: // data Feature m_feat; bool m_isFloat; public: ExFeatComparator(unsigned featID) : m_feat(Feature::get(featID)), m_isFloat(parameter::isFloat(m_feat)) {} ExFeatComparator(unsigned featID, bool isFloat) : m_feat(Feature::get(featID)), m_isFloat(isFloat) {} // allow modifying the comparator after it was constructed, to avoid ctor costs void setFeat(unsigned featID) { m_feat = Feature::get(featID);} void setIsFloat(bool isFloat) { m_isFloat = isFloat; } bool operator()(const Example* exPtr1, const Example* exPtr2) { return featComp(exPtr1->lookup(m_feat, m_isFloat), exPtr2->lookup(m_feat, m_isFloat), exPtr1->getRandomID(), exPtr2->getRandomID()); } bool featComp(const IFUnion_t& val1, const IFUnion_t& val2, unsigned id1, unsigned id2); }; /// compares example *pointers* on their weight /// N.B.: Larger weights come before smaller ones! template class ExWeightComparator { public: bool operator()(const Example* abstractExPtr1, const Example* abstractExPtr2); }; /// compares example *pointers* on their label and weight /// N.B.: Larger weights come before smaller ones! template class ExLabelWeightComparator { public: /// For boolean Examples, if true then label "true" is heavier /// than label "false", and vice versa if false. typename ex_t::m_predT m_greaterLabel; ExWeightComparator m_wcomp; ExLabelWeightComparator(typename ex_t::m_predT greaterLabel) : m_greaterLabel(greaterLabel) {}; bool operator()(const ex_t* abstractExPtr1, const ex_t* abstractExPtr2); }; } #include "ExComparator.ipp" #endif bdtl-1.0.3/src/bdtl/DecNode.hpp0000644000175000017500000002122312301176063015757 0ustar melamedmelamed/*! * \file DecNode.hpp * \class bdt::DecNode * \brief A Node in a DecTree. */ #ifndef _BDTDecNodeH__ #define _BDTDecNodeH__ #include #include #include #include #include "Dbug.hpp" #include "BTNode.hpp" #include "bdtl/parameter.hpp" #include "bdtl/Weights.hpp" #include "bdtl/FeatureTest.hpp" #include "bdtl/CompoundFeatureTest.hpp" #include "bdtl/ExVec.hpp" #ifndef DOXYGEN using namespace std; #endif /* DOXYGEN */ namespace bdt { /// A node in a decision tree class DecNode : public BTNode { typedef BTNode _super; typedef DecNode _self; public: typedef _self* nodePtr_t; DecNode(); DecNode(const NodeID newid, const NodeID parent, const unsigned depth, const CompoundFeatureTest& featureTest, const BSP& buildEptrs = BSP(), const BSP& trainExamples = BSP()); ~DecNode() { DEBUG(EX_DBG_LEVEL, "Deleting node " << m_id << endl); } /// getters inline NodeID getTrueChildID() const {return _super::rightChildID();} inline NodeID getFalseChildID() const {return _super::leftChildID();} // inline const Weights& getWeights() const { return m_totalWeight;} // inline Weights& getWeights() { return m_totalWeight;} inline double getPotential() const {return m_potential; } inline double getPenalizedLoss() const {return m_penalizedLoss; } inline double getUnpenDConf() const { return m_unpenDConf; } inline const FeatureTest& getFT() const { return m_featureTest; } inline const CompoundFeatureTest& getCFT() const {return m_CFT; } inline bool hasConfidence() const { return m_confidence; } inline double getConfidence() const { assert(m_confidence); return m_confidence.get(); } inline bool hasProvisionalConfidence() const { return m_provisionalConfidence; } inline double getProvisionalConfidence() const { assert(m_provisionalConfidence); return m_provisionalConfidence.get(); } inline unsigned int getFeatOrderBegin() const { return m_featOrderBegin; } inline unsigned int getFeatOrderEnd() const { return m_featOrderEnd; } ExamplePtrs& getBuildExamples() const { return *m_buildEptrs; } ExamplePtrs& getTrainExamples() const { return *m_trainEptrs; } /// Find the maximum (unpenalized) gain, either of this node or any /// of its possible children. /// \note Typically we only care about the best unpenalized gain /// at the root, which tells us a good decay schedule for the /// regularization coefficient. /// \pre isRoot() inline double steepestUnpenalizedLossGradient() const { assert(isRoot()); return m_steepestUnpenalizedLossGradient;} /// setters inline void setConfidence(double newConf) {m_confidence = newConf; } inline void setProvisionalConfidence(double newPC) {m_provisionalConfidence = newPC; } inline void setPotential(double newPot) {m_potential = newPot; } inline void setUnpenDConf(double newConf) {m_unpenDConf = newConf; } inline void setPenalizedLoss(double newLoss) {m_penalizedLoss = newLoss; } inline void setSULG(double newSULG) {m_steepestUnpenalizedLossGradient = newSULG; } /// \todo Make sure this isn't called twice. inline void setParentAndCFT(const NodeID parent, const CompoundFeatureTest& theCFT) { m_parentID = parent; m_CFT = theCFT; } inline void setFTandChildren(const FeatureTest& ft, const NodeID trueID, const NodeID falseID) { m_featureTest = ft; m_rightChildID = trueID; m_leftChildID = falseID; } inline void resetFTandChildren() { m_featureTest = FeatureTest(); m_rightChildID = NONODE; m_leftChildID = NONODE; } inline void setFeatOrderBegin(unsigned int FeatOrderBegin) { m_featOrderBegin = FeatOrderBegin; } inline void setFeatOrderEnd(unsigned int FeatOrderEnd) { m_featOrderEnd = FeatOrderEnd; } /// free the memory used by this object void freeMemory(); size_t size() const {return m_buildEptrs->size(); } string detailedStr(const string s="") const; /// Have we set the confidence and weight biases? /// \todo Make private bool hasConfidenceAndWeightBiases() const; /// stringify me string str(const string s="") const; friend ostream& operator<<(ostream &o, const DecNode& theNode) { string ns = theNode.str(); o << ns; return o; } friend istream& operator>>(istream &inStream, DecNode& theNode) { double conf; Weights weights; // unsigned count; string theLine, ignore; char c; inStream.get(c); assert(c == '\t'); // leading tab // getline(inStream, theLine); // cerr << theLine << endl; // istringstream theStream(theLine); if (inStream.peek() == 'L') { // leaf inStream >> ignore >> ws; assert(ignore == "Leaf#"); inStream >> theNode.m_id >> ws; inStream >> ignore >> ws; assert(ignore == "confidence="); inStream >> conf >> ws; theNode.m_confidence = conf; // inStream >> ignore >> ws; // assert(ignore == "ex. count"); // inStream >> count >> ws; // // the count doesn't get stored cuz they are just the size of the build set in the node theNode.m_rightChildID = NONODE; theNode.m_leftChildID = NONODE; } else { // non-leaf inStream >> ignore >> ws; assert(ignore == "Node#"); inStream >> theNode.m_id >> ws; inStream >> ignore >> ws; assert(ignore == "confidence="); inStream >> conf >> ws; theNode.m_confidence = conf; // inStream >> ignore >> ws; // assert(ignore == "ex. count"); // inStream >> count >> ws; // // the count doesn't get stored cuz they are just the size of the build set in the node inStream >> ignore >> ws; assert(ignore == "+child="); inStream >> theNode.m_rightChildID >> ws; inStream >> ignore >> ws; assert(ignore == "-child="); inStream >> theNode.m_leftChildID >> ws; inStream >> ignore >> ws; assert(ignore == "splitOn"); inStream >> ignore >> ws; assert(ignore == "("); inStream >> theNode.m_featureTest >> ws; inStream >> ignore >> ws; assert(ignore == ")"); } inStream >> ignore >> ws; assert(ignore == "depth="); inStream >> theNode.m_depth; // the following 2 lines should be removed when debugging using getline(), cuz it'll eat the newline inStream.get(c); assert(c == '\n'); // node-terminating newline return inStream; } protected: // methods /// Make sure the values of m_depth, m_parentID, and m_CFT all make sense void sanityCheck() const; private: // methods // double computeSteepestUnpenalizedLossGradient(const vector& weight) const; protected: // data /// FeatureTest that splits this node FeatureTest m_featureTest; /// CompoundFeatureTest representing the path from the root to this Node. CompoundFeatureTest m_CFT; /// confidence predicted by this node for any example that falls into it /// also, the change in confidence over the ensemble attributed to this node boost::optional m_confidence; /// provisional confidence, telling us what the confidence /// would be if this node were a leaf boost::optional m_provisionalConfidence; double m_unpenDConf; /// penalized loss over the training examples in this node, at the /// current stage of learning. This quantity is used only to /// validate a postulated split in attemptSplit(), so we might as /// well do that over all the training examples, not just the /// build examples. double m_penalizedLoss; /// The maximum (unpenalized) gain, either of this node or any /// of its possible children. double m_steepestUnpenalizedLossGradient; /// total loss gradient, or maximum possible gain, over the (sample of) examples in this node /// This is the same as Turian's Equation 4.34 without the constraint X_f(i) = 1. double m_potential; ///Every node stores two indices, one for the beginning and one ///for the end of the range in the feature order vector. ///Initially, these two indices point to the beginning and the end /// of the entire sorted feature vector. unsigned int m_featOrderBegin; unsigned int m_featOrderEnd; /// Pointers to the build Examples that fall in this leaf. /// Build examples are used for choosing splits, and are typically /// a sample of the full training Examples. BSP m_buildEptrs; /// Pointers to the training Examples that fall in this leaf. /// Training examples are used for confidence-rating the leaves, /// and their confidences will be updated when the Tree is done /// being built. BSP m_trainEptrs; }; } // namespace bdt #endif /* _BDTDecNodeH_ */ bdtl-1.0.3/src/bdtl/Trainer.hpp0000644000175000017500000002603312151020420016052 0ustar melamedmelamed/*! * \file Trainer.hpp * \class bdt::Trainer * \brief An Trainer of DecTree%s. */ #ifndef _BDTTrainerH__ #define _BDTTrainerH__ #include #include #include #include #include "bdtl/FeatureTest.hpp" #include "bdtl/CompoundFeatureTest.hpp" #include "bdtl/DecTree.hpp" // #include "UniqueID.hpp" #include "Vocabulary.hpp" #include "Tiebreak.hpp" #include "NBest.hpp" #include "StringWithSpaces.hpp" #include "bdtl/ExVec.hpp" #include "bdtl/IOHandler.hpp" #include "bdtl/parameter.hpp" #ifndef DOXYGEN using namespace std; #endif /* DOXYGEN */ namespace bdt { class Ensemble; class DecTree; class DecNode; /// used to store the n-best splits of a node typedef NBest, FeatureTest> > bestSplitV_t; typedef Pair fuPair; typedef Pair iuPair; class Trainer: public Dbug { public: // ctor Trainer(); // dtor ~Trainer(); /// reset the member hashes to prepare for the next split attempt void reset(); /// Wrapper for reading training parameters. /// \note The first three references (parameters) passed to this /// function are overwritten. /// \param argc Command-line argument count. /// \param argv Command-line argument vector. const po::variables_map& getTrainingParams(int argc, char *argv[]); /// Train an ensemble void train(int argc = 0, char **argv = NULL); template void prepareTrainingData(); /// short-cut for when we don't need to keep theEnsemble around template void trainLoopWrapper(string key = ""); /// prevDir is the optional name of a directory where a previous /// ensemble was induced. If supplied, the new ensemble will /// build on the old one. template void trainLoop(const string prevDir = ""); protected: template IOHandler* initializeTraining(const string& prevDir, unsigned& iteration); /// decrease the regularizer (regularization penalty). /// \param sulg is the steepestUnpenalizedLossGradient. /// \return the new regularization penalty double decreaseRegr(double sulg) const; /// get sample to train on, and possibly update sorted feature structure accordingly template void getSample(BSP& sampleEptrs, BSP& trainingEptrs); /// \param partialEnsemble ensemble from previous run /// \param iteration the current iteration, which will be updated by reference as necessary /// \param regn current reg'n penalty /// \param pPrevIOHandler is a pointer to an IOHandler for the previous ensemble template void updateFromPrevEnsemble(unsigned& iteration, double regn); /// Build a DecTree. Side-effect: The confidences of /// trainingExmpls will be updated with respect to this new /// DecTree. template BSP buildTree(BSP trainEptrs, BSP sampleEptrs = BSP(new ExamplePtrs), bool isProvisional = false); template bool handleNewDecLists(const BSP pTree, unsigned& interation); template bool handleNewDecTree(const BSP pTree, unsigned& interation); /// Choose confidences for all leaf nodes. template void chooseLeafConfidences(DecTree& theTree); /// Perform a node split. /// \param theTree which tree are we splitting in? /// \param n The NodeID of the node we are going to split. /// \param ft The feature test with which to split the node. /// \todo Check that none of n's parents have used the same ft /// \return true if the split succeeded template bool attemptSplit(DecTree& theTree, const NodeID n, const FeatureTest& ft); /// create new DecNode template NodeID newNode(DecTree& theTree, const NodeID parent, const unsigned depth, const CompoundFeatureTest& featureTest, const BSP buildExamples, const BSP trainExamples, bool provisional = false); /// Choose the confidence over getTrainExamples() that minimizes penalized loss /// \pre !hasConfidence() /// \post hasConfidence() /// \todo Make private template void setConfidence(DecNode& theNode); /// Set the provisional confidence of the node template void setProvisionalConfidence(DecNode& theNode); /// Find the best splitting feature tests for this node, and, for /// roots only, m_steepestUnpenalizedLossGradient unless they /// already exist. /// \note We might decay the regularization penalty after /// computing the root's steepestUnpenalizedLossGradient template double getBestSplitV(DecNode& theNode, bestSplitV_t& bestFTV, bool provisional = false); /// update the confidences of all Examples in the tree's leaves template void updateTreeExampleConfs(DecTree& theTree); /// update the confidences of all training Examples that fall in this leaf /// possibly update the current sample separately /// \pre hasConfidence() /// \param inverse if true then subtract the node's confidence instead of adding it template void updateSampleAndTrainConfs(DecNode& theNode, const bool inverse = false); template double updateTrainConfs(DecNode& theNode, const bool inverse = false); /// Update the provisional confidences on examples template void updateProvisionalExampleConfs(DecNode& theNode); private: // methods /// compute the gain from a given FeatureTest /// \param theNode /// \param ft /// \param trueWeights weights of examples satisfying the cft at theNode /// \param falseWeights weights of examples not satisfying the cft at theNode /// \param trueCounts counts of pos/neg examples satisfying the cft at theNode double featurePenalizedGain(const DecNode& theNode, const FeatureTest& ft, const pot_t satPotential, const pot_t nsatPotential, const size_t satCounts) const; void presortOnScalarFeatures(const BSP& trainingEptrs); template void getDecisionPointsForScalarFeatures(const DecNode& theNode); /// Call accumulatePotentialOnScalarFeatures and /// accumulatePotentialOnBinaryFeatures once for each example. /// Then, add in the unaccounted-for examples for each scalar /// feature. template void accumulatePotential(const DecNode& theNode, const bool provisional); /// accumulate into m_weightVV and m_countVV, while bookkeeping in m_includedWeights and m_includedCounts void accumulatePotentialOnScalarFeatures(const Example* exPtr, const pot_t& exPotential); /// accumulate into m_binaryFWeightVector and binaryFCountVector void accumulatePotentialOnBinaryFeatures(const Example* exPtr, const pot_t& exPotential); void getBestSplitOnBinaryFeature(const DecNode& theNode, bestSplitV_t& bestFTV, double& maxSatPotential, size_t& bestCount, Tiebreak& maxGain, double& maxUnpGain); void getBestSplitOnScalarFeature(const DecNode& theNode, bestSplitV_t& bestFTV, double& maxSatPotential, size_t& bestCount, Tiebreak& maxGain, double& maxUnpGain); void updateFeatureSorts(const FeatureTest& ft, DecNode* pParentNode, DecNode* trueChild, DecNode* falseChild); template void revertSplit(const NodeID& trueID, const NodeID& falseID, const CompoundFeatureTest& trueCFT, const CompoundFeatureTest& falseCFT, DecTree& theTree, DecNode* pParentNode, DecNode* trueChild, DecNode* falseChild); private: // data /// the ensemble currently being build Ensemble* m_pEnsemble; /// used when restarting a previous run and reading the previous partial ensemble from storage Ensemble* m_pPartialEnsemble; /// counter type when used as hash key typedef const unsigned int cuikey_t; /// data structures used in setting splits : VPH = vector of potentials hash typedef Vector potV_t; typedef Vector countV_t; potV_t m_binaryFPotentialVector; /// number of examples that satisfy each feature test countV_t m_binaryFSatCountVector; typedef google::dense_hash_map, HashNS::hash > dPoint_t; typedef google::dense_hash_map, HashNS::hash >::const_iterator dPointConstIterator_t; typedef google::dense_hash_map, HashNS::hash >::iterator dPointIterator_t; /// vector of vectors of decision points, one vector for each scalar feature dPoint_t m_decPointVV; typedef google::dense_hash_map > VPH_t; typedef google::dense_hash_map > VCH_t; /// potentials of examples that fall in each interval between decision points for scalar features VPH_t m_potVV; /// counts of examples that fall in each interval between decision points for scalar features VCH_t m_countVV; /// Scalar features sometimes have missing values, and we must do /// something about those examples. These hashes are used to /// record, for each scalar feature, how much potential has been /// accounted for. The difference between this number and the /// total potential of the node represents the "missing" /// potential. The current strategy is to assume that the missing /// feature value is the DEFAULT_FEATURE_VALUE, which is typically /// 0. A more sophisticated strategy would be to use the mean or /// median, or to normalize the feature values so that the mean is /// 0. google::dense_hash_map > m_includedPotential; google::dense_hash_map > m_includedCount; // These are for storing the order of features. We need two copies, one serves as a backup and the other is a working copy. typedef google::dense_hash_map, HashNS::hash > iOrder_t; typedef google::dense_hash_map, HashNS::hash >::const_iterator iOrderIterator_t; iOrder_t m_iFeatOrder, m_iFeatOrderBkp; //TODO: in my initial version of the code, I have also included the feature value along with the sorted indices, since the feature values // are redundant, we should get rid off them. typedef google::dense_hash_map, HashNS::hash > f1Order_t; typedef google::dense_hash_map, HashNS::hash >::const_iterator f1OrderIterator_t; f1Order_t m_fFeatOrder, m_fFeatOrderBkp; /// for diagnostics double m_unpenalizedTrainLoss, m_penalizedTrainLoss, m_ensemblePenalty; IOHandler* m_pPrevIOHandler; IOHandler* m_pCurrIOHandler; BSP m_spTrainingExmpls; }; } // namespace bdt #include "bdtl/Trainer.ipp" #endif /* _BDTTrainerH_ */ bdtl-1.0.3/src/bdtl/ExVec.hpp0000644000175000017500000000506112151020420015456 0ustar melamedmelamed/*! * \file ExVec.hpp * \brief Templatized-container of Example%s. * * \todo Unify Examples and ExamplePtrs into one class, which can detect * whether the template parameter type is a pointer or not. */ #ifndef _BDTExVecH__ #define _BDTExVecH__ #include #include #include "Vector.hpp" #include "bdtl/globals.hpp" #include "bdtl/FeatureTest.hpp" #include "bdtl/Weights.hpp" #include "bdtl/Example.hpp" #ifndef DOXYGEN using namespace std; #endif /* DOXYGEN */ namespace bdt { class ExamplePtrs : public Vector, public Dbug { private: boost::optional m_counts; public: // no template typedefs in C++ :( typedef Vector _super; // typedef _super::iterator iterator; // typedef _super::const_iterator const_iterator; // default constructors should suffice // ExamplePtrs() {}; // template // ExamplePtrs(const ExamplePtrs& e); // ~ExamplePtrs(); /// Normalize the weight biases of all Example%s to sum to 1. /// \pre parameter::normalizeWeightBiases() void normalize(); /// delete each example in the vector and then clear the vector itself. void deleteAll(); // /// Compute the total initial Weights of this Example set. // Weights getWeightBias() const; /// Add confidence conf to this Example set. void addConfidence(double conf); void addUnpenConfidence(double conf); /// Add provisional confidence provConf to this Example set void addProvisionalConfidence(double provConf); /// Uncompress every Example. void uncompress() const; /// Split into two ExamplePtrs, the ones that contain Feature f, /// and the ones that don't /// \return A pair, the first ExamplePtrs have Feature f, the /// second ExamplePtrs don't pair, BSP> split(const Feature&) const; /// Split into two ExamplePtrs, the ones that contain Feature f, /// and the ones that don't /// \return A pair, the first ExamplePtrs have Feature f, the /// second ExamplePtrs don't pair, BSP> split(const FeatureTest&) const; //! output friend ostream& operator<<(ostream& out, const ExamplePtrs& aVec){ return aVec.put(out); } virtual ostream& put(ostream& out) const { Vector::const_iterator i = Vector::begin(); for(; i != Vector::end(); ++i) out << **i << endl; return out; } }; } // namespace bdt #endif /* _BDTExVec.hpp_ */ bdtl-1.0.3/src/bdtl/CompoundExample.cpp0000644000175000017500000000561412151020420017543 0ustar melamedmelamed/*! * \file CompoundExample.cpp */ #include "bdtl/CompoundExample.hpp" namespace bdt { template<> string scalarSparseCompoundExample::name() { return "scalarSparseCompoundExample"; } template<> string boolSparseCompoundExample::name() { return "boolSparseCompoundExample"; } ostream& operator<<(ostream& out, const boolSparseCompoundExample& fe) { out << fe.name() << " "; fe.print(out); out << "; m_featSet ="; SparseFeatureSet::const_iterator hashci_a = fe.m_featSet->begin(); const SparseFeatureSet::const_iterator sentinel_a = fe.m_featSet->end(); for(; hashci_a != sentinel_a; ++hashci_a) { out << " " << *hashci_a; } out << "; and m_featMap = "; SparseFeatureMap::const_iterator hashci_b = fe.m_featMap->begin(); const SparseFeatureMap::const_iterator sentinel_b = fe.m_featMap->end(); for(; hashci_b != sentinel_b; ++hashci_b) { const Feature& theFeat = hashci_b->first; out << " " << theFeat << " "; const IFUnion_t& theVal = hashci_b->second; if (parameter::isFloat(theFeat)) out << theVal.f; else out << theVal.i; } return out; } ostream& operator<<(ostream& out, const scalarSparseCompoundExample& fe) { out << fe.name() << " "; fe.print(out); out << "; m_featSet ="; SparseFeatureSet::const_iterator hashci_a = fe.m_featSet->begin(); const SparseFeatureSet::const_iterator sentinel_a = fe.m_featSet->end(); for(; hashci_a != sentinel_a; ++hashci_a) { out << " " << *hashci_a; } out << "; and m_featMap = "; SparseFeatureMap::const_iterator hashci_b = fe.m_featMap->begin(); const SparseFeatureMap::const_iterator sentinel_b = fe.m_featMap->end(); for(; hashci_b != sentinel_b; ++hashci_b) { const Feature& theFeat = hashci_b->first; out << " " << theFeat << " "; const IFUnion_t& theVal = hashci_b->second; if (parameter::isFloat(theFeat)) out << theVal.f; else out << theVal.i; } return out; } // ostream& operator<<(ostream& out, const boolDenseCompoundExample& fe) { // out << fe.name() << " "; // fe.print(out); // out << "; m_featSet ="; // DenseFeatureSet::const_iterator hashci_a = fe.m_featSet->begin(); // const DenseFeatureSet::const_iterator sentinel_a = fe.m_featSet->end(); // for(; hashci_a != sentinel_a; ++hashci_a) { // out << " " << *hashci_a; // } // out << "; and m_featMap = "; // DenseFeatureMap::const_iterator hashci_b = fe.m_featMap->begin(); // const DenseFeatureMap::const_iterator sentinel_b = fe.m_featMap->end(); // for(; hashci_b != sentinel_b; ++hashci_b) { // const Feature& theFeat = hashci_b->first; // out << " " << theFeat << " "; // const IFUnion_t& theVal = hashci_b->second; // if (parameter::isFloat(theFeat)) // out << theVal.f; // else // out << theVal.i; // } // return out; // } } // namespace bdt bdtl-1.0.3/src/bdtl/Feature.hpp0000644000175000017500000000360612151020420016042 0ustar melamedmelamed/*! * \file Feature.hpp */ #ifndef _BDTFeatureH__ #define _BDTFeatureH__ #include "bdtl/globals.hpp" #include "bdtl/Vocab.hpp" #include "Vocabulary.hpp" #include "StringWithSpaces.hpp" #include #include #include namespace bdt { /// union type to use for feature values in examples, which might be /// a combination of integral and floating point typedef union { intVal_t i; fpVal_t f; } IFUnion_t; /// Feature of the Example% typedef string FeatureBase_t; // typedef unsigned FeatureBase_t; typedef Vocab Feature; // used for "external" featVocab's typedef Vocabulary featVocab_t; /// feature types: unknown/unset, float, integral, boolean, multinomial /// M currently unused enum feat_t {U, F, I, B, M}; typedef google::sparse_hash_map > SparseFeatureMap; typedef google::dense_hash_map > DenseFeatureMap; typedef google::sparse_hash_set > SparseFeatureSet; typedef google::dense_hash_set > DenseFeatureSet; /// A distinguished "empty" ID used by google::dense_hash_set. /// NB this is also distinct from ID.empty() extern Feature EMPTYFEATURE; } // namespace bdt namespace HashNS { //! Hash for IFUnion_t: just use it both ways template<> struct hash { size_t operator()(const bdt::IFUnion_t& u) const { return (size_t)(u.i * u.f); } }; template<> struct hash { size_t operator()(const bdt::IFUnion_t& u) const { return (size_t)(u.i * u.f); } }; template<> struct hash { size_t operator()(const bdt::IFUnion_t& u) const { return (size_t)(u.i * u.f); } }; } #endif /* _BDTFeatureH_ */ bdtl-1.0.3/src/bdtl/ExampleWithY.ipp0000644000175000017500000000154512135407673017055 0ustar melamedmelamed/*! * \file ExampleWithY.ipp */ #ifndef _BDTExampleWithYI__ #define _BDTExampleWithYI__ namespace bdt { template ExampleWithY::ExampleWithY() : //_super(-1u, 1.), // let _super initialize w/o params m_hasYValue(false) { sanityCheck(); } template ExampleWithY::ExampleWithY(unsigned partition, predict_t yValue, fpVal_t weightBias) : _super(partition, weightBias), // partition, weightBias m_yValue(yValue), m_hasYValue(true) {} template void ExampleWithY::print(ostream& theStream) const { theStream << getUniqueID() << " (" << getRandomID() << ") "; if (m_hasYValue) theStream << "yValue = " << m_yValue; if (HasConfidence) theStream << "; conf = " << Confidence; theStream << "; wBias = " << m_weightBias; } } // namespace bdt #endif bdtl-1.0.3/src/bdtl/LossFunctionBase.hpp0000644000175000017500000001322712151020420017670 0ustar melamedmelamed/*! * \file LossFunction.hpp * \brief Loss computation methods. * * Methods for computing losses, margins, and example weights. * \note Boosting specific. * * \todo Inline some of these functions? * \todo Rename "loss" to "gain" where applicable, to avoid confusion. */ #ifndef _BDTLossFuncBaseH_ #define _BDTLossFuncBaseH_ #include #include "globals.hpp" #include "bdtl/ExVec.hpp" #ifndef DOXYGEN using namespace std; #endif /* DOXYGEN */ namespace bdt { class Weights; class IOHandler; class LossFunctionBase { public: virtual ~LossFunctionBase() {}; /// Convert a margin to a loss. /// \param margin The margin of the prediction. /// \return The loss of this example. virtual double lossOf(double margin) const = 0; /// Convert an example's margin to a weight. /// \param margin The margin of the example. /// \return The weight of this example. virtual double weightOf(double margin) const = 0; /// Find the margin of a prediction. /// \param confidence The confidence of the prediction. /// \param yValue the correct prediction /// \return The margin of this prediction. /// for classification, we use Turian thesis equation 4.16 double marginOf(const double confidence, const bool yValue) const; /// For regression, use the inner term of the squared difference. /// When we get to other loss functions, this method will need to /// be templatized on the loss function in the LossFunction class /// template. double marginOf(const double confidence, const double yValue) const; /// Determine the per-feature regularization penalty double featprior(double conf, const string& classKey, unsigned short featFamily, short negLab) const; // Determine the regularization penalty associated with // a model confidence and its DPS normalizer double penalize(double confidence, double DPSNorm = 1.) const; // Find the loss of the Example, given a confidence change dconf. // \pre exPtr->hasConfidence() template double loss(const ex_t* exPtr, double dconf = 0.) const; // weight of margin, scaled by weight bias template double weight(const ex_t* exPtr) const; /// Set the WeightBias of an Example. /// If HasConfidence, then set WeightBias such that weight() == weight. /// Otherwise, set WeightBias to weight. template void scaleWeightBias(ex_t* exPtr, const double& newWeight) const; template double provisionalWeight(const ex_t* exPtr) const; template double unpenWeight(const ex_t* exPtr) const; // \pre hasConfidence() template double margin(const ex_t* exPtr) const; template double provisionalMargin(const ex_t* exPtr) const; template double unpenMargin(const ex_t* exPtr) const; template double currentPotential(const ExamplePtrs& exVec, bool provisional = false, bool penalized = true) const; template double unpenalizedLoss(const ExamplePtrs& exVec, double dconf = 0.) const; /// Decrease in loss as we adjust the current parameter value. /// \note If parameter::useUnpenalizedGain(), this function will /// be short-circuited and use unpenGain() double gain(const double unpenGradient, double parameterValue, unsigned short featFamily = 0, short polarity = 0, string classKey = string("def")) const; /// Absolute value of unpenalized loss gradient double unpenGain(const double unpenGradient) const; /// Choose the confidence update that minimizes penalized loss over a set of examples. /// currentConfidence is the current parameter value to be adjusted (and delta'd with) template double bestDConf(const ExamplePtrs& exs, pot_t potential, double currentConfidence, bool penalized, unsigned short featFamily = -1, short polarity = 0, const string& classKey = "def") const; /// method for normalizing the L1 Regularization term from Dudik, Phillips & Schapire (COLT'04). /// beta = beta0 * stddev(featureValue) / sqrt(numberOfExamples) /// where beta0 is the tied parameter optimized by cross-validation (e.g.) /// and stddev(featureValue) / sqrt(numberOfExamples) is the quantity returned by this method. /// Patrick Haffner (p.c., 2007) additionally recommended to lower-bound this quantity by /// beta0 / numberOfExamples ([sic] *NOT* sqrt(numberOfExamples)) double DPSNormalizer(const Weights& wei) const; protected: // the methods that take Weights as parameters pertain only to the old approach to classification used before rev'n 73 double regnTerm(const Weights& wei) const; double lossGradient(const Weights& wei) const; template double lineSearch(const ExamplePtrs& exs, pot_t potential, double currentConfidence, bool penalized, unsigned short featFamily, short polarity, const string& classKey) const; /// Compute the penalized loss over a set of examples for the given change in confidence template double postulatedLoss(const ExamplePtrs& exs, double currentConf, double dconf, bool penalized, unsigned short featFamily = 0, short polarity = 0, string classKey = "def") const; /// Decrease in loss as we adjust the current parameter value. double penalizedGain(const double unpenGradient, double parameterValue) const; double featPriorGain(const string& classKey, double parameterValue, unsigned short featFamily, short sign) const; }; // class } // namespace bdt #include "LossFunctionBase.ipp" #endif /* _BDTLossH_ */ bdtl-1.0.3/src/bdtl/ExBuilder.ipp0000644000175000017500000001066512151020420016336 0ustar melamedmelamed/*! * \file ExBuilder.ipp */ #include "ResourceUsage.hpp" #include "Time.hpp" #include "MoreStrings.hpp" // for split() #include "bdtl/ExBuilder.hpp" #include "bdtl/CompoundExample.hpp" #include "bdtl/parameter.hpp" namespace bdt { template template void ExBuilder::buildExContainer(exCont_t& theContainer, const string& filename, ExampleFormat exFormat) { iFile in(filename); string line, key; unsigned cnt = 0; while(getline(in, line)) { Example* pEx = constructorHelp2(key, line, exFormat, cnt); if (! pEx) { continue; } theContainer.push_back(pEx); ++cnt; } // shuffle the examples if so indicated, unless we are in sandbox mode // Do not shuffle in the preSortedFeatures mode, since this doesn't make a difference. if (parameter::randomizedTieBreaking() && !parameter::preSortedFeatures() && parameter::shuffleExamplesInExBuilder()) std::random_shuffle(theContainer.begin(), theContainer.end()); } template void ExBuilder::constructorHelp3(unsigned exCount) { if (exCount == 0) { DEBUG(WARNING_DBG_LEVEL, "No examples read"); } DEBUG(TREE_DBG_LEVEL + 1, "ExBuilder::ExBuilder() read " << exCount << " examples"); if (parameter::dumpVolatileDiagnostics()) { DEBUG(TREE_DBG_LEVEL + 1, " at " << nsTime::currentTime()); DEBUG(TREE_DBG_LEVEL + 1, endl << rUsage::resourceUsage()); } DEBUG(TREE_DBG_LEVEL + 1, endl); // Normalization is currently disabled. assert(!parameter::normalizeFeatureCounts()); } template Example* ExBuilder::constructorHelp2(string& key, const string& exString, const ExampleFormat exFormat, const unsigned exCount, const keySet_t& ks) { unsigned int partition; typename ex_t::m_predT yValue; double bias; // featDelim is the delimiter between features string featDelim; // nameValueDelim is the delimiter that separates the feature name from the feature value char nameValueDelim; featStringCont_t strs; const bool usingKeySet = (ks.size() > 0); if (exFormat == GenPar) { featDelim = " & "; } else if (exFormat == SVMLight) { featDelim = " "; } else THROW("Unknown example type" << endl); // Split exString into a container of feature strings using the appropriate delimiter // split() is in MoreStrings split(strs, exString, featDelim); const string metaData = strs.front(); if (exFormat == GenPar) { // partition and yValue modified by reference key = readMetaGP(metaData, yValue, partition, bias); //if we are using the keySet, then get only the examples that are relevant if(usingKeySet && ks.find(key) == ks.end()) { return NULL; } nameValueDelim = ' '; } else { // SVMLight format key = ""; partition = 0; readMetaSVML(metaData, yValue); bias = 1.; nameValueDelim = ':'; } ex_t* pEx = new ex_t(partition, yValue, bias); // remove the first (metadata) element, since we already processed it strs.pop_front(); pEx->setFeatures(strs, nameValueDelim); DEBUG(EX_DBG_LEVEL, "Finished reading example #" << exCount << endl); DEBUG(EX_DBG_LEVEL, "Created Ex " << *pEx << " from " << exString << endl); if (exCount % 1000 == 0) { DEBUG(NODE_DBG_LEVEL, "ExBuilder::ExBuilder() read " << exCount << " examples so far..." << endl); } else if (exCount % 10000 == 0) { DEBUG(TREE_DBG_LEVEL + 1, "ExBuilder::ExBuilder() read " << exCount << " examples so far..." << endl); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL + 1, rUsage::resourceUsage() << endl); } return pEx; } template string ExBuilder::readMetaGP(const string& metaData, typename ex_t::m_predT& yValue, unsigned& partition, double& bias){ istringstream iStr(metaData); string s; iStr >> s; readYValueGP(s, yValue); iStr >> s; // at this point, s may be "{" or the example bias if (s == "{") { bias = 1.0; } else { bias = boost::lexical_cast(s); iStr >> s; } iStr >> s; assert(s == "Inf#"); iStr >> s; // the Inf# iStr >> s; assert(s == "ST#"); iStr >> partition; iStr >> s; assert(s == "Key:"); iStr >> s; string ret(s); iStr >> s; // ignore iStr >> s; assert(s == "}"); return ret; } } // namespace bdt bdtl-1.0.3/src/bdtl/IntVocab.hpp0000644000175000017500000000613612151020420016155 0ustar melamedmelamed/*! * \file IntVocab.hpp */ #ifndef _BDTUIVocabH__ #define _BDTUIVocabH__ #include "bdtl/Vocab.hpp" #include "Vector.hpp" //#include "boost/dynamic_bitset.hpp" #ifndef DOXYGEN using namespace std; #endif /* DOXYGEN */ namespace bdt { // typedef this, so that if we want to use a different numeric // type in the future, we won't be confused between the 2 Vocab // template parameters typedef unsigned vocabNum_t; /// \brief more efficient Vocab template specialization for unsigned ints /// specialization of Vocab template class Vocab : boost::totally_ordered > { public: // As of April 2010, dynamic_bitset just doesn't behave for some reason. // typedef boost::dynamic_bitset<> container_t; typedef Vector container_t; Vocab() : m_uid(InvalidID) {}; Vocab(vocabNum_t n) : m_uid(n) { if (sm_currSize < n) { // container is 1-indexed sm_exists.resize(n + 1); // following loop woulid be redundant for boost::dynamic_bitset.resize() // cuz it automatically initializes new slots to false for(unsigned ind = sm_currSize + 1; ind < n; ++ind) { sm_exists[ind] = false; } sm_currSize = n; } // container is 1-indexed // sm_exists.set(n); sm_exists[n] = true; }; /// Get the Vocab item corresponding to some uid. static const Vocab& get(unsigned uid) { assert(sm_exists[uid] == true); static Vocab v; v.m_uid = uid; return v; }; /// Return the vocabNum_t value corresponding to this Vocab item. const vocabNum_t& operator()() const { static vocabNum_t ret; ret = vocabNum_t(m_uid); return ret; }; bool operator<(const Vocab& v) const { return m_uid < v.m_uid; }; bool operator==(const Vocab& v) const { return m_uid == v.m_uid; }; unsigned uid() const { return m_uid; }; static unsigned size() { return sm_currSize; }; static void dumpAll(ostream& out) { out << "Dumping Vocab " << I << endl; for (typename container_t::size_type ind = 0; ind <= sm_currSize; ++ind) if(sm_exists[ind]) { out << ind << " " << ind << endl; } }; friend ostream& operator<<(ostream &o, const Vocab& v) { v.sanityCheck(); o << v(); // << " ( uid = " << v.Uid << " )"; return o; } friend istream& operator>>(istream& i, Vocab& v) { vocabNum_t t; i >> t; v = Vocab(t); return i; } private: void sanityCheck() const { assert(m_uid != InvalidID); }; /// Unique identifier for this Vocab word. unsigned m_uid; static unsigned sm_currSize; static container_t sm_exists; }; template typename Vocab::container_t Vocab::sm_exists = typename Vocab::container_t(); template unsigned Vocab::sm_currSize = vocabNum_t(0); } // namespace bdt #endif /* _BDTUIVocabH_ */ bdtl-1.0.3/src/bdtl/globals.hpp0000644000175000017500000000412212135440030016071 0ustar melamedmelamed/*! * \file globals.hpp * \brief Global variables and typedefs */ #ifndef _BDTGlobalsH_ #define _BDTGlobalsH_ #include "Dbug.hpp" #include "File.hpp" #include #ifndef DOXYGEN using namespace std; #endif /* DOXYGEN */ #include "BoostMacros.hpp" #include "MoreExceptions.hpp" namespace bdt { const unsigned NOVALUE = (unsigned)-1; static const char WARNING_DBG_LEVEL = 1; static const char TREE_DBG_LEVEL = 3; static const char NODE_DBG_LEVEL = 5; static const char EX_DBG_LEVEL = 7; static const char FEAT_DBG_LEVEL = 9; static const char PEDANTIC_DBG_LEVEL = 11; static const bool NEW_TREE_FILE_FORMAT = true; /// feature value to use for examples that don't specify one static const double DEFAULT_FEATURE_VALUE = 0.0; /// Perform *slow* sanity checks that are not particularly critical (or, for that matter, /// likely to fail). /// Even if this is disabled, certain critical sanity checks will be performed. //#define SANITYCHECKS /// Where applicable, use logs for floating point operation. //#define USELOGS enum ExampleFormat {GenPar, SVMLight}; // moved here to avoid circular #include dependency // example label type enum label_t { BOOLEAN = 1, SCALAR = 2 }; // loss function family enum loss_t { EXPONENTIAL = 1, LOGISTIC = 2, SQUARED = 3 }; // regularization scheme enum regn_t { L1, L2 }; /// types to use for memory-intensive data-structures typedef int intVal_t; // can make them short or log-short if they take up too much memory typedef double fpVal_t; // can make them float if they take up too much memory const fpVal_t MINfpVAL = std::numeric_limits::epsilon(); typedef fpVal_t pot_t; typedef unsigned NodeID; /// \todo Cleaner way to define NONODE? Maybe start root node at NodeID 1? const NodeID NONODE = (NodeID)-1; // 0 is the root node // /// + or - 1 // /// \pre i != 0 // template int sign(const T& i, bool assertions=true) { // if (assertions) { // assert(i != 0); // } else { // if (i == 0) return 0; // } // return i > 0 ? +1 : -1; // } } // namespace bdt #endif /* _BDTGlobalsH_ */ bdtl-1.0.3/src/bdtl/Vocab.hpp0000644000175000017500000001176312151020420015504 0ustar melamedmelamed/*! * \file Vocab.hpp * \class bdt::Vocab * \brief An item in a typed vocabulary. * * The T value must be a total ordering. * */ #ifndef _BDTVocabH__ #define _BDTVocabH__ #include "MoreHashes.hpp" #include #include "BoostMacros.hpp" #include #ifndef DOXYGEN using namespace std; #endif /* DOXYGEN */ namespace bdt { const unsigned InvalidID = (unsigned)-1; template class Vocab : boost::totally_ordered > { private: // these typedefs are private cuz they're not needed in some specializations typedef google::sparse_hash_map, unsigned, HashNS::hash >, HashNS::equal_to_pointer>> toUIDT; typedef google::sparse_hash_map, HashNS::hash > fromUIDT; typedef typename fromUIDT::const_iterator const_iterator; public: Vocab(); Vocab(const T& t); // /// \todo REMOVEME // Vocab(unsigned uid) { // Uid = uid; // assert(uid < NextUid); // } // /// In case the user wants to construct Vocab items differently. // /// \param dummy Dummy boolean parameter, to allow compilation. // template Vocab(V v, bool dummy); /// Get the Vocab item corresponding to some uid. static const Vocab& get(unsigned uid); /// Return the T value corresponding to this Vocab item. const T& operator()() const; // /// Lock this vocabulary against further inserts. // static void lock() { assert(!Locked); Locked = true; } // static bool locked() { return Locked; } /// Note that comparisons are done using the uid(), not the actual value bool operator<(const Vocab& v) const; bool operator==(const Vocab& v) const; unsigned uid() const { return Uid; } static unsigned size() { return NextUid; } static ostream& dumpAll(ostream& out) { const_iterator ci = UidTo_t.begin(); const const_iterator sentinel = UidTo_t.end(); out << "Dumping Vocab " << I << endl; for(; ci != sentinel; ++ci) out << ci->first << " " << *(ci->second) << endl; return out; } friend ostream& operator<<(ostream &o, const Vocab& v) { v.sanityCheck(); o << v(); // << " ( uid = " << v.Uid << " )"; return o; } friend istream& operator>>(istream& i, Vocab& v) { T t; i >> t; v = Vocab(t); return i; } private: void sanityCheck() const; /// Unique identifier for this Vocab word. unsigned Uid; static set > All; static toUIDT T_toUid; static fromUIDT UidTo_t; // static bool Locked; ///< Is the vocabulary locked against further insertion? static unsigned NextUid; ///< The next T value to be used by some Vocab item. }; template set > Vocab::All; template typename Vocab::toUIDT Vocab::T_toUid; template typename Vocab::fromUIDT Vocab::UidTo_t; // template bool Vocab::Locked = false; template unsigned Vocab::NextUid = InvalidID+1; template Vocab::Vocab() : Uid(InvalidID) { } template Vocab::Vocab(const T& t) { const BSP tp(new T(t)); typename toUIDT::iterator i = T_toUid.find(tp); if (i == T_toUid.end()) { // assert(!Locked); T_toUid.insert(make_pair(tp, NextUid)); UidTo_t.insert(make_pair(NextUid, tp)); Uid = NextUid; NextUid++; All.insert(*this); assert(T_toUid.size() == UidTo_t.size()); assert(T_toUid.size() == All.size()); } else { assert(i != T_toUid.end()); Uid = i->second; } assert(T_toUid.find(tp)->second == Uid); sanityCheck(); } template void Vocab::sanityCheck() const { assert(Uid != InvalidID); } /// Return the T value corresponding to this Vocab item. template const T& Vocab::operator()() const { assert(UidTo_t.find(Uid) != UidTo_t.end()); return *UidTo_t.find(Uid)->second; } /// Get the Vocab item corresponding to some uid. template const Vocab& Vocab::get(unsigned uid) { static Vocab v; v.Uid = uid; assert(All.find(v) != All.end()); return *(All.find(v)); } template bool Vocab::operator<(const Vocab& v) const { return Uid < v.Uid; } template bool Vocab::operator==(const Vocab& v) const { return Uid == v.Uid; } } // namespace bdt #ifndef DOXYGEN namespace HashNS { //! Template specialization of STL's hash object. template struct hash > { size_t operator() (const bdt::Vocab& v) const { static hash hasher; return hasher(v.uid()); } }; } #endif /* DOXYGEN */ #include "bdtl/IntVocab.hpp" #endif /* _BDTVocabH_ */ bdtl-1.0.3/src/bdtl/IOHandler.cpp0000644000175000017500000001557212301176063016270 0ustar melamedmelamed#include "bdtl/IOHandler.hpp" #include "File.hpp" #include "bdtl/parameter.hpp" #include namespace bdt { IOHandler::IOHandler(string key, IOHandler* pNextHandler) : m_key(key), m_is_open(false), m_pNextHandler(pNextHandler), treeSequence(NULL), regrSequence(NULL), treeSeqFileName(*parameter::treeSeqFileName(key)), regrSeqFileName(*parameter::lastRegSeqFileName(key)), m_seqMarker(0) {} IOHandler::~IOHandler() { delete treeSequence; delete regrSequence; } void IOHandler::open(ios_base::openmode mode) { unsigned prec = parameter::outputFloatPrecision(); treeSequence = new oFile(treeSeqFileName, mode, prec); regrSequence = new oFile(regrSeqFileName, mode, prec); m_is_open = true; } void IOHandler::close() { THROWUNLESS(m_is_open, "IOHandler: attempted to use streams before opening"); treeSequence->close(); regrSequence->close(); m_is_open = false; } void IOHandler::flush() { THROWUNLESS(m_is_open, "IOHandler: attempted to use streams before opening"); treeSequence->flush(); regrSequence->flush(); } /// advance sequence marker past the given reg'r void IOHandler::advanceSeqMarker(double regr) { for (; m_seqMarker < regrVec.size(); ++m_seqMarker) { if (regrVec[m_seqMarker] < regr) break; } } void IOHandler::write(unsigned iteration, const DecTree& tree, double regnFactor) { THROWUNLESS(m_is_open, "IOHandler: attempted to use streams before opening"); const string& treeStr = tree.str(); if (NEW_TREE_FILE_FORMAT == true) { *treeSequence << iteration << " " << treeStr; } else { *treeSequence << treeStr; } THROWIF(treeSequence->fail(), "Fail bit set on file " << treeSeqFileName); *regrSequence << regnFactor << endl; THROWIF(regrSequence->fail(), "Fail bit set on file " << regrSeqFileName); } void IOHandler::getEnsemble(Ensemble*& ret, unsigned& iteration, double newMinRegr, bool updateIt) { // Find the index of the iteration where we drop below the newMinRegr unsigned loadTo = m_seqMarker; for (; loadTo < regrVec.size(); ++loadTo) { if (regrVec[loadTo] < newMinRegr) break; } unsigned skipped = 0; for(unsigned cnt = m_seqMarker; cnt < loadTo; ++cnt) { const DecTree& tree = treeVec[cnt]; if(tree.empty()) { ++skipped; continue; } if (updateIt) (*ret) += tree; ++iteration; // if we have a handle for the next handler :^, then use it to dump if (m_pNextHandler != NULL) { m_pNextHandler->write(iteration, tree, regrVec[cnt]); } } DEBUG(TREE_DBG_LEVEL, "Read in " << loadTo - m_seqMarker << " tree(s); wrote out " << loadTo - m_seqMarker - skipped << " of them." << endl); // remember how far we got m_seqMarker = loadTo; } void IOHandler::read() { assert(treeVec.empty()); try { regrVec.get(regrSeqFileName); } catch (exception& e) { DEBUG(TREE_DBG_LEVEL, "Unable to open " << regrSeqFileName << "; assuming empty"); regrVec.clear(); } unsigned regrVecSize = regrVec.size(); DEBUG(TREE_DBG_LEVEL, "Read in regularizer sequence of length " << regrVecSize << endl); treeVec.resize(regrVecSize); unsigned prevIter = 0; unsigned currIter = 0; unsigned treesRead = 0; bool needToRewriteTrees = false; bool needToRewriteRegr = false; iFile i(treeSeqFileName); THROWUNLESS(i.is_open(), "Could not open file for reading: " << treeSeqFileName); DecTree t; i >> ws; for(unsigned index = 0; index < regrVecSize; ++index) { if (i.eof()) { DEBUG(TREE_DBG_LEVEL, "Warning: tree sequence had only " << index << " trees, but regr sequence had " << regrVecSize << " entries." << endl); needToRewriteRegr = true; break; } try { if (NEW_TREE_FILE_FORMAT == true) { i >> currIter >> ws; treeVec[index].read(i); // THROWIF(i.fail(), "Fail bit set on " << file); if (currIter <= prevIter) { DEBUG(TREE_DBG_LEVEL, "Iterations garbled in treeSequence: " << currIter << " follows " << prevIter << endl); break; // leave the loop, without adding the last (garbled) tree to the ensemble } prevIter = currIter; treesRead = currIter; } else { treeVec[index].read(i); // THROWIF(i.fail(), "Fail bit set on " << file); treesRead = currIter; } i >> ws; DEBUG(TREE_DBG_LEVEL, "Finished reading tree # " << treesRead << endl); } catch (exception& e) { DEBUG(TREE_DBG_LEVEL, "After reading " << treesRead << " trees from " << treeSeqFileName << ", it stopped on " << e.what()); needToRewriteTrees = true; break; // leave the loop, without adding the last (garbled) tree to the ensemble } } DEBUG(TREE_DBG_LEVEL, "Read in tree sequence of length " << treesRead << endl); if (! i.eof()) { DEBUG(TREE_DBG_LEVEL, "Warning: regr sequence had " << regrVecSize << " entries, but tree sequence had more; truncating." << endl); needToRewriteTrees = true; } if (needToRewriteTrees ) { string tmpTreeSeqFileName = treeSeqFileName + ".tmp"; oFile oF(tmpTreeSeqFileName, ios_base::out, parameter::outputFloatPrecision()); THROWUNLESS(oF.is_open(), "Could not open file for writing: " << tmpTreeSeqFileName); currIter = 0; for(currIter = 0; currIter < treesRead; ++currIter) { const string& treeString(treeVec[currIter].str()); oF << currIter + 1 << " " << treeString; } std::rename(tmpTreeSeqFileName.c_str(), treeSeqFileName.c_str()); DEBUG(TREE_DBG_LEVEL, "Copied over " << currIter << " trees." << endl); } if (needToRewriteRegr) { string tmpRegrSeqFileName = regrSeqFileName + ".tmp"; ensureLength(treesRead, regrVec, tmpRegrSeqFileName); std::rename(tmpRegrSeqFileName.c_str(), regrSeqFileName.c_str()); } m_seqMarker = 0; } // Read the done training file // \return True iff the done training file exists (and contains "DONE") bool IOHandler::readDoneTraining(const string& key) { iFile inF(*parameter::doneTrainingFileName(key)); string s; inF >> s; assert(s == "DONE"); return true; } // write "DONE" to doneTraining // \param why String describing why we stopped. // \post IOHandler::readDoneTraining(key) == true void IOHandler::writeDone(const string& why) { DEBUG(TREE_DBG_LEVEL, "Stopping training: " << why << endl); if (parameter::dumpVolatileDiagnostics()) DEBUG(TREE_DBG_LEVEL + 1, rUsage::resourceUsage() << endl); oFile wd(*parameter::doneTrainingFileName(m_key)); THROWUNLESS(is_open(), "doneFile not open for writing"); wd << "DONE" << endl; THROWIF(wd.fail(), "Fail bit set"); wd.close(); assert(IOHandler::readDoneTraining(m_key)); } /// write the key to a file called 'key' void IOHandler::writeKey() { const string keyFileName = parameter::hypothesisDirectory() + "/key"; oFile keyFile(keyFileName); keyFile << m_key << endl; keyFile.close(); } } // namespace bdt bdtl-1.0.3/src/bdtl/CompoundExample.hpp0000644000175000017500000001243412151020420017546 0ustar melamedmelamed/*! * \class bdt::CompoundExample * * \brief An Example with two uncompressed Feature containers. One * feature container can be for one feature type, and the other * feature container for another. This was coded with usage in mind * to have one feature container for binary features and another for * scalar valued features. This allows for savings in memory, as well * as in time for some feature types. */ #ifndef _BDTCompoundExampleH__ #define _BDTCompoundExampleH__ #include #include "bdtl/ExampleWithY.hpp" #include "bdtl/globals.hpp" #include "bdtl/Feature.hpp" #include "bdtl/FeatureTest.hpp" #include "BoostMacros.hpp" namespace bdt { template class CompoundExample; typedef CompoundExample boolSparseCompoundExample; typedef CompoundExample boolDenseCompoundExample; typedef CompoundExample scalarSparseCompoundExample; typedef CompoundExample scalarDenseCompoundExample; template class CompoundExample : public ExampleWithY { public: // typedef Example _super; typedef fHashSet_t hashT_A; // needed for untemplatized reference to hash types typedef fHashMap_t hashT_B; // needed for untemplatized reference to hash types typedef list featStringCont_t; CompoundExample(unsigned partition = 0, predict_t yValue = predict_t(0), fpVal_t weightBias = 1.0); template CompoundExample(const fHashSet2T& fset_a, const fHashMap2T& fset_b, unsigned partition = 0, predict_t yValue = predict_t(0), fpVal_t weightBias = 1.0); template bool setFeatures(const fHashSet2T& fset_a, const fHashMap2T& fset_b); bool setFeatures(const featStringCont_t& strs, const char delim = ' '); // bool satisfies(const BinaryFeatureTest& ft) const; // bool satisfies(const ScalarFeatureTest& ft) const; bool satisfies(const FeatureTest& ft) const; bool satisfies(const Feature& feat, const rel_t& rel, const double& val) const; static bool satisfies(const typename fHashSet_t::const_iterator& fci_a, const typename fHashMap_t::const_iterator& fci_b, const typename fHashSet_t::const_iterator& sentinel_a, const rel_t& rel, const double& val); bool hasFeature(const Feature& f) const; // bool hasFeature(const FeatureTest& f) const; // bool hasFeature(const Feature* f) const; IFUnion_t lookup(const unsigned& featID) const; IFUnion_t lookup(const Feature& theFeat, const bool isFloat) const; /// concrete cursor over binary feature collection class binaryFeatCursor : public Cursor { protected: typename fHashSet_t::const_iterator m_fIter; const typename fHashSet_t::const_iterator m_sentinel; public: binaryFeatCursor(hashT_A& theHash) : m_fIter(theHash.begin()), m_sentinel(theHash.end()) {} bool isDone() { return m_fIter == m_sentinel; } const Feature& getNext() { return *m_fIter++; } }; /// concrete cursor over scalar feature collection class scalarFeatCursor : public Cursor { protected: typename fHashMap_t::const_iterator m_fIter; const typename fHashMap_t::const_iterator m_sentinel; public: scalarFeatCursor(hashT_B& theHash) : m_fIter(theHash.begin()), m_sentinel(theHash.end()) {} bool isDone() { return m_fIter == m_sentinel; } const typename fHashMap_t::value_type& getNext() { return *m_fIter++; } }; virtual BSP> getBinaryFeatCursor() const { BSP spBFC(new binaryFeatCursor(*m_featSet)); BSP> ret = static_pointer_cast>(spBFC); return ret; } virtual BSP> getScalarFeatCursor() const { BSP spSFC(new scalarFeatCursor(*m_featMap)); BSP> ret = static_pointer_cast>(spSFC); return ret; } static string name(); /// Uncompress the feature vector. /// Since the Example is already uncompressed, this method does nothing. void uncompress() const {} void clear(); friend ostream& operator<< (ostream& out, const CompoundExample& fe); protected: // methods bool setAhasFeature(const Feature& f) const; bool setBhasFeature(const Feature& f) const; bool satisfiesSet(const Feature& theFeat, const rel_t& rel, const double& testVal) const; static bool satisfiesSet(const rel_t& theRel, const double& val); bool satisfiesMap(const Feature& theFeat, const rel_t& rel, const double& testVal) const; static bool satisfiesMap(const typename fHashMap_t::const_iterator& fci, const rel_t& theRel, const double& val); void constructHelp(); protected: // data BSP m_featSet; BSP m_featMap; }; } // namespace bdt #include "CompoundExample.ipp" #endif /* _BDTCompoundExampleH_ */ bdtl-1.0.3/src/bdtl/Example.cpp0000644000175000017500000000110212151020420016022 0ustar melamedmelamed/*! * \file Example.cpp */ #include "bdtl/Example.hpp" namespace bdt { Example::Example() : m_weightBias(1.), Confidence(0.), HasConfidence(true), m_provisionalConfidence(0.), m_unpenConfidence(0.), m_partition(-1u) { sanityCheck(); } Example::Example(unsigned partition, fpVal_t weightBias) : m_weightBias(weightBias), Confidence(0.), HasConfidence(true), m_provisionalConfidence(0.), m_unpenConfidence(0.), m_partition(partition) { sanityCheck(); } void Example::sanityCheck() const { assert(m_weightBias > 0); assert(m_partition != -1u); } } // namespace bdt bdtl-1.0.3/src/bdtl/Weights.cpp0000644000175000017500000000247612151020420016060 0ustar melamedmelamed/*! * \file Weights.cpp */ #include "bdtl/Weights.hpp" namespace bdt { Weights::Weights() : _super(0.0,0.0) { } Weights::Weights(const Weights& w) : _super(w.neg(), w.pos()) { } Weights::Weights(const _super& w) : _super(w) { } Weights::Weights(const double& neg, const double& pos) : _super(neg, pos) { } void Weights::clear() { this->first = 0.0; this->second = 0.0; } Weights Weights::operator+(const Weights& w) const { return Weights(this->first + w.first, this->second + w.second); } void Weights::operator+=(const Weights& w) { first += w.first; second += w.second; } Weights Weights::operator-(const Weights& w) const { return Weights(this->first - w.first, this->second - w.second); } // Positive weight stored. const double& Weights::pos() const { return this->second; } // Negative weight stored. const double& Weights::neg() const { return this->first; } // Add weight. // \param w The weight to be added. // \param yValue Whether we should add positive or negative example weight. void Weights::add(const double& w, bool yValue) { if (yValue) this->second += w; else this->first += w; } // The sum of the positive and negative weight herein. double Weights::sum() const { return pos() + neg(); } } // namespace bdt bdtl-1.0.3/src/bdtl/CompoundFeatureTest.cpp0000644000175000017500000001275712171623344020431 0ustar melamedmelamed/*! * \file CompoundFeatureTest.cpp */ #include "bdtl/CompoundFeatureTest.hpp" #include "Dbug.hpp" #include "bdtl/globals.hpp" #include "bdtl/parameter.hpp" #include #include #include "BoostMacros.hpp" #include // for lower_bound namespace bdt { unsigned CompoundFeatureTest::nextCFTID = 0; // Can these operate in place so that we don't have to make copies? CompoundFeatureTest CompoundFeatureTest::operator+(const FeatureTest& newConjunct) const { CompoundFeatureTest f(*this); f.insert(newConjunct); return f; } CompoundFeatureTest CompoundFeatureTest::operator+(const pair& newConjunct) const { CompoundFeatureTest f(*this); f.insert(newConjunct); return f; } void CompoundFeatureTest::insert(const pair& newConjunct) { static const unsigned TRUE = 1; static const unsigned FALSE = 0; int theVal = newConjunct.second ? TRUE : FALSE; const FeatureTest theFT(newConjunct.first, eq, theVal); assert(! contains(theFT)); m_conjuncts.insert(theFT); /*if(parameter::featFamilyOutBuffer() || parameter::useFeatFamilyPriors()) { unsigned short f = parameter::getFeatFamily(boost::lexical_cast(theFT.m_feature())); short neg = (theFT.m_rel == ne || theFT.m_rel == le || !newConjunct.second) ? -1 : 1; m_lastFF = std::make_pair(f, neg); }*/ } void CompoundFeatureTest::insert(const FeatureTest& newConjunct) { if(parameter::featFamilyOutBuffer() || parameter::useFeatFamilyPriors()) { unsigned short f = parameter::getFeatFamily(boost::lexical_cast(newConjunct.m_feature())); short neg = (newConjunct.m_rel == ne || newConjunct.m_rel == le) ? -1 : 1; m_lastFF = std::make_pair(f, neg); } // DEBUG(EX_DBG_LEVEL, "CFT::insert of " << newConjunct << " into " << *this << endl); if (! m_conjuncts.empty()) { assert(! contains(newConjunct)); // check to see if this featureTest implies another or vice versa rel_t theRel = newConjunct.m_rel; if (theRel == le || theRel == lt || theRel == ge || theRel == gt) { iterator iter = lower_bound(m_conjuncts.begin(), m_conjuncts.end(), newConjunct); bool foundSameFeature = false; // depending on the feature test value, the relevant feature might be at the iterator, or before if (iter != m_conjuncts.end() && (iter->m_feature == newConjunct.m_feature) && (iter->m_rel == theRel)) { foundSameFeature = true; } else { if (iter != m_conjuncts.begin()) { --iter; if ((iter->m_feature == newConjunct.m_feature) && (iter->m_rel == theRel)) { foundSameFeature = true; } } } if (foundSameFeature) { // feature test with same feature and rel already exists, so combine it with the new one double theVal = newConjunct.m_val; double currVal = iter->m_val; if (theRel == le || theRel == lt) { if (currVal > theVal) iter->m_val = theVal; } else { // (theRel == ge || theRel == gt) if (currVal < theVal) iter->m_val = theVal; } return; } } } m_conjuncts.insert(newConjunct); } bool CompoundFeatureTest::contains(const FeatureTest& ft) const { // bool ret = ! (m_conjuncts.find(ft) == m_conjuncts.end()); bool ret = std::binary_search(m_conjuncts.begin(), m_conjuncts.end(), ft); return ret; } bool CompoundFeatureTest::operator==(const CompoundFeatureTest& other) const { //DEBUG(NODE_DBG_LEVEL, "CFT== comparing " << *this << " with " << other << endl); if (other.m_conjuncts.size() != m_conjuncts.size()) return false; // compare the tests by traversing both compounds simultaneously // this works only because the container is a set, which is ordered map_t::const_iterator iter1 = m_conjuncts.begin(); const map_t::const_iterator sentinel = m_conjuncts.end(); map_t::const_iterator iter2 = other.m_conjuncts.begin(); for (; iter1 != sentinel; ++iter1, ++iter2) { if (! (*iter1 == *iter2)) return false; } return true; } size_t CompoundFeatureTest::hashValue() const { size_t h = 0; for (map_t::const_iterator i = m_conjuncts.begin(); i != m_conjuncts.end(); i++) { const FeatureTest& theFT(*i); boost::hash_combine(h, theFT.hashValue()); } DEBUG(NODE_DBG_LEVEL, "CFT::hashValue got " << h << " for " << *this << endl); return h; } ostream& operator<<(ostream& o, const CompoundFeatureTest& f) { f.print(o); return o; } ostream& CompoundFeatureTest::print(ostream& o, const featVocab_t* featVocab) const { if(empty()) { o << ""; return o; } bool firstFeature = true; CompoundFeatureTest::map_t::const_iterator i = m_conjuncts.begin(); for (; i != m_conjuncts.end(); i++) { if (!firstFeature) o << " AND "; i->print(o,featVocab); firstFeature = false; } return o; } istream& operator>>(istream& theStream, CompoundFeatureTest& theCFT) { theCFT.clear(); string line, atomicFeatString; Feature feat; getline(theStream, line); if (line == "") return theStream; vector cftvec; split(cftvec, line, " AND "); vector::const_iterator vsci = cftvec.begin(); vector::const_iterator sentinel = cftvec.end(); for(; vsci != sentinel; ++vsci) { istringstream iss(*vsci); FeatureTest anFT; iss >> anFT; theCFT.insert(anFT); } return theStream; } } // namespace bdt bdtl-1.0.3/src/bdtl/DecTree.hpp0000644000175000017500000001515212301176063015775 0ustar melamedmelamed/*! * \file DecTree.hpp * \class bdt::DecTree * \brief A Decision Tree * * It contains DecNode%s, and keeps track of the frontier * (i.e.\ DecNode%s that might still be split.) * */ #ifndef _BDTDecTreeH__ #define _BDTDecTreeH__ #include "Vector.hpp" #include #include "bdtl/globals.hpp" #include "bdtl/DecNode.hpp" #ifndef DOXYGEN using namespace std; #endif /* DOXYGEN */ namespace bdt { class DecTree : public Dbug { public: typedef DecNode::nodePtr_t nodePtr_t; typedef Vector nodeContainer_t; typedef nodeContainer_t::const_iterator const_iterator; typedef nodeContainer_t::iterator iterator; typedef boost::unordered_set frontier_t; /// ctors /// Create an empty tree. DecTree(); /// No copy construction: doing so would require explicitly copying all the nodes thru their bare pointers, which is expensive. Instead, try to avoid the need for it. E.g., avoid ops that might re-allocate vectors of DecTrees. DecTree(const DecTree& t) { THROW("DecTree does not allow copy construction -- see code comments for reasons."); } ~DecTree() { // explicit clean-up of container of bare pointers iterator iter = getNodes().begin(); const const_iterator sentinel = getNodes().end(); for(; iter != sentinel; ++iter) delete(*iter); } // iterators const_iterator begin() const { return m_nodes.begin(); } iterator begin() { return m_nodes.begin(); } const_iterator end() const { return m_nodes.end(); } iterator end() { return m_nodes.end();} void clear(); /// Is this a degenerate Decision Tree? /// \return True if all leaves have confidence 0. bool empty() const { return m_nodes.empty(); } unsigned int size() const { return m_nodes.size(); } /// getters const nodeContainer_t& getNodes() const { return m_nodes;} nodeContainer_t& getNodes() { return m_nodes;} const frontier_t& getFrontier() const { return m_frontier;} frontier_t& getFrontier() { return m_frontier;} const nodeContainer_t& leaves() const; inline double getRootGain() const { return m_rootGain; } double sulg() const { return m_steepestUnpenalizedLossGradient; } nodePtr_t& rootNode() { return m_nodes.at(m_rootNode); } const nodePtr_t& rootNode() const { return m_nodes.at(m_rootNode); } const NodeID getRootID() const { return m_rootNode; } void setRootID(NodeID id) { m_rootNode = id; } nodePtr_t& getNode(NodeID id) { return m_nodes.at(id); } const nodePtr_t& getNode(NodeID id) const { return m_nodes.at(id); } /// setters void setRootGain(double newGain) {m_rootGain = newGain; } void setSULGFromRoot() { m_steepestUnpenalizedLossGradient = rootNode()->steepestUnpenalizedLossGradient(); } /// Find the confidence of some ex_t. /// \param e An ex_t. /// \return The confidence of that example. template double confidence(const ex_t& e) const; /// True iff this DecTree is an unsplit root. bool isJustRoot() const; /// If we were to add the confidence at each leaf to the training set Examples, /// what would be the loss? template double postulatedTrainingSetLoss() const; istream& read(istream &theStream); /// Convert a tree to a string. string str(const string s="") const; /// Convert a tree to a string. string detailedStr(const string s="") const; void insert(const nodePtr_t pNode) { m_nodes.push_back(pNode); m_frontier.insert(pNode->id()); } /// Remove NodeID n from Tree. /// \note This only works if n is the LAST node in the DecTree. void pruneNode(const NodeID n); void removeFromFrontier(const NodeID n); // template friend ostream& operator<< (ostream &out, const DecTree& t) { THROW("Templatized I/O operators don't compile under gcc-4.4.5 -- use str() and read() instead"); /** typename nodeContainer_t::const_iterator nodeIt = t.begin(); const typename nodeContainer_t::const_iterator sentinel = t.end(); out << "BeginTree" << endl; for (; nodeIt != sentinel; ++nodeIt) { const DecNode* pNode = static_cast(*nodeIt); out << "\t" << pNode->str(); } out << "EndTree" << endl; */ return out; } // template friend istream& operator>> (istream &theStream, DecTree& theTree) { THROW("Templatized I/O operators don't compile under gcc-4.4.5 -- use str() and read() instead"); /** theTree.clear(); string ignore; getline(theStream, ignore); THROWUNLESS(ignore == "BeginTree", "last line is: " << ignore); nodeContainer_t& nodeVec = theTree.m_nodes; char c = theStream.peek(); while (c == '\t') { // Each node is on a tab-indented line DecNode* newNodePtr(new DecNode); theStream >> (*newNodePtr); nodeVec.push_back(newNodePtr); c = theStream.peek(); } getline(theStream, ignore); THROWUNLESS(ignore == "EndTree", "last line is: " << ignore); // eat up trailing newlines c = theStream.peek(); while (c == '\n') { getline(theStream, ignore); c = theStream.peek(); } // Add parents and features for all nodes typename nodeContainer_t::iterator nIter = nodeVec.begin(); const typename nodeContainer_t::iterator sentinel = nodeVec.end(); unsigned i; for (i=0; nIter != sentinel; ++nIter, ++i) { const DecNode* pNode = static_cast(*nIter); const DecNode& theNode = *pNode; assert(theNode.id() == i); assert(nodeVec.at(i)->id() == i); if (! theNode.isLeaf()) { const FeatureTest posFT(theNode.getFT()); NodeID trueChild = theNode.getTrueChildID(); assert(nodeVec[trueChild]->parentID() == NONODE); // make sure it's not set yet CompoundFeatureTest cfpos(theNode.getCFT() + posFT); DecNode* tNode = static_cast(nodeVec[trueChild]); tNode->setParentAndCFT(i, cfpos); const FeatureTest negFT = posFT.negate(); NodeID falseChild = theNode.getFalseChildID(); assert(nodeVec[falseChild]->parentID() == NONODE); // make sure it's not set yet CompoundFeatureTest cfneg(theNode.getCFT() + negFT); DecNode* fNode = static_cast(nodeVec[falseChild]); fNode->setParentAndCFT(i, cfneg); } } */ return theStream; } private: // data /// NodeID of the root Node. NodeID m_rootNode; nodeContainer_t m_nodes; frontier_t m_frontier; double m_steepestUnpenalizedLossGradient; double m_rootGain; }; } // namespace bdt #include "bdtl/DecTree.ipp" #endif /* _BDTDecTreeH_ */ bdtl-1.0.3/src/bdtl/trainOne.cpp0000644000175000017500000000033012151020420016210 0ustar melamedmelamed/*! * \file trainOne.cpp * \brief Train one ensemble */ #include "bdtl/Trainer.hpp" using namespace bdt; int main(int argc, char **argv) { Trainer theTrainer; theTrainer.train(argc, argv); return 0; } bdtl-1.0.3/src/bdtl/FeatureTest.cpp0000644000175000017500000001067512151020420016701 0ustar melamedmelamed/*! * \file FeatureTest.cpp */ #include "bdtl/FeatureTest.hpp" #include "bdtl/globals.hpp" #include "bdtl/parameter.hpp" #include #include "MoreHashes.hpp" #include "MoreMath.hpp" namespace bdt { FeatureTest::FeatureTest(const Feature& f, const rel_t& r, const IFUnion_t& v) : m_feature(f), m_rel(r) { if (parameter::isFloat(f)) m_val = v.f; else if(parameter::isInteger(f)) m_val = (double)v.i; else m_val = 1.0; } FeatureTest::FeatureTest(const Feature& f, const rel_t& r, const int& i) : m_feature(f), m_rel(r), m_val(i) {} FeatureTest::FeatureTest(const Feature& f, const rel_t& r, const double& d) : m_feature(f), m_rel(r), m_val(d) {} bool FeatureTest::operator<(const FeatureTest& other) const { if (m_feature < other.m_feature) { return true; } else if (other.m_feature < m_feature) { return false; } if (m_rel < other.m_rel) { return true; } else if (other.m_rel < m_rel) { return false; } if (other.m_val - m_val > parameter::featValEpsilon()) { return true; } return false; } bool FeatureTest::operator==(const FeatureTest& other) const { if (m_feature != other.m_feature) return false; if (m_rel != other.m_rel) return false; if (MA::doubleEpsilonAdditiveEq(m_val, other.m_val, parameter::featValEpsilon())) return true; return false; } size_t FeatureTest::hashValue() const { static HashNS::hash floatHasher; size_t h = 0; const unsigned id = m_feature.uid(); boost::hash_combine(h, id); boost::hash_combine(h, static_cast(m_rel)); if(! parameter::isBinary(m_feature)) { // don't let precision errors lead to different hash values for the same floats const fpVal_t approxVal = MM::round(m_val, parameter::featValEpsilon()); const size_t stableVal = floatHasher(approxVal); boost::hash_combine(h, stableVal); } return h; } FeatureTest FeatureTest::negate() const { FeatureTest ret(*this); switch (m_rel) { case eq : ret.m_rel = ne; break; case ne : ret.m_rel = eq; break; case lt : ret.m_rel = ge; break; case gt : ret.m_rel = le; break; case le : ret.m_rel = gt; break; case ge : ret.m_rel = lt; break; default : THROW("Unknown relation type"); } return ret; } ostream& operator<<(ostream& o, const FeatureTest& ft) { ft.print(o); return o; } ostream& FeatureTest::print(ostream& o, const featVocab_t* featVocab) const { featVocab_t* FVPtr = const_cast(featVocab); if (FVPtr != NULL) { const FeatureBase_t& fb(m_feature()); const unsigned int GPFeatID = lexical_cast(fb); const string* featString( featVocab->getWord(GPFeatID)); o << *featString; } else { FVPtr = parameter::externalFeatVocab(); if (FVPtr != NULL) { const FeatureBase_t& fb(m_feature()); const VocabularyIndex aVI = lexical_cast(fb); const featVocab_t::word_t* featString( FVPtr->getWord(aVI)); o << *featString; } else { o << m_feature; } } switch (m_rel) { case eq : o << " == "; break; case ne : o << " != "; break; case lt : o << " < "; break; case gt : o << " > "; break; case le : o << " <= "; break; case ge : o << " >= "; break; default : THROW("Unknown relation type"); } o << m_val; return o; } istream& operator>>(istream& theStream, FeatureTest& theFT) { Feature theFeat; string relS, valS; theStream >> theFeat >> ws >> relS >> ws >> valS; theFT.m_feature = theFeat; rel_t theRel; if (relS == "==") { theRel = eq; } else if (relS == "!=") { theRel = ne; } else if (relS == "<") { theRel = lt; } else if (relS == ">") { theRel = gt; } else if (relS == ">=") { theRel = ge; } else if (relS == "<=") { theRel = le; } else { std::cerr << "relS = " << relS << std::endl; THROW("unrecognized relation"); } theFT.m_rel = theRel; double theVal = lexical_cast(valS); theFT.m_val = theVal; // the following code is incorrect, because even an int feature can have a non-int decision point // double isFloat; // // if there's a decimal point in the value then make it a float, o.w. an int // if (valS.find('.') == string::npos) { // isFloat = false; // } else { // isFloat = true; // } // parameter::setFloat(theFeat, isFloat); return theStream; } } // namespace bdt bdtl-1.0.3/src/bdtl/Feature.cpp0000644000175000017500000000024112151020420016025 0ustar melamedmelamed/*! * \file Feature.cpp */ #include "bdtl/Feature.hpp" #include "bdtl/globals.hpp" namespace bdt { Feature EMPTYFEATURE = Feature(); } // namespace bdt bdtl-1.0.3/src/bdtl/testOne.cpp0000644000175000017500000000043512151020420016060 0ustar melamedmelamed/*! * \file testOne.cpp * \brief classify examples using one Ensemble */ #include "bdtl/Ensemble.hpp" using namespace bdt; int main(int argc, char **argv) { ios::sync_with_stdio(false); // Do not keep C and C++ streams in sync Ensemble::predict(argc, argv); return 0; } bdtl-1.0.3/src/bdtl/ExBuilder.cpp0000644000175000017500000000202112151020420016313 0ustar melamedmelamed/*! * \file ExBuilder.cpp */ #include "ExBuilder.hpp" #include "BoostMacros.hpp" #include "bdtl/CompoundExample.hpp" namespace bdt { template<> void ExBuilder::readYValueGP(const string& yString, bool& yValue){ if (yString == "+"){ yValue = true; } else if (yString == "-"){ yValue = false; } else THROW("Unrecognized y-Value!"); } template<> void ExBuilder::readYValueGP(const string& yString, fpVal_t& yValue){ yValue = lexical_cast(yString); } template<> void ExBuilder::readMetaSVML(const string& metaData, fpVal_t& yValue){ yValue = lexical_cast(metaData); } template<> void ExBuilder::readMetaSVML(const string& metaData, bool& yValue){ if (metaData == "1") { yValue = true; } else if (metaData == "0") { yValue = false; } else THROW("Invalid example polarity symbol " << metaData); } } // namespace bdt bdtl-1.0.3/src/bdtl/PrioritizedExPtr.hpp0000644000175000017500000000313212122405072017740 0ustar melamedmelamed#ifndef _BDTPrioExPtr_ #define _BDTPrioExPtr_ #include #include "parameter.hpp" namespace bdt { template class PrioritizedExPtr { public: PrioritizedExPtr() : Empty(true) { } // \todo Use a Random type for random numbers. PrioritizedExPtr(const ex_t* ePtr) : Empty(false), ExamplePtr(ePtr) { const double rand = drand48(); Priority = parameter::pLossFunc->weight(ExamplePtr) / rand; }; bool operator<(const PrioritizedExPtr& p) const { return priority() < p.priority(); } bool operator>(const PrioritizedExPtr& p) const { return priority() > p.priority(); } bool operator==(const PrioritizedExPtr& p) const { return priority() == p.priority(); } bool operator!=(const PrioritizedExPtr& p) const { return priority() != p.priority(); } const double& priority() const { assert(!empty()); return Priority; } // const double& trueWeight() const { assert(!empty()); return TrueWeight; } const ex_t* examplePtr() const { assert(!empty()); return ExamplePtr; } bool empty() const { return Empty; } // We assume that all objects are distinct i.e. // that we cannot detect if the two Example%s are equal. bool objectEqualTo(const PrioritizedExPtr& p) const { //assert(example() != p.example()); return false; } friend ostream& operator<<(ostream &o, const PrioritizedExPtr& prioEx) { o << prioEx.Priority << " " << *(prioEx.ExamplePtr) << endl; return o; } private: bool Empty; double Priority; // double TrueWeight; const ex_t* ExamplePtr; }; } // namespace bdt #endif bdtl-1.0.3/src/bdtl/FeatureTest.hpp0000644000175000017500000000472412151020420016704 0ustar melamedmelamed#ifndef _BDTFeatureTestH__ #define _BDTFeatureTestH__ #include "bdtl/Feature.hpp" #include "Pair.hpp" #include "Vocabulary.hpp" #include "StringWithSpaces.hpp" //#include "HashMacros.hpp" namespace bdt { /// A FeatureTest is a structure that represents a feature test, /// comprising a feature (name/number), a numeric feature value, and /// a relationship to test on that feature value. The feature value /// is always floatingPoint, since even scalar features whose values /// are always integers can be tested for being less/greater than /// various non-integer values. E.g., for 0/1 features, test on f > 0.5. /// relation type: =, !=, <, >, <=, >= /// We're still modeling binary features using this featuretest format; /// If we want to test whether a feature exists (i.e. a binary feature is true) /// we're testing for "f == 1.0"; if we're testing for whether a feature does not /// exist, we are testing for "f == 0.0" (DAG) enum rel_t { eq, ne, lt, gt, le, ge }; class FeatureTest { public: // data Feature m_feature; rel_t m_rel; double m_val; // methods FeatureTest() {}; FeatureTest(const Feature& f, const rel_t& r, const IFUnion_t& v); FeatureTest(const Feature& f, const rel_t& r, const int& i); FeatureTest(const Feature& f, const rel_t& r, const double& d); bool operator<(const FeatureTest& other) const; bool operator==(const FeatureTest& other) const; size_t hashValue() const; // construct a feature test that is the logical opposite of this one FeatureTest negate() const; ostream& print(ostream& o, const featVocab_t* featVocab = NULL) const; friend ostream& operator<<(ostream &o, const FeatureTest& ft); friend istream& operator>>(istream &theStream, FeatureTest& theFT); }; } // namespace bdt namespace HashNS { //! Hash for rel_t: just convert to unsigned template<> struct hash { size_t operator()(const bdt::rel_t& r) const { return (size_t)(r); } }; template<> struct hash { size_t operator()(const bdt::rel_t& r) const { return (size_t)(r); } }; //! Hash for FeatureTest template<> struct hash { size_t operator()(const bdt::FeatureTest& ft) const { return ft.hashValue(); } }; template<> struct hash { size_t operator()(const bdt::FeatureTest& ft) const { return ft.hashValue(); } }; } #endif /* _BDTFeatureTestH_ */ bdtl-1.0.3/src/bdtl/SetExample.hpp0000644000175000017500000000735012151020420016516 0ustar melamedmelamed/*! * \class bdt::SetExample * \brief An Example with a fully uncompressed Feature vector. */ #ifndef _BDTSetExampleH__ #define _BDTSetExampleH__ #include #include "bdtl/ExampleWithY.hpp" #include "bdtl/globals.hpp" #include "bdtl/Feature.hpp" #include "bdtl/FeatureTest.hpp" #include "BoostMacros.hpp" namespace bdt { template class SetExample : public ExampleWithY { public: // typedef Example _super; typedef fHash_t hash_t; // needed for untemplatized type specs typedef list featStringCont_t; template SetExample(const fHashT2& fset, unsigned partition = 0, predict_t yValue = predict_t(0), fpVal_t weight = 1.0); template bool setFeatures(const fHashT2& fset); bool setFeatures(const featStringCont_t& strs, const char delim = ' '); // bool satisfies(const BinaryFeatureTest& ft) const; // bool satisfies(const ScalarFeatureTest& ft) const; bool satisfies(const FeatureTest& ft) const; bool satisfies(const Feature& feat, const rel_t& rel, const double& val) const; static bool satisfies(const typename fHash_t::const_iterator& fci, const rel_t& rel, const double& val); bool hasFeature(const Feature& f) const; // bool hasFeature(const FeatureTest& f) const; // bool hasFeature(const Feature* f) const; IFUnion_t lookup(const unsigned& featID) const; IFUnion_t lookup(const Feature& theFeat, const bool isFloat) const; BSP fset() const { return m_fSet; } static string name(); /// Uncompress the feature vector. /// Since the SetExample is already uncompressed, this method does nothing. void uncompress() const {} friend ostream& operator<< (ostream& out, const SetExample& fe); protected: void constructHelp(); BSP m_fSet; }; typedef SetExample SparseMapExample; typedef SetExample SparseSetExample; typedef SetExample DenseSetExample; // the following declarations prevent "specialization after instantiation" errors // template<> bool SparseMapExample::satisfies(const BinaryFeatureTest& ft) const; // template<> bool SparseMapExample::satisfies(const ScalarFeatureTest& ft) const; template<> bool SparseMapExample::satisfies(const FeatureTest& ft) const; template<> bool SparseMapExample::satisfies(const Feature& theFeat, const rel_t& theRel, const double& testVal) const; template<> bool SparseMapExample::satisfies(const SparseMapExample::hash_t::const_iterator& fci, const rel_t& rel, const double& val); template<> IFUnion_t SparseMapExample::lookup(const unsigned& featID) const; template<> IFUnion_t SparseMapExample::lookup(const Feature& theFeat, const bool isFloat) const; template<> bool SparseSetExample::satisfies(const FeatureTest& ft) const; // template<> bool SparseSetExample::satisfies(const BinaryFeatureTest& ft) const; // template<> bool SparseSetExample::satisfies(const ScalarFeatureTest& ft) const; template<> bool SparseSetExample::satisfies(const Feature& theFeat, const rel_t& theRel, const double& testVal) const; // template<> bool SparseSetExample::satisfies(const SparseSetExample::hash_t::const_iterator& fci, // const rel_t& rel, const double& val); template<> IFUnion_t SparseSetExample::lookup(const unsigned& featID) const; template<> IFUnion_t SparseSetExample::lookup(const Feature& theFeat, const bool isFloat) const; template<> bool SparseSetExample::hasFeature(const Feature& f) const; template<> bool SparseMapExample::hasFeature(const Feature& f) const; } // namespace bdt #endif /* _BDTSetExampleH_ */ bdtl-1.0.3/src/bdtl/DecTree.ipp0000644000175000017500000000356512151020420015767 0ustar melamedmelamed/*! * \file DecTree.ipp */ #include "Dbug.hpp" #include "bdtl/DecTree.hpp" #include "bdtl/DecNode.hpp" #include "bdtl/LossFunction.hpp" namespace bdt { // Find the confidence of some ex_t. // \param e An ex_t. // \return The confidence of that example. template double DecTree::confidence(const ex_t& e) const { if (m_nodes.empty()) return 0; NodeID currID = m_rootNode; DecNode* pNode = m_nodes.at(currID); while (! pNode->isLeaf()) { if (e.satisfies(pNode->getFT())) { if (parameter::traceConfidencePaths()) DEBUG(TREE_DBG_LEVEL, "\tex_t has feature " << pNode->getFT() << endl); //DEBUG(TREE_DBG_LEVEL, "\tex_t has feature " << pNode->getFT()->str() << endl); currID = pNode->getTrueChildID(); } else { if (parameter::traceConfidencePaths()) DEBUG(TREE_DBG_LEVEL, "\tex_t does not have feature " << pNode->getFT() << endl); //DEBUG(TREE_DBG_LEVEL, "\tex_t does not have feature " << pNode->getFT()->str() << endl); currID = pNode->getFalseChildID(); } pNode = m_nodes.at(currID); } return pNode->getConfidence(); } // If we were to add the confidence at each leaf to the training set Examples, // what would be the loss? template double DecTree::postulatedTrainingSetLoss() const { double l; double ret = 0; typename nodeContainer_t::const_iterator n = m_nodes.begin(); const typename nodeContainer_t::const_iterator sentinel = m_nodes.end(); for (; n != sentinel; ++n) { DecNode* dnPtr = *n; if (dnPtr->isLeaf()) { const DecNode* pNode = static_cast(dnPtr); l = parameter::pLossFunc->unpenalizedLoss(pNode->getTrainExamples(), pNode->getConfidence()); DEBUG(EX_DBG_LEVEL, "DecTree::postulatedTrainingSetLoss for node " << pNode->id() << " is " << l << endl); ret += l; } } return ret; } } // namespace bdt