pax_global_header00006660000000000000000000000064116062751130014514gustar00rootroot0000000000000052 comment=0b45daf4925ccac4f72f8bf022b990d7d659d32e svm-struct-matlab-1.0/000077500000000000000000000000001160627511300147415ustar00rootroot00000000000000svm-struct-matlab-1.0/.gitignore000066400000000000000000000003471160627511300167350ustar00rootroot00000000000000svm_struct_learn.mex* .DS_Store *.pbxuser *.perspective *.perspectivev3 build/ *.mode1v3 *.mode2v3 svm-struct-matlab.xcodeproj/xcuserdata/ svm-struct-matlab.xcodeproj/project.xcworkspace/xcuserdata/ svm-struct-matlab-*.tar.gz svm-struct-matlab-1.0/LICENSE.txt000066400000000000000000000043321160627511300165660ustar00rootroot00000000000000SVM-Struct ---------- Available at http://svmlight.joachims.org/ Author: Thorsten Joachims thorsten@joachims.org Cornell University Department of Computer Science 4153 Upson Hall Ithaca, NY 14853 USA LICENSING TERMS This program is granted free of charge for non-commercial research and education purposes. However you must obtain a license from the author to use it for commercial purposes. Scientific results produced using the software provided shall acknowledge the use of SVM-Struct. Please cite as I. Tsochantaridis, T. Joachims, T. Hofmann, and Y. Altun, Large Margin Methods for Structured and Interdependent Output Variables, Journal of Machine Learning Research (JMLR), 6(Sep):1453-1484, 2005. http://jmlr.csail.mit.edu/papers/volume6/tsochantaridis05a/tsochantaridis05a.pdf Moreover shall the author of SVM-Struct be informed about the publication. The software and derivatives of the software must not be distributed without prior permission of the author. By using SVM-Struct you agree to the licensing terms. NO WARRANTY BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. svm-struct-matlab-1.0/Makefile000066400000000000000000000076351160627511300164140ustar00rootroot00000000000000# file: Makefile # brief: Compile MEXified SVM-struct # author: Andrea Vedaldi MEX ?= mex VER = 1.0 PACKAGE = svm-struct-matlab # -------------------------------------------------------------------- # Auto-detect architecture # -------------------------------------------------------------------- Darwin_PPC_ARCH := mac Darwin_Power_Macintosh_ARCH := mac Darwin_i386_ARCH := maci64 Linux_i386_ARCH := glnx86 Linux_i686_ARCH := glnx86 Linux_unknown_ARC := glnx86 Linux_x86_64_ARCH := glnxa64 UNAME := $(shell uname -sm) ARCH ?= $($(shell echo "$(UNAME)" | tr \ _)_ARCH) # Mac OS X Intel 32 ifeq ($(ARCH),maci) SDKROOT ?= /Developer/SDKs/MacOSX10.6.sdk MACOSX_DEPLOYMENT_TARGET ?= 10.4 CFLAGS += -m32 -isysroot $(SDKROOT) -mmacosx-version-min=$(MACOSX_DEPLOYMENT_TARGET) LDFLAGS += --mmacosx-version-min=$(MACOSX_DEPLOYMENT_TARGET) MEXEXT = mexmaci endif # Mac OS X Intel 64 ifeq ($(ARCH),maci64) SDKROOT ?= /Developer/SDKs/MacOSX10.6.sdk MACOSX_DEPLOYMENT_TARGET ?= 10.4 CFLAGS += -m64 -isysroot $(SDKROOT) -mmacosx-version-min=$(MACOSX_DEPLOYMENT_TARGET) LDFLAGS += -mmacosx-version-min=$(MACOSX_DEPLOYMENT_TARGET) MEXEXT = mexmaci64 endif # Linux-32 ifeq ($(ARCH),glnx86) CFLAGS += -march=i686 LDFLAGS += MEXEXT = mexglx endif # Linux-64 ifeq ($(ARCH),glnxa64) LDFLAGS += MEXEXT = mexa64 endif MEXFLAGS = -largeArrayDims -$(ARCH) CFLAGS='$$CFLAGS $(CFLAGS) -Wall' LDFLAGS='$$LDFLAGS $(LDFLAGS)' BUILD = build/$(ARCH) # -------------------------------------------------------------------- # Build # -------------------------------------------------------------------- svm_light_objs := \ $(BUILD)/svm_light/svm_hideo.o \ $(BUILD)/svm_light/svm_learn.o \ $(BUILD)/svm_light/svm_common.o svm_struct_objs := \ $(BUILD)/svm_struct/svm_struct_learn.o \ $(BUILD)/svm_struct/svm_struct_common.o svm_custom_objs := \ $(BUILD)/svm_struct_api.o \ $(BUILD)/svm_struct_learn_custom.o $(BUILD)/%.o : %.c $(MEX) $(MEXFLAGS) -outdir "$(dir $@)" -c "$<" svm_struct_learn.$(MEXEXT) : svm_struct_learn_mex.c \ $(svm_custom_objs) \ $(svm_light_objs) \ $(svm_struct_objs) $(MEX) $(MEXFLAGS) $^ -output "$@" .PHONY: clean clean: rm -fv $(svm_custom_objs) $(svm_struct_objs) $(svm_light_objs) find . -name '*~' -delete .PHONY: distclean distclean: clean for ext in mexmaci mexmaci64 mexglx mexa64 ; \ do \ rm -fv svm_struct_learn.$${ext} ; \ done rm -rf build rm -rf $(PACKAGE)-*.tar.gz .PHONY: dist dist: git archive --format=tar --prefix=$(PACKAGE)-$(VER)/ v$(VER) | gzip >$(PACKAGE)-$(VER).tar.gz # svm_struct dependencies svm_struct_api.o: \ $(BUILD)/.dir \ svm_struct_api.c \ svm_struct_api.h \ svm_struct_api_types.h \ svm_struct/svm_struct_common.h svm_struct_learn_custom.o: \ $(BUILD)/.dir \ svm_struct_learn_custom.c \ svm_struct_api.h \ svm_struct_api_types.h \ svm_light/svm_common.h \ svm_struct/svm_struct_common.h svm_struct/svm_struct_mex.o : \ $(BUILD)/.dir \ svm_struct/svm_struct_mex.c svm_struct/svm_struct_common.o : \ $(BUILD)/.dir \ svm_struct/svm_struct_common.c \ svm_struct/svm_struct_common.h \ svm_light/svm_common.h \ svm_struct_api_types.h svm_struct/svm_struct_learn.o : \ $(BUILD)/.dir \ svm_struct/svm_struct_learn.c \ svm_struct/svm_struct_common.h \ svm_light/svm_common.h \ svm_light/svm_learn.h \ svm_struct_api_types.h \ svm_struct_api.h svm_struct/svm_struct_classify.o : \ $(BUILD)/.dir \ svm_struct/svm_struct_classify.c \ svm_struct/svm_struct_common.h \ svm_light/svm_common.h \ svm_struct_api_types.h \ svm_struct_api.h # svm_light dependencies svm_light/svm_learn.o : \ $(BUILD)/.dir \ svm_light/svm_learn.c \ svm_light/svm_learn.h \ svm_light/svm_common.h svm_light/svm_common.o : \ $(BUILD)/.dir \ svm_light/svm_common.c \ svm_light/svm_common.h \ svm_light/kernel.h svm_light/svm_hideo.o : \ $(BUILD)/.dir \ svm_light/svm_hideo.c svm-struct-matlab-1.0/README_MATLAB.txt000066400000000000000000000060001160627511300174530ustar00rootroot00000000000000 SVM^struct for MATLAB V1.0 Andrea Vedaldi This is a patch to Thorsten Joachims (http://www.joachims.org/) SVM-struct implementation that provides a simple-to-use MATLAB interface. OBTAINING: svm-struct-matlab homepage is http://www.vlfeat.org/~vedaldi/code/svm-struct-matlab.html. The GIT repository of the project can be downloaded at https://github.com/vedaldi/svm-srtuct-matlab. COMPILING: To compile this software use the included Makefile. It should work out of the box on Linux and Mac OS X provided that MATLAB MEX program is in the command line path. So just issue > make The only files that are needed to run the package with MATLAB are svm_struct_learn.mex* (MEX program) and svm_struct_learn.m (documentation). If MEX is not on the command line, the path can be specified as > make MEX=/bin/mex where is MATLAB root directory. Finally, it is also possible to specify manually the architecture > make ARCH=maci # Mac Intel 32 bit > make ARCH=maci64 # Mac Intel 64 bit > make ARCH=glnx86 # Linux 32 bit > make ARCH=glnxa64 # Linux 64 bit To clean the build products use > make clean # clean all build but the MEX file > make distclean # clean all build products USAGE: There is only one MATLAB command, i.e. SVM_STRUCT_LEARN. This commands take as input handles to the function implementing the maximal constraint violation search, the loss, and the feature map. See TEST_SVM_STRUCT_LEARN and TEST_SVM_STRUCT_LEARN_KER for example usage and SVM_STRUCT_LEARN built-in help for further information. CHANGES: 1.0 - Initial public release LICENSE: The MATLAB wrapper of T. Joachim's SVM^struct is distributed under the following ``MIT license''. This license covers only the additions to T. Joachims SVM^struct code that constitute the MATLAB interface (the "Software") and does not cover SVM^struct itself. See the file LICENSE.txt for the SVM^struct license. Copyright (C) 2011 by Andrea Vedaldi Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. svm-struct-matlab-1.0/README_STRUCT.txt000066400000000000000000000014671160627511300175530ustar00rootroot00000000000000Readme for the SVM-light structure learning API ----------------------------------------------- Thorsten Joachims, 03.07.2004 The API allows to implement different versions of the learning algorithm for learning different kinds of structures. To adapt to a new structure, one needs to modify the files svm_struct_api_types.h svm_struct_api.c Both files already contain empty templates. The first file contains the type definitions that need to be changed. PATTERN is the structure for storing the x-part of an example (x,y), LABEL is the y-part. The learned model will be stored in STRUCTMODEL. Finally, STRUCT_LEARN_PARM can be used to store any parameters that you might want to pass to the function. The second file contains the function you need to implement. See the documentation in the file for details. svm-struct-matlab-1.0/svm-struct-matlab.xcodeproj/000077500000000000000000000000001160627511300223225ustar00rootroot00000000000000svm-struct-matlab-1.0/svm-struct-matlab.xcodeproj/project.pbxproj000066400000000000000000000250011160627511300253740ustar00rootroot00000000000000// !$*UTF8*$! { archiveVersion = 1; classes = { }; objectVersion = 42; objects = { /* Begin PBXFileReference section */ 2D002F7B104FCC3200981B2C /* Makefile */ = {isa = PBXFileReference; explicitFileType = sourcecode.make; fileEncoding = 4; path = Makefile; sourceTree = ""; wrapsLines = 0; }; 2D002F7D104FCC3200981B2C /* kernel.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = kernel.h; sourceTree = ""; tabWidth = 8; }; 2D002F80104FCC3200981B2C /* svm_classify.c */ = {isa = PBXFileReference; fileEncoding = 4; indentWidth = 2; lastKnownFileType = sourcecode.c.c; path = svm_classify.c; sourceTree = ""; tabWidth = 8; usesTabs = 0; }; 2D002F81104FCC3200981B2C /* svm_common.c */ = {isa = PBXFileReference; fileEncoding = 4; indentWidth = 2; lastKnownFileType = sourcecode.c.c; path = svm_common.c; sourceTree = ""; tabWidth = 8; usesTabs = 0; }; 2D002F82104FCC3200981B2C /* svm_common.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = svm_common.h; sourceTree = ""; tabWidth = 8; }; 2D002F84104FCC3200981B2C /* svm_hideo.c */ = {isa = PBXFileReference; fileEncoding = 4; indentWidth = 2; lastKnownFileType = sourcecode.c.c; path = svm_hideo.c; sourceTree = ""; tabWidth = 8; usesTabs = 0; }; 2D002F86104FCC3200981B2C /* svm_learn.c */ = {isa = PBXFileReference; fileEncoding = 4; indentWidth = 2; lastKnownFileType = sourcecode.c.c; path = svm_learn.c; sourceTree = ""; tabWidth = 8; usesTabs = 0; }; 2D002F87104FCC3200981B2C /* svm_learn.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = svm_learn.h; sourceTree = ""; tabWidth = 8; }; 2D002F89104FCC3200981B2C /* svm_learn_main.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = svm_learn_main.c; sourceTree = ""; tabWidth = 8; }; 2D002F8A104FCC3200981B2C /* svm_loqo.c */ = {isa = PBXFileReference; fileEncoding = 4; indentWidth = 2; lastKnownFileType = sourcecode.c.c; path = svm_loqo.c; sourceTree = ""; tabWidth = 8; usesTabs = 0; }; 2D002F8D104FCC3200981B2C /* svm_struct_classify.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = svm_struct_classify.c; sourceTree = ""; tabWidth = 8; }; 2D002F8E104FCC3200981B2C /* svm_struct_common.c */ = {isa = PBXFileReference; fileEncoding = 4; indentWidth = 2; lastKnownFileType = sourcecode.c.c; path = svm_struct_common.c; sourceTree = ""; tabWidth = 8; usesTabs = 0; }; 2D002F8F104FCC3200981B2C /* svm_struct_common.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = svm_struct_common.h; sourceTree = ""; tabWidth = 8; }; 2D002F91104FCC3200981B2C /* svm_struct_learn.c */ = {isa = PBXFileReference; fileEncoding = 4; indentWidth = 2; lastKnownFileType = sourcecode.c.c; path = svm_struct_learn.c; sourceTree = ""; tabWidth = 8; usesTabs = 0; }; 2D002F92104FCC3200981B2C /* svm_struct_learn.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = svm_struct_learn.h; sourceTree = ""; tabWidth = 8; }; 2D002F94104FCC3200981B2C /* svm_struct_main.c */ = {isa = PBXFileReference; fileEncoding = 4; indentWidth = 2; lastKnownFileType = sourcecode.c.c; path = svm_struct_main.c; sourceTree = ""; tabWidth = 8; usesTabs = 0; }; 2D002F95104FCC3200981B2C /* svm_struct_api.c */ = {isa = PBXFileReference; fileEncoding = 4; indentWidth = 2; lastKnownFileType = sourcecode.c.c; lineEnding = 0; path = svm_struct_api.c; sourceTree = ""; tabWidth = 8; xcLanguageSpecificationIdentifier = xcode.lang.c; }; 2D002F96104FCC3200981B2C /* svm_struct_api.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = svm_struct_api.h; sourceTree = ""; tabWidth = 8; }; 2D002F97104FCC3200981B2C /* svm_struct_api_types.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = svm_struct_api_types.h; sourceTree = ""; tabWidth = 8; }; 2D002F98104FCC3200981B2C /* svm_struct_learn.m */ = {isa = PBXFileReference; explicitFileType = sourcecode; fileEncoding = 4; lineEnding = 0; path = svm_struct_learn.m; sourceTree = ""; tabWidth = 2; xcLanguageSpecificationIdentifier = xcode.lang.objc; }; 2D002F99104FCC3200981B2C /* svm_struct_learn_custom.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = svm_struct_learn_custom.c; sourceTree = ""; tabWidth = 8; }; 2D002F9A104FCC3200981B2C /* test_svm_struct_learn.m */ = {isa = PBXFileReference; explicitFileType = sourcecode; fileEncoding = 4; lineEnding = 0; path = test_svm_struct_learn.m; sourceTree = ""; tabWidth = 2; xcLanguageSpecificationIdentifier = xcode.lang.objc; }; 2D002F9B104FCC3200981B2C /* test_svm_struct_learn_ker.m */ = {isa = PBXFileReference; explicitFileType = sourcecode; fileEncoding = 4; lineEnding = 0; path = test_svm_struct_learn_ker.m; sourceTree = ""; tabWidth = 2; xcLanguageSpecificationIdentifier = xcode.lang.objc; }; 2D0030081050194100981B2C /* svm_struct_learn_mex.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = svm_struct_learn_mex.c; sourceTree = ""; }; 2D60637D13C8CAB70083CAB2 /* README_MATLAB.txt */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = README_MATLAB.txt; sourceTree = ""; wrapsLines = 0; }; /* End PBXFileReference section */ /* Begin PBXGroup section */ 2D002F66104FCBBE00981B2C = { isa = PBXGroup; children = ( 2D60637D13C8CAB70083CAB2 /* README_MATLAB.txt */, 2D002F7B104FCC3200981B2C /* Makefile */, 2D002F7C104FCC3200981B2C /* svm_light */, 2D002F8B104FCC3200981B2C /* svm_struct */, 2D002F95104FCC3200981B2C /* svm_struct_api.c */, 2D002F96104FCC3200981B2C /* svm_struct_api.h */, 2D002F97104FCC3200981B2C /* svm_struct_api_types.h */, 2D002F99104FCC3200981B2C /* svm_struct_learn_custom.c */, 2D0030081050194100981B2C /* svm_struct_learn_mex.c */, 2D002F98104FCC3200981B2C /* svm_struct_learn.m */, 2D002F9A104FCC3200981B2C /* test_svm_struct_learn.m */, 2D002F9B104FCC3200981B2C /* test_svm_struct_learn_ker.m */, ); sourceTree = ""; }; 2D002F7C104FCC3200981B2C /* svm_light */ = { isa = PBXGroup; children = ( 2D002F7D104FCC3200981B2C /* kernel.h */, 2D002F80104FCC3200981B2C /* svm_classify.c */, 2D002F81104FCC3200981B2C /* svm_common.c */, 2D002F82104FCC3200981B2C /* svm_common.h */, 2D002F84104FCC3200981B2C /* svm_hideo.c */, 2D002F86104FCC3200981B2C /* svm_learn.c */, 2D002F87104FCC3200981B2C /* svm_learn.h */, 2D002F89104FCC3200981B2C /* svm_learn_main.c */, 2D002F8A104FCC3200981B2C /* svm_loqo.c */, ); path = svm_light; sourceTree = ""; tabWidth = 8; }; 2D002F8B104FCC3200981B2C /* svm_struct */ = { isa = PBXGroup; children = ( 2D002F8D104FCC3200981B2C /* svm_struct_classify.c */, 2D002F8E104FCC3200981B2C /* svm_struct_common.c */, 2D002F8F104FCC3200981B2C /* svm_struct_common.h */, 2D002F91104FCC3200981B2C /* svm_struct_learn.c */, 2D002F92104FCC3200981B2C /* svm_struct_learn.h */, 2D002F94104FCC3200981B2C /* svm_struct_main.c */, ); path = svm_struct; sourceTree = ""; tabWidth = 8; }; /* End PBXGroup section */ /* Begin PBXLegacyTarget section */ 2D002F9C104FCC8300981B2C /* all */ = { isa = PBXLegacyTarget; buildArgumentsString = "$(ACTION)"; buildConfigurationList = 2D002FA7104FCCBB00981B2C /* Build configuration list for PBXLegacyTarget "all" */; buildPhases = ( ); buildToolPath = /usr/bin/make; dependencies = ( ); name = all; passBuildSettingsInEnvironment = 1; productName = all; }; /* End PBXLegacyTarget section */ /* Begin PBXProject section */ 2D002F68104FCBBE00981B2C /* Project object */ = { isa = PBXProject; buildConfigurationList = 2D002F6B104FCBBE00981B2C /* Build configuration list for PBXProject "svm-struct-matlab" */; compatibilityVersion = "Xcode 2.4"; developmentRegion = English; hasScannedForEncodings = 0; knownRegions = ( en, ); mainGroup = 2D002F66104FCBBE00981B2C; projectDirPath = ""; projectRoot = ""; targets = ( 2D002F9C104FCC8300981B2C /* all */, ); }; /* End PBXProject section */ /* Begin XCBuildConfiguration section */ 2D002F69104FCBBE00981B2C /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { ARCH = "$(ARCH_$(ARCHS))"; ARCHS = ( i386, x86_64, ); ARCH_i386 = maci; ARCH_x86_64 = maci64; DOXYGEN = doxygen; HEADER_SEARCH_PATHS = "$(MATLABROOT)/extern/include"; MACOSX_DEPLOYMENT_TARGET = 10.5; MATLABROOT = /Applications/Matlab09B.app; MEX = "$(MATLABROOT)/bin/mex"; ONLY_ACTIVE_ARCH = YES; SDKROOT = "$(DEVELOPER_SDK_DIR)/MacOSX10.6.sdk"; }; name = Debug; }; 2D002F6A104FCBBE00981B2C /* Release */ = { isa = XCBuildConfiguration; buildSettings = { ARCH = "$(ARCH_$(ARCHS))"; ARCHS = ( i386, x86_64, ); ARCH_i386 = maci; ARCH_x86_64 = maci64; DOXYGEN = doxygen; HEADER_SEARCH_PATHS = "$(MATLABROOT)/extern/include"; MACOSX_DEPLOYMENT_TARGET = 10.5; MATLABROOT = /Applications/Matlab09B.app; MEX = "$(MATLABROOT)/bin/mex"; NDEBUG = yes; ONLY_ACTIVE_ARCH = YES; SDKROOT = "$(DEVELOPER_SDK_DIR)/MacOSX10.6.sdk"; }; name = Release; }; 2D002F9D104FCC8300981B2C /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { }; name = Debug; }; 2D002F9E104FCC8300981B2C /* Release */ = { isa = XCBuildConfiguration; buildSettings = { GCC_ENABLE_FIX_AND_CONTINUE = NO; }; name = Release; }; /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ 2D002F6B104FCBBE00981B2C /* Build configuration list for PBXProject "svm-struct-matlab" */ = { isa = XCConfigurationList; buildConfigurations = ( 2D002F69104FCBBE00981B2C /* Debug */, 2D002F6A104FCBBE00981B2C /* Release */, ); defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; 2D002FA7104FCCBB00981B2C /* Build configuration list for PBXLegacyTarget "all" */ = { isa = XCConfigurationList; buildConfigurations = ( 2D002F9D104FCC8300981B2C /* Debug */, 2D002F9E104FCC8300981B2C /* Release */, ); defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; /* End XCConfigurationList section */ }; rootObject = 2D002F68104FCBBE00981B2C /* Project object */; } svm-struct-matlab-1.0/svm-struct-matlab.xcodeproj/project.xcworkspace/000077500000000000000000000000001160627511300263205ustar00rootroot00000000000000svm-struct-matlab-1.0/svm-struct-matlab.xcodeproj/project.xcworkspace/contents.xcworkspacedata000066400000000000000000000002421160627511300332600ustar00rootroot00000000000000 svm-struct-matlab-1.0/svm_light/000077500000000000000000000000001160627511300167355ustar00rootroot00000000000000svm-struct-matlab-1.0/svm_light/LICENSE.txt000066400000000000000000000042601160627511300205620ustar00rootroot00000000000000SVM-Light --------- Available at http://svmlight.joachims.org/ Author: Thorsten Joachims thorsten@joachims.org Cornell University Department of Computer Science 4153 Upson Hall Ithaca, NY 14853 USA LICENSING TERMS This program is granted free of charge for non-commercial research and education purposes. However you must obtain a license from the author to use it for commercial purposes. Scientific results produced using the software provided shall acknowledge the use of SVM-Light. Please cite as T. Joachims, Making large-Scale SVM Learning Practical. Advances in Kernel Methods - Support Vector Learning, B. Schölkopf and C. Burges and A. Smola (ed.), MIT-Press, 1999. http://www-ai.cs.uni-dortmund.de/DOKUMENTE/joachims_99a.pdf Moreover shall the author of SVM-Light be informed about the publication. The software or any derivative software must not be distributed without prior permission of the author. By using SVM-Light you agree to the licensing terms. NO WARRANTY BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. svm-struct-matlab-1.0/svm_light/Makefile000066400000000000000000000071731160627511300204050ustar00rootroot00000000000000# # makefile for svm_light # # Thorsten Joachims, 2002 # #Use the following to compile under unix or cygwin CC = gcc LD = gcc #Uncomment the following line to make CYGWIN produce stand-alone Windows executables #SFLAGS= -mno-cygwin CFLAGS= $(SFLAGS) -O3 # release C-Compiler flags LFLAGS= $(SFLAGS) -O3 # release linker flags #CFLAGS= $(SFLAGS) -pg -Wall -pedantic # debugging C-Compiler flags #LFLAGS= $(SFLAGS) -pg # debugging linker flags LIBS=-L. -lm # used libraries all: svm_learn_hideo svm_classify tidy: rm -f *.o rm -f pr_loqo/*.o clean: tidy rm -f svm_learn rm -f svm_classify rm -f libsvmlight.so help: info info: @echo @echo "make for SVM-light Thorsten Joachims, 1998" @echo @echo "Thanks to Ralf Herbrich for the initial version." @echo @echo "USAGE: make [svm_learn | svm_learn_loqo | svm_learn_hideo | " @echo " libsvmlight_hideo | libsvmlight_loqo | " @echo " svm_classify | all | clean | tidy]" @echo @echo " svm_learn builds the learning module (prefers HIDEO)" @echo " svm_learn_hideo builds the learning module using HIDEO optimizer" @echo " svm_learn_loqo builds the learning module using PR_LOQO optimizer" @echo " svm_classify builds the classfication module" @echo " libsvmlight_hideo builds shared object library that can be linked into" @echo " other code using HIDEO" @echo " libsvmlight_loqo builds shared object library that can be linked into" @echo " other code using PR_LOQO" @echo " all (default) builds svm_learn + svm_classify" @echo " clean removes .o and target files" @echo " tidy removes .o files" @echo # Create executables svm_learn and svm_classify svm_learn_hideo: svm_learn_main.o svm_learn.o svm_common.o svm_hideo.o $(LD) $(LFLAGS) svm_learn_main.o svm_learn.o svm_common.o svm_hideo.o -o svm_learn $(LIBS) #svm_learn_loqo: svm_learn_main.o svm_learn.o svm_common.o svm_loqo.o loqo # $(LD) $(LFLAGS) svm_learn_main.o svm_learn.o svm_common.o svm_loqo.o pr_loqo/pr_loqo.o -o svm_learn $(LIBS) svm_classify: svm_classify.o svm_common.o $(LD) $(LFLAGS) svm_classify.o svm_common.o -o svm_classify $(LIBS) # Create library libsvmlight.so, so that external code can get access to the # learning and classification functions of svm-light by linking this library. svm_learn_hideo_noexe: svm_learn_main.o svm_learn.o svm_common.o svm_hideo.o libsvmlight_hideo: svm_learn_main.o svm_learn.o svm_common.o svm_hideo.o $(LD) -shared svm_learn.o svm_common.o svm_hideo.o -o libsvmlight.so #svm_learn_loqo_noexe: svm_learn_main.o svm_learn.o svm_common.o svm_loqo.o loqo #libsvmlight_loqo: svm_learn_main.o svm_learn.o svm_common.o svm_loqo.o # $(LD) -shared svm_learn.o svm_common.o svm_loqo.o pr_loqo/pr_loqo.o -o libsvmlight.so # Compile components svm_hideo.o: svm_hideo.c $(CC) -c $(CFLAGS) svm_hideo.c -o svm_hideo.o #svm_loqo.o: svm_loqo.c # $(CC) -c $(CFLAGS) svm_loqo.c -o svm_loqo.o svm_common.o: svm_common.c svm_common.h kernel.h $(CC) -c $(CFLAGS) svm_common.c -o svm_common.o svm_learn.o: svm_learn.c svm_common.h $(CC) -c $(CFLAGS) svm_learn.c -o svm_learn.o svm_learn_main.o: svm_learn_main.c svm_learn.h svm_common.h $(CC) -c $(CFLAGS) svm_learn_main.c -o svm_learn_main.o svm_classify.o: svm_classify.c svm_common.h kernel.h $(CC) -c $(CFLAGS) svm_classify.c -o svm_classify.o #loqo: pr_loqo/pr_loqo.o #pr_loqo/pr_loqo.o: pr_loqo/pr_loqo.c # $(CC) -c $(CFLAGS) pr_loqo/pr_loqo.c -o pr_loqo/pr_loqo.o svm-struct-matlab-1.0/svm_light/kernel.h000066400000000000000000000054331160627511300203730ustar00rootroot00000000000000/************************************************************************/ /* */ /* kernel.h */ /* */ /* User defined kernel function. Feel free to plug in your own. */ /* */ /* Copyright: Thorsten Joachims */ /* Date: 16.12.97 */ /* */ /************************************************************************/ /* KERNEL_PARM is defined in svm_common.h The field 'custom' is reserved for */ /* parameters of the user defined kernel. You can also access and use */ /* the parameters of the other kernels. Just replace the line return((double)(1.0)); with your own kernel. */ /* Example: The following computes the polynomial kernel. sprod_ss computes the inner product between two sparse vectors. return((CFLOAT)pow(kernel_parm->coef_lin*sprod_ss(a,b) +kernel_parm->coef_const,(double)kernel_parm->poly_degree)); */ /* If you are implementing a kernel that is not based on a feature/value representation, you might want to make use of the field "userdefined" in SVECTOR. By default, this field will contain whatever string you put behind a # sign in the example file. So, if a line in your training file looks like -1 1:3 5:6 #abcdefg then the SVECTOR field "words" will contain the vector 1:3 5:6, and "userdefined" will contain the string "abcdefg". */ #include "../svm_struct_api_types.h" double custom_kernel(KERNEL_PARM *kernel_parm, SVECTOR *a, SVECTOR *b) /* plug in you favorite kernel */ { mxArray* structParm_array ; mxArray* kernelFn_array ; mxArray* args [6] ; mxArray* k_array ; double k ; int status ; { MexKernelInfo* info = (MexKernelInfo*) kernel_parm -> custom ; structParm_array = (mxArray*) info -> structParm ; kernelFn_array = (mxArray*) info -> kernelFn ; } args[0] = kernelFn_array ; args[1] = structParm_array ; args[2] = MexPhiCustomGetPattern (a->userdefined) ; args[3] = MexPhiCustomGetLabel (a->userdefined) ; args[4] = MexPhiCustomGetPattern (b->userdefined) ; args[5] = MexPhiCustomGetLabel (b->userdefined) ; status = mexCallMATLAB(1, &k_array, 6, args, "feval") ; if (status) { mexErrMsgTxt("Error while executing SPARM.KERNELFN") ; } if (mxGetClassID(k_array) == mxUNKNOWN_CLASS) { mexErrMsgTxt("SPARM.KERNELFN did not reutrn a result") ; } k = mxGetScalar (k_array) ; mxDestroyArray (k_array) ; return (k) ; } svm-struct-matlab-1.0/svm_light/svm_classify.c000066400000000000000000000153501160627511300216070ustar00rootroot00000000000000/***********************************************************************/ /* */ /* svm_classify.c */ /* */ /* Classification module of Support Vector Machine. */ /* */ /* Author: Thorsten Joachims */ /* Date: 02.07.02 */ /* */ /* Copyright (c) 2002 Thorsten Joachims - All rights reserved */ /* */ /* This software is available for non-commercial use only. It must */ /* not be modified and distributed without prior permission of the */ /* author. The author is not responsible for implications from the */ /* use of this software. */ /* */ /************************************************************************/ # include "svm_common.h" char docfile[200]; char modelfile[200]; char predictionsfile[200]; void read_input_parameters(int, char **, char *, char *, char *, long *, long *); void print_help(void); int main (int argc, char* argv[]) { DOC *doc; /* test example */ WORD *words; long max_docs,max_words_doc,lld; long totdoc=0,queryid,slackid; long correct=0,incorrect=0,no_accuracy=0; long res_a=0,res_b=0,res_c=0,res_d=0,wnum,pred_format; long j; double t1,runtime=0; double dist,doc_label,costfactor; char *line,*comment; FILE *predfl,*docfl; MODEL *model; read_input_parameters(argc,argv,docfile,modelfile,predictionsfile, &verbosity,&pred_format); nol_ll(docfile,&max_docs,&max_words_doc,&lld); /* scan size of input file */ max_words_doc+=2; lld+=2; line = (char *)my_malloc(sizeof(char)*lld); words = (WORD *)my_malloc(sizeof(WORD)*(max_words_doc+10)); model=read_model(modelfile); if(model->kernel_parm.kernel_type == 0) { /* linear kernel */ /* compute weight vector */ add_weight_vector_to_linear_model(model); } if(verbosity>=2) { printf("Classifying test examples.."); fflush(stdout); } if ((docfl = fopen (docfile, "r")) == NULL) { perror (docfile); exit (1); } if ((predfl = fopen (predictionsfile, "w")) == NULL) { perror (predictionsfile); exit (1); } while((!feof(docfl)) && fgets(line,(int)lld,docfl)) { if(line[0] == '#') continue; /* line contains comments */ parse_document(line,words,&doc_label,&queryid,&slackid,&costfactor,&wnum, max_words_doc,&comment); totdoc++; if(model->kernel_parm.kernel_type == LINEAR) {/* For linear kernel, */ for(j=0;(words[j]).wnum != 0;j++) { /* check if feature numbers */ if((words[j]).wnum>model->totwords) /* are not larger than in */ (words[j]).wnum=0; /* model. Remove feature if */ } /* necessary. */ } doc = create_example(-1,0,0,0.0,create_svector(words,comment,1.0)); t1=get_runtime(); if(model->kernel_parm.kernel_type == LINEAR) { /* linear kernel */ dist=classify_example_linear(model,doc); } else { /* non-linear kernel */ dist=classify_example(model,doc); } runtime+=(get_runtime()-t1); free_example(doc,1); if(dist>0) { if(pred_format==0) { /* old weired output format */ fprintf(predfl,"%.8g:+1 %.8g:-1\n",dist,-dist); } if(doc_label>0) correct++; else incorrect++; if(doc_label>0) res_a++; else res_b++; } else { if(pred_format==0) { /* old weired output format */ fprintf(predfl,"%.8g:-1 %.8g:+1\n",-dist,dist); } if(doc_label<0) correct++; else incorrect++; if(doc_label>0) res_c++; else res_d++; } if(pred_format==1) { /* output the value of decision function */ fprintf(predfl,"%.8g\n",dist); } if((int)(0.01+(doc_label*doc_label)) != 1) { no_accuracy=1; } /* test data is not binary labeled */ if(verbosity>=2) { if(totdoc % 100 == 0) { printf("%ld..",totdoc); fflush(stdout); } } } fclose(predfl); fclose(docfl); free(line); free(words); free_model(model,1); if(verbosity>=2) { printf("done\n"); /* Note by Gary Boone Date: 29 April 2000 */ /* o Timing is inaccurate. The timer has 0.01 second resolution. */ /* Because classification of a single vector takes less than */ /* 0.01 secs, the timer was underflowing. */ printf("Runtime (without IO) in cpu-seconds: %.2f\n", (float)(runtime/100.0)); } if((!no_accuracy) && (verbosity>=1)) { printf("Accuracy on test set: %.2f%% (%ld correct, %ld incorrect, %ld total)\n",(float)(correct)*100.0/totdoc,correct,incorrect,totdoc); printf("Precision/recall on test set: %.2f%%/%.2f%%\n",(float)(res_a)*100.0/(res_a+res_b),(float)(res_a)*100.0/(res_a+res_c)); } return(0); } void read_input_parameters(int argc, char **argv, char *docfile, char *modelfile, char *predictionsfile, long int *verbosity, long int *pred_format) { long i; /* set default */ strcpy (modelfile, "svm_model"); strcpy (predictionsfile, "svm_predictions"); (*verbosity)=2; (*pred_format)=1; for(i=1;(i=argc) { printf("\nNot enough input parameters!\n\n"); print_help(); exit(0); } strcpy (docfile, argv[i]); strcpy (modelfile, argv[i+1]); if((i+2) this help\n"); printf(" -v [0..3] -> verbosity level (default 2)\n"); printf(" -f [0,1] -> 0: old output format of V1.0\n"); printf(" -> 1: output the value of decision function (default)\n\n"); } svm-struct-matlab-1.0/svm_light/svm_common.c000066400000000000000000001342121160627511300212610ustar00rootroot00000000000000/************************************************************************/ /* */ /* svm_common.c */ /* */ /* Definitions and functions used in both svm_learn and svm_classify. */ /* */ /* Author: Thorsten Joachims */ /* Date: 02.07.04 */ /* */ /* Copyright (c) 2004 Thorsten Joachims - All rights reserved */ /* */ /* This software is available for non-commercial use only. It must */ /* not be modified and distributed without prior permission of the */ /* author. The author is not responsible for implications from the */ /* use of this software. */ /* */ /************************************************************************/ # include "ctype.h" # include "svm_common.h" # include "kernel.h" /* this contains a user supplied kernel */ #define MAX(x,y) ((x) < (y) ? (y) : (x)) #define MIN(x,y) ((x) > (y) ? (y) : (x)) #define SIGN(x) ((x) > (0) ? (1) : (((x) < (0) ? (-1) : (0)))) long verbosity; /* verbosity level (0-4) */ long kernel_cache_statistic; double classify_example(MODEL *model, DOC *ex) /* classifies one example */ { register long i; register double dist; if((model->kernel_parm.kernel_type == LINEAR) && (model->lin_weights)) return(classify_example_linear(model,ex)); dist=0; for(i=1;isv_num;i++) { dist+=kernel(&model->kernel_parm,model->supvec[i],ex)*model->alpha[i]; } return(dist-model->b); } double classify_example_linear(MODEL *model, DOC *ex) /* classifies example for linear kernel */ /* important: the model must have the linear weight vector computed */ /* use: add_weight_vector_to_linear_model(&model); */ /* important: the feature numbers in the example to classify must */ /* not be larger than the weight vector! */ { double sum=0; SVECTOR *f; for(f=ex->fvec;f;f=f->next) sum+=f->factor*sprod_ns(model->lin_weights,f); return(sum-model->b); } double kernel(KERNEL_PARM *kernel_parm, DOC *a, DOC *b) /* calculate the kernel function */ { double sum=0; SVECTOR *fa,*fb; if(kernel_parm->kernel_type == GRAM) { /* use value from explicitly */ if((a->kernelid>=0) && (b->kernelid>=0)) /* stored gram matrix */ return(kernel_parm->gram_matrix->element[MAX(a->kernelid,b->kernelid)] [MIN(a->kernelid,b->kernelid)]); else return(0); /* in case it is called for unknown vector */ } /* in case the constraints are sums of feature vector as represented as a list of SVECTOR's with their coefficient factor in the sum, take the kernel between all pairs */ for(fa=a->fvec;fa;fa=fa->next) { for(fb=b->fvec;fb;fb=fb->next) { if(fa->kernel_id == fb->kernel_id) sum+=fa->factor*fb->factor*single_kernel(kernel_parm,fa,fb); } } return(sum); } double single_kernel(KERNEL_PARM *kernel_parm, SVECTOR *a, SVECTOR *b) /* calculate the kernel function between two vectors */ { kernel_cache_statistic++; switch(kernel_parm->kernel_type) { case LINEAR: /* linear */ return(sprod_ss(a,b)); case POLY: /* polynomial */ return(pow(kernel_parm->coef_lin*sprod_ss(a,b)+kernel_parm->coef_const,(double)kernel_parm->poly_degree)); case RBF: /* radial basis function */ if(a->twonorm_sq<0) a->twonorm_sq=sprod_ss(a,a); if(b->twonorm_sq<0) b->twonorm_sq=sprod_ss(b,b); return(exp(-kernel_parm->rbf_gamma*(a->twonorm_sq-2*sprod_ss(a,b)+b->twonorm_sq))); case SIGMOID:/* sigmoid neural net */ return(tanh(kernel_parm->coef_lin*sprod_ss(a,b)+kernel_parm->coef_const)); case CUSTOM: /* custom-kernel supplied in file kernel.h*/ return(custom_kernel(kernel_parm,a,b)); default: printf("Error: Unknown kernel function\n"); exit(1); } } SVECTOR *create_svector(WORD *words, MexPhiCustom userdefined, double factor) { SVECTOR *vec; long fnum,i; fnum=0; while(words[fnum].wnum) { fnum++; } fnum++; vec = (SVECTOR *)my_malloc(sizeof(SVECTOR)); vec->words = (WORD *)my_malloc(sizeof(WORD)*(fnum)); for(i=0;iwords[i]=words[i]; } vec->twonorm_sq=-1; retainMexPhiCustom(userdefined) ; vec->userdefined = userdefined ; vec->kernel_id=0; vec->next=NULL; vec->factor=factor; return(vec); } SVECTOR *create_svector_shallow(WORD *words,MexPhiCustom userdefined,double factor) /* unlike 'create_svector' this does not copy words and userdefined */ { SVECTOR *vec; retainMexPhiCustom(userdefined) ; vec = (SVECTOR *)my_malloc(sizeof(SVECTOR)); vec->words = words; vec->twonorm_sq=-1; vec->userdefined=userdefined; vec->kernel_id=0; vec->next=NULL; vec->factor=factor; return(vec); } SVECTOR *create_svector_n(double *nonsparsevec, long maxfeatnum, MexPhiCustom userdefined, double factor) { return(create_svector_n_r(nonsparsevec,maxfeatnum,userdefined,factor,0)); } SVECTOR *create_svector_n_r(double *nonsparsevec, long maxfeatnum, MexPhiCustom userdefined, double factor, double min_non_zero) { SVECTOR *vec; long fnum,i; fnum=0; for(i=1;i<=maxfeatnum;i++) if((nonsparsevec[i]<-min_non_zero) || (nonsparsevec[i]>min_non_zero)) fnum++; vec = (SVECTOR *)my_malloc(sizeof(SVECTOR)); vec->words = (WORD *)my_malloc(sizeof(WORD)*(fnum+1)); fnum=0; for(i=1;i<=maxfeatnum;i++) { if((nonsparsevec[i]<-min_non_zero) || (nonsparsevec[i]>min_non_zero)) { vec->words[fnum].wnum=i; vec->words[fnum].weight=nonsparsevec[i]; fnum++; } } vec->words[fnum].wnum=0; vec->twonorm_sq=-1; retainMexPhiCustom(userdefined) ; vec->userdefined = userdefined ; vec->kernel_id=0; vec->next=NULL; vec->factor=factor; return(vec); } SVECTOR *copy_svector(SVECTOR *vec) { SVECTOR *newvec=NULL; if(vec) { newvec=create_svector(vec->words,vec->userdefined,vec->factor); newvec->kernel_id=vec->kernel_id; newvec->next=copy_svector(vec->next); } return(newvec); } SVECTOR *copy_svector_shallow(SVECTOR *vec) /* unlike 'copy_svector' this does not copy words and userdefined */ { SVECTOR *newvec=NULL; if(vec) { newvec=create_svector_shallow(vec->words,vec->userdefined,vec->factor); newvec->kernel_id=vec->kernel_id; newvec->next=copy_svector_shallow(vec->next); } return(newvec); } void free_svector(SVECTOR *vec) { SVECTOR *next; while(vec) { if(vec->words) free(vec->words); releaseMexPhiCustom (vec->userdefined) ; next=vec->next; free(vec); vec=next; } } void free_svector_shallow(SVECTOR *vec) /* unlike 'free_svector' this does not free words and userdefined */ { SVECTOR *next; while(vec) { next=vec->next; releaseMexPhiCustom(vec->userdefined) ; free(vec); vec=next; } } double sprod_ss(SVECTOR *a, SVECTOR *b) /* compute the inner product of two sparse vectors */ { register double sum=0; register WORD *ai,*bj; ai=a->words; bj=b->words; while (ai->wnum && bj->wnum) { if(ai->wnum > bj->wnum) { bj++; } else if (ai->wnum < bj->wnum) { ai++; } else { sum+=(ai->weight) * (bj->weight); ai++; bj++; } } return((double)sum); } SVECTOR* multadd_ss(SVECTOR *a, SVECTOR *b, double fa, double fb) { return(multadd_ss_r(a,b,fa,fb,0)); } SVECTOR* multadd_ss_r(SVECTOR *a,SVECTOR *b,double fa, double fb, double min_non_zero) /* compute fa*a+fb*b of two sparse vectors */ /* Note: SVECTOR lists are not followed, but only the first SVECTOR is used */ { SVECTOR *vec; register WORD *sum,*sumi; register WORD *ai,*bj; long veclength; double weight; ai=a->words; bj=b->words; veclength=0; while (ai->wnum && bj->wnum) { if(ai->wnum > bj->wnum) { veclength++; bj++; } else if (ai->wnum < bj->wnum) { veclength++; ai++; } else { veclength++; ai++; bj++; } } while (bj->wnum) { veclength++; bj++; } while (ai->wnum) { veclength++; ai++; } veclength++; sum=(WORD *)my_malloc(sizeof(WORD)*veclength); sumi=sum; ai=a->words; bj=b->words; while (ai->wnum && bj->wnum) { if(ai->wnum > bj->wnum) { (*sumi)=(*bj); sumi->weight*=fb; sumi++; bj++; } else if (ai->wnum < bj->wnum) { (*sumi)=(*ai); sumi->weight*=fa; sumi++; ai++; } else { weight=fa*(double)ai->weight+fb*(double)bj->weight; if((weight<-min_non_zero) || (weight>min_non_zero)) { sumi->wnum=ai->wnum; sumi->weight=weight; sumi++; } ai++; bj++; } } while (bj->wnum) { (*sumi)=(*bj); sumi->weight*=fb; sumi++; bj++; } while (ai->wnum) { (*sumi)=(*ai); sumi->weight*=fa; sumi++; ai++; } sumi->wnum=0; if(1) { /* potentially this wastes some memory, but saves malloc'ing */ vec=create_svector_shallow(sum,NULL,1.0); } else { /* this is more memory efficient */ vec=create_svector(sum,NULL,1.0); free(sum); } return(vec); } SVECTOR* sub_ss(SVECTOR *a, SVECTOR *b) /* compute the difference a-b of two sparse vectors */ /* Note: SVECTOR lists are not followed, but only the first SVECTOR is used */ { return(multadd_ss_r(a,b,1.0,-1.0,0)); } SVECTOR* sub_ss_r(SVECTOR *a, SVECTOR *b, double min_non_zero) /* compute the difference a-b of two sparse vectors and rounds to zero */ /* Note: SVECTOR lists are not followed, but only the first SVECTOR is used */ { return(multadd_ss_r(a,b,1.0,-1.0,min_non_zero)); } SVECTOR* add_ss(SVECTOR *a, SVECTOR *b) /* compute the sum a+b of two sparse vectors */ /* Note: SVECTOR lists are not followed, but only the first SVECTOR is used */ { return(multadd_ss_r(a,b,1.0,1.0,0)); } SVECTOR* add_ss_r(SVECTOR *a, SVECTOR *b, double min_non_zero) /* compute the sum a+b of two sparse vectors and rounds to zero */ /* Note: SVECTOR lists are not followed, but only the first SVECTOR is used */ { return(multadd_ss_r(a,b,1.0,1.0,min_non_zero)); } SVECTOR* add_list_ss(SVECTOR *a) { return(add_list_ss_r(a,0)); } SVECTOR* add_dual_list_ss_r(SVECTOR *a, SVECTOR *b, double min_non_zero) /* computes the linear combination of the two SVECTOR lists weighted by the factor of each SVECTOR */ { SVECTOR *f,*sum; for(f=a;f->next;f=f->next); /* find end of first vector list */ f->next=b; /* temporarily append the two vector lists */ sum=add_list_ss_r(a,min_non_zero); f->next=NULL; /* undo append */ return(sum); } SVECTOR* add_list_ss_r(SVECTOR *a, double min_non_zero) /* computes the linear combination of the SVECTOR list weighted by the factor of each SVECTOR */ { SVECTOR *oldsum,*sum,*f; WORD empty[2]; if(!a) { empty[0].wnum=0; sum=create_svector(empty,NULL,1.0); } else if(a && (!a->next)) { sum=smult_s(a,a->factor); } else { sum=multadd_ss_r(a,a->next,a->factor,a->next->factor,min_non_zero); for(f=a->next->next;f;f=f->next) { oldsum=sum; sum=multadd_ss_r(oldsum,f,1.0,f->factor,min_non_zero); free_svector(oldsum); } } return(sum); } int compareup_word(const void *a, const void *b) { double va,vb; va=((WORD *)a)->wnum; vb=((WORD *)b)->wnum; return((va > vb) - (va < vb)); } SVECTOR* add_list_sort_ss(SVECTOR *a) /* computes the linear combination of the SVECTOR list weighted by the factor of each SVECTOR. This should be a lot faster than add_list_ss for long lists. */ { return(add_list_sort_ss_r(a,0)); } SVECTOR* add_dual_list_sort_ss_r(SVECTOR *a, SVECTOR *b, double min_non_zero) /* computes the linear combination of the two SVECTOR lists weighted by the factor of each SVECTOR */ { SVECTOR *f,*sum; for(f=a;f->next;f=f->next); /* find end of first vector list */ f->next=b; /* temporarily append the two vector lists */ sum=add_list_sort_ss_r(a,min_non_zero); f->next=NULL; /* undo append */ return(sum); } SVECTOR* add_list_sort_ss_r(SVECTOR *a, double min_non_zero) /* Like add_list_sort_ss(SVECTOR *a), but rounds values smaller than min_non_zero to zero. */ { SVECTOR *sum,*f; WORD empty[2],*ai,*concat,*concati,*concat_read,*concat_write; long length,i; double weight; if(a){ /* count number or entries over all vectors */ length=0; for(f=a;f;f=f->next) { ai=f->words; while (ai->wnum) { length++; ai++; } } /* write all entries into one long array and sort by feature number */ concat=(WORD *)my_malloc(sizeof(WORD)*(length+1)); concati=concat; for(f=a;f;f=f->next) { ai=f->words; while (ai->wnum) { (*concati)=(*ai); concati->weight*=f->factor; concati++; ai++; } } qsort(concat,length,sizeof(WORD),compareup_word); concat_read=concat+1; concat_write=concat; for(i=0;(iwnum != concat_read->wnum);i++) { concat_write++; concat_read++; } weight=concat_write->weight; for(i=i;(iwnum == concat_read->wnum) { weight+=(double)concat_read->weight; concat_read++; } else { if((weight > min_non_zero) || (weight < -min_non_zero)) { concat_write->weight=weight; concat_write++; } (*concat_write)=(*concat_read); weight=concat_write->weight; concat_read++; } } if((length>0) && ((weight > min_non_zero) || (weight < -min_non_zero))) { concat_write->weight=weight; concat_write++; } concat_write->wnum=0; if(1) { /* this wastes some memory, but saves malloc'ing */ sum=create_svector_shallow(concat,NULL,1.0); } else { /* this is more memory efficient */ sum=create_svector(concat,NULL,1.0); free(concat); } } else { empty[0].wnum=0; sum=create_svector(empty,NULL,1.0); } return(sum); } SVECTOR* add_list_ns(SVECTOR *a) { return(add_list_ns_r(a,0)); } SVECTOR* add_dual_list_ns_r(SVECTOR *a, SVECTOR *b, double min_non_zero) /* computes the linear combination of the two SVECTOR lists weighted by the factor of each SVECTOR */ { SVECTOR *f,*sum; for(f=a;f->next;f=f->next); /* find end of first vector list */ f->next=b; /* temporarily append the two vector lists */ sum=add_list_ns_r(a,min_non_zero); f->next=NULL; /* undo append */ return(sum); } SVECTOR* add_list_ns_r(SVECTOR *a, double min_non_zero) /* computes the linear combination of the SVECTOR list weighted by the factor of each SVECTOR. assumes that the number of features is small compared to the number of elements in the list */ { SVECTOR *vec,*f; register WORD *ai; long totwords; double *sum; /* find max feature number */ totwords=0; for(f=a;f;f=f->next) { ai=f->words; while (ai->wnum) { if(totwordswnum) totwords=ai->wnum; ai++; } } sum=create_nvector(totwords); clear_nvector(sum,totwords); for(f=a;f;f=f->next) add_vector_ns(sum,f,f->factor); vec=create_svector_n_r(sum,totwords,NULL,1.0,min_non_zero); free(sum); return(vec); } void add_list_n_ns(double *vec_n, SVECTOR *vec_s, double faktor) { SVECTOR *f; for(f=vec_s;f;f=f->next) add_vector_ns(vec_n,f,f->factor*faktor); } void append_svector_list(SVECTOR *a, SVECTOR *b) /* appends SVECTOR b to the end of SVECTOR a. */ { SVECTOR *f; for(f=a;f->next;f=f->next); /* find end of first vector list */ f->next=b; /* append the two vector lists */ } void mult_svector_list(SVECTOR *a, double factor) /* multiplies the factor of each element in vector list with factor */ { SVECTOR *f; for(f=a;f;f=f->next) f->factor*=factor; } void setfactor_svector_list(SVECTOR *a, double factor) /* sets the factor of each element in vector list to factor */ { SVECTOR *f; for(f=a;f;f=f->next) f->factor=factor; } SVECTOR* smult_s(SVECTOR *a, double factor) /* scale sparse vector a by factor */ { SVECTOR *vec; register WORD *sum,*sumi; register WORD *ai; long veclength; ai=a->words; veclength=0; while (ai->wnum) { veclength++; ai++; } veclength++; sum=(WORD *)my_malloc(sizeof(WORD)*veclength); sumi=sum; ai=a->words; while (ai->wnum) { (*sumi)=(*ai); sumi->weight*=factor; if(sumi->weight != 0) sumi++; ai++; } sumi->wnum=0; vec=create_svector_shallow(sum,a->userdefined,1.0); return(vec); } SVECTOR* shift_s(SVECTOR *a, long shift) /* shifts the feature numbers by shift positions */ { SVECTOR *vec; register WORD *sum,*sumi; register WORD *ai; long veclength; ai=a->words; veclength=0; while (ai->wnum) { veclength++; ai++; } veclength++; sum=(WORD *)my_malloc(sizeof(WORD)*veclength); sumi=sum; ai=a->words; while (ai->wnum) { (*sumi)=(*ai); sumi->wnum+=shift; ai++; sumi++; } sumi->wnum=0; vec=create_svector_shallow(sum,a->userdefined,a->factor); return(vec); } int featvec_eq(SVECTOR *a, SVECTOR *b) /* tests two sparse vectors for equality */ { register WORD *ai,*bj; ai=a->words; bj=b->words; while (ai->wnum && bj->wnum) { if(ai->wnum > bj->wnum) { if((bj->weight) != 0) return(0); bj++; } else if (ai->wnum < bj->wnum) { if((ai->weight) != 0) return(0); ai++; } else { if((ai->weight) != (bj->weight)) return(0); ai++; bj++; } } return(1); } double model_length_s(MODEL *model) /* compute length of weight vector */ { register long i,j; register double sum=0,alphai; register DOC *supveci; KERNEL_PARM *kernel_parm=&(model->kernel_parm); for(i=1;isv_num;i++) { alphai=model->alpha[i]; supveci=model->supvec[i]; for(j=1;jsv_num;j++) { sum+=alphai*model->alpha[j] *kernel(kernel_parm,supveci,model->supvec[j]); } } return(sqrt(sum)); } double model_length_n(MODEL *model) /* compute length of weight vector */ { long i,totwords=model->totwords+1; double sum,*weight_n; SVECTOR *weight; if(model->kernel_parm.kernel_type != LINEAR) { printf("ERROR: model_length_n applies only to linear kernel!\n"); exit(1); } weight_n=create_nvector(totwords); clear_nvector(weight_n,totwords); for(i=1;isv_num;i++) add_list_n_ns(weight_n,model->supvec[i]->fvec,model->alpha[i]); weight=create_svector_n(weight_n,totwords,NULL,1.0); sum=sprod_ss(weight,weight); free(weight_n); free_svector(weight); return(sqrt(sum)); } void mult_vector_ns(double *vec_n, SVECTOR *vec_s, double faktor) { register WORD *ai; ai=vec_s->words; while (ai->wnum) { vec_n[ai->wnum]*=(faktor*(double)ai->weight); ai++; } } void add_vector_ns(double *vec_n, SVECTOR *vec_s, double faktor) { /* Note: SVECTOR lists are not followed, but only the first SVECTOR is used */ register WORD *ai; ai=vec_s->words; while (ai->wnum) { vec_n[ai->wnum]+=(faktor*(double)ai->weight); ai++; } } double sprod_ns(double *vec_n, SVECTOR *vec_s) { register double sum=0; register WORD *ai; ai=vec_s->words; while (ai->wnum) { sum+=(vec_n[ai->wnum]*(double)ai->weight); ai++; } return(sum); } void add_weight_vector_to_linear_model(MODEL *model) /* compute weight vector in linear case and add to model */ { long i; SVECTOR *f; model->lin_weights=create_nvector(model->totwords); clear_nvector(model->lin_weights,model->totwords); for(i=1;isv_num;i++) { for(f=(model->supvec[i])->fvec;f;f=f->next) add_vector_ns(model->lin_weights,f,f->factor*model->alpha[i]); } } DOC *create_example(long docnum, long queryid, long slackid, double costfactor, SVECTOR *fvec) { DOC *example; example = (DOC *)my_malloc(sizeof(DOC)); example->docnum=docnum; example->kernelid=docnum; example->queryid=queryid; example->slackid=slackid; example->costfactor=costfactor; example->fvec=fvec; return(example); } void free_example(DOC *example, long deep) { if(example) { if(deep) { if(example->fvec) free_svector(example->fvec); } free(example); } } int compare_randpair(const void *a, const void *b) { long va,vb; va=((RANDPAIR *)a)->sort; vb=((RANDPAIR *)b)->sort; return((va > vb) - (va < vb)); } long *random_order(long n) /* creates an array of the integers [0..n-1] in random order */ { long *randarray=(long *)my_malloc(sizeof(long)*n); RANDPAIR *randpair=(RANDPAIR *)my_malloc(sizeof(RANDPAIR)*n); long i; for(i=0;in=n; matrix->m=m; matrix->element=(double **)my_malloc(sizeof(double *)*n); for(i=0;ielement[i]=(double *)my_malloc(sizeof(double)*m); } return(matrix); } MATRIX *realloc_matrix(MATRIX *matrix, int n, int m) /* extends/shrinks matrix to n rows and m colums. Not that added elements are not initialized. */ { int i; if(!matrix) return(create_matrix(n,m)); for(i=n;in;i++) free(matrix->element[i]); matrix->element=(double **)realloc(matrix->element,sizeof(double *)*n); for(i=matrix->n;ielement[i]=(double *)my_malloc(sizeof(double)*m); for(i=0;in);i++) { matrix->element[i]=(double *)realloc(matrix->element[i],sizeof(double)*m); } matrix->n=n; matrix->m=m; return(matrix); } double *create_nvector(int n) /* creates a dense column vector with n+1 rows. unfortunately, there is part of the code that starts counting at 0, while the sparse vectors start counting at 1. So, it always allocates one extra row. */ { double *vector; vector=(double *)my_malloc(sizeof(double)*(n+1)); return(vector); } void clear_nvector(double *vec, long int n) { register long i; for(i=0;i<=n;i++) vec[i]=0; } MATRIX *copy_matrix(MATRIX *matrix) /* create deep copy of matrix */ { int i,j; MATRIX *copy; copy=create_matrix(matrix->n,matrix->m); for(i=0;in;i++) { for(j=0;jm;j++) { copy->element[i][j]=matrix->element[i][j]; } } return(copy); } void free_matrix(MATRIX *matrix) /* deallocates memory */ { int i; for(i=0;in;i++) { free(matrix->element[i]); } free(matrix->element); free(matrix); } void free_nvector(double *vector) /* deallocates memory */ { free(vector); } MATRIX *transpose_matrix(MATRIX *matrix) /* returns copy with transpose of matrix */ { int i,j; MATRIX *copy; copy=create_matrix(matrix->m,matrix->n); for(i=0;in;i++) { for(j=0;jm;j++) { copy->element[j][i]=matrix->element[i][j]; } } return(copy); } MATRIX *cholesky_matrix(MATRIX *A) /* Given a positive-definite symmetric matrix A[0..n-1][0..n-1], this routine constructs its Cholesky decomposition, A = L ∑ LT . On input, only the upper triangle of A need be given; A is not modified. The Cholesky factor L is returned in the lower triangle. */ { int i,j,k,n; double sum; MATRIX *L; if(A->m != A->n) { printf("ERROR: Matrix not quadratic. Cannot compute Cholesky!\n"); exit(1); } n=A->n; L=copy_matrix(A); for (i=0;ielement[i][j],k=i-1;k>=0;k--) sum -= L->element[i][k]*L->element[j][k]; if (i == j) { if (sum <= 0.0) printf("Cholesky: Matrix not positive definite"); L->element[i][i]=sqrt(sum); } else L->element[j][i]=sum/L->element[i][i]; } } /* set upper triange to zero */ for (i=0;ielement[i][j]=0; return(L); } double *find_indep_subset_of_matrix(MATRIX *A, double epsilon) /* Given a positive-semidefinite symmetric matrix A[0..n-1][0..n-1], this routine finds a subset of rows and colums that is linear independent. To do this, it constructs the Cholesky decomposition, A = L ∑ LT. On input, only the upper triangle of A need be given; A is not modified. The routine returns a vector in which non-zero elements indicate the linear independent subset. epsilon is the amount by which the diagonal entry of L has to be greater than zero. */ { int i,j,k,n; double sum,*indep; MATRIX *L; if(A->m != A->n) { printf("ERROR: Matrix not quadratic. Cannot compute Cholesky!\n"); exit(1); } n=A->n; L=copy_matrix(A); for (i=0;ielement[i][j],k=i-1;k>=0;k--) sum -= L->element[i][k]*L->element[j][k]; if (i == j) { if (sum <= epsilon) sum=0; L->element[i][i]=sqrt(sum); } else if(L->element[i][i] == 0) L->element[j][i]=0; else L->element[j][i]=sum/L->element[i][i]; } } /* Gather non-zero diagonal elements */ indep=create_nvector(n); for (i=0;ielement[i][i]; free_matrix(L); return(indep); } MATRIX *invert_ltriangle_matrix(MATRIX *L) /* Given a lower triangular matrix L, computes inverse L^-1 */ { int i,j,k,n; double sum; MATRIX *I; if(L->m != L->n) { printf("ERROR: Matrix not quadratic. Cannot invert triangular matrix!\n"); exit(1); } n=L->n; I=copy_matrix(L); for (i=0;ielement[i][i]=1.0/L->element[i][i]; for (j=i+1;jelement[j][k]*I->element[k][i]; I->element[j][i]=sum/L->element[j][j]; } } return(I); } double *prod_nvector_matrix(double *v, MATRIX *A) /* For column vector v and matrix A (assumed to match in size), computes w^T=v^T*A */ { int i,j; double sum; double *w; w=create_nvector(A->m); for (i=0;im;i++) { sum=0.0; for (j=0;jn;j++) { sum+=v[j]*A->element[j][i]; } w[i]=sum; } return(w); } double *prod_matrix_nvector(MATRIX *A, double *v) /* For column vector v and matrix A (assumed to match in size), computes w=A*v */ { int i,j; double sum; double *w; w=create_nvector(A->n); for (i=0;in;i++) { sum=0.0; for (j=0;jm;j++) { sum+=v[j]*A->element[i][j]; } w[i]=sum; } return(w); } double *prod_nvector_ltmatrix(double *v, MATRIX *A) /* For column vector v and a lower triangular matrix A (assumed to match in size), computes w^T=v^T*A */ { int i,j; double sum; double *w; w=create_nvector(A->m); for (i=0;im;i++) { sum=0.0; for (j=i;jn;j++) { sum+=v[j]*A->element[j][i]; } w[i]=sum; } return(w); } double *prod_ltmatrix_nvector(MATRIX *A, double *v) /* For column vector v and lower triangular matrix A (assumed to match in size), computes w=A*v */ { int i,j; double sum; double *w; w=create_nvector(A->n); for (i=0;in;i++) { sum=0.0; for (j=0;j<=i;j++) { sum+=v[j]*A->element[i][j]; } w[i]=sum; } return(w); } MATRIX *prod_matrix_matrix(MATRIX *A, MATRIX *B) /* For matrices A and B (assumed to match in size), computes C=A*B */ { int i,j,k; double sum; MATRIX *C; if(A->m != B->n) { printf("ERROR: Matrix size does not match. Cannot compute product!\n"); exit(1); } C=create_matrix(A->n,B->m); for (i=0;in;i++) { for (j=0;jm;j++) { sum=0.0; for (k=0;km;k++) { sum+=A->element[i][k]*B->element[k][j]; } C->element[i][j]=sum; } } return(C); } void print_matrix(MATRIX *matrix) /* prints matrix to STDOUT */ { int i,j; printf("\n"); printf("\n"); for(i=0;in;i++) { for(j=0;jm;j++) { printf("%4.3f\t",matrix->element[i][j]); } printf("\n"); } } /***************************** IO routines ***************************/ #if 0 void write_model(char *modelfile, MODEL *model) { FILE *modelfl; long j,i,sv_num; SVECTOR *v; MODEL *compact_model=NULL; if(verbosity>=1) { printf("Writing model file..."); fflush(stdout); } /* Replace SV with single weight vector */ if(0 && model->kernel_parm.kernel_type == LINEAR) { if(verbosity>=1) { printf("(compacting..."); fflush(stdout); } compact_model=compact_linear_model(model); model=compact_model; if(verbosity>=1) { printf("done)"); fflush(stdout); } } if ((modelfl = fopen (modelfile, "w")) == NULL) { perror (modelfile); exit (1); } fprintf(modelfl,"SVM-light Version %s\n",VERSION); fprintf(modelfl,"%ld # kernel type\n", model->kernel_parm.kernel_type); fprintf(modelfl,"%ld # kernel parameter -d \n", model->kernel_parm.poly_degree); fprintf(modelfl,"%.8g # kernel parameter -g \n", model->kernel_parm.rbf_gamma); fprintf(modelfl,"%.8g # kernel parameter -s \n", model->kernel_parm.coef_lin); fprintf(modelfl,"%.8g # kernel parameter -r \n", model->kernel_parm.coef_const); fprintf(modelfl,"%s# kernel parameter -u \n",model->kernel_parm.custom); fprintf(modelfl,"%ld # highest feature index \n",model->totwords); fprintf(modelfl,"%ld # number of training documents \n",model->totdoc); sv_num=1; for(i=1;isv_num;i++) { for(v=model->supvec[i]->fvec;v;v=v->next) sv_num++; } fprintf(modelfl,"%ld # number of support vectors plus 1 \n",sv_num); fprintf(modelfl,"%.8g # threshold b, each following line is a SV (starting with alpha*y)\n",model->b); for(i=1;isv_num;i++) { for(v=model->supvec[i]->fvec;v;v=v->next) { fprintf(modelfl,"%.32g ",model->alpha[i]*v->factor); for (j=0; (v->words[j]).wnum; j++) { fprintf(modelfl,"%ld:%.8g ", (long)(v->words[j]).wnum, (double)(v->words[j]).weight); } if(v->userdefined) fprintf(modelfl,"#%s\n",v->userdefined); else fprintf(modelfl,"#\n"); /* NOTE: this could be made more efficient by summing the alpha's of identical vectors before writing them to the file. */ } } fclose(modelfl); if(compact_model) free_model(compact_model,1); if(verbosity>=1) { printf("done\n"); } } #endif #if 0 MODEL *read_model(char *modelfile) { FILE *modelfl; long i,queryid,slackid; double costfactor; long max_sv,max_words,ll,wpos; char *line,*comment; WORD *words; char version_buffer[100]; MODEL *model; if(verbosity>=1) { printf("Reading model..."); fflush(stdout); } nol_ll(modelfile,&max_sv,&max_words,&ll); /* scan size of model file */ max_words+=2; ll+=2; words = (WORD *)my_malloc(sizeof(WORD)*(max_words+10)); line = (char *)my_malloc(sizeof(char)*ll); model = (MODEL *)my_malloc(sizeof(MODEL)); if ((modelfl = fopen (modelfile, "r")) == NULL) { perror (modelfile); exit (1); } fscanf(modelfl,"SVM-light Version %s\n",version_buffer); if(strcmp(version_buffer,VERSION)) { perror ("Version of model-file does not match version of svm_classify!"); exit (1); } fscanf(modelfl,"%ld%*[^\n]\n", &model->kernel_parm.kernel_type); fscanf(modelfl,"%ld%*[^\n]\n", &model->kernel_parm.poly_degree); fscanf(modelfl,"%lf%*[^\n]\n", &model->kernel_parm.rbf_gamma); fscanf(modelfl,"%lf%*[^\n]\n", &model->kernel_parm.coef_lin); fscanf(modelfl,"%lf%*[^\n]\n", &model->kernel_parm.coef_const); fscanf(modelfl,"%[^#]%*[^\n]\n", model->kernel_parm.custom); fscanf(modelfl,"%ld%*[^\n]\n", &model->totwords); fscanf(modelfl,"%ld%*[^\n]\n", &model->totdoc); fscanf(modelfl,"%ld%*[^\n]\n", &model->sv_num); fscanf(modelfl,"%lf%*[^\n]\n", &model->b); model->supvec = (DOC **)my_malloc(sizeof(DOC *)*model->sv_num); model->alpha = (double *)my_malloc(sizeof(double)*model->sv_num); model->index=NULL; model->lin_weights=NULL; for(i=1;isv_num;i++) { fgets(line,(int)ll,modelfl); if(!parse_document(line,words,&(model->alpha[i]),&queryid,&slackid, &costfactor,&wpos,max_words,&comment)) { printf("\nParsing error while reading model file in SV %ld!\n%s", i,line); exit(1); } model->supvec[i] = create_example(-1, 0,0, 0.0, create_svector(words,comment,1.0)); } fclose(modelfl); free(line); free(words); if(verbosity>=1) { fprintf(stdout, "OK. (%d support vectors read)\n",(int)(model->sv_num-1)); } return(model); } #endif MODEL *copy_model(MODEL *model) { MODEL *newmodel; long i; newmodel=(MODEL *)my_malloc(sizeof(MODEL)); (*newmodel)=(*model); newmodel->supvec = (DOC **)my_malloc(sizeof(DOC *)*model->sv_num); newmodel->alpha = (double *)my_malloc(sizeof(double)*model->sv_num); newmodel->index = NULL; /* index is not copied */ newmodel->supvec[0] = NULL; newmodel->alpha[0] = 0; for(i=1;isv_num;i++) { newmodel->alpha[i]=model->alpha[i]; newmodel->supvec[i]=create_example(model->supvec[i]->docnum, model->supvec[i]->queryid,0, model->supvec[i]->costfactor, copy_svector(model->supvec[i]->fvec)); } if(model->lin_weights) { newmodel->lin_weights = (double *)my_malloc(sizeof(double)*(model->totwords+1)); for(i=0;itotwords+1;i++) newmodel->lin_weights[i]=model->lin_weights[i]; } return(newmodel); } MODEL *compact_linear_model(MODEL *model) /* Makes a copy of model where the support vectors are replaced with a single linear weight vector. */ /* NOTE: It adds the linear weight vector also to newmodel->lin_weights */ /* WARNING: This is correct only for linear models! */ { MODEL *newmodel; newmodel=(MODEL *)my_malloc(sizeof(MODEL)); (*newmodel)=(*model); add_weight_vector_to_linear_model(newmodel); newmodel->supvec = (DOC **)my_malloc(sizeof(DOC *)*2); newmodel->alpha = (double *)my_malloc(sizeof(double)*2); newmodel->index = NULL; /* index is not copied */ newmodel->supvec[0] = NULL; newmodel->alpha[0] = 0.0; newmodel->supvec[1] = create_example(-1,0,0,0, create_svector_n(newmodel->lin_weights, newmodel->totwords, NULL,1.0)); newmodel->alpha[1] = 1.0; newmodel->sv_num=2; return(newmodel); } void free_model(MODEL *model, int deep) { long i; if(model->supvec) { if(deep) { for(i=1;isv_num;i++) { free_example(model->supvec[i],1); } } free(model->supvec); } if(model->alpha) free(model->alpha); if(model->index) free(model->index); if(model->lin_weights) free(model->lin_weights); free(model); } #if 0 void read_documents(char *docfile, DOC ***docs, double **label, long int *totwords, long int *totdoc) { char *line,*comment; WORD *words; long dnum=0,wpos,dpos=0,dneg=0,dunlab=0,queryid,slackid,max_docs; long max_words_doc, ll; double doc_label,costfactor; FILE *docfl; if(verbosity>=1) { printf("Scanning examples..."); fflush(stdout); } nol_ll(docfile,&max_docs,&max_words_doc,&ll); /* scan size of input file */ max_words_doc+=2; ll+=2; max_docs+=2; if(verbosity>=1) { printf("done\n"); fflush(stdout); } (*docs) = (DOC **)my_malloc(sizeof(DOC *)*max_docs); /* feature vectors */ (*label) = (double *)my_malloc(sizeof(double)*max_docs); /* target values */ line = (char *)my_malloc(sizeof(char)*ll); if ((docfl = fopen (docfile, "r")) == NULL) { perror (docfile); exit (1); } words = (WORD *)my_malloc(sizeof(WORD)*(max_words_doc+10)); if(verbosity>=1) { printf("Reading examples into memory..."); fflush(stdout); } dnum=0; (*totwords)=0; while((!feof(docfl)) && fgets(line,(int)ll,docfl)) { if(line[0] == '#') continue; /* line contains comments */ if(!parse_document(line,words,&doc_label,&queryid,&slackid,&costfactor, &wpos,max_words_doc,&comment)) { printf("\nParsing error in line %ld!\n%s",dnum,line); exit(1); } (*label)[dnum]=doc_label; /* printf("docnum=%ld: Class=%f ",dnum,doc_label); */ if(doc_label > 0) dpos++; if (doc_label < 0) dneg++; if (doc_label == 0) dunlab++; if((wpos>1) && ((words[wpos-2]).wnum>(*totwords))) (*totwords)=(words[wpos-2]).wnum; if((*totwords) > MAXFEATNUM) { printf("\nMaximum feature number exceeds limit defined in MAXFEATNUM!\n"); printf("LINE: %s\n",line); exit(1); } (*docs)[dnum] = create_example(dnum,queryid,slackid,costfactor, create_svector(words,comment,1.0)); /* printf("\nNorm=%f\n",((*docs)[dnum]->fvec)->twonorm_sq); */ dnum++; if(verbosity>=1) { if((dnum % 100) == 0) { printf("%ld..",dnum); fflush(stdout); } } } fclose(docfl); free(line); free(words); if(verbosity>=1) { fprintf(stdout, "OK. (%ld examples read)\n", dnum); } (*totdoc)=dnum; } int parse_document(char *line, WORD *words, double *label, long *queryid, long *slackid, double *costfactor, long int *numwords, long int max_words_doc, char **comment) { register long wpos,pos; long wnum; double weight; char featurepair[1000],junk[1000]; (*queryid)=0; (*slackid)=0; (*costfactor)=1; pos=0; (*comment)=NULL; while(line[pos] ) { /* cut off comments */ if((line[pos] == '#') && (!(*comment))) { line[pos]=0; (*comment)=&(line[pos+1]); } if(line[pos] == '\n') { /* strip the CR */ line[pos]=0; } pos++; } if(!(*comment)) (*comment)=&(line[pos]); /* printf("Comment: '%s'\n",(*comment)); */ wpos=0; /* check, that line starts with target value or zero, but not with feature pair */ if(sscanf(line,"%s",featurepair) == EOF) return(0); pos=0; while((featurepair[pos] != ':') && featurepair[pos]) pos++; if(featurepair[pos] == ':') { perror ("Line must start with label or 0!!!\n"); printf("LINE: %s\n",line); exit (1); } /* read the target value */ if(sscanf(line,"%lf",label) == EOF) return(0); pos=0; while(space_or_null((int)line[pos])) pos++; while((!space_or_null((int)line[pos])) && line[pos]) pos++; while((pos+=read_word(line+pos,featurepair)) && (featurepair[0]) && (wpos 0) (*slackid)=(long)wnum; else { perror ("Slack-id must be greater or equal to 1!!!\n"); printf("LINE: %s\n",line); exit (1); } } else if(sscanf(featurepair,"cost:%lf%s",&weight,junk)==1) { /* it is the example-dependent cost factor */ (*costfactor)=(double)weight; } else if(sscanf(featurepair,"%ld:%lf%s",&wnum,&weight,junk)==2) { /* it is a regular feature */ if(wnum<=0) { perror ("Feature numbers must be larger or equal to 1!!!\n"); printf("LINE: %s\n",line); exit (1); } if((wpos>0) && ((words[wpos-1]).wnum >= wnum)) { perror ("Features must be in increasing order!!!\n"); printf("LINE: %s\n",line); exit (1); } (words[wpos]).wnum=wnum; (words[wpos]).weight=(FVAL)weight; wpos++; } else { perror ("Cannot parse feature/value pair!!!\n"); printf("'%s' in LINE: %s\n",featurepair,line); exit (1); } } (words[wpos]).wnum=0; (*numwords)=wpos+1; return(1); } double *read_alphas(char *alphafile,long totdoc) /* reads the alpha vector from a file as written by the write_alphas function */ { FILE *fl; double *alpha; long dnum; if ((fl = fopen (alphafile, "r")) == NULL) { perror (alphafile); exit (1); } alpha = (double *)my_malloc(sizeof(double)*totdoc); if(verbosity>=1) { printf("Reading alphas..."); fflush(stdout); } dnum=0; while((!feof(fl)) && fscanf(fl,"%lf\n",&alpha[dnum]) && (dnum=1) { printf("done\n"); fflush(stdout); } return(alpha); } #endif void set_learning_defaults(LEARN_PARM *learn_parm, KERNEL_PARM *kernel_parm) { learn_parm->type=CLASSIFICATION; strcpy (learn_parm->predfile, "trans_predictions"); strcpy (learn_parm->alphafile, ""); learn_parm->biased_hyperplane=1; learn_parm->sharedslack=0; learn_parm->remove_inconsistent=0; learn_parm->skip_final_opt_check=0; learn_parm->svm_maxqpsize=10; learn_parm->svm_newvarsinqp=0; learn_parm->svm_iter_to_shrink=-9999; learn_parm->maxiter=100000; learn_parm->kernel_cache_size=40; learn_parm->svm_c=0.0; learn_parm->eps=0.1; learn_parm->transduction_posratio=-1.0; learn_parm->svm_costratio=1.0; learn_parm->svm_costratio_unlab=1.0; learn_parm->svm_unlabbound=1E-5; learn_parm->epsilon_crit=0.001; learn_parm->epsilon_a=1E-15; learn_parm->compute_loo=0; learn_parm->rho=1.0; learn_parm->xa_depth=0; kernel_parm->kernel_type=LINEAR; kernel_parm->poly_degree=3; kernel_parm->rbf_gamma=1.0; kernel_parm->coef_lin=1; kernel_parm->coef_const=1; strcpy(kernel_parm->custom,"empty"); } int check_learning_parms(LEARN_PARM *learn_parm, KERNEL_PARM *kernel_parm) { if((learn_parm->skip_final_opt_check) && (kernel_parm->kernel_type == LINEAR)) { printf("\nIt does not make sense to skip the final optimality check for linear kernels.\n\n"); learn_parm->skip_final_opt_check=0; } if((learn_parm->skip_final_opt_check) && (learn_parm->remove_inconsistent)) { printf("\nIt is necessary to do the final optimality check when removing inconsistent \nexamples.\n"); return(0); } if((learn_parm->svm_maxqpsize<2)) { printf("\nMaximum size of QP-subproblems not in valid range: %ld [2..]\n",learn_parm->svm_maxqpsize); return(0); } if((learn_parm->svm_maxqpsizesvm_newvarsinqp)) { printf("\nMaximum size of QP-subproblems [%ld] must be larger than the number of\n",learn_parm->svm_maxqpsize); printf("new variables [%ld] entering the working set in each iteration.\n",learn_parm->svm_newvarsinqp); return(0); } if(learn_parm->svm_iter_to_shrink<1) { printf("\nMaximum number of iterations for shrinking not in valid range: %ld [1,..]\n",learn_parm->svm_iter_to_shrink); return(0); } if(learn_parm->svm_c<0) { printf("\nThe C parameter must be greater than zero!\n\n"); return(0); } if(learn_parm->transduction_posratio>1) { printf("\nThe fraction of unlabeled examples to classify as positives must\n"); printf("be less than 1.0 !!!\n\n"); return(0); } if(learn_parm->svm_costratio<=0) { printf("\nThe COSTRATIO parameter must be greater than zero!\n\n"); return(0); } if(learn_parm->epsilon_crit<=0) { printf("\nThe epsilon parameter must be greater than zero!\n\n"); return(0); } if(learn_parm->rho<0) { printf("\nThe parameter rho for xi/alpha-estimates and leave-one-out pruning must\n"); printf("be greater than zero (typically 1.0 or 2.0, see T. Joachims, Estimating the\n"); printf("Generalization Performance of an SVM Efficiently, ICML, 2000.)!\n\n"); return(0); } if((learn_parm->xa_depth<0) || (learn_parm->xa_depth>100)) { printf("\nThe parameter depth for ext. xi/alpha-estimates must be in [0..100] (zero\n"); printf("for switching to the conventional xa/estimates described in T. Joachims,\n"); printf("Estimating the Generalization Performance of an SVM Efficiently, ICML, 2000.)\n"); } return(1); } void nol_ll(char *file, long int *nol, long int *wol, long int *ll) /* Grep through file and count number of lines, maximum number of spaces per line, and longest line. */ { FILE *fl; int ic; char c; long current_length,current_wol; if ((fl = fopen (file, "r")) == NULL) { perror (file); exit (1); } current_length=0; current_wol=0; (*ll)=0; (*nol)=1; (*wol)=0; while((ic=getc(fl)) != EOF) { c=(char)ic; current_length++; if(space_or_null((int)c)) { current_wol++; } if(c == '\n') { (*nol)++; if(current_length>(*ll)) { (*ll)=current_length; } if(current_wol>(*wol)) { (*wol)=current_wol; } current_length=0; current_wol=0; } } fclose(fl); } long minl(long int a, long int b) { if(ab) return(a); else return(b); } double get_runtime(void) { /* returns the current processor time in hundredth of a second */ clock_t start; start = clock(); return((double)start/((double)(CLOCKS_PER_SEC)/100.0)); } # ifdef _MSC_VER int isnan(double a) { return(_isnan(a)); } # endif int space_or_null(int c) { if (c==0) return 1; return isspace((unsigned char)c); } int read_word(char *in, char *out) { int found=0; while(isspace((int)(*in)) && (*in)) { /* skip over whitespace */ in++; found++; } while(!space_or_null((int)(*in))) { /* read non-whitespace string */ (*out)=(*in); in++; found++; out++; } (*out)=0; return(found); } void *my_malloc(size_t size) { void *ptr; if(size<=0) size=1; /* for AIX compatibility */ ptr=(void *)malloc(size); if(!ptr) { perror ("Out of memory!\n"); exit (1); } return(ptr); } void copyright_notice(void) { printf("\nCopyright: Thorsten Joachims, thorsten@joachims.org\n\n"); printf("This software is available for non-commercial use only. It must not\n"); printf("be modified and distributed without prior permission of the author.\n"); printf("The author is not responsible for implications from the use of this\n"); printf("software.\n\n"); } svm-struct-matlab-1.0/svm_light/svm_common.h000066400000000000000000000411631160627511300212700ustar00rootroot00000000000000/************************************************************************/ /* */ /* svm_common.h */ /* */ /* Definitions and functions used in both svm_learn and svm_classify. */ /* */ /* Author: Thorsten Joachims */ /* Date: 31.10.05 */ /* */ /* Copyright (c) 2005 Thorsten Joachims - All rights reserved */ /* */ /* This software is available for non-commercial use only. It must */ /* not be modified and distributed without prior permission of the */ /* author. The author is not responsible for implications from the */ /* use of this software. */ /* */ /************************************************************************/ #ifndef SVM_COMMON #define SVM_COMMON #include "mex.h" #include #define malloc(x) (mxMalloc(x)) #define realloc(x,y) (mxRealloc((x),(y))) #define free(x) (mxFree(x)) struct MexPhiCustomImpl_ { int counter ; mxArray * x ; mxArray * y ; } ; typedef struct MexPhiCustomImpl_ * MexPhiCustom ; __inline__ static MexPhiCustom newMexPhiCustomFromPatternLabel (mxArray const * x, mxArray const *y) { MexPhiCustom phi ; phi = mxMalloc (sizeof(struct MexPhiCustomImpl_)) ; phi -> counter = 1 ; phi -> x = mxDuplicateArray (x) ; phi -> y = mxDuplicateArray (y) ; return phi ; } __inline__ static void releaseMexPhiCustom (MexPhiCustom phi) { if (phi) { phi -> counter -- ; if (phi -> counter == 0) { mxDestroyArray (phi -> x) ; mxDestroyArray (phi -> y) ; mxFree (phi) ; } } } __inline__ static void retainMexPhiCustom (MexPhiCustom phi) { if (phi) { phi -> counter ++ ; } } __inline__ static mxArray * MexPhiCustomGetPattern (MexPhiCustom phi) { assert (phi) ; return phi -> x ; } __inline__ static mxArray * MexPhiCustomGetLabel (MexPhiCustom phi) { assert (phi) ; return phi -> y ; } # include # include # include # include # include # include # include # include # define VERSION "V6.20" # define VERSION_DATE "14.08.08" # define CFLOAT float /* the type of float to use for caching */ /* kernel evaluations. Using float saves */ /* us some memory, but you can use double, too */ # define FNUM int32_t /* the type used for storing feature ids */ # define FNUM_MAX 2147483647 /* maximum value that FNUM type can take */ # define FVAL float /* the type used for storing feature values */ # define MAXFEATNUM 99999999 /* maximum feature number (must be in valid range of FNUM type and long int!) */ # define LINEAR 0 /* linear kernel type */ # define POLY 1 /* polynomial kernel type */ # define RBF 2 /* rbf kernel type */ # define SIGMOID 3 /* sigmoid kernel type */ # define CUSTOM 4 /* userdefined kernel function from kernel.h */ # define GRAM 5 /* use explicit gram matrix from kernel_parm */ # define CLASSIFICATION 1 /* train classification model */ # define REGRESSION 2 /* train regression model */ # define RANKING 3 /* train ranking model */ # define OPTIMIZATION 4 /* train on general set of constraints */ # define MAXSHRINK 50000 /* maximum number of shrinking rounds */ typedef struct word { FNUM wnum; /* word number */ FVAL weight; /* word weight */ } WORD; typedef struct svector { WORD *words; /* The features/values in the vector by increasing feature-number. Feature numbers that are skipped are interpreted as having value zero. */ double twonorm_sq; /* The squared euclidian length of the vector. Used to speed up the RBF kernel. */ /* char *userdefined; */ MexPhiCustom userdefined ; /* You can put additional information here. This can be useful, if you are implementing your own kernel that does not work with feature/values representations (for example a string kernel). By default, svm-light will put here the string after the # sign from each line of the input file. */ long kernel_id; /* Feature vectors with different kernel_id's are orthogonal (ie. the feature number do not match). This is used for computing component kernels for linear constraints which are a sum of several different weight vectors. (currently not implemented). */ struct svector *next; /* Let's you set up a list of SVECTOR's for linear constraints which are a sum of multiple feature vectors. List is terminated by NULL. */ double factor; /* Factor by which this feature vector is multiplied in the sum. */ } SVECTOR; typedef struct doc { long docnum; /* Document ID. This has to be the position of the document in the training set array. */ long queryid; /* for learning rankings, constraints are generated for documents with the same queryID. */ double costfactor; /* Scales the cost of misclassifying this document by this factor. The effect of this value is, that the upper bound on the alpha for this example is scaled by this factor. The factors are set by the feature 'cost:' in the training data. */ long slackid; /* Index of the slack variable corresponding to this constraint. All constraints with the same slackid share the same slack variable. This can only be used for svm_learn_optimization. */ long kernelid; /* Position in gram matrix where kernel value can be found when using an explicit gram matrix (i.e. kernel_type=GRAM). */ SVECTOR *fvec; /* Feature vector of the example. The feature vector can actually be a list of feature vectors. For example, the list will have two elements, if this DOC is a preference constraint. The one vector that is supposed to be ranked higher, will have a factor of +1, the lower ranked one should have a factor of -1. */ } DOC; typedef struct learn_parm { long type; /* selects between regression and classification */ double svm_c; /* upper bound C on alphas */ double eps; /* regression epsilon (eps=1.0 for classification */ double svm_costratio; /* factor to multiply C for positive examples */ double transduction_posratio;/* fraction of unlabeled examples to be */ /* classified as positives */ long biased_hyperplane; /* if nonzero, use hyperplane w*x+b=0 otherwise w*x=0 */ long sharedslack; /* if nonzero, it will use the shared slack variable mode in svm_learn_optimization. It requires that the slackid is set for every training example */ long svm_maxqpsize; /* size q of working set */ long svm_newvarsinqp; /* new variables to enter the working set in each iteration */ long kernel_cache_size; /* size of kernel cache in megabytes */ double epsilon_crit; /* tolerable error for distances used in stopping criterion */ double epsilon_shrink; /* how much a multiplier should be above zero for shrinking */ long svm_iter_to_shrink; /* iterations h after which an example can be removed by shrinking */ long maxiter; /* number of iterations after which the optimizer terminates, if there was no progress in maxdiff */ long remove_inconsistent; /* exclude examples with alpha at C and retrain */ long skip_final_opt_check; /* do not check KT-Conditions at the end of optimization for examples removed by shrinking. WARNING: This might lead to sub-optimal solutions! */ long compute_loo; /* if nonzero, computes leave-one-out estimates */ double rho; /* parameter in xi/alpha-estimates and for pruning leave-one-out range [1..2] */ long xa_depth; /* parameter in xi/alpha-estimates upper bounding the number of SV the current alpha_t is distributed over */ char predfile[200]; /* file for predicitions on unlabeled examples in transduction */ char alphafile[200]; /* file to store optimal alphas in. use empty string if alphas should not be output */ /* you probably do not want to touch the following */ double epsilon_const; /* tolerable error on eq-constraint */ double epsilon_a; /* tolerable error on alphas at bounds */ double opt_precision; /* precision of solver, set to e.g. 1e-21 if you get convergence problems */ /* the following are only for internal use */ long svm_c_steps; /* do so many steps for finding optimal C */ double svm_c_factor; /* increase C by this factor every step */ double svm_costratio_unlab; double svm_unlabbound; double *svm_cost; /* individual upper bounds for each var */ long totwords; /* number of features */ } LEARN_PARM; typedef struct matrix { int n; /* number of rows */ int m; /* number of colums */ double **element; } MATRIX; typedef struct kernel_parm { long kernel_type; /* 0=linear, 1=poly, 2=rbf, 3=sigmoid, 4=custom, 5=matrix */ long poly_degree; double rbf_gamma; double coef_lin; double coef_const; char custom[50]; /* for user supplied kernel */ MATRIX *gram_matrix; /* here one can directly supply the kernel matrix. The matrix is accessed if kernel_type=5 is selected. */ } KERNEL_PARM; typedef struct model { long sv_num; long at_upper_bound; double b; DOC **supvec; double *alpha; long *index; /* index from docnum to position in model */ long totwords; /* number of features */ long totdoc; /* number of training documents */ KERNEL_PARM kernel_parm; /* kernel */ /* the following values are not written to file */ double loo_error,loo_recall,loo_precision; /* leave-one-out estimates */ double xa_error,xa_recall,xa_precision; /* xi/alpha estimates */ double *lin_weights; /* weights for linear case using folding */ double maxdiff; /* precision, up to which this model is accurate */ } MODEL; /* The following specifies a quadratic problem of the following form minimize g0 * x + 1/2 x' * G * x subject to ce*x - ce0 = 0 l <= x <= u */ typedef struct quadratic_program { long opt_n; /* number of variables */ long opt_m; /* number of linear equality constraints */ double *opt_ce,*opt_ce0; /* linear equality constraints opt_ce[i]*x - opt_ceo[i]=0 */ double *opt_g; /* hessian of objective */ double *opt_g0; /* linear part of objective */ double *opt_xinit; /* initial value for variables */ double *opt_low,*opt_up; /* box constraints */ } QP; typedef struct kernel_cache { long *index; /* cache some kernel evalutations */ CFLOAT *buffer; /* to improve speed */ long *invindex; long *active2totdoc; long *totdoc2active; long *lru; long *occu; long elems; long max_elems; long time; long activenum; long buffsize; } KERNEL_CACHE; typedef struct timing_profile { double time_kernel; double time_opti; double time_shrink; double time_update; double time_model; double time_check; double time_select; } TIMING; typedef struct shrink_state { long *active; long *inactive_since; long deactnum; double **a_history; /* for shrinking with non-linear kernel */ long maxhistory; double *last_a; /* for shrinking with linear kernel */ double *last_lin; /* for shrinking with linear kernel */ } SHRINK_STATE; typedef struct randpair { long val,sort; } RANDPAIR; double classify_example(MODEL *, DOC *); double classify_example_linear(MODEL *, DOC *); double kernel(KERNEL_PARM *, DOC *, DOC *); double single_kernel(KERNEL_PARM *, SVECTOR *, SVECTOR *); double custom_kernel(KERNEL_PARM *, SVECTOR *, SVECTOR *); SVECTOR *create_svector(WORD *, MexPhiCustom, double); SVECTOR *create_svector_shallow(WORD *, MexPhiCustom, double); SVECTOR *create_svector_n(double *, long, MexPhiCustom, double); SVECTOR *create_svector_n_r(double *, long, MexPhiCustom, double, double); SVECTOR *copy_svector(SVECTOR *); SVECTOR *copy_svector_shallow(SVECTOR *); void free_svector(SVECTOR *); void free_svector_shallow(SVECTOR *); double sprod_ss(SVECTOR *, SVECTOR *); SVECTOR* sub_ss(SVECTOR *, SVECTOR *); SVECTOR* sub_ss_r(SVECTOR *, SVECTOR *, double min_non_zero); SVECTOR* add_ss(SVECTOR *, SVECTOR *); SVECTOR* add_ss_r(SVECTOR *, SVECTOR *, double min_non_zero); SVECTOR* multadd_ss(SVECTOR *a, SVECTOR *b, double fa, double fb); SVECTOR* multadd_ss_r(SVECTOR *a,SVECTOR *b,double fa, double fb, double min_non_zero); SVECTOR* add_list_ns(SVECTOR *a); SVECTOR* add_dual_list_ns_r(SVECTOR *, SVECTOR *, double min_non_zero); SVECTOR* add_list_ns_r(SVECTOR *a, double min_non_zero); SVECTOR* add_list_ss(SVECTOR *); SVECTOR* add_dual_list_ss_r(SVECTOR *, SVECTOR *, double min_non_zero); SVECTOR* add_list_ss_r(SVECTOR *, double min_non_zero); SVECTOR* add_list_sort_ss(SVECTOR *); SVECTOR* add_dual_list_sort_ss_r(SVECTOR *, SVECTOR *, double min_non_zero); SVECTOR* add_list_sort_ss_r(SVECTOR *, double min_non_zero); void add_list_n_ns(double *vec_n, SVECTOR *vec_s, double faktor); void append_svector_list(SVECTOR *a, SVECTOR *b); void mult_svector_list(SVECTOR *a, double factor); void setfactor_svector_list(SVECTOR *a, double factor); SVECTOR* smult_s(SVECTOR *, double); SVECTOR* shift_s(SVECTOR *a, long shift); int featvec_eq(SVECTOR *, SVECTOR *); double model_length_s(MODEL *); double model_length_n(MODEL *); void mult_vector_ns(double *, SVECTOR *, double); void add_vector_ns(double *, SVECTOR *, double); double sprod_ns(double *, SVECTOR *); void add_weight_vector_to_linear_model(MODEL *); DOC *create_example(long, long, long, double, SVECTOR *); void free_example(DOC *, long); long *random_order(long n); void print_percent_progress(long *progress, long maximum, long percentperdot, char *symbol); MATRIX *create_matrix(int n, int m); MATRIX *realloc_matrix(MATRIX *matrix, int n, int m); double *create_nvector(int n); void clear_nvector(double *vec, long int n); MATRIX *copy_matrix(MATRIX *matrix); void free_matrix(MATRIX *matrix); void free_nvector(double *vector); MATRIX *transpose_matrix(MATRIX *matrix); MATRIX *cholesky_matrix(MATRIX *A); double *find_indep_subset_of_matrix(MATRIX *A, double epsilon); MATRIX *invert_ltriangle_matrix(MATRIX *L); double *prod_nvector_matrix(double *v, MATRIX *A); double *prod_matrix_nvector(MATRIX *A, double *v); double *prod_nvector_ltmatrix(double *v, MATRIX *A); double *prod_ltmatrix_nvector(MATRIX *A, double *v); MATRIX *prod_matrix_matrix(MATRIX *A, MATRIX *B); void print_matrix(MATRIX *matrix); MODEL *read_model(char *); MODEL *copy_model(MODEL *); MODEL *compact_linear_model(MODEL *model); void free_model(MODEL *, int); void read_documents(char *, DOC ***, double **, long *, long *); int parse_document(char *, WORD *, double *, long *, long *, double *, long *, long, char **); int read_word(char *in, char *out); double *read_alphas(char *,long); void set_learning_defaults(LEARN_PARM *, KERNEL_PARM *); int check_learning_parms(LEARN_PARM *, KERNEL_PARM *); void nol_ll(char *, long *, long *, long *); long minl(long, long); long maxl(long, long); double get_runtime(void); int space_or_null(int); void *my_malloc(size_t); void copyright_notice(void); # ifdef _MSC_VER int isnan(double); # endif extern long verbosity; /* verbosity level (0-4) */ extern long kernel_cache_statistic; #endif svm-struct-matlab-1.0/svm_light/svm_hideo.c000066400000000000000000000706471160627511300210740ustar00rootroot00000000000000/***********************************************************************/ /* */ /* svm_hideo.c */ /* */ /* The Hildreth and D'Espo solver specialized for SVMs. */ /* */ /* Author: Thorsten Joachims */ /* Date: 02.07.02 */ /* */ /* Copyright (c) 2002 Thorsten Joachims - All rights reserved */ /* */ /* This software is available for non-commercial use only. It must */ /* not be modified and distributed without prior permission of the */ /* author. The author is not responsible for implications from the */ /* use of this software. */ /* */ /***********************************************************************/ # include # include "svm_common.h" /* solve the quadratic programming problem minimize g0 * x + 1/2 x' * G * x subject to ce*x - ce0 = 0 l <= x <= u The linear constraint vector ce can only have -1/+1 as entries */ /* Common Block Declarations */ /* long verbosity; */ # define PRIMAL_OPTIMAL 1 # define DUAL_OPTIMAL 2 # define MAXITER_EXCEEDED 3 # define NAN_SOLUTION 4 # define ONLY_ONE_VARIABLE 5 # define LARGEROUND 0 # define SMALLROUND 1 /* /////////////////////////////////////////////////////////////// */ # define DEF_PRECISION 1E-5 # define DEF_MAX_ITERATIONS 200 # define DEF_LINDEP_SENSITIVITY 1E-8 # define EPSILON_HIDEO 1E-20 # define EPSILON_EQ 1E-5 double *optimize_qp(QP *, double *, long, double *, LEARN_PARM *); double *primal=0,*dual=0; long precision_violations=0; double opt_precision=DEF_PRECISION; long maxiter=DEF_MAX_ITERATIONS; double lindep_sensitivity=DEF_LINDEP_SENSITIVITY; double *buffer; long *nonoptimal; long smallroundcount=0; long roundnumber=0; /* /////////////////////////////////////////////////////////////// */ void *my_malloc(); int optimize_hildreth_despo(long,long,double,double,double,long,long,long,double,double *, double *,double *,double *,double *,double *, double *,double *,double *,long *,double *,double *); int solve_dual(long,long,double,double,long,double *,double *,double *, double *,double *,double *,double *,double *,double *, double *,double *,double *,double *,long); void linvert_matrix(double *, long, double *, double, long *); void lprint_matrix(double *, long); void ladd_matrix(double *, long, double); void lcopy_matrix(double *, long, double *); void lswitch_rows_matrix(double *, long, long, long); void lswitchrk_matrix(double *, long, long, long); double calculate_qp_objective(long, double *, double *, double *); void init_qp_solver() { primal = 0 ; dual = 0 ; precision_violations = 0 ; opt_precision = DEF_PRECISION ; maxiter = DEF_MAX_ITERATIONS ; lindep_sensitivity = DEF_LINDEP_SENSITIVITY ; buffer = 0 ; nonoptimal = 0 ; smallroundcount = 0 ; roundnumber = 0 ; } void free_qp_solver() { if(buffer) {free(buffer) ; buffer = 0 ;} if(nonoptimal) {free(nonoptimal) ; nonoptimal = 0; } if(dual) {free(dual) ; dual = 0 ;} if(primal) {free(primal) ; primal = 0;} } double *optimize_qp(qp,epsilon_crit,nx,threshold,learn_parm) QP *qp; double *epsilon_crit; long nx; /* Maximum number of variables in QP */ double *threshold; LEARN_PARM *learn_parm; /* start the optimizer and return the optimal values */ /* The HIDEO optimizer does not necessarily fully solve the problem. */ /* Since it requires a strictly positive definite hessian, the solution */ /* is restricted to a linear independent subset in case the matrix is */ /* only semi-definite. */ { long i,j; int result; double eq,progress; roundnumber++; if(!primal) { /* allocate memory at first call */ primal=(double *)my_malloc(sizeof(double)*nx); dual=(double *)my_malloc(sizeof(double)*((nx+1)*2)); nonoptimal=(long *)my_malloc(sizeof(long)*(nx)); buffer=(double *)my_malloc(sizeof(double)*((nx+1)*2*(nx+1)*2+ nx*nx+2*(nx+1)*2+2*nx+1+2*nx+ nx+nx+nx*nx)); (*threshold)=0; for(i=0;i=4) { /* really verbose */ printf("\n\n"); eq=qp->opt_ce0[0]; for(i=0;iopt_n;i++) { eq+=qp->opt_xinit[i]*qp->opt_ce[i]; printf("%f: ",qp->opt_g0[i]); for(j=0;jopt_n;j++) { printf("%f ",qp->opt_g[i*qp->opt_n+j]); } printf(": a=%.10f < %f",qp->opt_xinit[i],qp->opt_up[i]); printf(": y=%f\n",qp->opt_ce[i]); } if(qp->opt_m) { printf("EQ: %f*x0",qp->opt_ce[0]); for(i=1;iopt_n;i++) { printf(" + %f*x%ld",qp->opt_ce[i],i); } printf(" = %f\n\n",-qp->opt_ce0[0]); } } result=optimize_hildreth_despo(qp->opt_n,qp->opt_m, opt_precision,(*epsilon_crit), learn_parm->epsilon_a,maxiter, /* (long)PRIMAL_OPTIMAL, */ (long)0, (long)0, lindep_sensitivity, qp->opt_g,qp->opt_g0,qp->opt_ce,qp->opt_ce0, qp->opt_low,qp->opt_up,primal,qp->opt_xinit, dual,nonoptimal,buffer,&progress); if(verbosity>=3) { printf("return(%d)...",result); } if(learn_parm->totwords < learn_parm->svm_maxqpsize) { /* larger working sets will be linear dependent anyway */ learn_parm->svm_maxqpsize=maxl(learn_parm->totwords,(long)2); } if(result == NAN_SOLUTION) { lindep_sensitivity*=2; /* throw out linear dependent examples more */ /* generously */ if(learn_parm->svm_maxqpsize>2) { learn_parm->svm_maxqpsize--; /* decrease size of qp-subproblems */ } precision_violations++; } /* take one round of only two variable to get unstuck */ if((result != PRIMAL_OPTIMAL) || (!(roundnumber % 31)) || (progress <= 0)) { smallroundcount++; result=optimize_hildreth_despo(qp->opt_n,qp->opt_m, opt_precision,(*epsilon_crit), learn_parm->epsilon_a,(long)maxiter, (long)PRIMAL_OPTIMAL,(long)SMALLROUND, lindep_sensitivity, qp->opt_g,qp->opt_g0,qp->opt_ce,qp->opt_ce0, qp->opt_low,qp->opt_up,primal,qp->opt_xinit, dual,nonoptimal,buffer,&progress); if(verbosity>=3) { printf("return_srd(%d)...",result); } if(result != PRIMAL_OPTIMAL) { if(result != ONLY_ONE_VARIABLE) precision_violations++; if(result == MAXITER_EXCEEDED) maxiter+=100; if(result == NAN_SOLUTION) { lindep_sensitivity*=2; /* throw out linear dependent examples more */ /* generously */ /* results not valid, so return inital values */ for(i=0;iopt_n;i++) { primal[i]=qp->opt_xinit[i]; } } } } if(precision_violations > 50) { precision_violations=0; (*epsilon_crit)*=10.0; if(verbosity>=1) { printf("\nWARNING: Relaxing epsilon on KT-Conditions (%f).\n", (*epsilon_crit)); } } if((qp->opt_m>0) && (result != NAN_SOLUTION) && (!isnan(dual[1]-dual[0]))) (*threshold)=dual[1]-dual[0]; else (*threshold)=0; if(verbosity>=4) { /* really verbose */ printf("\n\n"); eq=qp->opt_ce0[0]; for(i=0;iopt_n;i++) { eq+=primal[i]*qp->opt_ce[i]; printf("%f: ",qp->opt_g0[i]); for(j=0;jopt_n;j++) { printf("%f ",qp->opt_g[i*qp->opt_n+j]); } printf(": a=%.30f",primal[i]); printf(": nonopti=%ld",nonoptimal[i]); printf(": y=%f\n",qp->opt_ce[i]); } printf("eq-constraint=%.30f\n",eq); printf("b=%f\n",(*threshold)); printf(" smallroundcount=%ld ",smallroundcount); } return(primal); } int optimize_hildreth_despo(n,m,precision,epsilon_crit,epsilon_a,maxiter,goal, smallround,lindep_sensitivity,g,g0,ce,ce0,low,up, primal,init,dual,lin_dependent,buffer,progress) long n; /* number of variables */ long m; /* number of linear equality constraints [0,1] */ double precision; /* solve at least to this dual precision */ double epsilon_crit; /* stop, if KT-Conditions approx fulfilled */ double epsilon_a; /* precision of alphas at bounds */ long maxiter; /* stop after this many iterations */ long goal; /* keep going until goal fulfilled */ long smallround; /* use only two variables of steepest descent */ double lindep_sensitivity; /* epsilon for detecting linear dependent ex */ double *g; /* hessian of objective */ double *g0; /* linear part of objective */ double *ce,*ce0; /* linear equality constraints */ double *low,*up; /* box constraints */ double *primal; /* primal variables */ double *init; /* initial values of primal */ double *dual; /* dual variables */ long *lin_dependent; double *buffer; double *progress; /* delta in the objective function between before and after */ { long i,j,k,from,to,n_indep,changed; double sum,bmin=0,bmax=0; double *d,*d0,*ig,*dual_old,*temp,*start; double *g0_new,*g_new,*ce_new,*ce0_new,*low_new,*up_new; double add,t; int result; double obj_before,obj_after; long b1,b2; double g0_b1,g0_b2,ce0_b; g0_new=&(buffer[0]); /* claim regions of buffer */ d=&(buffer[n]); d0=&(buffer[n+(n+m)*2*(n+m)*2]); ce_new=&(buffer[n+(n+m)*2*(n+m)*2+(n+m)*2]); ce0_new=&(buffer[n+(n+m)*2*(n+m)*2+(n+m)*2+n]); ig=&(buffer[n+(n+m)*2*(n+m)*2+(n+m)*2+n+m]); dual_old=&(buffer[n+(n+m)*2*(n+m)*2+(n+m)*2+n+m+n*n]); low_new=&(buffer[n+(n+m)*2*(n+m)*2+(n+m)*2+n+m+n*n+(n+m)*2]); up_new=&(buffer[n+(n+m)*2*(n+m)*2+(n+m)*2+n+m+n*n+(n+m)*2+n]); start=&(buffer[n+(n+m)*2*(n+m)*2+(n+m)*2+n+m+n*n+(n+m)*2+n+n]); g_new=&(buffer[n+(n+m)*2*(n+m)*2+(n+m)*2+n+m+n*n+(n+m)*2+n+n+n]); temp=&(buffer[n+(n+m)*2*(n+m)*2+(n+m)*2+n+m+n*n+(n+m)*2+n+n+n+n*n]); b1=-1; b2=-1; for(i=0;i=( up[i]-epsilon_a)) && (ce[i]>0.0))) ) { bmin=sum; b1=i; } if(((b2==-1) || (sum>=bmax)) && (!((init[i]<=(low[i]+epsilon_a)) && (ce[i]>0.0))) && (!((init[i]>=( up[i]-epsilon_a)) && (ce[i]<0.0))) ) { bmax=sum; b2=i; } } /* in case of unbiased hyperplane, the previous projection on */ /* equality constraint can lead to b1 or b2 being -1. */ if((b1 == -1) || (b2 == -1)) { b1=maxl(b1,b2); b2=maxl(b1,b2); } for(i=0;i g0_b2) { /* set b2 to upper bound */ /* printf("case +=>\n"); */ changed=1; t=up[b2]-init[b2]; if((init[b1]-low[b1]) < t) { t=init[b1]-low[b1]; } start[b1]=init[b1]-t; start[b2]=init[b2]+t; } } else if(((g[b1*n+b1]>0) || (g[b2*n+b2]>0))) { /* (ce[b1] != ce[b2]) */ /* printf("case +!\n"); */ t=((ce[b2]/ce[b1])*g0[b1]-g0[b2]+ce0[0]*(g[b1*n+b1]*ce[b2]/ce[b1]-g[b1*n+b2]/ce[b1]))/((ce[b2]*ce[b2]/(ce[b1]*ce[b1]))*g[b1*n+b1]+g[b2*n+b2]-2*(g[b1*n+b2]*ce[b2]/ce[b1]))-init[b2]; changed=1; if((up[b2]-init[b2]) < t) { t=up[b2]-init[b2]; } if((init[b2]-low[b2]) < -t) { t=-(init[b2]-low[b2]); } if((up[b1]-init[b1]) < t) { t=(up[b1]-init[b1]); } if((init[b1]-low[b1]) < -t) { t=-(init[b1]-low[b1]); } start[b1]=init[b1]+t; start[b2]=init[b2]+t; } } if((-g[b1*n+b2] == g[b1*n+b1]) && (-g[b1*n+b2] == g[b2*n+b2])) { /* printf("diffeuqal\n"); */ if(ce[b1] != ce[b2]) { if((g0_b1+g0_b2) < 0) { /* set b1 and b2 to upper bound */ /* printf("case -!<\n"); */ changed=1; t=up[b1]-init[b1]; if((up[b2]-init[b2]) < t) { t=up[b2]-init[b2]; } start[b1]=init[b1]+t; start[b2]=init[b2]+t; } else if((g0_b1+g0_b2) >= 0) { /* set b1 and b2 to lower bound */ /* printf("case -!>\n"); */ changed=1; t=init[b1]-low[b1]; if((init[b2]-low[b2]) < t) { t=init[b2]-low[b2]; } start[b1]=init[b1]-t; start[b2]=init[b2]-t; } } else if(((g[b1*n+b1]>0) || (g[b2*n+b2]>0))) { /* (ce[b1]==ce[b2]) */ /* printf("case -=\n"); */ t=((ce[b2]/ce[b1])*g0[b1]-g0[b2]+ce0[0]*(g[b1*n+b1]*ce[b2]/ce[b1]-g[b1*n+b2]/ce[b1]))/((ce[b2]*ce[b2]/(ce[b1]*ce[b1]))*g[b1*n+b1]+g[b2*n+b2]-2*(g[b1*n+b2]*ce[b2]/ce[b1]))-init[b2]; changed=1; if((up[b2]-init[b2]) < t) { t=up[b2]-init[b2]; } if((init[b2]-low[b2]) < -t) { t=-(init[b2]-low[b2]); } if((up[b1]-init[b1]) < -t) { t=-(up[b1]-init[b1]); } if((init[b1]-low[b1]) < t) { t=init[b1]-low[b1]; } start[b1]=init[b1]-t; start[b2]=init[b2]+t; } } } /* if we have a biased hyperplane, then adding a constant to the */ /* hessian does not change the solution. So that is done for examples */ /* with zero diagonal entry, since HIDEO cannot handle them. */ if((m>0) && ((fabs(g[b1*n+b1]) < lindep_sensitivity) || (fabs(g[b2*n+b2]) < lindep_sensitivity))) { /* printf("Case 0\n"); */ add+=0.093274; } /* in case both examples are linear dependent */ else if((m>0) && (g[b1*n+b2] != 0 && g[b2*n+b2] != 0) && (fabs(g[b1*n+b1]/g[b1*n+b2] - g[b1*n+b2]/g[b2*n+b2]) < lindep_sensitivity)) { /* printf("Case lindep\n"); */ add+=0.078274; } /* special case for zero diagonal entry on unbiased hyperplane */ if((m==0) && (b1>=0)) { if(fabs(g[b1*n+b1]) < lindep_sensitivity) { /* printf("Case 0b1\n"); */ for(i=0;i=0) start[b1]=low[b1]; } } if((m==0) && (b2>=0)) { if(fabs(g[b2*n+b2]) < lindep_sensitivity) { /* printf("Case 0b2\n"); */ for(i=0;i=0) start[b2]=low[b2]; } } /* printf("b1=%ld,b2=%ld\n",b1,b2); */ lcopy_matrix(g,n,d); if((m==1) && (add>0.0)) { for(j=0;j2) { /* switch, so that variables are better mixed */ lswitchrk_matrix(d,n,b1,(long)0); if(b2 == 0) lswitchrk_matrix(d,n,b1,(long)1); else lswitchrk_matrix(d,n,b2,(long)1); } if(smallround == SMALLROUND) { for(i=2;i0) { /* for biased hyperplane, pick two variables */ lin_dependent[0]=0; lin_dependent[1]=0; } else { /* for unbiased hyperplane, pick only one variable */ lin_dependent[0]=smallroundcount % 2; lin_dependent[1]=(smallroundcount+1) % 2; } } else { for(i=0;i2) { /* now switch back */ if(b2 == 0) { lswitchrk_matrix(ig,n,b1,(long)1); i=lin_dependent[1]; lin_dependent[1]=lin_dependent[b1]; lin_dependent[b1]=i; } else { lswitchrk_matrix(ig,n,b2,(long)1); i=lin_dependent[1]; lin_dependent[1]=lin_dependent[b2]; lin_dependent[b2]=i; } lswitchrk_matrix(ig,n,b1,(long)0); i=lin_dependent[0]; lin_dependent[0]=lin_dependent[b1]; lin_dependent[b1]=i; } /* lprint_matrix(d,n); */ /* lprint_matrix(ig,n); */ lcopy_matrix(g,n,g_new); /* restore g_new matrix */ if(add>0) for(j=0;j0) ce0_new[0]=-ce0[0]; for(i=0;i0) ce0_new[0]-=(start[i]*ce[i]); } } from=0; /* remove linear dependent vectors */ to=0; n_indep=0; for(i=0;i=3) { printf("real_qp_size(%ld)...",n_indep); } /* cannot optimize with only one variable */ if((n_indep<=1) && (m>0) && (!changed)) { for(i=n-1;i>=0;i--) { primal[i]=init[i]; } return((int)ONLY_ONE_VARIABLE); } if((!changed) || (n_indep>1)) { result=solve_dual(n_indep,m,precision,epsilon_crit,maxiter,g_new,g0_new, ce_new,ce0_new,low_new,up_new,primal,d,d0,ig, dual,dual_old,temp,goal); } else { result=PRIMAL_OPTIMAL; } j=n_indep; for(i=n-1;i>=0;i--) { if(!lin_dependent[i]) { j--; primal[i]=primal[j]; } else { primal[i]=start[i]; /* leave as is */ } temp[i]=primal[i]; } obj_before=calculate_qp_objective(n,g,g0,init); obj_after=calculate_qp_objective(n,g,g0,primal); (*progress)=obj_before-obj_after; if(verbosity>=3) { printf("before(%.30f)...after(%.30f)...result_sd(%d)...", obj_before,obj_after,result); } return((int)result); } int solve_dual(n,m,precision,epsilon_crit,maxiter,g,g0,ce,ce0,low,up,primal, d,d0,ig,dual,dual_old,temp,goal) /* Solves the dual using the method of Hildreth and D'Espo. */ /* Can only handle problems with zero or exactly one */ /* equality constraints. */ long n; /* number of variables */ long m; /* number of linear equality constraints */ double precision; /* solve at least to this dual precision */ double epsilon_crit; /* stop, if KT-Conditions approx fulfilled */ long maxiter; /* stop after that many iterations */ double *g; double *g0; /* linear part of objective */ double *ce,*ce0; /* linear equality constraints */ double *low,*up; /* box constraints */ double *primal; /* variables (with initial values) */ double *d,*d0,*ig,*dual,*dual_old,*temp; /* buffer */ long goal; { long i,j,k,iter; double sum,w,maxviol,viol,temp1,temp2,isnantest; double model_b,dist; long retrain,maxfaktor,primal_optimal=0,at_bound,scalemaxiter; double epsilon_a=1E-15,epsilon_hideo; double eq; if((m<0) || (m>1)) perror("SOLVE DUAL: inappropriate number of eq-constrains!"); /* printf("\n"); for(i=0;i0) { sum=0; /* dual hessian for eq constraints */ for(j=0;j0) { sum=0; /* dual linear component for eq constraints */ for(j=0;j 0) && (iter < (scalemaxiter*maxfaktor))) { iter++; while((maxviol > precision) && (iter < (scalemaxiter*maxfaktor))) { iter++; maxviol=0; for(i=0;i<2*(n+m);i++) { sum=d0[i]; for(j=0;j<2*(n+m);j++) { sum+=d[i*2*(n+m)+j]*dual_old[j]; } sum-=d[i*2*(n+m)+i]*dual_old[i]; dual[i]=-sum/d[i*2*(n+m)+i]; if(dual[i]<0) dual[i]=0; viol=fabs(dual[i]-dual_old[i]); if(viol>maxviol) maxviol=viol; dual_old[i]=dual[i]; } /* printf("%d) maxviol=%20f precision=%f\n",iter,maxviol,precision); */ } if(m>0) { for(i=0;i=(up[i])) { primal[i]=up[i]; } } if(m>0) model_b=dual[n+n+1]-dual[n+n]; else model_b=0; epsilon_hideo=EPSILON_HIDEO; for(i=0;i(low[i]+epsilon_hideo)) &&(dist>(1.0+epsilon_crit))) { epsilon_hideo=(primal[i]-low[i])*2.0; } } /* printf("\nEPSILON_HIDEO=%.30f\n",epsilon_hideo); */ for(i=0;i=(up[i]-epsilon_hideo)) { primal[i]=up[i]; } } retrain=0; primal_optimal=1; at_bound=0; for(i=0;(i(low[i]+epsilon_a)) && (dist > (1.0+epsilon_crit))) { retrain=1; primal_optimal=0; } if((primal[i]<=(low[i]+epsilon_a)) || (primal[i]>=(up[i]-epsilon_a))) { at_bound++; } /* printf("HIDEOtemp: a[%ld]=%.30f, dist=%.6f, b=%f, at_bound=%ld\n",i,primal[i],dist,model_b,at_bound); */ } if(m>0) { eq=-ce0[0]; /* check precision of eq-constraint */ for(i=0;i=(up[i]-epsilon_a)) { primal[i]=up[i]; } } } isnantest=0; for(i=0;i0) { temp1=dual[n+n+1]; /* copy the dual variables for the eq */ temp2=dual[n+n]; /* constraints to a handier location */ for(i=n+n+1;i>=2;i--) { dual[i]=dual[i-2]; } dual[0]=temp2; dual[1]=temp1; isnantest+=temp1+temp2; } if(isnan(isnantest)) { return((int)NAN_SOLUTION); } else if(primal_optimal) { return((int)PRIMAL_OPTIMAL); } else if(maxviol == 0.0) { return((int)DUAL_OPTIMAL); } else { return((int)MAXITER_EXCEEDED); } } void linvert_matrix(matrix,depth,inverse,lindep_sensitivity,lin_dependent) double *matrix; long depth; double *inverse,lindep_sensitivity; long *lin_dependent; /* indicates the active parts of matrix on input and output*/ { long i,j,k; double factor; for(i=0;i=0;i--) { if(!lin_dependent[i]) { factor=1/matrix[i*depth+i]; for(k=0;k=0;j--) { factor=matrix[j*depth+i]; matrix[j*depth+i]=0; for(k=0;k (y) ? (y) : (x)) #define SIGN(x) ((x) > (0) ? (1) : (((x) < (0) ? (-1) : (0)))) /* interface to QP-solver */ double *optimize_qp(QP *, double *, long, double *, LEARN_PARM *); /*---------------------------------------------------------------------------*/ /* Learns an SVM classification model based on the training data in docs/label. The resulting model is returned in the structure model. */ void svm_learn_classification(DOC **docs, double *class, long int totdoc, long int totwords, LEARN_PARM *learn_parm, KERNEL_PARM *kernel_parm, KERNEL_CACHE *kernel_cache, MODEL *model, double *alpha) /* docs: Training vectors (x-part) */ /* class: Training labels (y-part, zero if test example for transduction) */ /* totdoc: Number of examples in docs/label */ /* totwords: Number of features (i.e. highest feature index) */ /* learn_parm: Learning paramenters */ /* kernel_parm: Kernel paramenters */ /* kernel_cache:Initialized Cache of size totdoc, if using a kernel. NULL if linear.*/ /* model: Returns learning result (assumed empty before called) */ /* alpha: Start values for the alpha variables or NULL pointer. The new alpha values are returned after optimization if not NULL. Array must be of size totdoc. */ { long *inconsistent,i,*label; long inconsistentnum; long misclassified,upsupvecnum; double loss,model_length,example_length; double dualitygap,xisum,alphasum,xi; double maxdiff,*lin,*a,*c; double runtime_start,runtime_end; long iterations; long *unlabeled,transduction; long heldout; long loo_count=0,loo_count_pos=0,loo_count_neg=0,trainpos=0,trainneg=0; long loocomputed=0; double runtime_start_loo=0,runtime_start_xa=0; double heldout_c=0,r_delta_sq=0,r_delta,r_delta_avg; long *index,*index2dnum; double *weights; CFLOAT *aicache; /* buffer to keep one row of hessian */ double *xi_fullset; /* buffer for storing xi on full sample in loo */ double *a_fullset; /* buffer for storing alpha on full sample in loo */ TIMING timing_profile; SHRINK_STATE shrink_state; runtime_start=get_runtime(); timing_profile.time_kernel=0; timing_profile.time_opti=0; timing_profile.time_shrink=0; timing_profile.time_update=0; timing_profile.time_model=0; timing_profile.time_check=0; timing_profile.time_select=0; kernel_cache_statistic=0; learn_parm->totwords=totwords; /* make sure -n value is reasonable */ if((learn_parm->svm_newvarsinqp < 2) || (learn_parm->svm_newvarsinqp > learn_parm->svm_maxqpsize)) { learn_parm->svm_newvarsinqp=learn_parm->svm_maxqpsize; } init_shrink_state(&shrink_state,totdoc,(long)MAXSHRINK); label = (long *)my_malloc(sizeof(long)*totdoc); inconsistent = (long *)my_malloc(sizeof(long)*totdoc); unlabeled = (long *)my_malloc(sizeof(long)*totdoc); c = (double *)my_malloc(sizeof(double)*totdoc); a = (double *)my_malloc(sizeof(double)*totdoc); a_fullset = (double *)my_malloc(sizeof(double)*totdoc); xi_fullset = (double *)my_malloc(sizeof(double)*totdoc); lin = (double *)my_malloc(sizeof(double)*totdoc); learn_parm->svm_cost = (double *)my_malloc(sizeof(double)*totdoc); model->supvec = (DOC **)my_malloc(sizeof(DOC *)*(totdoc+2)); model->alpha = (double *)my_malloc(sizeof(double)*(totdoc+2)); model->index = (long *)my_malloc(sizeof(long)*(totdoc+2)); model->at_upper_bound=0; model->b=0; model->supvec[0]=0; /* element 0 reserved and empty for now */ model->alpha[0]=0; model->lin_weights=NULL; model->totwords=totwords; model->totdoc=totdoc; model->kernel_parm=(*kernel_parm); model->sv_num=1; model->loo_error=-1; model->loo_recall=-1; model->loo_precision=-1; model->xa_error=-1; model->xa_recall=-1; model->xa_precision=-1; inconsistentnum=0; transduction=0; r_delta=estimate_r_delta(docs,totdoc,kernel_parm); r_delta_sq=r_delta*r_delta; r_delta_avg=estimate_r_delta_average(docs,totdoc,kernel_parm); if(learn_parm->svm_c == 0.0) { /* default value for C */ learn_parm->svm_c=1.0/(r_delta_avg*r_delta_avg); if(verbosity>=1) printf("Setting default regularization parameter C=%.4f\n", learn_parm->svm_c); } learn_parm->eps=-1.0; /* equivalent regression epsilon for classification */ for(i=0;idocnum=i; inconsistent[i]=0; a[i]=0; lin[i]=0; c[i]=0.0; unlabeled[i]=0; if(class[i] == 0) { unlabeled[i]=1; label[i]=0; transduction=1; } if(class[i] > 0) { learn_parm->svm_cost[i]=learn_parm->svm_c*learn_parm->svm_costratio* docs[i]->costfactor; label[i]=1; trainpos++; } else if(class[i] < 0) { learn_parm->svm_cost[i]=learn_parm->svm_c*docs[i]->costfactor; label[i]=-1; trainneg++; } else { learn_parm->svm_cost[i]=0; } } if(verbosity>=2) { printf("%ld positive, %ld negative, and %ld unlabeled examples.\n",trainpos,trainneg,totdoc-trainpos-trainneg); fflush(stdout); } /* caching makes no sense for linear kernel */ if(kernel_parm->kernel_type == LINEAR) { /* kernel_cache = NULL; */ } /* compute starting state for initial alpha values */ if(alpha) { if(verbosity>=1) { printf("Computing starting state..."); fflush(stdout); } index = (long *)my_malloc(sizeof(long)*totdoc); index2dnum = (long *)my_malloc(sizeof(long)*(totdoc+11)); weights=(double *)my_malloc(sizeof(double)*(totwords+1)); aicache = (CFLOAT *)my_malloc(sizeof(CFLOAT)*totdoc); for(i=0;ilearn_parm->svm_cost[i]) alpha[i]=learn_parm->svm_cost[i]; } if(kernel_cache && (kernel_parm->kernel_type != LINEAR)) { for(i=0;i0) && (alpha[i]svm_cost[i]) && (kernel_cache_space_available(kernel_cache))) cache_kernel_row(kernel_cache,docs,i,kernel_parm); for(i=0;isvm_cost[i]) && (kernel_cache_space_available(kernel_cache))) cache_kernel_row(kernel_cache,docs,i,kernel_parm); } clear_nvector(weights,totwords); /* set weights to zero */ (void)compute_index(index,totdoc,index2dnum); update_linear_component(docs,label,index2dnum,alpha,a,index2dnum,totdoc, totwords,kernel_parm,kernel_cache,lin,aicache, weights); (void)calculate_svm_model(docs,label,unlabeled,lin,alpha,a,c, learn_parm,index2dnum,index2dnum,model); for(i=0;i=1) { printf("done.\n"); fflush(stdout); } } if(transduction) { learn_parm->svm_iter_to_shrink=99999999; if(verbosity >= 1) printf("\nDeactivating Shrinking due to an incompatibility with the transductive \nlearner in the current version.\n\n"); } if(transduction && learn_parm->compute_loo) { learn_parm->compute_loo=0; if(verbosity >= 1) printf("\nCannot compute leave-one-out estimates for transductive learner.\n\n"); } if(learn_parm->remove_inconsistent && learn_parm->compute_loo) { learn_parm->compute_loo=0; printf("\nCannot compute leave-one-out estimates when removing inconsistent examples.\n\n"); } if(learn_parm->compute_loo && ((trainpos == 1) || (trainneg == 1))) { learn_parm->compute_loo=0; printf("\nCannot compute leave-one-out with only one example in one class.\n\n"); } if(verbosity==1) { printf("Optimizing"); fflush(stdout); } /* train the svm */ iterations=optimize_to_convergence(docs,label,totdoc,totwords,learn_parm, kernel_parm,kernel_cache,&shrink_state,model, inconsistent,unlabeled,a,lin, c,&timing_profile, &maxdiff,(long)-1, (long)1); if(verbosity>=1) { if(verbosity==1) printf("done. (%ld iterations)\n",iterations); misclassified=0; for(i=0;(ib)*(double)label[i] <= 0.0) misclassified++; } printf("Optimization finished (%ld misclassified, maxdiff=%.5f).\n", misclassified,maxdiff); runtime_end=get_runtime(); if(verbosity>=2) { printf("Runtime in cpu-seconds: %.2f (%.2f%% for kernel/%.2f%% for optimizer/%.2f%% for final/%.2f%% for update/%.2f%% for model/%.2f%% for check/%.2f%% for select)\n", (runtime_end-runtime_start)/100.0, (100.0*timing_profile.time_kernel)/(runtime_end-runtime_start), (100.0*timing_profile.time_opti)/(runtime_end-runtime_start), (100.0*timing_profile.time_shrink)/(runtime_end-runtime_start), (100.0*timing_profile.time_update)/(runtime_end-runtime_start), (100.0*timing_profile.time_model)/(runtime_end-runtime_start), (100.0*timing_profile.time_check)/(runtime_end-runtime_start), (100.0*timing_profile.time_select)/(runtime_end-runtime_start)); } else { printf("Runtime in cpu-seconds: %.2f\n", (runtime_end-runtime_start)/100.0); } if(learn_parm->remove_inconsistent) { inconsistentnum=0; for(i=0;isv_num-1,inconsistentnum); } else { upsupvecnum=0; for(i=1;isv_num;i++) { if(fabs(model->alpha[i]) >= (learn_parm->svm_cost[(model->supvec[i])->docnum]- learn_parm->epsilon_a)) upsupvecnum++; } printf("Number of SV: %ld (including %ld at upper bound)\n", model->sv_num-1,upsupvecnum); } if((verbosity>=1) && (!learn_parm->skip_final_opt_check)) { loss=0; xisum=0; alphasum=0; model_length=0; for(i=0;ib)*(double)label[i]); if(xi > learn_parm->epsilon_crit) loss+=xi; xisum+=xi*learn_parm->svm_cost[i]; alphasum+=a[i]; model_length+=a[i]*label[i]*lin[i]; } model_length=sqrt(model_length); dualitygap=(0.5*model_length*model_length+xisum) -(alphasum-0.5*model_length*model_length); fprintf(stdout,"Upper bound on duality gap: gap=%.5f\n",dualitygap); fprintf(stdout,"Dual objective value: dval=%.5f\n", alphasum-0.5*model_length*model_length); fprintf(stdout,"L1 loss: loss=%.5f\n",loss); fprintf(stdout,"Norm of weight vector: |w|=%.5f\n",model_length); example_length=estimate_sphere(model); fprintf(stdout,"Norm of longest example vector: |x|=%.5f\n", length_of_longest_document_vector(docs,totdoc,kernel_parm)); fprintf(stdout,"Estimated VCdim of classifier: VCdim<=%.5f\n", estimate_margin_vcdim(model,model_length,example_length)); if((!learn_parm->remove_inconsistent) && (!transduction)) { runtime_start_xa=get_runtime(); if(verbosity>=1) { printf("Computing XiAlpha-estimates..."); fflush(stdout); } compute_xa_estimates(model,label,unlabeled,totdoc,docs,lin,a, kernel_parm,learn_parm,&(model->xa_error), &(model->xa_recall),&(model->xa_precision)); if(verbosity>=1) { printf("done\n"); } printf("Runtime for XiAlpha-estimates in cpu-seconds: %.2f\n", (get_runtime()-runtime_start_xa)/100.0); fprintf(stdout,"XiAlpha-estimate of the error: error<=%.2f%% (rho=%.2f,depth=%ld)\n", model->xa_error,learn_parm->rho,learn_parm->xa_depth); fprintf(stdout,"XiAlpha-estimate of the recall: recall=>%.2f%% (rho=%.2f,depth=%ld)\n", model->xa_recall,learn_parm->rho,learn_parm->xa_depth); fprintf(stdout,"XiAlpha-estimate of the precision: precision=>%.2f%% (rho=%.2f,depth=%ld)\n", model->xa_precision,learn_parm->rho,learn_parm->xa_depth); } else if(!learn_parm->remove_inconsistent) { estimate_transduction_quality(model,label,unlabeled,totdoc,docs,lin); } } if(verbosity>=1) { printf("Number of kernel evaluations: %ld\n",kernel_cache_statistic); } } /* leave-one-out testing starts now */ if(learn_parm->compute_loo) { /* save results of training on full dataset for leave-one-out */ runtime_start_loo=get_runtime(); for(i=0;ib)*(double)label[i]); if(xi_fullset[i]<0) xi_fullset[i]=0; a_fullset[i]=a[i]; } if(verbosity>=1) { printf("Computing leave-one-out"); } /* repeat this loop for every held-out example */ for(heldout=0;(heldoutrho*a_fullset[heldout]*r_delta_sq+xi_fullset[heldout] < 1.0) { /* guaranteed to not produce a leave-one-out error */ if(verbosity==1) { printf("+"); fflush(stdout); } } else if(xi_fullset[heldout] > 1.0) { /* guaranteed to produce a leave-one-out error */ loo_count++; if(label[heldout] > 0) loo_count_pos++; else loo_count_neg++; if(verbosity==1) { printf("-"); fflush(stdout); } } else { loocomputed++; heldout_c=learn_parm->svm_cost[heldout]; /* set upper bound to zero */ learn_parm->svm_cost[heldout]=0; /* make sure heldout example is not currently */ /* shrunk away. Assumes that lin is up to date! */ shrink_state.active[heldout]=1; if(verbosity>=2) printf("\nLeave-One-Out test on example %ld\n",heldout); if(verbosity>=1) { printf("(?[%ld]",heldout); fflush(stdout); } optimize_to_convergence(docs,label,totdoc,totwords,learn_parm, kernel_parm, kernel_cache,&shrink_state,model,inconsistent,unlabeled, a,lin,c,&timing_profile, &maxdiff,heldout,(long)2); /* printf("%.20f\n",(lin[heldout]-model->b)*(double)label[heldout]); */ if(((lin[heldout]-model->b)*(double)label[heldout]) <= 0.0) { loo_count++; /* there was a loo-error */ if(label[heldout] > 0) loo_count_pos++; else loo_count_neg++; if(verbosity>=1) { printf("-)"); fflush(stdout); } } else { if(verbosity>=1) { printf("+)"); fflush(stdout); } } /* now we need to restore the original data set*/ learn_parm->svm_cost[heldout]=heldout_c; /* restore upper bound */ } } /* end of leave-one-out loop */ if(verbosity>=1) { printf("\nRetrain on full problem"); fflush(stdout); } optimize_to_convergence(docs,label,totdoc,totwords,learn_parm, kernel_parm, kernel_cache,&shrink_state,model,inconsistent,unlabeled, a,lin,c,&timing_profile, &maxdiff,(long)-1,(long)1); if(verbosity >= 1) printf("done.\n"); /* after all leave-one-out computed */ model->loo_error=100.0*loo_count/(double)totdoc; model->loo_recall=(1.0-(double)loo_count_pos/(double)trainpos)*100.0; model->loo_precision=(trainpos-loo_count_pos)/ (double)(trainpos-loo_count_pos+loo_count_neg)*100.0; if(verbosity >= 1) { fprintf(stdout,"Leave-one-out estimate of the error: error=%.2f%%\n", model->loo_error); fprintf(stdout,"Leave-one-out estimate of the recall: recall=%.2f%%\n", model->loo_recall); fprintf(stdout,"Leave-one-out estimate of the precision: precision=%.2f%%\n", model->loo_precision); fprintf(stdout,"Actual leave-one-outs computed: %ld (rho=%.2f)\n", loocomputed,learn_parm->rho); printf("Runtime for leave-one-out in cpu-seconds: %.2f\n", (get_runtime()-runtime_start_loo)/100.0); } } if(learn_parm->alphafile[0]) write_alphas(learn_parm->alphafile,a,label,totdoc); shrink_state_cleanup(&shrink_state); free(label); free(inconsistent); free(unlabeled); free(c); free(a); free(a_fullset); free(xi_fullset); free(lin); free(learn_parm->svm_cost); } /* Learns an SVM regression model based on the training data in docs/label. The resulting model is returned in the structure model. */ void svm_learn_regression(DOC **docs, double *value, long int totdoc, long int totwords, LEARN_PARM *learn_parm, KERNEL_PARM *kernel_parm, KERNEL_CACHE **kernel_cache, MODEL *model) /* docs: Training vectors (x-part) */ /* class: Training value (y-part) */ /* totdoc: Number of examples in docs/label */ /* totwords: Number of features (i.e. highest feature index) */ /* learn_parm: Learning paramenters */ /* kernel_parm: Kernel paramenters */ /* kernel_cache:Initialized Cache, if using a kernel. NULL if linear. Note that it will be free'd and reassigned */ /* model: Returns learning result (assumed empty before called) */ { long *inconsistent,i,j; long inconsistentnum; long upsupvecnum; double loss,model_length,example_length; double maxdiff,*lin,*a,*c; double runtime_start,runtime_end; long iterations,kernel_cache_size; long *unlabeled; double r_delta_sq=0,r_delta,r_delta_avg; double *xi_fullset; /* buffer for storing xi on full sample in loo */ double *a_fullset; /* buffer for storing alpha on full sample in loo */ TIMING timing_profile; SHRINK_STATE shrink_state; DOC **docs_org; long *label; /* set up regression problem in standard form */ docs_org=docs; docs = (DOC **)my_malloc(sizeof(DOC)*2*totdoc); label = (long *)my_malloc(sizeof(long)*2*totdoc); c = (double *)my_malloc(sizeof(double)*2*totdoc); for(i=0;icostfactor,docs_org[i]->fvec); label[i]=+1; c[i]=value[i]; docs[j]=create_example(j,0,0,docs_org[i]->costfactor,docs_org[i]->fvec); label[j]=-1; c[j]=value[i]; } totdoc*=2; /* need to get a bigger kernel cache */ if(*kernel_cache) { kernel_cache_size=(*kernel_cache)->buffsize*sizeof(CFLOAT)/(1024*1024); kernel_cache_cleanup(*kernel_cache); (*kernel_cache)=kernel_cache_init(totdoc,kernel_cache_size); } runtime_start=get_runtime(); timing_profile.time_kernel=0; timing_profile.time_opti=0; timing_profile.time_shrink=0; timing_profile.time_update=0; timing_profile.time_model=0; timing_profile.time_check=0; timing_profile.time_select=0; kernel_cache_statistic=0; learn_parm->totwords=totwords; /* make sure -n value is reasonable */ if((learn_parm->svm_newvarsinqp < 2) || (learn_parm->svm_newvarsinqp > learn_parm->svm_maxqpsize)) { learn_parm->svm_newvarsinqp=learn_parm->svm_maxqpsize; } init_shrink_state(&shrink_state,totdoc,(long)MAXSHRINK); inconsistent = (long *)my_malloc(sizeof(long)*totdoc); unlabeled = (long *)my_malloc(sizeof(long)*totdoc); a = (double *)my_malloc(sizeof(double)*totdoc); a_fullset = (double *)my_malloc(sizeof(double)*totdoc); xi_fullset = (double *)my_malloc(sizeof(double)*totdoc); lin = (double *)my_malloc(sizeof(double)*totdoc); learn_parm->svm_cost = (double *)my_malloc(sizeof(double)*totdoc); model->supvec = (DOC **)my_malloc(sizeof(DOC *)*(totdoc+2)); model->alpha = (double *)my_malloc(sizeof(double)*(totdoc+2)); model->index = (long *)my_malloc(sizeof(long)*(totdoc+2)); model->at_upper_bound=0; model->b=0; model->supvec[0]=0; /* element 0 reserved and empty for now */ model->alpha[0]=0; model->lin_weights=NULL; model->totwords=totwords; model->totdoc=totdoc; model->kernel_parm=(*kernel_parm); model->sv_num=1; model->loo_error=-1; model->loo_recall=-1; model->loo_precision=-1; model->xa_error=-1; model->xa_recall=-1; model->xa_precision=-1; inconsistentnum=0; r_delta=estimate_r_delta(docs,totdoc,kernel_parm); r_delta_sq=r_delta*r_delta; r_delta_avg=estimate_r_delta_average(docs,totdoc,kernel_parm); if(learn_parm->svm_c == 0.0) { /* default value for C */ learn_parm->svm_c=1.0/(r_delta_avg*r_delta_avg); if(verbosity>=1) printf("Setting default regularization parameter C=%.4f\n", learn_parm->svm_c); } for(i=0;i 0) { learn_parm->svm_cost[i]=learn_parm->svm_c*learn_parm->svm_costratio* docs[i]->costfactor; } else if(label[i] < 0) { learn_parm->svm_cost[i]=learn_parm->svm_c*docs[i]->costfactor; } } /* caching makes no sense for linear kernel */ if((kernel_parm->kernel_type == LINEAR) && (*kernel_cache)) { printf("WARNING: Using a kernel cache for linear case will slow optimization down!\n"); } if(verbosity==1) { printf("Optimizing"); fflush(stdout); } /* train the svm */ iterations=optimize_to_convergence(docs,label,totdoc,totwords,learn_parm, kernel_parm,*kernel_cache,&shrink_state, model,inconsistent,unlabeled,a,lin,c, &timing_profile,&maxdiff,(long)-1, (long)1); if(verbosity>=1) { if(verbosity==1) printf("done. (%ld iterations)\n",iterations); printf("Optimization finished (maxdiff=%.5f).\n",maxdiff); runtime_end=get_runtime(); if(verbosity>=2) { printf("Runtime in cpu-seconds: %.2f (%.2f%% for kernel/%.2f%% for optimizer/%.2f%% for final/%.2f%% for update/%.2f%% for model/%.2f%% for check/%.2f%% for select)\n", (runtime_end-runtime_start)/100.0, (100.0*timing_profile.time_kernel)/(runtime_end-runtime_start), (100.0*timing_profile.time_opti)/(runtime_end-runtime_start), (100.0*timing_profile.time_shrink)/(runtime_end-runtime_start), (100.0*timing_profile.time_update)/(runtime_end-runtime_start), (100.0*timing_profile.time_model)/(runtime_end-runtime_start), (100.0*timing_profile.time_check)/(runtime_end-runtime_start), (100.0*timing_profile.time_select)/(runtime_end-runtime_start)); } else { printf("Runtime in cpu-seconds: %.2f\n", (runtime_end-runtime_start)/100.0); } if(learn_parm->remove_inconsistent) { inconsistentnum=0; for(i=0;isv_num-1,inconsistentnum); } else { upsupvecnum=0; for(i=1;isv_num;i++) { if(fabs(model->alpha[i]) >= (learn_parm->svm_cost[(model->supvec[i])->docnum]- learn_parm->epsilon_a)) upsupvecnum++; } printf("Number of SV: %ld (including %ld at upper bound)\n", model->sv_num-1,upsupvecnum); } if((verbosity>=1) && (!learn_parm->skip_final_opt_check)) { loss=0; model_length=0; for(i=0;ib)*(double)label[i] < (-learn_parm->eps+(double)label[i]*c[i])-learn_parm->epsilon_crit) loss+=-learn_parm->eps+(double)label[i]*c[i]-(lin[i]-model->b)*(double)label[i]; model_length+=a[i]*label[i]*lin[i]; } model_length=sqrt(model_length); fprintf(stdout,"L1 loss: loss=%.5f\n",loss); fprintf(stdout,"Norm of weight vector: |w|=%.5f\n",model_length); example_length=estimate_sphere(model); fprintf(stdout,"Norm of longest example vector: |x|=%.5f\n", length_of_longest_document_vector(docs,totdoc,kernel_parm)); } if(verbosity>=1) { printf("Number of kernel evaluations: %ld\n",kernel_cache_statistic); } } if(learn_parm->alphafile[0]) write_alphas(learn_parm->alphafile,a,label,totdoc); /* this makes sure the model we return does not contain pointers to the temporary documents */ for(i=1;isv_num;i++) { j=model->supvec[i]->docnum; if(j >= (totdoc/2)) { j=totdoc-j-1; } model->supvec[i]=docs_org[j]; } shrink_state_cleanup(&shrink_state); for(i=0;isvm_cost); } void svm_learn_ranking(DOC **docs, double *rankvalue, long int totdoc, long int totwords, LEARN_PARM *learn_parm, KERNEL_PARM *kernel_parm, KERNEL_CACHE **kernel_cache, MODEL *model) /* docs: Training vectors (x-part) */ /* rankvalue: Training target values that determine the ranking */ /* totdoc: Number of examples in docs/label */ /* totwords: Number of features (i.e. highest feature index) */ /* learn_parm: Learning paramenters */ /* kernel_parm: Kernel paramenters */ /* kernel_cache:Initialized pointer to Cache of size 1*totdoc, if using a kernel. NULL if linear. NOTE: Cache is getting reinitialized in this function */ /* model: Returns learning result (assumed empty before called) */ { DOC **docdiff; long i,j,k,totpair,kernel_cache_size; double *target,*alpha,cost; long *greater,*lesser; MODEL *pairmodel; SVECTOR *flow,*fhigh; totpair=0; for(i=0;iqueryid==docs[j]->queryid) && (rankvalue[i] != rankvalue[j])) { totpair++; } } } printf("Constructing %ld rank constraints...",totpair); fflush(stdout); docdiff=(DOC **)my_malloc(sizeof(DOC)*totpair); target=(double *)my_malloc(sizeof(double)*totpair); greater=(long *)my_malloc(sizeof(long)*totpair); lesser=(long *)my_malloc(sizeof(long)*totpair); k=0; for(i=0;iqueryid == docs[j]->queryid) { /* "Hijacked" costfactor to input rhs of constraints */ /* cost=(docs[i]->costfactor+docs[j]->costfactor)/2.0; */ cost=1; if(rankvalue[i] > rankvalue[j]) { if(kernel_parm->kernel_type == LINEAR) docdiff[k]=create_example(k,0,0,cost, sub_ss(docs[i]->fvec,docs[j]->fvec)); else { flow=copy_svector(docs[j]->fvec); flow->factor=-1.0; flow->next=NULL; fhigh=copy_svector(docs[i]->fvec); fhigh->factor=1.0; fhigh->next=flow; docdiff[k]=create_example(k,0,0,cost,fhigh); } target[k]=1+docs[i]->costfactor-docs[j]->costfactor; greater[k]=i; lesser[k]=j; k++; } else if(rankvalue[i] < rankvalue[j]) { if(kernel_parm->kernel_type == LINEAR) docdiff[k]=create_example(k,0,0,cost, sub_ss(docs[j]->fvec,docs[i]->fvec)); else { flow=copy_svector(docs[j]->fvec); flow->factor=1.0; flow->next=NULL; fhigh=copy_svector(docs[i]->fvec); fhigh->factor=-1.0; fhigh->next=flow; docdiff[k]=create_example(k,0,0,cost,fhigh); } target[k]=1+docs[j]->costfactor-docs[i]->costfactor; greater[k]=j; lesser[k]=i; k++; } } } } printf("done.\n"); fflush(stdout); /* need to get a bigger kernel cache */ if(*kernel_cache) { kernel_cache_size=(*kernel_cache)->buffsize*sizeof(CFLOAT)/(1024*1024); kernel_cache_cleanup(*kernel_cache); (*kernel_cache)=kernel_cache_init(totpair,kernel_cache_size); } /* must use unbiased hyperplane on difference vectors */ learn_parm->biased_hyperplane=0; pairmodel=(MODEL *)my_malloc(sizeof(MODEL)); svm_learn_optimization(docdiff,target,totpair,totwords,learn_parm, kernel_parm,(*kernel_cache),pairmodel,NULL); /* Transfer the result into a more compact model. If you would like to output the original model on pairs of documents, see below. */ alpha=(double *)my_malloc(sizeof(double)*totdoc); for(i=0;isv_num;i++) { alpha[lesser[(pairmodel->supvec[i])->docnum]]-=pairmodel->alpha[i]; alpha[greater[(pairmodel->supvec[i])->docnum]]+=pairmodel->alpha[i]; } model->supvec = (DOC **)my_malloc(sizeof(DOC *)*(totdoc+2)); model->alpha = (double *)my_malloc(sizeof(double)*(totdoc+2)); model->index = (long *)my_malloc(sizeof(long)*(totdoc+2)); model->supvec[0]=0; /* element 0 reserved and empty for now */ model->alpha[0]=0; model->sv_num=1; for(i=0;isupvec[model->sv_num]=docs[i]; model->alpha[model->sv_num]=alpha[i]; model->index[i]=model->sv_num; model->sv_num++; } else { model->index[i]=-1; } } model->at_upper_bound=0; model->b=0; model->lin_weights=NULL; model->totwords=totwords; model->totdoc=totdoc; model->kernel_parm=(*kernel_parm); model->loo_error=-1; model->loo_recall=-1; model->loo_precision=-1; model->xa_error=-1; model->xa_recall=-1; model->xa_precision=-1; free(alpha); free(greater); free(lesser); free(target); /* If you would like to output the original model on pairs of document, replace the following lines with '(*model)=(*pairmodel);' */ for(i=0;i rhs_i - \xi_i This corresponds to the -z o option. */ void svm_learn_optimization(DOC **docs, double *rhs, long int totdoc, long int totwords, LEARN_PARM *learn_parm, KERNEL_PARM *kernel_parm, KERNEL_CACHE *kernel_cache, MODEL *model, double *alpha) /* docs: Left-hand side of inequalities (x-part) */ /* rhs: Right-hand side of inequalities */ /* totdoc: Number of examples in docs/label */ /* totwords: Number of features (i.e. highest feature index) */ /* learn_parm: Learning paramenters */ /* kernel_parm: Kernel paramenters */ /* kernel_cache:Initialized Cache of size 1*totdoc, if using a kernel. NULL if linear.*/ /* model: Returns solution as SV expansion (assumed empty before called) */ /* alpha: Start values for the alpha variables or NULL pointer. The new alpha values are returned after optimization if not NULL. Array must be of size totdoc. */ { long i,*label; long misclassified,upsupvecnum; double loss,model_length,alphasum,example_length; double maxdiff,*lin,*a,*c; double runtime_start,runtime_end; long iterations,maxslackid,svsetnum; long *unlabeled,*inconsistent; double r_delta_avg; long *index,*index2dnum; double *weights,*slack,*alphaslack; CFLOAT *aicache; /* buffer to keep one row of hessian */ TIMING timing_profile; SHRINK_STATE shrink_state; runtime_start=get_runtime(); timing_profile.time_kernel=0; timing_profile.time_opti=0; timing_profile.time_shrink=0; timing_profile.time_update=0; timing_profile.time_model=0; timing_profile.time_check=0; timing_profile.time_select=0; kernel_cache_statistic=0; learn_parm->totwords=totwords; /* make sure -n value is reasonable */ if((learn_parm->svm_newvarsinqp < 2) || (learn_parm->svm_newvarsinqp > learn_parm->svm_maxqpsize)) { learn_parm->svm_newvarsinqp=learn_parm->svm_maxqpsize; } init_shrink_state(&shrink_state,totdoc,(long)MAXSHRINK); label = (long *)my_malloc(sizeof(long)*totdoc); unlabeled = (long *)my_malloc(sizeof(long)*totdoc); inconsistent = (long *)my_malloc(sizeof(long)*totdoc); c = (double *)my_malloc(sizeof(double)*totdoc); a = (double *)my_malloc(sizeof(double)*totdoc); lin = (double *)my_malloc(sizeof(double)*totdoc); learn_parm->svm_cost = (double *)my_malloc(sizeof(double)*totdoc); model->supvec = (DOC **)my_malloc(sizeof(DOC *)*(totdoc+2)); model->alpha = (double *)my_malloc(sizeof(double)*(totdoc+2)); model->index = (long *)my_malloc(sizeof(long)*(totdoc+2)); model->at_upper_bound=0; model->b=0; model->supvec[0]=0; /* element 0 reserved and empty for now */ model->alpha[0]=0; model->lin_weights=NULL; model->totwords=totwords; model->totdoc=totdoc; model->kernel_parm=(*kernel_parm); model->sv_num=1; model->loo_error=-1; model->loo_recall=-1; model->loo_precision=-1; model->xa_error=-1; model->xa_recall=-1; model->xa_precision=-1; r_delta_avg=estimate_r_delta_average(docs,totdoc,kernel_parm); if(learn_parm->svm_c == 0.0) { /* default value for C */ learn_parm->svm_c=1.0/(r_delta_avg*r_delta_avg); if(verbosity>=1) printf("Setting default regularization parameter C=%.4f\n", learn_parm->svm_c); } learn_parm->biased_hyperplane=0; /* learn an unbiased hyperplane */ learn_parm->eps=0.0; /* No margin, unless explicitly handcoded in the right-hand side in the training set. */ for(i=0;idocnum=i; a[i]=0; lin[i]=0; c[i]=rhs[i]; /* set right-hand side */ unlabeled[i]=0; inconsistent[i]=0; learn_parm->svm_cost[i]=learn_parm->svm_c*learn_parm->svm_costratio* docs[i]->costfactor; label[i]=1; } if(learn_parm->sharedslack) /* if shared slacks are used, they must */ for(i=0;islackid) { perror("Error: Missing shared slacks definitions in some of the examples."); exit(0); } /* print kernel matrix */ /* int j; for(i=0;i=1) { printf("Computing starting state..."); fflush(stdout); } index = (long *)my_malloc(sizeof(long)*totdoc); index2dnum = (long *)my_malloc(sizeof(long)*(totdoc+11)); if(kernel_parm->kernel_type == LINEAR) { weights=(double *)my_malloc(sizeof(double)*(totwords+1)); clear_nvector(weights,totwords); /* set weights to zero */ aicache=NULL; } else { weights=NULL; aicache = (CFLOAT *)my_malloc(sizeof(CFLOAT)*totdoc); } for(i=0;ilearn_parm->svm_cost[i]) alpha[i]=learn_parm->svm_cost[i]; } if(kernel_cache && (kernel_parm->kernel_type != LINEAR)) { for(i=0;i0) && (alpha[i]svm_cost[i]) && (kernel_cache_space_available(kernel_cache))) cache_kernel_row(kernel_cache,docs,i,kernel_parm); for(i=0;isvm_cost[i]) && (kernel_cache_space_available(kernel_cache))) cache_kernel_row(kernel_cache,docs,i,kernel_parm); } (void)compute_index(index,totdoc,index2dnum); update_linear_component(docs,label,index2dnum,alpha,a,index2dnum,totdoc, totwords,kernel_parm,kernel_cache,lin,aicache, weights); (void)calculate_svm_model(docs,label,unlabeled,lin,alpha,a,c, learn_parm,index2dnum,index2dnum,model); for(i=0;i=1) { printf("done.\n"); fflush(stdout); } } /* removing inconsistent does not work for general optimization problem */ if(learn_parm->remove_inconsistent) { learn_parm->remove_inconsistent = 0; printf("'remove inconsistent' not available in this mode. Switching option off!"); fflush(stdout); } /* caching makes no sense for linear kernel */ if(kernel_parm->kernel_type == LINEAR) { /* kernel_cache = NULL; */ } if(verbosity==1) { printf("Optimizing"); fflush(stdout); } /* train the svm */ if(learn_parm->sharedslack) iterations=optimize_to_convergence_sharedslack(docs,label,totdoc, totwords,learn_parm,kernel_parm, kernel_cache,&shrink_state,model, a,lin,c,&timing_profile, &maxdiff); else iterations=optimize_to_convergence(docs,label,totdoc, totwords,learn_parm,kernel_parm, kernel_cache,&shrink_state,model, inconsistent,unlabeled, a,lin,c,&timing_profile, &maxdiff,(long)-1,(long)1); if(verbosity>=1) { if(verbosity==1) printf("done. (%ld iterations)\n",iterations); misclassified=0; for(i=0;(ib)*(double)label[i] <= 0.0) misclassified++; } printf("Optimization finished (maxdiff=%.5f).\n",maxdiff); runtime_end=get_runtime(); if(verbosity>=2) { printf("Runtime in cpu-seconds: %.2f (%.2f%% for kernel/%.2f%% for optimizer/%.2f%% for final/%.2f%% for update/%.2f%% for model/%.2f%% for check/%.2f%% for select)\n", (runtime_end-runtime_start)/100.0, (100.0*timing_profile.time_kernel)/(runtime_end-runtime_start), (100.0*timing_profile.time_opti)/(runtime_end-runtime_start), (100.0*timing_profile.time_shrink)/(runtime_end-runtime_start), (100.0*timing_profile.time_update)/(runtime_end-runtime_start), (100.0*timing_profile.time_model)/(runtime_end-runtime_start), (100.0*timing_profile.time_check)/(runtime_end-runtime_start), (100.0*timing_profile.time_select)/(runtime_end-runtime_start)); } else { printf("Runtime in cpu-seconds: %.2f\n", (runtime_end-runtime_start)/100.0); } } if((verbosity>=1) && (!learn_parm->skip_final_opt_check)) { loss=0; model_length=0; alphasum=0; for(i=0;ib)*(double)label[i] < c[i]-learn_parm->epsilon_crit) loss+=c[i]-(lin[i]-model->b)*(double)label[i]; model_length+=a[i]*label[i]*lin[i]; alphasum+=rhs[i]*a[i]; } model_length=sqrt(model_length); fprintf(stdout,"Dual objective value: dval=%.5f\n", alphasum-0.5*model_length*model_length); fprintf(stdout,"Norm of weight vector: |w|=%.5f\n",model_length); } if(learn_parm->sharedslack) { index = (long *)my_malloc(sizeof(long)*totdoc); index2dnum = (long *)my_malloc(sizeof(long)*(totdoc+11)); maxslackid=0; for(i=0;islackid) maxslackid=docs[i]->slackid; } (void)compute_index(index,totdoc,index2dnum); slack=(double *)my_malloc(sizeof(double)*(maxslackid+1)); alphaslack=(double *)my_malloc(sizeof(double)*(maxslackid+1)); for(i=0;i<=maxslackid;i++) { /* init shared slacks */ slack[i]=0; alphaslack[i]=0; } for(i=0;islackid]+=a[i]; } compute_shared_slacks(docs,label,a,lin,c,index2dnum,learn_parm, slack,alphaslack); loss=0; model->at_upper_bound=0; svsetnum=0; for(i=0;i<=maxslackid;i++) { /* create full index */ loss+=slack[i]; if(alphaslack[i] > (learn_parm->svm_c - learn_parm->epsilon_a)) model->at_upper_bound++; if(alphaslack[i] > learn_parm->epsilon_a) svsetnum++; } free(index); free(index2dnum); free(slack); free(alphaslack); } if((verbosity>=1) && (!learn_parm->skip_final_opt_check)) { if(learn_parm->sharedslack) { printf("Number of SV: %ld\n", model->sv_num-1); printf("Number of non-zero slack variables: %ld (%ld slacks have non-zero alpha)\n", model->at_upper_bound,svsetnum); fprintf(stdout,"L1 loss: loss=%.5f\n",loss); } else { upsupvecnum=0; for(i=1;isv_num;i++) { if(fabs(model->alpha[i]) >= (learn_parm->svm_cost[(model->supvec[i])->docnum]- learn_parm->epsilon_a)) upsupvecnum++; } printf("Number of SV: %ld (including %ld at upper bound)\n", model->sv_num-1,upsupvecnum); fprintf(stdout,"L1 loss: loss=%.5f\n",loss); } example_length=estimate_sphere(model); fprintf(stdout,"Norm of longest example vector: |x|=%.5f\n", length_of_longest_document_vector(docs,totdoc,kernel_parm)); } if(verbosity>=1) { printf("Number of kernel evaluations: %ld\n",kernel_cache_statistic); } if(alpha) { for(i=0;ialphafile[0]) write_alphas(learn_parm->alphafile,a,label,totdoc); shrink_state_cleanup(&shrink_state); free(label); free(unlabeled); free(inconsistent); free(c); free(a); free(lin); free(learn_parm->svm_cost); } long optimize_to_convergence(DOC **docs, long int *label, long int totdoc, long int totwords, LEARN_PARM *learn_parm, KERNEL_PARM *kernel_parm, KERNEL_CACHE *kernel_cache, SHRINK_STATE *shrink_state, MODEL *model, long int *inconsistent, long int *unlabeled, double *a, double *lin, double *c, TIMING *timing_profile, double *maxdiff, long int heldout, long int retrain) /* docs: Training vectors (x-part) */ /* label: Training labels/value (y-part, zero if test example for transduction) */ /* totdoc: Number of examples in docs/label */ /* totwords: Number of features (i.e. highest feature index) */ /* laern_parm: Learning paramenters */ /* kernel_parm: Kernel paramenters */ /* kernel_cache: Initialized/partly filled Cache, if using a kernel. NULL if linear. */ /* shrink_state: State of active variables */ /* model: Returns learning result */ /* inconsistent: examples thrown out as inconstistent */ /* unlabeled: test examples for transduction */ /* a: alphas */ /* lin: linear component of gradient */ /* c: right hand side of inequalities (margin) */ /* maxdiff: returns maximum violation of KT-conditions */ /* heldout: marks held-out example for leave-one-out (or -1) */ /* retrain: selects training mode (1=regular / 2=holdout) */ { long *chosen,*key,i,j,jj,*last_suboptimal_at,noshrink; long inconsistentnum,choosenum,already_chosen=0,iteration; long misclassified,supvecnum=0,*active2dnum,inactivenum; long *working2dnum,*selexam; long activenum; double criterion,eq; double *a_old; double t0=0,t1=0,t2=0,t3=0,t4=0,t5=0,t6=0; /* timing */ long transductcycle; long transduction; double epsilon_crit_org; double bestmaxdiff; long bestmaxdiffiter,terminate; double *selcrit; /* buffer for sorting */ CFLOAT *aicache; /* buffer to keep one row of hessian */ double *weights; /* buffer for weight vector in linear case */ QP qp; /* buffer for one quadratic program */ epsilon_crit_org=learn_parm->epsilon_crit; /* save org */ if(kernel_parm->kernel_type == LINEAR) { learn_parm->epsilon_crit=2.0; /* kernel_cache=NULL; */ /* caching makes no sense for linear kernel */ } learn_parm->epsilon_shrink=2; (*maxdiff)=1; learn_parm->totwords=totwords; chosen = (long *)my_malloc(sizeof(long)*totdoc); last_suboptimal_at = (long *)my_malloc(sizeof(long)*totdoc); key = (long *)my_malloc(sizeof(long)*(totdoc+11)); selcrit = (double *)my_malloc(sizeof(double)*totdoc); selexam = (long *)my_malloc(sizeof(long)*totdoc); a_old = (double *)my_malloc(sizeof(double)*totdoc); aicache = (CFLOAT *)my_malloc(sizeof(CFLOAT)*totdoc); working2dnum = (long *)my_malloc(sizeof(long)*(totdoc+11)); active2dnum = (long *)my_malloc(sizeof(long)*(totdoc+11)); qp.opt_ce = (double *)my_malloc(sizeof(double)*learn_parm->svm_maxqpsize); qp.opt_ce0 = (double *)my_malloc(sizeof(double)); qp.opt_g = (double *)my_malloc(sizeof(double)*learn_parm->svm_maxqpsize *learn_parm->svm_maxqpsize); qp.opt_g0 = (double *)my_malloc(sizeof(double)*learn_parm->svm_maxqpsize); qp.opt_xinit = (double *)my_malloc(sizeof(double)*learn_parm->svm_maxqpsize); qp.opt_low=(double *)my_malloc(sizeof(double)*learn_parm->svm_maxqpsize); qp.opt_up=(double *)my_malloc(sizeof(double)*learn_parm->svm_maxqpsize); if(kernel_parm->kernel_type == LINEAR) { weights=create_nvector(totwords); clear_nvector(weights,totwords); /* set weights to zero */ } else weights=NULL; choosenum=0; inconsistentnum=0; transductcycle=0; transduction=0; if(!retrain) retrain=1; iteration=1; bestmaxdiffiter=1; bestmaxdiff=999999999; terminate=0; if(kernel_cache) { kernel_cache->time=iteration; /* for lru cache */ kernel_cache_reset_lru(kernel_cache); } for(i=0;iactive,totdoc,active2dnum); inactivenum=totdoc-activenum; clear_index(working2dnum); /* repeat this loop until we have convergence */ for(;retrain && (!terminate);iteration++) { if(kernel_cache) kernel_cache->time=iteration; /* for lru cache */ if(verbosity>=2) { printf( "Iteration %ld: ",iteration); fflush(stdout); } else if(verbosity==1) { printf("."); fflush(stdout); } if(verbosity>=2) t0=get_runtime(); if(verbosity>=3) { printf("\nSelecting working set... "); fflush(stdout); } if(learn_parm->svm_newvarsinqp>learn_parm->svm_maxqpsize) learn_parm->svm_newvarsinqp=learn_parm->svm_maxqpsize; i=0; for(jj=0;(j=working2dnum[jj])>=0;jj++) { /* clear working set */ if((chosen[j]>=(learn_parm->svm_maxqpsize/ minl(learn_parm->svm_maxqpsize, learn_parm->svm_newvarsinqp))) || (inconsistent[j]) || (j == heldout)) { chosen[j]=0; choosenum--; } else { chosen[j]++; working2dnum[i++]=j; } } working2dnum[i]=-1; if(retrain == 2) { choosenum=0; for(jj=0;(j=working2dnum[jj])>=0;jj++) { /* fully clear working set */ chosen[j]=0; } clear_index(working2dnum); for(i=0;ibiased_hyperplane) { eq=0; for(i=0;i learn_parm->epsilon_a);i++) { if((eq*label[i] > 0) && (a[i] > 0)) { chosen[i]=88888; choosenum++; if((eq*label[i]) > a[i]) { eq-=(a[i]*label[i]); a[i]=0; } else { a[i]-=(eq*label[i]); eq=0; } } } } compute_index(chosen,totdoc,working2dnum); } else { /* select working set according to steepest gradient */ if(iteration % 101) { already_chosen=0; if((minl(learn_parm->svm_newvarsinqp, learn_parm->svm_maxqpsize-choosenum)>=4) && (kernel_parm->kernel_type != LINEAR)) { /* select part of the working set from cache */ already_chosen=select_next_qp_subproblem_grad( label,unlabeled,a,lin,c,totdoc, (long)(minl(learn_parm->svm_maxqpsize-choosenum, learn_parm->svm_newvarsinqp) /2), learn_parm,inconsistent,active2dnum, working2dnum,selcrit,selexam,kernel_cache,1, key,chosen); choosenum+=already_chosen; } choosenum+=select_next_qp_subproblem_grad( label,unlabeled,a,lin,c,totdoc, minl(learn_parm->svm_maxqpsize-choosenum, learn_parm->svm_newvarsinqp-already_chosen), learn_parm,inconsistent,active2dnum, working2dnum,selcrit,selexam,kernel_cache,0,key, chosen); } else { /* once in a while, select a somewhat random working set to get unlocked of infinite loops due to numerical inaccuracies in the core qp-solver */ choosenum+=select_next_qp_subproblem_rand( label,unlabeled,a,lin,c,totdoc, minl(learn_parm->svm_maxqpsize-choosenum, learn_parm->svm_newvarsinqp), learn_parm,inconsistent,active2dnum, working2dnum,selcrit,selexam,kernel_cache,key, chosen,iteration); } } if(verbosity>=2) { printf(" %ld vectors chosen\n",choosenum); fflush(stdout); } if(verbosity>=2) t1=get_runtime(); if(kernel_cache) cache_multiple_kernel_rows(kernel_cache,docs,working2dnum, choosenum,kernel_parm); if(verbosity>=2) t2=get_runtime(); if(retrain != 2) { optimize_svm(docs,label,unlabeled,inconsistent,0.0,chosen,active2dnum, model,totdoc,working2dnum,choosenum,a,lin,c,learn_parm, aicache,kernel_parm,&qp,&epsilon_crit_org); } if(verbosity>=2) t3=get_runtime(); update_linear_component(docs,label,active2dnum,a,a_old,working2dnum,totdoc, totwords,kernel_parm,kernel_cache,lin,aicache, weights); if(verbosity>=2) t4=get_runtime(); supvecnum=calculate_svm_model(docs,label,unlabeled,lin,a,a_old,c, learn_parm,working2dnum,active2dnum,model); if(verbosity>=2) t5=get_runtime(); /* The following computation of the objective function works only */ /* relative to the active variables */ if(verbosity>=3) { criterion=compute_objective_function(a,lin,c,learn_parm->eps,label, active2dnum); printf("Objective function (over active variables): %.16f\n",criterion); fflush(stdout); } for(jj=0;(i=working2dnum[jj])>=0;jj++) { a_old[i]=a[i]; } if(retrain == 2) { /* reset inconsistent unlabeled examples */ for(i=0;(i=2) { t6=get_runtime(); timing_profile->time_select+=t1-t0; timing_profile->time_kernel+=t2-t1; timing_profile->time_opti+=t3-t2; timing_profile->time_update+=t4-t3; timing_profile->time_model+=t5-t4; timing_profile->time_check+=t6-t5; } /* checking whether optimizer got stuck */ if((*maxdiff) < bestmaxdiff) { bestmaxdiff=(*maxdiff); bestmaxdiffiter=iteration; } if(iteration > (bestmaxdiffiter+learn_parm->maxiter)) { /* long time no progress? */ terminate=1; retrain=0; if(verbosity>=1) printf("\nWARNING: Relaxing KT-Conditions due to slow progress! Terminating!\n"); } noshrink=0; if((!retrain) && (inactivenum>0) && ((!learn_parm->skip_final_opt_check) || (kernel_parm->kernel_type == LINEAR))) { if(((verbosity>=1) && (kernel_parm->kernel_type != LINEAR)) || (verbosity>=2)) { if(verbosity==1) { printf("\n"); } printf(" Checking optimality of inactive variables..."); fflush(stdout); } t1=get_runtime(); reactivate_inactive_examples(label,unlabeled,a,shrink_state,lin,c,totdoc, totwords,iteration,learn_parm,inconsistent, docs,kernel_parm,kernel_cache,model,aicache, weights,maxdiff); /* Update to new active variables. */ activenum=compute_index(shrink_state->active,totdoc,active2dnum); inactivenum=totdoc-activenum; /* reset watchdog */ bestmaxdiff=(*maxdiff); bestmaxdiffiter=iteration; /* termination criterion */ noshrink=1; retrain=0; if((*maxdiff) > learn_parm->epsilon_crit) retrain=1; timing_profile->time_shrink+=get_runtime()-t1; if(((verbosity>=1) && (kernel_parm->kernel_type != LINEAR)) || (verbosity>=2)) { printf("done.\n"); fflush(stdout); printf(" Number of inactive variables = %ld\n",inactivenum); } } if((!retrain) && (learn_parm->epsilon_crit>(*maxdiff))) learn_parm->epsilon_crit=(*maxdiff); if((!retrain) && (learn_parm->epsilon_crit>epsilon_crit_org)) { learn_parm->epsilon_crit/=2.0; retrain=1; noshrink=1; } if(learn_parm->epsilon_critepsilon_crit=epsilon_crit_org; if(verbosity>=2) { printf(" => (%ld SV (incl. %ld SV at u-bound), max violation=%.5f)\n", supvecnum,model->at_upper_bound,(*maxdiff)); fflush(stdout); } if(verbosity>=3) { printf("\n"); } if((!retrain) && (transduction)) { for(i=0;(iactive[i]=1; } activenum=compute_index(shrink_state->active,totdoc,active2dnum); inactivenum=0; if(verbosity==1) printf("done\n"); retrain=incorporate_unlabeled_examples(model,label,inconsistent, unlabeled,a,lin,totdoc, selcrit,selexam,key, transductcycle,kernel_parm, learn_parm); epsilon_crit_org=learn_parm->epsilon_crit; if(kernel_parm->kernel_type == LINEAR) learn_parm->epsilon_crit=1; transductcycle++; /* reset watchdog */ bestmaxdiff=(*maxdiff); bestmaxdiffiter=iteration; } else if(((iteration % 10) == 0) && (!noshrink)) { activenum=shrink_problem(docs,learn_parm,shrink_state,kernel_parm, active2dnum,last_suboptimal_at,iteration,totdoc, maxl((long)(activenum/10), maxl((long)(totdoc/500),100)), a,inconsistent); inactivenum=totdoc-activenum; if((kernel_cache) && (supvecnum>kernel_cache->max_elems) && ((kernel_cache->activenum-activenum)>maxl((long)(activenum/10),500))) { kernel_cache_shrink(kernel_cache,totdoc, minl((kernel_cache->activenum-activenum), (kernel_cache->activenum-supvecnum)), shrink_state->active); } } if((!retrain) && learn_parm->remove_inconsistent) { if(verbosity>=1) { printf(" Moving training errors to inconsistent examples..."); fflush(stdout); } if(learn_parm->remove_inconsistent == 1) { retrain=identify_inconsistent(a,label,unlabeled,totdoc,learn_parm, &inconsistentnum,inconsistent); } else if(learn_parm->remove_inconsistent == 2) { retrain=identify_misclassified(lin,label,unlabeled,totdoc, model,&inconsistentnum,inconsistent); } else if(learn_parm->remove_inconsistent == 3) { retrain=identify_one_misclassified(lin,label,unlabeled,totdoc, model,&inconsistentnum,inconsistent); } if(retrain) { if(kernel_parm->kernel_type == LINEAR) { /* reinit shrinking */ learn_parm->epsilon_crit=2.0; } } if(verbosity>=1) { printf("done.\n"); if(retrain) { printf(" Now %ld inconsistent examples.\n",inconsistentnum); } } } } /* end of loop */ free(chosen); free(last_suboptimal_at); free(key); free(selcrit); free(selexam); free(a_old); free(aicache); free(working2dnum); free(active2dnum); free(qp.opt_ce); free(qp.opt_ce0); free(qp.opt_g); free(qp.opt_g0); free(qp.opt_xinit); free(qp.opt_low); free(qp.opt_up); if(weights) free(weights); learn_parm->epsilon_crit=epsilon_crit_org; /* restore org */ model->maxdiff=(*maxdiff); return(iteration); } long optimize_to_convergence_sharedslack(DOC **docs, long int *label, long int totdoc, long int totwords, LEARN_PARM *learn_parm, KERNEL_PARM *kernel_parm, KERNEL_CACHE *kernel_cache, SHRINK_STATE *shrink_state, MODEL *model, double *a, double *lin, double *c, TIMING *timing_profile, double *maxdiff) /* docs: Training vectors (x-part) */ /* label: Training labels/value (y-part, zero if test example for transduction) */ /* totdoc: Number of examples in docs/label */ /* totwords: Number of features (i.e. highest feature index) */ /* learn_parm: Learning paramenters */ /* kernel_parm: Kernel paramenters */ /* kernel_cache: Initialized/partly filled Cache, if using a kernel. NULL if linear. */ /* shrink_state: State of active variables */ /* model: Returns learning result */ /* a: alphas */ /* lin: linear component of gradient */ /* c: right hand side of inequalities (margin) */ /* maxdiff: returns maximum violation of KT-conditions */ { long *chosen,*key,i,j,jj,*last_suboptimal_at,noshrink,*unlabeled; long *inconsistent,choosenum,already_chosen=0,iteration; long misclassified,supvecnum=0,*active2dnum,inactivenum; long *working2dnum,*selexam,*ignore; long activenum,retrain,maxslackid,slackset,jointstep; double criterion,eq_target; double *a_old,*alphaslack; double t0=0,t1=0,t2=0,t3=0,t4=0,t5=0,t6=0; /* timing */ double epsilon_crit_org,maxsharedviol; double bestmaxdiff; long bestmaxdiffiter,terminate; double *selcrit; /* buffer for sorting */ CFLOAT *aicache; /* buffer to keep one row of hessian */ double *weights; /* buffer for weight vector in linear case */ QP qp; /* buffer for one quadratic program */ double *slack; /* vector of slack variables for optimization with shared slacks */ epsilon_crit_org=learn_parm->epsilon_crit; /* save org */ if(kernel_parm->kernel_type == LINEAR) { learn_parm->epsilon_crit=2.0; /* kernel_cache=NULL; */ /* caching makes no sense for linear kernel */ } learn_parm->epsilon_shrink=2; (*maxdiff)=1; learn_parm->totwords=totwords; chosen = (long *)my_malloc(sizeof(long)*totdoc); unlabeled = (long *)my_malloc(sizeof(long)*totdoc); inconsistent = (long *)my_malloc(sizeof(long)*totdoc); ignore = (long *)my_malloc(sizeof(long)*totdoc); last_suboptimal_at = (long *)my_malloc(sizeof(long)*totdoc); key = (long *)my_malloc(sizeof(long)*(totdoc+11)); selcrit = (double *)my_malloc(sizeof(double)*totdoc); selexam = (long *)my_malloc(sizeof(long)*totdoc); a_old = (double *)my_malloc(sizeof(double)*totdoc); aicache = (CFLOAT *)my_malloc(sizeof(CFLOAT)*totdoc); working2dnum = (long *)my_malloc(sizeof(long)*(totdoc+11)); active2dnum = (long *)my_malloc(sizeof(long)*(totdoc+11)); qp.opt_ce = (double *)my_malloc(sizeof(double)*learn_parm->svm_maxqpsize); qp.opt_ce0 = (double *)my_malloc(sizeof(double)); qp.opt_g = (double *)my_malloc(sizeof(double)*learn_parm->svm_maxqpsize *learn_parm->svm_maxqpsize); qp.opt_g0 = (double *)my_malloc(sizeof(double)*learn_parm->svm_maxqpsize); qp.opt_xinit = (double *)my_malloc(sizeof(double)*learn_parm->svm_maxqpsize); qp.opt_low=(double *)my_malloc(sizeof(double)*learn_parm->svm_maxqpsize); qp.opt_up=(double *)my_malloc(sizeof(double)*learn_parm->svm_maxqpsize); if(kernel_parm->kernel_type == LINEAR) { weights=create_nvector(totwords); clear_nvector(weights,totwords); /* set weights to zero */ } else weights=NULL; maxslackid=0; for(i=0;islackid) maxslackid=docs[i]->slackid; } slack=(double *)my_malloc(sizeof(double)*(maxslackid+1)); alphaslack=(double *)my_malloc(sizeof(double)*(maxslackid+1)); for(i=0;i<=maxslackid;i++) { /* init shared slacks */ slack[i]=0; alphaslack[i]=0; } choosenum=0; retrain=1; iteration=1; bestmaxdiffiter=1; bestmaxdiff=999999999; terminate=0; if(kernel_cache) { kernel_cache->time=iteration; /* for lru cache */ kernel_cache_reset_lru(kernel_cache); } for(i=0;islackid]+=a[i]; a_old[i]=a[i]; last_suboptimal_at[i]=1; } activenum=compute_index(shrink_state->active,totdoc,active2dnum); inactivenum=totdoc-activenum; clear_index(working2dnum); /* call to init slack and alphaslack */ compute_shared_slacks(docs,label,a,lin,c,active2dnum,learn_parm, slack,alphaslack); /* repeat this loop until we have convergence */ for(;retrain && (!terminate);iteration++) { if(kernel_cache) kernel_cache->time=iteration; /* for lru cache */ if(verbosity>=2) { printf( "Iteration %ld: ",iteration); fflush(stdout); } else if(verbosity==1) { printf("."); fflush(stdout); } if(verbosity>=2) t0=get_runtime(); if(verbosity>=3) { printf("\nSelecting working set... "); fflush(stdout); } if(learn_parm->svm_newvarsinqp>learn_parm->svm_maxqpsize) learn_parm->svm_newvarsinqp=learn_parm->svm_maxqpsize; /* select working set according to steepest gradient */ jointstep=0; eq_target=0; if(iteration % 101) { slackset=select_next_qp_slackset(docs,label,a,lin,slack,alphaslack,c, learn_parm,active2dnum,&maxsharedviol); if((!(iteration % 100)) || (!slackset) || (maxsharedviolepsilon_crit)){ /* do a step with examples from different slack sets */ if(verbosity >= 2) { printf("(i-step)"); fflush(stdout); } i=0; for(jj=0;(j=working2dnum[jj])>=0;jj++) { /* clear old part of working set */ if((chosen[j]>=(learn_parm->svm_maxqpsize/ minl(learn_parm->svm_maxqpsize, learn_parm->svm_newvarsinqp)))) { chosen[j]=0; choosenum--; } else { chosen[j]++; working2dnum[i++]=j; } } working2dnum[i]=-1; already_chosen=0; if((minl(learn_parm->svm_newvarsinqp, learn_parm->svm_maxqpsize-choosenum)>=4) && (kernel_parm->kernel_type != LINEAR)) { /* select part of the working set from cache */ already_chosen=select_next_qp_subproblem_grad( label,unlabeled,a,lin,c,totdoc, (long)(minl(learn_parm->svm_maxqpsize-choosenum, learn_parm->svm_newvarsinqp) /2), learn_parm,inconsistent,active2dnum, working2dnum,selcrit,selexam,kernel_cache, (long)1,key,chosen); choosenum+=already_chosen; } choosenum+=select_next_qp_subproblem_grad( label,unlabeled,a,lin,c,totdoc, minl(learn_parm->svm_maxqpsize-choosenum, learn_parm->svm_newvarsinqp-already_chosen), learn_parm,inconsistent,active2dnum, working2dnum,selcrit,selexam,kernel_cache, (long)0,key,chosen); } else { /* do a step with all examples from same slack set */ if(verbosity >= 2) { printf("(j-step on %ld)",slackset); fflush(stdout); } jointstep=1; for(jj=0;(j=working2dnum[jj])>=0;jj++) { /* clear working set */ chosen[j]=0; } working2dnum[0]=-1; eq_target=alphaslack[slackset]; for(j=0;j=0;jj++) { */ if(docs[j]->slackid != slackset) ignore[j]=1; else { ignore[j]=0; learn_parm->svm_cost[j]=learn_parm->svm_c; /* printf("Inslackset(%ld,%ld)",j,shrink_state->active[j]); */ } } learn_parm->biased_hyperplane=1; choosenum=select_next_qp_subproblem_grad( label,unlabeled,a,lin,c,totdoc, learn_parm->svm_maxqpsize, learn_parm,ignore,active2dnum, working2dnum,selcrit,selexam,kernel_cache, (long)0,key,chosen); learn_parm->biased_hyperplane=0; } } else { /* once in a while, select a somewhat random working set to get unlocked of infinite loops due to numerical inaccuracies in the core qp-solver */ choosenum+=select_next_qp_subproblem_rand( label,unlabeled,a,lin,c,totdoc, minl(learn_parm->svm_maxqpsize-choosenum, learn_parm->svm_newvarsinqp), learn_parm,inconsistent,active2dnum, working2dnum,selcrit,selexam,kernel_cache,key, chosen,iteration); } if(verbosity>=2) { printf(" %ld vectors chosen\n",choosenum); fflush(stdout); } if(verbosity>=2) t1=get_runtime(); if(kernel_cache) cache_multiple_kernel_rows(kernel_cache,docs,working2dnum, choosenum,kernel_parm); if(verbosity>=2) t2=get_runtime(); if(jointstep) learn_parm->biased_hyperplane=1; optimize_svm(docs,label,unlabeled,ignore,eq_target,chosen,active2dnum, model,totdoc,working2dnum,choosenum,a,lin,c,learn_parm, aicache,kernel_parm,&qp,&epsilon_crit_org); learn_parm->biased_hyperplane=0; for(jj=0;(i=working2dnum[jj])>=0;jj++) /* recompute sums of alphas */ alphaslack[docs[i]->slackid]+=(a[i]-a_old[i]); for(jj=0;(i=working2dnum[jj])>=0;jj++) { /* reduce alpha to fulfill constraints */ if(alphaslack[docs[i]->slackid] > learn_parm->svm_c) { if(a[i] < (alphaslack[docs[i]->slackid]-learn_parm->svm_c)) { alphaslack[docs[i]->slackid]-=a[i]; a[i]=0; } else { a[i]-=(alphaslack[docs[i]->slackid]-learn_parm->svm_c); alphaslack[docs[i]->slackid]=learn_parm->svm_c; } } } for(jj=0;(i=active2dnum[jj])>=0;jj++) learn_parm->svm_cost[i]=a[i]+(learn_parm->svm_c -alphaslack[docs[i]->slackid]); model->at_upper_bound=0; for(jj=0;jj<=maxslackid;jj++) { if(alphaslack[jj]>(learn_parm->svm_c-learn_parm->epsilon_a)) model->at_upper_bound++; } if(verbosity>=2) t3=get_runtime(); update_linear_component(docs,label,active2dnum,a,a_old,working2dnum,totdoc, totwords,kernel_parm,kernel_cache,lin,aicache, weights); compute_shared_slacks(docs,label,a,lin,c,active2dnum,learn_parm, slack,alphaslack); if(verbosity>=2) t4=get_runtime(); supvecnum=calculate_svm_model(docs,label,unlabeled,lin,a,a_old,c, learn_parm,working2dnum,active2dnum,model); if(verbosity>=2) t5=get_runtime(); /* The following computation of the objective function works only */ /* relative to the active variables */ if(verbosity>=3) { criterion=compute_objective_function(a,lin,c,learn_parm->eps,label, active2dnum); printf("Objective function (over active variables): %.16f\n",criterion); fflush(stdout); } for(jj=0;(i=working2dnum[jj])>=0;jj++) { a_old[i]=a[i]; } retrain=check_optimality_sharedslack(docs,model,label,a,lin,c, slack,alphaslack,totdoc,learn_parm, maxdiff,epsilon_crit_org,&misclassified, active2dnum,last_suboptimal_at, iteration,kernel_parm); if(verbosity>=2) { t6=get_runtime(); timing_profile->time_select+=t1-t0; timing_profile->time_kernel+=t2-t1; timing_profile->time_opti+=t3-t2; timing_profile->time_update+=t4-t3; timing_profile->time_model+=t5-t4; timing_profile->time_check+=t6-t5; } /* checking whether optimizer got stuck */ if((*maxdiff) < bestmaxdiff) { bestmaxdiff=(*maxdiff); bestmaxdiffiter=iteration; } if(iteration > (bestmaxdiffiter+learn_parm->maxiter)) { /* long time no progress? */ terminate=1; retrain=0; if(verbosity>=1) printf("\nWARNING: Relaxing KT-Conditions due to slow progress! Terminating!\n"); } noshrink=0; if((!retrain) && (inactivenum>0) && ((!learn_parm->skip_final_opt_check) || (kernel_parm->kernel_type == LINEAR))) { if(((verbosity>=1) && (kernel_parm->kernel_type != LINEAR)) || (verbosity>=2)) { if(verbosity==1) { printf("\n"); } printf(" Checking optimality of inactive variables..."); fflush(stdout); } t1=get_runtime(); reactivate_inactive_examples(label,unlabeled,a,shrink_state,lin,c,totdoc, totwords,iteration,learn_parm,inconsistent, docs,kernel_parm,kernel_cache,model,aicache, weights,maxdiff); /* Update to new active variables. */ activenum=compute_index(shrink_state->active,totdoc,active2dnum); inactivenum=totdoc-activenum; /* check optimality, since check in reactivate does not work for sharedslacks */ compute_shared_slacks(docs,label,a,lin,c,active2dnum,learn_parm, slack,alphaslack); retrain=check_optimality_sharedslack(docs,model,label,a,lin,c, slack,alphaslack,totdoc,learn_parm, maxdiff,epsilon_crit_org,&misclassified, active2dnum,last_suboptimal_at, iteration,kernel_parm); /* reset watchdog */ bestmaxdiff=(*maxdiff); bestmaxdiffiter=iteration; /* termination criterion */ noshrink=1; retrain=0; if((*maxdiff) > learn_parm->epsilon_crit) retrain=1; timing_profile->time_shrink+=get_runtime()-t1; if(((verbosity>=1) && (kernel_parm->kernel_type != LINEAR)) || (verbosity>=2)) { printf("done.\n"); fflush(stdout); printf(" Number of inactive variables = %ld\n",inactivenum); } } if((!retrain) && (learn_parm->epsilon_crit>(*maxdiff))) learn_parm->epsilon_crit=(*maxdiff); if((!retrain) && (learn_parm->epsilon_crit>epsilon_crit_org)) { learn_parm->epsilon_crit/=2.0; retrain=1; noshrink=1; } if(learn_parm->epsilon_critepsilon_crit=epsilon_crit_org; if(verbosity>=2) { printf(" => (%ld SV (incl. %ld SV at u-bound), max violation=%.5f)\n", supvecnum,model->at_upper_bound,(*maxdiff)); fflush(stdout); } if(verbosity>=3) { printf("\n"); } if(((iteration % 10) == 0) && (!noshrink)) { activenum=shrink_problem(docs,learn_parm,shrink_state, kernel_parm,active2dnum, last_suboptimal_at,iteration,totdoc, maxl((long)(activenum/10), maxl((long)(totdoc/500),100)), a,inconsistent); inactivenum=totdoc-activenum; if((kernel_cache) && (supvecnum>kernel_cache->max_elems) && ((kernel_cache->activenum-activenum)>maxl((long)(activenum/10),500))) { kernel_cache_shrink(kernel_cache,totdoc, minl((kernel_cache->activenum-activenum), (kernel_cache->activenum-supvecnum)), shrink_state->active); } } } /* end of loop */ free(alphaslack); free(slack); free(chosen); free(unlabeled); free(inconsistent); free(ignore); free(last_suboptimal_at); free(key); free(selcrit); free(selexam); free(a_old); free(aicache); free(working2dnum); free(active2dnum); free(qp.opt_ce); free(qp.opt_ce0); free(qp.opt_g); free(qp.opt_g0); free(qp.opt_xinit); free(qp.opt_low); free(qp.opt_up); if(weights) free(weights); learn_parm->epsilon_crit=epsilon_crit_org; /* restore org */ model->maxdiff=(*maxdiff); return(iteration); } double compute_objective_function(double *a, double *lin, double *c, double eps, long int *label, long int *active2dnum) /* Return value of objective function. */ /* Works only relative to the active variables! */ { long i,ii; double criterion; /* calculate value of objective function */ criterion=0; for(ii=0;active2dnum[ii]>=0;ii++) { i=active2dnum[ii]; criterion=criterion+(eps-(double)label[i]*c[i])*a[i]+0.5*a[i]*label[i]*lin[i]; } return(criterion); } void clear_index(long int *index) /* initializes and empties index */ { index[0]=-1; } void add_to_index(long int *index, long int elem) /* initializes and empties index */ { register long i; for(i=0;index[i] != -1;i++); index[i]=elem; index[i+1]=-1; } long compute_index(long int *binfeature, long int range, long int *index) /* create an inverted index of binfeature */ { register long i,ii; ii=0; for(i=0;i=3) { printf("Running optimizer..."); fflush(stdout); } /* call the qp-subsolver */ a_v=optimize_qp(qp,epsilon_crit_target, learn_parm->svm_maxqpsize, &(model->b), /* in case the optimizer gives us */ /* the threshold for free. otherwise */ /* b is calculated in calculate_model. */ learn_parm); if(verbosity>=3) { printf("done\n"); } for(i=0;iepsilon_a)) { a[working2dnum[i]]=0; } else if(a_v[i]>=(learn_parm->svm_cost[working2dnum[i]]-learn_parm->epsilon_a)) { a[working2dnum[i]]=learn_parm->svm_cost[working2dnum[i]]; } */ } } void compute_matrices_for_optimization(DOC **docs, long int *label, long int *unlabeled, long *exclude_from_eq_const, double eq_target, long int *chosen, long int *active2dnum, long int *key, MODEL *model, double *a, double *lin, double *c, long int varnum, long int totdoc, LEARN_PARM *learn_parm, CFLOAT *aicache, KERNEL_PARM *kernel_parm, QP *qp) { register long ki,kj,i,j; register double kernel_temp; if(verbosity>=3) { fprintf(stdout,"Computing qp-matrices (type %ld kernel [degree %ld, rbf_gamma %f, coef_lin %f, coef_const %f])...",kernel_parm->kernel_type,kernel_parm->poly_degree,kernel_parm->rbf_gamma,kernel_parm->coef_lin,kernel_parm->coef_const); fflush(stdout); } qp->opt_n=varnum; qp->opt_ce0[0]=-eq_target; /* compute the constant for equality constraint */ for(j=1;jsv_num;j++) { /* start at 1 */ if((!chosen[(model->supvec[j])->docnum]) && (!exclude_from_eq_const[(model->supvec[j])->docnum])) { qp->opt_ce0[0]+=model->alpha[j]; } } if(learn_parm->biased_hyperplane) qp->opt_m=1; else qp->opt_m=0; /* eq-constraint will be ignored */ /* init linear part of objective function */ for(i=0;iopt_g0[i]=lin[key[i]]; } for(i=0;iopt_ce[i]=label[ki]; qp->opt_low[i]=0; qp->opt_up[i]=learn_parm->svm_cost[ki]; kernel_temp=(double)kernel(kernel_parm,docs[ki],docs[ki]); /* compute linear part of objective function */ qp->opt_g0[i]-=(kernel_temp*a[ki]*(double)label[ki]); /* compute quadratic part of objective function */ qp->opt_g[varnum*i+i]=kernel_temp; for(j=i+1;jopt_g0[i]-=(kernel_temp*a[kj]*(double)label[kj]); qp->opt_g0[j]-=(kernel_temp*a[ki]*(double)label[ki]); /* compute quadratic part of objective function */ qp->opt_g[varnum*i+j]=(double)label[ki]*(double)label[kj]*kernel_temp; qp->opt_g[varnum*j+i]=(double)label[ki]*(double)label[kj]*kernel_temp; } if(verbosity>=3) { if(i % 20 == 0) { fprintf(stdout,"%ld..",i); fflush(stdout); } } } for(i=0;iopt_xinit[i]=a[key[i]]; /* set linear part of objective function */ qp->opt_g0[i]=(learn_parm->eps-(double)label[key[i]]*c[key[i]])+qp->opt_g0[i]*(double)label[key[i]]; } if(verbosity>=3) { fprintf(stdout,"done\n"); } } long calculate_svm_model(DOC **docs, long int *label, long int *unlabeled, double *lin, double *a, double *a_old, double *c, LEARN_PARM *learn_parm, long int *working2dnum, long int *active2dnum, MODEL *model) /* Compute decision function based on current values */ /* of alpha. */ { long i,ii,pos,b_calculated=0,first_low,first_high; double ex_c,b_temp,b_low,b_high; if(verbosity>=3) { printf("Calculating model..."); fflush(stdout); } if(!learn_parm->biased_hyperplane) { model->b=0; b_calculated=1; } for(ii=0;(i=working2dnum[ii])>=0;ii++) { if((a_old[i]>0) && (a[i]==0)) { /* remove from model */ pos=model->index[i]; model->index[i]=-1; (model->sv_num)--; model->supvec[pos]=model->supvec[model->sv_num]; model->alpha[pos]=model->alpha[model->sv_num]; model->index[(model->supvec[pos])->docnum]=pos; } else if((a_old[i]==0) && (a[i]>0)) { /* add to model */ model->supvec[model->sv_num]=docs[i]; model->alpha[model->sv_num]=a[i]*(double)label[i]; model->index[i]=model->sv_num; (model->sv_num)++; } else if(a_old[i]==a[i]) { /* nothing to do */ } else { /* just update alpha */ model->alpha[model->index[i]]=a[i]*(double)label[i]; } ex_c=learn_parm->svm_cost[i]-learn_parm->epsilon_a; if(!learn_parm->sharedslack) { if((a_old[i]>=ex_c) && (a[i]at_upper_bound)--; } else if((a_old[i]=ex_c)) { (model->at_upper_bound)++; } } if((!b_calculated) && (a[i]>learn_parm->epsilon_a) && (a[i]b=((double)label[i]*learn_parm->eps-c[i]+lin[i]); /* model->b=(-(double)label[i]+lin[i]); */ b_calculated=1; } } /* No alpha in the working set not at bounds, so b was not calculated in the usual way. The following handles this special case. */ if(learn_parm->biased_hyperplane && (!b_calculated) && (model->sv_num-1 == model->at_upper_bound)) { first_low=1; first_high=1; b_low=0; b_high=0; for(ii=0;(i=active2dnum[ii])>=0;ii++) { ex_c=learn_parm->svm_cost[i]-learn_parm->epsilon_a; if(a[i]0) { b_temp=-(learn_parm->eps-c[i]+lin[i]); if((b_temp>b_low) || (first_low)) { b_low=b_temp; first_low=0; } } else { b_temp=-(-learn_parm->eps-c[i]+lin[i]); if((b_tempeps-c[i]+lin[i]); if((b_temp>b_low) || (first_low)) { b_low=b_temp; first_low=0; } } else { b_temp=-(learn_parm->eps-c[i]+lin[i]); if((b_tempb=-b_low; } else if(first_low) { model->b=-b_high; } else { model->b=-(b_high+b_low)/2.0; /* select b as the middle of range */ /* printf("\nb_low=%f, b_high=%f,b=%f\n",b_low,b_high,model->b); */ } } if(verbosity>=3) { printf("done\n"); fflush(stdout); } return(model->sv_num-1); /* have to substract one, since element 0 is empty*/ } long check_optimality(MODEL *model, long int *label, long int *unlabeled, double *a, double *lin, double *c, long int totdoc, LEARN_PARM *learn_parm, double *maxdiff, double epsilon_crit_org, long int *misclassified, long int *inconsistent, long int *active2dnum, long int *last_suboptimal_at, long int iteration, KERNEL_PARM *kernel_parm) /* Check KT-conditions */ { long i,ii,retrain; double dist,ex_c,target; if(kernel_parm->kernel_type == LINEAR) { /* be optimistic */ learn_parm->epsilon_shrink=-learn_parm->epsilon_crit+epsilon_crit_org; } else { /* be conservative */ learn_parm->epsilon_shrink=learn_parm->epsilon_shrink*0.7+(*maxdiff)*0.3; } retrain=0; (*maxdiff)=0; (*misclassified)=0; for(ii=0;(i=active2dnum[ii])>=0;ii++) { if((!inconsistent[i]) && label[i]) { dist=(lin[i]-model->b)*(double)label[i];/* 'distance' from hyperplane*/ target=-(learn_parm->eps-(double)label[i]*c[i]); ex_c=learn_parm->svm_cost[i]-learn_parm->epsilon_a; if(dist <= 0) { (*misclassified)++; /* does not work due to deactivation of var */ } if((a[i]>learn_parm->epsilon_a) && (dist > target)) { if((dist-target)>(*maxdiff)) /* largest violation */ (*maxdiff)=dist-target; } else if((a[i](*maxdiff)) /* largest violation */ (*maxdiff)=target-dist; } /* Count how long a variable was at lower/upper bound (and optimal).*/ /* Variables, which were at the bound and optimal for a long */ /* time are unlikely to become support vectors. In case our */ /* cache is filled up, those variables are excluded to save */ /* kernel evaluations. (See chapter 'Shrinking').*/ if((a[i]>(learn_parm->epsilon_a)) && (a[i]epsilon_a)) && (dist < (target+learn_parm->epsilon_shrink))) { last_suboptimal_at[i]=iteration; /* not likely optimal */ } else if((a[i]>=ex_c) && (dist > (target-learn_parm->epsilon_shrink))) { last_suboptimal_at[i]=iteration; /* not likely optimal */ } } } /* termination criterion */ if((!retrain) && ((*maxdiff) > learn_parm->epsilon_crit)) { retrain=1; } return(retrain); } long check_optimality_sharedslack(DOC **docs, MODEL *model, long int *label, double *a, double *lin, double *c, double *slack, double *alphaslack, long int totdoc, LEARN_PARM *learn_parm, double *maxdiff, double epsilon_crit_org, long int *misclassified, long int *active2dnum, long int *last_suboptimal_at, long int iteration, KERNEL_PARM *kernel_parm) /* Check KT-conditions */ { long i,ii,retrain; double dist,dist_noslack,ex_c=0,target; if(kernel_parm->kernel_type == LINEAR) { /* be optimistic */ learn_parm->epsilon_shrink=-learn_parm->epsilon_crit/2.0; } else { /* be conservative */ learn_parm->epsilon_shrink=learn_parm->epsilon_shrink*0.7+(*maxdiff)*0.3; } retrain=0; (*maxdiff)=0; (*misclassified)=0; for(ii=0;(i=active2dnum[ii])>=0;ii++) { /* 'distance' from hyperplane*/ dist_noslack=(lin[i]-model->b)*(double)label[i]; dist=dist_noslack+slack[docs[i]->slackid]; target=-(learn_parm->eps-(double)label[i]*c[i]); ex_c=learn_parm->svm_c-learn_parm->epsilon_a; if((a[i]>learn_parm->epsilon_a) && (dist > target)) { if((dist-target)>(*maxdiff)) { /* largest violation */ (*maxdiff)=dist-target; if(verbosity>=5) printf("sid %ld: dist=%.2f, target=%.2f, slack=%.2f, a=%f, alphaslack=%f\n",docs[i]->slackid,dist,target,slack[docs[i]->slackid],a[i],alphaslack[docs[i]->slackid]); if(verbosity>=5) printf(" (single %f)\n",(*maxdiff)); } } if((alphaslack[docs[i]->slackid]slackid]>0)) { if((slack[docs[i]->slackid])>(*maxdiff)) { /* largest violation */ (*maxdiff)=slack[docs[i]->slackid]; if(verbosity>=5) printf("sid %ld: dist=%.2f, target=%.2f, slack=%.2f, a=%f, alphaslack=%f\n",docs[i]->slackid,dist,target,slack[docs[i]->slackid],a[i],alphaslack[docs[i]->slackid]); if(verbosity>=5) printf(" (joint %f)\n",(*maxdiff)); } } /* Count how long a variable was at lower/upper bound (and optimal).*/ /* Variables, which were at the bound and optimal for a long */ /* time are unlikely to become support vectors. In case our */ /* cache is filled up, those variables are excluded to save */ /* kernel evaluations. (See chapter 'Shrinking').*/ if((a[i]<=learn_parm->epsilon_a) && (dist < (target+learn_parm->epsilon_shrink))) { last_suboptimal_at[i]=iteration; /* not likely optimal */ } else if((alphaslack[docs[i]->slackid]learn_parm->epsilon_a) && (fabs(dist_noslack - target) > -learn_parm->epsilon_shrink)) { last_suboptimal_at[i]=iteration; /* not at lower bound */ } else if((alphaslack[docs[i]->slackid]>=ex_c) && (a[i]>learn_parm->epsilon_a) && (fabs(target-dist) > -learn_parm->epsilon_shrink)) { last_suboptimal_at[i]=iteration; /* not likely optimal */ } } /* termination criterion */ if((!retrain) && ((*maxdiff) > learn_parm->epsilon_crit)) { retrain=1; } return(retrain); } void compute_shared_slacks(DOC **docs, long int *label, double *a, double *lin, double *c, long int *active2dnum, LEARN_PARM *learn_parm, double *slack, double *alphaslack) /* compute the value of shared slacks and the joint alphas */ { long jj,i; double dist,target; for(jj=0;(i=active2dnum[jj])>=0;jj++) { /* clear slack variables */ slack[docs[i]->slackid]=0.0; /* alphaslack[docs[i]->slackid]=0.0; */ } for(jj=0;(i=active2dnum[jj])>=0;jj++) { /* recompute slack variables */ dist=(lin[i])*(double)label[i]; target=-(learn_parm->eps-(double)label[i]*c[i]); if((target-dist) > slack[docs[i]->slackid]) slack[docs[i]->slackid]=target-dist; /* alphaslack[docs[i]->slackid]+=a[i]; */ } } long identify_inconsistent(double *a, long int *label, long int *unlabeled, long int totdoc, LEARN_PARM *learn_parm, long int *inconsistentnum, long int *inconsistent) { long i,retrain; /* Throw out examples with multipliers at upper bound. This */ /* corresponds to the -i 1 option. */ /* ATTENTION: this is just a heuristic for finding a close */ /* to minimum number of examples to exclude to */ /* make the problem separable with desired margin */ retrain=0; for(i=0;i=(learn_parm->svm_cost[i]-learn_parm->epsilon_a))) { (*inconsistentnum)++; inconsistent[i]=1; /* never choose again */ retrain=2; /* start over */ if(verbosity>=3) { printf("inconsistent(%ld)..",i); fflush(stdout); } } } return(retrain); } long identify_misclassified(double *lin, long int *label, long int *unlabeled, long int totdoc, MODEL *model, long int *inconsistentnum, long int *inconsistent) { long i,retrain; double dist; /* Throw out misclassified examples. This */ /* corresponds to the -i 2 option. */ /* ATTENTION: this is just a heuristic for finding a close */ /* to minimum number of examples to exclude to */ /* make the problem separable with desired margin */ retrain=0; for(i=0;ib)*(double)label[i]; /* 'distance' from hyperplane*/ if((!inconsistent[i]) && (!unlabeled[i]) && (dist <= 0)) { (*inconsistentnum)++; inconsistent[i]=1; /* never choose again */ retrain=2; /* start over */ if(verbosity>=3) { printf("inconsistent(%ld)..",i); fflush(stdout); } } } return(retrain); } long identify_one_misclassified(double *lin, long int *label, long int *unlabeled, long int totdoc, MODEL *model, long int *inconsistentnum, long int *inconsistent) { long i,retrain,maxex=-1; double dist,maxdist=0; /* Throw out the 'most misclassified' example. This */ /* corresponds to the -i 3 option. */ /* ATTENTION: this is just a heuristic for finding a close */ /* to minimum number of examples to exclude to */ /* make the problem separable with desired margin */ retrain=0; for(i=0;ib)*(double)label[i];/* 'distance' from hyperplane*/ if(dist=0) { (*inconsistentnum)++; inconsistent[maxex]=1; /* never choose again */ retrain=2; /* start over */ if(verbosity>=3) { printf("inconsistent(%ld)..",i); fflush(stdout); } } return(retrain); } void update_linear_component(DOC **docs, long int *label, long int *active2dnum, double *a, double *a_old, long int *working2dnum, long int totdoc, long int totwords, KERNEL_PARM *kernel_parm, KERNEL_CACHE *kernel_cache, double *lin, CFLOAT *aicache, double *weights) /* keep track of the linear component */ /* lin of the gradient etc. by updating */ /* based on the change of the variables */ /* in the current working set */ /* WARNING: Assumes that array of weights is initialized to all zero values for linear kernel! */ { register long i,ii,j,jj; register double tec; SVECTOR *f; if(kernel_parm->kernel_type==0) { /* special linear case */ /* clear_vector_n(weights,totwords); */ for(ii=0;(i=working2dnum[ii])>=0;ii++) { if(a[i] != a_old[i]) { for(f=docs[i]->fvec;f;f=f->next) add_vector_ns(weights,f, f->factor*((a[i]-a_old[i])*(double)label[i])); } } for(jj=0;(j=active2dnum[jj])>=0;jj++) { for(f=docs[j]->fvec;f;f=f->next) lin[j]+=f->factor*sprod_ns(weights,f); } for(ii=0;(i=working2dnum[ii])>=0;ii++) { if(a[i] != a_old[i]) { for(f=docs[i]->fvec;f;f=f->next) mult_vector_ns(weights,f,0.0); /* Set weights back to zero. */ } /* This is faster than init */ } /* weights to zero in each iter. */ } else { /* general case */ for(jj=0;(i=working2dnum[jj])>=0;jj++) { if(a[i] != a_old[i]) { get_kernel_row(kernel_cache,docs,i,totdoc,active2dnum,aicache, kernel_parm); for(ii=0;(j=active2dnum[ii])>=0;ii++) { tec=aicache[j]; lin[j]+=(((a[i]*tec)-(a_old[i]*tec))*(double)label[i]); } } } } } long incorporate_unlabeled_examples(MODEL *model, long int *label, long int *inconsistent, long int *unlabeled, double *a, double *lin, long int totdoc, double *selcrit, long int *select, long int *key, long int transductcycle, KERNEL_PARM *kernel_parm, LEARN_PARM *learn_parm) { long i,j,k,j1,j2,j3,j4,unsupaddnum1=0,unsupaddnum2=0; long pos,neg,upos,uneg,orgpos,orgneg,nolabel,newpos,newneg,allunlab; double dist,model_length,posratio,negratio; long check_every=2; double loss; static double switchsens=0.0,switchsensorg=0.0; double umin,umax,sumalpha; long imin=0,imax=0; static long switchnum=0; switchsens/=1.2; /* assumes that lin[] is up to date -> no inactive vars */ orgpos=0; orgneg=0; newpos=0; newneg=0; nolabel=0; allunlab=0; for(i=0;i 0) { orgpos++; } else { orgneg++; } } else { allunlab++; if(unlabeled[i]) { if(label[i] > 0) { newpos++; } else if(label[i] < 0) { newneg++; } } } if(label[i]==0) { nolabel++; } } if(learn_parm->transduction_posratio >= 0) { posratio=learn_parm->transduction_posratio; } else { posratio=(double)orgpos/(double)(orgpos+orgneg); /* use ratio of pos/neg */ } /* in training data */ negratio=1.0-posratio; learn_parm->svm_costratio=1.0; /* global */ if(posratio>0) { learn_parm->svm_costratio_unlab=negratio/posratio; } else { learn_parm->svm_costratio_unlab=1.0; } pos=0; neg=0; upos=0; uneg=0; for(i=0;ib); /* 'distance' from hyperplane*/ if(dist>0) { pos++; } else { neg++; } if(unlabeled[i]) { if(dist>0) { upos++; } else { uneg++; } } if((!unlabeled[i]) && (a[i]>(learn_parm->svm_cost[i]-learn_parm->epsilon_a))) { /* printf("Ubounded %ld (class %ld, unlabeled %ld)\n",i,label[i],unlabeled[i]); */ } } if(verbosity>=2) { printf("POS=%ld, ORGPOS=%ld, ORGNEG=%ld\n",pos,orgpos,orgneg); printf("POS=%ld, NEWPOS=%ld, NEWNEG=%ld\n",pos,newpos,newneg); printf("pos ratio = %f (%f).\n",(double)(upos)/(double)(allunlab),posratio); fflush(stdout); } if(transductcycle == 0) { j1=0; j2=0; j4=0; for(i=0;ib); /* 'distance' from hyperplane*/ if((label[i]==0) && (unlabeled[i])) { selcrit[j4]=dist; key[j4]=i; j4++; } } unsupaddnum1=0; unsupaddnum2=0; select_top_n(selcrit,j4,select,(long)(allunlab*posratio+0.5)); for(k=0;(k<(long)(allunlab*posratio+0.5));k++) { i=key[select[k]]; label[i]=1; unsupaddnum1++; j1++; } for(i=0;isvm_cost[i]=learn_parm->svm_c* learn_parm->svm_costratio_unlab*learn_parm->svm_unlabbound; } else if(label[i] == -1) { learn_parm->svm_cost[i]=learn_parm->svm_c* learn_parm->svm_unlabbound; } } } if(verbosity>=1) { /* printf("costratio %f, costratio_unlab %f, unlabbound %f\n", learn_parm->svm_costratio,learn_parm->svm_costratio_unlab, learn_parm->svm_unlabbound); */ printf("Classifying unlabeled data as %ld POS / %ld NEG.\n", unsupaddnum1,unsupaddnum2); fflush(stdout); } if(verbosity >= 1) printf("Retraining."); if(verbosity >= 2) printf("\n"); return((long)3); } if((transductcycle % check_every) == 0) { if(verbosity >= 1) printf("Retraining."); if(verbosity >= 2) printf("\n"); j1=0; j2=0; unsupaddnum1=0; unsupaddnum2=0; for(i=0;isvm_cost[i]=learn_parm->svm_c* learn_parm->svm_costratio_unlab*learn_parm->svm_unlabbound; } else if(label[i] == -1) { learn_parm->svm_cost[i]=learn_parm->svm_c* learn_parm->svm_unlabbound; } } } if(verbosity>=2) { /* printf("costratio %f, costratio_unlab %f, unlabbound %f\n", learn_parm->svm_costratio,learn_parm->svm_costratio_unlab, learn_parm->svm_unlabbound); */ printf("%ld positive -> Added %ld POS / %ld NEG unlabeled examples.\n", upos,unsupaddnum1,unsupaddnum2); fflush(stdout); } if(learn_parm->svm_unlabbound == 1) { learn_parm->epsilon_crit=0.001; /* do the last run right */ } else { learn_parm->epsilon_crit=0.01; /* otherwise, no need to be so picky */ } return((long)3); } else if(((transductcycle % check_every) < check_every)) { model_length=0; sumalpha=0; loss=0; for(i=0;ib); /* 'distance' from hyperplane*/ if((label[i]*dist)<(1.0-learn_parm->epsilon_crit)) { loss+=(1.0-(label[i]*dist))*learn_parm->svm_cost[i]; } } model_length=sqrt(model_length); if(verbosity>=2) { printf("Model-length = %f (%f), loss = %f, objective = %f\n", model_length,sumalpha,loss,loss+0.5*model_length*model_length); fflush(stdout); } j1=0; j2=0; j3=0; j4=0; unsupaddnum1=0; unsupaddnum2=0; umin=99999; umax=-99999; j4=1; while(j4) { umin=99999; umax=-99999; for(i=0;(ib); if((label[i]>0) && (unlabeled[i]) && (!inconsistent[i]) && (distumax)) { umax=dist; imax=i; } } if((umin < (umax+switchsens-1E-4))) { j1++; j2++; unsupaddnum1++; unlabeled[imin]=3; inconsistent[imin]=1; unsupaddnum2++; unlabeled[imax]=2; inconsistent[imax]=1; } else j4=0; j4=0; } for(j=0;(j0) { unlabeled[j]=2; } else if(label[j]<0) { unlabeled[j]=3; } /* inconsistent[j]=1; */ j3++; } } switchnum+=unsupaddnum1+unsupaddnum2; /* stop and print out current margin printf("switchnum %ld %ld\n",switchnum,kernel_parm->poly_degree); if(switchnum == 2*kernel_parm->poly_degree) { learn_parm->svm_unlabbound=1; } */ if((!unsupaddnum1) && (!unsupaddnum2)) { if((learn_parm->svm_unlabbound>=1) && ((newpos+newneg) == allunlab)) { for(j=0;(jpredfile,model,lin,a,unlabeled,label, totdoc,learn_parm); if(verbosity>=1) printf("Number of switches: %ld\n",switchnum); return((long)0); } switchsens=switchsensorg; learn_parm->svm_unlabbound*=1.5; if(learn_parm->svm_unlabbound>1) { learn_parm->svm_unlabbound=1; } model->at_upper_bound=0; /* since upper bound increased */ if(verbosity>=1) printf("Increasing influence of unlabeled examples to %f%% .", learn_parm->svm_unlabbound*100.0); } else if(verbosity>=1) { printf("%ld positive -> Switching labels of %ld POS / %ld NEG unlabeled examples.", upos,unsupaddnum1,unsupaddnum2); fflush(stdout); } if(verbosity >= 2) printf("\n"); learn_parm->epsilon_crit=0.5; /* don't need to be so picky */ for(i=0;isvm_cost[i]=learn_parm->svm_c* learn_parm->svm_costratio_unlab*learn_parm->svm_unlabbound; } else if(label[i] == -1) { learn_parm->svm_cost[i]=learn_parm->svm_c* learn_parm->svm_unlabbound; } } } return((long)2); } return((long)0); } /*************************** Working set selection ***************************/ long select_next_qp_subproblem_grad(long int *label, long int *unlabeled, double *a, double *lin, double *c, long int totdoc, long int qp_size, LEARN_PARM *learn_parm, long int *inconsistent, long int *active2dnum, long int *working2dnum, double *selcrit, long int *select, KERNEL_CACHE *kernel_cache, long int cache_only, long int *key, long int *chosen) /* Use the feasible direction approach to select the next qp-subproblem (see chapter 'Selecting a good working set'). If 'cache_only' is true, then the variables are selected only among those for which the kernel evaluations are cached. */ { long choosenum,i,j,k,activedoc,inum,valid; double s; for(inum=0;working2dnum[inum]>=0;inum++); /* find end of index */ choosenum=0; activedoc=0; for(i=0;(j=active2dnum[i])>=0;i++) { s=-label[j]; if(kernel_cache && cache_only) valid=(kernel_cache->index[j]>=0); else valid=1; if(valid && (!((a[j]<=(0+learn_parm->epsilon_a)) && (s<0))) && (!((a[j]>=(learn_parm->svm_cost[j]-learn_parm->epsilon_a)) && (s>0))) && (!chosen[j]) && (label[j]) && (!inconsistent[j])) { selcrit[activedoc]=(double)label[j]*(learn_parm->eps-(double)label[j]*c[j]+(double)label[j]*lin[j]); /* selcrit[activedoc]=(double)label[j]*(-1.0+(double)label[j]*lin[j]); */ key[activedoc]=j; activedoc++; } } select_top_n(selcrit,activedoc,select,(long)(qp_size/2)); for(k=0;(choosenum<(qp_size/2)) && (k<(qp_size/2)) && (kbiased_hyperplane || (selcrit[select[k]] > 0)) { */ i=key[select[k]]; chosen[i]=1; working2dnum[inum+choosenum]=i; choosenum+=1; if(kernel_cache) kernel_cache_touch(kernel_cache,i); /* make sure it does not get kicked out of cache */ /* } */ } activedoc=0; for(i=0;(j=active2dnum[i])>=0;i++) { s=label[j]; if(kernel_cache && cache_only) valid=(kernel_cache->index[j]>=0); else valid=1; if(valid && (!((a[j]<=(0+learn_parm->epsilon_a)) && (s<0))) && (!((a[j]>=(learn_parm->svm_cost[j]-learn_parm->epsilon_a)) && (s>0))) && (!chosen[j]) && (label[j]) && (!inconsistent[j])) { selcrit[activedoc]=-(double)label[j]*(learn_parm->eps-(double)label[j]*c[j]+(double)label[j]*lin[j]); /* selcrit[activedoc]=-(double)(label[j]*(-1.0+(double)label[j]*lin[j])); */ key[activedoc]=j; activedoc++; } } select_top_n(selcrit,activedoc,select,(long)(qp_size/2)); for(k=0;(choosenumbiased_hyperplane || (selcrit[select[k]] > 0)) { */ i=key[select[k]]; chosen[i]=1; working2dnum[inum+choosenum]=i; choosenum+=1; if(kernel_cache) kernel_cache_touch(kernel_cache,i); /* make sure it does not get kicked out of cache */ /* } */ } working2dnum[inum+choosenum]=-1; /* complete index */ return(choosenum); } long select_next_qp_subproblem_rand(long int *label, long int *unlabeled, double *a, double *lin, double *c, long int totdoc, long int qp_size, LEARN_PARM *learn_parm, long int *inconsistent, long int *active2dnum, long int *working2dnum, double *selcrit, long int *select, KERNEL_CACHE *kernel_cache, long int *key, long int *chosen, long int iteration) /* Use the feasible direction approach to select the next qp-subproblem (see section 'Selecting a good working set'). Chooses a feasible direction at (pseudo) random to help jump over numerical problem. */ { long choosenum,i,j,k,activedoc,inum; double s; for(inum=0;working2dnum[inum]>=0;inum++); /* find end of index */ choosenum=0; activedoc=0; for(i=0;(j=active2dnum[i])>=0;i++) { s=-label[j]; if((!((a[j]<=(0+learn_parm->epsilon_a)) && (s<0))) && (!((a[j]>=(learn_parm->svm_cost[j]-learn_parm->epsilon_a)) && (s>0))) && (!inconsistent[j]) && (label[j]) && (!chosen[j])) { selcrit[activedoc]=(j+iteration) % totdoc; key[activedoc]=j; activedoc++; } } select_top_n(selcrit,activedoc,select,(long)(qp_size/2)); for(k=0;(choosenum<(qp_size/2)) && (k<(qp_size/2)) && (k=0;i++) { s=label[j]; if((!((a[j]<=(0+learn_parm->epsilon_a)) && (s<0))) && (!((a[j]>=(learn_parm->svm_cost[j]-learn_parm->epsilon_a)) && (s>0))) && (!inconsistent[j]) && (label[j]) && (!chosen[j])) { selcrit[activedoc]=(j+iteration) % totdoc; key[activedoc]=j; activedoc++; } } select_top_n(selcrit,activedoc,select,(long)(qp_size/2)); for(k=0;(choosenum=0;ii++) { ex_c=learn_parm->svm_c-learn_parm->epsilon_a; if(alphaslack[docs[i]->slackid] >= ex_c) { dist=(lin[i])*(double)label[i]+slack[docs[i]->slackid]; /* distance */ target=-(learn_parm->eps-(double)label[i]*c[i]); /* rhs of constraint */ if((a[i]>learn_parm->epsilon_a) && (dist > target)) { if((dist-target)>maxdiff) { /* largest violation */ maxdiff=dist-target; maxdiffid=docs[i]->slackid; } } } } (*maxviol)=maxdiff; return(maxdiffid); } void select_top_n(double *selcrit, long int range, long int *select, long int n) { register long i,j; for(i=0;(i=0;j--) { if((j>0) && (selcrit[select[j-1]]0) { for(i=n;iselcrit[select[n-1]]) { for(j=n-1;j>=0;j--) { if((j>0) && (selcrit[select[j-1]]deactnum=0; shrink_state->active = (long *)my_malloc(sizeof(long)*totdoc); shrink_state->inactive_since = (long *)my_malloc(sizeof(long)*totdoc); shrink_state->a_history = (double **)my_malloc(sizeof(double *)*maxhistory); shrink_state->maxhistory=maxhistory; shrink_state->last_lin = (double *)my_malloc(sizeof(double)*totdoc); shrink_state->last_a = (double *)my_malloc(sizeof(double)*totdoc); for(i=0;iactive[i]=1; shrink_state->inactive_since[i]=0; shrink_state->last_a[i]=0; shrink_state->last_lin[i]=0; } } void shrink_state_cleanup(SHRINK_STATE *shrink_state) { free(shrink_state->active); free(shrink_state->inactive_since); if(shrink_state->deactnum > 0) free(shrink_state->a_history[shrink_state->deactnum-1]); free(shrink_state->a_history); free(shrink_state->last_a); free(shrink_state->last_lin); } long shrink_problem(DOC **docs, LEARN_PARM *learn_parm, SHRINK_STATE *shrink_state, KERNEL_PARM *kernel_parm, long int *active2dnum, long int *last_suboptimal_at, long int iteration, long int totdoc, long int minshrink, double *a, long int *inconsistent) /* Shrink some variables away. Do the shrinking only if at least minshrink variables can be removed. */ { long i,ii,change,activenum,lastiter; double *a_old; activenum=0; change=0; for(ii=0;active2dnum[ii]>=0;ii++) { i=active2dnum[ii]; activenum++; if(0 && learn_parm->sharedslack) lastiter=last_suboptimal_at[docs[i]->slackid]; else lastiter=last_suboptimal_at[i]; if(((iteration-lastiter) > learn_parm->svm_iter_to_shrink) || (inconsistent[i])) { change++; } } if((change>=minshrink) /* shrink only if sufficiently many candidates */ && (shrink_state->deactnummaxhistory)) { /* and enough memory */ /* Shrink problem by removing those variables which are */ /* optimal at a bound for a minimum number of iterations */ if(verbosity>=2) { printf(" Shrinking..."); fflush(stdout); } if(kernel_parm->kernel_type != LINEAR) { /* non-linear case save alphas */ a_old=(double *)my_malloc(sizeof(double)*totdoc); shrink_state->a_history[shrink_state->deactnum]=a_old; for(i=0;i=0;ii++) { i=active2dnum[ii]; if(0 && learn_parm->sharedslack) lastiter=last_suboptimal_at[docs[i]->slackid]; else lastiter=last_suboptimal_at[i]; if(((iteration-lastiter) > learn_parm->svm_iter_to_shrink) || (inconsistent[i])) { shrink_state->active[i]=0; shrink_state->inactive_since[i]=shrink_state->deactnum; } } activenum=compute_index(shrink_state->active,totdoc,active2dnum); shrink_state->deactnum++; if(kernel_parm->kernel_type == LINEAR) { shrink_state->deactnum=0; } if(verbosity>=2) { printf("done.\n"); fflush(stdout); printf(" Number of inactive variables = %ld\n",totdoc-activenum); } } return(activenum); } void reactivate_inactive_examples(long int *label, long int *unlabeled, double *a, SHRINK_STATE *shrink_state, double *lin, double *c, long int totdoc, long int totwords, long int iteration, LEARN_PARM *learn_parm, long int *inconsistent, DOC **docs, KERNEL_PARM *kernel_parm, KERNEL_CACHE *kernel_cache, MODEL *model, CFLOAT *aicache, double *weights, double *maxdiff) /* Make all variables active again which had been removed by shrinking. */ /* Computes lin for those variables from scratch. */ /* WARNING: Assumes that array of weights is initialized to all zero values for linear kernel! */ { register long i,j,ii,jj,t,*changed2dnum,*inactive2dnum; long *changed,*inactive; register double kernel_val,*a_old,dist; double ex_c,target; SVECTOR *f; if(kernel_parm->kernel_type == LINEAR) { /* special linear case */ /* clear_vector_n(weights,totwords); set weights to zero */ a_old=shrink_state->last_a; for(i=0;ifvec;f;f=f->next) add_vector_ns(weights,f, f->factor*((a[i]-a_old[i])*(double)label[i])); a_old[i]=a[i]; } } for(i=0;iactive[i]) { for(f=docs[i]->fvec;f;f=f->next) lin[i]=shrink_state->last_lin[i]+f->factor*sprod_ns(weights,f); } shrink_state->last_lin[i]=lin[i]; } for(i=0;ifvec;f;f=f->next) mult_vector_ns(weights,f,0.0); /* set weights back to zero */ } } else { changed=(long *)my_malloc(sizeof(long)*totdoc); changed2dnum=(long *)my_malloc(sizeof(long)*(totdoc+11)); inactive=(long *)my_malloc(sizeof(long)*totdoc); inactive2dnum=(long *)my_malloc(sizeof(long)*(totdoc+11)); for(t=shrink_state->deactnum-1;(t>=0) && shrink_state->a_history[t];t--) { if(verbosity>=2) { printf("%ld..",t); fflush(stdout); } a_old=shrink_state->a_history[t]; for(i=0;iactive[i]) && (shrink_state->inactive_since[i] == t)); changed[i]= (a[i] != a_old[i]); } compute_index(inactive,totdoc,inactive2dnum); compute_index(changed,totdoc,changed2dnum); for(ii=0;(i=changed2dnum[ii])>=0;ii++) { get_kernel_row(kernel_cache,docs,i,totdoc,inactive2dnum,aicache, kernel_parm); for(jj=0;(j=inactive2dnum[jj])>=0;jj++) { kernel_val=aicache[j]; lin[j]+=(((a[i]*kernel_val)-(a_old[i]*kernel_val))*(double)label[i]); } } } free(changed); free(changed2dnum); free(inactive); free(inactive2dnum); } (*maxdiff)=0; for(i=0;iinactive_since[i]=shrink_state->deactnum-1; if(!inconsistent[i]) { dist=(lin[i]-model->b)*(double)label[i]; target=-(learn_parm->eps-(double)label[i]*c[i]); ex_c=learn_parm->svm_cost[i]-learn_parm->epsilon_a; if((a[i]>learn_parm->epsilon_a) && (dist > target)) { if((dist-target)>(*maxdiff)) /* largest violation */ (*maxdiff)=dist-target; } else if((a[i](*maxdiff)) /* largest violation */ (*maxdiff)=target-dist; } if((a[i]>(0+learn_parm->epsilon_a)) && (a[i]active[i]=1; /* not at bound */ } else if((a[i]<=(0+learn_parm->epsilon_a)) && (dist < (target+learn_parm->epsilon_shrink))) { shrink_state->active[i]=1; } else if((a[i]>=ex_c) && (dist > (target-learn_parm->epsilon_shrink))) { shrink_state->active[i]=1; } else if(learn_parm->sharedslack) { /* make all active when sharedslack */ shrink_state->active[i]=1; } } } if(kernel_parm->kernel_type != LINEAR) { /* update history for non-linear */ for(i=0;ia_history[shrink_state->deactnum-1])[i]=a[i]; } for(t=shrink_state->deactnum-2;(t>=0) && shrink_state->a_history[t];t--) { free(shrink_state->a_history[t]); shrink_state->a_history[t]=0; } } } /****************************** Cache handling *******************************/ void get_kernel_row(KERNEL_CACHE *kernel_cache, DOC **docs, long int docnum, long int totdoc, long int *active2dnum, CFLOAT *buffer, KERNEL_PARM *kernel_parm) /* Get's a row of the matrix of kernel values This matrix has the same form as the Hessian, just that the elements are not multiplied by */ /* y_i * y_j * a_i * a_j */ /* Takes the values from the cache if available. */ { register long i,j,start; DOC *ex; ex=docs[docnum]; if(kernel_cache && (kernel_cache->index[docnum] != -1)) {/* row is cached? */ kernel_cache->lru[kernel_cache->index[docnum]]=kernel_cache->time;/* lru */ start=kernel_cache->activenum*kernel_cache->index[docnum]; for(i=0;(j=active2dnum[i])>=0;i++) { if(kernel_cache->totdoc2active[j] >= 0) { /* column is cached? */ buffer[j]=kernel_cache->buffer[start+kernel_cache->totdoc2active[j]]; } else { buffer[j]=(CFLOAT)kernel(kernel_parm,ex,docs[j]); } } } else { for(i=0;(j=active2dnum[i])>=0;i++) { buffer[j]=(CFLOAT)kernel(kernel_parm,ex,docs[j]); } } } void cache_kernel_row(KERNEL_CACHE *kernel_cache, DOC **docs, long int m, KERNEL_PARM *kernel_parm) /* Fills cache for the row m */ { register DOC *ex; register long j,k,l; register CFLOAT *cache; if(!kernel_cache_check(kernel_cache,m)) { /* not cached yet*/ cache = kernel_cache_clean_and_malloc(kernel_cache,m); if(cache) { l=kernel_cache->totdoc2active[m]; ex=docs[m]; for(j=0;jactivenum;j++) { /* fill cache */ k=kernel_cache->active2totdoc[j]; if((kernel_cache->index[k] != -1) && (l != -1) && (k != m)) { cache[j]=kernel_cache->buffer[kernel_cache->activenum *kernel_cache->index[k]+l]; } else { cache[j]=kernel(kernel_parm,ex,docs[k]); } } } else { perror("Error: Kernel cache full! => increase cache size"); } } } void cache_multiple_kernel_rows(KERNEL_CACHE *kernel_cache, DOC **docs, long int *key, long int varnum, KERNEL_PARM *kernel_parm) /* Fills cache for the rows in key */ { register long i; for(i=0;i=2) { printf(" Reorganizing cache..."); fflush(stdout); } keep=(long *)my_malloc(sizeof(long)*totdoc); for(j=0;jactivenum) && (scountactive2totdoc[jj]; if(!after[j]) { scount++; keep[j]=0; } } for(i=0;imax_elems;i++) { for(jj=0;jjactivenum;jj++) { j=kernel_cache->active2totdoc[jj]; if(!keep[j]) { from++; } else { kernel_cache->buffer[to]=kernel_cache->buffer[from]; to++; from++; } } } kernel_cache->activenum=0; for(j=0;jtotdoc2active[j] != -1)) { kernel_cache->active2totdoc[kernel_cache->activenum]=j; kernel_cache->totdoc2active[j]=kernel_cache->activenum; kernel_cache->activenum++; } else { kernel_cache->totdoc2active[j]=-1; } } kernel_cache->max_elems=(long)(kernel_cache->buffsize/kernel_cache->activenum); if(kernel_cache->max_elems>totdoc) { kernel_cache->max_elems=totdoc; } free(keep); if(verbosity>=2) { printf("done.\n"); fflush(stdout); printf(" Cache-size in rows = %ld\n",kernel_cache->max_elems); } } KERNEL_CACHE *kernel_cache_init(long int totdoc, long int buffsize) { long i; KERNEL_CACHE *kernel_cache; kernel_cache=(KERNEL_CACHE *)my_malloc(sizeof(KERNEL_CACHE)); kernel_cache->index = (long *)my_malloc(sizeof(long)*totdoc); kernel_cache->occu = (long *)my_malloc(sizeof(long)*totdoc); kernel_cache->lru = (long *)my_malloc(sizeof(long)*totdoc); kernel_cache->invindex = (long *)my_malloc(sizeof(long)*totdoc); kernel_cache->active2totdoc = (long *)my_malloc(sizeof(long)*totdoc); kernel_cache->totdoc2active = (long *)my_malloc(sizeof(long)*totdoc); kernel_cache->buffer = (CFLOAT *)my_malloc((size_t)(buffsize)*1024*1024); kernel_cache->buffsize=(long)(buffsize/sizeof(CFLOAT)*1024*1024); kernel_cache->max_elems=(long)(kernel_cache->buffsize/totdoc); if(kernel_cache->max_elems>totdoc) { kernel_cache->max_elems=totdoc; } if(verbosity>=2) { printf(" Cache-size in rows = %ld\n",kernel_cache->max_elems); printf(" Kernel evals so far: %ld\n",kernel_cache_statistic); } kernel_cache->elems=0; /* initialize cache */ for(i=0;iindex[i]=-1; kernel_cache->lru[i]=0; } for(i=0;ioccu[i]=0; kernel_cache->invindex[i]=-1; } kernel_cache->activenum=totdoc;; for(i=0;iactive2totdoc[i]=i; kernel_cache->totdoc2active[i]=i; } kernel_cache->time=0; return(kernel_cache); } void kernel_cache_reset_lru(KERNEL_CACHE *kernel_cache) { long maxlru=0,k; for(k=0;kmax_elems;k++) { if(maxlru < kernel_cache->lru[k]) maxlru=kernel_cache->lru[k]; } for(k=0;kmax_elems;k++) { kernel_cache->lru[k]-=maxlru; } } void kernel_cache_cleanup(KERNEL_CACHE *kernel_cache) { free(kernel_cache->index); free(kernel_cache->occu); free(kernel_cache->lru); free(kernel_cache->invindex); free(kernel_cache->active2totdoc); free(kernel_cache->totdoc2active); free(kernel_cache->buffer); free(kernel_cache); } long kernel_cache_malloc(KERNEL_CACHE *kernel_cache) { long i; if(kernel_cache_space_available(kernel_cache)) { for(i=0;imax_elems;i++) { if(!kernel_cache->occu[i]) { kernel_cache->occu[i]=1; kernel_cache->elems++; return(i); } } } return(-1); } void kernel_cache_free(KERNEL_CACHE *kernel_cache, long int i) { kernel_cache->occu[i]=0; kernel_cache->elems--; } long kernel_cache_free_lru(KERNEL_CACHE *kernel_cache) /* remove least recently used cache element */ { register long k,least_elem=-1,least_time; least_time=kernel_cache->time+1; for(k=0;kmax_elems;k++) { if(kernel_cache->invindex[k] != -1) { if(kernel_cache->lru[k]lru[k]; least_elem=k; } } } if(least_elem != -1) { kernel_cache_free(kernel_cache,least_elem); kernel_cache->index[kernel_cache->invindex[least_elem]]=-1; kernel_cache->invindex[least_elem]=-1; return(1); } return(0); } CFLOAT *kernel_cache_clean_and_malloc(KERNEL_CACHE *kernel_cache, long int docnum) /* Get a free cache entry. In case cache is full, the lru element is removed. */ { long result; if((result = kernel_cache_malloc(kernel_cache)) == -1) { if(kernel_cache_free_lru(kernel_cache)) { result = kernel_cache_malloc(kernel_cache); } } kernel_cache->index[docnum]=result; if(result == -1) { return(0); } kernel_cache->invindex[result]=docnum; kernel_cache->lru[kernel_cache->index[docnum]]=kernel_cache->time; /* lru */ return((CFLOAT *)((long)kernel_cache->buffer +(kernel_cache->activenum*sizeof(CFLOAT)* kernel_cache->index[docnum]))); } long kernel_cache_touch(KERNEL_CACHE *kernel_cache, long int docnum) /* Update lru time to avoid removal from cache. */ { if(kernel_cache && kernel_cache->index[docnum] != -1) { kernel_cache->lru[kernel_cache->index[docnum]]=kernel_cache->time; /* lru */ return(1); } return(0); } long kernel_cache_check(KERNEL_CACHE *kernel_cache, long int docnum) /* Is that row cached? */ { return(kernel_cache->index[docnum] != -1); } long kernel_cache_space_available(KERNEL_CACHE *kernel_cache) /* Is there room for one more row? */ { return(kernel_cache->elems < kernel_cache->max_elems); } /************************** Compute estimates ******************************/ void compute_xa_estimates(MODEL *model, long int *label, long int *unlabeled, long int totdoc, DOC **docs, double *lin, double *a, KERNEL_PARM *kernel_parm, LEARN_PARM *learn_parm, double *error, double *recall, double *precision) /* Computes xa-estimate of error rate, recall, and precision. See T. Joachims, Estimating the Generalization Performance of an SVM Efficiently, IMCL, 2000. */ { long i,looerror,looposerror,loonegerror; long totex,totposex; double xi,r_delta,r_delta_sq,sim=0; long *sv2dnum=NULL,*sv=NULL,svnum; r_delta=estimate_r_delta(docs,totdoc,kernel_parm); r_delta_sq=r_delta*r_delta; looerror=0; looposerror=0; loonegerror=0; totex=0; totposex=0; svnum=0; if(learn_parm->xa_depth > 0) { sv = (long *)my_malloc(sizeof(long)*(totdoc+11)); for(i=0;isv_num;i++) if(a[model->supvec[i]->docnum] < (learn_parm->svm_cost[model->supvec[i]->docnum] -learn_parm->epsilon_a)) { sv[model->supvec[i]->docnum]=1; svnum++; } sv2dnum = (long *)my_malloc(sizeof(long)*(totdoc+11)); clear_index(sv2dnum); compute_index(sv,totdoc,sv2dnum); } for(i=0;ib)*(double)label[i]); if(xi<0) xi=0; if(label[i]>0) { totposex++; } if((learn_parm->rho*a[i]*r_delta_sq+xi) >= 1.0) { if(learn_parm->xa_depth > 0) { /* makes assumptions */ sim=distribute_alpha_t_greedily(sv2dnum,svnum,docs,a,i,label, kernel_parm,learn_parm, (double)((1.0-xi-a[i]*r_delta_sq)/(2.0*a[i]))); } if((learn_parm->xa_depth == 0) || ((a[i]*kernel(kernel_parm,docs[i],docs[i])+a[i]*2.0*sim+xi) >= 1.0)) { looerror++; if(label[i]>0) { looposerror++; } else { loonegerror++; } } } totex++; } } (*error)=((double)looerror/(double)totex)*100.0; (*recall)=(1.0-(double)looposerror/(double)totposex)*100.0; (*precision)=(((double)totposex-(double)looposerror) /((double)totposex-(double)looposerror+(double)loonegerror))*100.0; free(sv); free(sv2dnum); } double distribute_alpha_t_greedily(long int *sv2dnum, long int svnum, DOC **docs, double *a, long int docnum, long int *label, KERNEL_PARM *kernel_parm, LEARN_PARM *learn_parm, double thresh) /* Experimental Code improving plain XiAlpha Estimates by computing a better bound using a greedy optimzation strategy. */ { long best_depth=0; long i,j,k,d,skip,allskip; double best,best_val[101],val,init_val_sq,init_val_lin; long best_ex[101]; CFLOAT *cache,*trow; cache=(CFLOAT *)my_malloc(sizeof(CFLOAT)*learn_parm->xa_depth*svnum); trow = (CFLOAT *)my_malloc(sizeof(CFLOAT)*svnum); for(k=0;kxa_depth;d++) { allskip=1; if(d>=1) { init_val_sq+=cache[best_ex[d-1]+svnum*(d-1)]; for(k=0;kxa_depth; } } free(cache); free(trow); /* printf("Distribute[%ld](%ld)=%f, ",docnum,best_depth,best); */ return(best); } void estimate_transduction_quality(MODEL *model, long int *label, long int *unlabeled, long int totdoc, DOC **docs, double *lin) /* Loo-bound based on observation that loo-errors must have an equal distribution in both training and test examples, given that the test examples are classified correctly. Compare chapter "Constraints on the Transductive Hyperplane" in my Dissertation. */ { long i,j,l=0,ulab=0,lab=0,labpos=0,labneg=0,ulabpos=0,ulabneg=0,totulab=0; double totlab=0,totlabpos=0,totlabneg=0,labsum=0,ulabsum=0; double r_delta,r_delta_sq,xi,xisum=0,asum=0; r_delta=estimate_r_delta(docs,totdoc,&(model->kernel_parm)); r_delta_sq=r_delta*r_delta; for(j=0;j 0) totlabpos++; else totlabneg++; } } for(j=1;jsv_num;j++) { i=model->supvec[j]->docnum; xi=1.0-((lin[i]-model->b)*(double)label[i]); if(xi<0) xi=0; xisum+=xi; asum+=fabs(model->alpha[j]); if(unlabeled[i]) { ulabsum+=(fabs(model->alpha[j])*r_delta_sq+xi); } else { labsum+=(fabs(model->alpha[j])*r_delta_sq+xi); } if((fabs(model->alpha[j])*r_delta_sq+xi) >= 1) { l++; if(unlabeled[model->supvec[j]->docnum]) { ulab++; if(model->alpha[j] > 0) ulabpos++; else ulabneg++; } else { lab++; if(model->alpha[j] > 0) labpos++; else labneg++; } } } printf("xacrit>=1: labeledpos=%.5f labeledneg=%.5f default=%.5f\n",(double)labpos/(double)totlab*100.0,(double)labneg/(double)totlab*100.0,(double)totlabpos/(double)(totlab)*100.0); printf("xacrit>=1: unlabelpos=%.5f unlabelneg=%.5f\n",(double)ulabpos/(double)totulab*100.0,(double)ulabneg/(double)totulab*100.0); printf("xacrit>=1: labeled=%.5f unlabled=%.5f all=%.5f\n",(double)lab/(double)totlab*100.0,(double)ulab/(double)totulab*100.0,(double)l/(double)(totdoc)*100.0); printf("xacritsum: labeled=%.5f unlabled=%.5f all=%.5f\n",(double)labsum/(double)totlab*100.0,(double)ulabsum/(double)totulab*100.0,(double)(labsum+ulabsum)/(double)(totdoc)*100.0); printf("r_delta_sq=%.5f xisum=%.5f asum=%.5f\n",r_delta_sq,xisum,asum); } double estimate_margin_vcdim(MODEL *model, double w, double R) /* optional: length of model vector in feature space */ /* optional: radius of ball containing the data */ { double h; /* follows chapter 5.6.4 in [Vapnik/95] */ if(w<0) { w=model_length_s(model); } if(R<0) { R=estimate_sphere(model); } h = w*w * R*R +1; return(h); } double estimate_sphere(MODEL *model) /* Approximates the radius of the ball containing */ /* the support vectors by bounding it with the */ { /* length of the longest support vector. This is */ register long j; /* pretty good for text categorization, since all */ double xlen,maxxlen=0; /* documents have feature vectors of length 1. It */ DOC *nulldoc; /* assumes that the center of the ball is at the */ WORD nullword; /* origin of the space. */ KERNEL_PARM *kernel_parm=&(model->kernel_parm); nullword.wnum=0; nulldoc=create_example(-2,0,0,0.0,create_svector(&nullword,NULL,1.0)); for(j=1;jsv_num;j++) { xlen=sqrt(kernel(kernel_parm,model->supvec[j],model->supvec[j]) -2*kernel(kernel_parm,model->supvec[j],nulldoc) +kernel(kernel_parm,nulldoc,nulldoc)); if(xlen>maxxlen) { maxxlen=xlen; } } free_example(nulldoc,1); return(maxxlen); } double estimate_r_delta(DOC **docs, long int totdoc, KERNEL_PARM *kernel_parm) { long i; double maxxlen,xlen; DOC *nulldoc; /* assumes that the center of the ball is at the */ WORD nullword; /* origin of the space. */ nullword.wnum=0; nulldoc=create_example(-2,0,0,0.0,create_svector(&nullword,NULL,1.0)); maxxlen=0; for(i=0;imaxxlen) { maxxlen=xlen; } } free_example(nulldoc,1); return(maxxlen); } double estimate_r_delta_average(DOC **docs, long int totdoc, KERNEL_PARM *kernel_parm) { long i; double avgxlen; DOC *nulldoc; /* assumes that the center of the ball is at the */ WORD nullword; /* origin of the space. */ nullword.wnum=0; nulldoc=create_example(-2,0,0,0.0,create_svector(&nullword,NULL,1.0)); avgxlen=0; for(i=0;imaxxlen) { maxxlen=xlen; } } return(maxxlen); } /****************************** IO-handling **********************************/ void write_prediction(char *predfile, MODEL *model, double *lin, double *a, long int *unlabeled, long int *label, long int totdoc, LEARN_PARM *learn_parm) { FILE *predfl; long i; double dist,a_max; if(verbosity>=1) { printf("Writing prediction file..."); fflush(stdout); } if ((predfl = fopen (predfile, "w")) == NULL) { perror (predfile); exit (1); } a_max=learn_parm->epsilon_a; for(i=0;ia_max)) { a_max=a[i]; } } for(i=0;i(learn_parm->epsilon_a))) { dist=(double)label[i]*(1.0-learn_parm->epsilon_crit-a[i]/(a_max*2.0)); } else { dist=(lin[i]-model->b); } if(dist>0) { fprintf(predfl,"%.8g:+1 %.8g:-1\n",dist,-dist); } else { fprintf(predfl,"%.8g:-1 %.8g:+1\n",-dist,dist); } } } fclose(predfl); if(verbosity>=1) { printf("done\n"); } } void write_alphas(char *alphafile, double *a, long int *label, long int totdoc) { FILE *alphafl; long i; if(verbosity>=1) { printf("Writing alpha file..."); fflush(stdout); } if ((alphafl = fopen (alphafile, "w")) == NULL) { perror (alphafile); exit (1); } for(i=0;i=1) { printf("done\n"); } } svm-struct-matlab-1.0/svm_light/svm_learn.h000066400000000000000000000202751160627511300211020ustar00rootroot00000000000000/***********************************************************************/ /* */ /* svm_learn.h */ /* */ /* Declarations for learning module of Support Vector Machine. */ /* */ /* Author: Thorsten Joachims */ /* Date: 02.07.02 */ /* */ /* Copyright (c) 2002 Thorsten Joachims - All rights reserved */ /* */ /* This software is available for non-commercial use only. It must */ /* not be modified and distributed without prior permission of the */ /* author. The author is not responsible for implications from the */ /* use of this software. */ /* */ /***********************************************************************/ #ifndef SVM_LEARN #define SVM_LEARN void svm_learn_classification(DOC **, double *, long, long, LEARN_PARM *, KERNEL_PARM *, KERNEL_CACHE *, MODEL *, double *); void svm_learn_regression(DOC **, double *, long, long, LEARN_PARM *, KERNEL_PARM *, KERNEL_CACHE **, MODEL *); void svm_learn_ranking(DOC **, double *, long, long, LEARN_PARM *, KERNEL_PARM *, KERNEL_CACHE **, MODEL *); void svm_learn_optimization(DOC **, double *, long, long, LEARN_PARM *, KERNEL_PARM *, KERNEL_CACHE *, MODEL *, double *); long optimize_to_convergence(DOC **, long *, long, long, LEARN_PARM *, KERNEL_PARM *, KERNEL_CACHE *, SHRINK_STATE *, MODEL *, long *, long *, double *, double *, double *, TIMING *, double *, long, long); long optimize_to_convergence_sharedslack(DOC **, long *, long, long, LEARN_PARM *, KERNEL_PARM *, KERNEL_CACHE *, SHRINK_STATE *, MODEL *, double *, double *, double *, TIMING *, double *); double compute_objective_function(double *, double *, double *, double, long *, long *); void clear_index(long *); void add_to_index(long *, long); long compute_index(long *,long, long *); void optimize_svm(DOC **, long *, long *, long *, double, long *, long *, MODEL *, long, long *, long, double *, double *, double *, LEARN_PARM *, CFLOAT *, KERNEL_PARM *, QP *, double *); void compute_matrices_for_optimization(DOC **, long *, long *, long *, double, long *, long *, long *, MODEL *, double *, double *, double *, long, long, LEARN_PARM *, CFLOAT *, KERNEL_PARM *, QP *); long calculate_svm_model(DOC **, long *, long *, double *, double *, double *, double *, LEARN_PARM *, long *, long *, MODEL *); long check_optimality(MODEL *, long *, long *, double *, double *, double *, long, LEARN_PARM *,double *, double, long *, long *, long *, long *, long, KERNEL_PARM *); long check_optimality_sharedslack(DOC **docs, MODEL *model, long int *label, double *a, double *lin, double *c, double *slack, double *alphaslack, long int totdoc, LEARN_PARM *learn_parm, double *maxdiff, double epsilon_crit_org, long int *misclassified, long int *active2dnum, long int *last_suboptimal_at, long int iteration, KERNEL_PARM *kernel_parm); void compute_shared_slacks(DOC **docs, long int *label, double *a, double *lin, double *c, long int *active2dnum, LEARN_PARM *learn_parm, double *slack, double *alphaslack); long identify_inconsistent(double *, long *, long *, long, LEARN_PARM *, long *, long *); long identify_misclassified(double *, long *, long *, long, MODEL *, long *, long *); long identify_one_misclassified(double *, long *, long *, long, MODEL *, long *, long *); long incorporate_unlabeled_examples(MODEL *, long *,long *, long *, double *, double *, long, double *, long *, long *, long, KERNEL_PARM *, LEARN_PARM *); void update_linear_component(DOC **, long *, long *, double *, double *, long *, long, long, KERNEL_PARM *, KERNEL_CACHE *, double *, CFLOAT *, double *); long select_next_qp_subproblem_grad(long *, long *, double *, double *, double *, long, long, LEARN_PARM *, long *, long *, long *, double *, long *, KERNEL_CACHE *, long, long *, long *); long select_next_qp_subproblem_rand(long *, long *, double *, double *, double *, long, long, LEARN_PARM *, long *, long *, long *, double *, long *, KERNEL_CACHE *, long *, long *, long); long select_next_qp_slackset(DOC **docs, long int *label, double *a, double *lin, double *slack, double *alphaslack, double *c, LEARN_PARM *learn_parm, long int *active2dnum, double *maxviol); void select_top_n(double *, long, long *, long); void init_shrink_state(SHRINK_STATE *, long, long); void shrink_state_cleanup(SHRINK_STATE *); long shrink_problem(DOC **, LEARN_PARM *, SHRINK_STATE *, KERNEL_PARM *, long *, long *, long, long, long, double *, long *); void reactivate_inactive_examples(long *, long *, double *, SHRINK_STATE *, double *, double*, long, long, long, LEARN_PARM *, long *, DOC **, KERNEL_PARM *, KERNEL_CACHE *, MODEL *, CFLOAT *, double *, double *); /* cache kernel evalutations to improve speed */ KERNEL_CACHE *kernel_cache_init(long, long); void kernel_cache_cleanup(KERNEL_CACHE *); void get_kernel_row(KERNEL_CACHE *,DOC **, long, long, long *, CFLOAT *, KERNEL_PARM *); void cache_kernel_row(KERNEL_CACHE *,DOC **, long, KERNEL_PARM *); void cache_multiple_kernel_rows(KERNEL_CACHE *,DOC **, long *, long, KERNEL_PARM *); void kernel_cache_shrink(KERNEL_CACHE *,long, long, long *); void kernel_cache_reset_lru(KERNEL_CACHE *); long kernel_cache_malloc(KERNEL_CACHE *); void kernel_cache_free(KERNEL_CACHE *,long); long kernel_cache_free_lru(KERNEL_CACHE *); CFLOAT *kernel_cache_clean_and_malloc(KERNEL_CACHE *,long); long kernel_cache_touch(KERNEL_CACHE *,long); long kernel_cache_check(KERNEL_CACHE *,long); long kernel_cache_space_available(KERNEL_CACHE *); void compute_xa_estimates(MODEL *, long *, long *, long, DOC **, double *, double *, KERNEL_PARM *, LEARN_PARM *, double *, double *, double *); double xa_estimate_error(MODEL *, long *, long *, long, DOC **, double *, double *, KERNEL_PARM *, LEARN_PARM *); double xa_estimate_recall(MODEL *, long *, long *, long, DOC **, double *, double *, KERNEL_PARM *, LEARN_PARM *); double xa_estimate_precision(MODEL *, long *, long *, long, DOC **, double *, double *, KERNEL_PARM *, LEARN_PARM *); void avg_similarity_of_sv_of_one_class(MODEL *, DOC **, double *, long *, KERNEL_PARM *, double *, double *); double most_similar_sv_of_same_class(MODEL *, DOC **, double *, long, long *, KERNEL_PARM *, LEARN_PARM *); double distribute_alpha_t_greedily(long *, long, DOC **, double *, long, long *, KERNEL_PARM *, LEARN_PARM *, double); double distribute_alpha_t_greedily_noindex(MODEL *, DOC **, double *, long, long *, KERNEL_PARM *, LEARN_PARM *, double); void estimate_transduction_quality(MODEL *, long *, long *, long, DOC **, double *); double estimate_margin_vcdim(MODEL *, double, double); double estimate_sphere(MODEL *); double estimate_r_delta_average(DOC **, long, KERNEL_PARM *); double estimate_r_delta(DOC **, long, KERNEL_PARM *); double length_of_longest_document_vector(DOC **, long, KERNEL_PARM *); void write_model(char *, MODEL *); void write_prediction(char *, MODEL *, double *, double *, long *, long *, long, LEARN_PARM *); void write_alphas(char *, double *, long *, long); typedef struct cache_parm_s { KERNEL_CACHE *kernel_cache; CFLOAT *cache; DOC **docs; long m; KERNEL_PARM *kernel_parm; long offset,stepsize; } cache_parm_t; #endif svm-struct-matlab-1.0/svm_light/svm_learn_main.c000066400000000000000000000331111160627511300220720ustar00rootroot00000000000000/***********************************************************************/ /* */ /* svm_learn_main.c */ /* */ /* Command line interface to the learning module of the */ /* Support Vector Machine. */ /* */ /* Author: Thorsten Joachims */ /* Date: 02.07.02 */ /* */ /* Copyright (c) 2000 Thorsten Joachims - All rights reserved */ /* */ /* This software is available for non-commercial use only. It must */ /* not be modified and distributed without prior permission of the */ /* author. The author is not responsible for implications from the */ /* use of this software. */ /* */ /***********************************************************************/ /* if svm-learn is used out of C++, define it as extern "C" */ #ifdef __cplusplus extern "C" { #endif # include "svm_common.h" # include "svm_learn.h" #ifdef __cplusplus } #endif char docfile[200]; /* file with training examples */ char modelfile[200]; /* file for resulting classifier */ char restartfile[200]; /* file with initial alphas */ void read_input_parameters(int, char **, char *, char *, char *, long *, LEARN_PARM *, KERNEL_PARM *); void wait_any_key(); void print_help(); int main (int argc, char* argv[]) { DOC **docs; /* training examples */ long totwords,totdoc,i; double *target; double *alpha_in=NULL; KERNEL_CACHE *kernel_cache; LEARN_PARM learn_parm; KERNEL_PARM kernel_parm; MODEL *model=(MODEL *)my_malloc(sizeof(MODEL)); read_input_parameters(argc,argv,docfile,modelfile,restartfile,&verbosity, &learn_parm,&kernel_parm); read_documents(docfile,&docs,&target,&totwords,&totdoc); if(restartfile[0]) alpha_in=read_alphas(restartfile,totdoc); if(kernel_parm.kernel_type == LINEAR) { /* don't need the cache */ kernel_cache=NULL; } else { /* Always get a new kernel cache. It is not possible to use the same cache for two different training runs */ kernel_cache=kernel_cache_init(totdoc,learn_parm.kernel_cache_size); } if(learn_parm.type == CLASSIFICATION) { svm_learn_classification(docs,target,totdoc,totwords,&learn_parm, &kernel_parm,kernel_cache,model,alpha_in); } else if(learn_parm.type == REGRESSION) { svm_learn_regression(docs,target,totdoc,totwords,&learn_parm, &kernel_parm,&kernel_cache,model); } else if(learn_parm.type == RANKING) { svm_learn_ranking(docs,target,totdoc,totwords,&learn_parm, &kernel_parm,&kernel_cache,model); } else if(learn_parm.type == OPTIMIZATION) { svm_learn_optimization(docs,target,totdoc,totwords,&learn_parm, &kernel_parm,kernel_cache,model,alpha_in); } if(kernel_cache) { /* Free the memory used for the cache. */ kernel_cache_cleanup(kernel_cache); } /* Warning: The model contains references to the original data 'docs'. If you want to free the original data, and only keep the model, you have to make a deep copy of 'model'. */ /* deep_copy_of_model=copy_model(model); */ write_model(modelfile,model); free(alpha_in); free_model(model,0); for(i=0;ibiased_hyperplane=atol(argv[i]); break; case 'i': i++; learn_parm->remove_inconsistent=atol(argv[i]); break; case 'f': i++; learn_parm->skip_final_opt_check=!atol(argv[i]); break; case 'q': i++; learn_parm->svm_maxqpsize=atol(argv[i]); break; case 'n': i++; learn_parm->svm_newvarsinqp=atol(argv[i]); break; case '#': i++; learn_parm->maxiter=atol(argv[i]); break; case 'h': i++; learn_parm->svm_iter_to_shrink=atol(argv[i]); break; case 'm': i++; learn_parm->kernel_cache_size=atol(argv[i]); break; case 'c': i++; learn_parm->svm_c=atof(argv[i]); break; case 'w': i++; learn_parm->eps=atof(argv[i]); break; case 'p': i++; learn_parm->transduction_posratio=atof(argv[i]); break; case 'j': i++; learn_parm->svm_costratio=atof(argv[i]); break; case 'e': i++; learn_parm->epsilon_crit=atof(argv[i]); break; case 'o': i++; learn_parm->rho=atof(argv[i]); break; case 'k': i++; learn_parm->xa_depth=atol(argv[i]); break; case 'x': i++; learn_parm->compute_loo=atol(argv[i]); break; case 't': i++; kernel_parm->kernel_type=atol(argv[i]); break; case 'd': i++; kernel_parm->poly_degree=atol(argv[i]); break; case 'g': i++; kernel_parm->rbf_gamma=atof(argv[i]); break; case 's': i++; kernel_parm->coef_lin=atof(argv[i]); break; case 'r': i++; kernel_parm->coef_const=atof(argv[i]); break; case 'u': i++; strcpy(kernel_parm->custom,argv[i]); break; case 'l': i++; strcpy(learn_parm->predfile,argv[i]); break; case 'a': i++; strcpy(learn_parm->alphafile,argv[i]); break; case 'y': i++; strcpy(restartfile,argv[i]); break; default: printf("\nUnrecognized option %s!\n\n",argv[i]); print_help(); exit(0); } } if(i>=argc) { printf("\nNot enough input parameters!\n\n"); wait_any_key(); print_help(); exit(0); } strcpy (docfile, argv[i]); if((i+1)svm_iter_to_shrink == -9999) { if(kernel_parm->kernel_type == LINEAR) learn_parm->svm_iter_to_shrink=2; else learn_parm->svm_iter_to_shrink=100; } if(strcmp(type,"c")==0) { learn_parm->type=CLASSIFICATION; } else if(strcmp(type,"r")==0) { learn_parm->type=REGRESSION; } else if(strcmp(type,"p")==0) { learn_parm->type=RANKING; } else if(strcmp(type,"o")==0) { learn_parm->type=OPTIMIZATION; } else if(strcmp(type,"s")==0) { learn_parm->type=OPTIMIZATION; learn_parm->sharedslack=1; } else { printf("\nUnknown type '%s': Valid types are 'c' (classification), 'r' regession, and 'p' preference ranking.\n",type); wait_any_key(); print_help(); exit(0); } if (!check_learning_parms(learn_parm, kernel_parm)) { wait_any_key(); print_help(); exit(0); } } void wait_any_key() { printf("\n(more)\n"); (void)getc(stdin); } void print_help() { printf("\nSVM-light %s: Support Vector Machine, learning module %s\n",VERSION,VERSION_DATE); copyright_notice(); printf(" usage: svm_learn [options] example_file model_file\n\n"); printf("Arguments:\n"); printf(" example_file-> file with training data\n"); printf(" model_file -> file to store learned decision rule in\n"); printf("General options:\n"); printf(" -? -> this help\n"); printf(" -v [0..3] -> verbosity level (default 1)\n"); printf("Learning options:\n"); printf(" -z {c,r,p} -> select between classification (c), regression (r),\n"); printf(" and preference ranking (p) (default classification)\n"); printf(" -c float -> C: trade-off between training error\n"); printf(" and margin (default [avg. x*x]^-1)\n"); printf(" -w [0..] -> epsilon width of tube for regression\n"); printf(" (default 0.1)\n"); printf(" -j float -> Cost: cost-factor, by which training errors on\n"); printf(" positive examples outweight errors on negative\n"); printf(" examples (default 1) (see [4])\n"); printf(" -b [0,1] -> use biased hyperplane (i.e. x*w+b>0) instead\n"); printf(" of unbiased hyperplane (i.e. x*w>0) (default 1)\n"); printf(" -i [0,1] -> remove inconsistent training examples\n"); printf(" and retrain (default 0)\n"); printf("Performance estimation options:\n"); printf(" -x [0,1] -> compute leave-one-out estimates (default 0)\n"); printf(" (see [5])\n"); printf(" -o ]0..2] -> value of rho for XiAlpha-estimator and for pruning\n"); printf(" leave-one-out computation (default 1.0) (see [2])\n"); printf(" -k [0..100] -> search depth for extended XiAlpha-estimator \n"); printf(" (default 0)\n"); printf("Transduction options (see [3]):\n"); printf(" -p [0..1] -> fraction of unlabeled examples to be classified\n"); printf(" into the positive class (default is the ratio of\n"); printf(" positive and negative examples in the training data)\n"); printf("Kernel options:\n"); printf(" -t int -> type of kernel function:\n"); printf(" 0: linear (default)\n"); printf(" 1: polynomial (s a*b+c)^d\n"); printf(" 2: radial basis function exp(-gamma ||a-b||^2)\n"); printf(" 3: sigmoid tanh(s a*b + c)\n"); printf(" 4: user defined kernel from kernel.h\n"); printf(" -d int -> parameter d in polynomial kernel\n"); printf(" -g float -> parameter gamma in rbf kernel\n"); printf(" -s float -> parameter s in sigmoid/poly kernel\n"); printf(" -r float -> parameter c in sigmoid/poly kernel\n"); printf(" -u string -> parameter of user defined kernel\n"); printf("Optimization options (see [1]):\n"); printf(" -q [2..] -> maximum size of QP-subproblems (default 10)\n"); printf(" -n [2..q] -> number of new variables entering the working set\n"); printf(" in each iteration (default n = q). Set n < q to \n"); printf(" prevent zig-zagging.\n"); printf(" -m [5..] -> size of cache for kernel evaluations in MB (default 40)\n"); printf(" The larger the faster...\n"); printf(" -e float -> eps: Allow that error for termination criterion\n"); printf(" [y [w*x+b] - 1] >= eps (default 0.001)\n"); printf(" -y [0,1] -> restart the optimization from alpha values in file\n"); printf(" specified by -a option. (default 0)\n"); printf(" -h [5..] -> number of iterations a variable needs to be\n"); printf(" optimal before considered for shrinking (default 100)\n"); printf(" -f [0,1] -> do final optimality check for variables removed\n"); printf(" by shrinking. Although this test is usually \n"); printf(" positive, there is no guarantee that the optimum\n"); printf(" was found if the test is omitted. (default 1)\n"); printf(" -y string -> if option is given, reads alphas from file with given\n"); printf(" and uses them as starting point. (default 'disabled')\n"); printf(" -# int -> terminate optimization, if no progress after this\n"); printf(" number of iterations. (default 100000)\n"); printf("Output options:\n"); printf(" -l string -> file to write predicted labels of unlabeled\n"); printf(" examples into after transductive learning\n"); printf(" -a string -> write all alphas to this file after learning\n"); printf(" (in the same order as in the training set)\n"); wait_any_key(); printf("\nMore details in:\n"); printf("[1] T. Joachims, Making Large-Scale SVM Learning Practical. Advances in\n"); printf(" Kernel Methods - Support Vector Learning, B. Schölkopf and C. Burges and\n"); printf(" A. Smola (ed.), MIT Press, 1999.\n"); printf("[2] T. Joachims, Estimating the Generalization performance of an SVM\n"); printf(" Efficiently. International Conference on Machine Learning (ICML), 2000.\n"); printf("[3] T. Joachims, Transductive Inference for Text Classification using Support\n"); printf(" Vector Machines. International Conference on Machine Learning (ICML),\n"); printf(" 1999.\n"); printf("[4] K. Morik, P. Brockhausen, and T. Joachims, Combining statistical learning\n"); printf(" with a knowledge-based approach - A case study in intensive care \n"); printf(" monitoring. International Conference on Machine Learning (ICML), 1999.\n"); printf("[5] T. Joachims, Learning to Classify Text Using Support Vector\n"); printf(" Machines: Methods, Theory, and Algorithms. Dissertation, Kluwer,\n"); printf(" 2002.\n\n"); } svm-struct-matlab-1.0/svm_light/svm_loqo.c000066400000000000000000000154011160627511300207410ustar00rootroot00000000000000/***********************************************************************/ /* */ /* svm_loqo.c */ /* */ /* Interface to the PR_LOQO optimization package for SVM. */ /* */ /* Author: Thorsten Joachims */ /* Date: 19.07.99 */ /* */ /* Copyright (c) 1999 Universitaet Dortmund - All rights reserved */ /* */ /* This software is available for non-commercial use only. It must */ /* not be modified and distributed without prior permission of the */ /* author. The author is not responsible for implications from the */ /* use of this software. */ /* */ /***********************************************************************/ # include # include "pr_loqo/pr_loqo.h" # include "svm_common.h" /* Common Block Declarations */ long verbosity; /* /////////////////////////////////////////////////////////////// */ # define DEF_PRECISION_LINEAR 1E-8 # define DEF_PRECISION_NONLINEAR 1E-14 double *optimize_qp(); double *primal=0,*dual=0; double init_margin=0.15; long init_iter=500,precision_violations=0; double model_b; double opt_precision=DEF_PRECISION_LINEAR; /* /////////////////////////////////////////////////////////////// */ void *my_malloc(); double *optimize_qp(qp,epsilon_crit,nx,threshold,learn_parm) QP *qp; double *epsilon_crit; long nx; /* Maximum number of variables in QP */ double *threshold; LEARN_PARM *learn_parm; /* start the optimizer and return the optimal values */ { register long i,j,result; double margin,obj_before,obj_after; double sigdig,dist,epsilon_loqo; int iter; if(!primal) { /* allocate memory at first call */ primal=(double *)my_malloc(sizeof(double)*nx*3); dual=(double *)my_malloc(sizeof(double)*(nx*2+1)); } if(verbosity>=4) { /* really verbose */ printf("\n\n"); for(i=0;iopt_n;i++) { printf("%f: ",qp->opt_g0[i]); for(j=0;jopt_n;j++) { printf("%f ",qp->opt_g[i*qp->opt_n+j]); } printf(": a%ld=%.10f < %f",i,qp->opt_xinit[i],qp->opt_up[i]); printf(": y=%f\n",qp->opt_ce[i]); } for(j=0;jopt_m;j++) { printf("EQ-%ld: %f*a0",j,qp->opt_ce[j]); for(i=1;iopt_n;i++) { printf(" + %f*a%ld",qp->opt_ce[i],i); } printf(" = %f\n\n",-qp->opt_ce0[0]); } } obj_before=0; /* calculate objective before optimization */ for(i=0;iopt_n;i++) { obj_before+=(qp->opt_g0[i]*qp->opt_xinit[i]); obj_before+=(0.5*qp->opt_xinit[i]*qp->opt_xinit[i]*qp->opt_g[i*qp->opt_n+i]); for(j=0;jopt_xinit[j]*qp->opt_xinit[i]*qp->opt_g[j*qp->opt_n+i]); } } result=STILL_RUNNING; qp->opt_ce0[0]*=(-1.0); /* Run pr_loqo. If a run fails, try again with parameters which lead */ /* to a slower, but more robust setting. */ for(margin=init_margin,iter=init_iter; (margin<=0.9999999) && (result!=OPTIMAL_SOLUTION);) { sigdig=-log10(opt_precision); result=pr_loqo((int)qp->opt_n,(int)qp->opt_m, (double *)qp->opt_g0,(double *)qp->opt_g, (double *)qp->opt_ce,(double *)qp->opt_ce0, (double *)qp->opt_low,(double *)qp->opt_up, (double *)primal,(double *)dual, (int)(verbosity-2), (double)sigdig,(int)iter, (double)margin,(double)(qp->opt_up[0])/4.0,(int)0); if(isnan(dual[0])) { /* check for choldc problem */ if(verbosity>=2) { printf("NOTICE: Restarting PR_LOQO with more conservative parameters.\n"); } if(init_margin<0.80) { /* become more conservative in general */ init_margin=(4.0*margin+1.0)/5.0; } margin=(margin+1.0)/2.0; (opt_precision)*=10.0; /* reduce precision */ if(verbosity>=2) { printf("NOTICE: Reducing precision of PR_LOQO.\n"); } } else if(result!=OPTIMAL_SOLUTION) { iter+=2000; init_iter+=10; (opt_precision)*=10.0; /* reduce precision */ if(verbosity>=2) { printf("NOTICE: Reducing precision of PR_LOQO due to (%ld).\n",result); } } } if(qp->opt_m) /* Thanks to Alex Smola for this hint */ model_b=dual[0]; else model_b=0; /* Check the precision of the alphas. If results of current optimization */ /* violate KT-Conditions, relax the epsilon on the bounds on alphas. */ epsilon_loqo=1E-10; for(i=0;iopt_n;i++) { dist=-model_b*qp->opt_ce[i]; dist+=(qp->opt_g0[i]+1.0); for(j=0;jopt_g[j*qp->opt_n+i]); } for(j=i;jopt_n;j++) { dist+=(primal[j]*qp->opt_g[i*qp->opt_n+j]); } /* printf("LOQO: a[%d]=%f, dist=%f, b=%f\n",i,primal[i],dist,dual[0]); */ if((primal[i]<(qp->opt_up[i]-epsilon_loqo)) && (dist < (1.0-(*epsilon_crit)))) { epsilon_loqo=(qp->opt_up[i]-primal[i])*2.0; } else if((primal[i]>(0+epsilon_loqo)) && (dist > (1.0+(*epsilon_crit)))) { epsilon_loqo=primal[i]*2.0; } } for(i=0;iopt_n;i++) { /* clip alphas to bounds */ if(primal[i]<=(0+epsilon_loqo)) { primal[i]=0; } else if(primal[i]>=(qp->opt_up[i]-epsilon_loqo)) { primal[i]=qp->opt_up[i]; } } obj_after=0; /* calculate objective after optimization */ for(i=0;iopt_n;i++) { obj_after+=(qp->opt_g0[i]*primal[i]); obj_after+=(0.5*primal[i]*primal[i]*qp->opt_g[i*qp->opt_n+i]); for(j=0;jopt_g[j*qp->opt_n+i]); } } /* if optimizer returned NAN values, reset and retry with smaller */ /* working set. */ if(isnan(obj_after) || isnan(model_b)) { for(i=0;iopt_n;i++) { primal[i]=qp->opt_xinit[i]; } model_b=0; if(learn_parm->svm_maxqpsize>2) { learn_parm->svm_maxqpsize--; /* decrease size of qp-subproblems */ } } if(obj_after >= obj_before) { /* check whether there was progress */ (opt_precision)/=100.0; precision_violations++; if(verbosity>=2) { printf("NOTICE: Increasing Precision of PR_LOQO.\n"); } } if(precision_violations > 500) { (*epsilon_crit)*=10.0; precision_violations=0; if(verbosity>=1) { printf("\nWARNING: Relaxing epsilon on KT-Conditions.\n"); } } (*threshold)=model_b; if(result!=OPTIMAL_SOLUTION) { printf("\nERROR: PR_LOQO did not converge. \n"); return(qp->opt_xinit); } else { return(primal); } } svm-struct-matlab-1.0/svm_struct/000077500000000000000000000000001160627511300171525ustar00rootroot00000000000000svm-struct-matlab-1.0/svm_struct/Makefile000066400000000000000000000026471160627511300206230ustar00rootroot00000000000000# Makefile for SVM-struct, 03.10.06 #Use the following to compile under unix or cygwin CC = gcc LD = gcc #Call 'make' using the following line to make CYGWIN produce stand-alone Windows executables # make 'SFLAGS=-mno-cygwin' CFLAGS = $(SFLAGS) -O3 -fomit-frame-pointer -ffast-math -Wall LDFLAGS = $(SFLAGS) -O3 -lm -Wall #CFLAGS = $(SFLAGS) -pg -Wall #LDFLAGS = $(SFLAGS) -pg -Wall all: svm_struct_noexe svm_struct_noexe: svm_struct_learn.o svm_struct_classify.o svm_struct_common.o svm_struct_main.o .PHONY: clean clean: rm -f *.o *.tcov *.d core gmon.out *.stackdump #----------------------# #---- STRUCT SVM ----# #----------------------# svm_struct_common.o: svm_struct_common.c svm_struct_common.h ../svm_struct_api_types.h $(CC) -c $(CFLAGS) svm_struct_common.c -o svm_struct_common.o svm_struct_learn.o: svm_struct_learn.c ../svm_light/svm_learn.h svm_struct_common.h ../svm_struct_api.h ../svm_struct_api_types.h $(CC) -c $(CFLAGS) svm_struct_learn.c -o svm_struct_learn.o svm_struct_main.o: svm_struct_main.c ../svm_light/svm_common.h ../svm_light/svm_learn.h svm_struct_learn.h svm_struct_common.h ../svm_struct_api.h ../svm_struct_api_types.h $(CC) -c $(CFLAGS) svm_struct_main.c -o svm_struct_main.o svm_struct_classify.o: svm_struct_classify.c svm_struct_common.h ../svm_struct_api_types.h ../svm_struct_api.h ../svm_light/svm_common.h $(CC) -c $(CFLAGS) svm_struct_classify.c -o svm_struct_classify.o svm-struct-matlab-1.0/svm_struct/svm_struct_classify.c000066400000000000000000000136321160627511300234310ustar00rootroot00000000000000/***********************************************************************/ /* */ /* svm_struct_classify.c */ /* */ /* Classification module of SVM-struct. */ /* */ /* Author: Thorsten Joachims */ /* Date: 03.07.04 */ /* */ /* Copyright (c) 2004 Thorsten Joachims - All rights reserved */ /* */ /* This software is available for non-commercial use only. It must */ /* not be modified and distributed without prior permission of the */ /* author. The author is not responsible for implications from the */ /* use of this software. */ /* */ /************************************************************************/ #include #ifdef __cplusplus extern "C" { #endif #include "../svm_light/svm_common.h" #ifdef __cplusplus } #endif #include "../svm_struct_api.h" #include "svm_struct_common.h" char testfile[200]; char modelfile[200]; char predictionsfile[200]; void read_input_parameters(int, char **, char *, char *, char *, STRUCT_LEARN_PARM *, long*, long *); void print_help(void); int main (int argc, char* argv[]) { long correct=0,incorrect=0,no_accuracy=0; long i; double t1,runtime=0; double avgloss=0,l; FILE *predfl; STRUCTMODEL model; STRUCT_LEARN_PARM sparm; STRUCT_TEST_STATS teststats; SAMPLE testsample; LABEL y; svm_struct_classify_api_init(argc,argv); read_input_parameters(argc,argv,testfile,modelfile,predictionsfile,&sparm, &verbosity,&struct_verbosity); if(struct_verbosity>=1) { printf("Reading model..."); fflush(stdout); } model=read_struct_model(modelfile,&sparm); if(struct_verbosity>=1) { fprintf(stdout, "done.\n"); } if(model.svm_model->kernel_parm.kernel_type == LINEAR) { /* linear kernel */ /* compute weight vector */ add_weight_vector_to_linear_model(model.svm_model); model.w=model.svm_model->lin_weights; } if(struct_verbosity>=1) { printf("Reading test examples..."); fflush(stdout); } testsample=read_struct_examples(testfile,&sparm); if(struct_verbosity>=1) { printf("done.\n"); fflush(stdout); } if(struct_verbosity>=1) { printf("Classifying test examples..."); fflush(stdout); } if ((predfl = fopen (predictionsfile, "w")) == NULL) { perror (predictionsfile); exit (1); } for(i=0;i=2) { if((i+1) % 100 == 0) { printf("%ld..",i+1); fflush(stdout); } } free_label(y); } avgloss/=testsample.n; fclose(predfl); if(struct_verbosity>=1) { printf("done\n"); printf("Runtime (without IO) in cpu-seconds: %.2f\n", (float)(runtime/100.0)); } if((!no_accuracy) && (struct_verbosity>=1)) { printf("Average loss on test set: %.4f\n",(float)avgloss); printf("Zero/one-error on test set: %.2f%% (%ld correct, %ld incorrect, %d total)\n",(float)100.0*incorrect/testsample.n,correct,incorrect,testsample.n); } print_struct_testing_stats(testsample,&model,&sparm,&teststats); free_struct_sample(testsample); free_struct_model(model); svm_struct_classify_api_exit(); return(0); } void read_input_parameters(int argc,char *argv[],char *testfile, char *modelfile,char *predictionsfile, STRUCT_LEARN_PARM *struct_parm, long *verbosity,long *struct_verbosity) { long i; /* set default */ strcpy (modelfile, "svm_model"); strcpy (predictionsfile, "svm_predictions"); (*verbosity)=0;/*verbosity for svm_light*/ (*struct_verbosity)=1; /*verbosity for struct learning portion*/ struct_parm->custom_argc=0; for(i=1;(icustom_argv[struct_parm->custom_argc++],argv[i]);i++; strcpy(struct_parm->custom_argv[struct_parm->custom_argc++],argv[i]);break; case 'v': i++; (*struct_verbosity)=atol(argv[i]); break; case 'y': i++; (*verbosity)=atol(argv[i]); break; default: printf("\nUnrecognized option %s!\n\n",argv[i]); print_help(); exit(0); } } if((i+1)>=argc) { printf("\nNot enough input parameters!\n\n"); print_help(); exit(0); } strcpy (testfile, argv[i]); strcpy (modelfile, argv[i+1]); if((i+2) this help\n"); printf(" -v [0..3] -> verbosity level (default 2)\n\n"); print_struct_help_classify(); } svm-struct-matlab-1.0/svm_struct/svm_struct_common.c000066400000000000000000000040271160627511300231020ustar00rootroot00000000000000/***********************************************************************/ /* */ /* svm_struct_common.h */ /* */ /* Functions and types used by multiple components of SVM-struct. */ /* */ /* Author: Thorsten Joachims */ /* Date: 03.07.04 */ /* */ /* Copyright (c) 2004 Thorsten Joachims - All rights reserved */ /* */ /* This software is available for non-commercial use only. It must */ /* not be modified and distributed without prior permission of the */ /* author. The author is not responsible for implications from the */ /* use of this software. */ /* */ /***********************************************************************/ #include #include #include #include "svm_struct_common.h" long struct_verbosity; /* verbosity level (0-4) */ void printIntArray(int* x, int n) { int i; for(i=0;i= rhs[i] */ int m; /* m is the total number of constrains */ DOC **lhs; double *rhs; } CONSTSET; /**** print methods ****/ void printIntArray(int*,int); void printDoubleArray(double*,int); void printWordArray(WORD*); void printModel(MODEL *); void printW(double *, long, long, double); extern long struct_verbosity; /* verbosity level (0-4) */ #endif svm-struct-matlab-1.0/svm_struct/svm_struct_learn.c000066400000000000000000001247541160627511300227250ustar00rootroot00000000000000/***********************************************************************/ /* */ /* svm_struct_learn.c */ /* */ /* Basic algorithm for learning structured outputs (e.g. parses, */ /* sequences, multi-label classification) with a Support Vector */ /* Machine. */ /* */ /* Author: Thorsten Joachims */ /* Date: 26.06.06 */ /* */ /* Copyright (c) 2006 Thorsten Joachims - All rights reserved */ /* */ /* This software is available for non-commercial use only. It must */ /* not be modified and distributed without prior permission of the */ /* author. The author is not responsible for implications from the */ /* use of this software. */ /* */ /***********************************************************************/ #include "svm_struct_learn.h" #include "svm_struct_common.h" #include "../svm_struct_api.h" #include #define MAX(x,y) ((x) < (y) ? (y) : (x)) #define MIN(x,y) ((x) > (y) ? (y) : (x)) void svm_learn_struct(SAMPLE sample, STRUCT_LEARN_PARM *sparm, LEARN_PARM *lparm, KERNEL_PARM *kparm, STRUCTMODEL *sm, int alg_type) { int i,j; int numIt=0; long argmax_count=0; long newconstraints=0, totconstraints=0, activenum=0; int opti_round, *opti, fullround, use_shrinking; long old_totconstraints=0; double epsilon,svmCnorm; long tolerance,new_precision=1,dont_stop=0; double lossval,factor,dist; double margin=0; double slack, *slacks, slacksum, ceps; double dualitygap,modellength,alphasum; long sizePsi; double *alpha=NULL; long *alphahist=NULL,optcount=0,lastoptcount=0; CONSTSET cset; SVECTOR *diff=NULL; SVECTOR *fy, *fybar, *f, **fycache=NULL; SVECTOR *slackvec; WORD slackv[2]; MODEL *svmModel=NULL; KERNEL_CACHE *kcache=NULL; LABEL ybar; DOC *doc; long n=sample.n; EXAMPLE *ex=sample.examples; double rt_total=0, rt_opt=0, rt_init=0, rt_psi=0, rt_viol=0; double rt1,rt2; rt1=get_runtime(); init_struct_model(sample,sm,sparm,lparm,kparm); sizePsi=sm->sizePsi+1; /* sm must contain size of psi on return */ /* initialize shrinking-style example selection heuristic */ if(alg_type == NSLACK_SHRINK_ALG) use_shrinking=1; else use_shrinking=0; opti=(int*)my_malloc(n*sizeof(int)); for(i=0;iC/n; if(sparm->slack_norm == 1) { lparm->svm_c=svmCnorm; /* set upper bound C */ lparm->sharedslack=1; } else if(sparm->slack_norm == 2) { lparm->svm_c=999999999999999.0; /* upper bound C must never be reached */ lparm->sharedslack=0; if(kparm->kernel_type != LINEAR) { printf("ERROR: Kernels are not implemented for L2 slack norm!"); fflush(stdout); exit(0); } } else { printf("ERROR: Slack norm must be L1 or L2!"); fflush(stdout); exit(0); } epsilon=100.0; /* start with low precision and increase later */ tolerance=MIN(n/3,MAX(n/100,5));/* increase precision, whenever less than that number of constraints is not fulfilled */ lparm->biased_hyperplane=0; /* set threshold to zero */ cset=init_struct_constraints(sample, sm, sparm); if(cset.m > 0) { alpha=(double *)realloc(alpha,sizeof(double)*cset.m); alphahist=(long *)realloc(alphahist,sizeof(long)*cset.m); for(i=0; iepsilon_crit=epsilon; if(kparm->kernel_type != LINEAR) kcache=kernel_cache_init(MAX(cset.m,1),lparm->kernel_cache_size); svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi+n, lparm,kparm,kcache,svmModel,alpha); if(kcache) kernel_cache_cleanup(kcache); add_weight_vector_to_linear_model(svmModel); sm->svm_model=svmModel; sm->w=svmModel->lin_weights; /* short cut to weight vector */ /* create a cache of the feature vectors for the correct labels */ if(USE_FYCACHE) { fycache=(SVECTOR **)my_malloc(n*sizeof(SVECTOR *)); for(i=0;ikernel_type == LINEAR) { diff=add_list_ss(fy); /* store difference vector directly */ free_svector(fy); fy=diff; } fycache[i]=fy; } } rt_init+=MAX(get_runtime()-rt1,0); rt_total+=MAX(get_runtime()-rt1,0); /*****************/ /*** main loop ***/ /*****************/ do { /* iteratively increase precision */ epsilon=MAX(epsilon*0.49999999999,sparm->epsilon); new_precision=1; if(epsilon == sparm->epsilon) /* for final precision, find all SV */ tolerance=0; lparm->epsilon_crit=epsilon/2; /* svm precision must be higher than eps */ if(struct_verbosity>=1) printf("Setting current working precision to %g.\n",epsilon); do { /* iteration until (approx) all SV are found for current precision and tolerance */ opti_round++; activenum=n; dont_stop=0; old_totconstraints=totconstraints; do { /* with shrinking turned on, go through examples that keep producing new constraints */ if(struct_verbosity>=1) { printf("Iter %i (%ld active): ",++numIt,activenum); fflush(stdout); } ceps=0; fullround=(activenum == n); for(i=0; iloss_type == SLACK_RESCALING) ybar=find_most_violated_constraint_slackrescaling(ex[i].x, ex[i].y,sm, sparm); else ybar=find_most_violated_constraint_marginrescaling(ex[i].x, ex[i].y,sm, sparm); rt_viol+=MAX(get_runtime()-rt2,0); if(empty_label(ybar)) { if(opti[i] != opti_round) { activenum--; opti[i]=opti_round; } if(struct_verbosity>=2) printf("no-incorrect-found(%i) ",i); continue; } /**** get psi(y)-psi(ybar) ****/ rt2=get_runtime(); if(fycache) fy=copy_svector(fycache[i]); else fy=psi(ex[i].x,ex[i].y,sm,sparm); fybar=psi(ex[i].x,ybar,sm,sparm); rt_psi+=MAX(get_runtime()-rt2,0); /**** scale feature vector and margin by loss ****/ lossval=loss(ex[i].y,ybar,sparm); if(sparm->slack_norm == 2) lossval=sqrt(lossval); if(sparm->loss_type == SLACK_RESCALING) factor=lossval; else /* do not rescale vector for */ factor=1.0; /* margin rescaling loss type */ for(f=fy;f;f=f->next) f->factor*=factor; for(f=fybar;f;f=f->next) f->factor*=-factor; margin=lossval; /**** create constraint for current ybar ****/ append_svector_list(fy,fybar);/* append the two vector lists */ doc=create_example(cset.m,0,i+1,1,fy); /**** compute slack for this example ****/ slack=0; for(j=0;jslackid == i+1) { if(sparm->slack_norm == 2) /* works only for linear kernel */ slack=MAX(slack,cset.rhs[j] -(classify_example(svmModel,cset.lhs[j]) -sm->w[sizePsi+i]/(sqrt(2*svmCnorm)))); else slack=MAX(slack, cset.rhs[j]-classify_example(svmModel,cset.lhs[j])); } /**** if `error' add constraint and recompute ****/ dist=classify_example(svmModel,doc); ceps=MAX(ceps,margin-dist-slack); if(slack > (margin-dist+0.0001)) { printf("\nWARNING: Slack of most violated constraint is smaller than slack of working\n"); printf(" set! There is probably a bug in 'find_most_violated_constraint_*'.\n"); printf("Ex %d: slack=%f, newslack=%f\n",i,slack,margin-dist); /* exit(1); */ } if((dist+slack)<(margin-epsilon)) { if(struct_verbosity>=2) {printf("(%i,eps=%.2f) ",i,margin-dist-slack); fflush(stdout);} if(struct_verbosity==1) {printf("."); fflush(stdout);} /**** resize constraint matrix and add new constraint ****/ cset.m++; cset.lhs=(DOC **)realloc(cset.lhs,sizeof(DOC *)*cset.m); if(kparm->kernel_type == LINEAR) { diff=add_list_ss(fy); /* store difference vector directly */ if(sparm->slack_norm == 1) cset.lhs[cset.m-1]=create_example(cset.m-1,0,i+1,1, copy_svector(diff)); else if(sparm->slack_norm == 2) { /**** add squared slack variable to feature vector ****/ slackv[0].wnum=sizePsi+i; slackv[0].weight=1/(sqrt(2*svmCnorm)); slackv[1].wnum=0; /*terminator*/ slackvec=create_svector(slackv,NULL,1.0); cset.lhs[cset.m-1]=create_example(cset.m-1,0,i+1,1, add_ss(diff,slackvec)); free_svector(slackvec); } free_svector(diff); } else { /* kernel is used */ if(sparm->slack_norm == 1) cset.lhs[cset.m-1]=create_example(cset.m-1,0,i+1,1, copy_svector(fy)); else if(sparm->slack_norm == 2) exit(1); } cset.rhs=(double *)realloc(cset.rhs,sizeof(double)*cset.m); cset.rhs[cset.m-1]=margin; alpha=(double *)realloc(alpha,sizeof(double)*cset.m); alpha[cset.m-1]=0; alphahist=(long *)realloc(alphahist,sizeof(long)*cset.m); alphahist[cset.m-1]=optcount; newconstraints++; totconstraints++; } else { printf("+"); fflush(stdout); if(opti[i] != opti_round) { activenum--; opti[i]=opti_round; } } free_example(doc,0); free_svector(fy); /* this also free's fybar */ free_label(ybar); } /**** get new QP solution ****/ if((newconstraints >= sparm->newconstretrain) || ((newconstraints > 0) && (i == n-1)) || (new_precision && (i == n-1))) { if(struct_verbosity>=1) { printf("*");fflush(stdout); } rt2=get_runtime(); free_model(svmModel,0); svmModel=(MODEL *)my_malloc(sizeof(MODEL)); /* Always get a new kernel cache. It is not possible to use the same cache for two different training runs */ if(kparm->kernel_type != LINEAR) kcache=kernel_cache_init(MAX(cset.m,1),lparm->kernel_cache_size); /* Run the QP solver on cset. */ svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi+n, lparm,kparm,kcache,svmModel,alpha); if(kcache) kernel_cache_cleanup(kcache); /* Always add weight vector, in case part of the kernel is linear. If not, ignore the weight vector since its content is bogus. */ add_weight_vector_to_linear_model(svmModel); sm->svm_model=svmModel; sm->w=svmModel->lin_weights; /* short cut to weight vector */ optcount++; /* keep track of when each constraint was last active. constraints marked with -1 are not updated */ for(j=0;j-1) && (alpha[j] != 0)) alphahist[j]=optcount; rt_opt+=MAX(get_runtime()-rt2,0); if(new_precision && (epsilon <= sparm->epsilon)) dont_stop=1; /* make sure we take one final pass */ new_precision=0; newconstraints=0; } rt_total+=MAX(get_runtime()-rt1,0); } /* end of example loop */ rt1=get_runtime(); if(struct_verbosity>=1) printf("(NumConst=%d, SV=%ld, CEps=%.4f, QPEps=%.4f)\n",cset.m, svmModel->sv_num-1,ceps,svmModel->maxdiff); /* Check if some of the linear constraints have not been active in a while. Those constraints are then removed to avoid bloating the working set beyond necessity. */ if(struct_verbosity>=2) printf("Reducing working set...");fflush(stdout); remove_inactive_constraints(&cset,alpha,optcount,alphahist, MAX(50,optcount-lastoptcount)); lastoptcount=optcount; if(struct_verbosity>=2) printf("done. (NumConst=%d)\n",cset.m); rt_total+=MAX(get_runtime()-rt1,0); } while(use_shrinking && (activenum > 0)); /* when using shrinking, repeat until all examples produced no constraint at least once */ } while(((totconstraints - old_totconstraints) > tolerance) || dont_stop); } while((epsilon > sparm->epsilon) || finalize_iteration(ceps,0,sample,sm,cset,alpha,sparm)); if(struct_verbosity>=1) { /**** compute sum of slacks ****/ /**** WARNING: If positivity constraints are used, then the maximum slack id is larger than what is allocated below ****/ slacks=(double *)my_malloc(sizeof(double)*(n+1)); for(i=0; i<=n; i++) { slacks[i]=0; } if(sparm->slack_norm == 1) { for(j=0;jslackid]=MAX(slacks[cset.lhs[j]->slackid], cset.rhs[j]-classify_example(svmModel,cset.lhs[j])); } else if(sparm->slack_norm == 2) { for(j=0;jslackid]=MAX(slacks[cset.lhs[j]->slackid], cset.rhs[j] -(classify_example(svmModel,cset.lhs[j]) -sm->w[sizePsi+cset.lhs[j]->slackid-1]/(sqrt(2*svmCnorm)))); } slacksum=0; for(i=1; i<=n; i++) slacksum+=slacks[i]; free(slacks); alphasum=0; for(i=0; imaxdiff,epsilon)); printf("Upper bound on duality gap: %.5f\n", dualitygap); printf("Dual objective value: dval=%.5f\n", alphasum-0.5*modellength*modellength); printf("Total number of constraints in final working set: %i (of %i)\n",(int)cset.m,(int)totconstraints); printf("Number of iterations: %d\n",numIt); printf("Number of calls to 'find_most_violated_constraint': %ld\n",argmax_count); if(sparm->slack_norm == 1) { printf("Number of SV: %ld \n",svmModel->sv_num-1); printf("Number of non-zero slack variables: %ld (out of %ld)\n", svmModel->at_upper_bound,n); printf("Norm of weight vector: |w|=%.5f\n",modellength); } else if(sparm->slack_norm == 2){ printf("Number of SV: %ld (including %ld at upper bound)\n", svmModel->sv_num-1,svmModel->at_upper_bound); printf("Norm of weight vector (including L2-loss): |w|=%.5f\n", modellength); } printf("Norm. sum of slack variables (on working set): sum(xi_i)/n=%.5f\n",slacksum/n); printf("Norm of longest difference vector: ||Psi(x,y)-Psi(x,ybar)||=%.5f\n", length_of_longest_document_vector(cset.lhs,cset.m,kparm)); printf("Runtime in cpu-seconds: %.2f (%.2f%% for QP, %.2f%% for Argmax, %.2f%% for Psi, %.2f%% for init)\n", rt_total/100.0, (100.0*rt_opt)/rt_total, (100.0*rt_viol)/rt_total, (100.0*rt_psi)/rt_total, (100.0*rt_init)/rt_total); } if(struct_verbosity>=4) printW(sm->w,sizePsi,n,lparm->svm_c); if(svmModel) { sm->svm_model=copy_model(svmModel); sm->w=sm->svm_model->lin_weights; /* short cut to weight vector */ } print_struct_learning_stats(sample,sm,cset,alpha,sparm); if(fycache) { for(i=0;ibatch_size<100) batch_size=sparm->batch_size*n/100.0; init_struct_model(sample,sm,sparm,lparm,kparm); sizePsi=sm->sizePsi+1; /* sm must contain size of psi on return */ if(sparm->slack_norm == 1) { lparm->svm_c=sparm->C; /* set upper bound C */ lparm->sharedslack=1; } else if(sparm->slack_norm == 2) { printf("ERROR: The joint algorithm does not apply to L2 slack norm!"); fflush(stdout); exit(0); } else { printf("ERROR: Slack norm must be L1 or L2!"); fflush(stdout); exit(0); } lparm->biased_hyperplane=0; /* set threshold to zero */ epsilon=100.0; /* start with low precision and increase later */ epsilon_cached=epsilon; /* epsilon to use for iterations using constraints constructed from the constraint cache */ cset=init_struct_constraints(sample, sm, sparm); if(cset.m > 0) { alpha=(double *)realloc(alpha,sizeof(double)*cset.m); alphahist=(long *)realloc(alphahist,sizeof(long)*cset.m); for(i=0; igram_matrix=NULL; if((alg_type == ONESLACK_DUAL_ALG) || (alg_type == ONESLACK_DUAL_CACHE_ALG)) kparm->gram_matrix=init_kernel_matrix(&cset,kparm); /* set initial model and slack variables */ svmModel=(MODEL *)my_malloc(sizeof(MODEL)); lparm->epsilon_crit=epsilon; svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi, lparm,kparm,NULL,svmModel,alpha); add_weight_vector_to_linear_model(svmModel); sm->svm_model=svmModel; sm->w=svmModel->lin_weights; /* short cut to weight vector */ /* create a cache of the feature vectors for the correct labels */ fycache=(SVECTOR **)my_malloc(n*sizeof(SVECTOR *)); for(i=0;ikernel_type == LINEAR) { /* store difference vector directly */ diff=add_list_sort_ss_r(fy,COMPACT_ROUNDING_THRESH); free_svector(fy); fy=diff; } } else fy=NULL; fycache[i]=fy; } /* initialize the constraint cache */ if(alg_type == ONESLACK_DUAL_CACHE_ALG) { ccache=create_constraint_cache(sample,sparm,sm); /* NOTE: */ for(i=0;ikernel_type == LINEAR) lhs_n=create_nvector(sm->sizePsi); /* randomize order or training examples */ if(batch_size=1) { printf("Iter %i: ",++numIt); fflush(stdout); } rt1=get_runtime(); /**** compute current slack ****/ alphasum=0; for(j=0;(j alphasum/cset.m) slack=MAX(0,cset.rhs[j]-classify_example(svmModel,cset.lhs[j])); slack=MAX(0,slack); rt_total+=MAX(get_runtime()-rt1,0); /**** find a violated joint constraint ****/ lhs=NULL; rhs=0; if(alg_type == ONESLACK_DUAL_CACHE_ALG) { rt1=get_runtime(); /* Compute violation of constraints in cache for current w */ if(struct_verbosity>=2) rt2=get_runtime(); update_constraint_cache_for_model(ccache, svmModel); if(struct_verbosity>=2) rt_cacheupdate+=MAX(get_runtime()-rt2,0); /* Is there is a sufficiently violated constraint in cache? */ viol=compute_violation_of_constraint_in_cache(ccache,epsilon_est/2); if(viol-slack > MAX(epsilon_est/10,sparm->epsilon)) { /* There is a sufficiently violated constraint in cache, so use this constraint in this iteration. */ if(struct_verbosity>=2) rt2=get_runtime(); viol=find_most_violated_joint_constraint_in_cache(ccache, epsilon_est/2,lhs_n,&lhs,&rhs); if(struct_verbosity>=2) rt_cacheconst+=MAX(get_runtime()-rt2,0); cached_constraint=1; } else { /* There is no sufficiently violated constraint in cache, so update cache by computing most violated constraint explicitly for batch_size examples. */ viol_est=0; progress=0; viol=compute_violation_of_constraint_in_cache(ccache,0); for(j=0;(jepsilon));j++) { if(struct_verbosity>=1) print_percent_progress(&progress,n,10,"."); uptr=uptr % n; if(randmapping) i=randmapping[uptr]; else i=uptr; /* find most violating fydelta=fy-fybar and rhs for example i */ find_most_violated_constraint(&fydelta,&rhs_i,&ex[i], fycache[i],n,sm,sparm, &rt_viol,&rt_psi,&argmax_count); /* add current fy-fybar and loss to cache */ if(struct_verbosity>=2) rt2=get_runtime(); viol+=add_constraint_to_constraint_cache(ccache,sm->svm_model, i,fydelta,rhs_i,0.0001*sparm->epsilon/n, sparm->ccache_size,&rt_cachesum); if(struct_verbosity>=2) rt_cacheadd+=MAX(get_runtime()-rt2,0); viol_est+=ccache->constlist[i]->viol; uptr++; } cached_constraint=(j=2) rt2=get_runtime(); if(cached_constraint) viol=find_most_violated_joint_constraint_in_cache(ccache, epsilon_est/2,lhs_n,&lhs,&rhs); else viol=find_most_violated_joint_constraint_in_cache(ccache,0,lhs_n, &lhs,&rhs); if(struct_verbosity>=2) rt_cacheconst+=MAX(get_runtime()-rt2,0); viol_est*=((double)n/j); epsilon_est=(1-(double)j/n)*epsilon_est+(double)j/n*(viol_est-slack); if((struct_verbosity >= 1) && (j!=n)) printf("(upd=%5.1f%%,eps^=%.4f,eps*=%.4f)", 100.0*j/n,viol_est-slack,epsilon_est); } lhsXw=rhs-viol; rt_total+=MAX(get_runtime()-rt1,0); } else { /* do not use constraint from cache */ rt1=get_runtime(); cached_constraint=0; if(kparm->kernel_type == LINEAR) clear_nvector(lhs_n,sm->sizePsi); progress=0; rt_total+=MAX(get_runtime()-rt1,0); for(i=0; i=1) print_percent_progress(&progress,n,10,"."); /* compute most violating fydelta=fy-fybar and rhs for example i */ find_most_violated_constraint(&fydelta,&rhs_i,&ex[i],fycache[i],n, sm,sparm,&rt_viol,&rt_psi,&argmax_count); /* add current fy-fybar to lhs of constraint */ if(kparm->kernel_type == LINEAR) { add_list_n_ns(lhs_n,fydelta,1.0); /* add fy-fybar to sum */ free_svector(fydelta); } else { append_svector_list(fydelta,lhs); /* add fy-fybar to vector list */ lhs=fydelta; } rhs+=rhs_i; /* add loss to rhs */ rt_total+=MAX(get_runtime()-rt1,0); } /* end of example loop */ rt1=get_runtime(); /* create sparse vector from dense sum */ if(kparm->kernel_type == LINEAR) lhs=create_svector_n_r(lhs_n,sm->sizePsi,NULL,1.0, COMPACT_ROUNDING_THRESH); doc=create_example(cset.m,0,1,1,lhs); lhsXw=classify_example(svmModel,doc); free_example(doc,0); viol=rhs-lhsXw; rt_total+=MAX(get_runtime()-rt1,0); } /* end of finding most violated joint constraint */ rt1=get_runtime(); /**** if `error', then add constraint and recompute QP ****/ if(slack > (rhs-lhsXw+0.000001)) { printf("\nWARNING: Slack of most violated constraint is smaller than slack of working\n"); printf(" set! There is probably a bug in 'find_most_violated_constraint_*'.\n"); printf("slack=%f, newslack=%f\n",slack,rhs-lhsXw); /* exit(1); */ } ceps=MAX(0,rhs-lhsXw-slack); if((ceps > sparm->epsilon) || cached_constraint) { /**** resize constraint matrix and add new constraint ****/ cset.lhs=(DOC **)realloc(cset.lhs,sizeof(DOC *)*(cset.m+1)); cset.lhs[cset.m]=create_example(cset.m,0,1,1,lhs); cset.rhs=(double *)realloc(cset.rhs,sizeof(double)*(cset.m+1)); cset.rhs[cset.m]=rhs; alpha=(double *)realloc(alpha,sizeof(double)*(cset.m+1)); alpha[cset.m]=0; alphahist=(long *)realloc(alphahist,sizeof(long)*(cset.m+1)); alphahist[cset.m]=optcount; cset.m++; totconstraints++; if((alg_type == ONESLACK_DUAL_ALG) || (alg_type == ONESLACK_DUAL_CACHE_ALG)) { if(struct_verbosity>=2) rt2=get_runtime(); kparm->gram_matrix=update_kernel_matrix(kparm->gram_matrix,cset.m-1, &cset,kparm); if(struct_verbosity>=2) rt_kernel+=MAX(get_runtime()-rt2,0); } /**** get new QP solution ****/ if(struct_verbosity>=1) { printf("*");fflush(stdout); } if(struct_verbosity>=2) rt2=get_runtime(); /* set svm precision so that higher than eps of most violated constr */ if(cached_constraint) { epsilon_cached=MIN(epsilon_cached,ceps); lparm->epsilon_crit=epsilon_cached/2; } else { epsilon=MIN(epsilon,ceps); /* best eps so far */ lparm->epsilon_crit=epsilon/2; epsilon_cached=epsilon; } free_model(svmModel,0); svmModel=(MODEL *)my_malloc(sizeof(MODEL)); /* Run the QP solver on cset. */ kernel_type_org=kparm->kernel_type; if((alg_type == ONESLACK_DUAL_ALG) || (alg_type == ONESLACK_DUAL_CACHE_ALG)) kparm->kernel_type=GRAM; /* use kernel stored in kparm */ svm_learn_optimization(cset.lhs,cset.rhs,cset.m,sizePsi, lparm,kparm,NULL,svmModel,alpha); kparm->kernel_type=kernel_type_org; svmModel->kernel_parm.kernel_type=kernel_type_org; /* Always add weight vector, in case part of the kernel is linear. If not, ignore the weight vector since its content is bogus. */ add_weight_vector_to_linear_model(svmModel); sm->svm_model=svmModel; sm->w=svmModel->lin_weights; /* short cut to weight vector */ optcount++; /* keep track of when each constraint was last active. constraints marked with -1 are not updated */ for(j=0;j-1) && (alpha[j] != 0)) alphahist[j]=optcount; if(struct_verbosity>=2) rt_opt+=MAX(get_runtime()-rt2,0); /* Check if some of the linear constraints have not been active in a while. Those constraints are then removed to avoid bloating the working set beyond necessity. */ if(struct_verbosity>=3) printf("Reducing working set...");fflush(stdout); remove_inactive_constraints(&cset,alpha,optcount,alphahist,50); if(struct_verbosity>=3) printf("done. "); } else { free_svector(lhs); } if(struct_verbosity>=1) printf("(NumConst=%d, SV=%ld, CEps=%.4f, QPEps=%.4f)\n",cset.m, svmModel->sv_num-1,ceps,svmModel->maxdiff); rt_total+=MAX(get_runtime()-rt1,0); } while(cached_constraint || (ceps > sparm->epsilon) || finalize_iteration(ceps,cached_constraint,sample,sm,cset,alpha,sparm) ); if(struct_verbosity>=1) { printf("Final epsilon on KKT-Conditions: %.5f\n", MAX(svmModel->maxdiff,ceps)); slack=0; for(j=0;jkernel_type == LINEAR) modellength=model_length_n(svmModel); else modellength=model_length_s(svmModel); dualitygap=(0.5*modellength*modellength+sparm->C*viol) -(alphasum-0.5*modellength*modellength); printf("Upper bound on duality gap: %.5f\n", dualitygap); printf("Dual objective value: dval=%.5f\n", alphasum-0.5*modellength*modellength); printf("Primal objective value: pval=%.5f\n", 0.5*modellength*modellength+sparm->C*viol); printf("Total number of constraints in final working set: %i (of %i)\n",(int)cset.m,(int)totconstraints); printf("Number of iterations: %d\n",numIt); printf("Number of calls to 'find_most_violated_constraint': %ld\n",argmax_count); printf("Number of SV: %ld \n",svmModel->sv_num-1); printf("Norm of weight vector: |w|=%.5f\n",modellength); printf("Value of slack variable (on working set): xi=%.5f\n",slack); printf("Value of slack variable (global): xi=%.5f\n",viol); printf("Norm of longest difference vector: ||Psi(x,y)-Psi(x,ybar)||=%.5f\n", length_of_longest_document_vector(cset.lhs,cset.m,kparm)); if(struct_verbosity>=2) printf("Runtime in cpu-seconds: %.2f (%.2f%% for QP, %.2f%% for kernel, %.2f%% for Argmax, %.2f%% for Psi, %.2f%% for init, %.2f%% for cache update, %.2f%% for cache const, %.2f%% for cache add (incl. %.2f%% for sum))\n", rt_total/100.0, (100.0*rt_opt)/rt_total, (100.0*rt_kernel)/rt_total, (100.0*rt_viol)/rt_total, (100.0*rt_psi)/rt_total, (100.0*rt_init)/rt_total,(100.0*rt_cacheupdate)/rt_total, (100.0*rt_cacheconst)/rt_total,(100.0*rt_cacheadd)/rt_total, (100.0*rt_cachesum)/rt_total); else if(struct_verbosity==1) printf("Runtime in cpu-seconds: %.2f\n",rt_total/100.0); } if(ccache) { long cnum=0; CCACHEELEM *celem; for(i=0;iconstlist[i];celem;celem=celem->next) cnum++; printf("Final number of constraints in cache: %ld\n",cnum); } if(struct_verbosity>=4) printW(sm->w,sizePsi,n,lparm->svm_c); if(svmModel) { sm->svm_model=copy_model(svmModel); sm->w=sm->svm_model->lin_weights; /* short cut to weight vector */ free_model(svmModel,0); } print_struct_learning_stats(sample,sm,cset,alpha,sparm); if(lhs_n) free_nvector(lhs_n); if(ccache) free_constraint_cache(ccache); for(i=0;igram_matrix) free_matrix(kparm->gram_matrix); } void find_most_violated_constraint(SVECTOR **fydelta, double *rhs, EXAMPLE *ex, SVECTOR *fycached, long n, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm, double *rt_viol, double *rt_psi, long *argmax_count) /* returns fydelta=fy-fybar and rhs scalar value that correspond to the most violated constraint for example ex */ { double rt2=0; LABEL ybar; SVECTOR *fybar, *fy; double factor,lossval; if(struct_verbosity>=2) rt2=get_runtime(); (*argmax_count)++; if(sparm->loss_type == SLACK_RESCALING) ybar=find_most_violated_constraint_slackrescaling(ex->x,ex->y,sm,sparm); else ybar=find_most_violated_constraint_marginrescaling(ex->x,ex->y,sm,sparm); if(struct_verbosity>=2) (*rt_viol)+=MAX(get_runtime()-rt2,0); if(empty_label(ybar)) { printf("ERROR: empty label was returned for example\n"); /* exit(1); */ /* continue; */ } /**** get psi(x,y) and psi(x,ybar) ****/ if(struct_verbosity>=2) rt2=get_runtime(); if(fycached) fy=copy_svector(fycached); else fy=psi(ex->x,ex->y,sm,sparm); fybar=psi(ex->x,ybar,sm,sparm); if(struct_verbosity>=2) (*rt_psi)+=MAX(get_runtime()-rt2,0); lossval=loss(ex->y,ybar,sparm); free_label(ybar); /**** scale feature vector and margin by loss ****/ if(sparm->loss_type == SLACK_RESCALING) factor=lossval/n; else /* do not rescale vector for */ factor=1.0/n; /* margin rescaling loss type */ mult_svector_list(fy,factor); mult_svector_list(fybar,-factor); append_svector_list(fybar,fy); /* compute fy-fybar */ (*fydelta)=fybar; (*rhs)=lossval/n; } void remove_inactive_constraints(CONSTSET *cset, double *alpha, long currentiter, long *alphahist, long mininactive) /* removes the constraints from cset (and alpha) for which alphahist indicates that they have not been active for at least mininactive iterations */ { long i,m; m=0; for(i=0;im;i++) { if((alphahist[i]<0) || ((currentiter-alphahist[i]) < mininactive)) { /* keep constraints that are marked as -1 or which have recently been active */ cset->lhs[m]=cset->lhs[i]; cset->lhs[m]->docnum=m; cset->rhs[m]=cset->rhs[i]; alpha[m]=alpha[i]; alphahist[m]=alphahist[i]; m++; } else { free_example(cset->lhs[i],1); } } if(cset->m != m) { cset->m=m; cset->lhs=(DOC **)realloc(cset->lhs,sizeof(DOC *)*cset->m); cset->rhs=(double *)realloc(cset->rhs,sizeof(double)*cset->m); /* alpha=realloc(alpha,sizeof(double)*cset->m); */ /* alphahist=realloc(alphahist,sizeof(long)*cset->m); */ } } MATRIX *init_kernel_matrix(CONSTSET *cset, KERNEL_PARM *kparm) /* assigns a kernelid to each constraint in cset and creates the corresponding kernel matrix. */ { int i,j; double kval; MATRIX *matrix; /* assign kernel id to each new constraint */ for(i=0;im;i++) cset->lhs[i]->kernelid=i; /* allocate kernel matrix as necessary */ matrix=create_matrix(i+50,i+50); for(j=0;jm;j++) { for(i=j;im;i++) { kval=kernel(kparm,cset->lhs[j],cset->lhs[i]); matrix->element[j][i]=kval; matrix->element[i][j]=kval; } } return(matrix); } MATRIX *update_kernel_matrix(MATRIX *matrix, int newpos, CONSTSET *cset, KERNEL_PARM *kparm) /* assigns new kernelid to constraint in position newpos and fills the corresponding part of the kernel matrix */ { int i,maxkernelid=0,newid; double kval; double *used; /* find free kernelid to assign to new constraint */ for(i=0;im;i++) if(i != newpos) maxkernelid=MAX(maxkernelid,cset->lhs[i]->kernelid); used=create_nvector(maxkernelid+2); clear_nvector(used,maxkernelid+2); for(i=0;im;i++) if(i != newpos) used[cset->lhs[i]->kernelid]=1; for(newid=0;used[newid];newid++); free_nvector(used); cset->lhs[newpos]->kernelid=newid; /* extend kernel matrix if necessary */ maxkernelid=MAX(maxkernelid,newid); if((!matrix) || (maxkernelid>=matrix->m)) matrix=realloc_matrix(matrix,maxkernelid+50,maxkernelid+50); for(i=0;im;i++) { kval=kernel(kparm,cset->lhs[newpos],cset->lhs[i]); matrix->element[newid][cset->lhs[i]->kernelid]=kval; matrix->element[cset->lhs[i]->kernelid][newid]=kval; } return(matrix); } CCACHE *create_constraint_cache(SAMPLE sample, STRUCT_LEARN_PARM *sparm, STRUCTMODEL *sm) /* create new constraint cache for training set */ { long n=sample.n; EXAMPLE *ex=sample.examples; CCACHE *ccache; int i; ccache=(CCACHE *)my_malloc(sizeof(CCACHE)); ccache->n=n; ccache->sm=sm; ccache->constlist=(CCACHEELEM **)my_malloc(sizeof(CCACHEELEM *)*n); ccache->avg_viol_gain=(double *)my_malloc(sizeof(double)*n); ccache->changed=(int *)my_malloc(sizeof(int)*n); for(i=0;iconstlist[i]=(CCACHEELEM *)my_malloc(sizeof(CCACHEELEM)); ccache->constlist[i]->fydelta=create_svector_n(NULL,0,NULL,1); ccache->constlist[i]->rhs=loss(ex[i].y,ex[i].y,sparm)/n; ccache->constlist[i]->viol=0; ccache->constlist[i]->next=NULL; ccache->avg_viol_gain[i]=0; ccache->changed[i]=0; } return(ccache); } void free_constraint_cache(CCACHE *ccache) /* frees all memory allocated for constraint cache */ { CCACHEELEM *celem,*next; int i; for(i=0; in; i++) { celem=ccache->constlist[i]; while(celem) { free_svector(celem->fydelta); next=celem->next; free(celem); celem=next; } } free(ccache->constlist); free(ccache->avg_viol_gain); free(ccache->changed); free(ccache); } double add_constraint_to_constraint_cache(CCACHE *ccache, MODEL *svmModel, int exnum, SVECTOR *fydelta, double rhs, double gainthresh, int maxconst, double *rt_cachesum) /* add new constraint fydelta*w>rhs for example exnum to cache, if it is more violated (by gainthresh) than the currently most violated constraint in cache. if this grows the number of cached constraints for this example beyond maxconst, then the least recently used constraint is deleted. the function assumes that update_constraint_cache_for_model has been run. */ { double viol,viol_gain,viol_gain_trunc; double dist_ydelta; DOC *doc_fydelta; SVECTOR *fydelta_new; CCACHEELEM *celem; int cnum; double rt2=0; /* compute violation of new constraint */ doc_fydelta=create_example(1,0,1,1,fydelta); dist_ydelta=classify_example(svmModel,doc_fydelta); free_example(doc_fydelta,0); viol=rhs-dist_ydelta; viol_gain=viol-ccache->constlist[exnum]->viol; viol_gain_trunc=viol-MAX(ccache->constlist[exnum]->viol,0); ccache->avg_viol_gain[exnum]=viol_gain; /* check if violation of new constraint is larger than that of the best cache element */ if(viol_gain > gainthresh) { fydelta_new=fydelta; if(struct_verbosity>=2) rt2=get_runtime(); if(svmModel->kernel_parm.kernel_type == LINEAR) { if(COMPACT_CACHED_VECTORS == 1) { /* eval sum for linear */ fydelta_new=add_list_sort_ss_r(fydelta,COMPACT_ROUNDING_THRESH); free_svector(fydelta); } else if(COMPACT_CACHED_VECTORS == 2) { fydelta_new=add_list_ss_r(fydelta,COMPACT_ROUNDING_THRESH); free_svector(fydelta); } else if(COMPACT_CACHED_VECTORS == 3) { fydelta_new=add_list_ns_r(fydelta,COMPACT_ROUNDING_THRESH); free_svector(fydelta); } } if(struct_verbosity>=2) (*rt_cachesum)+=MAX(get_runtime()-rt2,0); celem=ccache->constlist[exnum]; ccache->constlist[exnum]=(CCACHEELEM *)my_malloc(sizeof(CCACHEELEM)); ccache->constlist[exnum]->next=celem; ccache->constlist[exnum]->fydelta=fydelta_new; ccache->constlist[exnum]->rhs=rhs; ccache->constlist[exnum]->viol=viol; ccache->changed[exnum]+=2; /* remove last constraint in list, if list is longer than maxconst */ cnum=2; for(celem=ccache->constlist[exnum];celem && celem->next && celem->next->next;celem=celem->next) cnum++; if(cnum>maxconst) { free_svector(celem->next->fydelta); free(celem->next); celem->next=NULL; } } else { free_svector(fydelta); } return(viol_gain_trunc); } void update_constraint_cache_for_model(CCACHE *ccache, MODEL *svmModel) /* update the violation scores according to svmModel and find the most violated constraints for each example */ { int i; long progress=0; double maxviol=0; double dist_ydelta; DOC *doc_fydelta; CCACHEELEM *celem,*prev,*maxviol_celem,*maxviol_prev; doc_fydelta=create_example(1,0,1,1,NULL); for(i=0; in; i++) { /*** example loop ***/ if(struct_verbosity>=3) print_percent_progress(&progress,ccache->n,10,"+"); maxviol=0; prev=NULL; maxviol_celem=NULL; maxviol_prev=NULL; for(celem=ccache->constlist[i];celem;celem=celem->next) { doc_fydelta->fvec=celem->fydelta; dist_ydelta=classify_example(svmModel,doc_fydelta); celem->viol=celem->rhs-dist_ydelta; if((celem->viol > maxviol) || (!maxviol_celem)) { maxviol=celem->viol; maxviol_celem=celem; maxviol_prev=prev; } prev=celem; } ccache->changed[i]=0; if(maxviol_prev) { /* move max violated constraint to the top of list */ maxviol_prev->next=maxviol_celem->next; maxviol_celem->next=ccache->constlist[i]; ccache->constlist[i]=maxviol_celem; ccache->changed[i]=1; } } free_example(doc_fydelta,0); } double compute_violation_of_constraint_in_cache(CCACHE *ccache, double thresh) /* computes the violation of the most violated joint constraint in cache. assumes that update_constraint_cache_for_model has been run. */ /* NOTE: This function assumes that loss(y,y')>=0, and it is most efficient when loss(y,y)=0. */ { double sumviol=0; int i,n=ccache->n; /**** add all maximal violations ****/ for(i=0; iconstlist[i]->viol*n > thresh) sumviol+=ccache->constlist[i]->viol; } return(sumviol); } double find_most_violated_joint_constraint_in_cache(CCACHE *ccache, double thresh, double *lhs_n, SVECTOR **lhs, double *rhs) /* constructs most violated joint constraint from cache. assumes that update_constraint_cache_for_model has been run. */ /* NOTE: For kernels, this function returns only a shallow copy of the Psi vectors in lhs. So, do not use a deep free, otherwise the case becomes invalid. */ /* NOTE: This function assumes that loss(y,y')>=0, and it is most efficient when loss(y,y)=0. */ { double sumviol=0; int i,n=ccache->n; SVECTOR *fydelta; (*lhs)=NULL; (*rhs)=0; if(lhs_n) { /* linear case? */ clear_nvector(lhs_n,ccache->sm->sizePsi); } /**** add all maximally violated fydelta to joint constraint ****/ for(i=0; iconstlist[i]->viol*n > thresh)) { /* get most violating fydelta=fy-fybar for example i from cache */ fydelta=ccache->constlist[i]->fydelta; (*rhs)+=ccache->constlist[i]->rhs; sumviol+=ccache->constlist[i]->viol; if(lhs_n) { /* linear case? */ add_list_n_ns(lhs_n,fydelta,1.0); /* add fy-fybar to sum */ } else { /* add fy-fybar to vector list */ fydelta=copy_svector(fydelta); append_svector_list(fydelta,(*lhs)); (*lhs)=fydelta; } } } /* create sparse vector from dense sum */ if(lhs_n) /* linear case? */ (*lhs)=create_svector_n_r(lhs_n,ccache->sm->sizePsi,NULL,1.0, COMPACT_ROUNDING_THRESH); return(sumviol); } svm-struct-matlab-1.0/svm_struct/svm_struct_learn.h000066400000000000000000000106211160627511300227150ustar00rootroot00000000000000/***********************************************************************/ /* */ /* svm_struct_learn.h */ /* */ /* Basic algorithm for learning structured outputs (e.g. parses, */ /* sequences, multi-label classification) with a Support Vector */ /* Machine. */ /* */ /* Author: Thorsten Joachims */ /* Date: 03.07.04 */ /* */ /* Copyright (c) 2004 Thorsten Joachims - All rights reserved */ /* */ /* This software is available for non-commercial use only. It must */ /* not be modified and distributed without prior permission of the */ /* author. The author is not responsible for implications from the */ /* use of this software. */ /* */ /***********************************************************************/ #ifndef SVM_STRUCT_LEARN #define SVM_STRUCT_LEARN #ifdef __cplusplus extern "C" { #endif #include "../svm_light/svm_common.h" #include "../svm_light/svm_learn.h" #ifdef __cplusplus } #endif #include "svm_struct_common.h" #include "../svm_struct_api_types.h" #define SLACK_RESCALING 1 #define MARGIN_RESCALING 2 #define NSLACK_ALG 0 #define NSLACK_SHRINK_ALG 1 #define ONESLACK_PRIMAL_ALG 2 #define ONESLACK_DUAL_ALG 3 #define ONESLACK_DUAL_CACHE_ALG 4 typedef struct ccacheelem { SVECTOR *fydelta; /* left hand side of constraint */ double rhs; /* right hand side of constraint */ double viol; /* violation score under current model */ struct ccacheelem *next; /* next in linked list */ } CCACHEELEM; typedef struct ccache { int n; /* number of examples */ CCACHEELEM **constlist; /* array of pointers to constraint lists - one list per example. The first element of the list always points to the most violated constraint under the current model for each example. */ STRUCTMODEL *sm; /* pointer to model */ double *avg_viol_gain; /* array of average values by which violation of globally most violated constraint exceeds that of most violated constraint in cache */ int *changed; /* array of boolean indicating whether the most violated ybar change compared to last iter? */ } CCACHE; void find_most_violated_constraint(SVECTOR **fydelta, double *lossval, EXAMPLE *ex, SVECTOR *fycached, long n, STRUCTMODEL *sm,STRUCT_LEARN_PARM *sparm, double *rt_viol, double *rt_psi, long *argmax_count); CCACHE *create_constraint_cache(SAMPLE sample, STRUCT_LEARN_PARM *sparm, STRUCTMODEL *sm); void free_constraint_cache(CCACHE *ccache); double add_constraint_to_constraint_cache(CCACHE *ccache, MODEL *svmModel, int exnum, SVECTOR *fydelta, double rhs, double gainthresh, int maxconst, double *rt_cachesum); void update_constraint_cache_for_model(CCACHE *ccache, MODEL *svmModel); double compute_violation_of_constraint_in_cache(CCACHE *ccache, double thresh); double find_most_violated_joint_constraint_in_cache(CCACHE *ccache, double thresh, double *lhs_n, SVECTOR **lhs, double *rhs); void svm_learn_struct(SAMPLE sample, STRUCT_LEARN_PARM *sparm, LEARN_PARM *lparm, KERNEL_PARM *kparm, STRUCTMODEL *sm, int alg_type); void svm_learn_struct_joint(SAMPLE sample, STRUCT_LEARN_PARM *sparm, LEARN_PARM *lparm, KERNEL_PARM *kparm, STRUCTMODEL *sm, int alg_type); void svm_learn_struct_joint_custom(SAMPLE sample, STRUCT_LEARN_PARM *sparm, LEARN_PARM *lparm, KERNEL_PARM *kparm, STRUCTMODEL *sm); void remove_inactive_constraints(CONSTSET *cset, double *alpha, long i, long *alphahist, long mininactive); MATRIX *init_kernel_matrix(CONSTSET *cset, KERNEL_PARM *kparm); MATRIX *update_kernel_matrix(MATRIX *matrix, int newpos, CONSTSET *cset, KERNEL_PARM *kparm); #endif svm-struct-matlab-1.0/svm_struct/svm_struct_main.c000066400000000000000000000434321160627511300225410ustar00rootroot00000000000000/***********************************************************************/ /* */ /* svm_struct_main.c */ /* */ /* Command line interface to the alignment learning module of the */ /* Support Vector Machine. */ /* */ /* Author: Thorsten Joachims */ /* Date: 03.07.04 */ /* */ /* Copyright (c) 2004 Thorsten Joachims - All rights reserved */ /* */ /* This software is available for non-commercial use only. It must */ /* not be modified and distributed without prior permission of the */ /* author. The author is not responsible for implications from the */ /* use of this software. */ /* */ /***********************************************************************/ /* the following enables you to use svm-learn out of C++ */ #ifdef __cplusplus extern "C" { #endif #include "../svm_light/svm_common.h" #include "../svm_light/svm_learn.h" #ifdef __cplusplus } #endif # include "svm_struct_learn.h" # include "svm_struct_common.h" # include "../svm_struct_api.h" #include #include #include /* } */ char trainfile[200]; /* file with training examples */ char modelfile[200]; /* file for resulting classifier */ void read_input_parameters(int, char **, char *, char *,long *, long *, STRUCT_LEARN_PARM *, LEARN_PARM *, KERNEL_PARM *, int *); void wait_any_key(); void print_help(); int main (int argc, char* argv[]) { SAMPLE sample; /* training sample */ LEARN_PARM learn_parm; KERNEL_PARM kernel_parm; STRUCT_LEARN_PARM struct_parm; STRUCTMODEL structmodel; int alg_type; svm_struct_learn_api_init(argc,argv); read_input_parameters(argc,argv,trainfile,modelfile,&verbosity, &struct_verbosity,&struct_parm,&learn_parm, &kernel_parm,&alg_type); if(struct_verbosity>=1) { printf("Reading training examples..."); fflush(stdout); } /* read the training examples */ sample=read_struct_examples(trainfile,&struct_parm); if(struct_verbosity>=1) { printf("done\n"); fflush(stdout); } /* Do the learning and return structmodel. */ if(alg_type == 0) svm_learn_struct(sample,&struct_parm,&learn_parm,&kernel_parm,&structmodel,NSLACK_ALG); else if(alg_type == 1) svm_learn_struct(sample,&struct_parm,&learn_parm,&kernel_parm,&structmodel,NSLACK_SHRINK_ALG); else if(alg_type == 2) svm_learn_struct_joint(sample,&struct_parm,&learn_parm,&kernel_parm,&structmodel,ONESLACK_PRIMAL_ALG); else if(alg_type == 3) svm_learn_struct_joint(sample,&struct_parm,&learn_parm,&kernel_parm,&structmodel,ONESLACK_DUAL_ALG); else if(alg_type == 4) svm_learn_struct_joint(sample,&struct_parm,&learn_parm,&kernel_parm,&structmodel,ONESLACK_DUAL_CACHE_ALG); else if(alg_type == 9) svm_learn_struct_joint_custom(sample,&struct_parm,&learn_parm,&kernel_parm,&structmodel); else exit(1); /* Warning: The model contains references to the original data 'docs'. If you want to free the original data, and only keep the model, you have to make a deep copy of 'model'. */ if(struct_verbosity>=1) { printf("Writing learned model...");fflush(stdout); } write_struct_model(modelfile,&structmodel,&struct_parm); if(struct_verbosity>=1) { printf("done\n");fflush(stdout); } free_struct_sample(sample); free_struct_model(structmodel); svm_struct_learn_api_exit(); return 0; } /*---------------------------------------------------------------------------*/ void read_input_parameters(int argc,char *argv[],char *trainfile, char *modelfile, long *verbosity,long *struct_verbosity, STRUCT_LEARN_PARM *struct_parm, LEARN_PARM *learn_parm, KERNEL_PARM *kernel_parm, int *alg_type) { long i; char type[100]; /* set default */ (*alg_type)=DEFAULT_ALG_TYPE; struct_parm->C=-0.01; struct_parm->slack_norm=1; struct_parm->epsilon=DEFAULT_EPS; struct_parm->custom_argc=0; struct_parm->loss_function=DEFAULT_LOSS_FCT; struct_parm->loss_type=DEFAULT_RESCALING; struct_parm->newconstretrain=100; struct_parm->ccache_size=5; struct_parm->batch_size=100; strcpy (modelfile, "svm_struct_model"); strcpy (learn_parm->predfile, "trans_predictions"); strcpy (learn_parm->alphafile, ""); (*verbosity)=0;/*verbosity for svm_light*/ (*struct_verbosity)=1; /*verbosity for struct learning portion*/ learn_parm->biased_hyperplane=1; learn_parm->remove_inconsistent=0; learn_parm->skip_final_opt_check=0; learn_parm->svm_maxqpsize=10; learn_parm->svm_newvarsinqp=0; learn_parm->svm_iter_to_shrink=-9999; learn_parm->maxiter=100000; learn_parm->kernel_cache_size=40; learn_parm->svm_c=99999999; /* overridden by struct_parm->C */ learn_parm->eps=0.001; /* overridden by struct_parm->epsilon */ learn_parm->transduction_posratio=-1.0; learn_parm->svm_costratio=1.0; learn_parm->svm_costratio_unlab=1.0; learn_parm->svm_unlabbound=1E-5; learn_parm->epsilon_crit=0.001; learn_parm->epsilon_a=1E-10; /* changed from 1e-15 */ learn_parm->compute_loo=0; learn_parm->rho=1.0; learn_parm->xa_depth=0; kernel_parm->kernel_type=0; kernel_parm->poly_degree=3; kernel_parm->rbf_gamma=1.0; kernel_parm->coef_lin=1; kernel_parm->coef_const=1; strcpy(kernel_parm->custom,"empty"); strcpy(type,"c"); for(i=1;(ialphafile,argv[i]); break; case 'c': i++; struct_parm->C=atof(argv[i]); break; case 'p': i++; struct_parm->slack_norm=atol(argv[i]); break; case 'e': i++; struct_parm->epsilon=atof(argv[i]); break; case 'k': i++; struct_parm->newconstretrain=atol(argv[i]); break; case 'h': i++; learn_parm->svm_iter_to_shrink=atol(argv[i]); break; case '#': i++; learn_parm->maxiter=atol(argv[i]); break; case 'm': i++; learn_parm->kernel_cache_size=atol(argv[i]); break; case 'w': i++; (*alg_type)=atol(argv[i]); break; case 'o': i++; struct_parm->loss_type=atol(argv[i]); break; case 'n': i++; learn_parm->svm_newvarsinqp=atol(argv[i]); break; case 'q': i++; learn_parm->svm_maxqpsize=atol(argv[i]); break; case 'l': i++; struct_parm->loss_function=atol(argv[i]); break; case 'f': i++; struct_parm->ccache_size=atol(argv[i]); break; case 'b': i++; struct_parm->batch_size=atof(argv[i]); break; case 't': i++; kernel_parm->kernel_type=atol(argv[i]); break; case 'd': i++; kernel_parm->poly_degree=atol(argv[i]); break; case 'g': i++; kernel_parm->rbf_gamma=atof(argv[i]); break; case 's': i++; kernel_parm->coef_lin=atof(argv[i]); break; case 'r': i++; kernel_parm->coef_const=atof(argv[i]); break; case 'u': i++; strcpy(kernel_parm->custom,argv[i]); break; case '-': strcpy(struct_parm->custom_argv[struct_parm->custom_argc++],argv[i]);i++; strcpy(struct_parm->custom_argv[struct_parm->custom_argc++],argv[i]);break; case 'v': i++; (*struct_verbosity)=atol(argv[i]); break; case 'y': i++; (*verbosity)=atol(argv[i]); break; default: printf("\nUnrecognized option %s!\n\n",argv[i]); print_help(); exit(0); } } if(i>=argc) { printf("\nNot enough input parameters!\n\n"); wait_any_key(); print_help(); exit(0); } strcpy (trainfile, argv[i]); if((i+1)svm_iter_to_shrink == -9999) { learn_parm->svm_iter_to_shrink=100; } if((learn_parm->skip_final_opt_check) && (kernel_parm->kernel_type == LINEAR)) { printf("\nIt does not make sense to skip the final optimality check for linear kernels.\n\n"); learn_parm->skip_final_opt_check=0; } if((learn_parm->skip_final_opt_check) && (learn_parm->remove_inconsistent)) { printf("\nIt is necessary to do the final optimality check when removing inconsistent \nexamples.\n"); wait_any_key(); print_help(); exit(0); } if((learn_parm->svm_maxqpsize<2)) { printf("\nMaximum size of QP-subproblems not in valid range: %ld [2..]\n",learn_parm->svm_maxqpsize); wait_any_key(); print_help(); exit(0); } if((learn_parm->svm_maxqpsizesvm_newvarsinqp)) { printf("\nMaximum size of QP-subproblems [%ld] must be larger than the number of\n",learn_parm->svm_maxqpsize); printf("new variables [%ld] entering the working set in each iteration.\n",learn_parm->svm_newvarsinqp); wait_any_key(); print_help(); exit(0); } if(learn_parm->svm_iter_to_shrink<1) { printf("\nMaximum number of iterations for shrinking not in valid range: %ld [1,..]\n",learn_parm->svm_iter_to_shrink); wait_any_key(); print_help(); exit(0); } if(struct_parm->C<0) { printf("\nYou have to specify a value for the parameter '-c' (C>0)!\n\n"); wait_any_key(); print_help(); exit(0); } if(((*alg_type) < 0) || (((*alg_type) > 5) && ((*alg_type) != 9))) { printf("\nAlgorithm type must be either '0', '1', '2', '3', '4', or '9'!\n\n"); wait_any_key(); print_help(); exit(0); } if(learn_parm->transduction_posratio>1) { printf("\nThe fraction of unlabeled examples to classify as positives must\n"); printf("be less than 1.0 !!!\n\n"); wait_any_key(); print_help(); exit(0); } if(learn_parm->svm_costratio<=0) { printf("\nThe COSTRATIO parameter must be greater than zero!\n\n"); wait_any_key(); print_help(); exit(0); } if(struct_parm->epsilon<=0) { printf("\nThe epsilon parameter must be greater than zero!\n\n"); wait_any_key(); print_help(); exit(0); } if((struct_parm->ccache_size<=0) && ((*alg_type) == 4)) { printf("\nThe cache size must be at least 1!\n\n"); wait_any_key(); print_help(); exit(0); } if(((struct_parm->batch_size<=0) || (struct_parm->batch_size>100)) && ((*alg_type) == 4)) { printf("\nThe batch size must be in the interval ]0,100]!\n\n"); wait_any_key(); print_help(); exit(0); } if((struct_parm->slack_norm<1) || (struct_parm->slack_norm>2)) { printf("\nThe norm of the slacks must be either 1 (L1-norm) or 2 (L2-norm)!\n\n"); wait_any_key(); print_help(); exit(0); } if((struct_parm->loss_type != SLACK_RESCALING) && (struct_parm->loss_type != MARGIN_RESCALING)) { printf("\nThe loss type must be either 1 (slack rescaling) or 2 (margin rescaling)!\n\n"); wait_any_key(); print_help(); exit(0); } if(learn_parm->rho<0) { printf("\nThe parameter rho for xi/alpha-estimates and leave-one-out pruning must\n"); printf("be greater than zero (typically 1.0 or 2.0, see T. Joachims, Estimating the\n"); printf("Generalization Performance of an SVM Efficiently, ICML, 2000.)!\n\n"); wait_any_key(); print_help(); exit(0); } if((learn_parm->xa_depth<0) || (learn_parm->xa_depth>100)) { printf("\nThe parameter depth for ext. xi/alpha-estimates must be in [0..100] (zero\n"); printf("for switching to the conventional xa/estimates described in T. Joachims,\n"); printf("Estimating the Generalization Performance of an SVM Efficiently, ICML, 2000.)\n"); wait_any_key(); print_help(); exit(0); } parse_struct_parameters(struct_parm); } void wait_any_key() { printf("\n(more)\n"); (void)getc(stdin); } void print_help() { printf("\nSVM-struct learning module: %s, %s, %s\n",INST_NAME,INST_VERSION,INST_VERSION_DATE); printf(" includes SVM-struct %s for learning complex outputs, %s\n",STRUCT_VERSION,STRUCT_VERSION_DATE); printf(" includes SVM-light %s quadratic optimizer, %s\n",VERSION,VERSION_DATE); copyright_notice(); printf(" usage: svm_struct_learn [options] example_file model_file\n\n"); printf("Arguments:\n"); printf(" example_file-> file with training data\n"); printf(" model_file -> file to store learned decision rule in\n"); printf("General Options:\n"); printf(" -? -> this help\n"); printf(" -v [0..3] -> verbosity level (default 1)\n"); printf(" -y [0..3] -> verbosity level for svm_light (default 0)\n"); printf("Learning Options:\n"); printf(" -c float -> C: trade-off between training error\n"); printf(" and margin (default 0.01)\n"); printf(" -p [1,2] -> L-norm to use for slack variables. Use 1 for L1-norm,\n"); printf(" use 2 for squared slacks. (default 1)\n"); printf(" -o [1,2] -> Rescaling method to use for loss.\n"); printf(" 1: slack rescaling\n"); printf(" 2: margin rescaling\n"); printf(" (default %d)\n",DEFAULT_RESCALING); printf(" -l [0..] -> Loss function to use.\n"); printf(" 0: zero/one loss\n"); printf(" ?: see below in application specific options\n"); printf(" (default %d)\n",DEFAULT_LOSS_FCT); printf("Optimization Options (see [2][5]):\n"); printf(" -w [0,..,9] -> choice of structural learning algorithm (default %d):\n",(int)DEFAULT_ALG_TYPE); printf(" 0: n-slack algorithm described in [2]\n"); printf(" 1: n-slack algorithm with shrinking heuristic\n"); printf(" 2: 1-slack algorithm (primal) described in [5]\n"); printf(" 3: 1-slack algorithm (dual) described in [5]\n"); printf(" 4: 1-slack algorithm (dual) with constraint cache [5]\n"); printf(" 9: custom algorithm in svm_struct_learn_custom.c\n"); printf(" -e float -> epsilon: allow that tolerance for termination\n"); printf(" criterion (default %f)\n",DEFAULT_EPS); printf(" -k [1..] -> number of new constraints to accumulate before\n"); printf(" recomputing the QP solution (default 100) (-w 0 and 1 only)\n"); printf(" -f [5..] -> number of constraints to cache for each example\n"); printf(" (default 5) (used with -w 4)\n"); printf(" -b [1..100] -> percentage of training set for which to refresh cache\n"); printf(" when no epsilon violated constraint can be constructed\n"); printf(" from current cache (default 100%%) (used with -w 4)\n"); printf("SVM-light Options for Solving QP Subproblems (see [3]):\n"); printf(" -n [2..q] -> number of new variables entering the working set\n"); printf(" in each svm-light iteration (default n = q). \n"); printf(" Set n < q to prevent zig-zagging.\n"); printf(" -m [5..] -> size of svm-light cache for kernel evaluations in MB\n"); printf(" (default 40) (used only for -w 1 with kernels)\n"); printf(" -h [5..] -> number of svm-light iterations a variable needs to be\n"); printf(" optimal before considered for shrinking (default 100)\n"); printf(" -# int -> terminate svm-light QP subproblem optimization, if no\n"); printf(" progress after this number of iterations.\n"); printf(" (default 100000)\n"); printf("Kernel Options:\n"); printf(" -t int -> type of kernel function:\n"); printf(" 0: linear (default)\n"); printf(" 1: polynomial (s a*b+c)^d\n"); printf(" 2: radial basis function exp(-gamma ||a-b||^2)\n"); printf(" 3: sigmoid tanh(s a*b + c)\n"); printf(" 4: user defined kernel from kernel.h\n"); printf(" -d int -> parameter d in polynomial kernel\n"); printf(" -g float -> parameter gamma in rbf kernel\n"); printf(" -s float -> parameter s in sigmoid/poly kernel\n"); printf(" -r float -> parameter c in sigmoid/poly kernel\n"); printf(" -u string -> parameter of user defined kernel\n"); printf("Output Options:\n"); printf(" -a string -> write all alphas to this file after learning\n"); printf(" (in the same order as in the training set)\n"); printf("Application-Specific Options:\n"); print_struct_help(); wait_any_key(); printf("\nMore details in:\n"); printf("[1] T. Joachims, Learning to Align Sequences: A Maximum Margin Aproach.\n"); printf(" Technical Report, September, 2003.\n"); printf("[2] I. Tsochantaridis, T. Joachims, T. Hofmann, and Y. Altun, Large Margin\n"); printf(" Methods for Structured and Interdependent Output Variables, Journal\n"); printf(" of Machine Learning Research (JMLR), Vol. 6(Sep):1453-1484, 2005.\n"); printf("[3] T. Joachims, Making Large-Scale SVM Learning Practical. Advances in\n"); printf(" Kernel Methods - Support Vector Learning, B. Schölkopf and C. Burges and\n"); printf(" A. Smola (ed.), MIT Press, 1999.\n"); printf("[4] T. Joachims, Learning to Classify Text Using Support Vector\n"); printf(" Machines: Methods, Theory, and Algorithms. Dissertation, Kluwer,\n"); printf(" 2002.\n"); printf("[5] T. Joachims, T. Finley, Chun-Nam Yu, Cutting-Plane Training of Structural\n"); printf(" SVMs, Machine Learning Journal, to appear.\n"); } svm-struct-matlab-1.0/svm_struct_api.c000066400000000000000000000553111160627511300201540ustar00rootroot00000000000000/***********************************************************************/ /* */ /* svm_struct_api.c */ /* */ /* Definition of API for attaching implementing SVM learning of */ /* structures (e.g. parsing, multi-label classification, HMM) */ /* */ /* Author: Thorsten Joachims */ /* Date: 03.07.04 */ /* */ /* Copyright (c) 2004 Thorsten Joachims - All rights reserved */ /* */ /* This software is available for non-commercial use only. It must */ /* not be modified and distributed without prior permission of the */ /* author. The author is not responsible for implications from the */ /* use of this software. */ /* */ /***********************************************************************/ #include #include #include "svm_struct/svm_struct_common.h" #include "svm_struct_api.h" /** ------------------------------------------------------------------ ** @brief ** ** Called in learning part before anything else is done to allow any ** initializations that might be necessary. **/ void svm_struct_learn_api_init(int argc, char* argv[]) { } /** ------------------------------------------------------------------ ** @brief ** ** Called in learning part at the very end to allow any clean-up ** that might be necessary. **/ void svm_struct_learn_api_exit() { } /** ------------------------------------------------------------------ ** @brief ** ** Called in prediction part before anything else is done to allow ** any initializations that might be necessary. **/ void svm_struct_classify_api_init (int argc, char* argv[]) { } /** ------------------------------------------------------------------ ** @brief ** ** Called in prediction part at the very end to allow any clean-up ** that might be necessary. **/ void svm_struct_classify_api_exit() { } /** ------------------------------------------------------------------ ** @brief Initialize structured model ** ** Initialize structmodel sm. The weight vector w does not need to be ** initialized, but you need to provide the maximum size of the ** feature space in sizePsi. This is the maximum number of different ** weights that can be learned. Later, the weight vector w will ** contain the learned weights for the model. **/ void init_struct_model (SAMPLE sample, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm, LEARN_PARM *lparm, KERNEL_PARM *kparm) { if (kparm->kernel_type == LINEAR) { mxArray const * sizePsi_array = mxGetField(sparm->mex, 0, "dimension") ; if (! sizePsi_array) { mexErrMsgTxt("Field PARM.DIMENSION not found") ; } if (! uIsRealScalar(sizePsi_array)) { mexErrMsgTxt("PARM.DIMENSION must be a scalar") ; } sm->sizePsi = *mxGetPr(sizePsi_array) ; if (sm->sizePsi < 1) { mexErrMsgTxt("PARM.DIMENSION must be not smaller than 1") ; } } else { sm -> sizePsi = 0 ; } } /** ------------------------------------------------------------------ ** @brief Initialize structred model constraints ** ** Initializes the optimization problem. Typically, you do not need ** to change this function, since you want to start with an empty set ** of constraints. However, if for example you have constraints that ** certain weights need to be positive, you might put that in ** here. The constraints are represented as lhs[i]*w >= rhs[i]. lhs ** is an array of feature vectors, rhs is an array of doubles. m is ** the number of constraints. The function returns the initial set of ** constraints. **/ CONSTSET init_struct_constraints (SAMPLE sample, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm) { CONSTSET c; long sizePsi=sm->sizePsi; long i; WORD words[2]; if(1) { /* normal case: start with empty set of constraints */ c.lhs=NULL; c.rhs=NULL; c.m=0; } else { /* add constraints so that all learned weights are positive. WARNING: Currently, they are positive only up to precision epsilon set by -e. */ c.lhs=my_malloc(sizeof(DOC *)*sizePsi); c.rhs=my_malloc(sizeof(double)*sizePsi); for(i=0; isizePsi. If the ** function cannot find a label, it shall return an empty label as ** recognized by the function empty_label(y). **/ LABEL classify_struct_example (PATTERN x, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm) { LABEL y ; mxArray* fn_array ; mxArray* w_array ; mxArray* args [4] ; int status ; fn_array= mxGetField(sparm->mex, 0, "classifyFn") ; if (! fn_array) { mexErrMsgTxt("Field PARM.CLASSIFYFN not found") ; } if (! mxGetClassID(fn_array) == mxFUNCTION_CLASS) { mexErrMsgTxt("PARM.CLASSIFYFN must be a valid function handle") ; } /* encapsulate sm->w into a Matlab array */ w_array = mxCreateDoubleMatrix(sm->sizePsi, 1, mxREAL) ; memcpy(mxGetPr(w_array), sm->w + 1, sm->sizePsi * sizeof(double)) ; /* evaluate Matlab callback */ args[0] = fn_array ; args[1] = (mxArray*) sparm->mex ; /* model (discard conts) */ args[2] = w_array ; args[3] = x.mex ; status = mexCallMATLAB(1, &y.mex, 4, args, "feval") ; mxDestroyArray(w_array) ; if (status) { mexErrMsgTxt("Error while executing PARM.CLASSIFYFN") ; } if (mxGetClassID(y.mex) == mxUNKNOWN_CLASS) { mexErrMsgTxt("PARM.CLASSIFYFN did not reutrn a result") ; } /* flag this label has one tha should be freed when discared */ y.isOwner = 1 ; return (y) ; } /** ------------------------------------------------------------------ ** @brief Find the most violated constraint with slack rescaling ** ** Finds the label ybar for pattern x that that is responsible for ** the most violated constraint for the slack rescaling ** formulation. For linear slack variables, this is that label ybar ** that maximizes ** ** argmax_{ybar} loss(y,ybar)*(1-psi(x,y)+psi(x,ybar)) ** ** Note that ybar may be equal to y (i.e. the max is 0), which is ** different from the algorithms described in ** [Tschantaridis/05]. Note that this argmax has to take into account ** the scoring function in sm, especially the weights sm.w, as well ** as the loss function, and whether linear or quadratic slacks are ** used. The weights in sm.w correspond to the features defined by ** psi() and range from index 1 to index sm->sizePsi. Most simple is ** the case of the zero/one loss function. For the zero/one loss, ** this function should return the highest scoring label ybar (which ** may be equal to the correct label y), or the second highest ** scoring label ybar, if Psi(x,ybar)>Psi(x,y)-1. If the function ** cannot find a label, it shall return an empty label as recognized ** by the function empty_label(y). **/ LABEL find_most_violated_constraint_slackrescaling (PATTERN x, LABEL y, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm) { LABEL ybar ; mxArray* fn_array ; mxArray* model_array ; mxArray* args [5] ; int status ; fn_array = mxGetField(sparm->mex, 0, "constraintFn") ; if (! fn_array) { mexErrMsgTxt("Field PARM.CONSTRAINTFN not found") ; } if (! mxGetClassID(fn_array) == mxFUNCTION_CLASS) { mexErrMsgTxt("PARM.CONSTRAINTFN must be a valid function handle") ; } /* encapsulate sm->w into a Matlab array */ model_array = newMxArrayEncapsulatingSmodel (sm) ; args[0] = fn_array ; args[1] = (mxArray*) sparm->mex ; /* model (discard conts) */ args[2] = model_array ; args[3] = x.mex ; args[4] = y.mex ; /* Apparently we must make sure that model_array is not destoryed by the automatic cleaning up function. It seems that, because it is constructed by encapsulating input argument to the MEX file, it must be considered an hybrid array, i.e. an array which cannot be destroyed by MATLAB automatic cleanup system, invoked when an error is raised */ mexSetTrapFlag (1) ; status = mexCallMATLAB(1, &ybar.mex, 5, args, "feval") ; mexSetTrapFlag (0) ; destroyMxArrayEncapsulatingSmodel (model_array) ; if (status) { mxArray * error_array ; mexCallMATLAB(1, &error_array, 0, NULL, "lasterror") ; mexCallMATLAB(0, NULL, 1, &error_array, "disp") ; mexCallMATLAB(0, NULL, 1, &error_array, "rethrow") ; } if (mxGetClassID(ybar.mex) == mxUNKNOWN_CLASS) { mexErrMsgTxt("PARM.CONSTRAINTFN did not reutrn a result") ; } ybar.isOwner = 1 ; return(ybar); } /** ------------------------------------------------------------------ ** @brief Find the most violated constraint with margin rescaling ** Finds the label ybar for pattern x that that is responsible for ** the most violated constraint for the margin rescaling ** formulation. For linear slack variables, this is that label ybar ** that maximizes ** ** argmax_{ybar} loss(y,ybar)+psi(x,ybar) ** ** Note that ybar may be equal to y (i.e. the max is 0), which is ** different from the algorithms described in ** [Tschantaridis/05]. Note that this argmax has to take into account ** the scoring function in sm, especially the weights sm.w, as well ** as the loss function, and whether linear or quadratic slacks are ** used. The weights in sm.w correspond to the features defined by ** psi() and range from index 1 to index sm->sizePsi. Most simple is ** the case of the zero/one loss function. For the zero/one loss, ** this function should return the highest scoring label ybar (which ** may be equal to the correct label y), or the second highest ** scoring label ybar, if Psi(x,ybar)>Psi(x,y)-1. If the function ** cannot find a label, it shall return an empty label as recognized ** by the function empty_label(y). **/ LABEL find_most_violated_constraint_marginrescaling (PATTERN x, LABEL y, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm) { LABEL ybar ; mxArray* fn_array ; mxArray* model_array ; mxArray* args [5] ; int status ; fn_array = mxGetField(sparm->mex, 0, "constraintFn") ; if (! fn_array) { mexErrMsgTxt("Field PARM.CONSTRAINTFN not found") ; } if (! mxGetClassID(fn_array) == mxFUNCTION_CLASS) { mexErrMsgTxt("PARM.CONSTRAINTFN is not a valid function handle") ; } /* encapsulate sm->w into a Matlab array */ model_array = newMxArrayEncapsulatingSmodel (sm) ; args[0] = fn_array ; args[1] = (mxArray*) sparm->mex ; /* model (discard conts) */ args[2] = model_array ; args[3] = x.mex ; args[4] = y.mex ; mexSetTrapFlag (1) ; status = mexCallMATLAB(1, &ybar.mex, 5, args, "feval") ; mexSetTrapFlag (0) ; destroyMxArrayEncapsulatingSmodel (model_array) ; if (status) { mxArray * error_array ; mexCallMATLAB(1, &error_array, 0, NULL, "lasterror") ; mexCallMATLAB(0, NULL, 1, &error_array, "error") ; } if (mxGetClassID(ybar.mex) == mxUNKNOWN_CLASS) { mexErrMsgTxt("PARM.CONSTRAINTFN did not reutrn a result") ; } ybar.isOwner = 1 ; return (ybar) ; } /** ------------------------------------------------------------------ ** @brief Is the label empty? ** ** Returns true, if y is an empty label. An empty label might be ** returned by find_most_violated_constraint_???(x, y, sm) if there ** is no incorrect label that can be found for x, or if it is unable ** to label x at all. **/ int empty_label (LABEL y) { return (y.mex == NULL) ; } /** ------------------------------------------------------------------ ** @brief Evaluate Psi(x, y) ** ** Returns a feature vector describing the match between pattern x ** and label y. The feature vector is returned as a list of ** SVECTOR's. Each SVECTOR is in a sparse representation of pairs ** , where the last pair has ** featurenumber 0 as a terminator. Featurenumbers start with 1 and ** end with sizePsi. Featuresnumbers that are not specified default ** to value 0. As mentioned before, psi() actually returns a list of ** SVECTOR's. Each SVECTOR has a field 'factor' and 'next'. 'next' ** specifies the next element in the list, terminated by a NULL ** pointer. The list can be though of as a linear combination of ** vectors, where each vector is weighted by its 'factor'. This ** linear combination of feature vectors is multiplied with the ** learned (kernelized) weight vector to score label y for pattern ** x. Without kernels, there will be one weight in sm.w for each ** feature. Note that psi has to match ** find_most_violated_constraint_???(x, y, sm) and vice versa. In ** particular, find_most_violated_constraint_???(x, y, sm) finds that ** ybar!=y that maximizes psi(x,ybar,sm)*sm.w (where * is the inner ** vector product) and the appropriate function of the loss + ** margin/slack rescaling method. See that paper for details. **/ SVECTOR * psi (PATTERN x, LABEL y, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm) { SVECTOR *sv = NULL; /* The algorith can use either a linear kernel (explicit feature map) * or a custom kernel (implicit feature map). For the explicit feature * map, this function returns a sizePhi-dimensional vector. For * the implicit feature map this function returns a placeholder */ if (sm -> svm_model -> kernel_parm .kernel_type == LINEAR) { /* For the linear kernel computes the vector Phi(x,y) */ mxArray* out ; mxArray* fn_array ; mxArray* args [4] ; WORD* words = NULL ; double twonorm_sq = 0 ; int status ; fn_array = mxGetField(sparm->mex, 0, "featureFn") ; if (! fn_array) { mexErrMsgTxt("Field PARM.FEATUREFN not found") ; } if (! mxGetClassID(fn_array) == mxFUNCTION_CLASS) { mexErrMsgTxt("PARM.FEATUREFN must be a valid function handle") ; } args[0] = fn_array ; args[1] = (mxArray*) sparm->mex ; /* model (discard conts) */ args[2] = x.mex ; /* pattern */ args[3] = y.mex ; /* label */ status = mexCallMATLAB(1, &out, 4, args, "feval") ; if (status) { mexErrMsgTxt("Error while executing PARM.FEATUREFN") ; } if (mxGetClassID(out) == mxUNKNOWN_CLASS) { mexErrMsgTxt("PARM.FEATUREFN must reutrn a result") ; } if (! mxIsSparse(out) || ! mxGetClassID(out) == mxDOUBLE_CLASS || ! mxGetN(out) == 1 || ! mxGetM(out) == sm->sizePsi) { mexErrMsgTxt("PARM.FEATUREFN must return a sparse column vector " "of the prescribed size") ; } { double * data = mxGetPr(out) ; mwIndex * colOffsets = mxGetJc(out) ; mwIndex * rowIndexes = mxGetIr(out) ; int numNZ = colOffsets[1] - colOffsets[0] ; words = (WORD*) my_malloc (sizeof(WORD) * (numNZ + 1)) ; int i ; for (i = 0 ; i < numNZ ; ++ i) { words[i].wnum = rowIndexes[i] + 1 ; words[i].weight = data[i] ; twonorm_sq += data[i] * data[i] ; } words[numNZ].wnum = 0 ; words[numNZ].weight = 0 ; } sv = create_svector_shallow (words, NULL, 1.0) ; sv->twonorm_sq = twonorm_sq ; mxDestroyArray (out) ; } else { /* For the ustom kernel returns a placeholder for (x,y). */ MexPhiCustom phi = newMexPhiCustomFromPatternLabel(x.mex, y.mex) ; WORD * words = mxMalloc(sizeof(WORD)) ; words[0].wnum = 0 ; words[0].weight = 0 ; sv = create_svector_shallow(words, phi, 1.0) ; } return (sv) ; } /** ------------------------------------------------------------------ ** @brief Evaluate loss function Delta(y, ybar) **/ double loss (LABEL y, LABEL ybar, STRUCT_LEARN_PARM *sparm) { double loss_value ; mxArray* fn_array ; mxArray* out ; mxArray* args [4] ; int status ; fn_array = mxGetField(sparm->mex, 0, "lossFn") ; if (! fn_array) { mexErrMsgTxt("Field PARM.LOSSFN not found") ; } if (! mxGetClassID(fn_array) == mxFUNCTION_CLASS) { mexErrMsgTxt("PARM.LOSSFN must be a valid function handle") ; } args[0] = fn_array ; args[1] = (mxArray*) sparm->mex ; /* model (discard conts) */ args[2] = y.mex ; args[3] = ybar.mex ; status = mexCallMATLAB (1, &out, 4, args, "feval") ; if (status) { mexErrMsgTxt("Error while executing PARM.LOSSFN") ; } if (! uIsRealScalar(out)) { mexErrMsgTxt("PARM.LOSSFN must reutrn a scalar") ; } loss_value = *mxGetPr(out) ; mxDestroyArray(out) ; return (loss_value) ; } /** ------------------------------------------------------------------ ** This function is called just before the end of each cutting plane ** iteration. ceps is the amount by which the most violated ** constraint found in the current iteration was ** violated. cached_constraint is true if the added constraint was ** constructed from the cache. If the return value is FALSE, then the ** algorithm is allowed to terminate. If it is TRUE, the algorithm ** will keep iterating even if the desired precision sparm->epsilon ** is already reached. **/ int finalize_iteration (double ceps, int cached_constraint, SAMPLE sample, STRUCTMODEL *sm, CONSTSET cset, double *alpha, STRUCT_LEARN_PARM *sparm) { return (0) ; } /** ------------------------------------------------------------------ ** This function is called after training and allows final touches to ** the model sm. But primarly it allows computing and printing any ** kind of statistic (e.g. training error) you might want. **/ void print_struct_learning_stats (SAMPLE sample, STRUCTMODEL *sm, CONSTSET cset, double *alpha, STRUCT_LEARN_PARM *sparm) { } /** ------------------------------------------------------------------ ** This function is called after making all test predictions in ** svm_struct_classify and allows computing and printing any kind of ** evaluation (e.g. precision/recall) you might want. You can use the ** function eval_prediction to accumulate the necessary statistics for ** each prediction. **/ void print_struct_testing_stats (SAMPLE sample, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm, STRUCT_TEST_STATS *teststats) { } /** ------------------------------------------------------------------ ** This function allows you to accumlate statistic for how well the ** predicition matches the labeled example. It is called from ** svm_struct_classify. See also the function ** print_struct_testing_stats. **/ void eval_prediction (long exnum, EXAMPLE ex, LABEL ypred, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm, STRUCT_TEST_STATS *teststats) { if(exnum == 0) { /* this is the first time the function is called. So initialize the teststats */ } } /** ------------------------------------------------------------------ ** Frees the memory of x. **/ void free_pattern (PATTERN x) { } /** ------------------------------------------------------------------ ** Frees the memory of y. **/ void free_label (LABEL y) { if (y.isOwner && y.mex) { mxDestroyArray(y.mex) ; } } /** ------------------------------------------------------------------ ** Frees the memory of model. **/ void free_struct_model (STRUCTMODEL sm) { if(sm.svm_model) free_model(sm.svm_model, 1 ); /* add free calls for user defined data here */ } /** ------------------------------------------------------------------ ** Frees the memory of a sample. **/ void free_struct_sample (SAMPLE s) { int i; for(i=0;i custom parameters that can be adapted for struct\n"); printf(" learning. The * can be replaced by any character\n"); printf(" and there can be multiple options starting with --.\n"); } /** ------------------------------------------------------------------ ** Parses the command line parameters that start with -- **/ void parse_struct_parameters(STRUCT_LEARN_PARM *sparm) { int i; for(i=0;(icustom_argc) && ((sparm->custom_argv[i])[0] == '-');i++) { switch ((sparm->custom_argv[i])[2]) { case 'a': i++; /* strcpy(learn_parm->alphafile,argv[i]); */ break; case 'e': i++; /* sparm->epsilon=atof(sparm->custom_argv[i]); */ break; case 'k': i++; /* sparm->newconstretrain=atol(sparm->custom_argv[i]); */ break; default: printf("\nUnrecognized option %s!\n\n",sparm->custom_argv[i]); exit(0); } } } /** ------------------------------------------------------------------ ** Prints a help text that is appended to the common help text of ** svm_struct_classify. **/ void print_struct_help_classify() { printf(" --* string -> custom parameters that can be adapted for struct\n"); printf(" learning. The * can be replaced by any character\n"); printf(" and there can be multiple options starting with --.\n"); } /** ------------------------------------------------------------------ ** Parses the command line parameters that start with -- for the ** classification module. **/ void parse_struct_parameters_classify (STRUCT_LEARN_PARM *sparm) { int i; for(i=0;(icustom_argc) && ((sparm->custom_argv[i])[0] == '-');i++) { switch ((sparm->custom_argv[i])[2]) { /* case 'x': i++; strcpy(xvalue,sparm->custom_argv[i]); break; */ default: printf("\nUnrecognized option %s!\n\n",sparm->custom_argv[i]); exit(0); } } } svm-struct-matlab-1.0/svm_struct_api.h000066400000000000000000000075731160627511300201700ustar00rootroot00000000000000/***********************************************************************/ /* */ /* svm_struct_api.h */ /* */ /* Definition of API for attaching implementing SVM learning of */ /* structures (e.g. parsing, multi-label classification, HMM) */ /* */ /* Author: Thorsten Joachims */ /* Date: 03.07.04 */ /* */ /* Copyright (c) 2004 Thorsten Joachims - All rights reserved */ /* */ /* This software is available for non-commercial use only. It must */ /* not be modified and distributed without prior permission of the */ /* author. The author is not responsible for implications from the */ /* use of this software. */ /* */ /***********************************************************************/ #include "svm_struct_api_types.h" #include "svm_struct/svm_struct_common.h" #ifndef svm_struct_api #define svm_struct_api void svm_struct_learn_api_init(int argc, char* argv[]); void svm_struct_learn_api_exit(); void svm_struct_classify_api_init(int argc, char* argv[]); void svm_struct_classify_api_exit(); SAMPLE read_struct_examples(char *file, STRUCT_LEARN_PARM *sparm); void init_struct_model(SAMPLE sample, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm, LEARN_PARM *lparm, KERNEL_PARM *kparm); CONSTSET init_struct_constraints(SAMPLE sample, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm); LABEL find_most_violated_constraint_slackrescaling(PATTERN x, LABEL y, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm); LABEL find_most_violated_constraint_marginrescaling(PATTERN x, LABEL y, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm); LABEL classify_struct_example(PATTERN x, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm); int empty_label(LABEL y); SVECTOR *psi(PATTERN x, LABEL y, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm); double loss(LABEL y, LABEL ybar, STRUCT_LEARN_PARM *sparm); int finalize_iteration(double ceps, int cached_constraint, SAMPLE sample, STRUCTMODEL *sm, CONSTSET cset, double *alpha, STRUCT_LEARN_PARM *sparm); void print_struct_learning_stats(SAMPLE sample, STRUCTMODEL *sm, CONSTSET cset, double *alpha, STRUCT_LEARN_PARM *sparm); void print_struct_testing_stats(SAMPLE sample, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm, STRUCT_TEST_STATS *teststats); void eval_prediction(long exnum, EXAMPLE ex, LABEL prediction, STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm, STRUCT_TEST_STATS *teststats); void write_struct_model(char *file,STRUCTMODEL *sm, STRUCT_LEARN_PARM *sparm); STRUCTMODEL read_struct_model(char *file, STRUCT_LEARN_PARM *sparm); void write_label(FILE *fp, LABEL y); void free_pattern(PATTERN x); void free_label(LABEL y); void free_struct_model(STRUCTMODEL sm); void free_struct_sample(SAMPLE s); void print_struct_help(); void parse_struct_parameters(STRUCT_LEARN_PARM *sparm); void print_struct_help_classify(); void parse_struct_parameters_classify(STRUCT_LEARN_PARM *sparm); void svm_learn_struct_joint_custom(SAMPLE sample, STRUCT_LEARN_PARM *sparm, LEARN_PARM *lparm, KERNEL_PARM *kparm, STRUCTMODEL *sm); #endif svm-struct-matlab-1.0/svm_struct_api_types.h000066400000000000000000000231071160627511300214030ustar00rootroot00000000000000/***********************************************************************/ /* */ /* svm_struct_api_types.h */ /* */ /* Definition of API for attaching implementing SVM learning of */ /* structures (e.g. parsing, multi-label classification, HMM) */ /* */ /* Author: Thorsten Joachims */ /* Date: 13.10.03 */ /* */ /* Copyright (c) 2003 Thorsten Joachims - All rights reserved */ /* */ /* This software is available for non-commercial use only. It must */ /* not be modified and distributed without prior permission of the */ /* author. The author is not responsible for implications from the */ /* use of this software. */ /* */ /***********************************************************************/ #ifndef svm_struct_api_types #define svm_struct_api_types # include "svm_light/svm_common.h" # include "svm_light/svm_learn.h" # include typedef struct MexKernelInfo_ { mxArray const * structParm ; mxArray const * kernelFn ; } MexKernelInfo ; __inline__ static int uIsString(const mxArray* A, int L) { int M = mxGetM(A) ; int N = mxGetN(A) ; return mxIsChar(A) && mxGetNumberOfDimensions(A) == 2 && (M == 1 || (M == 0 && N == 0)) && (L < 0 || N == L) ; } __inline__ static int uIsReal (const mxArray* A) { return mxIsDouble(A) && ! mxIsComplex(A) ; } __inline__ static int uIsRealScalar(const mxArray* A) { return uIsReal (A) && mxGetNumberOfElements(A) == 1 ; } __inline__ static mxArray * newMxArrayFromDoubleVector (int n, double const* v) { mxArray* array = mxCreateDoubleMatrix(n, 1, mxREAL) ; memcpy(mxGetPr(array), v, sizeof(double) * n) ; return (array) ; } __inline__ static mxArray * newMxArrayFromSvector (int n, SVECTOR const* sv) { WORD* wi ; double *pr ; mwSize nz = 0 ; mwIndex *ir, *jc ; mxArray* sv_array ; /* count words */ for (wi = sv->words ; wi->wnum >= 1 ; ++ wi) nz ++ ; /* allocate sparse array */ sv_array = mxCreateSparse(n, 1, nz, mxREAL) ; /* mxSetPr(mxMalloc(sizeof(double) * nz)) ; mxSetIr(mxMalloc(sizeof(mwIndex) * nz)) ; mxSetJc(mxMalloc(sizeof(mwIndex) * 2)) ;*/ ir = mxGetIr (sv_array) ; jc = mxGetJc (sv_array) ; pr = mxGetPr (sv_array) ; /* copy fields */ for (wi = sv->words ; wi->wnum >= 1 ; ++ wi) { *pr ++ = wi -> weight ; *ir ++ = wi -> wnum ; if (wi -> wnum > n) { char str [512] ; snprintf(str, sizeof(str), "Component index %d larger than sparse vector dimension %d", wi -> wnum, n) ; mexErrMsgTxt(str) ; } } jc [0] = 0 ; jc [1] = nz ; return (sv_array) ; } # define INST_NAME "MATLAB MEX interface" # define INST_VERSION "V0.1" # define INST_VERSION_DATE "??.??.??" /* default precision for solving the optimization problem */ # define DEFAULT_EPS 0.1 /* default loss rescaling method: 1=slack_rescaling, 2=margin_rescaling */ # define DEFAULT_RESCALING 2 /* default loss function: */ # define DEFAULT_LOSS_FCT 0 /* default optimization algorithm to use: */ # define DEFAULT_ALG_TYPE 3 /* store Psi(x,y) (for ALG_TYPE 1) instead of recomputing it every time: */ # define USE_FYCACHE 1 /* decide whether to evaluate sum before storing vectors in constraint cache: 0 = NO, 1 = YES (best, if sparse vectors and long vector lists), 2 = YES (best, if short vector lists), 3 = YES (best, if dense vectors and long vector lists) */ # define COMPACT_CACHED_VECTORS 1 /* minimum absolute value below which values in sparse vectors are rounded to zero. Values are stored in the FVAL type defined in svm_common.h RECOMMENDATION: assuming you use FVAL=float, use 10E-15 if COMPACT_CACHED_VECTORS is 1 10E-10 if COMPACT_CACHED_VECTORS is 2 or 3 */ # define COMPACT_ROUNDING_THRESH 10E-15 typedef struct pattern { /* this defines the x-part of a training example, e.g. the structure for storing a natural language sentence in NLP parsing */ mxArray* mex ; } PATTERN; typedef struct label { /* this defines the y-part (the label) of a training example, e.g. the parse tree of the corresponding sentence. */ mxArray* mex ; int isOwner ; } LABEL; typedef struct structmodel { double *w; /* pointer to the learned weights */ MODEL *svm_model; /* the learned SVM model */ long sizePsi; /* maximum number of weights in w */ double walpha; /* other information that is needed for the stuctural model can be added here, e.g. the grammar rules for NLP parsing */ } STRUCTMODEL; typedef struct struct_learn_parm { double epsilon; /* precision for which to solve quadratic program */ double newconstretrain; /* number of new constraints to accumulate before recomputing the QP solution (used in w=1 algorithm) */ int ccache_size; /* maximum number of constraints to cache for each example (used in w=4 algorithm) */ double batch_size; /* size of the mini batches in percent of training set size (used in w=4 algorithm) */ double C; /* trade-off between margin and loss */ char custom_argv[50][300]; /* storage for the --* command line options */ int custom_argc; /* number of --* command line options */ int slack_norm; /* norm to use in objective function for slack variables; 1 -> L1-norm, 2 -> L2-norm */ int loss_type; /* selected loss type from -r command line option. Select between slack rescaling (1) and margin rescaling (2) */ int loss_function; /* select between different loss functions via -l command line option */ /* further parameters that are passed to init_struct_model() */ mxArray const * mex ; } STRUCT_LEARN_PARM ; typedef struct struct_test_stats { /* you can add variables for keeping statistics when evaluating the test predictions in svm_struct_classify. This can be used in the function eval_prediction and print_struct_testing_stats. */ } STRUCT_TEST_STATS; __inline__ static mxArray * newMxArrayEncapsulatingDoubleVector (int n, double * v) { #if 1 mxArray * v_array = mxCreateDoubleMatrix (0, 0, mxREAL) ; mxSetPr (v_array, v) ; mxSetM (v_array, n) ; mxSetN (v_array, 1) ; return v_array ; #else return newMxArrayFromDoubleVector (n, v) ; #endif } __inline__ static mxArray * newMxArrayEncapsulatingSmodel (STRUCTMODEL * smodel) { mwSize dims [] = {1, 1} ; char const * fieldNames [] = { "w", "alpha", "svPatterns", "svLabels" } ; mxArray * smodel_array = mxCreateStructArray (2, dims, 4, fieldNames) ; /* we cannot just encapsulate the arrays because we need to shift by * one */ if (smodel -> svm_model -> kernel_parm .kernel_type == LINEAR) { mxSetField (smodel_array, 0, "w", newMxArrayFromDoubleVector (smodel->sizePsi, smodel->w + 1) ) ; } else { int numFeatures = 0, fi, svi ; SVECTOR * sv ; double * alpha ; /* count how much space we need to store the expansion */ for (svi = 1 ; svi < smodel->svm_model->sv_num ; ++svi) { for (sv = smodel->svm_model->supvec[svi]->fvec ; sv ; sv = sv -> next) { ++ numFeatures ; } } mxArray * alpha_array = mxCreateDoubleMatrix (numFeatures, 1, mxREAL) ; mxArray * svPatterns_array = mxCreateCellMatrix (1, numFeatures) ; mxArray * svLabels_array = mxCreateCellMatrix (1, numFeatures) ; /* fill in the values */ alpha = mxGetPr (alpha_array) ; for (fi = 0, svi = 1 ; svi < smodel->svm_model->sv_num ; ++svi) { for (sv = smodel->svm_model->supvec[svi]->fvec ; sv ; sv = sv -> next, ++ fi) { alpha [fi] = smodel->svm_model->alpha[svi] * sv->factor ; mxSetCell(svPatterns_array, fi, MexPhiCustomGetPattern (sv->userdefined)) ; mxSetCell(svLabels_array, fi, MexPhiCustomGetLabel (sv->userdefined)) ; } } mxSetField (smodel_array, 0, "alpha", alpha_array) ; mxSetField (smodel_array, 0, "svPatterns", svPatterns_array) ; mxSetField (smodel_array, 0, "svLabels", svLabels_array) ; } return smodel_array ; } __inline__ static void destroyMxArrayEncapsulatingDoubleVector (mxArray * array) { if (array) { mxSetN (array, 0) ; mxSetM (array, 0) ; mxSetPr (array, NULL) ; mxDestroyArray (array) ; } } __inline__ static void destroyMxArrayEncapsulatingSmodel (mxArray * array) { if (array) { /* w and alpha are freed by mxDestroyArray, but we do not want this * to happen to the encapsulated patterns and labels yet (or are these shared?) */ int i, n ; mxArray * svPatterns_array = mxGetField (array, 0, "svPatterns") ; mxArray * svLabels_array = mxGetField (array, 0, "svLabels") ; if (svPatterns_array) { n = mxGetNumberOfElements (svPatterns_array) ; for (i = 0 ; i < n ; ++ i) { mxSetCell (svPatterns_array, i, NULL) ; mxSetCell (svLabels_array, i, NULL) ; } } mxDestroyArray (array) ; } } #endif svm-struct-matlab-1.0/svm_struct_learn.m000066400000000000000000000154031160627511300205140ustar00rootroot00000000000000% SVM_STRUCT_LEARN Calls the SVM-struct solver % MODEL = SVM_STRUCT_LEARN(ARGS, PARM) runs SVM-struct solver with % parameters ARGS on the problem PARM. See [1-6] for the % theory. SPARM is a structure of with the fields % % PATTERNS:: patterns (X) % A cell array of patterns. The entries can have any nature % (they can just be indexes of the actual data for example). % % LABELS:: labels (Y) % A cell array of labels. The entries can have any nature. % % LOSSFN:: loss function callback % A handle to the loss function. This function has the form % L = LOSS(PARAM, Y, YBAR) where PARAM is the SPARM structure, % Y a ground truth label, YBAR another label, and L a % non-negative scalar. % % CONSTRAINTFN:: constraint callback % A handle to the constraint generation function. This function % has the form YBAR = FUNC(PARAM, MODEL, X, Y) where PARAM is % the input PARM structure, MODEL is the a structure % representing the current model, X is an input pattern, and Y % is its ground truth label. YBAR is the most violated labels. % % FEATUREN:: feature map callback % A handle to the feature map. This function has the form PSI = % FEATURE(PARAM, X, Y) where PARAM is the input PARM structure, % X is a pattern, Y is a label, and PSI a sparse vector of % dimension PARM.DIMENSION. This handle does not need to be % specified if kernels are used. % % DIMENSION:: dimension of the feature map % The dimension of the feature map. This value does not need to % be specified i kernels are used. % % KERNELFN:: kernel function callback % A handle to the kernel function. This function has the form K % = KERN(PARAM, X, Y, XP, YP) where PARAM is the input PARM % structure, and X, Y and XP, YP are two pattern-label pairs, % input of the joint kernel. This handle does not need to be % specified if feature maps are used. % % MODEL is a structure with fields: % % W:: weight vector % This is a spare vector of size PARAM.DIMENSION. It is used % with feature maps. % % ALPHA:: dual variables % SVPATTERNS:: patterns which are support vectors % SVLABELS:: labels which are support vectors % Used with kernels. % % ARGS is a string specifying options in the usual struct % SVM. These are: % % General Options:: % -v [0..3] -> verbosity level (default 1) % -y [0..3] -> verbosity level for svm_light (default 0) % % Learning Options:: % -c float -> C: trade-off between training error % and margin (default 0.01) % -p [1,2] -> L-norm to use for slack variables. Use 1 for L1-norm, % use 2 for squared slacks. (default 1) % -o [1,2] -> Rescaling method to use for loss. % 1: slack rescaling % 2: margin rescaling % -l [0..] -> Loss function to use. % 0: zero/one loss % ?: see below in application specific options % % Optimization Options (see [2][5]):: % -w [0,..,9] -> choice of structural learning algorithm % 0: n-slack algorithm described in [2] % 1: n-slack algorithm with shrinking heuristic % 2: 1-slack algorithm (primal) described in [5] % 3: 1-slack algorithm (dual) described in [5] % 4: 1-slack algorithm (dual) with constraint cache [5] % 9: custom algorithm in svm_struct_learn_custom.c % -e float -> epsilon: allow that tolerance for termination % criterion % -k [1..] -> number of new constraints to accumulate before % recomputing the QP solution (default 100) (-w 0 and 1 only) % -f [5..] -> number of constraints to cache for each example % (default 5) (used with -w 4) % -b [1..100] -> percentage of training set for which to refresh cache % when no epsilon violated constraint can be constructed % from current cache (default 100%%) (used with -w 4) % % SVM-light Options for Solving QP Subproblems (see [3]):: % -n [2..q] -> number of new variables entering the working set % in each svm-light iteration (default n = q). % Set n < q to prevent zig-zagging. % -m [5..] -> size of svm-light cache for kernel evaluations in MB % (default 40) (used only for -w 1 with kernels) % -h [5..] -> number of svm-light iterations a variable needs to be % optimal before considered for shrinking (default 100) % -# int -> terminate svm-light QP subproblem optimization, if no % progress after this number of iterations. % (default 100000) % % Kernel Options:: % -t int -> type of kernel function: % 0: linear (default) % 1: polynomial (s a*b+c)^d % 2: radial basis function exp(-gamma ||a-b||^2) % 3: sigmoid tanh(s a*b + c) % 4: user defined kernel from kernel.h % -d int -> parameter d in polynomial kernel % -g float -> parameter gamma in rbf kernel % -s float -> parameter s in sigmoid/poly kernel % -r float -> parameter c in sigmoid/poly kernel % -u string -> parameter of user defined kernel % % Output Options:: % -a string -> write all alphas to this file after learning % (in the same order as in the training set) % References:: % [1] T. Joachims, Learning to Align Sequences: A Maximum Margin Approach. % Technical Report, September, 2003. % [2] I. Tsochantaridis, T. Joachims, T. Hofmann, and Y. Altun, Large Margin % Methods for Structured and Interdependent Output Variables, Journal % of Machine Learning Research (JMLR), Vol. 6(Sep):1453-1484, 2005. % [3] T. Joachims, Making Large-Scale SVM Learning Practical. Advances in % Kernel Methods - Support Vector Learning, B. Schölkopf and C. Burges and % A. Smola (ed.), MIT Press, 1999. % [4] T. Joachims, Learning to Classify Text Using Support Vector % Machines: Methods, Theory, and Algorithms. Dissertation, Kluwer, % 2002. % [5] T. Joachims, T. Finley, Chun-Nam Yu, Cutting-Plane Training of Structural % SVMs, Machine Learning Journal, to appear. % [6] http://svmlight.joachims.org/ % Authors:: Andrea Vedaldi (MATLAB MEX version) svm-struct-matlab-1.0/svm_struct_learn_custom.c000066400000000000000000000040441160627511300220730ustar00rootroot00000000000000/***********************************************************************/ /* */ /* svm_struct_learn_custom.c (instantiated for SVM-perform) */ /* */ /* Allows implementing a custom/alternate algorithm for solving */ /* the structual SVM optimization problem. The algorithm can use */ /* full access to the SVM-struct API and to SVM-light. */ /* */ /* Author: Thorsten Joachims */ /* Date: 09.01.08 */ /* */ /* Copyright (c) 2008 Thorsten Joachims - All rights reserved */ /* */ /* This software is available for non-commercial use only. It must */ /* not be modified and distributed without prior permission of the */ /* author. The author is not responsible for implications from the */ /* use of this software. */ /* */ /***********************************************************************/ #include #include #include #include "svm_struct_api.h" #include "svm_light/svm_common.h" #include "svm_struct/svm_struct_common.h" #include "svm_struct/svm_struct_learn.h" void svm_learn_struct_joint_custom(SAMPLE sample, STRUCT_LEARN_PARM *sparm, LEARN_PARM *lparm, KERNEL_PARM *kparm, STRUCTMODEL *sm) /* Input: sample (training examples) sparm (structural learning parameters) lparm (svm learning parameters) kparm (kernel parameters) Output: sm (learned model) */ { /* Put your algorithm here. See svm_struct_learn.c for an example of how to access this API. */ } svm-struct-matlab-1.0/svm_struct_learn_mex.c000066400000000000000000000352001160627511300213500ustar00rootroot00000000000000/***********************************************************************/ /* */ /* svm_struct_main.c */ /* */ /* Command line interface to the alignment learning module of the */ /* Support Vector Machine. */ /* */ /* Author: Thorsten Joachims */ /* Date: 03.07.04 */ /* */ /* Copyright (c) 2004 Thorsten Joachims - All rights reserved */ /* */ /* This software is available for non-commercial use only. It must */ /* not be modified and distributed without prior permission of the */ /* author. The author is not responsible for implications from the */ /* use of this software. */ /* */ /***********************************************************************/ #ifdef __cplusplus extern "C" { #endif #include "svm_light/svm_common.h" #include "svm_light/svm_learn.h" #ifdef __cplusplus } #endif # include "svm_struct/svm_struct_learn.h" # include "svm_struct/svm_struct_common.h" # include "svm_struct_api.h" #include #include #include void read_input_parameters (int, char **, long *, long *, STRUCT_LEARN_PARM *, LEARN_PARM *, KERNEL_PARM *, int *); void arg_split (char *string, int *argc, char ***argv) ; void init_qp_solver() ; void free_qp_solver() ; /** ------------------------------------------------------------------ ** @brief MEX entry point **/ void mexFunction (int nout, mxArray ** out, int nin, mxArray const ** in) { SAMPLE sample; /* training sample */ LEARN_PARM learn_parm; KERNEL_PARM kernel_parm; STRUCT_LEARN_PARM struct_parm; STRUCTMODEL structmodel; int alg_type; enum {IN_ARGS=0, IN_SPARM} ; enum {OUT_W=0} ; /* SVM-light is not fully reentrant, so we need to run this patch first */ init_qp_solver() ; verbosity = 0 ; kernel_cache_statistic = 0 ; if (nin != 2) { mexErrMsgTxt("Two arguments required") ; } /* Parse ARGS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ char arg [1024 + 1] ; int argc ; char ** argv ; if (! uIsString(in[IN_ARGS], -1)) { mexErrMsgTxt("ARGS must be a string") ; } mxGetString(in[IN_ARGS], arg, sizeof(arg) / sizeof(char)) ; arg_split (arg, &argc, &argv) ; svm_struct_learn_api_init(argc+1, argv-1) ; read_input_parameters (argc+1,argv-1, &verbosity, &struct_verbosity, &struct_parm, &learn_parm, &kernel_parm, &alg_type ) ; if (kernel_parm.kernel_type != LINEAR && kernel_parm.kernel_type != CUSTOM) { mexErrMsgTxt ("Only LINEAR or CUSTOM kerneles are supported") ; } /* Parse SPARM ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ mxArray const * sparm_array = in [IN_SPARM] ; mxArray const * patterns_array ; mxArray const * labels_array ; mxArray const * kernelFn_array ; int numExamples, ei ; if (! sparm_array) { mexErrMsgTxt("SPARM must be a structure") ; } struct_parm.mex = sparm_array ; patterns_array = mxGetField(sparm_array, 0, "patterns") ; if (! patterns_array || ! mxIsCell(patterns_array)) { mexErrMsgTxt("SPARM.PATTERNS must be a cell array") ; } numExamples = mxGetNumberOfElements(patterns_array) ; labels_array = mxGetField(sparm_array, 0, "labels") ; if (! labels_array || ! mxIsCell(labels_array) || ! mxGetNumberOfElements(labels_array) == numExamples) { mexErrMsgTxt("SPARM.LABELS must be a cell array " "with the same number of elements of " "SPARM.PATTERNS") ; } sample.n = numExamples ; sample.examples = (EXAMPLE *) my_malloc (sizeof(EXAMPLE) * numExamples) ; for (ei = 0 ; ei < numExamples ; ++ ei) { sample.examples[ei].x.mex = mxGetCell(patterns_array, ei) ; sample.examples[ei].y.mex = mxGetCell(labels_array, ei) ; sample.examples[ei].y.isOwner = 0 ; } if (struct_verbosity >= 1) { mexPrintf("There are %d training examples\n", numExamples) ; } kernelFn_array = mxGetField(sparm_array, 0, "kernelFn") ; if (! kernelFn_array && kernel_parm.kernel_type == CUSTOM) { mexErrMsgTxt("SPARM.KERNELFN must be define for CUSTOM kernels") ; } if (kernelFn_array) { MexKernelInfo * info ; if (mxGetClassID(kernelFn_array) != mxFUNCTION_CLASS) { mexErrMsgTxt("SPARM.KERNELFN must be a valid function handle") ; } info = (MexKernelInfo*) kernel_parm.custom ; info -> structParm = sparm_array ; info -> kernelFn = kernelFn_array ; } /* Learning ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ switch (alg_type) { case 0: svm_learn_struct(sample,&struct_parm,&learn_parm,&kernel_parm,&structmodel,NSLACK_ALG) ; break ; case 1: svm_learn_struct(sample,&struct_parm,&learn_parm,&kernel_parm,&structmodel,NSLACK_SHRINK_ALG); break ; case 2: svm_learn_struct_joint(sample,&struct_parm,&learn_parm,&kernel_parm,&structmodel,ONESLACK_PRIMAL_ALG); break ; case 3: svm_learn_struct_joint(sample,&struct_parm,&learn_parm,&kernel_parm,&structmodel,ONESLACK_DUAL_ALG); break ; case 4: svm_learn_struct_joint(sample,&struct_parm,&learn_parm,&kernel_parm,&structmodel,ONESLACK_DUAL_CACHE_ALG); break ; case 9: svm_learn_struct_joint_custom(sample,&struct_parm,&learn_parm,&kernel_parm,&structmodel); break ; default: mexErrMsgTxt("Unknown algorithm type") ; } /* Write output ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ /* Warning: The model contains references to the original data 'docs'. If you want to free the original data, and only keep the model, you have to make a deep copy of 'model'. */ mxArray * model_array = newMxArrayEncapsulatingSmodel (&structmodel) ; out[OUT_W] = mxDuplicateArray (model_array) ; destroyMxArrayEncapsulatingSmodel (model_array) ; free_struct_sample (sample) ; free_struct_model (structmodel) ; svm_struct_learn_api_exit () ; free_qp_solver () ; } /** ------------------------------------------------------------------ ** @brief Parse argument string **/ void read_input_parameters (int argc,char *argv[], long *verbosity,long *struct_verbosity, STRUCT_LEARN_PARM *struct_parm, LEARN_PARM *learn_parm, KERNEL_PARM *kernel_parm, int *alg_type) { long i ; (*alg_type)=DEFAULT_ALG_TYPE; /* SVM struct options */ (*struct_verbosity)=1; struct_parm->C=-0.01; struct_parm->slack_norm=1; struct_parm->epsilon=DEFAULT_EPS; struct_parm->custom_argc=0; struct_parm->loss_function=DEFAULT_LOSS_FCT; struct_parm->loss_type=DEFAULT_RESCALING; struct_parm->newconstretrain=100; struct_parm->ccache_size=5; struct_parm->batch_size=100; /* SVM light options */ (*verbosity)=0; strcpy (learn_parm->predfile, "trans_predictions"); strcpy (learn_parm->alphafile, ""); learn_parm->biased_hyperplane=1; learn_parm->remove_inconsistent=0; learn_parm->skip_final_opt_check=0; learn_parm->svm_maxqpsize=10; learn_parm->svm_newvarsinqp=0; learn_parm->svm_iter_to_shrink=-9999; learn_parm->maxiter=100000; learn_parm->kernel_cache_size=40; learn_parm->svm_c=99999999; /* overridden by struct_parm->C */ learn_parm->eps=0.001; /* overridden by struct_parm->epsilon */ learn_parm->transduction_posratio=-1.0; learn_parm->svm_costratio=1.0; learn_parm->svm_costratio_unlab=1.0; learn_parm->svm_unlabbound=1E-5; learn_parm->epsilon_crit=0.001; learn_parm->epsilon_a=1E-10; /* changed from 1e-15 */ learn_parm->compute_loo=0; learn_parm->rho=1.0; learn_parm->xa_depth=0; kernel_parm->kernel_type=0; kernel_parm->poly_degree=3; kernel_parm->rbf_gamma=1.0; kernel_parm->coef_lin=1; kernel_parm->coef_const=1; strcpy (kernel_parm->custom,"empty"); /* Parse -x options, delegat --x ones */ for(i=1;(ialphafile,argv[i]); break; case 'c': i++; struct_parm->C=atof(argv[i]); break; case 'p': i++; struct_parm->slack_norm=atol(argv[i]); break; case 'e': i++; struct_parm->epsilon=atof(argv[i]); break; case 'k': i++; struct_parm->newconstretrain=atol(argv[i]); break; case 'h': i++; learn_parm->svm_iter_to_shrink=atol(argv[i]); break; case '#': i++; learn_parm->maxiter=atol(argv[i]); break; case 'm': i++; learn_parm->kernel_cache_size=atol(argv[i]); break; case 'w': i++; (*alg_type)=atol(argv[i]); break; case 'o': i++; struct_parm->loss_type=atol(argv[i]); break; case 'n': i++; learn_parm->svm_newvarsinqp=atol(argv[i]); break; case 'q': i++; learn_parm->svm_maxqpsize=atol(argv[i]); break; case 'l': i++; struct_parm->loss_function=atol(argv[i]); break; case 'f': i++; struct_parm->ccache_size=atol(argv[i]); break; case 'b': i++; struct_parm->batch_size=atof(argv[i]); break; case 't': i++; kernel_parm->kernel_type=atol(argv[i]); break; case 'd': i++; kernel_parm->poly_degree=atol(argv[i]); break; case 'g': i++; kernel_parm->rbf_gamma=atof(argv[i]); break; case 's': i++; kernel_parm->coef_lin=atof(argv[i]); break; case 'r': i++; kernel_parm->coef_const=atof(argv[i]); break; case 'u': i++; strcpy(kernel_parm->custom,argv[i]); break; case 'v': i++; (*struct_verbosity)=atol(argv[i]); break; case 'y': i++; (*verbosity)=atol(argv[i]); break; case '-': strcpy(struct_parm->custom_argv[struct_parm->custom_argc++],argv[i]); i++; strcpy(struct_parm->custom_argv[struct_parm->custom_argc++],argv[i]); break; default: { char msg [1024+1] ; snprintf(msg, sizeof(msg)/sizeof(char), "Unrecognized option '%s'",argv[i]) ; mexErrMsgTxt(msg) ; } } } /* whatever is left is an error */ if (i < argc) { char msg [1024+1] ; snprintf(msg, sizeof(msg)/sizeof(char), "Unrecognized argument '%s'", argv[i]) ; mexErrMsgTxt(msg) ; } /* Check parameter validity */ if(learn_parm->svm_iter_to_shrink == -9999) { learn_parm->svm_iter_to_shrink=100; } if((learn_parm->skip_final_opt_check) && (kernel_parm->kernel_type == LINEAR)) { mexWarnMsgTxt("It does not make sense to skip the final optimality check for linear kernels."); learn_parm->skip_final_opt_check=0; } if((learn_parm->skip_final_opt_check) && (learn_parm->remove_inconsistent)) { mexErrMsgTxt("It is necessary to do the final optimality check when removing inconsistent examples."); } if((learn_parm->svm_maxqpsize<2)) { char msg [1025] ; snprintf(msg, sizeof(msg)/sizeof(char), "Maximum size of QP-subproblems not in valid range: %ld [2..]",learn_parm->svm_maxqpsize) ; mexErrMsgTxt(msg) ; } if((learn_parm->svm_maxqpsizesvm_newvarsinqp)) { char msg [1025] ; snprintf(msg, sizeof(msg)/sizeof(char), "Maximum size of QP-subproblems [%ld] must be larger than the number of" " new variables [%ld] entering the working set in each iteration.", learn_parm->svm_maxqpsize, learn_parm->svm_newvarsinqp) ; mexErrMsgTxt(msg) ; } if(learn_parm->svm_iter_to_shrink<1) { char msg [1025] ; snprintf(msg, sizeof(msg)/sizeof(char), "Maximum number of iterations for shrinking not in valid range: %ld [1,..]", learn_parm->svm_iter_to_shrink); mexErrMsgTxt(msg) ; } if(struct_parm->C<0) { mexErrMsgTxt("You have to specify a value for the parameter '-c' (C>0)!"); } if(((*alg_type) < 0) || (((*alg_type) > 5) && ((*alg_type) != 9))) { mexErrMsgTxt("Algorithm type must be either '0', '1', '2', '3', '4', or '9'!"); } if(learn_parm->transduction_posratio>1) { mexErrMsgTxt("The fraction of unlabeled examples to classify as positives must " "be less than 1.0 !!!"); } if(learn_parm->svm_costratio<=0) { mexErrMsgTxt("The COSTRATIO parameter must be greater than zero!"); } if(struct_parm->epsilon<=0) { mexErrMsgTxt("The epsilon parameter must be greater than zero!"); } if((struct_parm->ccache_size<=0) && ((*alg_type) == 4)) { mexErrMsgTxt("The cache size must be at least 1!"); } if(((struct_parm->batch_size<=0) || (struct_parm->batch_size>100)) && ((*alg_type) == 4)) { mexErrMsgTxt("The batch size must be in the interval ]0,100]!"); } if((struct_parm->slack_norm<1) || (struct_parm->slack_norm>2)) { mexErrMsgTxt("The norm of the slacks must be either 1 (L1-norm) or 2 (L2-norm)!"); } if((struct_parm->loss_type != SLACK_RESCALING) && (struct_parm->loss_type != MARGIN_RESCALING)) { mexErrMsgTxt("The loss type must be either 1 (slack rescaling) or 2 (margin rescaling)!"); } if(learn_parm->rho<0) { mexErrMsgTxt("The parameter rho for xi/alpha-estimates and leave-one-out pruning must" " be greater than zero (typically 1.0 or 2.0, see T. Joachims, Estimating the" " Generalization Performance of an SVM Efficiently, ICML, 2000.)!"); } if((learn_parm->xa_depth<0) || (learn_parm->xa_depth>100)) { mexErrMsgTxt("The parameter depth for ext. xi/alpha-estimates must be in [0..100] (zero" "for switching to the conventional xa/estimates described in T. Joachims," "Estimating the Generalization Performance of an SVM Efficiently, ICML, 2000.)") ; } parse_struct_parameters (struct_parm) ; } void arg_split (char *string, int *argc, char ***argv) { size_t size; char *d, *p; for (size = 1, p = string; *p; p++) { if (isspace((int) *p)) { size++; } } size++; /* leave space for final NULL pointer. */ *argv = (char **) my_malloc(((size * sizeof(char *)) + (p - string) + 1)); for (*argc = 0, p = string, d = ((char *) *argv) + size*sizeof(char *); *p != 0; ) { (*argv)[*argc] = NULL; while (*p && isspace((int) *p)) p++; if (*argc == 0 && *p == '#') { break; } if (*p) { char *s = p; (*argv)[(*argc)++] = d; while (*p && !isspace((int) *p)) p++; memcpy(d, s, p-s); d += p-s; *d++ = 0; while (*p && isspace((int) *p)) p++; } } } svm-struct-matlab-1.0/test_svm_struct_learn.m000066400000000000000000000053201160627511300215500ustar00rootroot00000000000000function test_svm_struct_learn % TEST_SVM_STRUCT_LEARN % A demo function for SVM_STRUCT_LEARN(). It shows how to use % SVM-struct to learn a standard linear SVM. randn('state',0) ; rand('state',0) ; % ------------------------------------------------------------------ % Generate data % ------------------------------------------------------------------ th = pi/3 ; c = cos(th) ; s = sin(th) ; patterns = {} ; labels = {} ; for i=1:100 patterns{i} = diag([2 .5]) * randn(2, 1) ; labels{i} = 2*(randn > 0) - 1 ; patterns{i}(2) = patterns{i}(2) + labels{i} ; patterns{i} = [c -s ; s c] * patterns{i} ; end % ------------------------------------------------------------------ % Run SVM struct % ------------------------------------------------------------------ parm.patterns = patterns ; parm.labels = labels ; parm.lossFn = @lossCB ; parm.constraintFn = @constraintCB ; parm.featureFn = @featureCB ; parm.dimension = 2 ; parm.verbose = 1 ; model = svm_struct_learn(' -c 1.0 -o 1 -v 1 ', parm) ; w = model.w ; % ------------------------------------------------------------------ % Plots % ------------------------------------------------------------------ figure(1) ; clf ; hold on ; x = [patterns{:}] ; y = [labels{:}] ; plot(x(1, y>0), x(2,y>0), 'g.') ; plot(x(1, y<0), x(2,y<0), 'r.') ; set(line([0 w(1)], [0 w(2)]), 'color', 'y', 'linewidth', 4) ; xlim([-3 3]) ; ylim([-3 3]) ; set(line(10*[w(2) -w(2)], 10*[-w(1) w(1)]), ... 'color', 'y', 'linewidth', 2, 'linestyle', '-') ; axis equal ; set(gca, 'color', 'b') ; w end % ------------------------------------------------------------------ % SVM struct callbacks % ------------------------------------------------------------------ function delta = lossCB(param, y, ybar) delta = double(y ~= ybar) ; if param.verbose fprintf('delta = loss(%3d, %3d) = %f\n', y, ybar, delta) ; end end function psi = featureCB(param, x, y) psi = sparse(y*x/2) ; if param.verbose fprintf('w = psi([%8.3f,%8.3f], %3d) = [%8.3f, %8.3f]\n', ... x, y, full(psi(1)), full(psi(2))) ; end end function yhat = constraintCB(param, model, x, y) % slack resaling: argmax_y delta(yi, y) (1 + - ) % margin rescaling: argmax_y delta(yi, y) + if dot(y*x, model.w) > 1, yhat = y ; else yhat = - y ; end if param.verbose fprintf('yhat = violslack([%8.3f,%8.3f], [%8.3f,%8.3f], %3d) = %3d\n', ... model.w, x, y, yhat) ; end end svm-struct-matlab-1.0/test_svm_struct_learn_ker.m000066400000000000000000000056141160627511300224170ustar00rootroot00000000000000function test_svm_struct_learn_ker % TEST_SVM_STRUCT_LEARN % Test function for SVM_STRUCT_LEARN(). It shows how to use % SVM-struct to learn a standard linear SVM while using the generic % kernel interface. randn('state',0) ; rand('state',0) ; % ------------------------------------------------------------------ % Generate data % ------------------------------------------------------------------ th = pi/3 ; c = cos(th) ; s = sin(th) ; patterns = {} ; labels = {} ; for i=1:100 patterns{i} = diag([2 .5]) * randn(2, 1) ; labels{i} = 2*(randn > 0) - 1 ; patterns{i}(2) = patterns{i}(2) + labels{i} ; patterns{i} = [c -s ; s c] * patterns{i} ; end % ------------------------------------------------------------------ % Run SVM struct % ------------------------------------------------------------------ parm.patterns = patterns ; parm.labels = labels ; parm.lossFn = @lossCB parm.constraintFn =@constraintCB ; parm.kernelFn = @kernelCB ; parm.verbose = 1 ; model = svm_struct_learn(' -c 1.0 -o 1 -v 1 -t 4 ', parm) ; w = cat(2, model.svPatterns{:}) * (model.alpha .* cat(1, model.svLabels{:})) / 2 ; % ------------------------------------------------------------------ % Plots % ------------------------------------------------------------------ figure(1) ; clf ; hold on ; x = [patterns{:}] ; y = [labels{:}] ; plot(x(1, y>0), x(2,y>0), 'g.') ; plot(x(1, y<0), x(2,y<0), 'r.') ; set(line([0 w(1)], [0 w(2)]), 'color', 'y', 'linewidth', 4) ; xlim([-3 3]) ; ylim([-3 3]) ; set(line(10*[w(2) -w(2)], 10*[-w(1) w(1)]), ... 'color', 'y', 'linewidth', 2, 'linestyle', '-') ; axis equal ; set(gca, 'color', 'b') ; w end % -------------------------------------------------------------------- % SVM struct callbacks % -------------------------------------------------------------------- function delta = lossCB(param, y, ybar) % loss function delta(y, ybar) delta = double(y ~= ybar) ; if param.verbose fprintf('delta = loss(%3d, %3d) = %f\n', y, ybar, delta) ; end end function k = kernelCB(param, x,y, xp, yp) k = x' * xp * y * yp / 4 ; end function yhat = constraintCB(param, model, x, y) % slack resaling: argmax_y delta(yi, y) (1 + - ) % margin rescaling: argmax_y delta(yi, y) + % the kernel is linear, get a weight vector back if size(model.svPatterns, 2) == 0 w = zeros(size(x)) ; else w = [model.svPatterns{:}] * (model.alpha .* [model.svLabels{:}]') / 2 ; end if dot(y*x, w) > 1, yhat = y ; else yhat = -y ; end if param.verbose fprintf('yhat = violslack([%8.3f,%8.3f], [%8.3f,%8.3f], %3d) = %3d\n', ... w, x, y, yhat) ; end end