pax_global_header00006660000000000000000000000064127017015340014512gustar00rootroot0000000000000052 comment=2998dc6dd011f3845f5562c61ca81261c9349c70 qminer-5.0.0/000077500000000000000000000000001270170153400130075ustar00rootroot00000000000000qminer-5.0.0/.gitignore000066400000000000000000000040031270170153400147740ustar00rootroot00000000000000# Compiled Object files *.slo *.lo *.o # Compiled Dynamic libraries *.so *.dylib # Compiled Static libraries *.lai *.la *.a # project files folder project # Autoconf config.log # NPM npm-debug.log /node_modules/* !/node_modules/qminer.js # project files from Eclipse .project .cproject .settings .metadata # project files from NetBeans nbproject # build results build doc Doxyfile-tmp log-doxygen.txt temp cppdoc qm.conf db *.dat *.opensdf *.sdf *.lib *.user *.suo Debug Release lock jsdoc_1.txt jsdoc_2.txt /examples/twitter/sandbox/twitter/1h_tweets.json /examples/twitter/sandbox/twitter/graph1.gv /examples/twitter/sandbox/twitter/graph2.gv /examples/twitter/sandbox/twitter/svmFilter.bin /examples/twitter/sandbox/twitter/svmSentiment.bin timeseriesHack twitterHack .DS_store /qminer.aps /qminer.rc /resource.h /test/javascript/tesi.batjavascriptdebug javascriptdebug linalg - Copy timeseries - Copy tesi /src/glib/net/sock - novak.h /src/qminer/intellisense/backup_linalg_handmade.txt /src/qminer/intellisense/cut_out.txt /src/qminer/intellisense/intelli_body.js /src/qminer/intellisense/intelli_body2.js /src/qminer/intellisense/intelli_head.js_ /src/qminer/intellisense/intelli_tail_old.js /src/qminer/intellisense/intellisense.js /src/qminer/intellisense/intellitesiOut.js /src/qminer/intellisense/intellitesi_body.js /src/qminer/intellisense/problems.js /src/qminer/intellisense/test.py /src/qminer/intellisense/time.primer unique_fields timeseriesPlot - Copy timeseries - Copy (2) old /examples/graphextract/cobisg.txt bug quax elycite symphony symphony-code out /test/cpp/run-all-tests /test/cpp/test1.txt /test/cpp/test.txt cha .DC_Store autom4te.cache autom4te.cache nodetempdoc /x64 /test/nodejs/*.bin /test/nodejs/g.html db1 db2 /test/nodejs/db-strict binding-1.gyp /test/nodejs/.tmpclfile /test/nodejs/tmpclfile /test/nodejs/tmpclfile2 /test/nodejs/recordset_plain.js /test/nodejs/file.txt /test/my/ /examples/streamaggregate/node_modules *.opendb /test/nodejs/store_and_index.js /test/nodejs/jointest.dbg.js qminer-5.0.0/.gitmodules000066400000000000000000000000001270170153400151520ustar00rootroot00000000000000qminer-5.0.0/.travis.yml000066400000000000000000000013651270170153400151250ustar00rootroot00000000000000sudo: false git: depth: 10 language: node_js addons: apt: sources: [ 'ubuntu-toolchain-r-test', 'llvm-toolchain-precise-3.6' ] packages: [ 'clang-3.6', 'gcc-4.8', 'g++-4.8', 'gcc-4.8-multilib', 'g++-4.8-multilib', 'libc6-dev-i386', 'linux-libc-dev:i386' ] node_js: - 5 env: - CXXFLAGS=-std=c++0x CC=gcc-4.8 CXX=g++-4.8 matrix: include: - node_js: 0.12 env: CXXFLAGS=-std=c++0x CXX=clang++-3.6 CC=clang NPMOPT=--clang=1 QMDEBUG=--debug install: - npm install --build-from-source $NPMOPT $QMDEBUG script: - npm install -g mocha - npm install -g mustache - ./tools/genExampleTests.sh - ./test/nodejs/test.sh os: linux notifications: email: - blaz.fortuna@ijs.si - jan.rupnik@ijs.si - viktor@carvic.si qminer-5.0.0/CHANGELOG.md000066400000000000000000000657571270170153400146440ustar00rootroot00000000000000# QMiner Change Log ### 8 Apr 2016 **New version 5.0.0** **Breaking with new features** New features: - Stores from same `TBase` share same `PBlobBs`. Speed improvements for 1000 empty stores: - create: 0.5s vs 21s - save: 0.6s vs 5s - load: 0.06s vs 4s - Removed unused flags from blob pointer, freeing 37.5% space per `TBlobPt` - Changed `TBlobPt` segment parameter from `uchar` to `uint16`, increasing max blob base size to **128TB** - KMeans reimplemented in C++: Templated and wrapped Stopar's KMeans, which is implemented ```clustering.h``` and ```clustering.cpp```. The javascript wrapper contains the same functions as before. Constructor parameters are: name | type | description -----------------|---------|--------------- **iter** | number | Number of iterations **k** | number | Number of centroids **verbose** | boolean | If false, the console output is supressed **centroidType** | string | Type of the centroids. Options: "Dense" or "Sparse" **distanceType** | string | Distance type used at the calculation. Options: "Euclid" or "Cos" **Properties** ```centroids```, ```medoids```, ```idxv```. **Methods** ```getParams```, ```setParams```, ```getModel```, ```fit```, ```fitAsync```, ```predict```, ```transform```, ```permuteCentroids```, ```explain```, ```save``` The ```fit``` method can be used asynchronously (```fitAsync```). ```javascript var qm = require('qminer'); var params = { iter: 10000, k: 2, verbose: false, distanceType: 'Euclid', centroidType: 'Dense' }; var kmeans = new qm.analytics.KMeans(params); // create the matrix var mat = new qm.la.Matrix({ rows: 1000, cols: 300, random: true }); //- Synchronous use of fit kmeans.fit(mat); //- Asynchronous use of fit kmeans.fitAsync(mat, function (err) { if (err) { console.log(err); } // successful fitting }); ``` Bug fixes: - fixed clang warnings - changed tabs to four spaces on qminer source files - Fixed issue 398 — move `stat.h` stuff to `xmath.h` - Fixed issue 399 - stream aggregate example description ### 25 Mar 2016 **New version 4.10.0** **Non-breaking with new features** New features: - `TStreamAggrOnAddFilter` class can be extended to override `CallOnAdd` method that takes a record and returns true if the aggregate should process it. Currently we have two filters: default that passes every record and `TOnAddSubsampler`, which can skip a given amount of records for every time the aggregate is actually updated. `TSimpleLinReg` is currently the only aggregate that supports filtering. Example: ``` var linReg = store.addStreamAggr({ filter: { type: "subsamplingFilter", skip: 3 }, type: 'simpleLinearRegression', inAggrX: winX.name, inAggrY: winY.name, storeX: "Function", storeY: "Function", quantiles: [0.25, 0.75] }); ``` Bug fixes: - #394. Added asserts for invalid record IDs in the buffer which are a result of store window being too short (incompatible with the window aggregate). - #264 (incorrect exception handling) `TJsonVal` has two new functions: `AssertObjKeyStr`, `AssertObjKeyNum`. The functions take a second parameter (function name), where `__FUNCTION__` can be used (not standard but works on msvc and gcc). Example: `ParamVal->AssertObjKeyStr("timestamp", __FUNCTION__);` - #372. all JS vectors have `toArray` function. - #350. arm publish script added Other: - deleted example tests (generated by travis) - removed datasets tests (will be moved to `qminer-data-loader`) - global mocha instalation is not required any more. tests can be run by calling `npm test` - made tests in `store_partial_flush.js` silent - updated GitHub Wiki (part of #351) ### 18 Mar 2016 **New vesion 4.9.1** **Non-breaking with a big fix** Bug fix: - Primary keys could be set to existing values belonging to other records. Now we throw exception in such cases. Added corresponding tests. ### 11 Mar 2016 **New version: 4.9.0** **Non-breaking with new features** New features: - New aggregates: - `TWinBufFltV` (type: `timeSeriesWinBufVector`), connects to `timeSeriesWinBuf`, implements `IFltV` (holds a vector of buffer values in memory) - `TSimpleLinReg` (type: `simpleLinearRegression`), connects to two `IFltV` aggregates, computes a linear fit and (optionally) quantile bands Bug fixes: - `TGix::PartialFlush` had a bug when saved item set got assigned a new `TBlobPt`. Fixed #386 - `TInMemStorage::DelVals` did wrong accounting when deleting bigger chunks of records - SVR using libsvm unallocated memory fix - VS2015 warning fix (`StackWalker`) - Queue made more efficient, added standard API (`Front()`, `Back()`), refactored variable names to be more meaningful and added unit tests. - Removed IOB error compensation in `TQQueue::GetSubValV` (now throws an exception) and renamed the method to `TQQueue::GetSubValVec`, so it doesn't compile (behaviour change). Any users should rename `GetSubValV` to `GetSubValVec` and check the correctness of indices when calling. - async MDS segfault fix - GUID method changed (can generate more than 10M IDs per second) Other: - Unit tests, documentation, examples: `timeSeriesWinBufVector`, `simpleLinearRegression` ### 4 Mar 2016 **New version: 4.8.0** **Non-breaking with new features** New features: - `analytics.MDS` now has async version - `la.svd` now has async version Bug fixes: - Renamed SparseMatrix.submat to SparseMatrix.getColSubmatrix. Fix #402 - `TQm::TBase` now again backwards compatible with respect to loading settings. Fix #401 Other: - BlobBs file size limit extended from 1GB to 2GB - `la.svd` and `la.qr` moved from `la_structures_nodejs.h` ### 26 Feb 2016 **New version: 4.7.0** **Non-breaking with new features** New features: - `TStrUtil` can transform `THashSet` to string - Async version of `analytics.MDS` - New stream aggregate `threshold`, that returns 1 if input number above threshold, 0 otherwise. - Added `getColSubmatrix` function to `SparseMatrix` that gets `IntVector` of column ids and returns sparse matrix constructed from selected columns. - Added `clear` function to `SparseMatrix` that clears its content and sets rows to -1. Bub fixes: - `TClust::TAbsKMeans` no longer returns empty clusters - `Numeric::InvFtr` wrongly de-normalized numbers - TVec Move constructor: no need to delete ValT - TVec Move assignment: Delete internal ValT pointer only if you own it. Other: - Added examples to `qm.la` module ### 12 Feb 2016 **New version: 4.6.0** **Non-breaking with new features** New features: - Added new query aggregate that performs simple counting of records over some datetime column with provided granularity. - Added move constructor and assignment operator to `TKeyDat` - Added move assignment operator to `THashKeyDat` Bug fixes: - Removed clang compile warnings for MDS and TTimeSpan - documentation generation: jsdoc-baseline version fixed Other: - Faster and more focused CI testing - new branch for releases (ci_matrix) that tests and publishes the full version ### 5 Feb 2016 **New version: 4.5.0** **Non-breaking with new features** New features: - Added support for default field-storage location for whole store. Add special tag `storage_location` in the `options` node of store schema. In the following example, Here, all fields of the store will be stored in cache as this is now the default for the whole store. Each individual field can still override this setting. ``` { "name": store_name, "fields": [ { "name": "name", "type": "string" }, { "name": "val", "type": "int" } ], "options": { "storage_location": "cache" } } ``` - Added on `onTime(unit64 TmMsec)` and `onStep()` functions that enable updating of aggregates without adding a new record in the store. `onTime(unit64 TmMsec)` is added to the NodeJs interface, while `onStep()` is an internal function. Bug fix: - MDS no longer generates compile warnings ### 29 Jan 2016 **New version: 4.4.0** **Non-breaking with new feature** New feature: - `Vector.sparse` takes an optional integer argument for the sparse vector dimension, which can be set to -1 for unknown Bug fixes: - SVR debug test fix - MDS no longer returns compile warnings and actually uses selected distance metric. - `TTDigest MergeValues()` tests no longer break in debug mode (`streamaggr.js` sequential insert test) Other: - debug builds are now tested ### 22 Jan 2016 **New version: 4.3.0** **Non-breaking with new features** Features: - Location of join fields can be defined in schema (memory or cache). - Nearest neighbor anomaly detector explain exposes first and last record ID - Nearest neighbor anomaly detector accepts vector of rates (as opposed to only single rate). Predict returns position of the rate that is reached starting with 1 (or 0 if none). - Can disable field name validation (`scrictNames` in base definition) - TLinAlg can solve generalized eigenvalue problems - added `TRecSet::TRecFilterByFieldUInt64` - added `TIndex::HasJoin(const int& JoinKeyId, const uint64& RecId) const` - added `THash::THash(const TVec >& KeyDatV)` constructor - added `TStr::GetNrNumFExt` can generate any number of leading zeros Big fixes: - GYP fixed to make `libsvm` work. - `qm.saveCsv` puts headers always in quotes - TZipIn::CreateZipProcess puts filename in quotes - Consolidated all references to records with frequency to Fq. ### 15 Jan 2016 **New version: 4.2.0** **Non-breaking with new features** Features: - Nearest neighbor init method exposed - `TNodeJsFtrSpace` factory constructor added - `IFtrSpace` interface added, implemented by `TWinBufFtrSpVec`, exposed in JS - Field-join binary representation: Field-joins can now be stored in more compact way. Develop can specify field-types to be used for storing the field-join's record id and frequency by providing `storage` tag. Default is `uint64-int`. It is also possible to set frequency type to empty string, which means that frequency will be always 1 and it wont take any space in the storage. Example: `joins: [{ name: 'parent', type: 'field', store: 'People', storage: 'int16-byte' }]` - `filterByField`: Recordset provides utility method `filterByField` that can now also operate on field joins: `recordset.filterByField("parent", parent_id, parent_id);`. It currently accepts record ids. Caller can provide a range of IDs (min and max). - added tdigest stream aggregator for estimating any percentile from streaming data Bug fixes: - Sparse vector normalize fixed - `TWinBufFtrSpVec` save/load fix - `writeJson` and `readJson` do not parse and stringify in C++ but instead use `JSON.stringify` and `JSON.parse` - added `readString` to `FIn` that complements `FOut.writeBinary` Other: - test stream aggregate `getFeatureSpace` - `TWinBufFtrSpVec` save/load test ### 8 Jan 2016 **New version: 4.1.0** **non-breaking with new features** Features: - Stream aggregate `TEmaSpVec` - exponential moving average for sparse vectors. Bug fixes: - `TNodeJsFuncStreamAggr` supports the `IsInit` method of the `TStreamAggr` interface provided by `init` function. Added unit tests. - `TOnlineHistogram` supports additional init logic by specifying minimum count (when we have less than the given min, init is false). - Code for new `int16` and `int64` types was copy-pasted in one place and not fixed. Other: - Added tests and documentation for Tokenizer and PCA ### 24 Dec 2015 **New version: 4.0.1** **patch** Bug fixes: - Fixed reflexion of storage related objects (example: store.allRecords now reports to be a getter instead of a value). API was not changed. This fixes Tonic crashes. ### 18 Dec 2015 **New version: 4.0.0** **breaking with new features** Features: - Now works with Node.js 4.x and 5.x - (breaking) `QMiner` now supports the following new types: - `byte` - unsigned value between 0 and 255 - `int16` - 16-bit integer - `uint16` - 16-bit unsigned integer - `uint` - 32-bit unsigned integer - `int64` - 64-bit integer - `sfloat` - single-precision float value (existing type `float` uses double precision) - `json` - arbitrary `javascript` object. Internally it will be stored as `JSON`. Automatically (de)serialized. - `blob` - binary buffer (uses `TMem` internally). When new record is created, this field needs to be sent in as base64-encoded string. When the record is accessed, the field is represented and can be manipulated as javascript's `Buffer` object. - `TStreamAggrOut` two interfaces are now templated `IValTmIO` and `TValVec` - `TFltTmIO` is a typedef for `IValTmIO` - `TFltVec` is a typedef for `TValVec` - `TWinBuf` is now templated (according to new templated interfaces) and an abstract class. A derived class must implement `TVal GetRecVal(const uint64& RecId)`. - Two derived classes: - the old `TWinBuf` is implemented as `TWinBufFlt : public TWinBuf` - `TWinBufFtrSpVec : TWinBuf` takes a JSON array (or a single JSON) of feature extractor descriptors and computes sparse vectors of type TIntFltKdV on records — Added new interface functions to stream aggregate Node.js API - Added support for JSON argument inputs for: `extractVector`, `extractSparseVector`, `extractMatrix`, `updateRecord`, `updateRecords`. The methods that expected a record set can now take an array of JSON objects, where each object respects the store schema. - Added sparse-vector sum aggregate for sparse vectors that maintains centroid vector of sparse vectors coming out of window buffer feature space aggregate. - Introduced `qminer-data-loader` NPM module to handle datasets for examples. - Node.js `TStore` implementation that let us wrap external data sources as stores. For now works with feature extractors - Joins have index type by default. Bug fixes: - MDS documentation fixed. Other: - Added recordSet.sortByFq to documentation - Added examples to linear algebra - Updated travis and appveyor to test: arch x64/x86 - node 0.12/4/5 - platform win/linux/osx - Made qm structures safe for Tonic notebooks (no crashing due to infinite recursion) ### 04 Dec 2015 **New version: 3.6.0** **non-breaking with new features** Features: - Implement full API for MDS in `qm.analytics` - Record set filter by boolean - `FeatureSpace.extractSparseVector` can directly accept JSON, no need to do `store.newRecord(JSON)` before. Bug fixes: - Assert valid names on stream aggregates - Fixed text query returning non-weighted results bug (issue #176) - Fixed record set weighted sampling to actually work as promised (issue #177) - `TStore::GarbageCollect()` works well for stores with only in-mem fields (issue #329) - Fixed `createExampleTests.js` to not remove `*` from code - Cleaned sparse matrix JS constructor - Optimised dense matrix multiplication for row-major - Propagate LIBSVM error messages (issue #303) - Use TNotify for debug and error messages in LIBSVM (issue #302) Other: - Added documentation and tests for timeWindow definition on stores (issue #329) - Added documentation and tests for MDS (issue #309) - Removed Eigen from repository, now included as git submodule - Added unit tests for LIBSVM (issue #301) ### 27 Nov 2015 **New version: 3.5.0** **non-breaking with new features** Features: - Stream aggregates that get time series on input now support delayed inputs (can get more then one value per iteration): online histogram, window aggregates (sum, max, min, mean, variance) - Time series tick and window buffer can read from numeric fields of type other then double Bug fixes: - LIBSVM sparse matrix bug-fix when working with sparse vectors - Multinomial fix for sparse vectors (does not store zero elements) - Nearest neighbor anomaly detector explains more in explain - ClassTPE defined reference counter is now protected and not private - Chi2 stream aggregate cleanup (save/load, etc.) - Stream aggregates implemented in JavaScript can (de)serialize their state - Renamed TNodeJsSA->TNodeJsStreamAggr - Renamed TNodeJsStreamAggr -> TNodeJsFuncStreamAggr ### 20 Nov 2015 **New version: 3.4.0** **non-breaking with new features** Feature: - Stream aggregates have `reset()` function that resets their state - Added serialisation to Chi^2 and online histogram - exposed FAcecss (mode in which base is opened) to js side in qm.base.getStats() method - Decision tree: explain for positive examples, correlation between attributes - Support for writing Node.js async code in C++: `TNodeTask`, macros for defining async functions, callback execution on main thread - Multinomial feature extractor can use numeric field as source for weight - Window stream aggregate: - possibility of delay before things go into the window - changed interface: input and output elements both vectors - does not store windowed elements anymore, keeping only pointers to store Bug fix: - Replaced `nodist` with `nvmw` to prepare binaries for Windows. (`nodist` started acting funny) - removed automatic closing and flushing file stream in `.save(fout)` and `.load(fin)` functions in online regression metric fixed unit tests according to previous commit - bugfix in resampler stream aggregate `.load` method - Compensation for numerical errors in `TSpecFunc::BetaCf` in `xmath.cpp`. Other: - Tests do not output to console anymore - Renamed `TWindowBuffer` to `TWinAggr` ### 13 Nov 2015 **New version: 3.3.0** **non-breaking with new features** Feature: - Added LIBSVM (algorithm name "LIBSVM"), currently we have SVC and SVR - Changed chi2 algorithm so it computes a two sample test - multidimensional scaling for data visualization - EIGEN support (gyp updated). EIGEN will be added to qminer repository (third_party) - save and load in TRecBuffer. The buffer now stores record IDs as opposed to records - online regression metrics now have save and load - spread sheet parser TSsParse can take stream as input - added Decision Tree (split: InfoGain, GainRatio, prune: min examples threshold) - async reading of CSV - added record by value vector in qm module for async processing - FeatureSpace.updateRecordsAsync - FeatureSpace.extractMatrixAsync Bugfixes: - Sort works with multiple threads and is more robust. Sort can take TRnd as argument. - undefined behaviour bug (works different on ARMv7): casting double to uint64 should be: (unsigned)(int64)(double) - portability problem with casting char * to double * (ARMv7 bus errors) Other: - qminer works on tonic: go to https://tonicdev.com/npm/qminer - qminer win 32bit and linux 32bit binaries are published in the cloud - moved logistic regression classifier to classification.h/cpp ### 6 Nov 2015 **New version: 3.2.0** **non-breaking with new features** Feature: - Added build time to flags (`require(‘qminer’).flags`) Other: - no longer depends on `libuuid` on Linux and Mac OSX, now we include `sole` library to handle this task. - can build on ARM v7 (RaspberryPi 2) ### 30 Oct 2015 **New version: 3.1.0** **non-breaking with new features and bug fixes** Features: - `TNodeJsUtil::ExecuteJson` — execute JS function that returns JSON. - ChiSquare stream aggregator - takes two IFltVec stream aggregates on the input and performs ChiSquare test - A new aggregator `TOnlineSlottedHistogram` was added that accumulates data from equivalent historical periods. For instance the same hours of the day or the same minutes of the hour. - A new aggregator `TVecDiff` was added that subtracts one vectors from another. This is useful e.g. when we have histogram for the last 2 hours and another histogram for the whole history. This new aggregate will output the difference in distributions. - JS Sparse matrix can save itself to text format supported by Matlab - Feature space can use only one of the extractors in extract* methods (given as second argument) - OneVsAll can get binary matrix as target label - `clustering.h` now has agglomerative approach - Added async methods (`fitAsnyc` on StreamStory) - Helper async methods available in `TNodeJsAsyncUtil` Bug fixes: - CI does not fail if same version was already published - Movies example now works - TSparseColMatrix and TSparseRowMatrix now compute dimensions - Bag of words feature generator clears word hash table on Clr() - Added checks for feature space and record compatibility - Fixed issue #187 - Undid skipped tests for fresh date, text feature extractor, prop hazards Other: - Added some C++ unit tests, removed old obsolete files. - Example tests for examples that work over http server (currently only for stream aggregate example) - Removed deasync dependency - Documented analytics.preprocessing ### 23 Oct 2015 **New version: 3.0.0** **breaking with new features and bug fixes** Features: - On each release binaries for Windows, Linux and Mac OS are automatically prepared and uploaded to Amazon S3. `npm install qminer` no longer needs to compile from source. To force recompile use `npm install qminer --build-from-source`. - `TNodeJsUtil::GetArgTmMSecs` can extract time from javascript function argument (ISO String, Date or timestamp) Bug fixes: - Time stamps coming from Node.js now treated as signed integer - Cleanup code for indicators in signalproc.h - Fixed serialization of KMeans in analytics.h (breaking) - Removed redundant constructors in stream aggregators (breaking) - Smaller release binaries on Linux (removed debug symbols) - KMeans in `clustering.h` did not update centroids, now it does - Tested and removed bugs in `metrics.ClassificationScore` and `metrics.PredictionCurves` in `analytics.js`. Other: - Increased timeout for tests, needed for slow runs on Travis or Appveyor CI - Documentation and tests for `metrics` in `analytics.js`. ### 16 Oct 2015 **New version: 2.6.0 ** **non-breaking with new features and bug fixes** Features: - Online histogram in `signalproc.h`, which can inc/dec bing counts - Histogram stream aggregate which can attach to tick (no forgetting) or window (forgetting) - `FindAll` and `FindAllSatisfy` added to `TVec` to get all IDs of occurrences of (a) given value, or (b) values that satisfy given function - `TSpecFunc::StudentCdf` student cumulative density functions - New cluster methods in `clustering.h`: K-Means, DP-Means - Hierarchical Markov Chain in `mc.h` working on continuous time - `fs.readLines(fin, onLine, onEnd, onError)` which iterates over file/inputs stream/node buffer and executes callback for each line - `Base.isClosed()`, returns true if base closed - Helper functions in `TNodeJsUtil` for checking if argument is undefined, getting function from argument and getting function, int and float from V8 object Bug fixes: - Bad parameters when creating stream aggregates in Node.JS do not crash the whole process - Unwrapping has additional checks to prevent crashing - Sort does error and exception checking around JavaScript callbacks - TTm to ISO String fixed to always have 3 digits for milliseconds - Stemmer no longer crashes on strange parameters - SVD works on 1-dimensional inputs- Fixed confusions between C++ and JS timestamps. C++ side now consistently uses Windows timestamps (milliseconds from 1601-01-01) and JS uses milliseconds since 1970-01-01 (same as Date.getTime()) - No more skipped tests for stream aggregates and resolved associated issues. - Issues closed: 197, 196, 189, 188, 183, 192, 230, 198 Other: - Improved documentation for base schemas, record sets, etc. ### 2 Oct 2015 **New version: 2.5.0** **non-breaking with new features and bug fixes** Features: - added BTree index for efficient numeric range searches. Supported data types: int, float, uint64, datatime - Regression error metrics: batch and online metrics - Recordset.filterByField - Added support for null values for numerics and datetime. Also, added support for datetime-filtering via string or uint64 (Unix msec-epoch). Bug fixes: Other: - Unit tests and documentation for NNet - Code cleanup - Documentation generation fixes and enhancements ### 25 Sep 2015 **New version: 2.4.0** **non-breaking with new features and bug fixes** Features: - `record.setField`, `store.newRec` and `recset.filterByField` accept unix timestamp as input for datetime fields - `fout.writeBinary` writes binary serialization of JS strings, numbers or jsons to GLib output stream (`TFOut`) - k-means has explain method which returns medoid of the cluster new instance is assigned to Bug fixes: - fixed memory leak when assigning emtpy TVec to another empty TVec - automatic removal of timestamp in generated javascript documentation (jsdoc) to avoid conflicts at merging documentation ### 18 Sep 2015 **New version: 2.3.0** **non-breaking with new features** Features: - KMeans can get seed centroids as parameters Bug fixes: - TFIn reading buffer beyond EOF - TVec::AddSorted made faster - Feature space constructor checkes parameters Other: - Cleaned and updated SNAP examples and documentation - Added required APIs, documentation and tests for logistic regression, proportional hazards and neural networks ### 11 Sep 2015 **New version: 2.2.1** **non-breaking without new features** Bug fixes: - SVC save fixed - twitter example fixed - time series example fixed - TStr::TrimLeft could crash - active learning fixed Other: - SVC.save (unit test, example) - recursive linear reg tests and documentation - logistic regression API update, doc, example, tests - proportional hazards model API update, doc, example, tests - licenses updated - /src/qminer/gui folder deleted - examples/timeseriesPlot deleted - examples/updatingTimeseriesPlot deleted - src/nodejs/ run.js scripts removed ### 4 Sep 2015 **New version: 2.2.0** **non-breaking with new features** Features: - TPath with functions for editing file and path strings - Added compile flags to `qm.flags` - Added recset.FilterByFieldStr which takes two strings and keeps all that are between. Exposed in JS API Bug fixes: - TUInt64 now has Mx and Mn fields of type uint64 - TInt64 now has Mx and Mn fields of type int64 ### 28 Aug 2015 **New version: 2.1.1** **non-breaking, no new features** Other: - Added tests for almost all documented classes and methods in `analytics.js` - All examples from documentation are now also converted to tests - Tests now have 10s to finish - Removed copy-paste from `binding.gyp`, MS Build toolset now defaults to v120 ### 21 Aug 2015 **New version: 2.1.0** **non-breaking with new features** New features: - k-Means - can reorder computed centroids (clusters) based on a given map - Nearest Neighbor anomaly detector reimplemented in C++, now much faster. Only works with sparse vectors. Bug fixes: - `TZipIn` does not hang when looking for length, on empty files, etc. - Fixed NaN issue in `TSigmoid`, now works normally Other: - Removed deprecated `TTempIndex` from `qminer_core` - Merged `qminer_gs` and `qminer_pbs` into `qminer_storage` - Moved TOAST functions from `qminer_core` to `qminer_storage` ### 7 Aug 2015 **New version: 2.0.0** **Breaking changes: binary format of storage changed, JS API modified (analytics cleanup)** New features: - New store implementation using Paged Blob: - TMem and TBase implement C++ move semantics - Optimized DeleteAllRecs on TStoreImpl and TStorePgb - TOAST support in Page Blob - Numeric feature extractor: new option for normalization (standardize) - Circular record buffer in `qm.js` - Nearest neighbor anomaly detection extended to cover streaming scenarios (partialFit) and serialization - TFIn, TFOut support for (de)-serializing JSON - kmeans: manually set initial centroids Other: - gix improvements - SVR optimized - unit tests: svc, svr, kmeans qminer-5.0.0/LICENSE000066400000000000000000000030741270170153400140200ustar00rootroot00000000000000BSD License For QMiner software Copyright (c) 2015, Jozef Stefan Institute, Quintelligence d.o.o. and contributors All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. The views and conclusions contained in the software and documentation are those of the authors and should not be interpreted as representing official policies, either expressed or implied, of the FreeBSD Project. qminer-5.0.0/README.md000066400000000000000000000101321270170153400142630ustar00rootroot00000000000000QMiner ====== [![Join the chat at https://gitter.im/qminer/qminer](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/qminer/qminer?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![NPM Version][npm-image]][npm-url] [![NPM Downloads][downloads-image]][downloads-url] [![Linux Build][travis-linux-image]][travis-linux-url] [![Linux Build][travis-mac-image]][travis-mac-url] [![Windows Build][appveyor-image]][appveyor-url] QMiner is an analytics platform for large-scale real-time streams containing structured and unstructured data. It is designed for scaling to millions of instances on high-end commodity hardware, providing efficient storage, retrieval and analytics mechanisms with real-time response. **[Project homepage](http://qminer.ijs.si/)** ### Examples - [Twitter sentiment extraction](https://tonicdev.com/rupnikj/qminer-sentiment-extraction) - [Recommendation system](https://tonicdev.com/blazf/qminer-recommendation) ### Prerequisites **node.js v5.x, v4.x or v0.12.x and npm 2.11 or higher** To test that your node.js version is correct, run ```node --version``` and ```npm --version```. Not compatible with nodejs v0.10 or older. **Windows** - [Visual C++ Redistributable Packages for Visual Studio 2013](https://www.microsoft.com/en-us/download/details.aspx?id=40784) download **vcredist_x64.exe** for node.js x64 or **vcredist_x86.exe** for node.js x86. ### Install npm install qminer **Test** node -e "require('qminer'); console.log('OK')" --- ### Compile from source + [Linux](https://github.com/qminer/qminer/wiki/Compiling-from-source-on-Linux) + [Mac OS X](https://github.com/qminer/qminer/wiki/Compiling-from-source-on-Mac-OSX) + [Windows](https://github.com/qminer/qminer/wiki/Compiling-from-source-on-Windows) ### Documentation + [Examples](https://github.com/qminer/qminer/wiki/Example) + [JavaScript API](https://rawgit.com/qminer/qminer/master/nodedoc/index.html) + [`qm` module](https://rawgit.com/qminer/qminer/master/nodedoc/module-qm.html) + [`la` module](https://rawgit.com/qminer/qminer/master/nodedoc/module-la.html) + [`analytics` module](https://rawgit.com/qminer/qminer/master/nodedoc/module-analytics.html) + [C++ Documentation](http://agava.ijs.si/~blazf/qminer/) + [Architecture](https://github.com/qminer/qminer/wiki/Architecture) + [Roadmap](https://github.com/qminer/qminer/wiki/Roadmap) + [Mailing list](https://groups.google.com/forum/#!forum/qminer) ## Acknowledgments [QMiner](http://qminer.ijs.si/) is developed by [AILab](http://ailab.ijs.si/) at [Jozef Stefan Institute](http://www.ijs.si/), [Quintelligence](http://quintelligence.com) and other contributors. The authors would like to acknowledge funding from the European Union Seventh Framework Programme, under Grant Agreements 288342 ([XLike](http://www.xlike.org/)), 611346 ([XLime](http://xlime.eu)), 611875 ([Symphony](http://projectsymphony.eu)), 317534 ([Sophocles](http://sophocles.eu/)), 318452 ([Mobis](https://sites.google.com/site/mobiseuprojecteu/)), 600074 ([NRG4Cast](http://nrg4cast.org)), 619437 ([Sunseed](http://sunseed-fp7.eu)), 632840 ([FI-Impact](http://fi-impact.net/home/)) and 612329 ([ProaSense](http://www.proasense.eu)). This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 636160-2 ([Optimum](http://www.optimumproject.eu/)). ![](http://ailab.ijs.si/~blazf/eu.png) Thanks to [Deleaker](http://deleaker.com) for supporting open-source projects. [npm-image]: https://img.shields.io/npm/v/qminer.svg [npm-url]: https://npmjs.org/package/qminer [downloads-image]: https://img.shields.io/npm/dm/qminer.svg [downloads-url]: https://npmjs.org/package/qminer [travis-linux-image]: https://img.shields.io/travis/qminer/qminer/master.svg?label=linux [travis-linux-url]: https://travis-ci.org/qminer/qminer [travis-mac-image]: https://img.shields.io/travis/qminer/qminer/osx-binaries.svg?label=mac [travis-mac-url]: https://travis-ci.org/qminer/qminer [appveyor-image]: https://img.shields.io/appveyor/ci/rupnikj/qminer/master.svg?label=windows [appveyor-url]: https://ci.appveyor.com/project/rupnikj/qminer qminer-5.0.0/appveyor.yml000066400000000000000000000014721270170153400154030ustar00rootroot00000000000000environment: matrix: - nodejs_version: "4" platform: - x64 configuration: - release shallow_clone: true install: # Set paths - SET PATH=C:\Program Files (x86)\MSBuild\12.0\bin\;%PATH% - if "%PLATFORM%" == "x64" set PATH=C:\Python27-x64;%PATH% - if "%PLATFORM%" == "x86" SET PATH=C:\python27;%PATH% # Check if we're publishing. Only punlish commits including [publish binary] at qminer/qminer (ignore pull requests) - SET CM=%APPVEYOR_REPO_COMMIT_MESSAGE% - ECHO commit message %CM% - ECHO %APPVEYOR_REPO_NAME% - ps: Install-Product node $env:nodejs_version $env:platform build_script: - npm install -g mustache - npm install --build-from-source --verbose --msvs_version=2013 --%CONFIGURATION% - cd tools - genExampleTests.bat - cd .. - .\test\nodejs\test.bat test: off deploy: off qminer-5.0.0/binding.gyp000066400000000000000000000222551270170153400151500ustar00rootroot00000000000000{ 'variables': { 'LIN_ALG_BLAS%': 'NBLAS', 'LIN_ALG_LAPACKE%': 'NLAPACKE', 'LIN_ALG_EIGEN%': 'NEIGEN', 'LIN_ALG_INCLUDE%': 'src/glib/base/', 'LIN_EIGEN_INCLUDE%': 'src/third_party/eigen/', #full path to lapack or openblas libraries 'LIN_ALG_LIB%': '', #64 bit indexing for BLAS 'INDEX_64%': 'NINDEX_64', 'INTEL%': 'NINTEL' }, 'target_defaults': { 'default_configuration': 'Release', 'configurations': { 'Debug': { 'defines': [ 'DEBUG', ], }, 'Release': { 'defines': [ 'NDEBUG' ], } }, 'defines': [ '<(LIN_ALG_BLAS)', '<(LIN_ALG_LAPACKE)', '<(LIN_ALG_EIGEN)', '<(INDEX_64)', '<(INTEL)' ], # hack for setting xcode settings based on example from # http://src.chromium.org/svn/trunk/o3d/build/common.gypi 'target_conditions': [ ['OS=="mac"', { 'xcode_settings': { 'MACOSX_DEPLOYMENT_TARGET': '10.7', 'GCC_ENABLE_CPP_RTTI': 'YES', 'GCC_ENABLE_CPP_EXCEPTIONS': 'YES', 'OTHER_CFLAGS': [ '-std=c++11', '-stdlib=libc++' ] }, }], ], 'conditions': [ # operating system specific parameters ['OS == "linux"', { "link_settings": { "libraries": [ '-lrt', '-fopenmp', '<(LIN_ALG_LIB)' ], }, # GCC flags 'cflags_cc!': [ '-fno-rtti', '-fno-exceptions' ], 'cflags_cc': [ '-std=c++0x', '-frtti', '-fexceptions' ], 'cflags': [ '-Wno-deprecated-declarations', '-fopenmp' ], # additional flags for ARM 'conditions': [ ['target_arch == "arm"', { "link_settings": { "ldflags": [ '-Wl,--allow-multiple-definition' ] }, 'cflags!': [ ], # add -g if low on memory and gcc fails in debug mode 'cflags': [ '-fsigned-char' ], # add -g if you need symbols in release mode 'defines': [ "ARM" ] }] ] }], ['OS == "win"', { 'msbuild_toolset': 'v120', 'msvs_settings': { 'VCCLCompilerTool': { #'RuntimeTypeInfo': 'true', # /GR : this should work but doesn't get picked up #'ExceptionHandling': 1, # /EHsc: this should work but doesn't get picked up 'OpenMP': 'true', "AdditionalOptions": [ "/EHsc /GR" ] # release mode displays D9025 warnings, which is a known issue https://github.com/nodejs/node-gyp/issues/335 }, 'VCLinkerTool': { 'SubSystem' : 1, # Console 'AdditionalOptions': ['<(LIN_ALG_LIB)'] }, }, }], ['OS == "mac"', { "default_configuration": "Release", "configurations": { "Debug": { "defines": [ "DEBUG", ], "xcode_settings": { "GCC_OPTIMIZATION_LEVEL": "0", "GCC_GENERATE_DEBUGGING_SYMBOLS": "YES" } }, "Release": { "defines": [ "NDEBUG" ], "xcode_settings": { "GCC_OPTIMIZATION_LEVEL": "3", "GCC_GENERATE_DEBUGGING_SYMBOLS": "NO", "DEAD_CODE_STRIPPING": "YES", "GCC_INLINES_ARE_PRIVATE_EXTERN": "YES" } } } }] ], }, 'targets': [ { # node qminer module 'target_name': 'qm', 'sources': [ # core qm module 'src/nodejs/qm/qm_nodejs.h', 'src/nodejs/qm/qm_nodejs.cpp', 'src/nodejs/qm/qm_nodejs_streamaggr.h', 'src/nodejs/qm/qm_nodejs_streamaggr.cpp', 'src/nodejs/qm/qm_nodejs_store.h', 'src/nodejs/qm/qm_nodejs_store.cpp', 'src/nodejs/qm/qm_param.h', # la 'src/nodejs/la/la_nodejs.h', 'src/nodejs/la/la_nodejs.cpp', 'src/nodejs/la/la_structures_nodejs.h', 'src/nodejs/la/la_structures_nodejs.cpp', 'src/nodejs/la/la_vector_nodejs.h', # analytics 'src/nodejs/analytics/analytics.h', 'src/nodejs/analytics/analytics.cpp', # fs 'src/nodejs/fs/fs_nodejs.h', 'src/nodejs/fs/fs_nodejs.cpp', # snap 'src/nodejs/snap/snap_nodejs.h', 'src/nodejs/snap/snap_nodejs.cpp', # ht 'src/nodejs/ht/ht_nodejs.h', 'src/nodejs/ht/ht_nodejs.cpp', # statistics 'src/nodejs/statistics/stat_nodejs.h', 'src/nodejs/statistics/stat_nodejs.cpp', # addon utilities 'src/nodejs/nodeutil.h', 'src/nodejs/nodeutil.cpp', # init functions 'src/nodejs/modinit.h', 'src/nodejs/modinit.cpp' ], 'include_dirs': [ 'src/nodejs/qm', 'src/nodejs/la', 'src/nodejs/analytics', 'src/nodejs/fs', 'src/nodejs/snap', 'src/nodejs/ht', 'src/nodejs/statistics', 'src/nodejs/', 'src/qminer/', 'src/glib/base/', 'src/glib/mine/', 'src/glib/misc/', 'src/third_party/sole/', 'src/third_party/Snap/snap-core', 'src/third_party/Snap/snap-adv', 'src/third_party/Snap/snap-exp', 'src/third_party/Snap/qlib-core', '<(LIN_ALG_INCLUDE)', '<(LIN_EIGEN_INCLUDE)' ], 'dependencies': [ 'glib', 'snap_lib', 'qminer', ], }, { # qminer library 'target_name': 'qminer', 'type': 'static_library', 'sources': [ 'src/qminer/qminer_core.h', 'src/qminer/qminer_core.cpp', 'src/qminer/qminer_storage.h', 'src/qminer/qminer_storage.cpp', 'src/qminer/qminer_ftr.h', 'src/qminer/qminer_ftr.cpp', 'src/qminer/qminer_aggr.h', 'src/qminer/qminer_aggr.cpp' ], 'include_dirs': [ 'src/qminer', 'src/glib/base/', 'src/glib/mine/', 'src/glib/misc/', 'src/third_party/sole/', '<(LIN_ALG_INCLUDE)', '<(LIN_EIGEN_INCLUDE)' ], }, { # snap external library 'target_name': 'snap_lib', 'type': 'static_library', 'sources': [ 'src/third_party/Snap/snap-core/Snap.cpp' ], 'include_dirs': [ 'src/third_party/Snap/snap-core', 'src/third_party/Snap/snap-adv', 'src/third_party/Snap/snap-exp', 'src/glib/base/', 'src/glib/mine/', 'src/glib/misc/', '<(LIN_ALG_INCLUDE)', '<(LIN_EIGEN_INCLUDE)' ], }, { # glib library 'target_name': 'glib', 'type': 'static_library', 'sources': [ 'src/glib/base/base.h', 'src/glib/base/base.cpp', 'src/glib/mine/mine.h', 'src/glib/mine/mine.cpp', 'src/third_party/sole/sole.cpp', 'src/third_party/libsvm/svm.h', 'src/third_party/libsvm/svm.cpp' ], 'include_dirs': [ 'src/glib/base/', 'src/glib/mine/', 'src/glib/misc/', 'src/third_party/sole/', 'src/third_party/libsvm/', '<(LIN_ALG_INCLUDE)', '<(LIN_EIGEN_INCLUDE)' ], }, { # needed for publishing binaries with node-pre-gyp 'target_name': 'action_after_build', 'type': 'none', 'dependencies': [ 'qm' ], 'copies': [{ 'files': [ '<(PRODUCT_DIR)/qm.node' ], 'destination': './out/' }] } ] } qminer-5.0.0/examples/000077500000000000000000000000001270170153400146255ustar00rootroot00000000000000qminer-5.0.0/examples/.gitignore000066400000000000000000000000061270170153400166110ustar00rootroot00000000000000/news qminer-5.0.0/examples/graphs/000077500000000000000000000000001270170153400161115ustar00rootroot00000000000000qminer-5.0.0/examples/graphs/graphs.js000066400000000000000000000041501270170153400177330ustar00rootroot00000000000000snap = require('../../index.js').snap; console.log("creating 1) undirected, 2) directed and 3) directe-multi graph"); // creating 1) undirected, 2) directed and 3) directed-multi graph g1 = new snap.UndirectedGraph; g2 = new snap.DirectedGraph; g3 = new snap.DirectedMultigraph; // adding nodes g1.addNode(1); g2.addNode(1); g3.addNode(1); g1.addNode(2); g2.addNode(2); g3.addNode(2); g1.addNode(3); g2.addNode(3); g3.addNode(3); g1.addNode(4); g2.addNode(4); g3.addNode(4); // adding edges g1.addEdge(1, 2); g2.addEdge(1, 2); g3.addEdge(1, 2); g1.addEdge(1, 2); g2.addEdge(1, 2); g3.addEdge(1, 2); g1.addEdge(2, 1); g2.addEdge(2, 3); g3.addEdge(2, 3); g1.addEdge(4, 3); g2.addEdge(1, 3); g3.addEdge(1, 3); g1.addEdge(3, 4); g2.addEdge(1, 4); g3.addEdge(1, 4); // iteration nodes and edges console.log("UNDIRECTED GRAPH (g1):"); console.log("nodes:"); g1.eachNode(function (N) { console.log("id: " + N.id + " deg: " + N.deg); }); console.log("edges:") g1.eachEdge(function (E) { console.log("n1: " + E.srcId + " n2: " + E.dstId); }); console.log("DIRECTED GRAPH (g2):"); console.log("nodes:"); g2.eachNode(function (N) { console.log("id: " + N.id + " deg: " + N.deg); }); console.log("edges:") g2.eachEdge(function (E) { console.log("n1: " + E.srcId + " n2: " + E.dstId); }); console.log("DIRECTED-MULTIGRAPH (g3):"); console.log("nodes:") g3.eachNode(function (N) { console.log("id: " + N.id + " deg: " + N.deg); }); console.log("edges:") g3.eachEdge(function (E) { console.log("n1: " + E.srcId + " n2: " + E.dstId); }); // connected components components1 = g1.components() console.log("CONNECTED COMPONENTS (g1)"); for (var i=0; i