requirements.txt0000664ڊ00111170000000037213243274037014274 0ustar klemmso0cvpr# Dependencies for Barista # To install, open terminal and use command "sudo pip install -r requirements.txt" h5py==2.6.0 leveldb==0.194 lmdb==0.92 matplotlib==1.5.3 seaborn==0.7.1 numpy>=1.12.0 ipython==5.5.0 ipywidgets==5.2.2 scikit-image==0.12.3 README.md0000664ڊ00111170000000270013243274037012264 0ustar klemmso0cvprBarista is an open-source graphical high-level interface for the Caffe deep learning framework written in Python. **Overview** * Barista is distributed under the MIT license (see LICENSE file). * For more information, please refer to https://www.uni-muenster.de/PRIA/Barista/ * Please note that Barista is currently mainly developed on Linux and that Windows and Mac OS support is still experimental. **Requirements** * Barista uses Python 2.7 * In order to use Barista, please install the Python requirements in the requirements.txt file, e.g. using pip: `pip install --user -r requirements.txt` * Barista also requires a running version of Caffe. For more information on installing Caffe and obtaining the newest version, please refer to https://github.com/BVLC/caffe and http://caffe.berkeleyvision.org/ **Using Barista** * To start Barista, run the file `main.py` * A user manual as well as a starter's tutorial on Barista can be found in the wiki of the project repository: https://zivgitlab.uni-muenster.de/pria/Barista/wikis/home If you have any questions regarding Barista, please feel free to contact the developers at barista(at)uni-muenster.de ** Citing Barista ** If you use Barista in your research, please cite it using the following reference: Klemm, S. and Scherzinger, A. and Drees, D. and Jiang, X. Barista - a Graphical Tool for Designing and Training Deep Neural Networks. arXiv preprint. arXiv:1802.04626. http://arxiv.org/abs/1802.04626 main.py0000775ڊ00111170000000762113243274037012315 0ustar klemmso0cvpr#!/usr/bin/python2 # To change this license header, choose License Headers in Project Properties. # To change this template file, choose Tools | Templates # and open the template in the prototxt_editor. import os import sys import signal import argparse import logging from backend.barista.project import dirIsProject parser = argparse.ArgumentParser() parser.add_argument('-o', '--open', help='skip Start Dialog and open a Project', type=str, default='') parser.add_argument('-n', '--net', help='start a network client - no GUI', action='store_true') parser.add_argument('-p', '--port', help='local port to receive connections', default='0', type=int) parser.add_argument('-d', '--dir', help='local directory to run all sessions in', type=str, default='.') parser.add_argument('-i', '--ip', help='IP reachable from Barista GUI', type=str) args = parser.parse_args() logging.basicConfig(format='[%(asctime)s] %(levelname)s @ %(funcName)s: %(message)s', datefmt='%d/%m/%Y %H:%M:%S', filename="baristalog.txt", level=logging.INFO) # SHOW_IPY = True SHOW_IPY = False if SHOW_IPY: try: from gui.ipy_widget import show as ipyshow except ImportError as e: print("Cannot load ipython: {}".format(e)) def ipyshow(parent): pass else: def ipyshow(parent): pass if __name__ == "__main__": if(args.net): # validate port if args.port < 0: logging.error("Port should be >=0") sys.stderr.write("Port should be >= 0\n") exit(2) if args.port > 65535: logging.error("Port should be <= 65535") sys.stderr.write("Port should be <= 65535 \n") exit(2) # validate path args.dir = os.path.abspath(args.dir) if not os.path.isdir(args.dir): logging.error("Path '%s' is not valid", args.dir) sys.stderr.write("Path '" + args.dir + "' is not valid\n") exit(3) if not os.access(args.dir, os.W_OK): logging.error("Path '%s' is not writeable", args.dir) sys.stderr.write("Path '" + args.dir + "' is not writeable\n") exit(4) from PyQt5.QtCore import QCoreApplication from backend.networking.barista_server import BaristaServer app = QCoreApplication(sys.argv) server = BaristaServer(app, args.ip, args.port, args.dir) signal.signal(signal.SIGINT, signal.SIG_DFL) else: from backend.barista.utils.settings import applicationQSetting from backend.caffe import path_loader from gui.start_dialog import StartDialog from PyQt5.QtWidgets import QApplication # Get Caffepath out of settings settings = applicationQSetting() settings.beginGroup("Path") path = settings.value("caffePath") app = QApplication(sys.argv) signal.signal(signal.SIGINT, signal.SIG_DFL) # Set global application stylesheet. with open('resources/styles.qss', "r") as stylesFile: stylesheet = stylesFile.read() app.setStyleSheet(stylesheet) # If settings doesnt contain valid caffepath start dialog to change caffepath if not path_loader.checkIfFileIsCaffeRoot(path): from gui.caffepath_dialog import CaffepathDialog caffedlg = CaffepathDialog() caffedlg.show() app.exec_() # ipyshow(startdlg._win.networkManager) startdlg = StartDialog() # Check if directory is a parameter if len(args.open) > 0: if not dirIsProject(args.open): print("The provided path is not a valid Barista project: Please run Barista without flags and create a project first.") sys.exit(2) args.open = os.path.abspath(args.open) startdlg.loadProject(args.open) else: startdlg.show() # Run the application. app.exec_() LICENSE0000664ڊ00111170000000204513243274037012014 0ustar klemmso0cvprCopyright 2017 University of Münster Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.__init__.py0000664ڊ00111170000000016413243274037013120 0ustar klemmso0cvpr# required for providing correct package import, see https://stackoverflow.com/questions/448271/what-is-init-py-for contributors.md0000664ڊ00111170000000114613243274037014067 0ustar klemmso0cvprBarista contributors (sorted alphabeticaly) ============================================ Project management: * Aaron Scherzinger * Dominik Drees * Sören Klemm Developers: * Aleksej Matis * Alexander Hunger * Alexandre Stüben * Anna Plogmaker * Johannes Weidner * Kai Hartz * Markus Schmetkamp * Michael Fujarski * Robin Ortkemper * Robin Rexeisen * Til Sander * Tim Hartz * Alexander Schulze-Schwering * Celien Sophie Bosma * Daniel Beckmann * Frederik Probst * Johannes Stricker * Johannes Toschki * Jonas Schiefner * Joschka Hannes Strüber * Leon Pielage * Lukas Jung * Philipp Torben Jüres * Yves Reker resources/gear.png0000664ڊ00111170000000235713243274037014453 0ustar klemmso0cvprPNG  IHDR@@`UgAMA a cHRMz&u0`:pQ<bKGD#2 pHYs B(xtIME 6:= IDATh͙YHTQhZ&-CA EXTHQ!`؃цD=DDE E/E+IKIhN9m{ιe=sChi"al?M]%-`$+@$c`L@6 UlLWQ*z`0CI\#6p.]#ӂ&LN/<)$Ún>H䛉Fn"?˹A)W$-Q#Vۧ!D~N#t1ǜ~ *ӯcclӬ,0bW &SIB mt "~t̎,IHJ6[LBbNFr+`m&0 w){ a,!Ct)>eRis0lM2743*X2ULCdfC/Ӽ޼$~5M|d2(mGZ4 r| ]r|v7ylB)0WY{:Ho]*\@i7 { e߶ciiZCv/q5uL.0O&0IrO1#♙J0fep\wrrHf]to"iq@âD)!~EEJUKQCœBf*:j-hj^E\$e_CХ')%tEXtdate:create2017-12-19T19:05:54+01:00m%tEXtdate:modify2017-12-19T19:05:54+01:00SLtEXtSoftwarewww.inkscape.org<IENDB`resources/group.svg0000664ڊ00111170000000753413243274037014706 0ustar klemmso0cvprgroup2resources/host_manager.png0000664ڊ00111170000000077213243274037016203 0ustar klemmso0cvprPNG  IHDR% SsBIT|d pHYs " "tEXtSoftwarewww.inkscape.org<wIDATXjA_ Sz|!"1B*{R/5a1tn:Ra0[ͷ3\Nq~ ^(MSIwCeBGUT @ۭMj<Q۬4RUiHUJ#Uk)UC!C[՜uV}>r|K|.Z5 J]5xq[)򕺻k#|,*nt:db6E܉h4Z>hykSp:&^oŹͱ%IL>GQ`TUn~1 ]~{r-ײ[8͈x*,"0"ɮڒ؃jŬ{IENDB`resources/icon.png0000775ڊ00111170000000221113243274037014455 0ustar klemmso0cvprPNG  IHDR@@iqsBIT|d pHYs 1 1L~tEXtSoftwarewww.inkscape.org<IDATxMElO!~DQAP*( Q&)P%TDî_Y?543;3=螪ի73 ]Uxhʯ؁[UO@p:eW W Dʊp0yg˂,qD5^%-x,؂SUg &/麾d!u p ~ gwZu\<+ܪcRu[yxp): +Sc|ik@iozE_]Ǭ، ւg~f& Oތ st9ϵ_i8ۊXW"P (:?,\8 <#Vd>? wy\v0c.awXJ;ǙfhG%|b &Z퇂Q$ $<,E&8J(tXa\y$Et0f(i?A; _ ١a'`6ϡ7eUL7MdN}YnXm:!z X fq,Xw 膀_K婿9,hY?S3|40,)/$kX/ B3wUlGUKR?cL4 ZAcL@}H l\~Տ~~Pc1IENDB`resources/input_manager.png0000664ڊ00111170000000253413243274037016363 0ustar klemmso0cvprPNG  IHDR i;sBIT|d pHYs B(xtEXtSoftwarewww.inkscape.org<IDATHMoY*Wݪ[NL)'c"3 DbE@# 65RKgЌZU!  %ADaח+Y0v'4 MүT:S}= e4k˲wwwӀ1 PXhT*<ۈgg`Gŕ>d2}???ŴmJ6%ɠ:RJhF \ץ8NRz=n4RJU2̷r2BKa͛7{nNJU^z<{,i ,=ZTdaa XYY͑u V7o3>|eYjD5jj/^8f)re!`rr!B("">|@ExG`6KKKT*"ɸoRR4^a$4M! C>E~Ǐ#@JI:0 lfffEQ>փڡ( mc 8JRJ\=R3uR*)]׿رcdC? C?~̍7Ak׮q677YXX`vvvF "0 &&&P$IzaHE{fIל:u/r_իWloo8vc GGt!t1B'ObIG0 2rZ oBHM$AJg8.CQ49s-* r9lfjjꋋ8FϟS"j6|rLc b0T vicX^^&ͲOyN Ƨh ަi8s%3.,ˢX,~q+J~ VUUw۷('?%8fss}6wޥZRe\>4M,BJ˲<(}V/SKJ .pҥ_1v{R""zQA`&i"`bb!|~|Zr},WP[CxMtRT\!B]%ĉ/_U,-۶l6;{ Ð^Gq֖Wm)u}z|4mѲpN r8=#S뺯(JjySǛ32XRIENDB`resources/invalid.png0000664ڊ00111170000000065413243274037015161 0ustar klemmso0cvprPNG  IHDRJ~sgAMA a cHRMz&u0`:pQ<bKGD#2 pHYs B(xtIME %3ZzIDAT8˕ weHMPMȁ#cF/Sp h5A a =!,=UYL y2#WY؟Wޗ" ZLX85[ yD% S{%tEXtdate:create2017-12-08T19:28:37+01:00Տ%tEXtdate:modify2017-12-08T19:28:37+01:007htEXtSoftwarewww.inkscape.org<IENDB`resources/more.png0000664ڊ00111170000002206013243274037014470 0ustar klemmso0cvprPNG  IHDRx pHYs.#.#x?v OiCCPPhotoshop ICC profilexڝSgTS=BKKoR RB&*! J!QEEȠQ, !{kּ> H3Q5 B.@ $pd!s#~<<+"x M0B\t8K@zB@F&S`cbP-`'{[! eDh;VEX0fK9-0IWfH  0Q){`##xFW<+*x<$9E[-qWW.(I+6aa@.y24x6_-"bbϫp@t~,/;m%h^ uf@Wp~<5j>{-]cK'Xto(hw?G%fIq^D$.Tʳ?D*A, `6B$BB dr`)B(Ͱ*`/@4Qhp.U=pa( Aa!ڈbX#!H$ ɈQ"K5H1RT UH=r9\F;2G1Q= C7F dt1r=6Ыhڏ>C03l0.B8, c˱" VcϱwE 6wB aAHXLXNH $4 7 Q'"K&b21XH,#/{C7$C2'ITFnR#,4H#dk9, +ȅ3![ b@qS(RjJ4e2AURݨT5ZBRQ4u9̓IKhhitݕNWGw Ljg(gwLӋT071oUX**| J&*/Tު UUT^S}FU3S ԖUPSSg;goT?~YYLOCQ_ cx,!k u5&|v*=9C3J3WRf?qtN (~))4L1e\kXHQG6EYAJ'\'GgSSݧ M=:.kDwn^Loy}/TmG X $ <5qo</QC]@Caaᄑ.ȽJtq]zۯ6iܟ4)Y3sCQ? 0k߬~OCOg#/c/Wװwa>>r><72Y_7ȷOo_C#dz%gA[z|!?:eAAA!h쐭!ΑiP~aa~ 'W?pX15wCsDDDޛg1O9-J5*>.j<74?.fYXXIlK9.*6nl {/]py.,:@LN8A*%w% yg"/6шC\*NH*Mz쑼5y$3,幄'L Lݛ:v m2=:1qB!Mggfvˬen/kY- BTZ(*geWf͉9+̳ې7ᒶKW-X潬j9(xoʿܔĹdff-[n ڴ VE/(ۻCɾUUMfeI?m]Nmq#׹=TR+Gw- 6 U#pDy  :v{vg/jBFS[b[O>zG499?rCd&ˮ/~јѡ򗓿m|x31^VwwO| (hSЧc3- cHRMz%u0`:o_F[IDATx[]u}9 %@}P@M2qjKәBG}@q cu(igzyq8Т}*xr!$rBQGsֱGHB.{̬a&Y^kfgg ˴] ((((((((( (((((((( (v((((((((( (((((((( ((((((((( /QHrdÓdYEI$Yd{mK$/9ɦ$ϵݐd}'Ydc6;;;|2k3.ɣ˨‚6tr?}]$oK$ܗ{IM35\ II.x}T(^m8ߟ$Ƕ6)[JrG0oAWQ9pN]IM4J)mInIlʨӜƗSH˒\n+e{: IOڇ^nd6iV3Ҝښd9:2*r*9^ IVn$v22Zgng*Ňw*YIzN[,}5VzU~ ༌fHʜ}`i$OB{6`@IN$G'yr@,s=p?@J[ܘa|h.u@fYZSHPӒ\2$h$ g8閼J78KX%Iӫ^ی>ӌǒkoM93:Wo$_wOG_lUu,v(gNL$$lw%gﮖ'WF{ѓ3''KN{Sڹ3|LyBSǃiAdN-mo߾KFҁ/|X`{$gcygg:,,=kesg<IL2S4>ʒΚ4=s9]1emm/5X[oI'4Ot샕uD{Ui G?=ܮخ_uJFmu^9dt-rZwtε:]F{:"c*ZAy|p1sc%Qa3de,]?X_.tYk^#$δTٱGmjZL2PΎ]o3C6$[뫯MB $%yЌtA-\`o:IWdiW9*X'[3 I fY*[/Q1YUx"Q^2jRfN pQZ/ZoH1)l4ѮR[4;;3oN#- w!. 2_{%$Yj5KZHo~ %uKkl9lv}gonZ)ܭ~zN46$9:ƳI޸Z6ot?In @Ol0'R|L9W$U+>-}U> 1U _+ys|_0%u ׵5D7%Qpc tҗ#-EV'9%/v?P$ٻ;DX@3$isKʵ*ɩ3Kv|K|?"(i8GO݃$Ӻd<=<ڃ&ɲ4O \(EeԾ0߆$Jk)ʯx"Ү(.Ȅd]s"szV jtw 񖥘mNOrqYZdN[ҋ.pG}%RO3zPìpE96Bi[m2UgmL+dۧet\%+>S^-<ٮl2\?//پ Zvuo+_ QSr3J-_g tzaχ'dt,TF rzE )+Kz8%\?Ir8u$99Hmj3z̩,] ^#2}~θ;ӾVHn]vܓq#PPmW]jb$V՘ w] 2GnLrR< O9[K~nkgv=&WxjiSٵ^#ks.n~癥5KWV}Ax&0vp%i*/5KR`L%;6?oS ׉ebet7!NـITرInϹ|޵a}$)3r]2Z_Frz~;kӇK 9#I$[|=J|7ehs\^dt9rZy>k{$viq}Um2zCeM79gu5t9{%9=u?InIԭQzF7%-ɿ2:{JWZ[N*]gљ54עh{o*fʨ>r¾i>sr[4_;lIg$kڌ#29=%ɩPILrX-K(Wj \9-I6MIL?J$Ϧ ׉ftaם4_m\FeTNVv((((((((( (((((((( ((((((((( (((((((( ((((((((( `((((((((@r3n-+IENDB`resources/pause.png0000664ڊ00111170000000040013243274037014635 0ustar klemmso0cvprPNG  IHDR pHYs  tIME  /<#tEXtCommentCreated with GIMPWzIDAT8 Et t5 ` E8؄߼6auY׋#0AA5>NZnʷhړ~{7 H-g5Wh=6h>NM} XבTIENDB`resources/pencil.png0000664ڊ00111170000000202613243274037015000 0ustar klemmso0cvprPNG  IHDR@@`UgAMA a cHRMz&u0`:pQ<bKGD#2 pHYs B(xtIME  8IDAThKhADԶI{DAmAЛmѤڇz/xEA|UII~2`'GwMx]T\@KY+I@7]N/x=M@5 'Mm{^J!/ahS{@.ʡ&~J*xBB{I qftUZ8Q2,.5v5?SjP&vˆn@@%mSRh6>ψ3 (GϽ@䲡!O9}x@r1^ \m),Sڞ޴-&4*h_V) N/Cp`D8^p9Җk)r ^˦.$%ŋOU0*?^%tEXtdate:create2017-12-19T19:09:10+01:007> %tEXtdate:modify2017-12-19T19:09:10+01:00jtEXtSoftwarewww.inkscape.org<IENDB`resources/property.svg0000664ڊ00111170000000173213243274037015430 0ustar klemmso0cvprpropertyresources/snap.png0000664ڊ00111170000000160413243274037014470 0ustar klemmso0cvprPNG  IHDR bKGDD< pHYs  tIME 1!tEXtCommentCreated with GIMPWIDAT8TMHTQ}o<g&'K #ȍc~E eM Ҧ4iAs) Yh&#Fi{ۢLm :p|sGD"D~Kg9P[>3N~"H(6~z(=~@pɛx\ oi) 21GϳȱlCPU0 vZ+++ L45F+^_O2/_T}ZƆ` @}5@@t8^Ԁ[ aRٺMAL&$qfZtt^Cݞ7l܈{wbC##̸($cy3 "BݎO@S.;4}tvue6::jGog{bB!a͚ŷld z S]SeA{ y<[{v! =6SӴ4UT%zj`*c@pB1[yEvu=(J[7oĒE>QϞ>r\O$2\NaZT*Mx:%^tsWC; Yb xH$$iWpAdIaj.տ-0T@]{#)C>BbZ@dS$6,K4-u۶d2PĀ9gtN#t9?swJЅ*IENDB`resources/sources.txt0000664ڊ00111170000000042313243274037015243 0ustar klemmso0cvprpencil.png from https://www.flaticon.com/authors/bogdan-rosu from www.flaticon.com is licensed by Creative Commons BY 3.0 gear.png, trash.png, valid.png and invalid.png from https://www.flaticon.com/authors/google from www.flaticon.com is licensed by Creative Commons BY 3.0 resources/start.png0000664ڊ00111170000000071713243274037014670 0ustar klemmso0cvprPNG  IHDR pHYs  tIME  / >YtEXtCommentCreated with GIMPWIIDAT8˵սjPs$-wS׼ n""mI77( P>BL6C*dI ^0y)R%0 `0xg$I{0^ň dY1V p@ ,aDn3cN1\$a3e/a+q`-qoryC:LQ7qsl6V !v'I2 ^+O'!",˟8o(EqieY3۶#Dji)"t]ϰHURG㔫:Vt[Dkf&IENDB`resources/stop.png0000664ڊ00111170000000035113243274037014512 0ustar klemmso0cvprPNG  IHDR pHYs  tIME  /22OtEXtCommentCreated with GIMPWcIDAT8픱 0 +%cdْ4 w7b'up57vFeo]WOI`843\ X>yB QIENDB`resources/styles.qss0000664ڊ00111170000000421113243274037015071 0ustar klemmso0cvpr/****************************************************************/ /* Below are global styles that can be used throughout the app. */ /****************************************************************/ QLabel#infoText { color: rgba(0, 0, 0, 125); } #noIndicator::menu-indicator { width: 0px; } /***************************************************/ /* Below are styles affecting the properties dock. */ /***************************************************/ QTreeView#properties::item { padding: 1px; min-height: 25px; } /*************************************************/ /* Below are styles affecting the input manager. */ /*************************************************/ QLabel#databaseLabel { background-color: #FAFAFA; border-style: solid; border-width: 1px; border-color: #555555; border-radius: 5px; text-align: center; } QLabel#databaseType { background-color: #EEEEEE; border-style: solid; border-width: 1px; border-color: #555555; border-radius: 5px; text-align: center; } QLabel#databaseStatus { background-color: #EE0000; color: #EEEEEE; border-style: solid; border-width: 1px; border-color: #555555; border-radius: 5px; text-align: center; } /*******************************************************/ /* Below are styles affecting the session item widget. */ /*******************************************************/ QPushButton#sessionBtn { background-color: #EEEEEE; border-style: solid; border-width: 1px; border-color: #555555; border-radius: 5px; text-align: center; } QPushButton#sessionBtn:hover { background-color: #BBBBBB; } QPushButton#sessionBtn:!hover { background-color: #EEEEEE; } QPushButton#sessionBtn:focus { background-color: #EEEEEE; } QPushButton#sessionBtn::menu-indicator { width: 0; height: 0; image: 0; } QLabel#sessionId { background-color: #EEEEEE; border-style: solid; border-width: 1px; border-color: #555555; border-radius: 5px; text-align: center; } QLabel#sessionStatus, QLabel#invalidMsg { background-color: #FFF; border-style: solid; border-width: 1px; border-color: #FFF; border-radius: 5px; text-align: center; color: #FFF; } resources/trash.png0000664ڊ00111170000000106113243274037014645 0ustar klemmso0cvprPNG  IHDR@@`UgAMA a cHRMz&u0`:pQ<bKGD#2 pHYs B(xtIME h"IDAThٱ @Ca&L!8 XYJB%lY(Ml4lȑ{TqwGL*D 4r"aj!lv{zmǪ ɣ{߁+eRv)'>2SBi+6pPx?C     OJث UbZ"A5u*dǟI[X#0V9PZ /m~)R$" +%tEXtdate:create2017-12-19T19:04:19+01:00Qi%tEXtdate:modify2017-12-19T19:04:19+01:00 HtEXtSoftwarewww.inkscape.org<IENDB`resources/valid.png0000664ڊ00111170000000070313243274037014625 0ustar klemmso0cvprPNG  IHDRJ~sgAMA a cHRMz&u0`:pQ<bKGD#2 pHYs B(xtIME (k IDAT8; @"r1{i4AR/3h5k8+N9|?뚍yA,¯Yy4\9y!#1sx#Z~Vr]'?:OK*"7LJ~|i`Sq%tEXtdate:create2017-12-08T19:27:40+01:00%tEXtdate:modify2017-12-08T19:27:40+01:00[rtEXtSoftwarewww.inkscape.org<IENDB`gui/main_window/docks/properties/property_widgets/__init__.py0000664ڊ00111170000000000013243274037025115 0ustar klemmso0cvprgui/main_window/docks/properties/property_widgets/data.py0000664ڊ00111170000010217713243274037024311 0ustar klemmso0cvprfrom PyQt5.QtCore import QObject, pyqtSignal import backend.caffe.proto_info as info from gui.main_window.docks.properties.property_widgets.types import * class ReasonUpdated: """ This class represents an update of a value in PropertyDataObject """ def __init__(self, oldValue, newValue): self.oldValue = oldValue self.newValue = newValue def __str__(self): return "Updated {} to {}".format(self.oldValue, self.newValue) def __eq__(self, other): if type(other) != type(self): return False return other.oldValue == self.oldValue and other.newValue == self.newValue class ReasonAdded: """ This class represents a push in a list in PropertyDataListObject """ def __init__(self, idx): self.idx = idx def __str__(self): return "Added {}".format(self.idx) def __eq__(self, other): if type(other) != type(self): return False return other.idx == self.idx class ReasonRemoved: """ This class represents a remove in a list in PropertyDataListObject """ def __init__(self, idx, val): self.idx = idx self.value = val def __str__(self): return "Removed {} - value {}".format(self.idx, self.value) def __eq__(self, other): if type(other) != type(self): return False return other.idx == self.idx class ReasonGiven: """ This class represents a new given value in PropertyData """ def __init__(self, name): self.name = name def __str__(self): return "Given {}".format(self.name) def __eq__(self, other): if type(other) != type(self): return False return other.name == self.name class ReasonUngiven: """ This class represents a value in PropertyData which became deleted """ def __init__(self, name): self.name = name def __str__(self): return "Ungiven {}".format(self.name) def __eq__(self, other): if type(other) != type(self): return False return other.name == self.name class ReasonKeyAdded: """ This class represents a key/value in PropertyDataDictObject which was added """ def __init__(self, key): self.key = key def __str__(self): return "Key added {}".format(self.key) def __eq__(self, other): if type(other) != type(self): return False return other.key == self.key class ReasonKeyDeleted: """ This class represents a key/value in PropertyDataDictObject which was deleted """ def __init__(self, key): self.key = key def __str__(self): return "Key deleted {}".format(self.key) def __eq__(self, other): if type(other) != type(self): return False return other.key == self.key class PropertyInfoBuilder: def buildInfo(self, name, protoparameter, uri=None): """ This function should build an info object (something which behaves like PropertyInfo) for the given parameter. name is the name of the info object protoparameter is any object which helps build the info object uri is a list whicht describes the position of this info """ pass class PropertyInfo(object): """ This Class gives infos about a type in properties """ def typeEntryString(self): """ Give the type as String. If the type is a list/dictionary, returns the type of the entries. Only needed if entryInfo won't be overwritten """ pass def entryInfo(self): """ Returns a info object which describes a entry object if the type is a list/dictionary """ return PropertyInfoListEntryWrapper(self) def typeString(self): """ Give the type as String (see StringType, IntType,...)""" pass def enumOptions(self): """ A list of options if enum """ pass def name(self): """ Return the name of this property """ pass def deprecated(self): """ Return true iff this property is deprecated """ pass def description(self): """ Return a decription of this property """ pass def prototype(self): """ Return a raw value of a default type. The result may be changed in other function, so you may want to return a copy of your prototype """ pass def subparameter(self): """ Should return a dictionary which keys are the subparameter and its values are the protoparameter for your PropertyInfoBuilder and can be from any type. E.g. {"subparameter1": SomeCaffeMetaInfosForExampleIfInfoBuilderWantsThis, "subparameter2": MaySomeVeryDifferentData } """ pass def isEditable(self): pass def isRequired(self): pass class PropertyInfoListEntryWrapper(PropertyInfo): """ Wraps info such that typeString returns the typeEntryString. Useful for dictionary or list subtypes """ def __init__(self, info): self._info = info #type: PropertyInfo def base(self): """ Return the base info-object this object wraps """ return self._info def typeString(self): return self._info.typeEntryString() def typeEntryString(self): return self._info.typeEntryString() def entryInfo(self): raise LookupError("EntryInfo should not be used twice by using PropertyInfoListEntryWrapper") def enumOptions(self): return self._info.enumOptions() def name(self): return self._info.name() def deprecated(self): return self._info.deprecated() def description(self): return self._info.description() def prototype(self): return self._info.prototype() def isEditable(self): return self._info.isEditable() def isRequired(self): return self._info.isRequired() def subparameter(self): return self._info.subparameter() def propertyObjectForInfo(info): """ Returns the subtype of PropertyDataObject which should be used with the given type in info """ if info.typeString() == GroupType: return PropertyDataGroupObject if info.typeString() == ListType: return PropertyDataListObject if info.typeString() == DictType: return PropertyDataDictObject return PropertyDataObject def dataBuild(parentContainer, idx, info, infobuilder, subtype=None, parent=None, uri=[], signalcontroller = None): """ Builds the value from the raw value given by the parent container (e.g. Dictionary or List) and idx. If raw value is primitive, returns a primitive. If value is a list, returns a list of PropertyData/-Object. If value is Group-Type, returns a PropertyData. info is from type PropertyInfo and contains the information of parentContainer[idx]. infobuilder: instance of PropertyInfoBuilder for generating meta informations parent is a PropertyDataObject which someChildPropertyChanged will be connected with its childs. uri is a identifier for this entry in the whole dictionary (type: list) """ # Here we build Data for an entry of an container (e.g. list, dict). # Without subtype=="property" an primitive can be returned, but here # this primitive is capsuled in a PropertyDataObject if subtype == "Property": PData = propertyObjectForInfo(info) res = PData(info, parentContainer, idx, infobuilder, uri=uri, signalcontroller=signalcontroller) if parent: res.someChildPropertyChanged.connect(parent.someChildPropertyChanged) return res # Group-Type -> Build PropertyData and connect change-signal-propagation if info.typeString() == GroupType: res = PropertyData(parentContainer[idx],info.subparameter(),infobuilder, uri=uri, signalcontroller=signalcontroller) if parent: res.someChildPropertyChanged.connect(parent.someChildPropertyChanged) return res # List -> Recursive build data for every entry # and connect change-signal-propagation for each if info.typeString() == ListType: res = [] for i in range(len(parentContainer[idx])): data = dataBuild(parentContainer[idx], i, info.entryInfo(), infobuilder, "Property", uri=uri+[i], signalcontroller=signalcontroller) if parent: data.someChildPropertyChanged.connect(parent.someChildPropertyChanged) res.append(data) return res # Dictionary -> Recursive build data for every key # and connect change-signal-propagation for each if info.typeString() == DictType: res = {} me = parentContainer[idx] for key in me: data = dataBuild(me, key, info.entryInfo(), infobuilder, "Property", uri=uri+[key], signalcontroller=signalcontroller) if parent: data.someChildPropertyChanged.connect(parent.someChildPropertyChanged) res[key] = data return res return parentContainer[idx] import traceback import sys class SignalController(object): """ Class which can delay signals """ def __init__(self): self.__locked = 0 # Counter how much lock-calls are present self._cache = [] # Cache every Signal, which is not emitted yet self.__loging = False # Debugging -> Logging to console def enableLoging(self): """ Let this object print a stack-trace on every emitLater call (Debugging purpose) """ self.__loging = True def disableLoging(self): """ Stop this object from printing a stack-trace on every emitLater call (Debugging purpose) """ self.__loging = False def isLocked(self): return self.__locked > 0 def emitLater(self, signal, *args): """ emit the signal if not locked, otherwhise wait until unlock """ if self.__loging: print("Signal log - Traceback") for item in traceback.extract_stack(): print(item) fun = lambda: signal.emit(*args) if self.isLocked(): self._cache.append(fun) else: fun() def lock(self): """ lock the controller, such that the signals get emit when it's unlocked """ self.__locked += 1 def unlock(self): """ unlock the controller and emit all cached signals """ if self.__locked == 0: return self.__locked -= 1 if not self.isLocked(): tmp = self._cache self._cache = [] for fun in tmp: fun() def lockWhileFunction(self,fun): """ Lock this object - then call fun - then unlock this object. It can happened that the developer forgot to unlock for example when the function fun returns at some point. It is saver to use the function with lockWhileFunction """ self.lock() res = fun() self.unlock() return res class PropertyDataObject(QObject): """ Represent a property with a value """ propertyChanged = pyqtSignal(object, object) # second argument is reason for changing someChildPropertyChanged = pyqtSignal(list, object) def __init__(self, info, parentcontainer, idx, infobuilder, uri=[], parent=None, signalcontroller=None): """ info: PropertyInfo like object, parentdict: the dictionary/list containing this value idx: the idx, can be integer or string uri: a identifier for this property e.g. uri = ['exclude', 2, 'phase'] infobuilder: instance of PropertyInfoBuilder singalController: instance of SignalController for delay signals """ super(PropertyDataObject, self).__init__(parent) self._uri = uri self._info = info #type: PropertyInfo self._parentcontainer = parentcontainer #type: list self._idx = idx self._propvalue = None self.propertyChanged.connect(lambda _, reason: self.someChildPropertyChanged.emit(self._uri, reason)) self._infobuilder = infobuilder self._signalcontroller = signalcontroller # type: SignalController def signalController(self): """ Returns an object which can delay signals. It will delay signals for all docks in this hierarchy. That means that the child and parents DataObjects share this signalController and locking it will cause every object here to delay its signals. """ return self._signalcontroller def uri(self): return self._uri def _setRawValue(self, val): """ Set the value in the wrapped dictionary """ self._parentcontainer[self._idx] = val def _rawValue(self): """ Get the value from the wrapped dictionary """ return self._parentcontainer[self._idx] def _buildValue(self): """ Build the suitable PropertyData.. for the represented value """ res = dataBuild(self._parentcontainer, self._idx, self._info,self._infobuilder, parent=self, uri=self.uri(), signalcontroller=self._signalcontroller) return res def info(self): """Should return an PropertyInfo like Object""" return self._info def value(self): """ Return the value. If it is primitiv, the primitiv. If it is a group returns a PropertyData. If it is repeated returns a list of the PropertyDataObject """ if self._propvalue is None: self._propvalue = self._buildValue() return self._propvalue def listGetValue(self): """ Return the value, which should be returned on ___getitem___ of the parent""" return self.value() def setValue(self, value): """ Update the value of this PropertyDataObject. value should be a raw type not something like another PropertyDataObject. If the value has a wrong type an exception get raised. E.g. propertydata.setValue(42) propertydata.setValue(["a","b"]) """ if not checkIsType(value, self.info().typeString()): raise ValueError("Expected {} found type {} at {}".format(self.info().typeString(), type(value),self._uri)) oldVal = self._rawValue() if oldVal == value: return self._setRawValue(value) self._propvalue = None self._emitChange(value, ReasonUpdated(oldVal, value)) def _emitChange(self, value, reason): """ Emit change signal with support for SignalController """ if self._signalcontroller: self._signalcontroller.emitLater(self.propertyChanged, value, reason) else: self.propertyChanged.emit(value, reason) def valueOf(self, relative_uri): """ Return the value of a child element. relative_uri is the uri (a dictionary with the path like ["prop1", "prop2"]) which is relative to this PropertyDataObject. E.g. if we are prop1 the relative uri would look like ["prop2"]. If no such element found None will be returned. """ val = self.value() # We want this object if relative_uri == []: return val # We want a child object idx = relative_uri[0] if isinstance(val, PropertyData): return val.valueOf(relative_uri) # We have no childs => invalid => return None return None def __str__(self): return "{} at {}".format(self.__class__.__name__,self.uri()) def prototypeOfType(info): """ Return some default value for the given type in info. Useful for a entry in a list. """ info = info #type: PropertyInfo entrytype = info.entryInfo().typeString() # May return info.entryInfo().prototype() a better solution?! if entrytype == StringType: return "" if entrytype == BoolType: return False if entrytype == IntType: return 0 if entrytype == FloatType: return 0.0 if entrytype == GroupType: return {} if entrytype == EnumType: return info.enumOptions()[0] if entrytype == UnknownType: return None if entrytype == DictType: return {} class PropertyDataBuildingException(Exception): pass class PropertyDataGroupObject(PropertyDataObject): """ Represent a PropertyDataObject containing PropertyData It pass through the getitem, setitem, deltitem, contains operations """ def listGetValue(self): """ Return the value, which should be returned on ___getitem___ of the parent""" return self.value() def __getitem__(self, idx): return self.value()[idx] def __setitem__(self, idx, val): self.value()[idx] = val def __delitem__(self, idx): del self.value()[idx] def __contains__(self,key): return key in self.value() class PropertyDataDictObject(PropertyDataObject): """ Represent a PropertyDataObject containing a dictionary""" class itemiter(object): def __init__(self, parent): self.parent = parent # type: PropertyDataDictObject def __iter__(self): for key in self.parent: yield key, self.parent[key] def iteritems(self): return PropertyDataDictObject.itemiter(self) def keys(self): return self._rawValue().keys() def __getitem__(self, idx): return self.value()[idx] def __setitem__(self, idx, val): if idx in self: if self.info().entryInfo().typeString() in [ListType, GroupType]: # We can not change PropertyData e.g. because # they are built by a fix procedure raise ValueError("Can only change primitive types through this function, not list or groups") self.value()[idx].setValue(val) else: self._rawValue()[idx] = val newdata = dataBuild(self._rawValue(), idx, self.info().entryInfo(), self._infobuilder, "Property", uri=self.uri()+[idx], signalcontroller=self._signalcontroller) newdata.someChildPropertyChanged.connect(self.someChildPropertyChanged) self._propvalue[idx] = newdata #self._propvalue = self._buildValue() self._emitChange(self.value(), ReasonKeyAdded(idx)) def __delitem__(self, idx): v=self._rawValue()#type: list del v[idx] #self._propvalue = self._buildValue() del self._propvalue[idx] self._emitChange(self.value(), ReasonKeyDeleted(idx)) def __contains__(self,key): return key in self.value() def __iter__(self): for key in self.value(): yield key def getBoringDict(self): """ Return the underlying dictionary this object operates on """ return self._rawValue() def listGetValue(self): """ Return the value, which should be returned on ___getitem___ of the parent""" return self def setValue(self, value): if not checkIsType(value, self.info().typeString()): raise ValueError("Expected {} found type {} at {}".format(self.info().typeString(), type(value), self._uri)) oldVal = self._rawValue() if oldVal == value: return self._setRawValue(value) self._propvalue = self._buildValue() self._emitChange(self.value(), ReasonUpdated(oldVal, value)) def valueOf(self, relative_uri): val = self.value() # We want this object if relative_uri == []: return val # We want a child object idx = relative_uri[0] # Uri element id not an index or not valid if type(idx) != str or not idx in self: return None # Call the function of child with right relative uri if isinstance(val[idx], PropertyDataObject) or isinstance(val[idx], PropertyData): return val[idx].valueOf(relative_uri[1:]) # If the child is a primitive value but the uri want to # step into the next element => Invalid => return None return None class PropertyDataListObject(PropertyDataObject): """ Represent a property with multiple values """ def listGetValue(self): """ Return the value, which should be returned on ___getitem___ of the parent""" return self def pushBack(self, value): """ Append the raw value to the list """ idx = len(self._rawValue()) self._rawValue().append(value) if self._propvalue is None: self._propvalue = self._buildValue() else: newdata = dataBuild(self._rawValue(), idx, self.info().entryInfo(), self._infobuilder, "Property", uri=self.uri()+[idx], signalcontroller=self._signalcontroller) newdata.someChildPropertyChanged.connect(self.someChildPropertyChanged) self._propvalue.append(newdata) #self._propvalue = self._buildValue() self._emitChange(self.value(), ReasonAdded(len(self._rawValue())-1)) def removeObject(self, idx): v=self._rawValue()#type: list oldval = v[idx] del v[idx] self._propvalue = self._buildValue() self._emitChange(self.value(), ReasonRemoved(idx, oldval)) def insert(self, idx, val): v=self._rawValue()#type: list v.insert(idx,val) self._propvalue = self._buildValue() self._emitChange(self.value(), ReasonAdded(idx)) def remove(self, val): v=self._rawValue()#type: list idx = v.index(val) v.remove(val) self._propvalue = self._buildValue() self._emitChange(self.value(), ReasonRemoved(idx, val)) def index(self,val): """ Returns the index of val """ return self._rawValue().index(val) def setValue(self, value): if not checkIsType(value, self.info().typeString()): raise ValueError("Expected {} found type {} at {}".format(self.info().typeString(), type(value), self._uri)) oldVal = self._rawValue() if oldVal == value: return self._setRawValue(value) self._propvalue = self._buildValue() self._emitChange(self.value(), ReasonUpdated(oldVal, value)) def prototypeValue(self): """ Returns a raw value (not something like PropertyDataObject) from an new object in this list """ return prototypeOfType(self.info()) def valueOf(self, relative_uri): """ Return the value of a child element. relative_uri is the uri (a dictionary with the path like ["listprop", 1, "prop2"]) which is relative to this PropertyDataListObject. E.g. if we are listprop the relative uri would look like [1, "prop2"]. If no such element found None will be returned. """ val = self.value() # We want this object if relative_uri == []: return val # We want a child object idx = relative_uri[0] # Uri element id not an index or not valid if type(idx) != int or idx < 0 or idx >= len(val): return None # Call the function of child with right relative uri if isinstance(val[idx], PropertyDataObject) or isinstance(val[idx], PropertyData): return val[idx].valueOf(relative_uri[1:]) # If the child is a primitive value but the uri want to # step into the next element => Invalid => return None return None def __getitem__(self, idx): item = self.value()[idx] return item.listGetValue() def __setitem__(self, idx, val): item = self.value()[idx] if item.info().typeString() in [ListType, GroupType]: # We can not change PropertyData e.g. because # they are builded by a fix procedure raise ValueError("Can only change primitive types through this function, not list or groups") item.setValue(val) def __delitem__(self, idx): self.removeObject(idx) def __iter__(self): for el in self.value(): yield el.listGetValue() def __len__(self): return len(self._rawValue()) def __contains__(self, item): for el in self: if el == item: return True return False class PropertyData(QObject): """ Reresent a collection of properties """ # Gets emit, whenever some property get changed # Get called with an identifier list which looks like this # ["propname","propnameoflist",3,"antoherchild","someItem"] # and a object which represent the reason for emitting # (e.g. ReasonUpdated, ReasonGiven, ...) someChildPropertyChanged = pyqtSignal(list, object) # propertyGotGiven (value: PropertyDataObject) propertyGotGiven = pyqtSignal(object) # propertyGotUnGiven (name: str) propertyGotUnGiven = pyqtSignal(str) def __init__(self, valuedict, metadict, infobuilder, uri=[],parent=None, signalcontroller = None): """ valudict: Recursive list of {key: value} metadict: List of {key: protoinfo.Parameter } infobuilder: instance of PropertyInfoBuilder which generates the meta informations used here """ super(PropertyData, self).__init__(parent) if type(valuedict) != dict: raise PropertyDataBuildingException("valuedict should be a dict, is {} instead at {}".format(type(valuedict), uri)) self.valuedict = valuedict #type: dict if type(metadict) != dict: raise PropertyDataBuildingException("Metadict should be a dict, is {} instead at {}".format(type(metadict), uri)) self.metadict = metadict #type: dict self._buildProps = None self._uri = uri self._infobuilder = infobuilder #type: PropertyInfoBuilder self._signalcontroller = signalcontroller #type: SignalController self.__allavailprops = None def __str__(self): return "{} at {}".format(self.__class__.__name__,self.uri()) def uri(self): return self._uri def getBoringDict(self): """ Return the dictionary this class writes and read on """ return self.valuedict def valueOf(self, relative_uri): """ Return the value of a child element. relative_uri is the uri (a dictionary with the path like ["prop1", "listprop", 1, "prop2"]) which is relative to this PropertyData. E.g. if we are in PropertyData of prop1 the list would look like ["listprop", 1, "prop2"]. If no such element found None will be returned. """ if relative_uri == []: return None for el in self.givenProperties(): #type: PropertyDataObject if el.info().name() == relative_uri[0]: return el.valueOf(relative_uri[1:]) return None def signalController(self): """ Returns an object which can delay signals. It will delay signals for all docks in this hierarchy. That means that the child and parents DataObjects share this signalController and locking it will cause every object here to delay its signals. """ return self._signalcontroller def __buildProperty(self, name, meta): meta = meta #type: info.Parameter uri = self.uri()+[name] info = self._infobuilder.buildInfo(name, meta, uri) PropertyType = propertyObjectForInfo(info) res = PropertyType(info, self.valuedict, name, self._infobuilder, uri=uri, signalcontroller = self._signalcontroller) res.someChildPropertyChanged.connect(self.someChildPropertyChanged) return res def __buildProperties(self): res = [] for el in self.valuedict: res.append(self.__buildProperty(el, self.metadict[el])) return res def giveProperty(self, name, value): """ Add the property "name" to the given one and sets its value to "value". value should be the raw value, not something like PropertyDataObject. If the property is already given, it gets overwritten """ item = self.getChildDataObject(name) allprops = self.allAvailableProperties() if not name in allprops: raise ValueError("Unexpected key {} in {}".format(name,self._uri)) if not checkIsType(value,allprops[name].typeString()): raise ValueError("Expected {} found type {}".format(allprops[name].typeString(), type(value))) if item: item.setValue(value) return self.valuedict[name] = value if self._buildProps is None: self.__buildProperties() else: newProperty = self.__buildProperty(name, self.metadict[name]) self._buildProps.append(newProperty) self._emitGiven(newProperty, name) def givePropertyDefault(self, name): """ Like giveProperty but instead of give an explicit value the property will be initialize with a default value (info.prototype() of this property). If the value name is already given, nothing happen. """ if item in self: return props = self.allAvailableProperties() #type: Dict[str,PropertyInfo] if not name in props.keys(): raise ValueError("Property {} is not a valid one".format(name)) return self.giveProperty(name, props[name].prototype()) def _emitGiven(self, newProp, name): """ Emit someChildPropertyChanged-signal for giving a property with support for SignalController """ if self._signalcontroller: self._signalcontroller.emitLater(self.propertyGotGiven, newProp) self._signalcontroller.emitLater(self.someChildPropertyChanged, self.uri()+[name], ReasonGiven(name)) else: self.propertyGotGiven.emit(newProp) self.someChildPropertyChanged.emit(self.uri()+[name], ReasonGiven(name)) def ungiveProperty(self, name): """ Unset the property with the given name """ del self.valuedict[name] for idx, el in enumerate(self._buildProps): if el.info().name() == name: del self._buildProps[idx] break self._emitUngiven(name) def _emitUngiven(self, name): """ Emit someChildPropertyChanged-signal for ungiving a property with support for SignalController """ if self._signalcontroller: self._signalcontroller.emitLater(self.propertyGotUnGiven, name) self._signalcontroller.emitLater(self.someChildPropertyChanged, self.uri()+[name], ReasonUngiven(name)) else: self.propertyGotUnGiven.emit(name) self.someChildPropertyChanged.emit(self.uri()+[name], ReasonUngiven(name)) def allAvailableProperties(self): """ Returns a dictionary, which looks like this: { "name_of_property1": PropertyInfo1,...} """ if self.__allavailprops: # Lazy initialization return self.__allavailprops res = {} for key in self.metadict: # type: str res[key] = self._infobuilder.buildInfo(key, self.metadict[key], self.uri()+[key]) self.__allavailprops = res return res def givenProperties(self): """ Returns a list with infos about all properties: Structure looks like this: [ PropertyDataObject(), ... ] """ if self._buildProps is None: self._buildProps = self.__buildProperties() return self._buildProps def getChildDataObject(self, key): """ Return the child PropertyDataObject of the given key """ props = self.givenProperties() for element in props: # type: PropertyDataObject if element.info().name() == key: return element return None def __getitem__(self, key): """ Returns the value of the child object with the given key, if it's given """ el = self.getChildDataObject(key) if el is None: raise ValueError("PropertyData does not have the value {} defined at {}".format(key, self._uri)) return el.listGetValue() def __setitem__(self, key, val): """ Sets the property key to the value val, if it is a primitive (not a list or another group) """ allProps = self.allAvailableProperties() if not key in allProps: raise ValueError("Property {} cannot be set this property is not valid".format(key)) info = self.allAvailableProperties()[key] if info.typeString() in [ListType, GroupType]: # We can not change PropertyData e.g. because # they are builded by a fix procedure raise ValueError("Can only change primitive types through this function, not list or groups") self.giveProperty(key,val) def __contains__(self,key): """ Return true iff the value key is a given property """ props = self.givenProperties() for element in props: #type: PropertyDataObject if element.info().name() == key: return True return False def __delitem__(self, key): self.ungiveProperty(key) def __iter__(self): props = self.givenProperties() for element in props: #type: PropertyDataObject yield element.info().name() gui/main_window/docks/properties/property_widgets/generic_info.py0000664ڊ00111170000001654513243274037026032 0ustar klemmso0cvprimport copy from gui.main_window.docks.properties.property_widgets import data from gui.main_window.docks.properties.property_widgets.types import * class GenericBuilderException(Exception): pass class GenericInfoBuilder(data.PropertyInfoBuilder): """ This class can can build infos from a description which is a dictionary which looks like this: { "type": GroupType, # type information, not optional "name": "Bla", # name of this property, not optional "prototype": {}, # a value of which a copy will be a default value, not optional. If it is a function, the function get called, but nothing will be copied then "entries": [ # if Group, we have need the infos of the entries, not optional for this type { "name": "BlaSub", "type": IntType, # type is int "prototype": 0 # the default value }, { "name": "BlaSubList", "type": ListType, # A list "prototype": [] "entry": { # A list contains homogeneous entries # Which should be described here "name": "ListEntry", "type": IntType, "prototype": 0 } # alternatively to entry we can set # "typeEntry": IntType }, { "name": "BlaSubDict", "type": DictType, # A Dictionary "prototype": {} "entry": { # A dictionary contains homogeneous values # (and String-Keys) # Which should be described here "name": "DictEntry", "type": FloatType, "prototype": 0.2 } # alternatively to entry we can set # "typeEntry": FloatType }, { "name": "SomeEnum", "type": EnumType, "prototype": "A", "enumOptions": ["A","B"] }, { # If we want to redirect infos to another # builder, we can use the "builder"-attribute. # With the "info"-entry, this class will represents the info. # If the argument is a function, the function get called # If info and builder are set, there is no need for other keys "info": SomeOtherInfoClass # Builder should always been set if info is set and the type in info is GroupType "builder": SomeOtherInfoBuilder, # A function changes the uri for # the builder. # If not set the builder gets the whole uri "urifunc": lambda uri: uri[:3] } ] # Other available entries have the same name # as the function in PropertyInfo class } """ def __init__(self, infodescritpion): self._infodescritpion = infodescritpion def buildInfo(self, name, protoparameter, uri=None): descr = self._infodescritpion def req(name): try: return descr[name] except KeyError: raise GenericBuilderException("Key '{}' needed for {}".format(name, descr)) for i, uripart in enumerate(uri): if "info" in descr: descrtype = descr["info"].typeString() else: descrtype = req( "type" ) if not descrtype in [ListType, GroupType, DictType]: raise GenericBuilderException("Cannot build info for {} because {} is a primitive".format(uri, uri[:i+1])) if "builder" in descr: urifunc = lambda x: x if "urifunc" in descr: urifunc = descr["urifunc"] return descr["builder"].buildInfo(name, protoparameter, urifunc(uri)) elif descrtype == GroupType: entries = descr["entries"] for entry in entries: try: if "info" in entry: if entry["info"].name() == uri[i]: descr = entry break elif entry["name"] == uri[i]: descr = entry break except KeyError: raise GenericBuilderException("Key 'name' needed for {}".format(entry)) elif descrtype in [ListType, DictType]: descr = req( "entry" ) if "info" in descr: return descr["info"] else: return GenericInfoWrapper(descr) def buildRoot(self): if "info" in self._infodescritpion: return self._infodescritpion["info"] return GenericInfoWrapper(self._infodescritpion) class GenericInfoWrapper(data.PropertyInfo): """ Builds PropertyInfo on a info description (see the dictionaries GenericInfoBuilder) """ def __init__(self, description): self._description = description def name(self): return self.__requiredType("name") def typeEntryString(self): if "typeEntry" in self._description: return self._description["typeEntry"] return self.typeString() def typeString(self): return self.__requiredType("type") def __requiredType(self, name): # Make own exception if key does not exist try: return self._description[name] except KeyError: raise GenericBuilderException("Entry {} needed in info-description".format(name)) def __entryOr(self, name, default): # Returns default if name is not in self._description if name in self._description: return self._description[name] return default def entryInfo(self): if "entry" in self._description: entry = self._description["entry"] if "info" in entry: return entry["info"] return GenericInfoWrapper(entry) else: return super(GenericInfoWrapper, self).entryInfo() def enumOptions(self): return self.__entryOr("enumOptions", []) def deprecated(self): return self.__entryOr("deprecated", False) def description(self): return self.__entryOr("description", "") def prototype(self): prototype = self.__requiredType("prototype") if type(prototype) == function: return prototype() return copy.deepcopy(prototype) def subparameter(self): if self.typeString() == GroupType: subdescr = self.__requiredType("entries") res = {} for descr in subdescr: # We set them to None, because GenericInfoBuilder # does not handle protoparameter directly. # So every value should be fine if "info" in descr: res[descr["info"].name()] = None elif "name" in descr: res[descr["name"]] = None else: raise GenericBuilderException("{} needs a name".format(descr)) return res return [] def isEditable(self): return self.__entryOr("isEditable", True) def isRequired(self): return self.__entryOr("isRequired", True) gui/main_window/docks/properties/property_widgets/layer_info.py0000664ڊ00111170000000663113243274037025525 0ustar klemmso0cvprimport backend.caffe.proto_info as info from gui.main_window.docks.properties.property_widgets import data from gui.main_window.docks.properties.property_widgets.types import * class LayerPropertyInfoBuilder(data.PropertyInfoBuilder): def buildInfo(self, name, protoparameter, uri): if uri == []: return LayerParamInfo(name, protoparameter) return PropertyInfo(name, protoparameter, uri) def buildRootInfo(self, name, type): """ Builds a info object for the whole layer type should be an instance of ProtoInfo.LayerType name is the name which the info type should have """ params = type.parameters() return self.buildInfo(name, params, []) class PropertyInfo(data.PropertyInfo): """ This Class gives infos about a type in properties """ def __init__(self, name, meta,uri): self._name = name #type: str self.meta = meta #type: info.Parameter self.uri = uri #type: list def typeEntryString(self): """ Give the type as String. If the type is a list, returns the type of the entries. """ if self.meta.isInt(): return IntType if self.meta.isDouble() or self.meta.isFloat(): return FloatType if self.meta.isString(): return StringType if self.meta.isEnum(): return EnumType if self.meta.isParameterGroup(): return GroupType if self.meta.isBool(): return BoolType def typeString(self): """ Give the type as String (see StringType, IntType,...)""" if self.meta.isRepeated(): return ListType return self.typeEntryString() def enumOptions(self): return self.meta.availableValues() def name(self): """ Return the name of this property """ return self._name def deprecated(self): """ Return true iff this property is deprecated """ return self.meta.isDeprecated() def description(self): """ Return a decription of this property """ return self.meta.description() def prototype(self): """ Returns a raw value of a default type """ default = self.meta.defaultValue() if default is None: return data.prototypeOfType(self) return default def subparameter(self): return self.meta.parameter() def isEditable(self): uri = self.uri if uri and len(uri) > 0: if uri in [["type"]]: return False if uri[0] in ["bottom"]: return False return True def isRequired(self): uri = self.uri if uri and len(uri) > 0: return uri in [["name"], ["type"]] return False class LayerParamInfo(data.PropertyInfo): """ Give infos about the root parameters of a layer""" def __init__(self, name, parameters): self._name = name self.params = parameters def typeEntryString(self): return self.typeString() def typeString(self): return GroupType def enumOptions(self): return [] def name(self): return self._name def deprecated(self): return False def description(self): return "" def prototype(self): return {} def subparameter(self): return self.params def isEditable(self): return False def isRequired(self): return False gui/main_window/docks/properties/property_widgets/network_dict_info.py0000664ڊ00111170000001313613243274037027103 0ustar klemmso0cvprimport backend.caffe.proto_info as info from gui.main_window.docks.properties.property_widgets import data from gui.main_window.docks.properties.property_widgets import layer_info from gui.main_window.docks.properties.property_widgets.types import * class NetworkDictInfoBuilder(data.PropertyInfoBuilder): """ Builds info docks for the network-part of the state-dictionary """ def __init__(self, netdict): if type(netdict) == dict: self.netdictfunc = lambda: netdict else: self.netdictfunc = netdict self.layerinfo = layer_info.LayerPropertyInfoBuilder() def buildInfo(self, name, protoparameter, uri=None): if uri == []: return NetworkDictInfo(name, protoparameter) elif uri == ["layers"]: return LayersInfo("layers", LayerIdInfo("")) elif uri[0] == "layers": id = uri[1] layer = self.netdictfunc()["layers"][id] if len(uri) == 2: return LayerIdInfo(id) if uri[2] == "type": return LayerTypeInfo() if uri[2] == "parameters": if uri == ["layers",id,"parameters"]: return self.layerinfo.buildRootInfo(name, layer["type"]) else: return self.layerinfo.buildInfo(name, protoparameter, uri[3:]) elif uri == ["layerOrder"]: return LayerOrderInfo("layerOrder") else: return NetworkInfo(name, protoparameter, uri) def buildRootInfo(self, name=""): """ Builds the information about "network" in the state dictionary """ prot = info.CaffeMetaInformation().availableNetParameters().copy() del prot["layers"] del prot["layer"] prot["layers"] = "LayerDict" prot["layerOrder"] = "LayerOrder" return self.buildInfo(name,prot,[]) class LayersInfo(data.PropertyInfo): """ Give infos about the entry 'layers' in the network dictionary """ def __init__(self, name, layerIdInfo): self._name = name self.paramInfo = layerIdInfo def entryInfo(self): return self.paramInfo def typeString(self): return DictType def enumOptions(self): return [] def name(self): return self._name def deprecated(self): return False def description(self): return "" def prototype(self): return {} def subparameter(self): """ Returns the subparameter of an entry in this dictionary (the layer id subparemter = "parameters","type")""" return self.paramInfo.subparameter() def isEditable(self): return True def isRequired(self): return True class LayerOrderInfo(data.PropertyInfo): def __init__(self, name): self._name = name def typeEntryString(self): return StringType def typeString(self): return ListType def enumOptions(self): return [] def name(self): return self._name def deprecated(self): return False def description(self): return "" def prototype(self): return [] def subparameter(self): return {} def isEditable(self): return True def isRequired(self): return True class LayerIdInfo(data.PropertyInfo): """ Give infos about a layer (with type and parameters) in network dictionary """ def __init__(self, id): self.id = id def typeEntryString(self): return self.typeString() def typeString(self): return GroupType def enumOptions(self): return [] def name(self): return self.id def deprecated(self): return False def description(self): return "" def prototype(self): return {"type": None, "parameters": {}} def subparameter(self): return {"type": "type", "parameters": "parameters"} def isEditable(self): return False def isRequired(self): return False class LayerTypeInfo(data.PropertyInfo): """ Give infos about the type-attribute of a layer network dictionary""" def __init__(self): pass def typeEntryString(self): return self.typeString() def typeString(self): return UnknownType def enumOptions(self): return [] def name(self): return "type" def deprecated(self): return False def description(self): return "" def prototype(self): return None def subparameter(self): return {} def isEditable(self): return False def isRequired(self): return True class NetworkInfo(layer_info.PropertyInfo): """ Give infos about a non-layer parameter in the network-dictionary""" def __init__(self, name, meta, uri): self._name = name self.meta = meta self.uri = uri def isEditable(self): return True def isRequired(self): return self.uri in [[]] class NetworkDictInfo(data.PropertyInfo): """ Give infos about the root of the network dictionary""" def __init__(self, name, prot): self.prot = prot self._name = name def typeEntryString(self): return self.typeString() def typeString(self): return GroupType def enumOptions(self): return [] def name(self): return self._name def deprecated(self): return False def description(self): return "" def prototype(self): return {} def subparameter(self): return self.prot def isEditable(self): return True def isRequired(self): return True gui/main_window/docks/properties/property_widgets/program_state_info.py0000664ڊ00111170000000646713243274037027267 0ustar klemmso0cvprimport backend.caffe.dict_helper as helper from gui.main_window.docks.properties.property_widgets import network_dict_info from gui.main_window.docks.properties.property_widgets import solver_info from gui.main_window.docks.properties.property_widgets.generic_info import GenericInfoBuilder from gui.main_window.docks.properties.property_widgets.types import * def defaultState(): net = helper.bareNet("Unnamed") return { "network": net, "position": {}, "selection": [], "solver": {}, "hidden_connections": {} } class ProgramStateInfoBuilder(GenericInfoBuilder): def __init__(self, statedict): self.statedict = statedict self.netinfo = network_dict_info.NetworkDictInfoBuilder(lambda: self.statedict["network"]) self.solverinfo = solver_info.SolverPropertyInfoBuilder() infodescription = { "type": GroupType, "name": "", "prototype": defaultState, "entries": [ { "info": self.netinfo.buildRootInfo("network"), "builder": self.netinfo, "urifunc": lambda uri: uri[1:] }, { "name": "position", "type": DictType, "prototype": [], "entry": { "name": "coord", "type": ListType, "prototype": [0,0], "typeEntry": IntType } }, { "name": "selection", "type": ListType, "prototype": [], "typeEntry": StringType }, { "info": self.solverinfo.buildRootInfo("solver"), "builder": self.solverinfo, "urifunc": lambda uri: uri[1:] }, { "name": "hidden_connections", "type": ListType, "prototype": [], "isRequired": False, "entry": { "name": "hidden_connection", "type": GroupType, "prototype": {}, "entries": [ { "name": "topLayerId", "type": StringType, "prototype": "" }, { "name": "topBlobIdx", "type": IntType, "prototype": 0 }, { "name": "bottomLayerId", "type": StringType, "prototype": "" }, { "name": "bottomBlobIdx", "type": IntType, "prototype": 0 } ] } } ] } GenericInfoBuilder.__init__(self,infodescription) gui/main_window/docks/properties/property_widgets/solver_info.py0000664ڊ00111170000000521013243274037025713 0ustar klemmso0cvprimport backend.caffe.proto_info as info from gui.main_window.docks.properties.property_widgets import data from gui.main_window.docks.properties.property_widgets import layer_info from gui.main_window.docks.properties.property_widgets.types import * class SolverPropertyInfoBuilder(data.PropertyInfoBuilder): """ Builds info docks for the solver part of the state dictionary """ def buildInfo(self, name, protoparameter, uri=None): if uri == []: return SolverRootInfo(name, protoparameter) return PropertyInfo(name, protoparameter, uri) def buildRootInfo(self, name=""): """ Builds "solver" info object of the state dictionary """ import backend.caffe.path_loader as caffeLoader proto = caffeLoader.importProto() protosolver = proto.SolverParameter() descr = info.ParameterGroupDescriptor(protosolver) params = descr.parameter().copy() return self.buildInfo(name,params, []) class SolverRootInfo(data.PropertyInfo): """ Represent the info about the parameters of a solver""" def __init__(self, name, params): self._name = name self.params = params def typeEntryString(self): return self.typeString() def typeString(self): return GroupType def enumOptions(self): return [] def name(self): return self._name def deprecated(self): return False def description(self): return "" def prototype(self): return {} def subparameter(self): """ Returns the subparameter of an entry in this dictionary (the layer id subparemter = "parameters","type")""" return self.params def isEditable(self): return True def isRequired(self): return True class PropertyInfo(layer_info.PropertyInfo): def __init__(self, name, meta, uri): super(PropertyInfo,self).__init__(name,meta,uri) def isEditable(self): uri = self.uri if uri and len(uri) > 0: if uri in [["net"], ["net_param"]]: return False return True def isRequired(self): uri = self.uri if uri and len(uri) > 0: if uri in [["net"]]: return True return False def typeString(self): if self.uri == ["type"]: return EnumType return super(PropertyInfo, self).typeString() def enumOptions(self): if self.uri == ["type"]: return info.CaffeMetaInformation().availableSolverTypes() return super(PropertyInfo, self).enumOptions() def subparameter(self): params = super(PropertyInfo,self).subparameter() #type: dict return params gui/main_window/docks/properties/property_widgets/type_widgets.py0000664ڊ00111170000002173313243274037026105 0ustar klemmso0cvprfrom PyQt5 import QtCore, QtGui from PyQt5 import QtWidgets from PyQt5.QtCore import pyqtSignal from gui.main_window.docks.properties.property_widgets.data import StringType, IntType, EnumType, FloatType, BoolType, PropertyDataObject class TwoTypeWdg: """ TwoTypeWdg is a class which defines setupInTableLayout using exactly two widget: * one label for the name * one widget defines by its subclasses in _initValueWdg Child classes can use self._parent to get the parent widget. They can use self.data to get the PropertyDataObject this Widget represents. """ def __init__(self, data, parent=None): """ Build TwoTypeWdg with given parent widget and the instance of PropertyDatObject named data. The name of the name-label will be the value of data.info().name(). """ # super(TwoTypeWdg, self).__init__(parent) self.data = data #type: PropertyDataObject self._parent = parent self.nameWdg = QtWidgets.QLabel(self.data.info().name(),parent) self.valueWdg = self._initValueWdg() self.valueWdg.setSizePolicy(QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed)) self.nameWdg.setSizePolicy(QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed)) font = self.nameWdg.font() #type: QtGui.QFont # self.nameWdg.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter) if self.data.info().deprecated(): font.setStrikeOut(True) self.nameWdg.setFont(font) self.nameWdg.setToolTip(getTooltipString(self.data.info().description())) self.valueWdg.setToolTip(getTooltipString(self.data.info().description())) if not data.info().isEditable(): self.nameWdg.setEnabled(False) self.valueWdg.setEnabled(False) def setupInTableLayout(self, lay, row, remButton): """ Setup this object into a QGridLayout row is the specific row to insert the widgets remButton is the Remove-Button which should be inserted into the GridLayout by this function, too """ lay = lay #type: QtWidgets.QGridLayout lay.addWidget(self.nameWdg, row, 0,1,1) lay.addWidget(self.valueWdg, row, 1,1,1) lay.addWidget(remButton, row, 2,1,1) def _initValueWdg(self): """ This function should be overriden by every child It should return the Widget which represents the value of the PropertyDataObject. """ pass class StringWdg(TwoTypeWdg): """ A Line-Editor for edit simple text """ class Editor(QtWidgets.QLineEdit): """ The LineEdit-Widget """ """ This signal is emitted every time the value of this prototxt_editor should propagate changes. For example not every text change emits this signals. Only when the prototxt_editor gets deselected or enter is pressed this signal is emitted. """ saveChange = pyqtSignal(str) def __init__(self, text="", parent=None): super(StringWdg.Editor, self).__init__(text,parent) self.customContextMenuRequested.connect(self.openCtxMenu) self.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) self.returnPressed.connect(self._checkForChangesAndEmitIf) self.__textInData = text def openCtxMenu(self,pos): """ Add "From File" and "From Path" to the context menu """ menu = self.createStandardContextMenu() # type: QtWidgets.QMenu menu.addSeparator() menu.addAction(self.tr("From &File"), self.selectFile) menu.addAction(self.tr("From &Path"), self.selectPath) menu.exec_(self.mapToGlobal(pos)) def setTextWithChangeDetection(self, text): """ This method should be use from other to set the text of this widget. It updates the widget text but also updates the variable represents the last text data. So the function _checkForChangesAndEmitIf works. """ self.__textInData = text self.setText(text) def _checkForChangesAndEmitIf(self): """ Check is the new text does not equal the last one and emit signals if not equal """ if self.text() != self.__textInData: self.saveChange.emit(self.text()) def focusOutEvent(self, event): super(StringWdg.Editor, self).focusOutEvent(event) self._checkForChangesAndEmitIf() def selectFile(self): """ When clicking 'From File' """ filename,_ = QtWidgets.QFileDialog.getOpenFileName(self) if len(filename) == 0: return self.setText(filename) def selectPath(self): """ When clicking 'From Path' """ dirname = QtWidgets.QFileDialog.getExistingDirectory(self) if len(dirname) == 0: return self.setText(dirname) def _initValueWdg(self): wdg = StringWdg.Editor(self.data.value(),self._parent) wdg.saveChange.connect(lambda: self.data.setValue(wdg.text())) wdg.childEvent self.data.propertyChanged.connect(wdg.setTextWithChangeDetection) return wdg class IntWdg(TwoTypeWdg): """ A Spinbox which represent integer values """ # The maximal and minimal values of an Integer MAX_RANGE = -pow(2, 31), pow(2, 31-1) class SpinBox(QtWidgets.QSpinBox): """ The SpinBox widget """ def __init__(self, parent=0): super(IntWdg.SpinBox, self).__init__(parent) self.setFocusPolicy(QtCore.Qt.StrongFocus ) # idea from http://stackoverflow.com/questions/5821802/qspinbox-inside-a-qscrollarea-how-to-prevent-spin-box-from-stealing-focus-when def wheelEvent(self, event): if not self.hasFocus(): event.ignore() else: QtWidgets.QSpinBox.wheelEvent(self, event) def _initValueWdg(self): wdg = IntWdg.SpinBox(self._parent) wdg.setRange(*self.MAX_RANGE) wdg.setValue(self.data.value()) wdg.valueChanged.connect(self.data.setValue) self.data.propertyChanged.connect(wdg.setValue) return wdg class FloatWdg(TwoTypeWdg): """ A Spinbox which represent float values """ MAX_RANGE = -pow(2, 31), pow(2, 31-1) class SpinBox(QtWidgets.QDoubleSpinBox): """ The Spinbox Widget """ def __init__(self, parent=0): super(FloatWdg.SpinBox, self).__init__(parent) self.setFocusPolicy(QtCore.Qt.StrongFocus ) # self.setDecimals(323) self.setDecimals(15) self.setSingleStep(0.05) # idea from http://stackoverflow.com/questions/5821802/qspinbox-inside-a-qscrollarea-how-to-prevent-spin-box-from-stealing-focus-when def wheelEvent(self, event): if not self.hasFocus(): event.ignore() else: QtWidgets.QDoubleSpinBox.wheelEvent(self, event) def textFromValue(self, value): """ Change the text of this SpinBox such that the widget does not show unneeded zeros after the decimal point """ stringValue = super(FloatWdg.SpinBox, self).textFromValue(value) while len(stringValue) > 2 and stringValue[-1] == "0" and stringValue[-2] != ",": stringValue = stringValue[0:-1] return stringValue def _initValueWdg(self): wdg = FloatWdg.SpinBox(self._parent) wdg.setRange(*self.MAX_RANGE) wdg.setValue(self.data.value()) wdg.valueChanged.connect(self.data.setValue) self.data.propertyChanged.connect(wdg.setValue) return wdg class EnumWdg(TwoTypeWdg): """ Combobox for Enums """ def _initValueWdg(self): wdg = QtWidgets.QComboBox(self._parent) wdg.addItems(self.data.info().enumOptions()) wdg.currentTextChanged.connect(self.data.setValue) self.data.propertyChanged.connect(wdg.setCurrentText) wdg.setCurrentText(self.data.value()) return wdg class BoolWdg(TwoTypeWdg): """ Checkbox for Bools""" def _initValueWdg(self): wdg = QtWidgets.QCheckBox(self._parent) wdg.stateChanged.connect(lambda s: self.data.setValue(wdg.isChecked())) self.data.propertyChanged.connect(wdg.setChecked) wdg.setChecked(self.data.value()) return wdg # Export Widget WdgCatalog={ StringType: StringWdg, IntType: IntWdg, FloatType: FloatWdg, EnumType: EnumWdg, BoolType: BoolWdg, } def getTooltipString(description): """Ensures that tooltips are shown with newlines. """ newDescription = "" if not description == "": newDescription = "" newDescription += description newDescription += "" return newDescription gui/main_window/docks/properties/property_widgets/types.py0000664ڊ00111170000000173713243274037024544 0ustar klemmso0cvpr StringType = "string" IntType = "int" FloatType = "float" EnumType = "enum" GroupType = "group" ListType = "list" BoolType = "bool" DictType = "dict" UnknownType = "unknown" def checkIsType(instance, typestring): """ Checks if instance match the type expected from typestring. E.g. checkIsType(22, IntType) == True checkIsType(true, StringType) == False """ if typestring in [StringType, EnumType]: return type(instance) in [str, unicode] if typestring == IntType: return type(instance) in [int, long] if typestring == FloatType: return type(instance) in [int, long, float] if typestring in [GroupType, DictType]: return type(instance) == dict if typestring == BoolType: return type(instance) == bool if typestring in [ListType]: return type(instance) == list if typestring == UnknownType: return True raise ValueError("Unhandable typestring {}".format(typestring)) gui/main_window/docks/properties/property_widgets/widgets.py0000664ڊ00111170000003561613243274037025051 0ustar klemmso0cvprimport platform from PyQt5 import QtCore, QtGui from PyQt5 import QtWidgets from PyQt5.QtCore import pyqtSignal from gui.main_window.docks.properties.property_widgets.data import PropertyDataObject, PropertyDataListObject from gui.main_window.docks.properties.property_widgets.type_widgets import WdgCatalog from gui.main_window.docks.properties.property_widgets.types import * def clearLayout(lay): """ Removes every widget in the given layout """ for i in reversed(range(lay.count())): item = lay.itemAt(i) # type: QtWidgets.QLayoutItem lay.removeItem(item) wdg = item.widget() if wdg is None: childLay = item.layout() clearLayout(childLay) else: wdg.setParent(None) wdg.deleteLater() class PropertyEditWdg(QtWidgets.QScrollArea): """ This is the main widget for showing the properties. It should be used in the dock element for example. """ def __init__(self, parent=None, data = None): """ Construct this object with the parent (a widget) and the data (PropertyData/-Object). Through this widget the user can manipulate the data. data can be None, so there is no data set. """ super(PropertyEditWdg, self).__init__(parent) self.setWidgetResizable(True) self.content = None if data: self.setData(data) def setData(self, data): """ Set the data (may replace old one) with the given one """ self.content = GroupWdg(data) # lay = QtWidgets.QVBoxLayout() # lay.addWidget(val) # self.setLayout(lay) self.setWidget(self.content) def disableEditing(self, disable): """ Disable Editing of the data """ if self.content: self.content.setDisabled(disable) class RepeatedWrapper(QtWidgets.QGroupBox): """ A Widget for showing a list of values """ def __init__(self, data, parent=None): super(RepeatedWrapper, self).__init__(parent) # Layout to add the child widgets # Have to be a GridLayout. self._childlay = QtWidgets.QGridLayout() self._childlay.setHorizontalSpacing(2) # Layout for the top-remove button self._topLay = QtWidgets.QHBoxLayout() # Layout of the whole object self._lay = QtWidgets.QVBoxLayout() self._lay.addLayout(self._topLay) self._lay.addLayout(self._childlay) self._lay.addStretch() # self._lay.setContentsMargins(1,1,1,1) self.setLayout(self._lay) self.data = data #type: PropertyDataListObject self.nameLabel = QtWidgets.QLabel(data.info().name()) self.setToolTip(getTooltipString(self.data.info().description())) # \/ exclusive value self.lastChildRow = self._buildChilds(self.data.value()) self.data.propertyChanged.connect(self._buildChilds) # Layout for the add-Button bottomLay = QtWidgets.QHBoxLayout() self._lay.addLayout(bottomLay) bottomLay.addWidget(self.nameLabel,self.lastChildRow) bottomLay.addStretch() addButton = QtWidgets.QToolButton(self) addButton.clicked.connect(self.addOne) addButton.setText("+") bottomLay.addWidget(addButton,self.lastChildRow) def _clear(self): """ Clear the Layout so there is no widget anymore """ clearLayout(self._childlay) self.lastChildRow = 0 def _buildChilds(self, childs): """ Build the widget for every entry in the list """ self._clear() # if remove button is clicked def removeCallback(idx): return lambda: self.data.removeObject(idx) i = 1 # Current entry-idx in the GridLayout for (idx,element) in enumerate(childs): #type: (int,PropertyDataObject) # Build the widget wdg = WdgCatalog[element.info().typeString()](element,self) # The remove button remButton = QtWidgets.QToolButton(self) trashIcon = QtGui.QPixmap('resources/trash.png') remButton.setIcon(QtGui.QIcon(trashIcon)) remButton.clicked.connect(removeCallback(idx)) wdg.setupInTableLayout(self._childlay,idx,remButton) i += 1 self.nameLabel.setVisible(len(childs) == 0) return i def setupInTableLayout(self,lay, row, remButton): """ Install this widget in the QGridLayout lay """ def hline(): line = QtWidgets.QFrame(self) line.setFrameShape(QtWidgets.QFrame.HLine) line.setFrameShadow(QtWidgets.QFrame.Sunken) return line # Add the remove button to the top # The remove button is given as an argument # to this function. self._topLay.addWidget(hline()) self._topLay.addWidget(remButton) self._topLay.addWidget(hline()) # And this widget to the GridLayout lay.addWidget(self, row,0,1,3) def addOne(self): """ Add a new entry to the list using the default value (prototypeValue) of the PropertyInfo of the data """ self.data.pushBack(self.data.prototypeValue()) class GroupWdg(QtWidgets.QGroupBox): """ Widget for a group of values """ class Model(QtCore.QAbstractListModel): """ List Model for the autocompletion in 'New Property' Text-Field """ def __init__(self, data, parent=None): super(GroupWdg.Model, self).__init__(parent) self.data = data #type: PropertyData def newProperties(self): """ Return a list of properties which are not set in the GroupData """ all = self.data.allAvailableProperties() given = [prop.info().name() for prop in self.data.givenProperties()] res = [] for name in all: if not name in given: res.append(all[name]) # sort properties alphabetically res.sort(key=lambda x: x.name()) # move deprecated properties to the end of the list resFinal = [] resDeprecated = [] for property in res: if property.deprecated(): resDeprecated.append(property) else: resFinal.append(property) resFinal.extend(resDeprecated) return resFinal def rowCount(self, idx): return len(self.newProperties()) def data(self, idx, role): row = idx.row() prop = self.newProperties()[row] if role == QtCore.Qt.DisplayRole: return prop if role ==QtCore.Qt.EditRole: return prop.name() return None class Delegate(QtWidgets.QStyledItemDelegate): """ The visual Item of a entry in the autocompletion """ def __init__(self, parent=None): super(GroupWdg.Delegate,self).__init__(parent) def paint(self, painter, option, idx): # super(Delegate, self).paint(painter, option, idx) data = idx.data() painter = painter #type: QtGui.QPainter painter.save() # If the mouse is over this item if option.state & QtWidgets.QStyle.State_Selected: painter.fillRect(option.rect,option.palette.highlight()) painter.setBrush(option.palette.highlightedText()) else: painter.fillRect(option.rect,option.palette.base()) font = painter.font() #type: QtGui.QFont # Strike out if the data is deprecated font.setStrikeOut(data.deprecated()) painter.setFont(font) painter.drawText(option.rect, QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter, data.name()) painter.restore() def __init__(self, data, parent=None): super(GroupWdg, self).__init__(parent) self.name = "" self._topLay = QtWidgets.QHBoxLayout() self._editable = True # If data is a ProeprtyDataObject we get more # infos which we can use. if isinstance(data, PropertyDataObject): self.setToolTip(getTooltipString(data.info().description())) self.name = data.info().name() self._editable = data.info().isEditable() data = data.value() # Titel der Group-Box self.setTitle(self.name) self.data = data #type: PropertyData self.data.propertyGotGiven.connect(self._addProperty) self.data.propertyGotUnGiven.connect(self._buildChilds) lay = QtWidgets.QVBoxLayout() self.childLay = QtWidgets.QGridLayout() # self.childLay.setContentsMargins(2,2,2,2) lay.setContentsMargins(3,3,3,3) self.childLay.setHorizontalSpacing(2) lay.addLayout(self._topLay) lay.addLayout(self.childLay) lay.addStretch() # \/ exclusive self.lastChildRow = 0 self._buildChilds() lay.addLayout(self._buildAddLayout()) self.setLayout(lay) self.wdgDict = {} def _addProperty(self, property): """ Add the property to this group when it become given. property is an instance of PropertyDataObject or PropertyData """ def removeCallback(name): return lambda: self._removeElement(name) # We create a widget depending on the type of the property. wdg = WdgCatalog[property.info().typeString()](property,self) # Remove Button button = QtWidgets.QToolButton(self) trashIcon = QtGui.QPixmap('resources/trash.png') button.setIcon(QtGui.QIcon(trashIcon)) wdg.setupInTableLayout(self.childLay, self.lastChildRow, button) button.setEnabled(not property.info().isRequired()) name = property.info().name() button.clicked.connect(removeCallback(name)) self.lastChildRow+=1 self.wdgDict[property.info().name()] = wdg class CompleteLineEdit(QtWidgets.QLineEdit): """ The LineEdit with autocompletion """ """ Signal is emitted when a new text has been chosen by the user """ gotChoice = pyqtSignal(str) def __init__(self, parent=None): super(GroupWdg.CompleteLineEdit, self).__init__(parent) self.returnPressed.connect(lambda: self.gotChoice.emit(self.text())) self.gotChoice.connect(self._clear) def _clear(self): """ Clear completer and text """ self.completer().setCompletionPrefix("") self.clear() def focusInEvent(self,ev): """ Show completion if the prototxt_editor gets in focus """ super(GroupWdg.CompleteLineEdit, self).focusInEvent(ev) self.showComplete() def setCompleter(self, completer): super(GroupWdg.CompleteLineEdit,self).setCompleter(completer) # completer.activated.connect(self.gotChoice) def showComplete(self): """ Shows the completion if an completer is set. It give the right prefix to the completion. """ if self.completer() is None: return self.completer().setCompletionPrefix(self.text()) self.completer().complete() def keyPressEvent(self, event): """ Ctrl+Space should show the completer """ super(GroupWdg.CompleteLineEdit,self).keyPressEvent(event) ctrl = QtCore.Qt.ControlModifier # Qt replace Ctrl with Cmd on macOS but we do not want that if platform.system() == "Darwin": ctrl = QtCore.Qt.MetaModifier if event.modifiers() == ctrl and event.key() == QtCore.Qt.Key_Space: self.showComplete() def _addElement(self, name): """ Add a new property initialized with the default (protoype) value. If the name is invalid (e.g. is not a property for this group ), the user gets a Popup which informs him """ newOne = self._model.newProperties() for item in newOne: if item.name() == name: self.data.giveProperty(name,item.prototype()) return label = QtWidgets.QLabel(self.tr("This property does not exists")) label.setWindowFlags(QtCore.Qt.Popup) label.move(self.mapToGlobal(self.editor.geometry().topLeft()+QtCore.QPoint(0,-10))) label.show() QtCore.QTimer.singleShot(2000, lambda: label.close()) def _buildAddLayout(self): """ Build layout for the autocompletion-LineEdit""" lay = QtWidgets.QHBoxLayout() # Editor self.editor = GroupWdg.CompleteLineEdit() self.editor.setPlaceholderText(self.tr("New Property")) # Model self._model = GroupWdg.Model(self.data) # Completer completer = QtWidgets.QCompleter() completer.setModel(self._model) completer.setModelSorting(QtWidgets.QCompleter.CaseInsensitivelySortedModel) completer.popup().setItemDelegate(GroupWdg.Delegate(self)) completer.setCompletionMode(QtWidgets.QCompleter.PopupCompletion) self.editor.setCompleter(completer) lay.addStretch() lay.addWidget(self.editor) # Signals self.editor.gotChoice.connect(self._addElement) self.editor.setEnabled(self._editable) return lay def setupInTableLayout(self,lay, row, remButton): """ Setup this widget in parent QGridLayout """ def hline(): line = QtWidgets.QFrame(self) line.setFrameShape(QtWidgets.QFrame.HLine) line.setFrameShadow(QtWidgets.QFrame.Sunken) return line # Remove button at the top self._topLay.addWidget(hline()) self._topLay.addWidget(remButton) self._topLay.addWidget(hline()) lay.addWidget(self, row,0,1,3) def _removeElement(self, name): """ Remove the property with the given name from the data and this Widget """ self.data.ungiveProperty(name) def _clear(self): """ Clears all children """ clearLayout(self.childLay) self.lastChildRow = 0 self.wdgDict = {} def _buildChilds(self): """ Build the children widget which represent the given properties of the PropertyDataGroupObject """ self._clear() d = dict([(prop.info().name(), prop) for prop in self.data.givenProperties()]) keys = d.keys() keys.sort() for key in keys: property = d[key]#type: PropertyDataObject self._addProperty(property) def getTooltipString(description): """Ensures that tooltips are shown with newlines. """ newDescription = "" if not description == "": newDescription = "" newDescription += description newDescription += "" return newDescription WdgCatalog[GroupType] = GroupWdg WdgCatalog[ListType] = RepeatedWrapper gui/main_window/node_editor/items/renderer/__init__.py0000664ڊ00111170000000000013243274037023426 0ustar klemmso0cvprgui/main_window/node_editor/items/renderer/connection_item_renderer.py0000664ڊ00111170000000566713243274037026762 0ustar klemmso0cvprfrom PyQt5.QtGui import QPainter, QPen from gui.main_window.node_editor.node_editor_constants import Constants class ConnectionItemRenderer: """ Class is used to separate the drawing from the logic. This class renders a given connection to a graphics scene """ def __init__(self, connection): self.__connection = connection def paint(self, painter): # use antialiasing painter.setRenderHint(QPainter.Antialiasing) # connections, connected to layers with the include.phase parameter, # have a different opacity than normal connections opacity = Constants.itemOpacityInPhase if not self.__connection.getNodeEditor().isCurrentPhase(self.__connection.getPhase()): opacity = Constants.itemOpacityNotInPhase # connections between a layer and a in-place working layer have a different color than normal connections connectionColor = Constants.connectionItemColor if self.__connection.getIsInPlace(): connectionColor = Constants.itemInPlaceColor # set local color object alpha connectionColor.setAlpha(opacity) # draw highlight if the connection is selected if self.__connection.isSelected(): selectedColor = Constants.selectedColor selectedColor.setAlpha(opacity) painter.setPen(QPen(selectedColor, Constants.connectionItemSelectionSize)) painter.drawPath(self.__connection.path()) pen = QPen(connectionColor, Constants.connectionItemSize) # if the connection is hidden, draw a dashed line at the start and end of the connection if self.__connection.getHidden(): dashes = [] # calculate the dashes at the start of the connection for i in range(0, Constants.connectionItemHiddenDashCount - 1): dashes.append(Constants.connectionItemHiddenDashSize / pen.width()) dashes.append(Constants.connectionItemHiddenDashSpace / pen.width()) dashes.append(Constants.connectionItemHiddenDashSize / pen.width()) # calculate the size of the space between the dashed start and dashed end middleSpace = self.__connection.path().length() middleSpace -= 2 * Constants.connectionItemHiddenDashCount * Constants.connectionItemHiddenDashSize middleSpace -= 2 * (Constants.connectionItemHiddenDashCount - 1) * Constants.connectionItemHiddenDashSpace dashes.append(middleSpace / pen.width()) # calculate the dashes at the end of the connection for i in range(0, Constants.connectionItemHiddenDashCount): dashes.append(Constants.connectionItemHiddenDashSize / pen.width()) dashes.append(Constants.connectionItemHiddenDashSpace / pen.width()) pen.setDashPattern(dashes) # draw the connection painter.setPen(pen) painter.drawPath(self.__connection.path()) gui/main_window/node_editor/items/renderer/connector_item_renderer.py0000664ڊ00111170000000435413243274037026605 0ustar klemmso0cvprfrom PyQt5.QtCore import QRectF from PyQt5.QtGui import QPen, QBrush from gui.main_window.node_editor.node_editor_constants import Constants class ConnectorItemRenderer: """ Class is used to separate the drawing from the logic. This class renders a given connector to a graphics scene """ def __init__(self, connector): self.__connector = connector # get the outer and inner size of the enclosing rect (inner size is the size of the hole) outerSize = Constants.connectorItemOuterSize innerSize = Constants.connectorItemInnerSize # calculate the Qt rects centered at the center of the rect self.__outerRect = QRectF(-(outerSize / 2), -(outerSize / 2),outerSize, outerSize) self.__innerRect = QRectF(-(innerSize / 2), -(innerSize / 2), innerSize, innerSize) def boundingRect(self): return self.__outerRect def paint(self, painter): # connectors of a layer with the include.phase parameter # have a different opacity than normal connectors opacity = Constants.itemOpacityInPhase if not self.__connector.getNodeEditor().isCurrentPhase(self.__connector.getPhase()): opacity = Constants.itemOpacityNotInPhase # get colors colorInner = Constants.itemBackgroundColorLight colorOuter = Constants.itemBackgroundColorDark borderColor = Constants.itemBorderColor # in place layer connectors have a different color than normal connectors if self.__connector.isInPlace(): colorOuter = Constants.itemInPlaceColor # set the colors opacity colorInner.setAlpha(opacity) colorOuter.setAlpha(opacity) borderColor.setAlpha(opacity) # draw enclosing rect and rect border painter.setPen(QPen(borderColor, 1)) painter.fillRect(self.__outerRect, QBrush(colorOuter)) painter.drawRect(self.__outerRect) # if the connector is connected, fill the inner rect if self.__connector.isConnected(): painter.fillRect(self.__innerRect, QBrush(colorOuter)) else: painter.fillRect(self.__innerRect, QBrush(colorInner)) # draw the border of the inner rect painter.drawRect(self.__innerRect)gui/main_window/node_editor/items/renderer/node_item_renderer.py0000664ڊ00111170000003142313243274037025535 0ustar klemmso0cvprfrom PyQt5.QtCore import QRectF, Qt, QPointF from PyQt5.QtGui import QPainter, QColor, QPen, QBrush, QLinearGradient from gui.main_window.node_editor.node_editor_constants import Constants from gui.main_window.node_editor.layer_color_definitions import LayerColorDefinitions class TextRect: """ Used to store values required for text positioning and rendering """ def __init__(self, text, rect, font = None): self.text = text self.rect = rect self.font = font class NodeItemRenderer: """ Class is used to separate the drawing from the logic. This class renders a given node to a graphics scene """ __blobConnectorSizeWithPadding = Constants.nodeItemConnectorPaddding * 2 + Constants.connectorItemOuterSize def __init__(self, nodeItem): self.__nodeItem = nodeItem # create lists to store TextRects for the node item self.__headRects = list() self.__blobTopNameRects = list() self.__blobBottomNameRects = list() # create Qt rects for the head and body area self.__rectHead = QRectF(0, 0, 1, 1) self.__rectBlobArea = QRectF(0, 0, 1, 1) # create Qt rect for enclosing rects used as bounding rects self.__rectAll = QRectF(0, 0, 1, 1) self.__rectAllSelected = QRectF(0, 0, 1, 1) self.__hasCurrentPhase = True self.update() def boundingRect(self): # selected layers have a outer 'glow', so selected layers are bigger than not selected layers if self.__nodeItem.isSelected(): return self.__rectAllSelected else: return self.__rectAll def update(self): """ Recalculates positions of name, type, blobs, etc. """ # clear lists del self.__headRects[:] del self.__blobTopNameRects[:] del self.__blobBottomNameRects[:] # get size of the name nameWidth = Constants.nodeItemFontNameMetrics.width(self.__nodeItem.getName()) + Constants.nodeItemHeadPadding nameHeight = Constants.nodeItemFontTypeMetrics.height() self.__headRects.append(TextRect(self.__nodeItem.getName(), QRectF(0, 0, nameWidth, nameHeight), Constants.nodeItemFontName)) # get size of the type typeWidth = Constants.nodeItemFontTypeMetrics.width(self.__nodeItem.getTypeString()) + Constants.nodeItemHeadPadding typeHeight = Constants.nodeItemFontTypeMetrics.height() self.__headRects.append(TextRect(self.__nodeItem.getTypeString(), QRectF(0, 0, typeWidth, typeHeight), Constants.nodeItemFontType)) # get size of the phase (if the node has the paramter include.phase if len(self.__nodeItem.getPhase()) > 0: phaseString = "Phase: " + self.__nodeItem.getPhase() phaseWidth = Constants.nodeItemFontTypeMetrics.width(phaseString) + Constants.nodeItemHeadPadding phaseHeight = Constants.nodeItemFontTypeMetrics.height() self.__headRects.append(TextRect(phaseString, QRectF(0, 0, phaseWidth, phaseHeight), Constants.nodeItemFontType)) # calculate rect width and height for names of blobs topBlobsInfo = self.__buildBlobNameRectList(self.__nodeItem.getTopConnectors()) bottomBlobsInfo = self.__buildBlobNameRectList(self.__nodeItem.getBottomConnectors()) self.__blobTopNameRects = topBlobsInfo[0] self.__blobBottomNameRects = bottomBlobsInfo[0] blobAreaWidth = 2 * self.__blobConnectorSizeWithPadding + Constants.nodeItemConnectorPaddding + \ bottomBlobsInfo[1] + topBlobsInfo[1] blobAreaHeight = max(topBlobsInfo[2], bottomBlobsInfo[2], Constants.nodeItemMinBlobAreaHeight) # calculate the total width of the node rectWidth = blobAreaWidth for textRect in self.__headRects: rectWidth = max(rectWidth, textRect.rect.width()) # change the width of the head rects to be the full width of the node self.__updateHeadRects(rectWidth, self.__headRects) # calculate the total node height rectHeight = self.__rectHead.bottom() + blobAreaHeight # create Qt rects for the body and enclosing width the top left at (0, 0) self.__rectBlobArea = QRectF(0, self.__rectHead.bottom(), rectWidth, blobAreaHeight) self.__rectAll = QRectF(0, 0, rectWidth, rectHeight) # calculate a larger box to support selection (outer glow) selSize = Constants.nodeItemSelectionSize self.__rectAllSelected = QRectF(self.__rectAll.left() - selSize, self.__rectAll.top() - selSize, self.__rectAll.width() + 2 * selSize, self.__rectAll.height() + 2 * selSize) # after calculating the total width, adjust the name rects to align at the left/right self.__adjustBlobNameRectPositions(self.__blobTopNameRects, self.__rectAll.width(), self.__rectHead.bottom(), False) self.__adjustBlobNameRectPositions(self.__blobBottomNameRects, self.__rectAll.width(), self.__rectHead.bottom(), True) def __updateHeadRects(self, finalWidth, headRects): """ Resizes the size of the head text rect after the total needed width was calculated """ totalHeadHeight = 0 for x in range(0, len(headRects)): # special case for the first head text -> start at (0, 0) and has space at the top if x == 0: headRects[x].rect = QRectF(0, 0, finalWidth, headRects[x].rect.height() + Constants.nodeItemTextMargin) # special case for the last head text -> has space at the bottom elif x == len(headRects) - 1: headRects[x].rect = QRectF(0, headRects[x-1].rect.bottom(), finalWidth, headRects[x].rect.height() + Constants.nodeItemTextMargin) else: headRects[x].rect = QRectF(0, headRects[x-1].rect.bottom(), finalWidth, headRects[x].rect.height()) # add the rects height to the total head rect height totalHeadHeight += headRects[x].rect.height() # create Qt rect for the head part self.__rectHead = QRectF(0, 0, finalWidth, totalHeadHeight) def __buildBlobNameRectList(self, connectors): """ Creates rects for all top/bottom connectors (bounding rect for the text only) """ # check if the connector or the blob name text is higher blobNameHeightRect = max(Constants.nodeItemFontBlobMetrics.height(), self.__blobConnectorSizeWithPadding) blobNameRectList = list() maxWidth = 0 height = 0 # for each connector calculate the width of the name and create a Qt rect. # calculate the max width of all blob names and the total height for item in connectors: blobName = item.getBlobName() blobNameWidth = Constants.nodeItemFontBlobMetrics.width(blobName) + Constants.nodeItemTextMargin blobRect = QRectF(0, 0, blobNameWidth, blobNameHeightRect) blobNameRectList.append(TextRect(blobName, blobRect)) maxWidth = max(maxWidth, blobNameWidth) height += blobNameHeightRect return blobNameRectList, maxWidth, height def __adjustBlobNameRectPositions(self, blobRectNames, totalWidth, aboveHeight, atLeftBorder): """ Recalculates the blob name rects after the total width of the node was calculated """ i = 0 for item in blobRectNames: # position at the left side of the node x = self.__blobConnectorSizeWithPadding # or position at the right side of the node if not atLeftBorder: x = totalWidth - self.__blobConnectorSizeWithPadding - item.rect.width() # calculate starting position (head size + position in body) item.rect = QRectF(x, aboveHeight + i * item.rect.height(), item.rect.width(), item.rect.height()) i += 1 def updateConnectorPositions(self, connectors, atLeftBorder): """ Repositions the connector items after the new size of the node was calculated """ i = 0 for item in connectors: # position the connector item at the left border of the node,left of the blob name rect x = Constants.nodeItemConnectorPaddding + Constants.connectorItemOuterSize / 2 # or at the right border of the node, right of the blob name rect if not atLeftBorder: x = self.__rectAll.width() - Constants.nodeItemConnectorPaddding - Constants.connectorItemOuterSize + \ Constants.connectorItemOuterSize / 2 # set the y coordinate of the connector item to be at the center of the blob name rect y = self.__rectHead.bottom() + Constants.nodeItemConnectorPaddding + Constants.connectorItemOuterSize / 2 + \ i * self.__blobConnectorSizeWithPadding item.setPos(QPointF(x, y)) i += 1 def paint(self, painter): # use antialiasing for both text and box painter.setRenderHint(QPainter.Antialiasing, True) painter.setRenderHint(QPainter.TextAntialiasing, True) # get the opacity of the node item, layers with the include.phase parameter have a different opacity opacity = Constants.itemOpacityInPhase if not self.__nodeItem.getNodeEditor().isCurrentPhase(self.__nodeItem.getPhase()): opacity = Constants.itemOpacityNotInPhase # draw a outer 'glow' around the node item if the node is selected if self.__nodeItem.isSelected(): selectedColor = Constants.selectedColor selectedColor.setAlpha(opacity) painter.setPen(QPen(selectedColor, Constants.nodeItemSelectionSize)) painter.drawRect(self.__rectAllSelected) painter.fillRect(self.__rectAllSelected, QBrush(selectedColor)) # get the type color for the header (name and type) typeColor = LayerColorDefinitions.getTypeColor(self.__nodeItem.getType()) typeColor.setRgb(typeColor.red(), typeColor.green(), typeColor.blue(), opacity) # get background color backgroundColor = Constants.itemBackgroundColorLight backgroundColor.setAlpha(opacity) # set a linear gradient for the header (type color -> background color) gradient = QLinearGradient(0, self.__rectHead.top(), 0, self.__rectHead.bottom()) gradient.setColorAt(0, typeColor) gradient.setColorAt(0.5, backgroundColor) # draw background and border for the header painter.fillRect(self.__rectHead, QBrush(gradient)) # draw background for the blob area painter.fillRect(self.__rectBlobArea, QBrush(backgroundColor)) borderColor = Constants.itemBorderColor if self.__nodeItem.getIsInPlace(): borderColor = Constants.itemInPlaceColor borderColor.setAlpha(opacity) painter.setPen(QPen(borderColor, Constants.nodeItemBorderSize)) # draw outer border around the node painter.drawRect(self.__rectAll) # draw a line to separate header and connectors borSize = Constants.nodeItemBorderSize painter.setPen(QPen(borderColor, borSize)) painter.drawLine(self.__rectHead.left() + borSize / 2, self.__rectHead.bottom() - borSize / 2, self.__rectHead.right() - borSize / 2, self.__rectHead.bottom() - borSize / 2) painter.setPen(QPen(QColor(0, 0, 0, opacity))) # draw text of header if len(self.__headRects) > 1: # align the first head text at the bottom to provide some space at the top painter.setFont(self.__headRects[0].font) painter.drawText(self.__headRects[0].rect, Qt.AlignHCenter | Qt.AlignBottom, self.__headRects[0].text) # align other head texts at the center for i in range(1, len(self.__headRects) - 1): painter.setFont(self.__headRects[i].font) painter.drawText(self.__headRects[i].rect, Qt.AlignCenter, self.__headRects[i].text) # align the last head text at the top to provide some space at the bottom painter.setFont(self.__headRects[-1].font) painter.drawText(self.__headRects[-1].rect, Qt.AlignHCenter | Qt.AlignTop, self.__headRects[-1].text) # there is only one head text, so align it at the center elif len(self.__headRects) == 1: painter.setFont(self.__headRects[0].font) painter.drawText(self.__headRects[0].rect, Qt.AlignHCenter | Qt.AlignCenter, self.__headRects[0].text) # draw blob names painter.setFont(Constants.nodeItemFontBlob) for item in self.__blobBottomNameRects: painter.drawText(item.rect, Qt.AlignVCenter | Qt.AlignLeft, item.text) for item in self.__blobTopNameRects: painter.drawText(item.rect, Qt.AlignVCenter | Qt.AlignRight, item.text) gui/main_window/docks/plotter/__init__.py0000664ڊ00111170000000000013243274037021000 0ustar klemmso0cvprgui/main_window/docks/plotter/plot_canvas.py0000664ڊ00111170000000402513243274037021565 0ustar klemmso0cvpr from PyQt5.QtWidgets import QSizePolicy from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from matplotlib.figure import Figure def handleNonPositives(yValues): allNonPositive = [] for i in range(0,len(yValues)): for j in range(0,len(yValues[i])): if len(allNonPositive) < j + 1: allNonPositive.append(True) if yValues[i][j] > 0: allNonPositive[j] = False for i in range(0,len(yValues)): yValues[i] = [yValues[i][j] for j in range(0,len(allNonPositive)) if not allNonPositive[j]] return yValues def allNonNegative(yValues): for ys in yValues: for y in ys: if y > 0: return False return True class PlotCanvas(FigureCanvas): """ This plotter canvas draws the actual plots within the plotter widget. """ def __init__(self, parent=None, width=5, height=4, dpi=100): self.fig = Figure(figsize=(width, height), dpi=dpi) self.axes = self.fig.add_subplot(111) #self.axes.hold(True) # MatPlotLib hold depricated http://matplotlib.org/api/api_changes.html self.axes.grid(True) FigureCanvas.__init__(self, self.fig) self.setParent(parent) FigureCanvas.setSizePolicy(self, QSizePolicy.Expanding, QSizePolicy.Expanding) FigureCanvas.updateGeometry(self) def plotDictList(self, xValues, yValues, logarithmic): """ Plot a multiple charts on the axes. Return the Line2D docks. """ self.axes.grid(True) if logarithmic and not allNonNegative(yValues): try: return self.axes.semilogy(xValues, handleNonPositives(yValues)) except: pass return self.axes.plot(xValues, yValues) def legend(self, lines, labels): self.axes.legend(lines, labels, loc = 0, fontsize=10) def clear(self): self.axes.clear() self.axes.grid(True) gui/main_window/docks/plotter/qt5_plotter.py0000664ڊ00111170000003423513243274037021544 0ustar klemmso0cvprimport csv import time import seaborn from collections import OrderedDict from PyQt5 import QtCore from PyQt5.QtWidgets import QVBoxLayout, QWidget, QFrame from backend.parser.parser_listener import ParserListener from backend.parser.parser import Parser from backend.parser.parser_dummy import ParserDummy from gui.main_window.docks.plotter.plot_canvas import PlotCanvas # Own class for plotter # Select Train/Test # Select one or more value types: training rate, acc0, acc1, loss # Plot against time or iteration # Label the coordinates # More features? Multiple Plots... class Qt5Plotter(QFrame): """ This plotter takes a list of dictionaries as input and plottes selected data """ # available metrics global TIME, ITERATION, LEARNING_RATE TIME = "Seconds" ITERATION = "NumIters" LEARNING_RATE = "LearningRate" # How will be plottet? Time <-> Iterations, Linear <-> Logarithmic __againstTime = False __logarithmic = False # Each parser (i.e. log) has a list of metrics of train and test values # that the user wants to be plotted # Each list entry is a set of the metric that will be plotted # logId -> {metric} __testMetrics = OrderedDict() __trainMetrics = OrderedDict() # Each parser (i.e. log) has a list of train and test data # Each list entry is again a dictionary of the parsed key-value-pairs # logId -> [key -> value] __trainData = OrderedDict() __testData = OrderedDict() # Logs can be plotted with an offset in time/iterations # This is a usefull parameter for concatenating logs __timeOffsets = OrderedDict() __iterationOffsets = OrderedDict() # The list of parsers corresponds to the list of logs that are observed. # Each log (file or stream) must be added using a parser # logId -> parser __parsers = OrderedDict() __listener = OrderedDict() __plotterGUI = None class ParserConnection(ParserListener): """ Implements the required methods for the ParserListener so that the parsed data can be added to the data that will be plotted. """ def __init__(self, logId, testData, trainData, plotter, plotOnUpdate=False): """ Register the test and train data lists of the plotter. Optionaly set plotOnUpdate to True to replot the data every time the parser reads a new row and triggers an update. """ self.logId = logId self.plotter = plotter self.testData = testData self.trainData = trainData self.plotOnUpdate = plotOnUpdate def update(self, phase, row): """ Pass the update signal to the plotter. Replot if that is wanted. """ if phase == "TEST": self.testData.append(row) if phase == "TRAIN": self.trainData.append(row) if self.plotOnUpdate: self.plotter.plot(False, [self.logId]) def handle(self, event, message, groups): # Ignore event. # Not the plotters business. pass def registerKey(self, phase, key): """ Pass the signal to the gui so that it can create a new checkbox for the key. """ gui = self.plotter.getPlotterGUI() if gui: gui.registerKeySignal.emit(self.logId, phase, key) # end ParserConnection def __init__(self, parent=None): super(Qt5Plotter, self).__init__(parent) self.setFrameShape(QFrame.StyledPanel) # this is the Canvas Widget that displays the `figure` self.canvas = PlotCanvas(self) # this is the Navigation widget # it takes the Canvas widget and a parent # self.toolbar = NavigationToolbar(self.canvas, self) # set the layout layout = QVBoxLayout() layout.setContentsMargins(0,0,0,0) # layout.addWidget(self.toolbar) layout.addWidget(self.canvas) self.setLayout(layout) def resizeEvent(self, event): # This keeps the plot area as large as possible even after resizing the widget try: self.canvas.fig.tight_layout() except: pass def setPlotterGUI(self, plotterGUI): self.__plotterGUI = plotterGUI def getPlotterGUI(self): return self.__plotterGUI def getParser(self, logId): """ Get the parser that corresponds to the given log name. """ return self.__parsers[logId][1] def putLog(self, logId, logName, parser, plotOnUpdate=False, timeOffset=0, iterationOffset=0): """ Add a new log to be plotted. A log is added by adding a parser that reads the log file/stream. A logId is used to identify the logs/parsers It is possible to add offset values for time and iteration numbers This is usually only interesting for the time, e.g. if a training is interruped and resumed (usually the iteration number in the log-file is still constistent. """ if (isinstance(parser, Parser) | isinstance(parser, ParserDummy)) & isinstance(logId, str): self.__testData[logId] = [] self.__trainData[logId] = [] self.__testMetrics[logId] = [] self.__trainMetrics[logId] = [] self.__timeOffsets[logId] = timeOffset self.__iterationOffsets[logId] = iterationOffset self.__listener[logId] = self.ParserConnection( logId, self.__testData[logId], self.__trainData[logId], self, plotOnUpdate) parser.addListener(self.__listener[logId]) self.__parsers[logId] = [logName, parser] def removeLog(self, logId): """ Remove the given log from the plot. """ if logId: if logId in self.__parsers: if logId in self.__listener: self.__parsers[logId][1].removeListener(self.__listener[logId]) self.__parsers.pop(logId) def showMetricTest(self, logId, metric, show=True): """ Add a metric to the test plot if show is true, otherwise remove the metric. logId is the name of the logfile which is plotted. """ if logId not in self.__testMetrics: return if show and metric not in self.__testMetrics[logId]: self.__testMetrics[logId].append(metric) elif not show and metric in self.__testMetrics[logId]: self.__testMetrics[logId].remove(metric) def isMetricTestShown(self, logId, metric): return metric in self.__testMetrics[logId] def showMetricTrain(self, logId, metric, show=True): """ Add a metric to the train plot if show is true, otherwise remove the metric. logId is the name of the logfile which is plotted. """ if logId not in self.__testMetrics: return if show and metric not in self.__trainMetrics[logId]: self.__trainMetrics[logId].append(metric) elif not show and metric in self.__trainMetrics[logId]: self.__trainMetrics[logId].remove(metric) def isMetricTrainShown(self, logId, metric): return metric in self.__trainMetrics[logId] def plotAgainstTime(self): self.__againstTime = True def plotAgainstIterations(self): self.__againstTime = False def plotLogarithmic(self): self.__logarithmic = True def plotLinear(self): self.__logarithmic = False # # Some helping methodes for the plotting # # # # # # # # # # # # # def __xLabel(self): if self.__againstTime: return "Time (in s)" else: return "Iterations" def __xValues(self, dictList, logId, againstTime): xValues = [] if againstTime: for dict in dictList: xValues.append(dict.get(TIME) + self.__timeOffsets[logId]) else: for dict in dictList: xValues.append(dict.get(ITERATION) + self.__iterationOffsets[logId]) return xValues def __yValues(self, dictList, showMetrics): yValues = [] for record in dictList: values = [] for metric in showMetrics: values.append(record.get(metric)) yValues.append(values) return yValues # Count the metrics the user wants to plot def numGraphs(self): numGraphs = 0 for logId in self.__parsers: numGraphs = numGraphs + len(self.__trainMetrics[logId]) + len(self.__testMetrics[logId]) return numGraphs def plot(self, evenIfThereIsNothingToPlot = True, updateOnly = None): """ Plot the data. This method will don't call the final plot function of the actual plotting library if there is no data selected for plotting. Otherwise there could result an unpretty visual flickering in the widget. One can plot anyway by setting the first parameter to False. In addition one can further specify which logs should be checked in order to avoid unnecessary replotting. """ # Count the metrics the user wants to plot # Only plot it if the number is greater than 0 or if the user wants to plot the empty plot numGraphs = 0 relevantParsers = self.__parsers if updateOnly is not None: relevantParsers = updateOnly for logId in relevantParsers: numGraphs = numGraphs + len(self.__trainMetrics[logId]) + len(self.__testMetrics[logId]) if numGraphs == 0 and not evenIfThereIsNothingToPlot: return # Begin plotting self.canvas.clear() lines = [] labels = [] for logId in self.__parsers: if len(self.__trainMetrics[logId]) != 0: xValues = self.__xValues(self.__trainData[logId], logId, self.__againstTime) yValues = self.__yValues(self.__trainData[logId], self.__trainMetrics[logId]) lines.extend(self.canvas.plotDictList(xValues, yValues, self.__logarithmic)) train = lambda s: self.__parsers[logId][0] + ".train." + s labels.extend(map(train, self.__trainMetrics[logId])) if len(self.__testMetrics[logId]) != 0: xValues = self.__xValues(self.__testData[logId], logId, self.__againstTime) yValues = self.__yValues(self.__testData[logId], self.__testMetrics[logId]) lines.extend(self.canvas.plotDictList(xValues, yValues, self.__logarithmic)) test = lambda s: self.__parsers[logId][0] + ".test." + s labels.extend(map(test, self.__testMetrics[logId])) self.canvas.legend(lines, labels) try: self.canvas.draw() except: pass def getLastTimeValue(self, logId, ofTestData=False): if ofTestData: return self.__testData[logId][-1][TIME] else: return self.__trainData[logId][-1][TIME] def getLastIterationValue(self, logId, ofTestData=False): if ofTestData: return self.__testData[logId][-1][ITERATION] else: return self.__trainData[logId][-1][ITERATION] # Methods for CSV export # # # # # # # # # # # # # # # # # # # # # # # def __arrayToString(self, array, delimiter): if array is None or len(array) == 0: return "" else: string = array[0] for i in range(1, len(array)): string = string + delimiter + " " + array[i] return string def exportCSVToFile(self, path): if not path: return with open(path, 'wb') as csvfile: spamwriter = csv.writer(csvfile, delimiter=';', quotechar='|', quoting=csv.QUOTE_MINIMAL) spamwriter.writerow(["# Barista CSV-Export (" + time.ctime() + ")"]) for logId in self.__parsers: names = [] if len(self.__trainMetrics[logId]) != 0: xIterValues = self.__xValues(self.__trainData[logId], logId, False) xTimeValues = self.__xValues(self.__trainData[logId], logId, True) yValues = self.__yValues(self.__trainData[logId], self.__trainMetrics[logId]) train = lambda s: self.__parsers[logId][0] + ".train." + s names = map(train, self.__trainMetrics[logId]) spamwriter.writerow([]) spamwriter.writerow([]) spamwriter.writerow(["# " + self.__parsers[logId][0] + " TRAIN"]) row = ["Iterations", "Time"] row.extend(names) spamwriter.writerow(row) for i in range(0, len(xIterValues)): row = [xIterValues[i], xTimeValues[i]] row.extend(yValues[i]) spamwriter.writerow(row) if len(self.__testMetrics[logId]) != 0: xIterValues = self.__xValues(self.__testData[logId], logId, False) xTimeValues = self.__xValues(self.__testData[logId], logId, True) yValues = self.__yValues(self.__testData[logId], self.__testMetrics[logId]) test = lambda s: self.__parsers[logId][0] + ".test." + s spamwriter.writerow([]) spamwriter.writerow([]) spamwriter.writerow(["# " + self.__parsers[logId][0] + " TEST"]) names = map(test, self.__testMetrics[logId]) row = ["Iterations", "Time"] row.extend(names) spamwriter.writerow(row) for i in range(0, len(xIterValues)): row = [int(xIterValues[i]), xTimeValues[i]] row.extend(yValues[i]) spamwriter.writerow(row) gui/main_window/docks/plotter/qt5_plotter_gui.py0000664ڊ00111170000005316513243274037022413 0ustar klemmso0cvprimport os from PyQt5 import QtGui, QtCore, QtWidgets from collections import OrderedDict from PyQt5.QtCore import Qt, pyqtSignal, QVariant from PyQt5.QtWidgets import ( QComboBox, QHBoxLayout, QVBoxLayout, QGridLayout, QPushButton, QCheckBox, QListWidget, QFileDialog, QStackedWidget, QWidget, QLabel, QToolButton, QMenu, QWidgetAction, QStyle, QApplication, QMessageBox, QListWidgetItem) from backend.barista.utils.logger import Log from backend.parser.parser import Parser from gui.main_window.docks.dock import DockElement from gui.main_window.docks.plotter.qt5_plotter import Qt5Plotter class DockElementPlotter(DockElement): selected = None registerKeySignal = pyqtSignal(str, str, str) __settings = {} def __init__(self, mainWindow, title): DockElement.__init__(self, mainWindow, 'Plotter') self.name = title self.setWindowTitle(title) self.setupUi() def setupUi(self): self.resize(500, 300) self.settingsWidget = QWidget() self.settingsWidget.setMinimumSize(900,150) # Create the main widget as a container widget. self.mainWidget = QWidget(self) self.setWidget(self.mainWidget) mainLayout = QVBoxLayout(self.mainWidget) # Create 'Settings' dropdown button. self.settingsButton = QToolButton(self) self.settingsButton.setText("Settings") # Remove the menu indicator by applying a global style. self.settingsButton.setObjectName("noIndicator") # And replace it with a down arrow icon instead (which looks better). self.settingsButton.setArrowType(QtCore.Qt.DownArrow) self.settingsButton.setToolButtonStyle(QtCore.Qt.ToolButtonTextBesideIcon) # Set the popup that opens on button click. self.settingsButton.setPopupMode(QToolButton.InstantPopup) action = QWidgetAction(self.settingsButton) action.setDefaultWidget(self.settingsWidget) self.settingsButton.addAction(action) # Create a horizontal layout for the settings button. buttonLayout = QHBoxLayout() buttonLayout.addWidget(self.settingsButton) buttonLayout.addStretch() mainLayout.addLayout(buttonLayout) # Create the plotter widget. self.plotter = Qt5Plotter() self.plotter.setPlotterGUI(self) self.plotter.setMinimumSize(300,200) self.plotter.plotLinear() mainLayout.addWidget(self.plotter) # Create the settings dialog. self.setupSettingsDialogUi() # Set keys that are plotted by default self.defaultKeys = {"TEST": ["loss"], "TRAIN": ["loss"]} def setupSettingsDialogUi(self): self.settingsDialogLayout = QGridLayout() # DropDown for plotAgainstTime or Iterations self.dbTimeIter = QComboBox() self.dbTimeIter.addItem("Iterations") self.dbTimeIter.addItem("Time") self.dbTimeIter.setFixedWidth(130) self.dbTimeIter.currentIndexChanged.connect(self.plotAgainstTimeIter) self.settingsDialogLayout.addWidget(self.dbTimeIter, 1, 3) # DropDown for plotting scale linear or logarithmic self.dbLinLoga = QComboBox() self.dbLinLoga.addItem("Linear") self.dbLinLoga.addItem("Logarithmic") self.dbLinLoga.setFixedWidth(130) self.dbLinLoga.currentIndexChanged.connect(self.plotLinLoga) self.settingsDialogLayout.addWidget(self.dbLinLoga, 3,3) # Dynamic checkboxes # Key lists from parser that define the table rows for train and test self.test_dict_list = [] self.train_dict_list = [] # Lists for checkboxes for train and test self.test_cb = [] self.train_cb = [] # Text labels for better display testLabel = QLabel("Test keys:") trainLabel = QLabel("Train keys:") timeIterLabel = QLabel("Plot against:") LinLogaLabel = QLabel("Plot scale:") self.settingsDialogLayout.addWidget(testLabel,0,4) self.settingsDialogLayout.addWidget(trainLabel,2,4) self.settingsDialogLayout.addWidget(timeIterLabel,0,3) self.settingsDialogLayout.addWidget(LinLogaLabel,2,3) # create default Checkboxes self.createCheckboxes(self.test_dict_list, self.train_dict_list) # Button select Logfiles self.btn1 = QPushButton(QtGui.QIcon(QApplication.style().standardIcon(QStyle.SP_DialogOpenButton)), "") self.btn1.setFixedWidth(40) self.btn1.clicked.connect(self.getFiles) self.settingsDialogLayout.addWidget(self.btn1,3,0) # Button remove item self.btn2 = QPushButton(QtGui.QIcon(QApplication.style().standardIcon(QStyle.SP_TrashIcon)), "") self.btn2.setFixedWidth(40) self.btn2.clicked.connect(self.removeItem) self.settingsDialogLayout.addWidget(self.btn2,3,1) # Button CSV export self.btn3 = QPushButton("Export as CSV") self.btn3.setFixedWidth(110) self.btn3.clicked.connect(self.exportCSV) self.settingsDialogLayout.addWidget(self.btn3,3,2) # Create Logfile list self.leftlist = QListWidget() self.leftlist.setFixedSize(200,100) self.Stack = QStackedWidget(self) self.settingsDialogLayout.addWidget(self.Stack,0,0,3,3) self.settingsDialogLayout.addWidget(self.leftlist,0,0,3,3) self.leftlist.currentRowChanged.connect(self.display) self.leftlist.currentItemChanged.connect(self.selectedLogItem) self.settingsWidget.setLayout(self.settingsDialogLayout) # Signal handling self.registerKeySignal.connect(self.registerKey) def btnstatetest(self, checkbox): """ Plot the selected checkbox for Test-Data """ if self.selected: self.plotter.showMetricTest(self.selected, checkbox.text(), checkbox.isChecked()) self.displayPlot() self.__settings["checkBoxes"][self.selected]["TEST"][checkbox.text()] = checkbox.isChecked() def btnstatetrain(self, checkbox): """ Plot the selected checkbox for Train-Data """ if self.selected: self.plotter.showMetricTrain(self.selected, checkbox.text(), checkbox.isChecked()) self.displayPlot() self.__settings["checkBoxes"][self.selected]["TRAIN"][checkbox.text()] = checkbox.isChecked() def displayPlot(self): """ Update the plotter. """ if self.selected: self.plotter.plot() self.plotter.show() self.createNecessaryDocks() def createNecessaryDocks(self): """ Create the necessary docks if they don't exist. """ if self.__settings is None: self.__settings = {} if "checkBoxes" not in self.__settings: self.__settings["checkBoxes"] = {} if self.selected not in self.__settings["checkBoxes"] or \ isinstance(self.__settings["checkBoxes"][self.selected], list): self.__settings["checkBoxes"][self.selected] = {"TEST": {}, "TRAIN": {}} def existsKeySelection(self): """ Returns true if there is a selection by the user of keys to be plotted """ if self.__settings is None: self.__settings = {} if "checkBoxes" not in self.__settings: self.__settings["checkBoxes"] = {} if self.selected not in self.__settings["checkBoxes"]: return False return True def createCheckboxes(self, test_list, train_list): """ Dynamically creates checkboxes, which are in the lists: test_list and train_list """ self.test_dict_list = test_list self.train_dict_list = train_list # As long as there are elements in TEST list: create Checkboxes if self.test_dict_list is not None: for i in range(len(self.test_dict_list)): self.test_cb.append(QCheckBox(self.test_dict_list[i], self)) self.settingsDialogLayout.addWidget(self.test_cb[i], 1, 4+i) self.test_cb[i].stateChanged.connect(lambda checked, i=i: self.btnstatetest(self.test_cb[i])) if self.existsKeySelection() and self.test_cb[i].text() in \ self.__settings["checkBoxes"][self.selected]["TEST"]: # Compatibility for older projects try: self.test_cb[i].setChecked( self.__settings["checkBoxes"][self.selected]["TEST"].get(self.test_cb[i].text(), True)) except AttributeError: self.test_cb[i].setChecked(True) else: self.test_cb[i].setChecked(self.test_cb[i].text() in self.defaultKeys["TEST"]) self.btnstatetest(self.test_cb[i]) # As long as there are elements in TRAIN list: create Checkboxes if self.train_dict_list is not None: for i in range(len(self.train_dict_list)): self.train_cb.append(QCheckBox(self.train_dict_list[i], self)) self.settingsDialogLayout.addWidget(self.train_cb[i], 3, 4+i) self.train_cb[i].stateChanged.connect(lambda checked, i=i: self.btnstatetrain(self.train_cb[i])) if self.existsKeySelection() and self.train_cb[i].text() in \ self.__settings["checkBoxes"][self.selected]["TRAIN"]: # Compatibility for older projects try: self.train_cb[i].setChecked( self.__settings["checkBoxes"][self.selected]["TRAIN"].get(self.train_cb[i].text(), True)) except AttributeError: self.train_cb[i].setChecked(True) else: self.train_cb[i].setChecked(self.train_cb[i].text() in self.defaultKeys["TRAIN"]) self.btnstatetrain(self.train_cb[i]) self.settingsWidget.setLayout(self.settingsDialogLayout) def display(self, i): self.Stack.setCurrentIndex(i) def loadFiles(self, filenames): """ Loads every log file in the list of filenames and adds them to the list of logs. """ for i in range(len(filenames)): f = filenames[i] if not os.path.exists(f): Log.error("External log not found: " + f, Log.getCallerId("plotter")) else: parser = Parser(open(f, 'r'), OrderedDict(), Log.getCallerId(str(f))) head, tail = os.path.split(str(f)) logId = "external_" + tail logName = "[ext] " + tail self.putLog(logId, parser, logName=logName) parser.parseLog() # This is for saving the loaded logs in the project file # (create the necessary docks if they don't exist) if self.__settings is None: self.__settings = {"logFiles": {logId: f}} elif "logFiles" not in self.__settings: self.__settings["logFiles"] = {logId: f} else: self.__settings["logFiles"][logId] = f def getFiles(self): """ Dialog for adding different data files to list for plotting """ dlg = QFileDialog() dlg.setFileMode(QFileDialog.ExistingFiles) if dlg.exec_(): self.loadFiles(dlg.selectedFiles()) self.selectLast() def plotAgainstTimeIter(self): if self.dbTimeIter.currentText() == "Time": self.plotter.plotAgainstTime() self.__settings["againstTime"] = "Time" self.plotter.plot() else: if self.dbTimeIter.currentText() == "Iterations": self.plotter.plotAgainstIterations() self.__settings["againstTime"] = "Iterations" self.plotter.plot() def plotLinLoga(self): if self.dbLinLoga.currentText() == "Linear": self.plotter.plotLinear() self.__settings["logarithmic"] = "Linear" self.plotter.plot() else: if self.dbLinLoga.currentText() == "Logarithmic": self.plotter.plotLogarithmic() self.__settings["logarithmic"] = "Logarithmic" self.plotter.plot() def putLog(self, logId, parser, plotOnUpdate=False, logName=None): """ Add a log (i.e. parser) to the list of shown logs. If a log with the log name already exists primes are added to the name until it is unique. """ for i in range(self.leftlist.count()): if logId == self.leftlist.item(i).data(Qt.UserRole): return if not logName: logName = logId self.test_dict_list = [] self.train_dict_list = [] while self.leftlist.findItems(logName, Qt.MatchExactly): logName += "'" self.plotter.putLog(logId, logName, parser, plotOnUpdate) logItem = QListWidgetItem(logName) v = QVariant(logId) logItem.setData(Qt.UserRole, v) self.leftlist.insertItem(self.leftlist.count(), logItem) self.selectLast() def removeLog(self, logId, logName): try: self.__settings["checkBoxes"].pop(logId) except KeyError: pass self.plotter.removeLog(logId) item = self.leftlist.findItems(logName, Qt.MatchExactly) if item: row = self.leftlist.row(item[0]) self.leftlist.takeItem(row) self.test_dict_list = [] self.train_dict_list = [] self.selectLast() self.plotter.plot() self.plotter.show() def registerKey(self, logId, phase, key): """ Registers the key given by the signal. This is only of interest if the key belongs to the selected log file. A new checkbox is then created for that key. """ if self.leftlist.currentItem(): if str(self.leftlist.currentItem().data(Qt.UserRole)) == logId: if phase == "TEST": self.test_dict_list.append(key) if phase == "TRAIN": self.train_dict_list.append(key) self.updateCheckboxes(self.test_dict_list, self.train_dict_list) def removeItem(self): """ Removes the selected item from the list and also from plot. """ item = self.leftlist.takeItem(self.leftlist.currentRow()) if item is not None: self.removeLog(item.data(Qt.UserRole), item.text()) self.displayPlot() self.selectLast() if (self.__settings is not None and "logFiles" in self.__settings and str(item.data(Qt.UserRole)) in self.__settings["logFiles"]): del self.__settings["logFiles"][str(item.data(Qt.UserRole))] def selectLast(self): if len(self.leftlist) > 0: self.leftlist.setCurrentItem(self.leftlist.item(len(self.leftlist) - 1)) self.selectedLogItem(self.leftlist.item(len(self.leftlist) - 1)) else: self.test_dict_list = [] self.train_dict_list = [] self.updateCheckboxes(self.test_dict_list,self.train_dict_list) def selectedLogItem(self, item, update=True): """ Memorize which logfile is selected, block signals, and asks plotter if the checkbox is selected or not. """ if item: if update: self.updateCheckboxes( list(self.plotter.getParser(str(item.data(Qt.UserRole))).getKeys('TEST')), list(self.plotter.getParser(str(item.data(Qt.UserRole))).getKeys('TRAIN'))) self.selected = str(str(item.data(Qt.UserRole))) self.tickBoxes() self.__settings["selectedIndex"] = self.leftlist.row(item) def updateCheckboxes(self, test_list, train_list): """ Clear all checkboxes and clear the layer and create new checkboxes and add them to the layer again """ for i in range(len(self.test_cb)): self.settingsDialogLayout.removeWidget(self.test_cb[i]) self.test_cb[i].deleteLater() self.test_cb[i] = None for i in range(len(self.train_cb)): self.settingsDialogLayout.removeWidget(self.train_cb[i]) self.train_cb[i].deleteLater() self.train_cb[i] = None self.test_cb = [] self.train_cb = [] # create new Checkboxes with new Keys self.createCheckboxes(test_list, train_list) self.tickBoxes() def tickBoxes(self): """ Whether a plot is shown is stored in the plotter object. This method ticks the checkboxes of the keys whose plot is shown in the plotter. """ for i in range(len(self.test_cb)): self.test_cb[i].blockSignals(True) self.test_cb[i].setChecked(self.plotter.isMetricTestShown(self.selected, self.test_cb[i].text())) self.test_cb[i].blockSignals(False) for i in range(len(self.train_cb)): self.train_cb[i].blockSignals(True) self.train_cb[i].setChecked(self.plotter.isMetricTrainShown(self.selected, self.train_cb[i].text())) self.train_cb[i].blockSignals(False) def setProject(self, project): """ This sets the project object. The plotter settings have the following structure: "plotter": { "checkBoxes": { "logName1": { "TEST": { "accuracy": true, "loss": false }, "TRAIN": { "LearningRate": true, "loss": false } }, "logName2": { ... }, ... }, "selectedIndex": 0, "againstTime": "Iterations", "logarithmic": "Logarithmic" } """ if "plotter" not in project.getSettings(): project.getSettings()["plotter"] = self.__settings else: self.__settings = project.getSettings()["plotter"] selectedIndex = -1 if "selectedIndex" in self.__settings: selectedIndex = self.__settings["selectedIndex"] if "logFiles" in self.__settings: self.loadFiles(list(self.__settings["logFiles"].values())) if "checkBoxes" in self.__settings: for name in self.__settings["checkBoxes"]: for key in self.__settings["checkBoxes"][name]["TEST"]: # Compatibility for older projects try: self.plotter.showMetricTest(name, key, self.__settings["checkBoxes"][name]["TEST"].get(key)) except AttributeError: self.plotter.showMetricTest(name, key, True) for key in self.__settings["checkBoxes"][name]["TRAIN"]: # Compatibility for older projects try: self.plotter.showMetricTrain(name, key, self.__settings["checkBoxes"][name]["TRAIN"].get(key)) except AttributeError: self.plotter.showMetricTrain(name, key, True) if "againstTime" in self.__settings: index = self.dbTimeIter.findText(self.__settings["againstTime"], Qt.MatchFixedString) if index >= 0: self.dbTimeIter.setCurrentIndex(index) if "logarithmic" in self.__settings: index = self.dbLinLoga.findText(self.__settings["logarithmic"], Qt.MatchFixedString) if index >= 0: self.dbLinLoga.setCurrentIndex(index) #for sid in project.getSessions(): # session = project.getSession(sid) # self.putLog(str(session.getLogId()), session.getParser(), True, str(session.getLogFileName(True))) self.plotter.plot() if selectedIndex >= 0: self.leftlist.setCurrentItem(self.leftlist.item(selectedIndex)) self.selectedLogItem(self.leftlist.item(selectedIndex), False) else: self.selectLast() # Delete plot if session is deleted project.deleteSession.connect(lambda sid: self.removeLog(str(project.getSession(sid).getLogId()), str(project.getSession(sid).getLogFileName(True)))) project.deleteSession.connect(lambda sid: project.deletePlotterSettings(str(project.getSession(sid).getLogId()))) project.resetSession.connect(lambda sid: self.removeLog(str(project.getSession(sid).getLogId()), str(project.getSession(sid).getLogFileName(True)))) project.resetSession.connect(lambda sid: project.deletePlotterSettings(str(project.getSession(sid).getLogId()))) def exportCSV(self): """ Opens a file dialog and the plotter saves the shown plots to the selected file. The file dialog only appears if there is at least one plot. """ if self.plotter.numGraphs() == 0: QMessageBox.question(self, 'PyQt5 message', "Nothing selected!", QMessageBox.Ok, QMessageBox.Ok) else: path = str(QFileDialog.getSaveFileName(filter="CSV table (*.csv) ;; Text file (*.txt)")[0]) if path: if not (path.endswith(".csv") or path.endswith(".txt")): path = path + ".csv" self.plotter.exportCSVToFile(path) gui/main_window/docks/properties/__init__.py0000664ڊ00111170000000000013243274037021503 0ustar klemmso0cvprgui/main_window/docks/properties/dock_layer_properties.py0000664ڊ00111170000000472313243274037024354 0ustar klemmso0cvprfrom PyQt5 import QtWidgets from gui.main_window.docks.dock import DockElement from gui.network_manager.network_manager import NetworkManager from gui.main_window.docks.properties.properties_data_model import PropertiesDataModel from gui.main_window.docks.properties.properties_widget import PropertiesWidget class LayerProperties(DockElement): def __init__(self, mainWindow, title): DockElement.__init__(self, mainWindow, "Layer Properties") self.tab = QtWidgets.QTabWidget() self.tab.setDocumentMode(True) self.setWidget(self.tab) self.ids = [] self.datadict = {} self._mainWindow = mainWindow self.disableEditing(mainWindow.getDisabled()) def setTab(self, id): """ Set the layer whose properties should be shown by its id """ self.tab.clear() self.addTab(id) def addTab(self, id): """ Add a new property editor for the layer with the given id """ network = self._networkmanager.network falloutlen = len(network.uri()) layer = network["layers"][id] data = layer["parameters"] widget = PropertiesWidget(self) widget.setModel(PropertiesDataModel(data)) self.tab.addTab(widget, data["name"]) self.ids.append(id) self.datadict[id] = data self.disableEditing(self._mainWindow.getDisabled()) def setNetworkManager(self, networkManager): self._networkmanager = networkManager # type: NetworkManager def setTabs(self, ids): """ Set the layers whose properties should be shown by its id. ids is a list of string. """ self.clearProperties() for id in ids: self.addTab(id) def clearProperties(self): """ Let the dock show no properties for any layer """ self.ids = [] self.datadict = {} self.tab.clear() def updateDock(self): """ Rebuild all property editors in this dock """ ids = self.ids self.clearProperties() self.setTabs(ids) def removeTab(self, id): """ Remove the property editor for the layer with the given id """ idx = self.ids.index(id) self.tab.removeTab(idx) self.ids.remove(id) del self.datadict[id] def disableEditing(self, disable): """ Disable Editing of each widget in this dock """ for i in range(0, self.tab.count()): tab = self.tab.widget(i) tab.disableEditing(disable) gui/main_window/docks/properties/dock_solver_properties.py0000664ڊ00111170000000306213243274037024545 0ustar klemmso0cvprfrom PyQt5 import QtCore, QtWidgets, QtGui from PyQt5.QtCore import pyqtSignal from gui.main_window.docks.properties.properties_data_model import PropertiesDataModel from gui.main_window.docks.properties.properties_widget import PropertiesWidget from gui.main_window.docks.dock import DockElement class SolverProperties(DockElement): # This signal will be emitted when any of the properties of the solver change. solverChanged = pyqtSignal(list, object) def __init__(self, mainWindow, title): DockElement.__init__(self, mainWindow, "Solver Properties") self._data = None self.setToolTip('This shows the Properties the loaded Solver.') self.setupUi() self._mainWindow = mainWindow self.disableEditing(mainWindow.getDisabled()) def setupUi(self): self.tree = PropertiesWidget(self) self.setWidget(self.tree) def setSolver(self, reactiveSolver): """ Set the solver whose properties should be shown. reactiveSolver has to be an instance of PropertyData """ if self._data: self._data.someChildPropertyChanged.disconnect(self.solverChanged) self._data = reactiveSolver self.tree.setModel(PropertiesDataModel(self._data, self.tree)) self._data.someChildPropertyChanged.connect(self.solverChanged) self.disableEditing(self._mainWindow.getDisabled()) def data(self): return self._data def disableEditing(self, disable): """ Disable Editing of this dock """ self.tree.disableEditing(disable) gui/main_window/docks/properties/properties_data_model.py0000664ڊ00111170000002337213243274037024332 0ustar klemmso0cvprfrom PyQt5 import QtCore, QtGui, QtWidgets from PyQt5.Qt import QApplication from PyQt5.QtCore import QModelIndex, QVariant from gui.main_window.docks.properties.property_widgets.data import * from gui.main_window.docks.properties.property_widgets.types import * def getChild(data, row): if isinstance(data, PropertyDataGroupObject): data = data.value() if isinstance(data, PropertyData): key = sortPropertyKeys(data.getBoringDict().keys())[row] c = data.getChildDataObject(key) elif isinstance(data, PropertyDataDictObject): key = sortPropertyKeys(data.value().keys())[row] c = data.value()[key] else: c = data.value()[row] return c def getChildCount(data): if isinstance(data, PropertyDataGroupObject): data = data.value() if isinstance(data, PropertyData): return len(data.givenProperties()) elif isinstance(data, PropertyDataListObject): return len(data.value()) elif isinstance(data, PropertyDataDictObject): return len(data.value()) return 0 def getColumnCount(data): return 2 def getParent(data, root): # IMPORTANT: PropertyDataGroupObject and contained PropertyData have the same # uri. if root == data: return None # Find uri from root to parent. uri = data.uri()[len(root.uri()):-1] for key in uri: try: if isinstance(root, PropertyData): root = root.getChildDataObject(key) elif isinstance(root, PropertyDataGroupObject): root = root.value().getChildDataObject(key) else: root = root.value()[key] except: print("[ERROR] Key '%s' unexpectedly not found in property '%s'." % (str(key), str(root.value()))) return None return root def getRow(data, root): parent = getParent(data, root) key = data.uri()[-1] if isinstance(parent, PropertyData): return sortPropertyKeys(parent.getBoringDict().keys()).index(key) elif isinstance(parent, PropertyDataDictObject): return sortPropertyKeys(parent.values().keys()).index(key) elif isinstance(parent, PropertyDataListObject): return key return 0 def getName(data): return str(data.uri()[-1]) def getValue(data): if type(data) != PropertyDataObject: return '' return data.value() def isRequired(data): return not isinstance(data, PropertyDataObject) or data.info().isRequired() def isEditable(data): return type(data) == PropertyDataObject and data.info().isEditable() def hasProperty(data, key): try: return key in data except: return False def sortPropertyKeys(keys): # Sort the keys normally. result = sorted(keys) # Then bring the 'name' and 'type' properties to the front of the list if # they are available. if 'type' in result: result.insert(0, result.pop(result.index('type'))) if 'name' in result: result.insert(0, result.pop(result.index('name'))) return result class PropertiesDataModel(QtCore.QAbstractItemModel): def __init__(self, data, parent=None): """ data : PropertyData """ super(PropertiesDataModel, self).__init__(parent) self.root = data def data(self, index, role): """ Only return display data. """ if not index.isValid(): return QVariant() node = index.internalPointer() if role == QtCore.Qt.DecorationRole and index.column() == 0: if type(node) == PropertyDataObject: return QtGui.QIcon('resources/property.svg') # return QApplication.instance().style().standardIcon(getattr(QtWidgets.QStyle, 'SP_FileIcon')) return QtGui.QIcon('resources/group.svg') # return QApplication.instance().style().standardIcon(getattr(QtWidgets.QStyle, 'SP_DirIcon')) if role == QtCore.Qt.ForegroundRole and isinstance(node, PropertyDataObject) and not isEditable(node) and index.column() > 0: return QtGui.QBrush(QtGui.QColor(125, 125, 125)) if role == QtCore.Qt.ToolTipRole and isinstance(node, PropertyDataObject): if node.info().description() == '': return 'No info available.' return '' + node.info().description() + '' if role == QtCore.Qt.DisplayRole: return getName(node) if index.column() == 0 else getValue(node) return QVariant() def rowCount(self, parent): # If parent is invalid, we show the root node only, so we have 1 row. if not parent.isValid(): return 1 parentNode = parent.internalPointer() return getChildCount(parentNode) def columnCount(self, index): if not index.isValid(): return 2 # parentNode = self.root else: parentNode = index.internalPointer() return getColumnCount(parentNode) def flags(self, index): if index.isValid() and index.column() > 0 and isEditable(index.internalPointer()): return QtCore.Qt.ItemIsEditable | super(PropertiesDataModel, self).flags(index) return super(PropertiesDataModel, self).flags(index) def headerData(self, section, orientation, role): if orientation == QtCore.Qt.Horizontal and role == QtCore.Qt.DisplayRole: if section == 0: return "Property" if section == 1: return "Value" return QVariant() def index(self, row, column, parent): if not self.hasIndex(row, column, parent): return QModelIndex() if not parent.isValid(): return self.createIndex(row, column, self.root) # parentNode = self.root else: parentNode = parent.internalPointer() childNode = getChild(parentNode, row) if childNode is not None: return self.createIndex(row, column, childNode) return QModelIndex() def parent(self, index): if not index.isValid(): return QModelIndex() childNode = index.internalPointer() parentNode = getParent(childNode, self.root) if parentNode is None: return QModelIndex() row = getRow(parentNode, self.root) if row is None: return QModelIndex() return self.createIndex(row, 0, parentNode) # Below is everything that's required for writing to the model. def setData(self, index, value, role): if not index.isValid(): return False node = index.internalPointer() if type(node) != PropertyDataObject or role != QtCore.Qt.EditRole: return False try: if index.internalPointer().info().typeString() == IntType: value = int(value) elif index.internalPointer().info().typeString() == FloatType: value = float(value) elif index.internalPointer().info().typeString() == BoolType: value = bool(value) # TODO: EnumType index.internalPointer().setValue(value) self.dataChanged.emit(index, index) except: print('[ERROR] failed to set value of %s, because it had an invalid type.' % (getName(node))) return False return True def setHeaderData(self, section, orientation, value, role): return NotImplementedError() def addToGroup(self, parent, key, value=None): # Make sure that we add this as a child of the first column, to prevent # GUI update issues. parent = self.index(parent.row(), 0, parent.parent()) parentNode = parent.internalPointer() if not isinstance(parentNode, PropertyDataGroupObject) and not parentNode == self.root: return False node = parentNode if parentNode == self.root else parentNode.value() keys = node.getBoringDict().keys() if key in keys: return False keys.append(key) row = sortPropertyKeys(keys).index(key) self.beginInsertRows(parent, row, row) if value is None: node.givePropertyDefault(key) else: node.giveProperty(key, value) self.endInsertRows() return True def addToList(self, parent, value): # Make sure that we add this as a child of the first column, to prevent # GUI update issues. parent = self.index(parent.row(), 0, parent.parent()) parentNode = parent.internalPointer() if not isinstance(parentNode, PropertyDataListObject): return False row = len(parentNode.value()) self.beginInsertRows(parent, row, row) parentNode.pushBack(value) self.endInsertRows() return True def removeNode(self, index): node = index.internalPointer() parent = self.parent(index) parentNode = getParent(node, self.root) # We are only allowed to remove nodes when the parent is root, or and # instance of PropertyDataGroupObject or PropertyDataListObject. if parentNode == self.root: self.beginRemoveRows(parent, index.row(), index.row()) parentNode.ungiveProperty(node._idx) self.endRemoveRows() elif isinstance(parentNode, PropertyDataGroupObject): self.beginRemoveRows(parent, index.row(), index.row()) parentNode.value().ungiveProperty(node._idx) self.endRemoveRows() elif isinstance(parentNode, PropertyDataListObject): self.beginRemoveRows(parent, index.row(), index.row()) parentNode.removeObject(node._idx) self.endRemoveRows() # Otherwise we print an error message. else: print("[ERROR] Cannot remove property from node of type %s" % str(type(parentNode))) return False gui/main_window/docks/properties/properties_delegate.py0000664ڊ00111170000000742713243274037024016 0ustar klemmso0cvprfrom PyQt5 import QtCore, QtGui, QtWidgets from PyQt5.QtCore import QModelIndex, QVariant from gui.main_window.docks.properties.property_widgets.data import * from gui.main_window.docks.properties.property_widgets.types import * class PropertiesDelegate(QtWidgets.QStyledItemDelegate): def __init__(self, parent=None): super(PropertiesDelegate, self).__init__(parent) def createEditor(self, parent, options, index): if index.column() == 0: return super(PropertiesDelegate, self).createEditor(parent, options, index) if index.internalPointer().info().typeString() == EnumType: return self.createEnumEditor(parent, options, index) elif index.internalPointer().info().typeString() == FloatType: return self.createFloatEditor(parent, options, index) elif index.internalPointer().info().typeString() == IntType: return self.createIntEditor(parent, options, index) elif index.internalPointer().info().typeString() == BoolType: return self.createBoolEditor(parent, options, index) return super(PropertiesDelegate, self).createEditor(parent, options, index) def createEnumEditor(self, parent, options, index): data = index.internalPointer() comboBox = QtWidgets.QComboBox(parent) comboBox.addItems(data.info().enumOptions()) comboBox.setCurrentText(data.value()) return comboBox def createBoolEditor(self, parent, options, index): data = index.internalPointer() comboBox = QtWidgets.QComboBox(parent) comboBox.addItem("true") comboBox.addItem("false") comboBox.setCurrentText(str(data.value()).lower()) return comboBox def createFloatEditor(self, parent, options, index): data = index.internalPointer() spinBox = QtWidgets.QDoubleSpinBox(parent) spinBox.setRange(-pow(2, 31), pow(2, 31-1)) spinBox.setFocusPolicy(QtCore.Qt.StrongFocus ) spinBox.setDecimals(15) spinBox.setSingleStep(0.05) spinBox.setValue(data.value()) return spinBox def createIntEditor(self, parent, options, index): data = index.internalPointer() spinBox = QtWidgets.QSpinBox(parent) spinBox.setRange(-pow(2, 31), pow(2, 31-1)) spinBox.setFocusPolicy(QtCore.Qt.StrongFocus ) spinBox.setValue(data.value()) return spinBox def setEditorData(self, editor, index): data = index.internalPointer() if isinstance(editor, QtWidgets.QComboBox): if index.internalPointer().info().typeString() == BoolType: editor.setCurrentText(str(data.value()).lower()) else: editor.setCurrentText(data.value()) elif isinstance(editor, QtWidgets.QSpinBox) or isinstance(editor, QtWidgets.QDoubleSpinBox): editor.setValue(data.value()) elif isinstance(editor, QtWidgets.QLineEdit): editor.setText(data.value()) else: super(PropertiesDelegate, self).setEditorData(editor, index) def setModelData(self, editor, model, index): if isinstance(editor, QtWidgets.QComboBox): if index.internalPointer().info().typeString() == BoolType: model.setData(index, True if editor.currentText() == "true" else False, QtCore.Qt.EditRole) else: model.setData(index, editor.currentText(), QtCore.Qt.EditRole) elif isinstance(editor, QtWidgets.QSpinBox) or isinstance(editor, QtWidgets.QDoubleSpinBox): model.setData(index, editor.value(), QtCore.Qt.EditRole) elif isinstance(editor, QtWidgets.QLineEdit): model.setData(index, editor.text(), QtCore.Qt.EditRole) else: super(PropertiesDelegate, self).setModelData(editor, model, index) gui/main_window/docks/properties/properties_suggestion_model.py0000664ڊ00111170000000414413243274037025604 0ustar klemmso0cvprfrom PyQt5 import QtCore, QtGui from PyQt5.QtCore import QModelIndex, QVariant from gui.main_window.docks.properties.property_widgets.data import * class PropertiesSuggestionModel(QtCore.QAbstractListModel): def __init__(self, data=None, parent=None): super(PropertiesSuggestionModel, self).__init__(parent) self.data = data def newProperties(self): """ Return a list of properties which are not set in the GroupData """ if self.data is None: return [] # Find suggestions as list of available properties that have not been given. availableProperties = self.data.allAvailableProperties() givenProperties = [prop.info().name() for prop in self.data.givenProperties()] suggestions = [prop for name, prop in availableProperties.items() if name not in givenProperties] # Sort properties alphabetically. suggestions.sort(key=lambda x: x.name()) # Move deprecated properties to the end of the list. suggestionsFinal = [] suggestionsDeprecated = [] for prop in suggestions: if prop.deprecated(): suggestionsDeprecated.append(prop) else: suggestionsFinal.append(prop) suggestionsFinal.extend(suggestionsDeprecated) return suggestionsFinal def rowCount(self, index): return len(self.newProperties()) def data(self, index, role): if not index.isValid(): return QVariant() row = index.row() if row >= len(self.newProperties()): return QVariant() prop = self.newProperties()[row] if role == QtCore.Qt.ToolTipRole: if prop.description() == '': return 'No info available.' return '' + prop.description() + '' if role == QtCore.Qt.DisplayRole or role == QtCore.Qt.EditRole: return prop.name() + ' [' + prop.typeString() + ']' if role == QtCore.Qt.UserRole: return prop return QVariant() def update(self): self.modelReset.emit() gui/main_window/docks/properties/properties_widget.py0000664ڊ00111170000001312413243274037023516 0ustar klemmso0cvprfrom PyQt5 import QtCore, QtGui, QtWidgets from PyQt5.QtCore import QModelIndex, QVariant from PyQt5.QtWidgets import QAbstractItemView from gui.main_window.docks.properties.properties_delegate import PropertiesDelegate from gui.main_window.docks.properties.properties_data_model import PropertiesDataModel from gui.main_window.docks.properties.properties_suggestion_model import PropertiesSuggestionModel from gui.main_window.docks.properties.property_widgets.data import * class PropertiesWidget(QtWidgets.QWidget): def __init__(self, parent=None): super(PropertiesWidget, self).__init__(parent) self.disabled = False self.setupUi() def setupUi(self): mainLayout = QtWidgets.QVBoxLayout(self) # Create the treeview to display the properties. self.model = None self.tree = QtWidgets.QTreeView(self) self.tree.setObjectName("properties") self.tree.setItemDelegate(PropertiesDelegate(self.tree)) self.tree.setEditTriggers(QtWidgets.QAbstractItemView.AllEditTriggers) self.tree.header().setStretchLastSection(False) self.tree.header().setSectionResizeMode(QtWidgets.QHeaderView.Stretch) mainLayout.addWidget(self.tree) # Add a label that shows node descriptions. self.descriptionLabel = QtWidgets.QLabel("", self) mainLayout.addWidget(self.descriptionLabel) self.descriptionLabel.setObjectName('infoText') self.descriptionLabel.setWordWrap(True) # Create buttons to add and remove properties. buttonLayout = QtWidgets.QHBoxLayout() buttonLayout.addStretch() mainLayout.addLayout(buttonLayout) # Add ComboBox to select properties. self.propertyBox = QtWidgets.QComboBox(self) self.propertyBox.setModel(PropertiesSuggestionModel()) buttonLayout.addWidget(self.propertyBox, 1) # Add Buttons to add and remove properties. self.addButton = QtWidgets.QPushButton("Add", self) self.addButton.clicked.connect(self.addClickedHandler) buttonLayout.addWidget(self.addButton, 0) self.removeButton = QtWidgets.QPushButton("Remove", self) buttonLayout.addWidget(self.removeButton, 0) self.removeButton.clicked.connect(self.removeClickedHandler) self.rowChangedHandler(QModelIndex(), QModelIndex()) def setModel(self, model): self.model = model self.tree.setModel(model) self.tree.selectionModel().currentRowChanged.connect(self.rowChangedHandler) self.tree.expandAll() self.tree.setCurrentIndex(self.model.index(0, 0, QModelIndex())) def rowChangedHandler(self, current, previous): node = current.internalPointer() isRoot = node is not None and node == self.model.root self.propertyBox.setEnabled((isinstance(node, PropertyDataGroupObject) or isRoot) and not self.disabled) self.propertyBox.setVisible(isinstance(node, PropertyDataGroupObject) or isRoot) self.addButton.setEnabled((not current.isValid() or isinstance(node, PropertyDataGroupObject) or isRoot or isinstance(node, PropertyDataListObject)) and not self.disabled) self.removeButton.setEnabled(current.isValid() and not isRoot and not node.info().isRequired() and not self.disabled) if isinstance(node, PropertyDataGroupObject): self.propertyBox.setModel(PropertiesSuggestionModel(node.value())) elif isRoot: self.propertyBox.setModel(PropertiesSuggestionModel(node)) else: self.propertyBox.setModel(PropertiesSuggestionModel()) #self.propertyBox.setDisabled(True) #self.addButton.setDisabled(True) #self.removeButton.setDisabled(True) # Show description. if node is not None: if isinstance(node, PropertyDataObject): self.descriptionLabel.setText(node.info().description()) else: self.descriptionLabel.setText('') def addClickedHandler(self): index = self.tree.selectionModel().currentIndex() if not index.isValid(): return node = index.internalPointer() isRoot = node is not None and node == self.model.root # Add to group or root. if isinstance(node, PropertyDataGroupObject) or isRoot: prop = self.propertyBox.currentData(QtCore.Qt.UserRole) try: self.model.addToGroup(index, prop.name(), prop.prototype()) except Exception as e: QtWidgets.QMessageBox.critical(self, self.tr("Failed to add property."), self.tr(type(e).__name__ + ": " + str(e))) return self.propertyBox.model().update() # Add to list. elif isinstance(node, PropertyDataListObject): value = node.prototypeValue() self.model.addToList(index, value) def removeClickedHandler(self): index = self.tree.selectionModel().currentIndex() if index.isValid() and not index.internalPointer().info().isRequired(): self.model.removeNode(index) self.propertyBox.model().update() def disableEditing(self, disable): """ Disable Editing of the tree """ self.disabled = disable if disable: self.tree.setEditTriggers(QAbstractItemView.NoEditTriggers) self.propertyBox.setDisabled(disable) self.addButton.setDisabled(disable) self.removeButton.setDisabled(disable) else: self.tree.setEditTriggers(QtWidgets.QAbstractItemView.AllEditTriggers) gui/main_window/docks/session/__init__.py0000664ڊ00111170000000000013243274037020772 0ustar klemmso0cvprgui/main_window/docks/session/dock_session_list.py0000664ڊ00111170000002557613243274037023002 0ustar klemmso0cvprimport os from PyQt5.QtCore import QPoint, Qt, pyqtSignal from PyQt5.QtGui import QCursor from PyQt5.QtGui import QStandardItem from PyQt5.QtWidgets import ( QWidget, QPushButton, QMessageBox, QComboBox, QListWidget, QToolButton, QMenu, QAction, QVBoxLayout, QHBoxLayout, QListWidgetItem, QAbstractItemView, QFileDialog ) import backend.caffe.loader as loader from gui.main_window.docks.dock import DockElement from gui.main_window.docks.session.session_item import SessionWidget from gui.gui_util import askFromList class DockElementSessions(DockElement): def __init__(self, mainWindow, title): DockElement.__init__(self, mainWindow, title) self.mainWindow = mainWindow self.default_actions = mainWindow.actions self.name = title self.resize(300, 500) self.__setupGui() self.controller = None self.displayed_sessions = None def showDialog(self, text, info, title, buttons, location=None): msg = QMessageBox() msg.setIcon(QMessageBox.Information) msg.setText(text) msg.setInformativeText(info) msg.setWindowTitle(title) msg.setStandardButtons(buttons) if location is None: location = QCursor.pos() msg.move(location - QPoint(msg.sizeHint().width() / 2, msg.sizeHint().height() / 2)) return msg.exec_() def __setupGui(self): """ Create the widgets for this view. """ self.widget = QWidget() vbox = QVBoxLayout() hbox = QHBoxLayout() # # # Create buttons # create a new Session self.new = QPushButton() self.new.setToolTip('Create Session') self.new.setText('New') self.new.setEnabled(True) self.new.clicked.connect(self._onCreate) hbox.addWidget(self.new) # start a session # self.start = QToolButton() # self.start.setToolTip('Start Training') # self.start.setText('Start Training') # self.start.setEnabled(False) # self.start.setPopupMode(QToolButton.MenuButtonPopup) # self.start_menu = QMenu(self.start) # self.start.setMenu(self.start_menu) # solverStart = QAction('Start from solverstate', self.start_menu) # solverStart.triggered.connect(self.onStartFromSolverstate) # self.start_menu.addAction(solverStart) # modelStart = QAction('Start from caffemodel', self.start_menu) # modelStart.triggered.connect(self.onStartFromCaffemodel) # self.start_menu.addAction(modelStart) # hbox.addWidget(self.start) self.status_combo_box = QComboBox() self.status_combo_box.clear() self.status_combo_box.move(120, 30) model = self.status_combo_box.model() states = [ 'ALL', 'RUNNING', 'PAUSED', 'WAITING', 'FAILED', 'FINISHED', 'INVALID' ] for state in states: item = QStandardItem(state) model.appendRow(item) hbox.addWidget(self.status_combo_box) # self.delete = QPushButton() # self.delete.setText('Delete') # self.delete.setToolTip("Delete Sessions") # self.delete.clicked.connect(self.onDelete) # self.delete.setEnabled(False) # hbox.addWidget(self.delete) vbox.addLayout(hbox) self.session_list = QListWidget() self.session_list.setSelectionMode(QAbstractItemView.SingleSelection) vbox.addWidget(self.session_list) self.widget.setLayout(vbox) self.setWidget(self.widget) def showSessions(self, sessions): """ Show the session widgets in the list. """ self.session_list.clear() for sid in reversed(sorted(sessions.keys())): session = sessions[sid] item = QListWidgetItem(self.session_list) item.setData(Qt.UserRole, sid) widget = SessionWidget(session, self.controller, self.session_list) item.setSizeHint(widget.sizeHint()) self.session_list.addItem(item) self.session_list.setItemWidget(item, widget) self.displayed_sessions = sessions def sessionStateChanged(self, session, state): """ Notify the widget of the given session about changed state. """ for item in self.session_list.findItems('', Qt.MatchRegExp): widget = self.session_list.itemWidget(item) if widget: if widget.getSession() is session: widget.sessionStateChanged(state) def sessionIterationChanged(self, session): """ Notify the widget of the given session about iteration-change. """ for item in self.session_list.findItems('', Qt.MatchRegExp): widget = self.session_list.itemWidget(item) if widget: if widget.getSession() is session: widget.sessionIterationChanged() def sessionSnapshotAdded(self, session): """ Notify the widget of the given session about an added snapshot. """ for item in self.session_list.findItems('', Qt.MatchRegExp): widget = self.session_list.itemWidget(item) if widget: if widget.getSession() is session: widget.addAllSnapshots() def setSelectedSID(self, SID): """ Sets the list widgets selection to the session with SID. """ numItems = self.session_list.count() for idx in range(numItems): buff = self.session_list.model().index(idx, 0) _sid = buff.data(Qt.UserRole) if _sid == SID: self.session_list.setCurrentIndex(buff) def getSelectedSessions(self): """ Return the currently selected sessions as a list. """ ret = [] for item in self.session_list.selectedItems(): sid = item.data(Qt.UserRole) sess = self.displayed_sessions[sid] if sess: ret.append(sess) return ret def _onCreate(self): list = self.mainWindow.hostManager.getActiveHostList() # type: list if len(list) == 0: self.controller.justCreateNewSession() return list.insert(0, ["local", "Local Host"]) ret = askFromList(self, list, "Select Host for new Session", "You have configured remote Hosts.\nPlease select where to create the Session.") if ret: if ret is "local": self.controller.justCreateNewSession() else: remote = self.mainWindow.hostManager.getHostDataById(ret) self.controller.justCreateNewSession(remote) pass # todo self.controller.createRemoteSession(ret) return def onStart(self): """ Start session button was clicked. """ status = self.showDialog("Save Changes?", "Do you want to save unsaved changes before starting a new session?", "Unsaved Changes", QMessageBox.Yes | QMessageBox.No | QMessageBox.Cancel) if status == QMessageBox.Yes: if self.default_actions.saveProject(): self.controller.createNewSession() elif status == QMessageBox.Cancel: return elif status == QMessageBox.No: self.controller.createNewSession() def onDelete(self): """ Delete session button was clicked. NOTE: This is currently not connected to any signal. Instead the session_item.onDelete() will fire when an item is deleted from the list. """ sc = str(len(self.session_list.selectedItems())) status = self.showDialog("Do you really want to delete the "+sc+" selected sessions?", "All session data, logs and models will be deleted!", "Delete selected sessions?", QMessageBox.Ok | QMessageBox.Cancel) if status == QMessageBox.Ok: # Get all selected sessions. sessions = self.getSelectedSessions() # Remove them from the project. self.controller.removeSessions(self.getSelectedSessions()) # Find all rows that correspond to the selected sessions. sessionIDs = [session.getSessionId() for session in sessions] rows = [self.session_list.row(item) for item in self.session_list.selectedItems() if int(item.data(Qt.UserRole)) in sessionIDs] # And remove those rows. for row in reversed(sorted(rows)): item = self.session_list.takeItem(row) item = None def onSelectionChanged(self): """ Called when the list selection changed. """ if len(self.session_list.selectedItems()) == 1: sid = self.getSelectedSessions()[0].getSessionId() self.default_actions.changeSession(newSID=sid) def onStartFromSolverstate(self): """ Called when the start toolbutton for solverstate action was pressed. """ filename, _ = QFileDialog.getOpenFileName(self.mainWindow, self.tr("Resume from solverstate"), "", "Solverstate (*.solverstate *.solverstate.h5) ;; All Files (*)") if filename: model_file = loader.getCaffemodelFromSolverstate(filename) caffemodel = os.path.join(os.path.dirname(filename), model_file) if os.path.exists(caffemodel): self.controller.createNewSessionFromSnapshot(snapshot=filename) else: self.showDialog("The corresponding caffemodel file '"+model_file+"' was not found.", "The solverstate could not be resumed!", "Caffemodel not found!", QMessageBox.Ok) def onStartFromCaffemodel(self): """ Called when the start toolbutton for caffemodel action was pressed. """ filename, _ = QFileDialog.getOpenFileName(self.mainWindow, self.tr("Resume from caffemodel"), "", "Caffemodel (*.caffemodel *.caffemodel.h5) ;; All Files (*)") if filename: self.controller.createNewSession(caffemodel=filename) def getFilterState(self): """ Return the text of the currently filtered state. """ return str(self.status_combo_box.currentText()) def setController(self, controller): """ Set the controller and connect actions to contoller. """ self.controller = controller self.controller.setSessionGui(self) # self.start.setEnabled(True) # self.start.clicked.connect(self.onStart) self.status_combo_box.currentIndexChanged.connect( self.controller.filterState) self.session_list.itemSelectionChanged.connect( self.onSelectionChanged) gui/main_window/docks/session/session_controller.py0000664ڊ00111170000002541513243274037023202 0ustar klemmso0cvprfrom collections import OrderedDict from PyQt5.Qt import QObject from PyQt5.QtCore import pyqtSignal, Qt from backend.barista.constraints.session_run.training import checkMinimumTrainingRequirements from backend.barista.session.client_session import ClientSession from backend.barista.session.session import State from backend.barista.session.session_pool import SessionPool from backend.barista.utils.logger import Log from backend.barista.utils.logger import LogCaller class SessionController(QObject, LogCaller): """ View Controller for DockElementSessions. This controller handles events from the sessions widget and dispatches sessions when a user requests to train his configured net. """ poolSignal = pyqtSignal(object) sessionSelectionChanged = pyqtSignal(object) sessionStateRefreshed = pyqtSignal(object) def __init__(self): super(SessionController, self).__init__() self.project = None self.caller_id = None self.plotter = None self.sessionGui = None self.wPlotter = None # self.activeSession = None # stores the session that is currently worked on def setProject(self, project): self.project = project if project: for sid, session in project.getSessions().iteritems(): self.sessionConnects(session) session.snapshotAdded.connect(lambda: self.wPlotter.updatePlotter(self.project.getSessions())) self.filterState() pool = SessionPool() self.poolSignal.connect(lambda: self.wPlotter.updatePlotter(self.project.getSessions())) pool.activate(lambda: self.poolSignal.emit('hey')) self.updateWeightPlotter() project.activeSessionChanged.connect(self.refreshAccordingToActiveSession) def sessionConnects(self, session): """ Connects different signals to the given session. """ # Without the QueuedConnection a running remote session can't communicate with the server session.stateChanged.connect( lambda state=None, session=session: self.sessionStateChanged(session, state), Qt.QueuedConnection) session.iterationChanged.connect( lambda session=session: self.sessionIterationChanged(session), Qt.QueuedConnection) session.snapshotAdded.connect( lambda _, session=session: self.sessionSnapshotAdded(session), Qt.QueuedConnection) def sessionStateChanged(self, session, state=None): """ Handle state-change of the given session. """ self.sessionGui.sessionStateChanged(session, state) self.refreshAccordingToActiveSession() if session.getState() == State.RUNNING: self.displaySession(session.getSessionId()) def refreshAccordingToActiveSession(self): """ Refresh docks, text editor and node editor according to the active session. """ session = self.project.getActiveSession() state = session.getState() # emit signal to handle state change in view_manager self.sessionStateRefreshed.emit(state) def sessionIterationChanged(self, session): """ Handle iteration-change of the given session. """ self.sessionGui.sessionIterationChanged(session) def sessionSnapshotAdded(self, session): """ Handle new snapshot for a given session. """ self.sessionGui.sessionSnapshotAdded(session) def setPlotterGui(self, plotter): self.plotter = plotter def setSessionGui(self, sessionGui): self.sessionGui = sessionGui def setActiveSID(self, sessionID): """Sets the active sessionID. If a valid sessionID is provided, this method sets the session active and updates all GUI elements accordingly. A sessionID is valid if a session item does exist. Session states and connectivity are not checked.""" if sessionID in self.project.getValidSIDs(): self.filterState() self.sessionSelectionChanged.emit(sessionID) self.sessionGui.setSelectedSID(sessionID) # update session list self.updateWeightPlotter() def setWeightPlotter(self, wPlotter): self.wPlotter = wPlotter def justCreateNewSession(self, remote=None): """ Creates a new session and returns its session id. """ if remote is not None: sessionID = self.project.createRemoteSession(remote, state_dictionary=self.sessionGui.mainWindow.networkManager.getStateDictionary()) else: sessionID = self.project.createSession( state_dictionary=self.sessionGui.mainWindow.networkManager.getStateDictionary()) self.filterState() if sessionID is not None: session = self.project.getSession(sessionID) self.sessionConnects(session) self.sessionGui.mainWindow.actions.checkSessionState(session, showWarnings=False) self.setActiveSID(sessionID) return sessionID def loadRemoteSession(self, remote, sessionUID): sessionID = self.project.loadRemoteSession(remote, sessionUID) self.filterState() if sessionID is not None: session = self.project.getSession(sessionID) session.fetchParserData() self.sessionConnects(session) self.setActiveSID(sessionID) def createNewSession(self, caffemodel=None): """ Create a new session and start the training. Create a new session, add new parser to plotter and create a new session widget for the session """ if checkMinimumTrainingRequirements(self.project, self.sessionGui): sessionID = self.project.createSession( state_dictionary=self.sessionGui.mainWindow.networkManager.getStateDictionary()) if sessionID: session = self.project.getSessions[sessionID] session.snapshotAdded.connect(lambda: self.wPlotter.updatePlotter(self.project.getSessions())) self.updateWeightPlotter() # Since this is not set at startup, the parsing has already been done, thus the session should send signals session.setParserInitialized() session.start(caffemodel=caffemodel) self.filterState() else: Log.error('Failed to create session!', self.getCallerId()) def displaySession(self, sessionID): """ Displays a session in the plotter. """ session = self.project.getSession(sessionID) if self.plotter and session is not None: parser = session.getParser() if parser: # add plot (True -> plotOnUpdate = realtime) self.plotter.putLog(str(session.getLogId()), parser, True, str(session.getLogFileName(True))) def updateWeightPlotter(self, input=None): """ Updates Plotter""" self.wPlotter.updatePlotter(self.project.getSessions()) def createNewSessionFromSnapshot(self, snapshot, old_session=None): """ Create a new session as a clone of the given session, start with the snapshot state. """ sessionID = self.project.cloneSession(snapshot, old_session) session = self.project.getSession(sessionID) if session: session.setState(State.PAUSED) self.sessionConnects(session) session.snapshotAdded.connect(lambda: self.wPlotter.updatePlotter(self.project.getSessions())) self.setActiveSID(sessionID) else: cid = None if old_session: cid = old_session.getCallerId() else: cid = self.getCallerId() Log.error('Failed to clone session!', cid) def removeSessions(self, sessions): """ Delete the given sessions and make sure that a new valid session will be selected. If the last session is deleted, we create a new one, to keep a sane project state. """ sessionIDs = [s.getSessionId() for s in sessions] # Get all session ids that remain valid after the sessions have been deleted. validSIDs = [s for s in self.project.getValidSIDs() if s not in sessionIDs] # If we delete all valid sessions, we create a new one, because the project # shouldn't exist without a valid session. if len(validSIDs) == 0: sid = self.justCreateNewSession() else: sid = validSIDs[-1] # Now we select a new session from the remaining valid sessions, to prevent # a conflict when deleting. self.project.setActiveSID(sid) # Remove the session. for session in sessions: self.plotter.removeLog(str(session.getLogId()), str(session.getLogFileName(True))) self.project.removeSessions(sessions) # Update the weight plotter. self.wPlotter.updatePlotter(self.project.getSessions()) # Update the session list widget, to show the remaining session items. self.filterState() # And finally we call the default action to load the newly selected session. # TODO: this should eventually be called from sessionGui.setSelectedSID() # anyway, however without this line it doesn't seem to work. self.sessionGui.mainWindow.actions.loadSessionState(sid) # And select the newly selected session from the list widgets. self.sessionGui.setSelectedSID(sid) # TODO: Are the deleted sessions garbage collected at this point or do we # have a memory leak? def filterState(self): """ Filter the sessions for the state that has been selected in the GUI. """ filtered = OrderedDict() state = self.sessionGui.getFilterState() if state == 'ALL': filtered = self.project.getSessions() else: for sid, session in self.project.getSessions().iteritems(): sesstate = session.getState() if ((state == 'RUNNING' and sesstate is State.RUNNING) or (state == 'FAILED' and sesstate is State.FAILED) or (state == 'FINISHED' and sesstate is State.FINISHED) or (state == 'PAUSED' and sesstate is State.PAUSED) or (state == 'INVALID' and sesstate is State.INVALID) or (state == 'WAITING' and sesstate is State.WAITING)): filtered[sid] = session self.sessionGui.showSessions(filtered) # LogCaller def getCallerId(self): """ Return the unique caller id for this session """ if self.caller_id is None: self.caller_id = Log.getCallerId('SessionController') return self.caller_id def resetSession(self, session): session.reset() self.sessionStateChanged(session) gui/main_window/docks/session/session_item.py0000664ڊ00111170000003465313243274037021761 0ustar klemmso0cvprimport os from PyQt5.QtCore import QPoint, Qt from PyQt5.QtGui import QPixmap, QIcon, QCursor from PyQt5.QtWidgets import ( QWidget, QPushButton, QLabel, QMessageBox, QVBoxLayout, QHBoxLayout, QProgressBar, QMenu, QAction) from backend.barista.session.session_utils import State from backend.barista.session.client_session import ClientSession class SessionWidget(QWidget): """ A instance of this widget represents an entry for a session in the list of session within the running jobs dock element. """ #TODO: add an item to select a session as visible/editable i.e. 'active'? def __init__(self, session, controller, parent=None): super(SessionWidget, self).__init__(parent) self.session = session self.controller = controller mainLayout = QVBoxLayout(self) buttonLayout = QHBoxLayout() progressLayout = QHBoxLayout() # Add a network label if this is a remote session. if isinstance(self.session, ClientSession): networkLabel = QLabel() remote = self.session.remote networkLabel.setText(remote[0] + ":" + str(remote[1])) networkLabel.setAlignment(Qt.AlignLeft) mainLayout.addWidget(networkLabel) # Create a label for the session id. sessionIdLabel = QLabel(str(session.getSessionId())) sessionIdLabel.setObjectName("sessionId") sessionIdLabel.setFixedSize(30, 30) sessionIdLabel.setAlignment(Qt.AlignCenter) buttonLayout.addWidget(sessionIdLabel) # Create session status label. self.statusLabel = QLabel() self.statusLabel.setObjectName("sessionStatus") self.statusLabel.setFixedHeight(30) self.statusLabel.setMinimumWidth(100) self.statusLabel.setAlignment(Qt.AlignCenter) buttonLayout.addWidget(self.statusLabel) # Create snapshot button. self.snapshotBtn = QPushButton() self.snapshotBtn.setObjectName("sessionBtn") snapshotIcon = QPixmap('resources/snap.png') self.snapshotBtn.setIcon(QIcon(snapshotIcon)) self.snapshotBtn.setToolTip("Take a snapshot") self.snapshotBtn.setFixedSize(30, 30) self.snapshotBtn.clicked.connect(self.onSnapshot) buttonLayout.addWidget(self.snapshotBtn) # Create delete button. self.deleteBtn = QPushButton() self.deleteBtn.setObjectName("sessionBtn") deleteIcon = QPixmap('resources/trash.png') self.deleteBtn.setIcon(QIcon(deleteIcon)) self.deleteBtn.setToolTip("Delete Session") self.deleteBtn.setFixedSize(30, 30) self.deleteBtn.clicked.connect(self.onDelete) buttonLayout.addWidget(self.deleteBtn) # Create more button. self.moreBtn = QPushButton() self.moreBtn.setObjectName("sessionBtn") moreIcon = QPixmap('resources/more.png') self.moreBtn.setIcon(QIcon(moreIcon)) self.moreBtn.setToolTip("Show more actions") self.moreBtn.setFixedSize(30, 30) # Add actions to the more button menu. moreBtnMenu = QMenu(self.moreBtn) self.cloneSessionMenu = moreBtnMenu.addMenu("Clone Session") self.moreBtn.setMenu(moreBtnMenu) resetAct = QAction("Reset", moreBtnMenu) resetAct.triggered.connect(self.onReset) moreBtnMenu.addAction(resetAct) self.addAllSnapshots() buttonLayout.addWidget(self.moreBtn) # Add a stretch, so that the horizontal items are left aligned. mainLayout.addLayout(buttonLayout) # Create start button. self.startBtn = QPushButton() self.startBtn.setObjectName("sessionBtn") startIcon = QPixmap('resources/start.png') self.startBtn.setIcon(QIcon(startIcon)) self.startBtn.setToolTip("Start Training") self.startBtn.setFixedSize(30, 30) self.startBtn.clicked.connect(self.onStart) self.startBtn.setEnabled(False) progressLayout.addWidget(self.startBtn) # Create pause button. self.pauseBtn = QPushButton() self.pauseBtn.setObjectName("sessionBtn") pauseIcon = QPixmap('resources/pause.png') self.pauseBtn.setIcon(QIcon(pauseIcon)) self.pauseBtn.setToolTip("Pause Training") self.pauseBtn.setFixedSize(30, 30) self.pauseBtn.clicked.connect(self.onPause) self.pauseBtn.setEnabled(False) progressLayout.addWidget(self.pauseBtn) # Add a progress bar. self.progress = QProgressBar() self.progress.setTextVisible(True) self.sessionIterationChanged() progressLayout.addWidget(self.progress) mainLayout.addLayout(progressLayout) #Add text label for invalid sessions self.invalidLabel = QLabel() self.invalidLabel.setObjectName("invalidMsg") self.invalidLabel.setFixedHeight(30) self.invalidLabel.setMinimumWidth(80) self.invalidLabel.setAlignment(Qt.AlignCenter) self.invalidLabel.setEnabled(False) progressLayout.addWidget(self.invalidLabel) # Update the current session state. self.sessionStateChanged(self.session.getState()) def getSession(self): """ Return the session of this widget. """ return self.session def getSID(self): """ Returns the session ID related to this session item. """ return self.session.getSessionId() def showDialog(self, text, info, title, location=None, buttons=None): """ Show a message box to the user. Return the user decision. """ msg = QMessageBox() msg.setIcon(QMessageBox.Information) msg.setText(text) msg.setInformativeText(info) msg.setWindowTitle(title) if buttons is not None: try: msg.setStandardButtons(buttons) except TypeError: # if something wrong is passed to the buttons Parameter, show the standard message. msg.setStandardButtons(QMessageBox.Ok | QMessageBox.Cancel) else: msg.setStandardButtons(QMessageBox.Ok | QMessageBox.Cancel) if location is None: location = QCursor.pos() msg.move(location - QPoint(msg.sizeHint().width() / 2, msg.sizeHint().height() / 2)) return msg.exec_() def sessionIterationChanged(self): """ The sessions iteration changed. """ iteration = self.session.getIteration() maxIteration = self.session.getMaxIteration() try: i = float(iteration) * 100.0 m = float(maxIteration) self.progress.setValue(i/m) except Exception: pass text = str(int(iteration))+' / '+str(maxIteration) self.progress.setFormat(text) def addAllSnapshots(self): """ Add all snapshots to the clone session submenu. """ snapshots = self.session.getSnapshots() self.cloneSessionMenu.clear() for iter_num in sorted(snapshots.keys()): self.addSnapshot(snapshots[iter_num]) snapshots = self.session.getSnapshots() snapshotCount = len(snapshots) if snapshotCount == 0: self.cloneSessionMenu.setEnabled(False) else: self.cloneSessionMenu.setEnabled(True) def addSnapshot(self, snapshot): """ Add a snapshot to the snapshot menu. """ snapAct = QAction(os.path.basename(snapshot), self.cloneSessionMenu) snapAct.triggered.connect(lambda: self.createFromSnapshot(snapshot)) self.cloneSessionMenu.addAction(snapAct) def createFromSnapshot(self, snapshot=None): """ Called when the snapshot tool button was clicked or when a snapshot was selected in the snapshot dropdown. """ if isinstance(self.session, ClientSession): self.showDialog("Currently cloning of remote Sessions is not supported", "", "", buttons=QMessageBox.Ok) return if type(snapshot) is not str and type(snapshot) is not unicode: snapshot = self.session.getLastSnapshot() info = "A new session will be created from the snapshot " + snapshot if self.showDialog("Do you want to continue from this snapshot?", info, "") == QMessageBox.Ok: self.controller.createNewSessionFromSnapshot(snapshot, self.session) def sessionStateChanged(self, state=None): """ Called when the session state changed. """ if state is None: state = self.session.getState() if state is State.WAITING: self.__buttons_wait() self.__progress_show() self.setStatus('Ready', 'blue') self.statusLabel.setToolTip('Ready to train!') elif state is State.RUNNING: self.__buttons_run() self.__progress_show() self.setStatus('Running', 'green') self.statusLabel.setToolTip('Currently training..') elif state is State.PAUSED: self.__buttons_wait() self.__progress_show() self.setStatus('Paused', 'orange') self.statusLabel.setToolTip('Training paused.') elif state is State.FINISHED: self.__buttons_disable() self.__progress_show() self.setStatus('Finished', 'gray') self.statusLabel.setToolTip('Finished, training completed.') elif state is State.FAILED: self.__buttons_disable() self.__progress_show() self.setStatus('Failed', 'red') self.statusLabel.setToolTip('Something went wrong.') elif state is State.UNDEFINED: self.__buttons_disable() self.__progress_show() self.setStatus('Undefined', 'gray') self.statusLabel.setToolTip('') elif state is State.INVALID: self.__buttons_disable() self.setStatus('Invalid', 'gray') self.statusLabel.setToolTip('Session is not ready to train.') self.__progress_hide() errorList = self.session.getErrorList() ''' ErrorList is a list of all constraints that are broken for this session. Each tuple contains a full error message displayed in the tooltip (index 0) and a key word to be shown in the status box. ''' if errorList is not None and len(errorList) > 0: self.invalidLabel.setToolTip(errorList[0][0]) self.setInvalidMsg((errorList[0])[1]) elif state is State.NOTCONNECTED: self.__buttons_disable() self.__progress_hide() self.setStatus('Invalid', 'gray') self.setInvalidMsg('No connection to host found.') self.invalidLabel.setToolTip('') self.statusLabel.adjustSize() def setStatus(self, text, color='gray'): """ Set the style of the status label. """ self.statusLabel.setText(text) # If we just want to overwrite single properties of the stylesheet, just # append them to the current stylesheet. self.statusLabel.setStyleSheet(self.statusLabel.styleSheet() + " QLabel { background-color: "+color+"; border-color: "+color+" }" ) def setInvalidMsg(self, text, color='gray'): self.invalidLabel.setText(text) self.invalidLabel.setStyleSheet( self.invalidLabel.styleSheet() + " QLabel { background-color: " + color + "; border-color: " + color + " }") def __buttons_run(self): self.startBtn.setEnabled(False) self.startBtn.hide() self.pauseBtn.setEnabled(True) self.pauseBtn.show() self.snapshotBtn.setEnabled(True) def __buttons_wait(self): self.startBtn.setEnabled(True) self.startBtn.show() self.pauseBtn.setEnabled(False) self.pauseBtn.hide() self.snapshotBtn.setEnabled(False) def __buttons_disable(self): self.startBtn.setEnabled(False) self.startBtn.show() self.pauseBtn.setEnabled(False) self.pauseBtn.hide() self.snapshotBtn.setEnabled(False) def __progress_hide(self): self.startBtn.setEnabled(False) self.startBtn.show() self.progress.setEnabled(False) self.progress.hide() self.invalidLabel.setEnabled(True) self.invalidLabel.show() def __progress_show(self): self.progress.setEnabled(True) self.progress.show() self.invalidLabel.setEnabled(False) self.invalidLabel.hide() def onStart(self): """ Start button was clicked. """ if self.session.getState() is State.WAITING: self.session.start() elif self.session.getState() is State.PAUSED: self.session.proceed() def onPause(self): self.session.pause() def onSnapshot(self): self.session.snapshot() def onDelete(self): """ Delete button was clicked. """ sid = self.getSID() if self.session.getState() == State.RUNNING: confirm = self.showDialog("The selected session is still running.\nDo you want to stop it " "and delete Session {} anyway?".format(sid), "", "Delete Session") == QMessageBox.Ok else: confirm = self.showDialog("Do you want to delete Session {}?".format(sid), "", "Delete Session") == QMessageBox.Ok if confirm: self.controller.removeSessions((self.session,)) def onReset(self): """ Reset button was clicked. """ sid = self.getSID() if self.session.getState() == State.RUNNING: confirm = self.showDialog( "The selected session is still running.\nDo you want to stop and reset Session {} anyway?\n" "All progress (including logs) will be lost!".format(sid), "", "Reset Session") == QMessageBox.Ok elif self.session.getState() == State.PAUSED: confirm = self.showDialog( "The selected session was trained before.\nDo you want to reset Session {} anyway?\n" "All progress (including logs) will be lost!".format(sid), "", "Reset Session") == QMessageBox.Ok else: confirm = self.showDialog("Do you want to reset Session {}?".format(sid), "", "Reset Session") == QMessageBox.Ok if confirm: self.controller.resetSession(self.session) gui/main_window/docks/weight_visualization/__init__.py0000664ڊ00111170000000000013243274037023557 0ustar klemmso0cvprgui/main_window/docks/weight_visualization/dock_weight_visualization.py0000664ڊ00111170000003573213243274037027314 0ustar klemmso0cvprimport os from PyQt5.QtWidgets import ( QVBoxLayout, QHBoxLayout, QComboBox, QMessageBox, QWidget, QPushButton, QFileDialog, QSizePolicy ) from backend.barista.utils.logger import Log from backend.barista.utils.logger import LogCaller from backend.caffe import loader from backend.barista.session.session import Session from backend.barista.session.client_session import ClientSession from gui.main_window.docks.dock import DockElement from gui.main_window.docks.weight_visualization.image_canvas import ImageCanvas from gui.main_window.docks.weight_visualization.weights import calculateConvWeights from gui.main_window.docks.weight_visualization.weights import loadNetParameter class DockElementWeightPlotter(DockElement, LogCaller): """ This dock element displays the weights of a layer in a selected session's snapshot. Currently only convolutional layer are supported. The canvas renders the convolutional filter's weights in a grid layout. The selection of sessions, snapshots and layers is possible by selecting them in one of the three comboboxes. """ ALLOWED_LAYERTYPES = {"Convolution"} class SelectComboBox(QComboBox): """ This class is a template for the combobox needed to choose the snapshots, layers, and session""" # emptyMessage gets printed on the Combobox when no element has been added to it. def __init__(self, emptyMessage): QComboBox.__init__(self) # Change layout to neatly fit the text self.setSizePolicy(QSizePolicy.Maximum, QSizePolicy.Minimum) self.setSizeAdjustPolicy(self.AdjustToContents) self.__emptyMessage = emptyMessage def replaceItems(self, idList): """ Replace all items with the current List""" self.clear() if idList is None or idList == []: self.setEnabled(False) self.addItems([self.__emptyMessage]) self.setCurrentText(self.__emptyMessage) else: self.setEnabled(True) QComboBox.addItems(self, idList) def __init__(self, mainWindow, title): DockElement.__init__(self, mainWindow, title) self.caller_id = None self.__setupGui() # Initialize the comboboxes needed self.sessionComboBox.activated['QString'].connect(self.__setCurrentSession) self.snapshotComboBox.activated['QString'].connect(self.__setCurrentSnapshot) self.layerComboBox.activated['QString'].connect(self.__setCurrentLayer) self.__currentSessionId = None self.__currentSnapshotId = None self.__currentLayerName = None self.__currentNet = None self.__sessionDict = {} # Stores the snapshots which has been selected by the user, to load them faster the next time they are needed. self.__alreadyOpenSnapshots = {} def __setupGui(self): """Add layouts and populate them with widgets""" self.mainWidget = QWidget() self.mainLayout = QVBoxLayout(self.mainWidget) self.setWidget(self.mainWidget) # Create the comboboxes to select a visualization. self.selectionLayout = QHBoxLayout() self.mainLayout.addLayout(self.selectionLayout) self.sessionComboBox = DockElementWeightPlotter.SelectComboBox("No sessions") # Set the sessionComboBox's size policy to force it to display the whole name. self.sessionComboBox.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Minimum) self.selectionLayout.addWidget(self.sessionComboBox) self.snapshotComboBox = DockElementWeightPlotter.SelectComboBox("No snapshots") self.selectionLayout.addWidget(self.snapshotComboBox) self.layerComboBox = DockElementWeightPlotter.SelectComboBox("No layers") self.selectionLayout.addWidget(self.layerComboBox) # Create the 'Save as Image..' button right aligned and connect it's signal. self.selectionLayout.addStretch() self.saveImageButton = QPushButton("Save as Image..") self.saveImageButton.clicked.connect(self.__saveImage) self.selectionLayout.addWidget(self.saveImageButton) # Create the canvas to display the plot. self.canvasWidget = ImageCanvas() self.mainLayout.addWidget(self.canvasWidget) def __setCurrentSession(self, sid): if self.__currentSessionId == sid: return self.__currentSessionId = sid self.__updateSnapshotList() self.__updateLayerList() self.__updateCanvas() def __setCurrentSnapshot(self, sid): if self.__currentSnapshotId == sid: return self.__currentSnapshotId = sid self.__updateCanvas() def __setCurrentLayer(self, lid): if self.__currentLayerName == lid: return self.__currentLayerName = lid self.__updateCanvas() def __updateCanvas(self): """ Updates the current shown picture with the current settings """ # Load model self.__currentNet = self.__getNetwork() if self.__currentNet: self.saveImageButton.setEnabled(True) image = calculateConvWeights(self.__currentNet, self.__currentLayerName) if image is not None: self.canvasWidget.showImage(image) self.canvasWidget.show() else: # Hide canvas when no snapshot, session or layer can be chosen self.canvasWidget.hide() self.saveImageButton.setEnabled(False) def __getNetwork(self, sess_id=None, snap_id=None): """ Return the caffe network of the current session and snapshot. """ if sess_id is None: sess_id = self.__currentSessionId if snap_id is None: snap_id = self.__currentSnapshotId if sess_id and snap_id: # Creates a Dictionary of Sessions Ids, which contains Snapshot Ids # which point to Layer Id which point to # already opened Networks. net = None if sess_id in self.__alreadyOpenSnapshots: session_snapshots = self.__alreadyOpenSnapshots[sess_id] if snap_id in session_snapshots: net = session_snapshots[snap_id] if net: # cached net found return net else: # snapshot was accessed for the first time # create and cache the net session = self.__sessionDict[sess_id] snapName = snap_id.replace('solverstate', 'caffemodel') if isinstance(session, ClientSession): net = session.loadNetParameter(snapName) if net is None: return else: snapshotPath = session.getSnapshotDirectory() snapshotPath = str(os.path.join(snapshotPath, snapName)) if not os.path.exists(snapshotPath): Log.error('Snapshot file '+snapshotPath+' does not exist!', self.getCallerId()) return net = loadNetParameter(snapshotPath) if net is not None: if sess_id not in self.__alreadyOpenSnapshots.keys(): self.__alreadyOpenSnapshots[sess_id] = {snap_id: net} else: self.__alreadyOpenSnapshots[sess_id][snap_id] = net return net else: # Show a warning message Log.error('The hdf5 snapshot format is not supported for the weight visualization! ' 'This can be changed by setting the snapshot_format parameter in the solver properties.', self.getCallerId()) def updatePlotter(self, sessionDict): """ Updates the comboboxes and redraws the weight image. """ # convert ids to string self.__sessionDict = { "Session " + str(sid): session for sid, session in sessionDict.iteritems() } self.__updateSessionsList() self.__updateSnapshotList() self.__updateLayerList() self.__updateCanvas() def __updateSessionsList(self): """ Should be called, if the Id-List changes. Updates the session Combobox. """ # get a ordered List from the session-dictionary idList = map( lambda tuple: tuple[0], sorted(self.__sessionDict.items(), key=lambda id: id[1].getSessionId())) # updates the sessionCombobox with the current sessions self.sessionComboBox.replaceItems(idList) # Tries to apply the last selected session, if it exist (could be # deleted). If not, select the last one(assume sorted) if self.__currentSessionId is None or self.__currentSessionId not in idList: if idList == []: self.__currentSessionId = None else: self.__currentSessionId = idList[-1] self.sessionComboBox.setCurrentText(self.__currentSessionId) def __updateSnapshotList(self): """ Should be called, if the Id-List changes. Updates the snapshot Combobox. """ # assert that there is a current session to show if self.__currentSessionId is not None: snapshotDict = self.__sessionDict[self.__currentSessionId].getSnapshots() snapshotsIdList = map( lambda tuple: tuple[1], sorted(snapshotDict.items(), key=lambda id: id[0])) else: snapshotsIdList = [] # update the snapshotCombobox with the current snapshots self.snapshotComboBox.replaceItems(snapshotsIdList) # Select the last snapshot, or None if there are none if not snapshotsIdList == []: self.__currentSnapshotId = snapshotsIdList[-1] else: self.__currentSnapshotId = None self.snapshotComboBox.setCurrentText(self.__currentSnapshotId) def __updateLayerList(self): """ Update the layer list with available layers found in the net description. """ # getLayers if self.__currentSessionId is not None: session = self.__sessionDict[self.__currentSessionId] if isinstance(session, ClientSession): netInternal = session.loadInternalNetFile() currentNetwork = loader.loadNet(netInternal) layerNames = map( lambda layer: layer["parameters"]["name"], filter( lambda layer: layer["type"].name() in self.ALLOWED_LAYERTYPES, currentNetwork["layers"].values() ) ) layerNameList = sorted(layerNames) else: try: currentNetworkPath = session.getInternalNetFile() file = open(currentNetworkPath, 'r') currentNetwork = loader.loadNet(file.read()) # get all the names of the layers, which match the desired type layerNames = map( lambda layer: layer["parameters"]["name"], filter( lambda layer: layer["type"].name() in self.ALLOWED_LAYERTYPES, currentNetwork["layers"].values() ) ) layerNameList = sorted(layerNames) except IOError: callerId = Log.getCallerId('weight-plotter') Log.error("Could not open the network of this session.", callerId) layerNameList =[] else: layerNameList =[] # updates the layer Combobox with the current layers self.layerComboBox.replaceItems(layerNameList) if self.__currentLayerName is None or self.__currentLayerName not in layerNameList: if not layerNameList == []: self.__currentLayerName = layerNameList[-1] else: self.__currentLayerName = None self.layerComboBox.setCurrentText(self.__currentLayerName) def __saveImage(self): """ Gives the user the opportunity to save the image on disk. The user can choose from allowed extension, and is alerted if the entered extension is not valid. If no extension is entered by the user, the extension is set to png. """ allowedExtensions = [".png", ".bmp", ".jpeg", ".jpg"] allowedAsString = str(reduce(lambda x, y: x + y, map(lambda x: " *" + x, allowedExtensions))) callerId = self.getCallerId() try: filename = "" # While the entered extension does not matches the allowd extensions while not self.__validateFilename(filename, allowedExtensions): fileDialog = QFileDialog() fileDialog.setDefaultSuffix("png") filenameArray = fileDialog.getSaveFileName( self, "Save File", filename, allowedAsString ) filename = filenameArray[0] if filename != "": _, extension = os.path.splitext(filename) # If no extension has been entered, append .png if extension == "": filename += ".png" else: # If user clicks on abort, leave the loop break # Show an alert message, when an unknown extension has been entered if not self.__validateFilename(filename, allowedExtensions): msg = QMessageBox() msg.setIcon(QMessageBox.Warning) msg.setWindowTitle("Warning") msg.setText("Please enter a valid extension:\n" + allowedAsString) msg.exec_() if filename != "": if self.__currentNet: image = calculateConvWeights(self.__currentNet, self.__currentLayerName) self.canvasWidget.saveImage(image, filename) Log.log("Saved image under " + filename, callerId) else: Log.error("Try to save image of weights without a network being loaded.", callerId) except Exception as e: Log.error("Saving the file failed. " + str(e), callerId) # LogCaller def getCallerId(self): """ Return the unique caller id for this session """ if self.caller_id is None: self.caller_id = Log.getCallerId('weight-plotter') return self.caller_id def __validateFilename(self, filename, allowedExtensions): """ Checks if filename is valid, returns corresponding boolean""" _, extension = os.path.splitext(filename) if extension not in allowedExtensions: return False else: return True gui/main_window/docks/weight_visualization/image_canvas.py0000664ڊ00111170000000453713243274037024460 0ustar klemmso0cvprfrom PyQt5.QtWidgets import ( QGraphicsView, QGraphicsScene, QGraphicsPixmapItem ) from PyQt5 import QtCore from PyQt5.QtGui import QPixmap, QImage, QColor class ImageCanvas(QGraphicsView): """ This class manages the view of the weights. """ def __init__(self): QGraphicsView.__init__(self) self.__scene = QGraphicsScene(self) self.__item = QGraphicsPixmapItem() self.__scene.addItem(self.__item) self.setScene(self.__scene) self.setTransformationAnchor(QGraphicsView.AnchorUnderMouse) self.setResizeAnchor(QGraphicsView.AnchorUnderMouse) # Change resize policy when hidden, to ensure it takes the same space pol = self.sizePolicy() pol.setRetainSizeWhenHidden(True) self.setSizePolicy(pol) def center(self): """ Centers the pictures and adjust the zoom-level to maximize the picture.""" pictureRect = QtCore.QRectF(self.__item.pixmap().rect()) self.fitInView(pictureRect, QtCore.Qt.KeepAspectRatio) def showImage(self, rawImage): """ Takes a raw Image and show it in the view. """ self.__image = self._getPixImage(rawImage) self.setDragMode(QGraphicsView.ScrollHandDrag) self.__item.setPixmap(self.__image) self.__scene = QGraphicsScene(self) self.__scene.addItem(self.__item) self.setScene(self.__scene) #self.center() def _getPixImage(self, image): """ generate a QPixImage from the raw input Image. """ height = image.shape[0] width = image.shape[1] qImage = QImage(width, height, QImage.Format_RGB32) for x in range(0, width): for y in range(0, height): val = int(255*image[y][x]) qImage.setPixel(x, y, QColor.fromRgb(val, val, val, 1).rgb()) return QPixmap.fromImage(qImage) def wheelEvent(self, event): """ Overrides wheel event to allow zooming. """ mouseDelta = event.angleDelta().y()/120.0 scale = 1.0 + 0.1*mouseDelta self.scale(scale, scale) # idea from http://stackoverflow.com/questions/6915106/saving-a-numpy-array-as-an-image-instructions def saveImage(self, rawImage, filename): image = self._getPixImage(rawImage) if not image.save(filename): raise IOError("Could not save file in the chosen folder.") gui/main_window/docks/weight_visualization/run_weights.py0000664ڊ00111170000000130413243274037024366 0ustar klemmso0cvpr# from https://www.eriksmistad.no/visualizing-learned-features-of-a-caffe-neural-network/ # test code, will be changed import backend.caffe.path_loader as caffeLoader caffe = caffeLoader.importCaffe() from gui.main_window.docks.weight_visualization import visualize_weights # Load model net = caffe.Net('/Users/sandtil/Documents/Developer/pyenv/Barista/test/examples/mnist/sessions/20161221_151232_203/net.prototxt', '/Users/sandtil/Documents/Developer/pyenv/Barista/test/examples/mnist/sessions/20161221_151232_203/snapshots/lenet_iter_10000.caffemodel', caffe.TEST) visualize_weights(net, 'conv1', filename='conv1') #visualize_weights(net, 'conv2', filename='conv2.png') gui/main_window/docks/weight_visualization/weights.py0000664ڊ00111170000000540213243274037023505 0ustar klemmso0cvpr# from https://www.eriksmistad.no/visualizing-learned-features-of-a-caffe-neural-network/ import numpy as np def loadNetParameter(caffemodel): """ Return a NetParameter protocol buffer loaded from the caffemodel. """ import backend.caffe.path_loader as caffeLoader proto = caffeLoader.importProto() net = proto.NetParameter() try: with open(caffemodel, 'rb') as f: net.ParseFromString(f.read()) return net except: pass def loadNetParamFromString(paramstring): import backend.caffe.path_loader as caffeLoader proto = caffeLoader.importProto() net = proto.NetParameter() try: net.ParseFromString(paramstring) return net except: pass def calculateConvWeights(net, layer_name, padding=2): """ Return a grayscale image array which displays the weights of the convolutional layer in the net. """ for layer in net.layer: if layer.name == layer_name: for blob in layer.blobs: if len(blob.shape.dim) == 4: # The parameters are a list of [weights, biases] data = np.copy(blob.data).reshape(blob.shape.dim) # N is the total number of convolutions N = blob.shape.dim[0]*blob.shape.dim[1] # Ensure the resulting image is square filters_per_row = int(np.ceil(np.sqrt(N))) # Assume the filters are square filter_size = blob.shape.dim[2] # Size of the result image including padding result_size = filters_per_row*(filter_size + padding) - padding # Initialize result image to all zeros result = np.zeros((result_size, result_size)) # Tile the filters into the result image filter_x = 0 filter_y = 0 for n in range(blob.shape.dim[0]): for c in range(blob.shape.dim[1]): if filter_x == filters_per_row: filter_y += 1 filter_x = 0 for i in range(filter_size): for j in range(filter_size): x = filter_y*(filter_size + padding) + i y = filter_x*(filter_size + padding) + j result[x, y] = data[n, c, i, j] filter_x += 1 # Normalize image to 0-1 min = result.min() max = result.max() result = (result - min) / (max - min) return result gui/main_window/node_editor/items/__init__.py0000664ڊ00111170000000000013243274037021620 0ustar klemmso0cvprgui/main_window/node_editor/items/connection_item.py0000664ڊ00111170000002025413243274037023253 0ustar klemmso0cvprfrom PyQt5.QtWidgets import QGraphicsPathItem, QMenu from PyQt5.QtCore import QPointF from PyQt5.QtGui import QPainterPath, QPainterPathStroker from gui.main_window.node_editor.items.renderer.connection_item_renderer import ConnectionItemRenderer from gui.main_window.node_editor.node_editor_constants import Constants class ConnectionItem(QGraphicsPathItem): """ A ConnectionItem represents a top-bottom connection in the graphics scene """ def __init__(self, nodeEditor, parent=None): super(ConnectionItem, self).__init__(parent) # make selectable self.setFlag(QGraphicsPathItem.ItemIsSelectable, True) # draw behind node items self.setZValue(0.5) self.__phase = "" self.__isInPlace = False self.__nodeEditor = nodeEditor self.__hidden = False self.__topConnector = None self.__bottomConnector = None self.__start = QPointF(0, 0) self.__end = QPointF(0, 0) # create a renderer to separate drawing from logic self.__renderer = ConnectionItemRenderer(self) self.__updatePath() def paint(self, painter, option, widget=None): self.__renderer.paint(painter) def shape(self): stroker = QPainterPathStroker() stroker.setWidth(Constants.connectionItemSize *3) return stroker.createStroke(self.path()) def setStart(self, start): """ Sets the start position of the connection and recalculates the path """ self.__start = start self.__updatePath() def setEnd(self, end): """ Sets the end position of the connection and recalculates the path """ self.__end = end self.__updatePath() def getTopConnector(self): """ Returns the connections top connector """ return self.__topConnector def getBottomConnector(self): """ Returns the connections bottom connector """ return self.__bottomConnector def setConnector(self, connector): """ Sets the top/bottom connector of the connection. Whether the top or bottom connector gets set, is decided by information provided by the connector """ if connector.isTopConnector(): self.setTopConnector(connector) else: self.setBottomConnector(connector) def setTopConnector(self, connector): """ Sets the top connector and updates the connections path """ self.__topConnector = connector self.__start = connector.scenePos() self.updateData() self.__updatePath() def setBottomConnector(self, connector): """ Sets the bottom connector and updates the connections path """ self.__bottomConnector = connector self.__end = connector.scenePos() self.updateData() self.__updatePath() def setHidden(self, hidden): """ Sets the connection to be hidden/shown and updates the rendering """ self.__hidden = hidden self.update() def updateMousePosition(self, pos): """ Updates the start/end of the connection if the connection is getting dragged (created) """ if self.__topConnector is None: self.__start = pos elif self.__bottomConnector is None: self.__end = pos self.__updatePath() def updateData(self): """ Updates the connections internal in-place and phase variables to update the rendering """ # set in-place if self.__bottomConnector is not None: self.__isInPlace = self.__bottomConnector.isInPlace() else: self.__isInPlace = False # set phase self.__phase = "" if self.__topConnector is not None: self.__phase = self.__topConnector.getPhase() if self.__bottomConnector is not None and self.__phase == "": self.__phase = self.__bottomConnector.getPhase() # update self.update() def __updatePath(self): """ Updates the connections path, using the start and end points """ path = QPainterPath(self.__start) # the end point is further left than the start point, so draw two half circles and a straight line if self.__start.x() > self.__end.x(): yDiff = abs(self.__end.y() - self.__start.y()) middleY = (self.__start.y() + self.__end.y()) / 2 # calculate the control points needed for the first half circle curve1End = QPointF(self.__start.x(), middleY) curve1CP1 = QPointF(self.__start.x() + yDiff / 2, self.__start.y()) curve1CP2 = QPointF(self.__start.x() + yDiff / 2, middleY) # calculate the control points needed for the second half circle curve2Start = QPointF(self.__end.x(), middleY) curve2CP1 = QPointF(self.__end.x() - yDiff / 2, middleY) curve2CP2 = QPointF(self.__end.x() - yDiff / 2, self.__end.y()) # draw the first half circle path.cubicTo(curve1CP1.x(), curve1CP1.y(), curve1CP2.x(), curve1CP2.y(), curve1End.x(), curve1End.y()) # draw the straight line path.lineTo(curve2Start.x(), curve2Start.y()) # draw the second half circle path.cubicTo(curve2CP1.x(), curve2CP1.y(), curve2CP2.x(), curve2CP2.y(), self.__end.x(), self.__end.y()) # the start point is further left than the end point, so draw a Bezier curve between both else: path.cubicTo((self.__start.x() + self.__end.x()) / 2, self.__start.y(), (self.__start.x() + self.__end.x()) / 2, self.__end.y(), self.__end.x(), self.__end.y()) self.setPath(path) self.update() def checkSameConnectorTypeRestriction(self, connector): """ Check, whether the connector type (top/bottom) is already set in the connection """ if self.__topConnector is not None and connector.isTopConnector(): return False elif self.__bottomConnector is not None and not connector.isTopConnector(): return False return True def getConnectorIfNotFullyConnected(self): """ Returns the only connected connector, if only one connector is connected """ if self.__topConnector is not None and self.__bottomConnector is None: return self.__topConnector elif self.__topConnector is None and self.__bottomConnector is not None: return self.__bottomConnector return None def getHidden(self): """ Returns whether the connection is hidden """ return self.__hidden def getNodeEditor(self): """ Returns the node prototxt_editor object (for the renderer) """ return self.__nodeEditor def getPhase(self): """ Returns the internal phase """ return self.__phase def getIsInPlace(self): """ Returns the internal in-place value """ return self.__isInPlace def contextMenuEvent(self, event): """ Creates the context menu """ contextMenu = QMenu() menuText = "Hide" if self.__hidden: menuText = "Show" # add action to show/hide the connection toggleHideAction = contextMenu.addAction(menuText) if not self.__nodeEditor.disable: removeAction = contextMenu.addAction("Remove") # show context menu action = contextMenu.exec_(event.screenPos()) if action is not None: if action == toggleHideAction: self.__hidden = not self.__hidden if self.__hidden: self.__nodeEditor.tryToAddHiddenConnection(self) else: self.__nodeEditor.tryToRemoveHiddenConnection(self) self.update() elif not self.__nodeEditor.disable and action == removeAction: # deselect all Layers self.__nodeEditor.tryToClearSelection() # deselect all other connections for item in self.__nodeEditor.getScene().selectedItems(): item.setSelected(False) # select this connection self.setSelected(True) # remove this connection self.__nodeEditor.tryToDeleteSelection() gui/main_window/node_editor/items/connector_bottom_item.py0000664ڊ00111170000000471313243274037024474 0ustar klemmso0cvprfrom PyQt5.QtWidgets import QMenu from gui.main_window.node_editor.items.connector_item import ConnectorItem class ConnectorBottomItem(ConnectorItem): """ Class to provide bottom connector functionality """ def __init__(self, index, nodeItem, nodeEditor, parent=None): super(ConnectorBottomItem, self).__init__(index, nodeItem, nodeEditor, parent) def isTopConnector(self): """ Returns whether the connector is a top connector (implementation for parent class) """ return False def isInPlace(self): """ Returns whether the connector is owned by an in-place working layer (implementation for parent class) """ return self._nodeItem.getIsInPlace() def getConnectedNodes(self): """ Returns a list of node items, connected to this connector (implementation for parent class) """ nodes = list() # for each connection get the node connected to the top of the connection for connection in self._connections: connectionsTopConnector = connection.getTopConnector() if connectionsTopConnector is not None: nodes.append(connectionsTopConnector.getNodeItem()) return nodes def addConnection(self, connection): """ Adds a connection to the connector and sets the end of the connection to this connectors position (implementation for parent class) """ self._connections.append(connection) connection.setEnd(self.scenePos()) def updateConnectionPositions(self): """ Updates the connected connections, sets the end of all connected connections to this connectors position (implementation for parent class) """ for connection in self._connections: connection.setEnd(self.scenePos()) def contextMenuEvent(self, event): """ Context menu for the bottom connector """ contextMenu = QMenu() disconnectBottom = contextMenu.addAction("Disconnect") if self.getConnectionCount() == 0: disconnectBottom.setEnabled(False) removeBottom = contextMenu.addAction("Remove") action = contextMenu.exec_(event.screenPos()) if action is not None: if action == removeBottom: self._nodeEditor.tryToRemoveBottomBlob(self._nodeItem.getLayerID(), self._index) elif action == disconnectBottom: self._nodeEditor.disconnectBottomBlob(self._nodeItem.getLayerID(), self._index) gui/main_window/node_editor/items/connector_item.py0000664ڊ00111170000001026513243274037023107 0ustar klemmso0cvprfrom PyQt5.QtWidgets import QGraphicsItem from gui.main_window.node_editor.items.renderer.connector_item_renderer import ConnectorItemRenderer class ConnectorItem(QGraphicsItem): """ Class to provide basic functionality for top and bottom connectors. This class should not get instanced. """ def __init__(self, index, nodeItem, nodeEditor, parent=None): super(ConnectorItem, self).__init__(parent) # set flag to get position changed to update connected connections self.setFlag(QGraphicsItem.ItemSendsScenePositionChanges, True) # draw above the node item self.setZValue(2.0) self._nodeEditor = nodeEditor self._nodeItem = nodeItem self._index = index self._blobName = "" self._connections = list() # create a renderer to draw the connector self._renderer = ConnectorItemRenderer(self) def boundingRect(self): return self._renderer.boundingRect() def addConnection(self, connection): """ Adds a connection to the connector (should get implemented in subclass) """ return def removeConnection(self, connection): """ Removes a connection """ self._connections.remove(connection) def paint(self, painter, option, widget=None): """ Draws the connector """ self._renderer.paint(painter) def setIndex(self, index): """ Sets the index of the connector in the node item """ self._index = index def isConnected(self): """ Returns True if there is any connection, False otherwise""" return len(self._connections) > 0 def isInPlace(self): """ Returns whether the connector is connected to a in-place working layer (should get implemented in subclass) """ return def getNodeEditor(self): """ Returns the NodeEditor object (for the renderer) """ return self._nodeEditor def getNodeItem(self): """ Returns the owning node item """ return self._nodeItem def setBlobName(self, blobName): """ Sets the blobs name """ self._blobName = blobName def getBlobName(self): """ Returns the blob name """ return self._blobName def getPhase(self): """ Returns the phase of the connector (phase of the owning node item) """ return self._nodeItem.getPhase() def getIndex(self): """ Returns the index of hte connector in the node item """ return self._index def isTopConnector(self): """ Returns whether the connector is a top connector (should get implemented in subclass)""" return def updateConnectionPositions(self): """ Updates the connected connections (should get implemented in subclass) """ return def getConnectedNodes(self): """ Returns a list of node items, connected to this connector (should get implemented in subclass) """ return def getConnections(self): """ Returns a list of connection items, connected to this connector """ return self._connections def getConnectionCount(self): """ Returns the number of connections, connected to this connector """ return len(self._connections) def hasPhaseConnection(self, phase): """ Checks whether the connector is connected to a layer in the given phase """ for connection in self._connections: if connection.getPhase() == "" or connection.getPhase() == phase: return True return False def getConnectedLayers(self): """ Returns a list of layer IDs of all connected layers """ layerIDs = list() connectedNodes = self.getConnectedNodes() for node in connectedNodes: layerIDs.append(node.getLayerID()) return layerIDs def itemChange(self, change, value): """ Update all connections, connected to this connector """ if change == QGraphicsItem.ItemScenePositionHasChanged: self.updateConnectionPositions() return super(ConnectorItem, self).itemChange(change, value) def disableEditing(self, disable): ''' Disable the context menu of the blob. ''' self.setEnabled(not disable)gui/main_window/node_editor/items/connector_top_item.py0000664ڊ00111170000000541613243274037023773 0ustar klemmso0cvprfrom PyQt5.QtWidgets import QMenu from gui.main_window.node_editor.items.connector_item import ConnectorItem class ConnectorTopItem(ConnectorItem): """ Class to provide top connector functionality """ def __init__(self, index, nodeItem, nodeEditor, parent=None): super(ConnectorTopItem, self).__init__(index, nodeItem, nodeEditor, parent) def isTopConnector(self): """ Returns whether the connector is a top connector (implementation for parent class) """ return True def isInPlace(self): """ Returns whether the connector is connected to a in-place working layer A top connector is in place if any connected bottom connector is in place. (implementation for parent class) """ for connection in self._connections: if connection.getIsInPlace(): return True return False def getConnectedNodes(self): """ Returns a list of node items, connected to this connector (implementation for parent class) """ nodes = list() # for each connection get the node connected to the bottom of the connection for connection in self._connections: connectionsBottomConnector = connection.getBottomConnector() if connectionsBottomConnector is not None: nodes.append(connectionsBottomConnector.getNodeItem()) return nodes def addConnection(self, connection): """ Adds a connection to the connector and sets the start of the connection to this connectors position (implementation for parent class) """ self._connections.append(connection) connection.setStart(self.scenePos()) def updateConnectionPositions(self): """ Updates the connected connections, sets the start of all connected connections to this connectors position (implementation for parent class) """ for connection in self._connections: connection.setStart(self.scenePos()) def contextMenuEvent(self, event): """ Context menu for the top connector """ contextMenu = QMenu() renameTop = contextMenu.addAction("Change name") disconnectTop = contextMenu.addAction("Disconnect") if self.getConnectionCount() == 0: disconnectTop.setEnabled(False) removeTop = contextMenu.addAction("Remove") action = contextMenu.exec_(event.screenPos()) if action is not None: if action == removeTop: self._nodeEditor.tryToRemoveTopBlob(self._nodeItem.getLayerID(), self._index) elif action == renameTop: self._nodeEditor.tryToRenameTopBlob(self) elif action == disconnectTop: self._nodeEditor.disconnectTopBlob(self._nodeItem.getLayerID(), self._index) gui/main_window/node_editor/items/node_item.py0000664ڊ00111170000001726313243274037022047 0ustar klemmso0cvprfrom PyQt5.QtCore import QRectF from PyQt5.QtWidgets import QGraphicsItem from gui.main_window.node_editor.items.connector_bottom_item import ConnectorBottomItem from gui.main_window.node_editor.items.connector_top_item import ConnectorTopItem from gui.network_manager.layer_helper import LayerHelper from gui.main_window.node_editor.items.renderer.node_item_renderer import NodeItemRenderer class NodeItem(QGraphicsItem): def __init__(self, layerID, layerData, nodeEditor, parent=None): super(NodeItem, self).__init__(parent) # Tell Qt to send change events self.setFlag(QGraphicsItem.ItemIsMovable, True) self.setFlag(QGraphicsItem.ItemIsSelectable, True) self.setFlag(QGraphicsItem.ItemSendsScenePositionChanges, True) self.setZValue(1.0) self.__nodeEditor = nodeEditor self.__layerID = layerID self.__layerData = layerData self.__phase = "" self.__isInPlace = False self.__bottomConnectors = list() self.__topConnectors = list() # create a renderer to split logic from rendering self.__renderer = NodeItemRenderer(self) self.updateLayerData() def updateLayerData(self): """ Updates the layers internal variables to represent changes made to the layer and updates the rendering """ self.__phase = LayerHelper.getLayerPhase(self.__layerData) self.__updateConnectorNames() self.__isInPlace = LayerHelper.isLayerInPlace(self.__layerData) self.__renderer.update() self.__renderer.updateConnectorPositions(self.__bottomConnectors, True) self.__renderer.updateConnectorPositions(self.__topConnectors, False) self.update() def __updateConnectorNames(self): """ Sets the internal connector names to correspond to the blob names in the underling data structure """ if "top" in self.__layerData["parameters"]: for index in range(0, len(self.__topConnectors)): blobName = self.__layerData["parameters"]["top"][index] self.__topConnectors[index].setBlobName(blobName) if "bottom" in self.__layerData["parameters"]: for index in range(0, len(self.__bottomConnectors)): blobName = self.__layerData["parameters"]["bottom"][index] self.__bottomConnectors[index].setBlobName(blobName) def addTopConnector(self, blobName): """ Adds a top connector with the given name to the node item """ connector = ConnectorTopItem(len(self.__topConnectors), self, self.__nodeEditor, self) connector.setBlobName(blobName) self.__topConnectors.append(connector) self.updateLayerData() def addBottomConnector(self, blobName): """ Adds a bottom connector with the given name to the node item """ connector = ConnectorBottomItem(len(self.__bottomConnectors), self, self.__nodeEditor, self) connector.setBlobName(blobName) self.__bottomConnectors.append(connector) self.updateLayerData() def removeBottomConnector(self, blobIndex): """ Removes the bottom connector with the given index """ self.__removeConnector(self.__bottomConnectors, blobIndex) def removeTopConnector(self, blobIndex): """ Removes the top connector with the given index """ self.__removeConnector(self.__topConnectors, blobIndex) def boundingRect(self): return self.__renderer.boundingRect() def paint(self, painter, option, widget): self.__renderer.paint(painter) def getTopConnectorCount(self): """ Returns the number of top connectors """ return len(self.__topConnectors) def getBottomConnectorCount(self): """ Returns the number of bottom connectors """ return len(self.__bottomConnectors) def getTopConnectors(self): """ Returns a list of all top connector items """ return self.__topConnectors def getBottomConnectors(self): """ Returns a list of all bottom connector items """ return self.__bottomConnectors def getNodeEditor(self): return self.__nodeEditor def getLayerID(self): """ Returns the internal layer ID of the layer represented by this node item """ return self.__layerID def getName(self): """ Returns the name of the layer, represented by this node item """ return self.__layerData["parameters"]["name"] def getTypeString(self): """ Returns the type name as a string """ return self.__layerData["type"].name() def getType(self): """ Returns the internal type object """ return self.__layerData["type"] def getPhase(self): """ Returns this node items phase """ return self.__phase def getIsInPlace(self): """ Returns, whether the layer, represented by this node item, works in-place """ return self.__isInPlace def getNodesConnectedToTops(self): """ Returns a list of node items connected to any top connector of this node item """ return self.__getNodesConnectedToConnectors(self.__topConnectors) def __removeConnector(self, connectors, indexToRemove): """ Removes the given connector from this node item """ self.scene().removeItem(connectors[indexToRemove]) del connectors[indexToRemove] self.__updateConnectorIndices(connectors) self.updateLayerData() def __updateConnectorIndices(self, connectors): """ Recalculates the connector indices after deleting a connector """ for i in range(0, len(connectors)): connectors[i].setIndex(i) i += 1 def getNodesConnectedToBottoms(self): """ Returns a list of node items connected to any bottom connector of this node item """ return self.__getNodesConnectedToConnectors(self.__bottomConnectors) def __getNodesConnectedToConnectors(self, connectors): """ Returns a list of node items connected to any connector in the given list of connectors """ connected = list() for connector in connectors: connectedNodes = connector.getConnectedNodes() for node in connectedNodes: if node not in connected: connected.append(node) return connected def itemChange(self, change, value): """ Check, whether the item is out of the scenes bounds """ if change == QGraphicsItem.ItemPositionChange: # get the new position newPos = value oldRect = self.sceneBoundingRect() newRect = QRectF(newPos.x(), newPos.y(), oldRect.width(), oldRect.height()) sceneRect = QRectF(self.scene().sceneRect()) # check if the left or right side of the new rect is outside the scenes bounds -> move it back inside if newRect.left() < sceneRect.left(): newPos.setX(sceneRect.left()) elif newRect.left() + newRect.width() > sceneRect.right(): newPos.setX(sceneRect.right() - newRect.width()) # check if the top or bottom side of the new rect is outside the scenes bounds -> move it back inside if newRect.bottom() > sceneRect.bottom(): newPos.setY(sceneRect.bottom() - newRect.height()) elif newRect.top() < sceneRect.top(): newPos.setY(sceneRect.top()) return newPos return super(NodeItem, self).itemChange(change, value) def disableBlobs(self, disable): ''' Disable the context menu of the tob and bottom blobs. ''' for blob in self.getBottomConnectors(): blob.disableEditing(disable) for blob in self.getTopConnectors(): blob.disableEditing(disable) gui/main_window/docks/__init__.py0000664ڊ00111170000000000013243274037017307 0ustar klemmso0cvprgui/main_window/docks/dock.py0000664ڊ00111170000000143213243274037016502 0ustar klemmso0cvpr# To change this license header, choose License Headers in Project Properties. # To change this template file, choose Tools | Templates # and open the template in the prototxt_editor. from PyQt5 import QtGui, QtCore, QtWidgets import functools class DockElement(QtWidgets.QDockWidget): def __init__(self, mainWindow, title): QtWidgets.QDockWidget.__init__(self, title) palette = self.palette() palette.setColor(QtGui.QPalette.Window, QtCore.Qt.white) self.setPalette(palette) self.setAutoFillBackground(1) self.adjustSize() self.setObjectName(title) # Connect the visibility changed event to the view manager self.visibilityChanged.connect(functools.partial(mainWindow.viewManager.onVisibilityChange))gui/main_window/docks/dock_active_layers.py0000664ڊ00111170000005540613243274037021426 0ustar klemmso0cvprimport functools from PyQt5 import QtCore, QtWidgets from PyQt5.QtCore import Qt from PyQt5.QtWidgets import QMenu, QAction, QMessageBox import backend.caffe.dict_helper as helper import gui.main_window.docks.properties.property_widgets.data as PropData from gui.main_window.docks.dock import DockElement class DockElementActivLayers(DockElement): def __init__(self, mainWindow, title): DockElement.__init__(self, mainWindow, 'Network Layers') self.name = title # Set the minimum size parameters self.setMinimumHeight(265) self.setMinimumWidth(250) widget = QtWidgets.QWidget() self.layout = QtWidgets.QVBoxLayout() widget.setLayout(self.layout) self.setWidget(widget) self.addTextfieldForSearch() self.addTableWidget() def addTextfieldForSearch(self): textField = QtWidgets.QLineEdit(self) textField.setPlaceholderText("Search for layers") textField.textChanged.connect(self.searchForLayers) self.layout.addWidget(textField) def addTableWidget(self): self.table = activLayerTable() self.layout.addWidget(self.table) def searchForLayers(self, layerName): self.table.searchForLayers(layerName) def getLayersListWidget(self): return self.table def mousePressEvent(self, QMouseEvent): return def disableEditing(self, disable): ''' Disable the table with all active layers. ''' if self.table: self.table.setDisabled(disable) class activLayerTable(QtWidgets.QTableWidget): def __init__(self): QtWidgets.QTableWidget.__init__(self) # Set behaviour and layout self.setHorizontalHeader(HorizontalHeader(QtCore.Qt.Horizontal, self)) self.horizontalHeader().setTableWidget(self) self.horizontalHeader().setSectionResizeMode(QtWidgets.QHeaderView.ResizeToContents) self.horizontalHeader().setStretchLastSection(True) self.setSelectionMode(QtWidgets.QAbstractItemView.SingleSelection) self.setSelectionBehavior(QtWidgets.QAbstractItemView.SelectRows) # Set the number of columns as well as their headers, also disabled the vertical headers self.setColumnCount(3) self.verticalHeader().setVisible(False) self.setHorizontalHeaderItem(0, QtWidgets.QTableWidgetItem("Order")) self.setHorizontalHeaderItem(1, QtWidgets.QTableWidgetItem("Name")) self.setHorizontalHeaderItem(2, QtWidgets.QTableWidgetItem("Type")) # Add signal handling when cell is changed by user input self.itemChanged.connect(self.cellIsChanged) # Initialize internal variables self.tableItems = [] self.tableItemsSearch = [] self.selectedItemRows = [] self.selectedItemIds = [] self.controlHold = False self.shiftHold = False # Store if the selection was changed by the keyboard or some other event self.selectionByKeyboard = 0 # Set up a variable to store the sorting order, default being Ascending by Name self.sortingOrder = (0, QtCore.Qt.AscendingOrder) self.reactiveState = None def addItem(self, id, name,order, type): # Add a new item to the internal data list and update the ui accordingly self.tableItems.append(TableItem(id, name,order, self.networkManager.getToolTip(id), type)) self.updateWithAllElements() # Sort the items so that the new item fits in at the right place self.sort() def __onUpdate(self, uri, reason): if uri[0] == "selection": self.clearSelection() for id in self.reactiveState["selection"]: self.addItemToSelection(id) if uri[0] == "network": network = self.reactiveState["network"] uri = uri[1:] if uri == ["layerOrder"]: for i, id in enumerate(network["layerOrder"]): self.updateItemOrder(id, i) self.updateWithAllElements() return if len(uri) > 1: id = uri[1] if uri[2:] == ["parameters","name"]: if isinstance(reason, PropData.ReasonUpdated): self.updateItemName(id, reason.newValue) else: if isinstance(reason, PropData.ReasonKeyAdded): id = reason.key if id in network["layerOrder"]: order = network["layerOrder"].index(id) else: order = 0 name = network["layers"][id]["parameters"]["name"] typename = network["layers"][id]["type"].name() self.addItem(id, name, order, typename) if isinstance(reason, PropData.ReasonKeyDeleted): id = reason.key self.deleteItemsByID([id]) def setReactiveState(self, state): if self.reactiveState: self.reactiveState.someChildPropertyChanged.disconnect(self.__onUpdate) self.reactiveState = state if self.reactiveState: self.reactiveState.someChildPropertyChanged.connect(self.__onUpdate) def setSelectedItem(self,id): self.selectionByKeyboard -= 1 self.clearSelection() for row in range(0, self.rowCount()): if id == self.item(row, 1).id: self.item(row, 0).setSelected(True) self.item(row, 1).setSelected(True) self.item(row, 2).setSelected(True) # Set this item as the last one selected self.selectedItemRows.append(row) self.selectedItemIds.append(id) # Make sure the item is visible by scrolling to it self.scrollToItem(self.item(row,0)) break def addItemToSelection(self, id): self.selectionByKeyboard -= 1 for row in range(0, self.rowCount()): if id == self.item(row, 1).id: self.item(row, 0).setSelected(True) self.item(row, 1).setSelected(True) self.item(row, 2).setSelected(True) # Set this item as the last one selected self.selectedItemRows.append(row) self.selectedItemIds.append(id) # Make sure the item is visible by scrolling to it self.scrollToItem(self.item(row, 0)) break def removeItemFromSelection(self, id): self.selectionByKeyboard -= 1 for row in range(0, self.rowCount()): if id == self.item(row, 1).id: self.item(row, 0).setSelected(False) self.item(row, 1).setSelected(False) self.item(row, 2).setSelected(False) # Set this item as the last one selected self.selectedItemRows.remove(row) self.selectedItemIds.remove(id) # Make sure the item is visible by scrolling to it self.scrollToItem(self.item(row, 0)) break def setSelectedItems(self, ids): self.selectionByKeyboard -= 1 self.clearSelection() for row in range(0, self.rowCount()): for id in ids: if id == self.item(row, 1).id: self.item(row, 0).setSelected(True) self.item(row, 1).setSelected(True) self.item(row, 2).setSelected(True) # Set this item as the last one selected self.selectedItemRows.append(row) self.selectedItemIds.append(id) # If there is only one selected item, make it visible by scrolling to it if len(ids) == 1: self.scrollToItem(self.item(row, 0)) break def clearSelection(self): self.setCurrentItem(None) self.selectedItemRows = [] self.selectedItemIds = [] def deleteItemsByID(self, ids): for id in ids: for i in range(0, len(self.tableItems)): if self.tableItems[i].id == id: del self.tableItems[i] break; # Update the list with all elements self.updateWithAllElements() def clearTable(self): self.clearContents() self.tableItems = [] self.tableItemsSearch = [] def updateWithAllElements(self): self.update(self.tableItems) def update(self, list): # Only delete the contents so that the headers are still left self.clearContents() # Set the size of the widget self.setRowCount(len(list)) # Set all items with their corresponding positions, then sort them for row in range(0, len(list)): self.setItem(row, 0, TableItemWidgetOrder(list[row].order, list[row].id, list[row].toolTip)) self.setItem(row, 1, TableItemWidgetName(list[row].name, list[row].id, list[row].toolTip)) self.setItem(row, 2, TableItemWidgetType(list[row].type, list[row].toolTip)) self.sort() # Re-select the items that had been selected for row in range(0, self.rowCount()): for id in self.selectedItemIds: if self.item(row, 1).id == id: self.item(row, 0).setSelected(True) self.item(row, 1).setSelected(True) self.item(row, 2).setSelected(True) break def sort(self): 'Sorts the items according to the sorting option used by user, default is Ascending Order by Name' self.sortItems(self.sortingOrder[0], self.sortingOrder[1]) def updateItemName(self, id, name): usedNames = [] for row in range(0, len(self.tableItems)): usedNames.append(self.tableItems[row].getName()) if name in usedNames: QMessageBox.about(self, "Warning!", "A layer with that name already exists. " "During training, caffe will assume shared weights among equally named layers. " "Training will fail if dimensions don't match.") for row in range(0, len(self.tableItems)): if self.tableItems[row].getID() == id: self.tableItems[row].setName(name) break # Item has been updated, call table update self.updateWithAllElements() def updateItemOrder(self, id, order): for row in range(0,len(self.tableItems)): if self.tableItems[row].getID() == id: self.tableItems[row].setOrder(order) break def searchForLayers(self, layerName): self.tableItemsSearch = [] for i in range(0, len(self.tableItems)): if layerName.lower() in self.tableItems[i].name.lower(): self.tableItemsSearch.append(self.tableItems[i]) # Update the table with the searched items, so that only those are shown self.update(self.tableItemsSearch) def scrollToItemByID(self, id): for row in range(0, self.rowCount() - 1): if self.item(row, 0).getID() == id: self.scrollToItem(self.item(row, 0)) def cellIsChanged(self, tableItem): if isinstance(tableItem, TableItemWidgetName): if(tableItem.isEdited()): # Store the table item ID id = tableItem.getID() # Change the name of the item in the internal table and all other elements #self.networkManager.updateName(tableItem.getID(), tableItem.text()) h = helper.DictHelper(self.networkManager.network) # type: helper.DictHelper h.layerParams(id)["name"] = tableItem.text() # Scroll to the renamed item to make sure that it is visible self.scrollToItemByID(id) def mousePressEvent(self, QMouseEvent): # Check if the user has clicked an item if self.itemAt(QMouseEvent.pos()) != None: # Get the current row and id of the item belonging to it row = self.itemAt(QMouseEvent.pos()).row() id = self.item(row, 1).getID() # On left button change the selection if QMouseEvent.button() == Qt.LeftButton: # Depending on whether the user is pressing the ctrl or shift key, add to or range selection if self.controlHold: # If the item is already selected, remove it from selection if self.item(row, 1).isSelected(): self.networkManager.removeLayerFromSelection(id) else: self.networkManager.addLayerToSelection(id) elif self.shiftHold: # The Row of the last selected element is stored as the last element of selectedItemRows lastItemRow = self.selectedItemRows[len(self.selectedItemRows) - 1] # Clear the selection self.networkManager.clearSelection() # Select items depending on the position of the clicked item if lastItemRow < row: # Start at row and select everything to the last selected element for i in range(lastItemRow, row + 1): self.networkManager.addLayerToSelection(self.item(i, 1).getID()) else: # Start at last selected item and select everything up to row for i in range(lastItemRow, row - 1, -1): self.networkManager.addLayerToSelection(self.item(i, 1).getID()) else: self.networkManager.setSelection(id) # On right button open the context menu if QMouseEvent.button() == Qt.RightButton: # Set the selection to the clicked item self.networkManager.setSelection(id) # Show the context menu self.showContextMenu(QMouseEvent.globalPos(), id) else: # Otherwise, clear the selection self.networkManager.clearSelection() def mouseMoveEvent(self, QMouseEvent): return def mouseReleaseEvent(self, QMouseEvent): return def mouseDoubleClickEvent(self, QMouseEvent): # Check if user clicked an actual item clickedItem = self.itemAt(QMouseEvent.pos()) # Enable editing of the item on double click, editing is only enabled for the name column if isinstance(clickedItem, TableItemWidgetName): clickedItem.setEdited(True) self.editItem(self.itemAt(QMouseEvent.pos())) def keyPressEvent(self, QKeyEvent): # Enable adding to selection on ctrl key if QKeyEvent.key() == QtCore.Qt.Key_Control: self.networkManager.setControlHold(True) # Enable range selection on shift key elif QKeyEvent.key() == QtCore.Qt.Key_Shift: self.networkManager.setShiftHold(True) def keyReleaseEvent(self, QKeyEvent): # Delete on del key if QKeyEvent.key() == QtCore.Qt.Key_Delete: self.networkManager.deleteSelectedLayers() # Switch selection to next item on tab elif QKeyEvent.key() == QtCore.Qt.Key_Tab: # Get the row of the last selected element row = self.selectedItemRows[len(self.selectedItemRows) - 1] # If the last item has been reached start at the beginning if row == self.rowCount() - 1: self.networkManager.setSelection(self.item(0, 1).id) else: self.networkManager.setSelection(self.item(row + 1, 1).id) # Switch to or add to selection on arrow down elif QKeyEvent.key() == QtCore.Qt.Key_Down: # Get the row of the last selected element row = self.selectedItemRows[len(self.selectedItemRows) - 1] # If last selection event was not done via keyboard, clear the selection if self.selectionByKeyboard < 0: self.networkManager.setSelection(self.item(row, 1).id) # Save that the last edit was done by keyboard self.selectionByKeyboard = 1 # If the last item has been reached do nothing if not row == self.rowCount() - 1: # Add to the selection if shift is held down if self.shiftHold: # If the next item is already selected, remove the current from selection if self.selectedItemRows.__contains__(row + 1): if self.selectedItemRows.__contains__(row): self.networkManager.removeLayerFromSelection(self.item(row, 1).id) else: self.networkManager.addLayerToSelection(self.item(row + 1, 1).id) else: self.networkManager.setSelection(self.item(row + 1, 1).id) # Switch to or add to selection on arrow up elif QKeyEvent.key() == QtCore.Qt.Key_Up: # Get the row of the last selected element row = self.selectedItemRows[len(self.selectedItemRows) - 1] # If last selection event was not done via keyboard, clear the selection if self.selectionByKeyboard < 0: self.networkManager.setSelection(self.item(row, 1).id) # Save that the last edit was done by keyboard self.selectionByKeyboard = 1 # If the first item has been reached do nothing if not row == 0: # Add to the selection if shift is held down if self.shiftHold: if self.selectedItemRows.__contains__(row - 1): if self.selectedItemRows.__contains__(row): self.networkManager.removeLayerFromSelection(self.item(row, 1).id) else: self.networkManager.addLayerToSelection(self.item(row - 1, 1).id) else: self.networkManager.setSelection(self.item(row - 1, 1).id) # Disabled adding to selection if ctrl key is released elif QKeyEvent.key() == QtCore.Qt.Key_Control: self.networkManager.setControlHold(False) # Disable range selected if shift key is release elif QKeyEvent.key() == QtCore.Qt.Key_Shift: self.networkManager.setShiftHold(False) def showContextMenu(self, pos, id): # Create a menu with context options menu = QMenu(self) # Add a focus option to context menu focusAction = QAction("Focus", self) focusAction.triggered.connect(functools.partial(self.networkManager.focusLayer, id)) menu.addAction(focusAction) # Add a delete option to context menu deleteAction = QAction('Delete', self) deleteAction.triggered.connect(functools.partial(self.networkManager.deleteLayer, id)) menu.addAction(deleteAction) # Show the menu at the current mouse position menu.popup(pos) def setNetworkManager(self, networkManager): self.networkManager = networkManager def getNetworkManager(self): return self.networkManager def setControlHold(self, bool): self.controlHold = bool def getControlHold(self): return self.controlHold def setShiftHold(self, bool): self.shiftHold = bool def getShiftHold(self): return self.shiftHold def setSortingOrder(self, column, order): self.sortingOrder = (column, order) def getSortingOrder(self): return self.sortingOrder class TableItem(): def __init__(self, id, name,order, toolTip, type): self.name = name self.id = id self.order = order self.toolTip = toolTip self.type = type def getID(self): return self.id def getName(self): return self.name def setName(self,name): self.name = name def setOrder(self,order): self.order = order class TableItemWidgetOrder(QtWidgets.QTableWidgetItem): def __init__(self, order,id, tooltip): QtWidgets.QTableWidgetItem.__init__(self, self._longString(order)) self.id = id self.order = order #self.setToolTip(tooltip) self.setFlags(QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled) def getID(self): return self.id def setOrder(self, order): self.order = order def getOrder(self): return self.order def _longString(self,number): strg = str(number) while len(strg)<3: strg = "0"+strg return strg class TableItemWidgetName(QtWidgets.QTableWidgetItem): def __init__(self, name, id, tooltip): QtWidgets.QTableWidgetItem.__init__(self, name) self.name = name self.id = id self.edited = False #self.setToolTip(tooltip) def setEdited(self, bool): self.edited = bool def isEdited(self): return self.edited def getID(self): return self.id def setName(self, name): self.name = name def getName(self): return self.name class TableItemWidgetType(QtWidgets.QTableWidgetItem): def __init__(self, type, tooltip): QtWidgets.QTableWidgetItem.__init__(self, type) self.setFlags(QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled) #self.setToolTip(tooltip) class HorizontalHeader(QtWidgets.QHeaderView): def __init(self, orientation, parent): QtWidgets.QHeaderView.__init__(orientation, parent) def setTableWidget(self, tableWidget): # Save the parent widget for access to current sorting order self.tableWidget = tableWidget # Set Sorting indicator so that it is Ascending Order by Name, then show sorting indicator self.setSortIndicator(1, QtCore.Qt.AscendingOrder) self.setSortIndicatorShown(True) def mousePressEvent(self, QMouseEvent): # Get the index from the element itself and the current order from the table widget index = self.logicalIndexAt(QMouseEvent.pos()) currentOrder = self.tableWidget.getSortingOrder()[1] # Set the new sorting order depending on the current order and clicked element if currentOrder == QtCore.Qt.AscendingOrder: self.tableWidget.setSortingOrder(index, QtCore.Qt.DescendingOrder) self.tableWidget.sort() self.setSortIndicator(index, QtCore.Qt.DescendingOrder) if currentOrder == QtCore.Qt.DescendingOrder: self.tableWidget.setSortingOrder(index, QtCore.Qt.AscendingOrder) self.tableWidget.sort() self.setSortIndicator(index, QtCore.Qt.AscendingOrder) # Show the sorting indicator self.setSortIndicatorShown(True) def mouseMoveEvent(self, QMouseEvent): return def mouseReleaseEvent(self, QMouseEvent): return gui/main_window/docks/dock_available_layers.py0000664ڊ00111170000001122613243274037022063 0ustar klemmso0cvprfrom PyQt5 import QtCore, QtWidgets from PyQt5 import QtGui from backend.caffe.proto_info import CaffeMetaInformation from gui.main_window.docks.dock import DockElement class DockElementLayers(DockElement): def __init__(self, mainWindow, title): # Set title and initialize the dock widget DockElement.__init__(self, mainWindow, title) self.name = title widget = QtWidgets.QWidget() self.layout = QtWidgets.QVBoxLayout() # Add the text field and the layer list to the widget self.addDescription() self.addTextfieldForSearch() self.addListOfAllLayers() widget.setLayout(self.layout) self.setWidget(widget) def addDescription(self): descriptionLabel = QtWidgets.QLabel("Drag and drop any of the available layers to the network canvas to add a new layer.", self) descriptionLabel.setObjectName("infoText") descriptionLabel.setWordWrap(True) # Set the object name so that the global stylesheet for info text will be applied. # descriptionLabel.setStyleSheet("color: rgba(0, 0, 0, 125)") self.layout.addWidget(descriptionLabel) def addTextfieldForSearch(self): textField = QtWidgets.QLineEdit(self) textField.setPlaceholderText("Search for Layers") textField.textChanged.connect(self.searchForLayers) self.layout.addWidget(textField) def addListOfAllLayers(self): #used: http://pyqt.sourceforge.net/Docs/PyQt4/qlistwidget.html self.allLayersWindow = QtWidgets.QListWidget(self) # Enable dragging of list elements, connect item pressed action so that current item is saves for drag and drop self.allLayersWindow.setDragEnabled(True) self.allLayersWindow.itemPressed.connect(self.startDragAction) #get all LayerTypes from CaffeMetaInformation self.caffeInfo = CaffeMetaInformation() self.allLayersDict = self.caffeInfo.availableLayerTypes() #convert the dictionary into a list self.allLayers = self.dictionaryToList(self.allLayersDict) self.setAllLayersNewLayerList(self.allLayers) #show all Layers in list self.updateAllLayersWindowNewList(self.allLayers) self.layout.addWidget(self.allLayersWindow) def dictionaryToList(self,dictionary): list = [None]*len(dictionary) for i in range(0,len(dictionary)): list[i] = dictionary.keys()[i] return list def setAllLayersNewLayerList(self,list): self.allLayers = list def setAllLayersAddLayerList(self, list): '''this function adds the layername that are in list to the list self.allLayers''' for i in range(0,len(list)): isIn =1 for j in range(0,len(self.allLayers)): if list[i] == self.allLayers[j]: isIn = 0 if(isIn): self.allLayers.append(list[i]) def updateAllLayersWindowNewList(self,newList): '''this function updates the list that shows the Layer with a given list, in this main_window are only the names of the layers represented''' self.allLayersWindow.clear() for i in range(0,len(newList)): self.allLayersWindow.addItem(newList[i]) self.allLayersWindow.sortItems(0) def updateAllLayersWindowAddList(self,listToAdd): '''this function updates the layerwindow and adds the given list to the layers that are allready shown, in this main_window are only the name of the layers represented''' for i in range(0,len(listToAdd)): if len(self.allLayersWindow.findItems(listToAdd[i],QtCore.Qt.MatchExactly)) is 0: self.allLayersWindow.addItem(listToAdd[i]) self.allLayersWindow.sortItems(0) #todo: write a function to search for a special layer and show only that one def searchForLayers(self,layerName): list = [] #search in list of all Layers which are layerName for i in range(0,len(self.allLayers)): if layerName.lower() in self.allLayers[i].lower(): list.append(self.allLayers[i]) self.updateAllLayersWindowNewList(list) #todo: create a function to drag and drop the layer into the main main_window to create a new network def startDragAction(self): mimeData = QtCore.QMimeData() mimeData.setText(self.allLayersWindow.currentItem().text()) drag = QtGui.QDrag(self) drag.setMimeData(mimeData) drag.exec_() def disableEditing(self, disable): """ Disable the table with all available layers. """ if self.allLayersWindow: self.allLayersWindow.setDisabled(disable) gui/main_window/docks/dock_console.py0000664ڊ00111170000001556513243274037020240 0ustar klemmso0cvpr# -*- coding: utf-8 -*- import traceback import PyQt5.QtGui as QtGui from PyQt5.Qt import QTextCursor from PyQt5.QtWidgets import QWidget, QTextEdit, QLabel, QComboBox, QVBoxLayout, QHBoxLayout from backend.barista.utils.logger import Log, Logger from gui.main_window.docks.dock import DockElement class DockElementConsole(DockElement): def __init__(self, mainWindow, title): DockElement.__init__(self, mainWindow, 'Console') self.name = title self.resize(700, 500) self.widget = QWidget() Log.newLine.connect(self.appendLineToConsole) Log.sigRefreshGui.connect(self.appendLinesToConsole) Log.sigRefreshCallers.connect(self.fillCallerComboBox) self.__callersCopy = [] self.__currentCallerFilter = -1 self.__currentTypeFilter = -1 self.__msgTypes = [Logger.ALL, Logger.ERROR, Logger.TEXT] self.__logOutput = None self.__callerComboBox = None self.__scrollDown = True self.__setupGui() def clearLog(self): ''' clears all input in the edittext field ''' self.__logOutput.clear() def appendLineToConsole(self, logline): ''' adds a line of input the the console with prefix if it is not filtered ''' if(self.__shouldDisplayLogLine(logline)): try: self.__setupFont(logline.caller, logline.msgType) sb = self.__logOutput.verticalScrollBar() if sb.value() >= sb.maximum(): self.__scrollDown = True else: self.__scrollDown = False # self.__writePrefix(logline.caller) not used because there is no append.. logText = logline.line self.__logOutput.append(self.__getPrefix(logline.caller, logline.time) + logText) except: errMsg = traceback.format_exc() self.__logOutput.append("Error during printing a log line: {}".format( errMsg )) def __writePrefix(self, caller): ''' NOT USED because of no appen of plain text and self.__setCursorToEnd() leads to lag of gui writes the prefix to the log without line feed ''' font = self.__logOutput.font() font.setBold(True) self.__logOutput.setCurrentFont(font) self.__logOutput.insertPlainText(self.__getPrefix(caller)) font = self.__logOutput.font() font.setBold(False) self.__logOutput.setCurrentFont(font) def __getPrefix(self, caller, time): ''' the prefix that have to be added ''' prefix = "" if self.__currentCallerFilter == -1: prefix = "[" + time + ", " + caller.description + "]" return prefix def __shouldDisplayLogLine(self, logline): ''' returns whether this line should be displayed. Filtered on Caller and MessageType ''' if(self.__currentCallerFilter == -1): if(self.__currentTypeFilter != -1): return (logline.msgType.typeId == self.__currentTypeFilter) return True elif(logline.caller.callerId == self.__callersCopy[self.__currentCallerFilter].callerId): if(self.__currentTypeFilter == -1): return True else: return logline.msgType.typeId == self.__currentTypeFilter return False def appendLinesToConsole(self, loglines): ''' adds multiple lines to the console ''' for logline in loglines: self.appendLineToConsole(logline) def __setupFont(self, caller, msgType): ''' setup the font ''' font = self.__logOutput.font() font.setBold(False) color = caller.color if(msgType == Logger.ERROR): color = msgType.color font.setFamily("Courier") font.setPointSize(10) self.__logOutput.setCurrentFont(font) self.__logOutput.setTextColor(color) def __setupGui(self): ''' adds the wigets to the gui ''' self.__logOutput = QTextEdit(self.widget) self.__logOutput.setReadOnly(True) self.__logOutput.setLineWrapMode(QTextEdit.NoWrap) self.__logOutput.verticalScrollBar().rangeChanged.connect(self.handleSroll) labelCallers = QLabel("Show output generated by ") self.__callerComboBox = QComboBox() self.__callerComboBox.currentIndexChanged.connect(self.__filterCallers) labelTypes = QLabel("Show messages of type ") self.__typeComboBox = QComboBox() self.__typeComboBox.currentIndexChanged.connect(self.__filterTypes) for type in self.__msgTypes: self.__typeComboBox.addItem(type.description) Log.refreshCallers() vbox = QVBoxLayout() hbox = QHBoxLayout() hbox.addWidget(labelCallers) hbox.addWidget(self.__callerComboBox) hbox.addWidget(labelTypes) hbox.addWidget(self.__typeComboBox) hbox.addStretch(1) vbox.addLayout(hbox) vbox.addWidget(self.__logOutput) self.widget.setLayout(vbox) self.widget.resize(700, 500) self.setWidget(self.widget) def __setCursorToEnd(self): ''' used to not insert new lines at the users selection ''' self.__logOutput.moveCursor(QTextCursor.End) def fillCallerComboBox(self, callers): ''' inserts the data for the filter caller combo box callers is a dictionary callerid (int) -> Caller object ''' #Display only callers that have content logged filterCallers = lambda caller: caller.used == True #save copy to get right index self.__callersCopy = list(filter(filterCallers, callers.values())) self.__callerComboBox.clear() model = self.__callerComboBox.model() # Appen All item = QtGui.QStandardItem(str(Logger.ALL.description)) # no Error model.appendRow(item) # AppendCallers for caller in self.__callersCopy : item = QtGui.QStandardItem(str(caller.description)) item.setForeground(caller.color) model.appendRow(item) def handleSroll(self): ''' cares about whether the scrollbar has to be on bottom of the console or not ''' sb = self.__logOutput.verticalScrollBar() if(self.__scrollDown): sb.setValue(sb.maximum()) def __filterCallers(self, i): if(i == -1): return; # invalid filter, empty box or resetet self.__currentCallerFilter = i-1 # -1 Because of "ALL" self.clearLog() Log.refreshConsole() def __filterTypes(self, i): self.__currentTypeFilter = i-1 # -1 Because of "ALL" self.clearLog() Log.refreshConsole() gui/main_window/node_editor/__init__.py0000664ڊ00111170000000000013243274037020477 0ustar klemmso0cvprgui/main_window/node_editor/layer_color_definitions.py0000664ڊ00111170000000415113243274037023660 0ustar klemmso0cvprfrom PyQt5.QtGui import QColor from backend.caffe.proto_info import LayerType from gui.main_window.node_editor.node_editor_constants import Constants class LayerColorDefinitions: __LAYER_CATEGORY_COLORS = {LayerType.CATEGORY_DATA : QColor( 0, 150, 0), LayerType.CATEGORY_LOSS : QColor( 0, 100, 255), LayerType.CATEGORY_ACTIVATION : QColor( 200, 200, 50), LayerType.CATEGORY_NONE : Constants.itemBackgroundColorLight} __LAYER_GROUP_INVALID = -1 __LAYER_GROUP_PYTHON_LAYER = 0 __LAYER_GROUP_STRUCTURAL_LAYER = 1 __LAYER_GROUP_COLORS = {__LAYER_GROUP_PYTHON_LAYER : QColor(50, 255, 255), __LAYER_GROUP_STRUCTURAL_LAYER : QColor(255, 100, 0)} __LAYER_GROUP_DEFINITIONS = {"Python" : __LAYER_GROUP_PYTHON_LAYER, "Convolution" : __LAYER_GROUP_STRUCTURAL_LAYER, "InnerProduct": __LAYER_GROUP_STRUCTURAL_LAYER, "Deconvolution": __LAYER_GROUP_STRUCTURAL_LAYER, "LSTM": __LAYER_GROUP_STRUCTURAL_LAYER, "RNN": __LAYER_GROUP_STRUCTURAL_LAYER, "BatchNorm": __LAYER_GROUP_STRUCTURAL_LAYER} def __init__(self): return @staticmethod def getTypeColor(nodeType): """ Returns the color for a given layer type based on the types category and type name """ # First try to get the group by type name. If the type name is found, try to get the groups color. # If the type name has an invalid group, the layers category is used to get the color. # If the type name can not be found, use the layer category as well. return LayerColorDefinitions.__LAYER_GROUP_COLORS.get( LayerColorDefinitions.__LAYER_GROUP_DEFINITIONS.get(nodeType.name(), LayerColorDefinitions.__LAYER_GROUP_INVALID), LayerColorDefinitions.__LAYER_CATEGORY_COLORS[nodeType.category()]) gui/main_window/node_editor/node_editor.py0000664ڊ00111170000006527413243274037021263 0ustar klemmso0cvprfrom PyQt5.QtCore import QPointF from PyQt5.QtWidgets import QInputDialog, QMessageBox from gui.main_window.node_editor.items.connection_item import ConnectionItem from gui.main_window.node_editor.items.node_item import NodeItem from gui.main_window.node_editor.node_editor_event_handler import NodeEditorEventHandler from gui.main_window.node_editor.node_editor_view import NodeEditorView from gui.main_window.node_editor.node_sort import NodeSort class NodeEditor: def __init__(self, parent=None): self.__view = NodeEditorView(self, parent) self.__scene = self.__view.scene() self.__networkManager = None self.__currentPhase = "" self.__nodes = dict() self.__hiddenConnections = list() self.__eventHandler = NodeEditorEventHandler(self, self.__view, self.__scene) self.__view.setEventHandler(self.__eventHandler) self.__scene.setEventHandler(self.__eventHandler) self.disable = False def getNodes(self): """ Returns the dict of all node items, the layer ID is the key """ return self.__nodes def setNetworkManager(self, networkManager): """ Sets the NetworkManager """ self.__networkManager = networkManager self.__eventHandler.setNetworkManager(networkManager) def getNetworkManager(self): """ Returns the NetworkManager """ return self.__networkManager def setCopySelection(self): self.__eventHandler.setCopySelection() def clearAll(self): """ Removes all items from the scene and clears the internal dicts """ self.__nodes.clear() self.__scene.clear() def createLayerPositionDict(self): """ Creates and returns a dict with the layer ID as key and the layers position in the scene as value """ # create a new dict positions = dict() # fill dict for layerID, nodeItem in self.__nodes.iteritems(): pos = nodeItem.scenePos() positions[layerID] = (pos.x(), pos.y()) return positions def getPositionOfLayer(self, id): """ Returns the position of a node item with a given layer ID """ pos = self.__nodes[id].scenePos() return pos.x(), pos.y() def setPositionOfLayer(self, id, pos): """ Sets the position of a node item with the given layer ID """ self.__nodes[id].setPos(pos[0],pos[1]) def applyLayerPositionDict(self, positionsDict): """ Sets the positions of all node items to the positions defined in positionsDict, layerID is the key """ for layerID, pos in positionsDict.iteritems(): self.__nodes[layerID].setPos(pos[0], pos[1]) def addLayer(self, layerID, layerData, scenePosX = 0.0, scenePosY = 0.0): """ Creates a new NodeItem to represent the layer """ # create the new node Item nodeItem = NodeItem(layerID, layerData, self) # set the tooltip containing the parameters to the node item parameters = self.__networkManager.getToolTip(layerID) nodeItem.setToolTip(parameters) # add top connectors to the node item if "top" in layerData["parameters"]: for top in layerData["parameters"]["top"]: nodeItem.addTopConnector(top) # add bottom connectors to the node item if "bottom" in layerData["parameters"]: for bottom in layerData["parameters"]["bottom"]: nodeItem.addBottomConnector(bottom) # add the node item to the scene and the internal dict self.__scene.addItem(nodeItem) self.__nodes[layerID] = nodeItem nodeItem.setPos(QPointF(scenePosX, scenePosY)) def setKeyboardFocus(self): self.__view.setFocus() def focusLayer(self, layerID): """ Focuses the viewport to the node item with the given layer ID """ position = self.__nodes[layerID].pos() self.__view.centerOn(position) def addBottomBlob(self, layerID, blobName): """ Adds a bottom connector to the node item with the given layer ID """ nodeItem = self.__nodes[layerID] nodeItem.addBottomConnector(blobName) def removeBottomBlob(self, layerID, blobIndex): """ Removes a bottom connector from the node item with the given layer ID """ self.__nodes[layerID].removeBottomConnector(blobIndex) # Update the tooltip as well self.updateTooltip(layerID) def addTopBlob(self, layerID, blobName): """ Adds a top connector to the node item with the given layer ID """ nodeItem = self.__nodes[layerID] nodeItem.addTopConnector(blobName) bottomConnectors = self.__nodes[layerID].getBottomConnectors() for bottomConnector in bottomConnectors: connections = bottomConnector.getConnections() for connection in connections: connection.updateData() # Update the tooltip as well self.updateTooltip(layerID) def renameTopBlob(self, layerID, blobIndex, newName): """ Renames a top connector and changes the name of all connected bottom connectors """ # update the top connectors name connector = self.__nodes[layerID].getTopConnectors()[blobIndex] connector.setBlobName(newName) self.__nodes[layerID].updateLayerData() # get all connected connectors and change their name connections = connector.getConnections() for connection in connections: bottomConnector = connection.getBottomConnector() self.__networkManager.renameBottomBlob(bottomConnector.getNodeItem().getLayerID(), bottomConnector.getIndex(), newName) connection.updateData() # update all connections bottomConnectors = self.__nodes[layerID].getBottomConnectors() for bottomConnector in bottomConnectors: connections = bottomConnector.getConnections() for connection in connections: connection.updateData() # Update the tooltip as well self.updateTooltip(layerID) def renameBottomBlob(self, layerID, blobIndex, newName): """ Renames a bottom connector and removes all connections getting invalid by this operation """ # rename the bottom connector bottomConnector = self.__nodes[layerID].getBottomConnectors()[blobIndex] bottomConnector.setBlobName(newName) self.__nodes[layerID].updateLayerData() # get all top connectors and check if they have the same name. If not -> remove connection connections = list(bottomConnector.getConnections()) for connection in connections: topConnector = connection.getTopConnector() if topConnector.getBlobName() != newName: self.__removeConnection(connection) # Update the tooltip as well self.updateTooltip(layerID) def removeTopBlob(self, layerID, blobIndex): """ Removes a top connector from the node item with the given layer ID """ self.__nodes[layerID].removeTopConnector(blobIndex) # Update the tooltip as well self.updateTooltip(layerID) def disconnectBottomBlob(self, layerID, blobIndex): """ Removes all connections connected to the bottom connector with the given index of the node item with the given layer ID """ connector = self.__nodes[layerID].getBottomConnectors()[blobIndex] self.__disconnectAllConnectorConnections(connector) # Update the tooltip as well self.updateTooltip(layerID) def disconnectTopBlob(self, layerID, blobIndex): """ Removes all connections connected to the top connector with the given index of the node item with the given layer ID """ connector = self.__nodes[layerID].getTopConnectors()[blobIndex] self.__disconnectAllConnectorConnections(connector) # Update the tooltip as well self.updateTooltip(layerID) def createConnection(self, topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex): """ Creates a new connection between the top connector at topBlobIndex of the node item with the topLayerID and the bottom connector at bottomBlobIndex of the node item with the bottomLayerID """ # get the connectors topConnector = self.__nodes[topLayerID].getTopConnectors()[topBlobIndex] bottomConnector = self.__nodes[bottomLayerID].getBottomConnectors()[bottomBlobIndex] # create a new connection item and add it to the scene connection = ConnectionItem(self) self.__scene.addItem(connection) # set the connection/connectors in the connection/connectors connection.setTopConnector(topConnector) connection.setBottomConnector(bottomConnector) bottomConnector.addConnection(connection) topConnector.addConnection(connection) def rearrangeNodes(self): """ Automatically sorts the node items """ nodesort = NodeSort(self.getNodes(), self.__view) self.__networkManager.checkLayersForMovement(self.getNodeIds()) def rearrangeNodesVertical(self): """ Automatically sorts the node items Vertical """ nodesort = NodeSort(self.getNodes(), self.__view, True) self.__networkManager.checkLayersForMovement(self.getNodeIds()) def clearSelection(self): """ Clears the selection """ self.__scene.clearSelection() def setSelectedLayer(self, layerID): """ Sets a single layer as selected. Already selected layers get deselected. """ # remove all node items from the selection self.clearSelection() # get the node item with layer ID and set it as selected for item in self.__scene.items(): if isinstance(item, NodeItem): if item.getLayerID() == layerID: item.setSelected(True) break def setSelectedLayers(self, layerIDs): """ Sets a list of layers as selected. Alread selected layers get deselected. """ # remove all node items from the selection self.clearSelection() # get all items in the scene layerList = list(layerIDs) items = self.__scene.items() # check all items and set the as selected if they are in the given list while(len(layerList) > 0): for item in items: if isinstance(item, NodeItem): if item.getLayerID() == layerList[0]: item.setSelected(True) break del layerList[0] def addLayerToSelection(self, layerID): """ Selects a node item without clearing other layers selection """ items = self.__scene.items() for item in items: if isinstance(item, NodeItem): if item.getLayerID() == layerID: item.setSelected(True) break def removeLayerFromSelection(self, layerID): """ Deselects a node item without clearing other layers selection """ selectedItems = self.__scene.selectedItems() for item in selectedItems: if isinstance(item, NodeItem): if item.getLayerID() == layerID: item.setSelected(False) break def updateLayerData(self, layerID): """ Notifies a node item to update the layer data to render correctly after change """ self.__nodes[layerID].updateLayerData() # Update the tooltip as well self.updateTooltip(layerID) def disconnectConnectionsOfSelection(self): """ Removes all connections from the selection and all connections connecting any selected node item """ # get all selected items selectedItems = self.__scene.selectedItems() # remove all selected connections for item in selectedItems: if isinstance(item, ConnectionItem): self.__removeConnection(item) # remove all connections connected to selected layers for item in selectedItems: if isinstance(item, NodeItem): self.__disconnectAllLayerConnections(item) def __disconnectAllLayerConnections(self, nodeItem): """ Removes all connections connected to any top or bottom connector of a given node item """ # remove connections of top connectors for topConnector in nodeItem.getTopConnectors(): self.__disconnectAllConnectorConnections(topConnector) # remove connections of bottom connectors for bottomConnector in nodeItem.getBottomConnectors(): self.__disconnectAllConnectorConnections(bottomConnector) def __disconnectAllConnectorConnections(self, connector): """ Removes all connections connected to a connector """ connections = list(connector.getConnections()) for connection in connections: self.__removeConnection(connection) def __removeConnection(self, connection): """ Removes a connection """ # get all selected items selectedItems = self.__scene.selectedItems() # remove connection from connectors topConnector = connection.getTopConnector() bottomConnector = connection.getBottomConnector() topConnector.removeConnection(connection) bottomConnector.removeConnection(connection) # get layer ID and connector index to notify data change bottomLayerID = bottomConnector.getNodeItem().getLayerID() bottomBlobIndex = bottomConnector.getIndex() # if the bottom connector has no connections left and is not in the selection, # remove the underling data relation of top-bottom blob if bottomConnector.getConnectionCount() == 0 and bottomConnector.getNodeItem() not in selectedItems: self.__networkManager.disconnectLayer(bottomLayerID, bottomBlobIndex) # remove the connection item from the scene self.__scene.removeItem(connection) def deleteItemsByID(self, list): """ Removes node items from the scene based on the list of layer IDs """ for layerID in list: items = self.__scene.items() for item in items: if isinstance(item, NodeItem): if item.getLayerID() == layerID: self.__scene.removeItem(item) index = self.__nodes.values().index(item) key = self.__nodes.keys()[index] del self.__nodes[key] def updateLayerName(self, layerID, name): """ Notify the node item to update the rendering. """ self.__nodes[layerID].updateLayerData() # Update the tooltip as well self.updateTooltip(layerID) def updateTooltip(self, layerID): """ Updates the tooltip of the node item with the given layer ID """ parameters = self.__networkManager.getToolTip(layerID) self.__nodes[layerID].setToolTip(parameters) def setControlHold(self, bool): self.__eventHandler.setControlHold(bool) def createListFromHiddenConnections(self): """ Creates a list of tuples containing the hidden connections, the tuples have the form (topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex) """ outList = list() for connection in self.__hiddenConnections: topConnector = connection.getTopConnector() bottomConnector = connection.getBottomConnector() outList.append((topConnector.getNodeItem().getLayerID(), topConnector.getIndex(), bottomConnector.getNodeItem().getLayerID(), bottomConnector.getIndex())) return outList def setHiddenConnectionsFromList(self, hiddenList): """ Sets all connections to show and then hides the connections specified in hiddenList, hiddenList contains tuples (topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex) """ self.__hiddenConnections = [] for data in hiddenList: connection = self.__getConnection(data[0], data[1], data[2], data[3]) if connection is not None: connection.setHidden(True) self.__hiddenConnections.append(connection) def tryToAddHiddenConnection(self, connection): """ Called by connections to notify a hidden state change. """ if connection not in self.__hiddenConnections: self.__hiddenConnections.append(connection) self.__networkManager.connectionsHiddenStateChanged(self.createListFromHiddenConnections()) def tryToRemoveHiddenConnection(self, connection): """ Called by connections to notify a hidden state change. """ if connection in self.__hiddenConnections: self.__hiddenConnections.remove(connection) self.__networkManager.connectionsHiddenStateChanged(self.createListFromHiddenConnections()) def tryToCreateLayer(self, type, scenePosX, scenePosY): """ Notifies the NetworkManager to create a new Layer. """ self.__networkManager.addLayer(type, scenePosX, scenePosY) def tryToAddBottomBlob(self, layerID): """ Notifies the NetworkManager to create a new bottom connector for the given layer. """ self.__networkManager.addBottomBlob(layerID) def tryToRemoveBottomBlob(self, layerID, blobIndex): """ Notifies the NetworkManager to try to remove the bottom connector with the given index in the given layer. """ self.__networkManager.removeBottomBlob(layerID, blobIndex) def getNode(self, layerID): return self.__nodes[layerID] def tryToAddTopBlob(self, layerID, blobName): """ Notifies the NetworkManager to add a top connector to the layer with the given name. """ self.__networkManager.addTopBlob(layerID, blobName) def tryToRenameTopBlob(self, connector): """ Shows a input dialog to ask for a new blob name and notifies the NetworkManager to change the blobs name. """ # show input dialog name, ok = QInputDialog.getText(self.__view, "Change name of top blob", "Top blob name:") name = name.strip() # if the user did not provide a valid name, ask again while ok and len(name) == 0: QMessageBox.warning(self.__view, "Top blob name is empty", "The name for the top blob can't be empty or all whitespace characters.") name, ok = QInputDialog.getText(self.__view, "Change name of top blob", "Top blob name:") name = name.strip() # the user provided a valid name if ok: # check if any bottom connector has multiple connections as a bottom connector can only have one name bottomConnectorWithMultipleConnectionsFound = False connections = connector.getConnections() for connection in connections: bottomConnector = connection.getBottomConnector() if bottomConnector.getConnectionCount() > 1: bottomConnectorWithMultipleConnectionsFound = True # ask the user whether they want to continue (break invalid connections) ore cancel shouldRename = True if bottomConnectorWithMultipleConnectionsFound: reply = QMessageBox.question(self.__view, "Removing connections", "A bottom blob, connected to the top blob, has multiple connections. " "By renaming the top blob, the other connections will get removed. " "Continue?", QMessageBox.Yes, QMessageBox.No) if reply != QMessageBox.Yes: shouldRename = False # if the user did not cancel -> notify the Networkmanager if shouldRename: self.__networkManager.renameTopBlob(connector.getNodeItem().getLayerID(), connector.getIndex(), name) def tryToRemoveTopBlob(self, layerID, blobIndex): """ Notifies the NetworkManager to remove the bottom connector with the given index of the given layer """ self.__networkManager.removeTopBlob(layerID, blobIndex) def tryToClearSelection(self): """ Notifies the NetworkManager to clear the selection """ self.__networkManager.clearSelectionWithoutSavingHistory() def tryToSetSelectionList(self, layerIDs): """ Notifies the NetworkManager to update the selected layer list """ self.__networkManager.setSelectionList(layerIDs) def tryToDeleteSelection(self): """ Notifies the NetworkManager to remove all selected layers """ self.__networkManager.deleteSelectedLayers() def tryToConnect(self, topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex): """ Checks wheter the new connection would be valid and notifies the NetworkManager to create the new connection """ topConnector = self.__nodes[topLayerID].getTopConnectors()[topBlobIndex] bottomConnector = self.__nodes[bottomLayerID].getBottomConnectors()[bottomBlobIndex] alreadyConnectedBottomLayers = topConnector.getConnectedLayers() # check id the new connections is between two connectors of the same layer7 if topConnector.getNodeItem() == bottomConnector.getNodeItem(): return False # check if connection between both connectors exists if bottomLayerID in alreadyConnectedBottomLayers: return False # check if both layers have the same phase (if both are not in all phases) if topConnector.getPhase() != "" and bottomConnector.getPhase() != "": if topConnector.getPhase() != bottomConnector.getPhase(): return False # if the top layer is in all phases, the bottom layer is not allowed to have any connection at the connector if topConnector.getPhase() == "": if bottomConnector.getConnectionCount() > 0: return False # if the top layer is not in all phases, check if the # bottom layer connector has another connection in the same phase else: if bottomConnector.hasPhaseConnection(topConnector.getPhase()): return False elif bottomConnector.getBlobName() != "": if topConnector.getBlobName() != bottomConnector.getBlobName(): return False if topConnector.isInPlace(): if topConnector.hasPhaseConnection(bottomConnector.getPhase()): return False # layers can get connected self.__networkManager.connectLayers(topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex) return True def isCurrentPhase(self, phase): """ Returns whether the phase is equals the current displayed phase. """ if len(phase) == 0 or self.__currentPhase == phase: return True else: return False def __getConnection(self, topID, topIndex, bottomID, bottomIndex): """ Returns the connection between the layers with the IDs topID and bottomID, connected at the connectors with the indices topIndex and bottomIndex """ topConnector = self.__nodes[topID].getTopConnectors()[topIndex] connections = topConnector.getConnections() for connection in connections: bottomConnector = connection.getBottomConnector() if bottomConnector.getNodeItem().getLayerID() == bottomID and bottomConnector.getIndex() == bottomIndex: return connection return None def calculateLayerOrder(self): """ Returns a list of layer ID representing the order of connection """ order = list() # get a list of all node items with no bottom connectors # or with no connections connected to any bottom connector currentNodes = self.__getStartNodes() waitingNodes = dict() # unused touched = {} # repeat until all nodes have been processed while len(currentNodes) > 0: currentNode = currentNodes[0] touched[currentNode.getLayerID()] = currentNode.getName() # add the current node to the order list order.append(currentNode.getLayerID()) # get a list of all nodes connected to any top connector of the current node followingNodes = currentNode.getNodesConnectedToTops() for node in followingNodes: touched[node.getLayerID()] = node.getName() # for all following nodes, check if they are waiting (not all node items connected to bottoms # have been processed) and decrees the waiting count. If all needed nodes have benn processed, # add the node to the processable nodes for following in followingNodes: # the layer has already been processed if following.getLayerID() in order: continue # layer has other unprocessed pre nodes if following in waitingNodes: if waitingNodes[following] > 1: waitingNodes[following] -= 1 else: currentNodes.append(following) del waitingNodes[following] else: inputNodeCount = len(following.getNodesConnectedToBottoms()) if inputNodeCount > 1: waitingNodes[following] = inputNodeCount - 1 else: currentNodes.append(following) currentNodes.remove(currentNode) return order def getView(self): return self.__view def getScene(self): return self.__view.scene() def __getStartNodes(self): """ Returns a list of all node items with no bottom connectors or no connections connected to bottom connectors. """ startNodes = list() for layerID, item in self.__nodes.iteritems(): if item.getBottomConnectorCount() == 0: startNodes.append(item) else: connected = False for connector in item.getBottomConnectors(): if connector.getConnectionCount() > 0: connected = True break if not connected: startNodes.append(item) return startNodes def getNodeIds(self): """ Returns a list of all layer IDs of nodes currently in the scene. """ return self.__nodes.keys() def disableEditing(self, disable): """ Disable relevant changes of the net. """ self.disable = disable self.__eventHandler.setDisabled(disable) self.__view.scene().disableEditing(disable) for _, node in self.getNodes().iteritems(): node.disableBlobs(disable) gui/main_window/node_editor/node_editor_constants.py0000664ڊ00111170000000471513243274037023350 0ustar klemmso0cvprfrom PyQt5.QtGui import QFont, QFontMetricsF, QColor class Constants: """ This class stores constant values used by the node prototxt_editor """ # grid size and pattern sceneGridSize = 20 sceneLargeGridCells = 5 # scene size sceneInitialWidth = 32000 sceneInitialHeight = 32000 # used for zoom in/out using the mouse wheel viewScaleFactor = 1.1 viewMaxScale = 0.2 viewMinScale = 5.0 # use default font font = QFont() # font used for the layer name nodeItemFontName = QFont(font) nodeItemFontName.setPointSize(10) nodeItemFontName.setBold(True) nodeItemFontNameMetrics = QFontMetricsF(nodeItemFontName) # font used for the layer type nodeItemFontType = QFont(font) nodeItemFontType.setPointSize(8) nodeItemFontTypeMetrics = QFontMetricsF(nodeItemFontType) # font used for the blob names nodeItemFontBlob = QFont(font) nodeItemFontBlob.setPointSize(8) nodeItemFontBlobMetrics = QFontMetricsF(nodeItemFontBlob) # constants used to add free spaces in the node item rendering nodeItemMinBlobAreaHeight = 10 nodeItemHeadPadding = 20 nodeItemSelectionSize = 3 nodeItemBorderSize = 3 nodeItemTextMargin = 4 nodeItemConnectorPaddding = 6 # connector size connectorItemOuterSize = 12 connectorItemInnerSize = 6 # opacity for items in phase and normal items itemOpacityInPhase = 200 itemOpacityNotInPhase = 100 # node item background colors itemBackgroundColorDark = QColor(100, 100, 100) itemBackgroundColorLight = QColor(200, 200, 200) itemBorderColor = QColor(0, 0, 0) # color of items working in-place itemInPlaceColor = QColor(30, 30, 200) # connection item constants connectionItemColor = QColor(100, 100, 100) connectionItemSelectionSize = 9 connectionItemSize = 5 connectionItemHiddenDashCount = 4 connectionItemHiddenDashSize = 10 connectionItemHiddenDashSpace = 10 # color for selected items (node item, connection) selectedColor = QColor(150, 150, 0) def __init__(self): return @staticmethod def setPaintDevice(device): """ Used to setup font metrics correctly """ Constants.nodeItemFontNameMetrics = QFontMetricsF(Constants.nodeItemFontName, device) Constants.nodeItemFontTypeMetrics = QFontMetricsF(Constants.nodeItemFontType, device) Constants.nodeItemFontBlobMetrics = QFontMetricsF(Constants.nodeItemFontBlob, device) gui/main_window/node_editor/node_editor_event_handler.py0000664ڊ00111170000006574513243274037024164 0ustar klemmso0cvprfrom PyQt5.QtCore import Qt, QPointF, QPoint, QRect, QRectF, QSize, QSizeF from PyQt5.QtGui import QTransform, QCursor, QPainterPath from PyQt5.QtWidgets import QGraphicsView, QRubberBand, QMenu, QInputDialog, QMessageBox from gui.main_window.node_editor.items.connection_item import ConnectionItem from gui.main_window.node_editor.items.connector_item import ConnectorItem from gui.main_window.node_editor.items.node_item import NodeItem from gui.main_window.node_editor.node_editor_constants import Constants class NodeEditorEventHandler: def __init__(self, nodeEditor, view, scene): self.nodeEditor = nodeEditor self.view = view self.scene = scene # Declare local variables used in mouse and keyboard events self.previousGlobalMousePos = QPointF(0.0, 0.0) self.previousLocalMousePos = QPointF(0.0, 0.0) self.clickOrigin = QPoint(0, 0) self.clearSelection = False self.setSelectionToItem = None self.removeFromSelection = False self.removeFromSelectionItemID = None # Save the last item the mouse has moved over self.lastHoveredItem = None # Save whether a button of the mouse or keyboard is currently being held down by user self.leftMouseButtonHold = False self.middleMouseButtonHold = False self.controlHold = False self.copySelection = [] self.blockTranslate = False # Save whether there is an ongoing rubber band selection self.dragMode = "Drag" self.dragModeBlocked = False self.view.setCursor(QCursor(Qt.OpenHandCursor)) self.view.setTransformationAnchor(QGraphicsView.NoAnchor) # Set standard drag mode and create rubber band element self.rubberBand = QRubberBand(QRubberBand.Rectangle, view) # Save if editing is allowed self.disabled = False def wheelEvent(self, QGraphicsSceneWheelEvent): # Handle the wheel event, which changes the zoom of the view oldScalePos = self.view.mapToScene(QGraphicsSceneWheelEvent.pos()) if QGraphicsSceneWheelEvent.angleDelta().y() > 0: self.view.scaleSymmetric(Constants.viewScaleFactor) else: self.view.scaleSymmetric(1 / Constants.viewScaleFactor) # Store the position of the mouse with the new scaling factor newScalePos = self.view.mapToScene(QGraphicsSceneWheelEvent.pos()) # Translate the view so that the mouse is the anchor for the zoom translationFactor = newScalePos - oldScalePos self.view.translate(translationFactor.x(), translationFactor.y()) def mousePressEvent(self, QMouseEvent): # Block all changes of the dragging mode self.dragModeBlocked = True # Transform position into scene position scenePos = self.view.mapToScene(QMouseEvent.pos()) # Take action according to pressed mouse button if QMouseEvent.button() == Qt.LeftButton: # Store that left mouse button is held down self.leftMouseButtonHold = True # Further actions according to current mouse mode if self.dragMode == "Drag": # First of, clear former selection self.networkManager.clearSelection() for item in self.scene.selectedItems(): item.setSelected(False) # Store the global position to calculate translation of view on mouse move self.previousGlobalMousePos = QMouseEvent.pos() elif self.dragMode == "Selection": # First of, clear former selection self.networkManager.clearSelection() # Start the rubber band rectangle selection self.clickOrigin = scenePos self.rubberBand.setGeometry(QRect(self.view.mapFromScene(self.clickOrigin), QSize())) self.rubberBand.show() elif self.dragMode == "Items": # Selection change when left mouse button is pressed, also prepare for moving selected elements self.previousLocalMousePos = scenePos self.pressedItem = self.scene.itemAt(scenePos, QTransform()) if isinstance(self.pressedItem, NodeItem): #check if a connection is in the selection, if yes, clear it from the selection for item in self.scene.selectedItems(): if isinstance(item, ConnectionItem): item.setSelected(False) # Check if ctrl key is held down if self.controlHold: # Check if there are connection elements in the selection, if yes, clear selection for item in self.scene.selectedItems(): if isinstance(item, ConnectionItem): self.networkManager.clearSelection() break # Check if the item has already been selected if self.pressedItem.isSelected(): # Start removing from selection. only finish if items are not moved before mouse release self.removeFromSelection = True self.removeFromSelectionItemID = self.pressedItem.getLayerID() else: self.networkManager.addLayerToSelection(self.pressedItem.getLayerID()) else: self.networkManager.clearSelection() # Check if there is no currently selected Item if len(self.scene.selectedItems()) == 0: # Select the new Item self.networkManager.setSelection(self.pressedItem.getLayerID()) elif len(self.scene.selectedItems()) == 1: # Check if the item has already been selected if self.pressedItem.isSelected(): # Start clearing selection, only finish if item is not moved before mouse release self.clearSelection = True else: self.networkManager.setSelection(self.pressedItem.getLayerID()) elif len(self.scene.selectedItems()) > 1: # Prepare for the item that has been pressed to be the only selected Item, if it is not moved in the meantime self.setSelectionToItem = scenePos elif isinstance(self.pressedItem, ConnectorItem): # Connector has been clicked, start the connection process (further handling in mouse move event) self.scene.startConnection(self.pressedItem, scenePos) elif isinstance(self.pressedItem, ConnectionItem): if self.controlHold: # Check if there are node items in the selection, if yes, clear selection if len(self.scene.selectedItems()) > 0: for item in self.scene.selectedItems(): if isinstance(item, NodeItem): self.networkManager.clearSelection() break if self.pressedItem.isSelected(): self.pressedItem.setSelected(False) else: self.pressedItem.setSelected(True) else: if self.pressedItem.isSelected(): self.pressedItem.setSelected(False) else: self.pressedItem.setSelected(True) else: # Check if the item is already selected if self.pressedItem.isSelected(): if len(self.scene.selectedItems()) > 0: self.networkManager.clearSelection() for item in self.scene.selectedItems(): item.setSelected(False) self.pressedItem.setSelected(True) else: self.pressedItem.setSelected(False) else: self.networkManager.clearSelection() for item in self.scene.selectedItems(): item.setSelected(False) self.pressedItem.setSelected(True) elif QMouseEvent.button() == Qt.RightButton: # Store the item that has been pressed by the mouse pressedItem = self.scene.itemAt(scenePos, QTransform()) if isinstance(pressedItem, NodeItem): # Make sure item selection is correct if not pressedItem.isSelected(): if self.controlHold: self.networkManager.addLayerToSelection(pressedItem.getLayerID()) else: self.networkManager.setSelection(pressedItem.getLayerID()) contextMenu = QMenu() addTopAction = None addBottomAction = None removeAction = None if not self.disabled: addTopAction = contextMenu.addAction("Add top") addBottomAction = contextMenu.addAction("Add bottom") copyItemsAction = contextMenu.addAction("Copy") if not self.disabled: removeAction = contextMenu.addAction("Remove") action = contextMenu.exec_(QMouseEvent.screenPos().toPoint()) if action is not None: if action == addTopAction: name, ok = QInputDialog.getText(self.view, "Add top blob", "Top blob name:") name = name.strip() while ok and len(name) == 0: QMessageBox.warning(self.view, "Top blob name is empty", "The name for the top blob can't be empty or all whitespace characters.") name, ok = QInputDialog.getText(self.view, "Add top blob", "Top blob name:") name = name.strip() if ok: self.nodeEditor.tryToAddTopBlob(pressedItem.getLayerID(), name) elif action == addBottomAction: self.nodeEditor.tryToAddBottomBlob(pressedItem.getLayerID()) elif action == copyItemsAction: self.nodeEditor.setCopySelection() elif action == removeAction: self.networkManager.deleteSelectedLayers() elif isinstance(pressedItem, ConnectionItem): self.networkManager.clearSelection() for item in self.scene.selectedItems(): item.setSelected(False) pressedItem.setSelected(True) elif pressedItem == None: # Open a context menu. contextMenu = QMenu() # If an item has been copied, show the 'Paste' action. if len(self.copySelection) > 0 and not self.disabled: pastAction = contextMenu.addAction("Paste") contextMenu.addSeparator() # Add the 'Arrange Network' actions. horArrangeAction = contextMenu.addAction("Arrange Network Horizontally") vertArrangeAction = contextMenu.addAction("Arrange Network Vertically") # Execute the menu and act according to the selected action. action = contextMenu.exec_(QMouseEvent.screenPos().toPoint()) if action and action.text() == 'Paste': self.networkManager.duplicateLayers(self.copySelection, self.view.mapToScene(QMouseEvent.pos())) elif action and action.text() == 'Arrange Network Horizontally': self.view.arrangeHorizontallyClicked.emit() elif action and action.text() == 'Arrange Network Vertically': self.view.arrangeVerticallyClicked.emit() def mouseMoveEvent(self, QMouseEvent): # Transform position into scene position scenePos = self.view.mapToScene(QMouseEvent.pos()) if self.leftMouseButtonHold: if self.dragMode == "Drag": # Set the new mouse cursor self.view.setCursor(QCursor(Qt.ClosedHandCursor)) # Calculate the movement for the view translation offset = self.previousGlobalMousePos - QMouseEvent.pos() self.previousGlobalMousePos = QMouseEvent.pos() offset *= 1/self.view.getScale() self.view.translate(-offset.x(), -offset.y()) elif self.dragMode == "Selection": # Check if the user has left the scene with the mouse, if yes, translate accordingly if QMouseEvent.pos().x() > self.view.viewport().width() and QMouseEvent.pos().y() > self.view.viewport().height(): QCursor.setPos(self.view.mapToGlobal(QPoint(self.view.viewport().width(), self.view.viewport().height()))) self.view.translate(-3 * self.view.getScale(), -3 * self.view.getScale()) elif QMouseEvent.pos().x() < 0 and QMouseEvent.pos().y() < 0: QCursor.setPos(self.view.mapToGlobal(QPoint(1, 1))) self.view.translate(3 * self.view.getScale(), 3 * self.view.getScale()) elif QMouseEvent.pos().x() > self.view.viewport().width() and QMouseEvent.pos().y() < 0: QCursor.setPos(self.view.mapToGlobal(QPoint(self.view.viewport().width(), 1))) self.view.translate(-3 * self.view.getScale(), 3 * self.view.getScale()) elif QMouseEvent.pos().x() < 0 and QMouseEvent.pos().y() > self.view.viewport().height(): QCursor.setPos(self.view.mapToGlobal(QPoint(1, self.view.viewport().height()))) self.view.translate(3 * self.view.getScale(), -3 * self.view.getScale()) elif QMouseEvent.pos().x() > self.view.viewport().width(): QCursor.setPos(self.view.mapToGlobal(QPoint(self.view.viewport().width(), QMouseEvent.pos().y()))) self.view.translate(-3 * self.view.getScale(), 0) elif QMouseEvent.pos().y() > self.view.viewport().height(): QCursor.setPos(self.view.mapToGlobal(QPoint(QMouseEvent.pos().x(), self.view.viewport().height()))) self.view.translate(0, -3 * self.view.getScale()) elif QMouseEvent.pos().x() < -1: QCursor.setPos(self.view.mapToGlobal(QPoint(1, QMouseEvent.pos().y()))) self.view.translate(3 * self.view.getScale(), 0) elif QMouseEvent.pos().y() < -1: QCursor.setPos(self.view.mapToGlobal(QPoint(QMouseEvent.pos().x(), 1))) self.view.translate(0, 3 * self.view.getScale()) # Map origin coordinates to current view coordinates clickOriginView = self.view.mapFromScene(self.clickOrigin) # Calculate the new top left point as well as width and height topLeft = QPoint(min(clickOriginView.x(), self.view.mapFromGlobal(QCursor.pos()).x()), min(clickOriginView.y(), self.view.mapFromGlobal(QCursor.pos()).y())) size = QSize(abs(clickOriginView.x() - self.view.mapFromGlobal(QCursor.pos()).x()), abs(clickOriginView.y() - self.view.mapFromGlobal(QCursor.pos()).y())) self.rubberBand.setGeometry(QRect(topLeft, size)) elif self.dragMode == "Items": if self.scene.currentlyConnecting is not None and not self.disabled: self.scene.currentlyConnecting.updateMousePosition(scenePos) else: # Moving should only be enabled for node items if isinstance(self.pressedItem, NodeItem): # Cancel the selection clearing if self.clearSelection: self.clearSelection = False elif self.removeFromSelection: self.removeFromSelection = False self.removeFromSelectionItemID = None elif self.setSelectionToItem is not None: self.setSelectionToItem = None # Calculate the change in mouse position currentLocalMousePos = scenePos translation = currentLocalMousePos - self.previousLocalMousePos self.previousLocalMousePos = currentLocalMousePos # Move all selected Elements according to mouse movement resetMouse = self.translateItems(translation.x(), translation.y()) if resetMouse: QCursor.setPos(QCursor.pos().x() - translation.x(), QCursor.pos().y() - translation.y()) else: if self.lastHoveredItem != self.scene.itemAt(scenePos, QTransform()): # If no button is held down, change mouse pointer according to item it hovers over if self.scene.itemAt(scenePos, QTransform()) is None: if self.controlHold: self.view.setCursor(QCursor(Qt.CrossCursor)) self.dragMode = "Selection" else: self.view.setCursor(QCursor(Qt.OpenHandCursor)) self.dragMode = "Drag" else: self.view.setCursor(QCursor(Qt.ArrowCursor)) self.dragMode = "Items" # Set a new last hovered item self.lastHoveredItem = self.scene.itemAt(scenePos, QTransform()) def mouseReleaseEvent(self, QMouseEvent): # Transform position into scene position scenePos = self.view.mapToScene(QMouseEvent.pos()) if QMouseEvent.button() == Qt.LeftButton: # Store that button is no longer being held down self.leftMouseButtonHold = False if self.dragMode == "Drag": self.view.setCursor(Qt.OpenHandCursor) elif self.dragMode == "Selection": # Disable the rubber band selection and hide it self.isRubberBandSelection = False self.rubberBand.hide() # Set the scene selection to all elements within the rubber band area painterPath = QPainterPath() # Calculate rectangle selection position, using the rubber band will fail at this point clickOriginView = self.view.mapFromScene(self.clickOrigin) sceneOrigin = self.view.mapToScene(clickOriginView) topLeft = QPointF(min(sceneOrigin.x(), scenePos.x()), min(sceneOrigin.y(), scenePos.y())) size = QSizeF(abs(sceneOrigin.x() - scenePos.x()), abs(sceneOrigin.y() - scenePos.y())) # Select every element within the rectangle selection painterPath.addRect(QRectF(topLeft, size)) self.scene.setSelectionArea(painterPath, QTransform()) # Get the ids of all selected elements to set selection to other ui elements selectedIDs = [] # Kick all connection items out of that selection for item in self.scene.selectedItems(): if isinstance(item, NodeItem): selectedIDs.append(item.getLayerID()) item.setSelected(False) self.networkManager.setSelectionList(selectedIDs) elif self.dragMode == "Items": if self.scene.currentlyConnecting is not None: self.scene.connectCurrentConnection(scenePos) elif isinstance(self.pressedItem, ConnectorItem): for item in self.scene.selectedItems(): item.setSelected(False) self.networkManager.clearSelection() for connection in self.pressedItem.getConnections(): connection.setSelected(True) elif self.clearSelection: self.networkManager.clearSelection() self.clearSelection = False elif self.removeFromSelection and self.removeFromSelectionItemID is not None: self.networkManager.removeLayerFromSelection(self.removeFromSelectionItemID) self.removeFromSelection = False self.removeFromSelectionItemID = None elif self.setSelectionToItem is not None: self.networkManager.setSelection(self.scene.itemAt(self.setSelectionToItem, QTransform()).getLayerID()) # Unblock the drag mode self.dragModeBlocked = False # Change the cursor type according to the bool value if self.lastHoveredItem is None: if self.controlHold: self.view.setCursor(QCursor(Qt.CrossCursor)) self.dragMode = "Selection" else: self.view.setCursor(QCursor(Qt.OpenHandCursor)) self.dragMode = "Drag" else: self.view.setCursor(QCursor(Qt.ArrowCursor)) self.dragMode = "Items" def keyPressEvent(self, QKeyEvent): if QKeyEvent.key() == Qt.Key_Control: self.networkManager.setControlHold(True) elif QKeyEvent.key() == Qt.Key_Shift: self.networkManager.setShiftHold(True) else: if QKeyEvent.key() == Qt.Key_Up: self.translateItems(0, - 5 * self.view.getScale()) if QKeyEvent.key() == Qt.Key_Down: self.translateItems(0, 5 * self.view.getScale()) if QKeyEvent.key() == Qt.Key_Right: self.translateItems(5 * self.view.getScale(), 0) if QKeyEvent.key() == Qt.Key_Left: self.translateItems(- 5 * self.view.getScale(), 0) if QKeyEvent.key() == Qt.Key_C: if self.controlHold: self.copySelection = [] for item in self.scene.selectedItems(): # If getLayerId gives an error, do not copy the element try: self.copySelection.append(item.getLayerID()) except AttributeError: None if QKeyEvent.key() == Qt.Key_V and not self.disabled: if len(self.copySelection) > 0: self.networkManager.duplicateLayers(self.copySelection) def keyReleaseEvent(self, QKeyEvent): if QKeyEvent.key() == Qt.Key_Control: self.networkManager.setControlHold(False) elif QKeyEvent.key() == Qt.Key_Shift: self.networkManager.setShiftHold(False) elif QKeyEvent.key() == Qt.Key_Delete and not self.disabled: self.networkManager.deleteSelectedLayers() def setNetworkManager(self, networkManager): self.networkManager = networkManager def setControlHold(self, bool): self.controlHold = bool # Only change state when user is not currently selecting anything if not self.dragModeBlocked: # Change the cursor type according to the bool value if self.lastHoveredItem is None: if self.controlHold: self.view.setCursor(QCursor(Qt.CrossCursor)) self.dragMode = "Selection" else: self.view.setCursor(QCursor(Qt.OpenHandCursor)) self.dragMode = "Drag" else: self.view.setCursor(QCursor(Qt.ArrowCursor)) self.dragMode = "Items" def translateItems(self, deltaX, deltaY): # Moving should only be enabled for node items if self.scene.selectedItems() and isinstance(self.scene.selectedItems()[0], NodeItem): # Prevent multiple layers from changing relative positions when the scenes border is reached selectedItems = self.scene.selectedItems() if len(selectedItems) > 0: selectedItemsBoundingRect = selectedItems[0].sceneBoundingRect() for i in range(1, len(selectedItems)): selectedItemsBoundingRect = selectedItemsBoundingRect.united(selectedItems[i].sceneBoundingRect()) # Fix for wrong scene bounding rects right and bottom selectedItemsBoundingRect.setWidth(selectedItemsBoundingRect.width() + 3) selectedItemsBoundingRect.setHeight(selectedItemsBoundingRect.height() + 3) newBoundingRect = QRectF(selectedItemsBoundingRect) newBoundingRect.moveTo(selectedItemsBoundingRect.topLeft() + QPointF(deltaX, deltaY)) sceneRect = self.scene.sceneRect() moved = False if newBoundingRect.left() < sceneRect.left(): deltaX = sceneRect.left() - selectedItemsBoundingRect.left() moved = True elif newBoundingRect.right() > sceneRect.right(): deltaX = sceneRect.right() - selectedItemsBoundingRect.right() moved = True if newBoundingRect.top() < sceneRect.top(): deltaY = sceneRect.top() - selectedItemsBoundingRect.top() moved = True elif newBoundingRect.bottom() > sceneRect.bottom(): deltaY = sceneRect.bottom() - selectedItemsBoundingRect.bottom() moved = True for item in self.scene.selectedItems(): item.setPos(item.scenePos() + QPointF(deltaX, deltaY)) # Check if any of the objects have reached the boundary and the view should be translated if len(selectedItems) > 0: selectedItemsBoundingRect = selectedItems[0].sceneBoundingRect() for i in range(1, len(selectedItems)): selectedItemsBoundingRect = selectedItemsBoundingRect.united( selectedItems[i].sceneBoundingRect()) newRect = self.view.mapFromScene(selectedItemsBoundingRect).boundingRect() if newRect.left() < 0 or newRect.right() > self.view.viewport().width() or newRect.top() < 0 or newRect.bottom() > self.view.viewport().height(): self.view.translate(-deltaX, -deltaY) moved = True return moved def setCopySelection(self): self.copySelection = [] for item in self.scene.selectedItems(): self.copySelection.append(item.getLayerID()) def setDisabled(self, disabled): self.disabled = disabled gui/main_window/node_editor/node_editor_scene.py0000664ڊ00111170000002473613243274037022436 0ustar klemmso0cvprfrom PyQt5.QtCore import QLineF from PyQt5.QtGui import QPen, QColor, QCursor, QTransform from PyQt5.QtWidgets import QGraphicsScene, QMessageBox from gui.main_window.node_editor.items.connection_item import ConnectionItem from gui.main_window.node_editor.items.connector_item import ConnectorItem from gui.main_window.node_editor.items.node_item import NodeItem from gui.main_window.node_editor.node_editor_constants import Constants class NodeEditorScene(QGraphicsScene): def __init__(self, view, initialWidth, initialHeight, nodeEditor): super(NodeEditorScene, self).__init__(-(initialWidth / 2), -(initialHeight / 2), initialWidth, initialHeight, view) self.__view = view self.__nodeEditor = nodeEditor self.currentlyConnecting = None self.disabled = False # Create the event handler for this scene self.eventHandler = None def setEventHandler(self, eventHandler): self.eventHandler = eventHandler def getNodeEditor(self): return self.__nodeEditor def getView(self): return self.__view def startConnection(self, connector, scenePosition): """ Creates a new connection item, connected to the connector """ self.currentlyConnecting = ConnectionItem(self.__nodeEditor) self.currentlyConnecting.setConnector(connector) self.currentlyConnecting.updateMousePosition(scenePosition) self.addItem(self.currentlyConnecting) def connectCurrentConnection(self, position): """ Try to connect the currently connecting connection. If no valid connector was found, the connection disappears """ # get the item at the position itemAt = self.itemAt(position.toPoint(), self.getView().transform()) # remove the connection (a new connection will get added if there is a valid connector) self.removeItem(self.currentlyConnecting) connection = self.currentlyConnecting self.currentlyConnecting = None """ if itemAt is a Connector (Top/Bottom) item (if you pull onto a Blob) """ if itemAt is not None and isinstance(itemAt, ConnectorItem) and not self.disabled: # check, whether the connection is already connected to connector of the given type (top/bottom) if connection.checkSameConnectorTypeRestriction(itemAt): # get the connectors if itemAt.isTopConnector(): topConnector = itemAt bottomConnector = connection.getConnectorIfNotFullyConnected() else: topConnector = connection.getConnectorIfNotFullyConnected() bottomConnector = itemAt # get data needed to notify the underling data structure topLayerID = topConnector.getNodeItem().getLayerID() bottomLayerID = bottomConnector.getNodeItem().getLayerID() topBlobIndex = topConnector.getIndex() bottomBlobIndex = bottomConnector.getIndex() # notify to change the data self.__nodeEditor.tryToConnect(topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex) """ if itemAt is a Node Item (if you pull onto a layer) """ if itemAt is not None and isinstance(itemAt, NodeItem) and not self.disabled: # test if connector starts at a top Blob if connection.getConnectorIfNotFullyConnected().isTopConnector(): # bottomNode is itemAt bottomNode = itemAt # get layer IDs topLayerID = connection.getConnectorIfNotFullyConnected().getNodeItem().getLayerID() bottomLayerID = bottomNode.getLayerID() topBlobIndex = connection.getConnectorIfNotFullyConnected().getIndex() # get the Index of the new Blob, should it be necessary to create one # (determined in the following for loop) bottomBlobIndex = bottomNode.getBottomConnectorCount() # current connection top name and phase topBlobName = connection.getConnectorIfNotFullyConnected().getBlobName() topBlobPhase = connection.getConnectorIfNotFullyConnected().getPhase() # check if there is a connected Node that has a different phase than the currently # connecting Node, but has a connection with the same top Blob Name topBlobFound = False for bottomBlob in bottomNode.getBottomConnectors(): if len(bottomBlob.getConnectedNodes()) > 0: for topNode in bottomBlob.getConnectedNodes(): for topBlob in topNode.getTopConnectors(): if topBlob.getBlobName() == topBlobName and topBlob.getPhase() != topBlobPhase: bottomBlobIndex = bottomBlob.getIndex() topBlobFound = True break # otherwise (if no corresponding top Blob was found) # get Index of first empty bottom blob (if available) counter = -1 emptyBlobAvailable = False if not topBlobFound: for blob in bottomNode.getBottomConnectors(): counter += 1 if len(blob.getConnectedNodes()) == 0: bottomBlobIndex = counter emptyBlobAvailable = True break # add empty bottom blob property if not emptyBlobAvailable and not topBlobFound: self.__nodeEditor.tryToAddBottomBlob(bottomLayerID) # connect nodes connected = self.__nodeEditor.tryToConnect(topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex) # if the connection did not work but a new blob was created, remove it if not connected and not emptyBlobAvailable and not topBlobFound: bottomNode.removeBottomConnector(bottomBlobIndex) def dragEnterEvent(self, event): """ Verifies that the data dragged into the widget is correct """ mimeData = event.mimeData().text() if not self.__nodeEditor.getNetworkManager().isValidLayerType(mimeData): event.ignore() def dragMoveEvent(self, event): """ Empty reimplementation """ def dropEvent(self, event): mimeData = event.mimeData().text() if self.__nodeEditor.getNetworkManager().isValidLayerType(mimeData): # When calculating the replace option, use QCursor because pos() of event is (0, 0) for some reason item = self.itemAt(self.__view.mapToScene(self.__view.mapFromGlobal(QCursor.pos())), QTransform()) if isinstance(item, NodeItem): if QMessageBox.warning(self.__view, "Replace Layer", "Would you like to replace " + item.getName() + " with the new layer?", QMessageBox.Yes, QMessageBox.No) == QMessageBox.No: item = None # Create the new node item self.__nodeEditor.tryToCreateLayer(mimeData, event.scenePos().x(), event.scenePos().y()) # Check if the user wants to replace an old layer with this one if item != None: # Get the new layer id newItem = self.selectedItems()[0] # Add top connectors for i in range(0, item.getTopConnectorCount()): self.__nodeEditor.tryToAddTopBlob(newItem.getLayerID(), item.getTopConnectors()[i].getBlobName()) for connection in item.getTopConnectors()[i].getConnections(): connection.setTopConnector(newItem.getTopConnectors()[i]) item.getTopConnectors()[i].removeConnection(connection) newItem.getTopConnectors()[i].addConnection(connection) # Add bottom connectors for i in range(0, item.getBottomConnectorCount()): self.__nodeEditor.tryToAddBottomBlob(newItem.getLayerID()) for connection in item.getBottomConnectors()[i].getConnections(): connection.setBottomConnector(newItem.getBottomConnectors()[i]) item.getBottomConnectors()[i].removeConnection(connection) newItem.getBottomConnectors()[i].addConnection(connection) # Translate new item to the exact place of the old item newItem.setPos(item.x(), item.y()) self.__nodeEditor.getNetworkManager().deleteLayer(item.getLayerID()) # Set keyboard focus to the node prototxt_editor view self.__nodeEditor.setKeyboardFocus() event.accept() def drawBackground(self, painter, rect): """ Draws a large and a small grid on the scene """ super(NodeEditorScene, self).drawBackground(painter, rect) # SMALL grid # do not draw small grid if zoomed out smallGridSize = Constants.sceneGridSize if self.getView().getScale() < smallGridSize / 10: self.drawGrid(rect, smallGridSize, QColor(230, 230, 230), painter) # LARGE grid # do not draw large grid if zoomed out largeGridSize = Constants.sceneGridSize * Constants.sceneLargeGridCells if self.getView().getScale() < largeGridSize / 10: self.drawGrid(rect, largeGridSize, QColor(200, 200, 200), painter) def drawGrid(self, rect, gridSize, color, painter): """ Draw visible grid lines in the current view rect Concept of grid drawing by Bitto: http://www.qtcentre.org/threads/5609-Drawing-grids-efficiently-in-QGraphicsScene""" lines = list() startX = rect.left() - (rect.left() % gridSize) startY = rect.top() - (rect.top() % gridSize) # add vertical lines x = startX while x < rect.right(): lines.append(QLineF(x, rect.top(), x, rect.bottom())) x += gridSize # add horizontal lines y = startY while y < rect.bottom(): lines.append(QLineF(rect.left(), y, rect.right(), y)) y += gridSize # draw the lines painter.setPen(QPen(color)) painter.drawLines(lines) def disableEditing(self, disable): """ Disable automatic generation of new blobs. """ self.disabled = disable gui/main_window/node_editor/node_editor_view.py0000664ڊ00111170000000451513243274037022304 0ustar klemmso0cvprfrom PyQt5.QtWidgets import QGraphicsView from PyQt5.QtCore import Qt, QPoint, pyqtSignal from gui.main_window.node_editor.node_editor_constants import Constants from gui.main_window.node_editor.node_editor_scene import NodeEditorScene class NodeEditorView(QGraphicsView): # These signals are emitted from within the NodeEditorEventHandler when the # actions are clicked from the context menu. arrangeVerticallyClicked = pyqtSignal() arrangeHorizontallyClicked = pyqtSignal() def __init__(self, nodeEditor, parent=None): super(NodeEditorView, self).__init__(parent) Constants.setPaintDevice(self) self.__scene = NodeEditorScene(self, Constants.sceneInitialWidth, Constants.sceneInitialHeight, nodeEditor) self.setScene(self.__scene) self.setTransformationAnchor(QGraphicsView.AnchorUnderMouse) self.setViewportUpdateMode(QGraphicsView.FullViewportUpdate) self.eventHandler = None self.centerOn(0, 0) def setEventHandler(self, eventHandler): self.eventHandler = eventHandler def scaleSymmetric(self, scaleFactor): """ set scale and zooms the view, regarding the max and min zoom """ scale = self.getScale()*scaleFactor # Prevent over scale if scale > Constants.viewMinScale: factor = (1/self.getScale())*Constants.viewMinScale self.scale(factor, factor) return elif scale < Constants.viewMaxScale: factor = (1/self.getScale())*Constants.viewMaxScale self.scale(factor, factor) return # scale self.scale(scaleFactor, scaleFactor) def getScale(self): return self.transform().m11() def wheelEvent(self, event): self.eventHandler.wheelEvent(event) def mouseDoubleClickEvent(self, QMouseEvent): return def mousePressEvent(self, QMouseEvent): self.eventHandler.mousePressEvent(QMouseEvent) def mouseMoveEvent(self, QMouseEvent): self.eventHandler.mouseMoveEvent(QMouseEvent) def mouseReleaseEvent(self, QMouseEvent): self.eventHandler.mouseReleaseEvent(QMouseEvent) def keyPressEvent(self, QKeyEvent): self.eventHandler.keyPressEvent(QKeyEvent) def keyReleaseEvent(self, QKeyEvent): self.eventHandler.keyReleaseEvent(QKeyEvent) gui/main_window/node_editor/node_sort.py0000664ڊ00111170000002102213243274037020743 0ustar klemmso0cvprfrom backend.barista.utils.logger import Log from gui.main_window.node_editor.items.node_item import NodeItem # TODO: # -try avoiding intersecting connections (arrange ordered by connectors) callerId = Log.getCallerId('GraphLayouter') class NodeSort(): class Node: OFFSET = 50 CENTER_X = 0 CENTER_Y = 0 def __init__(self, guiNode, listOfConnectedNodes): self.guiNode = guiNode # contains only output nodes of type Node self.listOfConnectedNodes = listOfConnectedNodes self.height = self.guiNode.boundingRect().height() self.width = self.guiNode.boundingRect().width() self.x = -1 # mark pos not set yet self.y = -1 # mark pos not set yet self.visited = False def setPosition(self, columnWidths, rowHeights): xpos = -NodeSort.Node.CENTER_X for x in range(0, self.x): xpos += columnWidths[x]+self.OFFSET ypos = -NodeSort.Node.CENTER_Y for y in range(0, self.y): ypos += rowHeights[y]+self.OFFSET self.guiNode.setPos(xpos, ypos) # Special methods, not really needed noticed to late, # handles only if there is a connection between topand bottom def getIndexWhereTopEqualsBottom(self): ''' Needed for special cases where the top is equal to the bottom (where input is equal to output) ''' if (len(self.listOfConnectedNodes) > 0 and self in self.listOfConnectedNodes): return self.listOfConnectedNodes.index(self) return -1 def getOutputNodesWithoutSelf(self): nodes = self.listOfConnectedNodes index = self.getIndexWhereTopEqualsBottom() if(index != -1): # remove index from list return nodes[:index] + nodes[index+1:] return nodes def __init__(self, layers, view, vertical=False): self.vertical = vertical if(vertical): NodeSort.Node.OFFSET = 70 else: NodeSort.Node.OFFSET = 30 self.__hasCycle = False self.columnWidths = [] self.rowHeights = [] # all abstract nodes (type Node) that exist self.nodes = [] # all gui nodes that exist self.guiNodes = [] if(type(layers) is list): Log.error("wrong type of layers", callerId) return # fills nodes and guiNodes self.getAbstractNodes(layers.values()) if(self.__hasCycle): Log.error("There is a Cycle in that graph, cannot be handled yet", callerId) return if(not layers.values()): return self.sort(self.nodes, view) def getAbstractNodes(self, graphItems): for graphItem in graphItems: if(type(graphItem) is NodeItem): self.createNodeItem(graphItem, []) def createNodeItem(self, guiNode, visitedGuiNodes): # if node not handled yet, add it if(guiNode not in self.guiNodes): outputNodes = [] # every connector can hold more connections connectedGuiNodes = [] for guiConnectorItem in guiNode.getTopConnectors(): connectedGuiNodes += guiConnectorItem.getConnectedNodes() for guiNodeItem in connectedGuiNodes: if(guiNodeItem in visitedGuiNodes): # TODO: GuiNode.setErrorMArker() self.__hasCycle = True return newVisitedNodes = list(visitedGuiNodes) newVisitedNodes.append(guiNodeItem) if guiNodeItem not in self.guiNodes: self.createNodeItem(guiNodeItem, newVisitedNodes) if(self.__hasCycle): return outputNodes.append(self.nodes[self.guiNodes.index(guiNodeItem)]) node = NodeSort.Node(guiNode, outputNodes) self.nodes.append(node) self.guiNodes.append(guiNode) def sort(self, listOfNodes, view): firstNodes = self.getListOfFirstNodes(listOfNodes) # just reset, firstNOdes used this self.setAllUnvisited(listOfNodes) self.stepSize = 1 self.x = 0 self.y = 0 self.maxY = 0 self.maxX = 0 self.traverseConnectedNodes(firstNodes, 0) self.setCellSizes(listOfNodes) if(self.vertical): self.verticalGrid() self.setViewBounds(view) self.updateGuiNodes(listOfNodes) def getListOfFirstNodes(self, listOfNodes): firstNodes = [] for ele in listOfNodes: ele.visited = True for ele2 in listOfNodes: if(ele in ele2.listOfConnectedNodes): ele.visited = False break if(ele.visited): firstNodes.append(ele) return firstNodes def setAllUnvisited(self, listOfNodes): for ele in listOfNodes: ele.visited = False def traverseConnectedNodes(self, nodes, x): y = 0 for ele in nodes: # ele could be visited before, so just if it was not good (to low) calculated reset it if ele.x < x: ele.x = x if ele.y < y: # if ele.x changed there must be recalculation for y y = self.getCurrentYMaxInColumn(ele.x) ele.y = y # maximum for maybe needed for make it pretty if self.maxX < ele.x: self.maxX = ele.x if self.maxY < ele.y: self.maxY = ele.y self.traverseConnectedNodes(ele.listOfConnectedNodes, ele.x+self.stepSize) def getCurrentYMaxInColumn(self, x): nodesAtX = self.getNodesInColumn(self.nodes, x) if(len(nodesAtX) != 0): maxY = max(node.y for node in nodesAtX) + self.stepSize else: maxY = 0 return maxY def updateGuiNodes(self, listOfNodes): # set Position for all nodes for node in listOfNodes: node.setPosition(self.columnWidths, self.rowHeights) def getNodesInColumn(self, nodes, x): nodesInColumnX = [] for ele in nodes: if(ele.x == x): nodesInColumnX.append(ele) return nodesInColumnX def getNodesInRow(self, nodes, y): nodesInRowY = [] for ele in nodes: if(ele.y == y): nodesInRowY.append(ele) return nodesInRowY def setCellSizes(self, nodes): # Set Column Width for x in range(0, self.maxX+self.stepSize): # maxX included because its the maximum USED nodesAtX = self.getNodesInColumn(nodes, x) # get maxwidth, every node in the column should have this columnwidth maxWidth = max(node.width for node in nodesAtX) if(self.vertical): maxWidth = max(node.height for node in nodesAtX) self.columnWidths.append(maxWidth) # Set Row Height for y in range(0, self.maxY+self.stepSize): # maxY included because its the maximum USED nodesAtY = self.getNodesInRow(nodes, y) # get Maxheight, every node in the row should have this rowheight if(len(nodesAtY) != 0): maxHeight = max(node.height for node in nodesAtY) if(self.vertical): maxHeight = max(node.width for node in nodesAtY) else: maxHeight = -1 self.rowHeights.append(maxHeight) def setViewBounds(self, view): neededWidth = sum(self.columnWidths)+NodeSort.Node.OFFSET*len(self.columnWidths) neededHeight = sum(self.rowHeights)+NodeSort.Node.OFFSET*len(self.rowHeights) NodeSort.Node.CENTER_X = int(round(neededWidth/2)) NodeSort.Node.CENTER_Y = int(round(neededHeight/2)) marginScreen = -(NodeSort.Node.OFFSET/2) x = marginScreen-NodeSort.Node.CENTER_X y = marginScreen-NodeSort.Node.CENTER_Y view.fitInView(x, y, neededWidth, neededHeight, True) # doesnt scale anything, only if there is too much zoom view.scaleSymmetric(1) # Layout vertical def verticalGrid(self): for node in self.nodes: # swap x and y position tmp = node.x node.x = node.y node.y = tmp tmp = self.columnWidths self.columnWidths = self.rowHeights self.rowHeights = tmp gui/host_manager/__init__.py0000664ڊ00111170000000000013243274037016340 0ustar klemmso0cvprgui/host_manager/hardware_selector.py0000664ڊ00111170000001356513243274037020322 0ustar klemmso0cvprfrom PyQt5.QtWidgets import QDialog, QHBoxLayout,QVBoxLayout, QPushButton, QListWidget, QLabel, QListView, QScrollArea from PyQt5.QtCore import QAbstractListModel, Qt from PyQt5.QtGui import QFontDatabase from backend.networking.net_util import sendMsgToHost, buildTransaction from backend.networking.protocol import Protocol class HardwareSelector(QDialog): def __init__(self, host, port, parent = None): QDialog.__init__(self, parent) self.host = host self.port = port self._layout = QVBoxLayout(self) self.setWindowTitle("Hardware on " + self.host + ":" + str(self.port)) self.setMinimumSize(800, 600) self.hardware = None self.selectedHW = None self._model = None self._lbl_status = QLabel() self._layout.addWidget(self._lbl_status) self._layoutH = QHBoxLayout() self._layout.addLayout(self._layoutH) self._hwlist = QListView() self._layoutH.addWidget(self._hwlist) self._lbl_hw = QLabel() self._lbl_hw.setMinimumWidth(400) self._lbl_hw.setMargin(10) self._lbl_hw.setFont(QFontDatabase.systemFont(QFontDatabase.FixedFont)) #self._layoutH.addWidget(self._hwlabel) self._hwscroll = QScrollArea() self._hwscroll.setWidget(self._lbl_hw) self._layoutH.addWidget(self._hwscroll) self._layoutB = QHBoxLayout() self._layout.addLayout(self._layoutB) self._btn_select = QPushButton("Select") self._layoutB.addWidget(self._btn_select) self._btn_rescan = QPushButton("Rescan") self._layoutB.addWidget(self._btn_rescan) self._btn_cancel = QPushButton("Cancel") self._layoutB.addWidget(self._btn_cancel) self._btn_select.clicked.connect(self._select) self._btn_cancel.clicked.connect(lambda: (self._quitScan(), self.close())) self._btn_rescan.clicked.connect(self._prepareScan) self._disable() self._start() def _start(self, ret=None): if not ret: self._prepareScan() ret = self._getHardware() if ret: self.hardware = ret[0] if len(self.hardware) == 0: self._lbl_status.setText("No Hardware detected. Is Caffepath set?") elif ret[1] == 0: self._lbl_status.setText("Currently selected CPU: " + ret[0][0]["name"]) else: self._lbl_status.setText("Currently selected GPU " + str(ret[1]) + ": " + ret[0][ret[1]]["name"]) self._model = self.HardwareListModel(self.hardware) self._hwlist.setModel(self._model) self._hwlist.selectionModel().currentChanged.connect(self._onSelection) self._enable() def _enable(self): self._hwlist.setEnabled(True) self._btn_rescan.setEnabled(True) def _disable(self): self._hwlist.setEnabled(False) self._btn_select.setEnabled(False) self._btn_rescan.setEnabled(False) self._lbl_hw.setText("") if self._hwlist.selectionModel(): self._hwlist.selectionModel().currentChanged.disconnect() self._hwlist.setModel(None) def _select(self): self.selectedHW = self._hwlist.currentIndex().row() self.close() def _onSelection(self): row = self._hwlist.currentIndex().row() if row != -1: self._lbl_hw.setText(self._getText(row)) self._lbl_hw.adjustSize() self._btn_select.setEnabled(True) else: self._lbl_hw.setText("") def _getText(self, row): log = self.hardware[row]["log"] text = "" for l in log: text += l + "\n" return text def _prepareScan(self): self._disable() ct = buildTransaction(self.host, self.port) if ct: self.ct = ct msg = {"key": Protocol.SCANHARDWARE} self.ct.send(msg) self.ct.bufferReady.connect(self._processScan) self.ct.socketClosed.connect(self._socketClosed) self._lbl_status.setText("Start scanning...") else: self._lbl_status.setText("Failed to start Scan.") def _processScan(self): msg = self.ct.asyncRead() if msg["status"]: if "finished" in msg.keys(): if msg["finished"]: self._quitScan() self._lbl_status.setText("Scan finished.") self._start([msg["hardware"], msg["current"]]) else: if "id" in msg.keys(): self._lbl_status.setText("Found GPU " + str(msg["id"]) + ": " + msg["name"]) else: self._lbl_status.setText("Found CPU: " + msg["name"]) else: self._lbl_status.setText("Scan failed.") def _socketClosed(self): self._quitScan() self._enable() self._lbl_status.setText("Connection lost!") def _quitScan(self): if hasattr(self, "ct"): self.ct.bufferReady.disconnect() self.ct.socketClosed.disconnect() self.ct.close() del self.ct def _getHardware(self): msg = {"key": Protocol.GETHARDWARE} ret = sendMsgToHost(self.host, self.port, msg) if ret and "hardware" in ret: return [ret["hardware"], ret["current"]] class HardwareListModel(QAbstractListModel): def __init__(self, data): QAbstractListModel.__init__(self) self.data = data def rowCount(self, parent=None, *args, **kwargs): return len(self.data) def data(self, index, role=None): row = index.row() if role == Qt.DisplayRole: if row==0: return "CPU: " + self.data[row]["name"] return "GPU " + str(self.data[row]["id"]) + ": " + self.data[row]["name"] gui/host_manager/host_manager.py0000664ڊ00111170000010254513243274037017271 0ustar klemmso0cvprimport uuid from PyQt5 import QtWidgets from PyQt5 import QtCore from PyQt5.QtCore import Qt, pyqtSlot as Slot, QMetaObject, QTimer, QObject from PyQt5.QtGui import QIcon from PyQt5.QtWidgets import * from backend.barista.utils.settings import applicationQSetting from backend.networking.net_util import sendMsgToHost, buildTransaction from backend.networking.protocol import Protocol from gui.gui_util import askFromList from gui.host_manager.remote_file_dialog import RemoteFileDialog from gui.host_manager.hardware_selector import HardwareSelector class HostManager(QWidget): def __init__(self, parent=None): QWidget.__init__(self,parent) # remember the parent for future use self.parent = parent self.dict = None # self.logid = Log.getCallerId("InputManager") # main_window settings self.setWindowRole("QDialog") self.setWindowModality(Qt.ApplicationModal) self.setWindowFlags(self.windowFlags() & QtCore.Qt.CustomizeWindowHint) self.setWindowFlags(self.windowFlags() | Qt.Window | Qt.WindowMaximizeButtonHint | Qt.WindowCloseButtonHint) self.setWindowFlags(self.windowFlags() & ~QtCore.Qt.WindowMinimizeButtonHint) self.setWindowTitle("Host Manager") self._layout = QVBoxLayout(self) self._buttonlayout = QHBoxLayout() self._layout.addLayout(self._buttonlayout) # add "new host" button self._pb_add = QPushButton("Add new Host") self._buttonlayout.addWidget(self._pb_add) # delete button self._pb_remove = QPushButton("Remove selected Host") self._pb_remove.setEnabled(False) self._buttonlayout.addWidget(self._pb_remove) # listwidget with all dbs self._hostscroll = QListWidget() self._hostscroll.setSelectionMode(QAbstractItemView.ExtendedSelection) self._hostscroll.setVerticalScrollMode(QAbstractItemView.ScrollPerPixel) self._layout.addWidget(self._hostscroll) self.dict = {"hostorder": [], "hostlist": {}} self.resize(800, 600) self._pb_add.clicked.connect(self._addNewHost) self._pb_remove.clicked.connect(self.onDelete) self._hostscroll.itemSelectionChanged.connect(self._selectionChange) self._loadFromSettings() self._updateListWidget() def _loadFromSettings(self): settings = applicationQSetting() settings.beginGroup("Host") val = settings.value("dict", "") settings.endGroup() if val != "": self.dict = self._fromDict(val) def _saveToSettings(self): settings = applicationQSetting() settings.beginGroup("Host") settings.setValue("dict", self._toDict()) settings.endGroup() def _toDict(self): """ Converts the dict of Hosts into a serializable dictionary """ serialdict = {} serialdict["hostlist"] = {} for id in self.dict["hostorder"]: host = self.dict["hostlist"][id] serialdict["hostlist"][id] = {"host": host.host, "port": host.port, "name": host.name} serialdict["hostorder"] = self.dict["hostorder"] return serialdict def _fromDict(self, dict): """ Converts the serializable dictionary into the dict of Hosts """ hostdict = {} hostdict["hostlist"] ={} if "hostorder" in dict.keys(): hostdict["hostorder"] = dict["hostorder"] for id in dict["hostorder"]: host = Host(self, dict["hostlist"][id]["host"], dict["hostlist"][id]["port"], dict["hostlist"][id]["name"]) host.setID(id) hostdict["hostlist"][id] = host return hostdict def _selectionChange(self): """disable and enable the delete button on selection""" if len(self._hostscroll.selectedItems()) <= 0: self._pb_remove.setEnabled(False) else: self._pb_remove.setEnabled(True) def getActiveHostList(self): ret = [] for id in self.dict["hostorder"]: if self.dict["hostlist"][id].connect(): host = [id, "Remote Host: '" + self.dict["hostlist"][id].name + "'"] ret.append(host) return ret def getHostDataById(self, id): if id in self.dict["hostorder"]: return [self.dict["hostlist"][id].host, self.dict["hostlist"][id].port] def makePathRelative(self, id, path): if id in self.dict["hostorder"]: return self.makePathRealtiveDirect(self.dict["hostlist"][id].host, self.dict["hostlist"][id].port, path) def makePathRealtiveDirect(self, host, port, path): msg = {"key": Protocol.MAKEPATHRELATIVE, "path": path} ret = sendMsgToHost(host, port, msg) if ret: return ret["path"] def onDelete(self): """remove selected hosts""" ic = len(self._hostscroll.selectedItems()) if ic > 0: gramNum = "" if ic > 1: gramNum = "s" # confirm delete ret = QMessageBox.question(self, "Remove {0} selected Host{1}".format(str(ic), gramNum), "Do you really want to delete the selected host{}?".format(gramNum), QMessageBox.Ok, QMessageBox.No) if ret == QMessageBox.Ok: for item in self._hostscroll.selectedItems(): self.deletebyID(item.data(Qt.UserRole)) # instead of using the _updateListWidget function it is more efficient to remove the widgets by hand rows = [] # get all the rows for item in self._hostscroll.selectedItems(): rows.append(self._hostscroll.row(item)) # remove in reverse order to conserve the row-number for row in reversed(sorted(rows)): item = self._hostscroll.takeItem(row) item = None self._saveToSettings() self._updateSizeHint() def onKeyDelete(self): self.onDelete() def keyReleaseEvent(self, QKeyEvent): if QKeyEvent.key() == QtCore.Qt.Key_Delete: self.onKeyDelete() super(HostManager, self).keyReleaseEvent(QKeyEvent) def deletebyID(self, id): """given an ID remove the entry from the host dict""" if id: if id in self.dict["hostorder"]: self.dict["hostorder"].remove(id) # this removes it from the order list self.dict["hostlist"][id].onDelete() del self.dict["hostlist"][id]# this removes it from the dict def setName(self, id, name): self.dict["hostlist"][id].name = name self._saveToSettings() self._updateSizeHint() def _addNewHost(self): mask = HostSelect(self) mask.exec_() ret = mask.returnvalue if ret["valid"]: host = Host(self, ret["host"], ret["port"], ret["host"]) id = str(uuid.uuid4()) host.setID(id) self.dict["hostorder"].append(id) self.dict["hostlist"][id] = host self._saveToSettings() self._updateListWidget() def _updateListWidget(self): self._hostscroll.clear() for id in self.dict["hostorder"]: host = self.dict["hostlist"][id] widget = HostWidget(self) widget.setHost(host) host.widget = widget widget.updateState() item = QListWidgetItem() item.setSizeHint(widget.sizeHint()) item.setData(Qt.UserRole, id) self._hostscroll.addItem(item) self._hostscroll.setItemWidget(item, widget) self._updateSizeHint() @Slot() def _updateSizeHint(self): """update the size of a widget inside the listwidget""" for index in range(0, self._hostscroll.count()): item = self._hostscroll.item(index) widget = self._hostscroll.itemWidget(item) item.setSizeHint(widget.sizeHint()) self._hostscroll.update() def moveUp(self, id): """move the item one position up""" if id: i = self.dict["hostorder"].index(id) if i > 0: # you can move the item ... tmp = self.dict["hostorder"][i - 1] self.dict["hostorder"][i - 1] = self.dict["hostorder"][i] self.dict["hostorder"][i] = tmp # ... but you can't move the widget. you need to create the widget anew host = self.dict["hostlist"][id] widget = HostWidget(self) host.widget = widget widget.setHost(host) widget.updateState() scroll = self._hostscroll.cursor() row = self._getIndexinList(id) item = self._hostscroll.takeItem(row) item.setSizeHint(widget.sizeHint()) self._hostscroll.insertItem(row - 1, item) self._hostscroll.setItemWidget(item, widget) self._hostscroll.setCursor(scroll) def moveDown(self, id): """move the item one position down""" if id: i = self.dict["hostorder"].index(id) if i < len(self.dict["hostorder"]) - 1: # you can move the item... tmp = self.dict["hostorder"][i + 1] self.dict["hostorder"][i + 1] = self.dict["hostorder"][i] self.dict["hostorder"][i] = tmp # ... but you can't move the widget. you need to create the widget anew host = self.dict["hostlist"][id] widget = HostWidget(self) host.widget = widget widget.setHost(host) widget.updateState() scroll = self._hostscroll.cursor() row = self._getIndexinList(id) item = self._hostscroll.takeItem(row) item.setSizeHint(widget.sizeHint()) self._hostscroll.insertItem(row + 1, item) self._hostscroll.setItemWidget(item, widget) self._hostscroll.setCursor(scroll) def _getIndexinList(self, id): """given an id what is the index of the item in the listwidget""" for index in range(0, self._hostscroll.count()): item = self._hostscroll.item(index) if item.data(Qt.UserRole) == id: return index return -1 def _showLineDialog(self, title, text, default=""): """display a simple dialog asking for a line input""" # create a dialog dialog = QDialog(self) dialog.setWindowTitle(title) # layouts and widgets layout = QVBoxLayout(dialog) label = QLabel(text) layout.addWidget(label) line = QLineEdit(default) layout.addWidget(line) button = QPushButton("Ok") layout.addWidget(button) button.clicked.connect(lambda: dialog.close()) # exec and get the line content dialog.exec_() return line.text() class HostSelect(QDialog): def __init__(self, parent): self.hostlist = parent.dict["hostlist"] QDialog.__init__(self, parent) self.setWindowTitle("Add new Host") self.layout = QVBoxLayout(self) self.lbl_host = QLabel("Specifiy IP address or public hostname:") self.layout.addWidget(self.lbl_host) self.le_host = QLineEdit() self.layout.addWidget(self.le_host) self.lbl_port = QLabel("Specify port:") self.layout.addWidget(self.lbl_port) self.sb_port = QSpinBox() self.sb_port.setMinimum(1) self.sb_port.setSingleStep(1) self.sb_port.setMaximum(65535) self.sb_port.setButtonSymbols(QAbstractSpinBox.NoButtons) self.layout.addWidget(self.sb_port) self.sublayout = QHBoxLayout() self.layout.addLayout(self.sublayout) self.pb_confirm = QPushButton("OK") self.sublayout.addWidget(self.pb_confirm) self.pb_cancel = QPushButton("Cancel") self.sublayout.addWidget(self.pb_cancel) self.returnvalue = {"host": "", "port": 0, "valid": False} self.pb_confirm.clicked.connect(self.checkInput) self.pb_cancel.clicked.connect(self.close) def checkInput(self): if self.le_host.text() == "": QMessageBox.critical(self, "Invalid", "Host can not be empty!") return for id in self.hostlist.keys(): if self.hostlist[id].host == self.le_host.text() and \ self.hostlist[id].port == self.sb_port.value(): QMessageBox.critical(self, "Already existing", "This host has already been added.") return self.returnvalue["host"] = self.le_host.text() self.returnvalue["port"] = self.sb_port.value() self.returnvalue["valid"] = True self.close() class HostWidget(QWidget): def __init__(self, manager): super(self.__class__, self).__init__() self.manager = manager self.host = None # self.port = None # self.name = "" # self.id = None # add layouts self.layout = QVBoxLayout(self) self.hbox1 = QHBoxLayout() self.hbox1.setAlignment(Qt.AlignLeft) self.hbox2 = QHBoxLayout() self.hbox2.setAlignment(Qt.AlignLeft) self.hbox3 = QHBoxLayout() self.hbox3.setAlignment(Qt.AlignLeft) self.layout.addLayout(self.hbox1) # add labels self.lbl_status = QLabel("state") self.lbl_status.setFixedWidth(45) self.hbox1.addWidget(self.lbl_status) # create all resources ico_rename = QIcon("resources/pencil.png") pix_reload = qApp.style().standardPixmap(QStyle.SP_BrowserReload) ico_reload = QIcon(pix_reload) pix_up = qApp.style().standardPixmap(QStyle.SP_ArrowUp) ico_up = QIcon(pix_up) pix_down = qApp.style().standardPixmap(QStyle.SP_ArrowDown) ico_down = QIcon(pix_down) #pix_path = qApp.style().standardPixmap(QStyle.SP_CommandLink) #ico_path = QIcon(pix_path) ico_path = QIcon("resources/gear.png") pix_hw = qApp.style().standardPixmap(QStyle.SP_ComputerIcon) ico_hw = QIcon(pix_hw) pix_add = qApp.style().standardPixmap(QStyle.SP_FileIcon) ico_add = QIcon(pix_add) # make all buttons self.pb_rename = QPushButton(ico_rename, "") self.pb_rename.setFixedSize(30, 30) self.pb_rename.setToolTip("Rename") self.hbox1.addWidget(self.pb_rename) self.pb_reload = QPushButton(ico_reload, "") self.pb_reload.setFixedSize(30, 30) self.pb_reload.setToolTip("Reload") self.hbox1.addWidget(self.pb_reload) self.pb_up = QPushButton(ico_up, "") self.pb_up.setFixedSize(30, 30) self.pb_up.setToolTip("Move Up") self.hbox1.addWidget(self.pb_up) self.pb_down = QPushButton(ico_down, "") self.pb_down.setFixedSize(30, 30) self.pb_down.setToolTip("Move Down") self.hbox1.addWidget(self.pb_down) self.pb_path = QPushButton(ico_path, "") self.pb_path.setFixedSize(30, 30) self.pb_path.setToolTip("Set Caffe Path") self.hbox1.addWidget(self.pb_path) self.pb_hw = QPushButton(ico_hw, "") self.pb_hw.setFixedSize(30, 30) self.pb_hw.setToolTip("Manage server hardware") self.hbox1.addWidget(self.pb_hw) self.pb_add = QPushButton(ico_add, "") self.pb_add.setFixedSize(30, 30) self.pb_add.setToolTip("Add Session from Host") self.hbox1.addWidget(self.pb_add) # rest of labels self.lbl_name = QLabel() font = self.lbl_name.font() font.setBold(True) self.lbl_name.setFont(font) self.hbox1.addWidget(self.lbl_name) self.lbl_hostconfig = QLabel() self.hbox2.addWidget(self.lbl_hostconfig) self.lbl_hosttraining = QLabel() self.hbox2.addWidget(self.lbl_hosttraining) self.layout.addLayout(self.hbox2) self.lbl_hardwaremode = QLabel() self.hbox3.addWidget(self.lbl_hardwaremode) self.lbl_hardwarename = QLabel() self.hbox3.addWidget(self.lbl_hardwarename) self.layout.addLayout(self.hbox3) self.lbl_addr = QLabel() self.layout.addWidget(self.lbl_addr) self.lbl_caffepath = QLabel() self.layout.addWidget(self.lbl_caffepath) self.lbl_sessionpath = QLabel() self.layout.addWidget(self.lbl_sessionpath) self.lbl_sessioncount = QLabel() self.layout.addWidget(self.lbl_sessioncount) self.lbl_projectsession = QLabel() self.layout.addWidget(self.lbl_projectsession) self.lbl_projectsession_running = QLabel() self.layout.addWidget(self.lbl_projectsession_running) self.lbl_projectsession_finished = QLabel() self.layout.addWidget(self.lbl_projectsession_finished) self.lbl_projectsession_waiting = QLabel() self.layout.addWidget(self.lbl_projectsession_waiting) self.lbl_projectsession_other = QLabel() self.layout.addWidget(self.lbl_projectsession_other) self.lbl_connectioncount = QLabel() self.layout.addWidget(self.lbl_connectioncount) self.collapseGui() # Number of sessions all together # caffepath # button: select caffe path self.pb_rename.clicked.connect(self._rename, Qt.QueuedConnection) self.pb_reload.clicked.connect(self._onReload, Qt.QueuedConnection) self.pb_up.clicked.connect(lambda: self.manager.moveUp(self.host.id), Qt.QueuedConnection) self.pb_down.clicked.connect(lambda: self.manager.moveDown(self.host.id), Qt.QueuedConnection) self.pb_path.clicked.connect(self._updateCaffePath) self.pb_hw.clicked.connect(self.showHardwareSelect) self.pb_add.clicked.connect(self.addSessionFromHost) # self.transaction = None # self.timer = QTimer() # self.timer.setSingleShot(True) # self.timer.setInterval(15000) # self.timer.timeout.connect(self._disconnected, Qt.QueuedConnection) def showHardwareSelect(self): self.hw = HardwareSelector(self.host.host, self.host.port, self) self.hw.exec_() if self.hw.selectedHW is not None: ret = self.host.setHardware(self.hw.selectedHW) if ret: self.updateState() if ret["status"]: return QMessageBox.critical(self, "Set Hardware", "Failed to set Hardware") def addSessionFromHost(self): if not hasattr(self.manager.parent.viewManager, "project"): return pid = self.manager.parent.viewManager.project.projectId ret = self.host.getSessionsOfProject(pid) seslist = None if ret: if ret["status"]: if len(ret["sessions"]) > 0: seslist = ret["sessions"] else: QMessageBox.information(self, "Sessions on '" + self.host.name + "'", "No sessions for this project were found.") else: QMessageBox.critical(self, "Sessions on '" + self.host.name + "'", "No connection could be established to this host!") if seslist is None: return ses = askFromList(self, seslist, "Sessions on '" + self.host.name + "'", "Please select a session to add.") if ses is None: return self.manager.parent.viewManager.sessionController.loadRemoteSession((self.host.host, self.host.port), ses) def setHost(self, host): self.host = host def _onReload(self): self.manager.parent.viewManager.sessionController.filterState() self.updateState() def updateState(self): # self._setState() self.lbl_name.setText(self.host.name) self.lbl_addr.setText("Remote address: \t" + self.host.host + ":" + str(self.host.port)) self.host.updateNetwork() def _rename(self): '''ask for a new name an set it''' name = self.manager._showLineDialog("Rename", "Enter a new name:", self.lbl_name.text()) self.lbl_name.setText(name) self.manager.setName(self.host.id, name) def _updateCaffePath(self): if self.host.host and self.host.port: rfd = RemoteFileDialog(self.host.host, self.host.port, "Select Caffe Path", dirSelect=True) rfd.exec_() path = rfd.returnvalue if path is not "": ret = self.host.setCaffePath(path) if ret and ret["status"]: return QMessageBox.critical(self, "Set Caffepath", "Could not set Caffepath.\nIs path valid?") def setState(self, alive=False): if alive: self.lbl_status.setText("alive") else: self.lbl_status.setText("dead") # set the stylesheet self.lbl_status.setStyleSheet(""" QLabel{ """ + self._statusColor(alive) + """ border-style: solid; border-width: 1px; border-color: #555555; border-radius: 5px; text-align: center; } """) def _statusColor(self, status): '''set the color for the status''' if status: return """background-color: #00EE00;color:#000000;""" return """background-color:#EE0000;color:#EEEEEE;""" def _setHardware(self, id, name): color = """background-color: #00EECC;color:#000000;""" if id is 0: self.lbl_hardwaremode.setText("CPU") color = """background-color: #FFEE00;color:#000000;""" else: self.lbl_hardwaremode.setText("GPU " + str(id-1)) self.lbl_hardwaremode.setStyleSheet(""" QLabel{ """ + color + """ border-style: solid; border-width: 1px; border-color: #555555; border-radius: 5px; text-align: center; } """) self.lbl_hardwarename.setText(name) def _setConfig(self, valid): if valid: self.lbl_hostconfig.setText("Valid config") else: self.lbl_hostconfig.setText("Invalid config") self.lbl_hostconfig.setStyleSheet(""" QLabel{ """ + self._statusColor(valid) + """ border-style: solid; border-width: 1px; border-color: #555555; border-radius: 5px; text-align: center; } """) def _setTraining(self, valid): if valid: self.lbl_hosttraining.setText("Host is free for training") else: self.lbl_hosttraining.setText("Training in progress") self.lbl_hosttraining.setStyleSheet(""" QLabel{ """ + self._statusColor(valid) + """ border-style: solid; border-width: 1px; border-color: #555555; border-radius: 5px; text-align: center; } """) def collapseGui(self): obj = [self.lbl_caffepath, self.lbl_sessionpath, self.lbl_hostconfig, self.lbl_hostconfig, self.lbl_hosttraining, self.lbl_hardwaremode, self.lbl_hardwarename, self.lbl_sessioncount, self.lbl_connectioncount] obj = obj + [self.lbl_projectsession, self.lbl_projectsession_running, self.lbl_projectsession_waiting, self.lbl_projectsession_finished, self.lbl_projectsession_other] obj = obj + [self.pb_path, self.pb_hw, self.pb_add] for o in obj: o.setHidden(True) def _expandGui(self, projectsessions=False): obj = [self.lbl_caffepath, self.lbl_sessionpath, self.lbl_hostconfig, self.lbl_hostconfig, self.lbl_hosttraining, self.lbl_hardwaremode, self.lbl_hardwarename, self.lbl_sessioncount, self.lbl_connectioncount, self.lbl_projectsession] if projectsessions: obj = obj + [self.lbl_projectsession_running, self.lbl_projectsession_waiting, self.lbl_projectsession_finished, self.lbl_projectsession_other] obj = obj + [self.pb_path, self.pb_hw, self.pb_add] for o in obj: o.setHidden(False) # @Slot() # def _updateNetwork(self): # if self._connect(): # msg = {"key": Protocol.GETSTATUS} # if hasattr(self.manager.parent, "viewManager"): # if hasattr(self.manager.parent.viewManager, "project"): # msg["projectid"] = self.manager.parent.viewManager.project.projectId # self.transaction.send(msg) # return # self.collapseGui() # self.setState() # self.timer.stop() # if self.host and self.port: # msg = {"key": Protocol.GETSTATUS} # if hasattr(self.manager.parent, "viewManager"): # if hasattr(self.manager.parent.viewManager, "project"): # msg["projectid"] = self.manager.parent.viewManager.project.projectId # # # ret = sendMsgToHost(self.host, self.port, msg) # if ret: # if "data" in ret.keys(): # self._parseData(ret["data"]) # # self.timer.start() # return # def parseData(self, data): self._expandGui("projectsessions" in data.keys()) self.setState(True) if "caffepath" in data.keys(): self.lbl_caffepath.setText("Caffepath: \t" + data["caffepath"]) if "sessionpath" in data.keys(): self.lbl_sessionpath.setText("Sessionpath: \t" + data["sessionpath"]) if "trainOnHW" in data.keys(): name = "" if "hardware" in data.keys(): if len(data["hardware"]) >= data["trainOnHW"] and len(data["hardware"]) is not 0: name = data["hardware"][data["trainOnHW"]] self._setHardware(data["trainOnHW"], name) if "sessioncount" in data.keys(): self.lbl_sessioncount.setText("Sessions on server: \t" + str(data["sessioncount"])) if "connections" in data.keys(): self.lbl_connectioncount.setText("Connections open: \t" + str(data["connections"])) if "config" in data.keys(): self._setConfig(data["config"]) if "training" in data.keys(): self._setTraining(data["training"]) if "projectsessions" in data.keys(): pdat = data["projectsessions"] self.lbl_projectsession.setText("Sessions of this project: \t" + str(pdat["count"])) self.lbl_projectsession_running.setText(" running/paused: \t" + str(pdat["running"])) self.lbl_projectsession_finished.setText(" finished: \t\t" + str(pdat["finished"])) self.lbl_projectsession_waiting.setText(" waiting: \t\t" + str(pdat["waiting"])) other = pdat["count"] - pdat["finished"] - pdat["waiting"] - pdat["running"] self.lbl_projectsession_other.setText(" other: \t\t" + str(other)) else: self.lbl_projectsession.setText("Sessions of this project: \t0") QMetaObject.invokeMethod(self.manager, "_updateSizeHint", Qt.QueuedConnection) # def _connect(self): # if self.transaction is None: # ct = buildTransaction(self.host, self.port) # if ct is not None: # self.transaction = ct # self.transaction.socketClosed.connect(self._disconnected) # self.transaction.bufferReady.connect(self._heartbeat) # self.timer.start() # return True # return self.transaction is not None # def _disconnected(self): # self.timer.stop() # # self.transaction.socketClosed.disconnect() # # self.transaction.bufferReady.disconnect() # if self.transaction.isConnected(): # self.transaction.close() # del self.transaction # self.transaction = None # self.collapseGui() # self.setState() # QMetaObject.invokeMethod(self.manager, "_updateSizeHint", Qt.QueuedConnection) # def _heartbeat(self): # ret = self.transaction.asyncRead() # if ret is not None: # if ret["key"] is Protocol.GETSTATUS: # if "data" in ret.keys(): # self.timer.stop() # self.timer.start() # self.parseData(ret["data"]) # # if hasattr(self.manager.parent, "viewManager"): # if hasattr(self.manager.parent.viewManager, "project"): # pid = self.manager.parent.viewManager.project.projectId # if "projectsessions" in ret["data"].keys(): # if "pid" in ret["data"]["projectsessions"].keys(): # if not ret["data"]["projectsessions"]["pid"] == pid: # self._updateNetwork() # else: # # broken protocol # self._disconnected() # else: # self._updateNetwork() class Host(QObject): """ This class encapsulates a host known by Barista(GUI). All communication with the actual server and corresponding data is managed here. """ def __init__(self, parent, host, port, name): super(self.__class__, self).__init__() self.manager = parent self.widget = None #type: HostWidget self.host = host self.port = port self.name = name self.id = None self.transaction = None self.timer = QTimer() self.timer.setSingleShot(True) self.timer.setInterval(15000) self.timer.timeout.connect(self.disconnected, Qt.QueuedConnection) def setID(self, id): self.id = id def setHardware(self, hid): msg = {"key": Protocol.SETHARDWARE, "hid": hid} return sendMsgToHost(self.host, self.port, msg) def getSessionsOfProject(self, pid): msg = {"key": Protocol.GETSESSIONS, "pid": pid} return sendMsgToHost(self.host, self.port, msg) def setCaffePath(self, path): msg = {"key": Protocol.SETCAFFEPATH, "path": path} return sendMsgToHost(self.host, self.port, msg) def connect(self): if self.transaction is None: ct = buildTransaction(self.host, self.port) if ct is not None: self.transaction = ct self.transaction.socketClosed.connect(self.disconnected) self.transaction.bufferReady.connect(self.heartbeat) self.timer.start() return True return self.transaction is not None def disconnected(self): self.timer.stop() # self.transaction.socketClosed.disconnect() # self.transaction.bufferReady.disconnect() if self.transaction.isConnected(): self.transaction.close() del self.transaction self.transaction = None self.widget.collapseGui() self.widget.setState() QMetaObject.invokeMethod(self.manager, "_updateSizeHint", Qt.QueuedConnection) def heartbeat(self): ret = self.transaction.asyncRead() if ret is not None: if ret["key"] is Protocol.GETSTATUS: if "data" in ret.keys(): self.timer.stop() self.timer.start() self.widget.parseData(ret["data"]) if hasattr(self.manager.parent, "viewManager"): if hasattr(self.manager.parent.viewManager, "project"): pid = self.manager.parent.viewManager.project.projectId if "projectsessions" in ret["data"].keys(): if "pid" in ret["data"]["projectsessions"].keys(): if not ret["data"]["projectsessions"]["pid"] == pid: self.updateNetwork() else: # broken protocol self.disconnected() else: self.updateNetwork() @Slot() def updateNetwork(self): if self.connect(): msg = {"key": Protocol.GETSTATUS} if hasattr(self.manager.parent, "viewManager"): if hasattr(self.manager.parent.viewManager, "project"): msg["projectid"] = self.manager.parent.viewManager.project.projectId self.transaction.send(msg) return self.widget.collapseGui() self.widget.setState() def onDelete(self): if self.transaction: self.transaction.close() self.transaction = None gui/host_manager/remote_file_dialog.py0000664ڊ00111170000002715213243274037020433 0ustar klemmso0cvprfrom os.path import normpath from PyQt5.QtCore import QAbstractTableModel, Qt, QModelIndex, pyqtSignal from PyQt5.QtGui import QColor, QIcon from PyQt5.QtWidgets import QDialog, QGridLayout, QTableView, QAbstractItemView, QComboBox, QPushButton, QLineEdit, \ QStyle, qApp, QFileIconProvider, QLabel from backend.networking.net_util import sendMsgToHost from backend.networking.protocol import Protocol class RemoteFileDialog(QDialog): def __init__(self, host, port, title="", fileFilter="All (*)", dirSelect=False, parent=None): QDialog.__init__(self, parent) self.fileFilter = fileFilter # Set the window Title if (title == ""): title = "Remote" windowtitle = title + " @ " + host + ":" + str(port) self.setWindowTitle(windowtitle) self.dirselect = dirSelect self.returnvalue = "" # Build the GUI self.layout = QGridLayout(self) self.tabview = self.KeyEventTableView() self.tabview.verticalHeader().hide() self.tabview.setSelectionMode(QAbstractItemView.SingleSelection) self.tabview.setSelectionBehavior(QAbstractItemView.SelectRows) self.tabview.setFocus() self.layout.addWidget(self.tabview, 1, 0, 1, 4) # Import Model self.model = self.RemoteFileItemModel(host, port, self.dirselect) self.tabview.setModel(self.model) # Parent Button pix_parentdir = qApp.style().standardPixmap(QStyle.SP_FileDialogToParent) ico_parentdir = QIcon(pix_parentdir) self.pb_parentdir = QPushButton(ico_parentdir, "") self.pb_parentdir.setFixedSize(35, 35) self.pb_parentdir.setFocusPolicy(Qt.NoFocus) self.layout.addWidget(self.pb_parentdir, 0, 0) # Refresh Button pix_reload = qApp.style().standardPixmap(QStyle.SP_BrowserReload) ico_reload = QIcon(pix_reload) self.pb_refresh = QPushButton(ico_reload, "") self.pb_refresh.setFixedSize(35, 35) self.pb_refresh.setFocusPolicy(Qt.NoFocus) self.layout.addWidget(self.pb_refresh, 0, 1) # Current Dir self.le_path = QLineEdit() self.le_path.setEnabled(False) self.le_path.setFocusPolicy(Qt.NoFocus) self.layout.addWidget(self.le_path, 0, 2, 1, 2) # File Filter self.cb_filter = QComboBox() self.cb_filter.addItems([fi.split()[0] for fi in fileFilter.split(";;")]) if len(fileFilter.split(";;")) == 1: self.cb_filter.setEnabled(False) self.layout.addWidget(self.cb_filter, 2, 3, 1, 1) # Status Label self.lbl_status = QLabel("") self.layout.addWidget(self.lbl_status, 2, 0, 1, 3) # Confirm Button self.pb_select = QPushButton("Select") self.pb_select.setFocusPolicy(Qt.NoFocus) self.layout.addWidget(self.pb_select, 3, 2, 1, 1) # Cancel Button self.pb_cancel = QPushButton("Cancel") self.pb_cancel.setFocusPolicy(Qt.NoFocus) self.layout.addWidget(self.pb_cancel, 3, 3, 1, 1) # Resize Tabel self.tabview.setColumnWidth(0, 400) self.tabview.hideColumn(1) self.tabview.setColumnWidth(2, 100) self.tabview.setColumnWidth(3, 200) self.tabview.setMinimumWidth(715) self.tabview.setMinimumHeight(400) self.adjustSize() # Connect Signals self.tabview.doubleClicked.connect(self._processInput) # Confirm on doubleclick self.tabview.enterKey.connect(self._processInput) # Confirm on enter/return self.tabview.backKey.connect(self._goToParentDir) # ParentDir on Backspace self.tabview.selectionModel().currentChanged.connect(self._updateLineEdit) # update current dir self.model.updateStatus.connect(self._setStatus) # update Host connection status self.cb_filter.currentIndexChanged.connect(self._updateFilter) # update FileFilter self.pb_parentdir.clicked.connect(self._goToParentDir) # Button ParentDir self.pb_refresh.clicked.connect(self._updateCurrentDir) # Button Refresh self.pb_cancel.clicked.connect(self._cancel) # Button Cancel self.pb_select.clicked.connect(self._processInputButton) # Button Confirm / Select # Fill Model with /home self.model.updateModel("/home/", self._createFilter()) self._updateLineEdit() self._updateSelection() def _processInput(self): """update path or accept selection on double click, enter/return key or button """ if (self._getCurrentItemType()): self.model.updateModel(self._getCurrentItemPath(), self._createFilter()) self._updateLineEdit() self.tabview.scrollToTop() self._updateSelection() else: self._accept() def _processInputButton(self): if self.dirselect: self.returnvalue = self.le_path.text() self.close() else: self._processInput() def _createFilter(self): """create the filter from the Qt-syntax based on the current selection""" current_filter = self.fileFilter.split(";;")[self.cb_filter.currentIndex()] current_filter = current_filter.split(" ", 1)[1] current_filter = current_filter[1:len(current_filter) - 1] current_filter = current_filter.split() return current_filter def _updateFilter(self): """update the model with the current filter""" self.model.updateModel(self.model.currentPath, self._createFilter()) self._updateSelection() def _updateLineEdit(self): """set the current dir in the line edit""" self.le_path.setText(normpath(self.model.currentPath)) def _getCurrentItemType(self): """check if selection is dir""" row = self.tabview.currentIndex().row() index = self.model.createIndex(row, 1) return self.model.data(index, Qt.UserRole) def _getCurrentItemPath(self): """get the path of the current selection""" row = self.tabview.currentIndex().row() index = self.model.createIndex(row, 1) item = self.model.itemData(index) if len(item) == 0: return "" return item[Qt.DisplayRole] def _updateSelection(self): """change the selection to the first item""" self.tabview.scrollToTop() self.tabview.setCurrentIndex(self.model.createIndex(0, 0, )) def _goToParentDir(self): """go to the parent directory""" path = self.model.currentPath + "/.." self.model.updateModel(path, self._createFilter()) self._updateLineEdit() self._updateSelection() def _updateCurrentDir(self): """refresh current directory""" self.model.updateModel(self.model.currentPath, self._createFilter()) self._updateLineEdit() self._updateSelection() def _cancel(self): """close this dialog""" self.close() def _accept(self): """close this dialog and write the returnvalue""" self.returnvalue = self._getCurrentItemPath() self.close() def _setStatus(self, status): """set the host connection status in the status label""" if status: self.lbl_status.setText("") else: self.lbl_status.setText("Host unreachable!") class KeyEventTableView(QTableView): """A small helper class to catch the Keys on the TableView""" enterKey = pyqtSignal() backKey = pyqtSignal() def __init__(self): QTableView.__init__(self) def keyPressEvent(self, keyEvent): """Detect the key presses and fire signals""" if keyEvent.key() == Qt.Key_Enter or keyEvent.key() == Qt.Key_Return: self.enterKey.emit() return if keyEvent.key() == Qt.Key_Backspace: self.backKey.emit() return super(self.__class__, self).keyPressEvent(keyEvent) class RemoteFileItemModel(QAbstractTableModel): """Model to convert the file dictionary into a table""" updateStatus = pyqtSignal(bool) def __init__(self, host, port, dirselect): QAbstractTableModel.__init__(self) self.host = host self.port = port self.dirselect = dirselect self.currentPath = "" self.fileList = list() def rowCount(self, QModelIndex_parent=None, *args, **kwargs): """return the current row count""" return len(self.fileList) def columnCount(self, QModelIndex_parent=None, *args, **kwargs): """reuturn the current column count""" return 4 def data(self, index, role=None): """return the data of the table for different roles""" row = index.row() if (row < 0 or row >= len(self.fileList)): return col = index.column() file = self.fileList[row] if role == Qt.UserRole: # use the UserRole to check if item is dir if file["isDir"]: return True return False if role == Qt.DisplayRole: # the text that is shown if col == 0: return file["name"] if col == 1: return file["path"] if col == 2: return file["fileSize"] if col == 3: return file["lastChange"] if role == Qt.DecorationRole: # the icon in the first column if col == 0: fip = QFileIconProvider() if file["isDir"]: ico = fip.icon(QFileIconProvider.Folder) # type: QIcon return ico.pixmap(20, 20) ico = fip.icon(QFileIconProvider.File) # type: QIcon return ico.pixmap(20, 20) if role == Qt.BackgroundColorRole: # make stripes if row % 2 == 0: return QColor(Qt.white) return QColor(Qt.lightGray).lighter(120) def headerData(self, p_int, orient, int_role=None): """Change the column names in the header""" if int_role == Qt.DisplayRole and orient == Qt.Horizontal: return ["Name", "Path", "FileSize", "Last Changed"][p_int] def updateModel(self, path="/home/", fileFilter=["*"]): """update the model by connecting to the host and asking for a new dir dictionary""" msg = {"key": Protocol.GETDIR, "path": path, "filter": fileFilter, "dirSelect": False} if self.dirselect: msg["dirSelect"] = True res = sendMsgToHost(self.host, self.port, msg) if res: self.currentPath = res["path"] oldlen = len(self.fileList) newlen = len(res["data"]) diff = oldlen - newlen if diff > 0: self.beginRemoveRows(QModelIndex(), newlen, oldlen - 1) self.fileList = res["data"] # has to be done here to prevent invalid index exception self.endRemoveRows() if diff < 0: self.beginInsertRows(QModelIndex(), oldlen, newlen - 1) self.fileList = res["data"] # has to be done here to prevent invalid index exception self.endInsertRows() if diff == 0: self.fileList = res["data"] self.dataChanged.emit(self.createIndex(0, 0), self.createIndex(newlen, 2)) self.updateStatus.emit(True) return self.updateStatus.emit(False) gui/input_manager/__init__.py0000664ڊ00111170000000000013243274037016522 0ustar klemmso0cvprgui/input_manager/database_object.py0000664ڊ00111170000000650713243274037020077 0ustar klemmso0cvprfrom backend.input_db.lmdb_input import * from backend.input_db.leveldb_input import * from backend.input_db.hdf5_txt_input import * class DatabaseObject: '''abstraction to make accessing databases easy and consistent across types''' def __init__(self): self._legalTypes = ["LMDB", "LEVELDB", "HDF5TXT"] self._db = None self._path = None self._dbType = None self._openPath = None self._openDBType = None def __del__(self): '''on delete: close''' if self._db is not None: self._db.close() def setDB(self, path, type): '''set the path and type of a database.''' if self.isLegalType(type): self._path = unicode(path) self._dbType = unicode(type) def open(self): '''open a database.''' if self._db is not None: self.close() if self._path is not None and self.isLegalType(self._dbType): if self._dbType == 'LMDB': self._db = LmdbInput() if self._dbType == 'LEVELDB': self._db = LeveldbInput() if self._dbType == 'HDF5TXT': self._db = Hdf5TxtInput() if self._projectPath: self._db.setProjectPath(self._projectPath) self._db.setPath(self._path) self._db.open() self._openPath = self._path self._openDBType = self._dbType def openFromPath(self, path, type): '''directly open db from path.''' if self.isLegalType(type): self._path = unicode(path) self._dbType = unicode(type) self.open() def close(self): '''close the db.''' if self._db is not None: self._db.close() self._db = None self._openPath = None self._openDBType = None def getDataCount(self): '''get the number of entries in the database.''' if self._db is not None: return self._db.getDataCount() def getDimensions(self): '''get the dimensions.''' if self._db is not None: return self._db.getDimensions() def verifyConsistency(self): '''check if all entries have the same channel number and size. This may take a lot of time since every entry has to be checked.''' if self._db is not None: return self._db.verifyConsistency() def getPath(self): '''get the path of the current open db. if no db is open get the path of the to be opened db.''' if self._db is not None: return self._openPath return self._path def getDBType(self): '''get the type of the current open db. if no db is open get the type of the to be opened db.''' if self._db is not None: return self._openDBType return self._dbType def isOpen(self): '''check if a db is open.''' if self._db is not None: return self._db.isOpen() return False def isLegalType(self, type): '''check if the passed type is legal.''' if type in self._legalTypes: return True return False def setProjectPath(self, path): '''set the project path for the HDF5TXT format''' if self._dbType == "HDF5TXT": self._projectPath = pathgui/input_manager/input_manager.py0000664ڊ00111170000016550413243274037017641 0ustar klemmso0cvprimport os import uuid import sys from PyQt5 import QtWidgets from PyQt5.QtCore import QUrl, QMetaObject, QDir from PyQt5.QtCore import pyqtSlot as Slot from PyQt5.QtGui import QDesktopServices from gui.host_manager.remote_file_dialog import * from gui.input_manager.database_object import * from gui.network_manager.layer_helper import * from backend.barista.utils.logger import * import backend.barista.utils.db_util as db_util from backend.networking.net_util import sendMsgToHost from backend.networking.protocol import Protocol from gui.prototxt_editor.editor_widget import EditorWidget from gui.gui_util import askFromList from input_manager_dialogs import CheckBoxDialog, TableEditor from operator import itemgetter class InputManager(QtWidgets.QWidget): # DB Dict # databases = {id1 = {path = "..", "name="...", type=".."}, id2 =...} # dborder = [id1,id2,id3] def __init__(self, parent=None): QtWidgets.QWidget.__init__(self, parent) # remember the parent for future use self.parent = parent self.dict = None self.logid = Log.getCallerId("InputManager") # main_window settings self.setWindowRole("QDialog") self.setWindowModality(Qt.ApplicationModal) self.setWindowFlags(self.windowFlags() & QtCore.Qt.CustomizeWindowHint) self.setWindowFlags(self.windowFlags() | Qt.Window | Qt.WindowMaximizeButtonHint | Qt.WindowCloseButtonHint) self.setWindowFlags(self.windowFlags() & ~QtCore.Qt.WindowMinimizeButtonHint) self.setWindowTitle("Input Manager") # buttons and labels self._pb_add = QtWidgets.QPushButton("Add new Database") self._pb_searchadd = QtWidgets.QPushButton("Search and add new Databases") self._pb_createtxt = QtWidgets.QPushButton("New HDF5TXT File") self._type_label = QtWidgets.QLabel("Type:") # combobox for filters self._filterCombo = QtWidgets.QComboBox() types = ["ALL", "LMDB", "LEVELDB", "HDF5TXT"] self._filterCombo.addItems(types) self._filterCombo.currentIndexChanged.connect(self.onFilter) # listwidget with all dbs self._dbscroll = self.ScrollList() self._dbscroll.setSelectionMode(QtWidgets.QAbstractItemView.ExtendedSelection) # layouts self._layout = QtWidgets.QVBoxLayout(self) self._buttonlayout = QtWidgets.QHBoxLayout() self._buttonlayout.addWidget(self._pb_add) self._buttonlayout.addWidget(self._pb_searchadd) self._buttonlayout.addWidget(self._pb_createtxt) self._buttonlayout.addStretch() self._buttonlayout.addWidget(self._type_label) self._buttonlayout.addWidget(self._filterCombo) self._layout.addLayout(self._buttonlayout) self._dbscroll.setVerticalScrollMode(QAbstractItemView.ScrollPerPixel) self._layout.addWidget(self._dbscroll) self._dbscroll.dropped.connect(self.fileDropEvent) self.dbs = [] self.resize(800, 600) self.disable = False self._pb_add.clicked.connect(self.opendb) self._pb_searchadd.clicked.connect(self.searchAddDB) self._pb_createtxt.clicked.connect(self.createTxt) def keyReleaseEvent(self, QKeyEvent): if QKeyEvent.key() == QtCore.Qt.Key_Delete: self.onKeyDelete() super(InputManager, self).keyReleaseEvent(QKeyEvent) """ creates a new db entry for a selected database. this is handled differently depending on whether or not the db source is local or remote by calling 'opendbLocal' or 'opendbRemote' @author ..., j_stru18 """ def opendb(self): list = self.parent.hostManager.getActiveHostList() # type: list if len(list) == 0: self.opendbLocal() else: list.insert(0, ["local", "Local Host"]) ret = askFromList(self, list, "Select DB Source", "You have configured remote Hosts.\nPlease select the Source of the Database.") if ret: if ret == "local": self.opendbLocal() else: self.opendbRemote(ret) """ open a file dialog and create a new db entry with the resulting absolute path if the selected file is *.hd5 or *.hdf5 a new *.txt file will be created, which contains the path of the selected database file, because caffe expects a txt file @author ..., f_prob02, j_stru18, l_hu0002 """ def opendbLocal(self): projectPath = self.parent.actions.getProjectPath() if projectPath: ret = QtWidgets.QFileDialog.getOpenFileName(self, "Select a Database", QDir.homePath(), "ALL (*.mdb CURRENT *.ldb *.txt *.h5 *.hdf5);;" + "LMDB (*.mdb);;" + "LEVELDB (CURRENT *.ldb);;" + "HDF5 (*.h5 *.hdf5);;" + "HDF5TXT (*.txt)") # ret = [PATH, TYPE] if len(ret[0]) != 0: success = self.addDBbyPathLocal(ret[0], ret[1]) if success: self.updateListWidget() """ does the same as opendbLocal but instead of the GUI File Dialog a RemoteFileDialog is used for the remote source additionally the db dict that is added contains information about the host and port of the remote source @author ..., f_prob02, j_stru18, l_hu0002 @param id the id of the remote source that was selected """ def opendbRemote(self, id): data = self.parent.hostManager.getHostDataById(id) if data: filter = "ALL (*.mdb CURRENT *.ldb *.hdf5 *.h5 *.txt);;LMDB (*.mdb);;LEVELDB (CURRENT *.ldb);;" \ "HDF5 (*.h5 *.hdf5);;HDF5TXT (*.txt)" rfd = RemoteFileDialog(data[0], data[1], "Select a Database", filter, parent=self) rfd.exec_() path = rfd.returnvalue if rfd.returnvalue is not "": if path and path != "": success = self.addDBbyPathRemote(path, data[0], data[1]) if success: # a new db was added self.updateListWidget() def fileDropEvent(self, list): """called when an item is dragged and dropped onto the Input Manager List""" for path in list: if os.path.isdir(path): self.searchAddDB(path) elif os.path.isfile(path): path, ext = os.path.splitext(path) # make sure extension is valid, .ldb files need a CURRENT file if not ext in [".mdb", ".txt", ""]: continue # make sure .mdb files are named data if ext == ".mdb" and os.path.basename(path) != "data": continue # make sure if extension is empty the filename is CURRENT if ext == "" and os.path.basename(path) != "CURRENT": continue # stick path and ext back together path = path+ext self.addDBbyPathLocal(path) self.updateListWidget() """ Adds a db to the input manager's dict. Parses for the file's type. In case of a .h5 or .hdf5 file a new HDF5TXT is created. The db's files (in case of lmdb a single one and in case of leveldb and hdf5 multiple files) are hashed and used as ID to prevent duplicates. @author j_stru18 @param path : the path to the db file @param typeString : if this method is called from openDBLocal, a typeString can be added as second parameter that implies the dbs datatype @return : whether or not the adding to the db dict was successful """ def addDBbyPathLocal(self, path, typeString=None): filename = os.path.basename(path) type = "" # parse for type if typeString is None or "ALL" in typeString: type = db_util.getType(filename) elif "LMDB" in typeString: type = 'LMDB' elif "LEVELDB" in typeString: type = "LEVELDB" elif "HDF5TXT" in typeString: type = "HDF5TXT" elif "HDF5" in typeString: type = "HDF5" newFileCreated = False if type == "HDF5": h5path = path index = filename.rfind(".", 0, len(filename)) defaultName = filename[:index] filename = self._showLineDialog("Create new HDF5TXT", "Enter a name for the HDF5TXT file:", defaultName) if filename is "": # Dialog was closed return path = os.path.join(os.path.dirname(path), filename) + ".txt" if os.path.exists(path): Log.error("File already exists: " + path + ", can't create new HDF5TXT ", self.logid) QtWidgets.QMessageBox.critical(self, "Can't create new HDF5TXT", "File already exists:\n" + path) return with open(path, 'w') as file: # add a new line to be sure file.write(h5path) type = "HDF5TXT" newFileCreated = True # trim path for folder based dbs if type == "LMDB" or type == "LEVELDB": path = os.path.dirname(path) + os.path.sep # create entry in db dict db = dict() db["name"] = type + " Database" db["path"] = path db["type"] = type addingSuccessful = self._addDB(db) if not addingSuccessful and newFileCreated: # adding the hdf5txt wasn't successful: remove newly created file os.remove(path) self.parent.networkManager.setModifiedFlag() return addingSuccessful """ Adds a db to the input manager's dict. Parses for the file's type. In case of a .h5 or .hdf5 file a new HDF5TXT is created. @author j_stru18 @param path : the path to the db file @param host : the hostname of the remote db file @param port : the host's port @return : whether or not the adding to the db dict was successful """ def addDBbyPathRemote(self, path, host, port): filename = os.path.basename(path) type = db_util.getType(filename) hdf5ID = None typeWasHDF5 = False if type == "HDF5": typeWasHDF5 = True type = "HDF5TXT" h5path = path index = filename.rfind(".", 0, len(filename)) filename = filename[:index] filename = self._showLineDialog("Create new HDF5TXT", "Enter a name for the HDF5TXT file:", filename) if filename is "": # Dialog was closed return path = os.path.join(os.path.dirname(path), filename) + ".txt" # we hash the hdf5 file: if it already exists in the input manager, we don't have to create a new HDF5TXT msg = {"key": Protocol.GETHASH, "path": h5path, "type": "HDF5"} ret = sendMsgToHost(host, port, msg) if ret and ret["status"]: hashValue = ret["hashValue"] portHash = db_util.getStringHash(str(port)) hdf5ID = str(hashValue ^ portHash) if not self.dbcontains(hdf5ID): msg = {"key": Protocol.MAKEHDF5TXT, "path": path, "hdf5": h5path} ret = sendMsgToHost(host, port, msg) if not ret and not ret["status"]: QtWidgets.QMessageBox.critical(self, "Can't create new HDF5TXT", "Creation of File failed:\n" + path) Log.error("Creation of HDF5TXT " + path + " failed", self.logid) return else: Log.error("Remote host sent no message.", self.logid) if type == "LMDB" or type == "LEVELDB": path = os.path.dirname(path) + os.path.sep # create entry in db dict db = dict() db["name"] = type + " Database" db["path"] = path db["type"] = type db["isRemote"] = True db["host"] = host db["port"] = port addingSuccessful = False if typeWasHDF5: addingSuccessful = self._addDBbyID(db, hdf5ID) else: addingSuccessful = self._addDB(db) self.parent.networkManager.setModifiedFlag() return addingSuccessful def searchAddDB(self, topdir=None): """ searches a given Folder and presents all Files to be selected or deselected to the user """ # if topdir is empty, the user has to select one, else topdir will be already given (e.g. drag and drop) if not topdir: # get Top directory and project path topdir = QtWidgets.QFileDialog.getExistingDirectory(self, "Select a Database Folder", QDir.homePath()) if topdir == "": return # user cancelled # get all files with their type that lie under the top directory fileList = [] for root, dirs, files in os.walk(topdir): for name in reversed(files): # parse for type type = "" # determine the type by analyzing file Extensions filePath, extension = os.path.splitext(os.path.join(root, name)) if extension == ".mdb": type = "LMDB" elif extension == ".txt": type = "HDF5TXT" elif extension == "": if filePath[-7:] == "CURRENT": type = "LEVELDB" # if file has a valid type add it to the fileList if len(type) > 0: if type == "LMDB" and filePath[-4:] != "data": pass # if db is from type lmdb but not named "data" else: fileList.append({"filepath": filePath+extension, "type": type}) if not fileList: QtWidgets.QMessageBox.warning( self, self.tr("Error: No database found"), self.tr("Neither \"{}\" nor any subfolders contain valid Barista databases!".format(topdir)) ) return # if there is just 1 item in the list (or the first item is a txt), skip the dialog if len(fileList) > 1 or fileList[0]["type"] == "HDF5TXT": # sort list by type fileList = sorted(fileList, key=itemgetter("type"), reverse=True) # call dialog dialog = CheckBoxDialog(fileList, topdir) if dialog.exec_(): # checks if dialog was accepted ("ok pressed") or aborted ("cancel" pressed) fileList = dialog.getFileList() else: return # if dialog was aborted, nothing needs to be imported # add files to input manager for file in fileList: if file["type"] == "LMDB" or file["type"] == "LEVELDB": file["filepath"] = os.path.dirname(file["filepath"]) + os.path.sep elif file["type"] == "HDF5TXT": file["filepath"] = file["filepath"] db = dict() db["name"] = file["type"] + " Database" if file["type"] != "HDF5TXT" else "HDF5 Database" db["path"] = file["filepath"] db["type"] = file["type"] self._addDB(db) self.updateListWidget() self.parent.networkManager.setModifiedFlag() def createTxt(self): self.editor = TableEditor() def importdb(self, dict): '''receive a new db dict, clear the old and open all''' self.dict = None self._initDict() if len(dict) > 0: k = dict.keys() # for every entry # check consistency if "dborder" in k and "dblist" in k: for dbid in dict["dborder"]: if dbid in dict["dblist"]: self._addDBbyID(dict["dblist"][dbid], dbid) self.updateListWidget() else: Log.error("Invalid Data on Import", self.logid) def deletebyID(self, id): '''given an ID remove the entry from the db dict''' if id: if self.dbcontains(id): self.dict["dborder"].remove(id) # this removes it from the order list del self.dict["dblist"][id] # this removes it from the dict self.parent.networkManager.setModifiedFlag() # notify change assert not self.dbcontains(id) def clearInputManager(self): '''clear the inputmanager on changing the project''' if self.dict is not None: self.dict = None self._initDict() self.updateListWidget() """ add a new database to the dict. only for databases WITHOUT IDs in case of a local db the relevant files are hashed and the hash is used as ID, in case of remote dbs a unique id is generated @author j_stru18 @param db a dictionary that holds the attributes of a database object @return whether or not the adding of the db was successful """ def _addDB(self, db): if "isRemote" in db and db["isRemote"]: return self._addDBRemote(db) else: return self._addDBLocal(db) """ add this new database to the dict. only for databases WITHOUT IDs generates an ID of the database by hashing the file to ensure no database is added twice @author ..., f_prob02, j_stru18, l_hu0002 @param db a dictionary that holds the attributes of a database object @return True if the adding was correct, False if the same database was already loaded or if the type, name or path was invalid """ def _addDBLocal(self, db): isValidDB = False if db["path"] is not None: dbPaths = db_util.getDBPathsByType(db["path"], db["type"]) hashValue = db_util.getMultipleHash(dbPaths) id = str(hashValue) isValidDB = self._addDBbyID(db, id) return isValidDB """ Generates an ID by hashing a remote database and the remote host's port, which is used to add the database to the input manager. WARNING: until #401 is implemented, we have to hash the port of the remote host as well, to ensure that duplicate files on different hosts can be added. This has to be changed afterwards. @author: j_stru18 @param db a dictionary that holds the attributes of a remote database object @return True if the adding was correct, False if the type, name or path was invalid @pre ensure that db is a db from remote host @pre ensure that db["path"] is absolute """ def _addDBRemote(self, db): isValidDB = False if db["path"] is not None: msg = {"key": Protocol.GETHASH, "path": db["path"], "type": db["type"]} ret = sendMsgToHost(db["host"], db["port"], msg) if ret and ret["status"]: hashValue = ret["hashValue"] portHash = db_util.getStringHash(str(db["port"])) id = str(hashValue ^ portHash) isValidDB = self._addDBbyID(db, id) else: Log.error("Remote host sent no message.", self.logid) return isValidDB """ Adds a new database to the input manager's dict for a given ID. When the DB is already loaded, the user is warned. When the db dict isn't constructed in the right way and doesn't contain the right keys and doesn't have the right type or is already imported, this method returns false. Else, if the adding was successful, it returns false. @author : j_stru18 @param db : a dict containing the information of a database file @id : the id of a database file as string @return : whether or not the adding to the db dict was successful or not """ def _addDBbyID(self, db, id): self._initDict() addingSuccessful = False if self._dbCheck(db): if not self.dbcontains(id): if "isRemote" not in db: db["isRemote"] = False self.dict["dborder"].append(id) self.dict["dblist"].update({id: db}) addingSuccessful = True else: duplicatePath = self.getPathByID(id) QtWidgets.QMessageBox.critical(self, "Duplicate Error on Import!", "The selected file was already " "imported:\n{}".format(duplicatePath)) Log.error("Duplicate error on import.", self.logid) assert self.dbcontains(id) return addingSuccessful """ returns the path of the saved db that is associated with the given ID @author j_stru18 @param id a db's unique ID @return the file path of the database that is associated with this ID """ def getPathByID(self, id): path = None self._initDict() if id in self.dict["dborder"]: path = self.dict["dblist"][id]["path"] type = self.dict["dblist"][id]["type"] if type == 'LMDB': path = os.path.join(path, "data.mdb") elif type == 'LEVELDB': path = os.path.join(path, "CURRENT") elif type == 'HDF5TXT': # in case of HDF5TXT the path already contains the filename pass return path def dbcontains(self, id): '''check if an ID is already in the dict''' self._initDict() if id in self.dict["dborder"]: return True return False def _initDict(self): '''this creates an empty dict if there is none''' if self.dict is None: self.dict = dict() self.dict["dblist"] = dict() self.dict["dborder"] = list() """ checks if a db dict is constructed in a valid way (contains keys for type, name and path and type is one of the three expected types @param db a dict containing """ def _dbCheck(self, db): validDB = True if "type" not in db.keys(): Log.error("'type' not in Database key", self.logid) validDB = False elif "name" not in db.keys(): Log.error("'name' not in Database key", self.logid) validDB = False elif "path" not in db.keys(): Log.error("'path' not in Database key", self.logid) validDB = False elif db["type"] not in ["LMDB", "LEVELDB", "HDF5TXT"]: Log.error("Database of wrong type: " + db["type"], self.logid) validDB = False return validDB def getDBDict(self): '''export the dict''' self._initDict() return self.dict def getLogID(self): '''return log id of input manager''' return self.logid def updateListWidget(self): '''remove all the widgets from the listwidget and create new. easiest way of updating the list widget''' self._dbscroll.clear() # for every db in dict for id in self.dict["dborder"]: db = self.dict["dblist"][id] # create a new widget and give it all necessary info dbwidget = self.DatabaseWidget(db["type"], self) dbwidget.setId(id) dbwidget.setPath(db["path"]) dbwidget.setName(db["name"]) dbwidget.showUsage(self.parent.networkManager.network) dbwidget.disableEditing(self.disable) if db["isRemote"]: dbwidget.setRemote(db["host"], db["port"]) dbwidget.open() # create a new listwidget item and give it all necessary info item = QtWidgets.QListWidgetItem(self._dbscroll) item.setData(Qt.UserRole, id) item.setData(255, db["type"]) item.setSizeHint(dbwidget.sizeHint()) # add the item and set the widget self._dbscroll.addItem(item) self._dbscroll.setItemWidget(item, dbwidget) self.onFilter() def onFilter(self): '''apply the filter to all items by hiding and unhiding''' type = self._filterCombo.currentText() for index in range(0, self._dbscroll.count()): item = self._dbscroll.item(index) if item.data(255) == type or type == "ALL": item.setHidden(False) else: item.setHidden(True) def onButtonDelete(self, id = None): '''delete db by id and update ListWidget''' ret = QtWidgets.QMessageBox.question(self, "Remove selected Database", "Do you really want to remove this database?", QtWidgets.QMessageBox.Ok, QtWidgets.QMessageBox.No) if ret == QtWidgets.QMessageBox.Ok: if not id is None: self.deletebyID(id) self.updateListWidget() def onKeyDelete(self): selection = self._dbscroll.selectedItems() count = len(selection) if count > 0: gramNum = "" thisThese = "this" if len(selection) > 1: gramNum = "s" thisThese = "these" ret = QtWidgets.QMessageBox.question(self, "Remove {0} selected Database{1}".format(str(count), gramNum), "Do you really want to remove {0} database{1}?".format(thisThese, gramNum), QtWidgets.QMessageBox.Ok, QtWidgets.QMessageBox.No) if ret == QtWidgets.QMessageBox.Ok: for item in selection: self.deletebyID(item.data(Qt.UserRole)) self.updateListWidget() def setName(self, id, name): '''change the name to a db''' self.dict["dblist"][id]["name"] = name self.parent.networkManager.setModifiedFlag() self._updateSizeHint() @Slot() def _updateSizeHint(self): '''update the size of a widget inside the listwidget''' for index in range(0, self._dbscroll.count()): item = self._dbscroll.item(index) widget = self._dbscroll.itemWidget(item) item.setSizeHint(widget.sizeHint()) self._dbscroll.update() def _getIndexinList(self, id): '''given an id what is the index of the item in the listwidget''' for index in range(0, self._dbscroll.count()): item = self._dbscroll.item(index) if item.data(Qt.UserRole) == id: return index return -1 def moveUp(self, id): '''move the item one position up''' if id: i = self.dict["dborder"].index(id) if i > 0: # you can move the item... tmp = self.dict["dborder"][i - 1] self.dict["dborder"][i - 1] = self.dict["dborder"][i] self.dict["dborder"][i] = tmp # ... but you can't move the widget. you need to create the widget anew dbwidget = self.DatabaseWidget(self.dict["dblist"][id]["type"], self) dbwidget.setId(id) dbwidget.setPath(self.dict["dblist"][id]["path"]) dbwidget.setName(self.dict["dblist"][id]["name"]) if self.dict["dblist"][id]["isRemote"]: dbwidget.setRemote(self.dict["dblist"][id]["host"], self.dict["dblist"][id]["port"]) dbwidget.open() scroll = self._dbscroll.cursor() row = self._getIndexinList(id) item = self._dbscroll.takeItem(row) item.setSizeHint(dbwidget.sizeHint()) self._dbscroll.insertItem(row - 1, item) self._dbscroll.setItemWidget(item, dbwidget) self._dbscroll.setCursor(scroll) self.parent.networkManager.setModifiedFlag() #update list widget self.updateListWidget() def moveDown(self, id): '''move the item one position down''' if id: i = self.dict["dborder"].index(id) if i < len(self.dict["dborder"]) - 1: # you can move the item... tmp = self.dict["dborder"][i + 1] self.dict["dborder"][i + 1] = self.dict["dborder"][i] self.dict["dborder"][i] = tmp # ... but you can't move the widget. you need to create the widget anew dbwidget = self.DatabaseWidget(self.dict["dblist"][id]["type"], self) dbwidget.setId(id) dbwidget.setPath(self.dict["dblist"][id]["path"]) dbwidget.setName(self.dict["dblist"][id]["name"]) if self.dict["dblist"][id]["isRemote"]: dbwidget.setRemote(self.dict["dblist"][id]["host"], self.dict["dblist"][id]["port"]) dbwidget.open() scroll = self._dbscroll.cursor() row = self._getIndexinList(id) item = self._dbscroll.takeItem(row) item.setSizeHint(dbwidget.sizeHint()) self._dbscroll.insertItem(row + 1, item) self._dbscroll.setItemWidget(item, dbwidget) self._dbscroll.setCursor(scroll) self.parent.networkManager.setModifiedFlag() # update list widget self.updateListWidget() def assignToLayer(self, id): '''get all layers of a the corresponding type and set the path''' # TODO: instead of setting the abs-path modify this in the future to allow: # - use relative paths for the dbs in the dict # - assign a referece to the layer to get the path at runtime from the input manager # get type an path type = self.dict["dblist"][id]["type"] path = self.dict["dblist"][id]["path"] # define corresponding layertypes layertypes = [] if type == "LMDB" or type == "LEVELDB": layertypes = ["Data"] if type == "HDF5TXT": layertypes = ["HDF5Data"] # get the layers network = self.parent.networkManager.network mylayer = LayerHelper.getIdandName(layertypes, network) if len(mylayer) == 0: # no layer of this type QtWidgets.QMessageBox.warning(self, type, "There are no Layer of this Type") return askFromListText = "Select a Layer for Session " + str(self.parent.viewManager.getProject().getActiveSID()) sel = askFromList(self, mylayer, type, askFromListText, False) if not sel: # nothin to do here return # set the path LayerHelper.setPathforType(sel, path, type, network) # update list widget self.updateListWidget() def _showLineDialog(self, title, text, default=""): '''display a simple dialog asking for a line input''' # create a dialog dialog = QtWidgets.QDialog(self) dialog.setWindowTitle(title) # layouts and widgets layout = QtWidgets.QVBoxLayout(dialog) label = QtWidgets.QLabel(text) layout.addWidget(label) line = QtWidgets.QLineEdit(default) layout.addWidget(line) button = QtWidgets.QPushButton("Ok") layout.addWidget(button) button.clicked.connect(lambda: dialog.accept()) # exec and get the line content isAccepted = dialog.exec_() if isAccepted == QDialog.Accepted: return line.text() else: return "" def disableEditing(self, disable): """ Set attribute disable """ self.disable = disable class DatabaseWidget(QtWidgets.QWidget): '''a widget to display a db in the listwidget''' def __init__(self, type, manager, parent=None): QtWidgets.QWidget.__init__(self, parent) self.db = None self.id = None self.isRemote = False self.remoteHost = None # remember the manager self.manager = manager # add layouts self.layout = QtWidgets.QVBoxLayout(self) self.hbox1 = QtWidgets.QHBoxLayout() self.hbox1.setAlignment(Qt.AlignLeft) self.hbox2 = QtWidgets.QHBoxLayout() self.hbox2.setAlignment(Qt.AlignLeft) self.hbox3 = QtWidgets.QHBoxLayout() self.hbox3.setAlignment(Qt.AlignLeft) self.hbox4 = QtWidgets.QHBoxLayout() self.hbox4.setAlignment(Qt.AlignLeft) # add labels self.lbl_remote = QtWidgets.QLabel() self.lbl_path = QtWidgets.QLabel("path") self.hbox3.addWidget(self.lbl_remote) self.hbox3.addWidget(self.lbl_path) self.layout.addLayout(self.hbox1) self.layout.addLayout(self.hbox2) self.layout.addLayout(self.hbox3) self.layout.addLayout(self.hbox4) # type label self.lbl_type = QtWidgets.QLabel(type) self.lbl_type.setObjectName("databaseType") # Overwrite the type color by appending them to the current stylesheet. self.lbl_type.setStyleSheet( self.lbl_type.styleSheet() + " QLabel { background-color: "+self._typeColor(type)+" }") self.lbl_type.setFixedWidth(100) self.hbox1.addWidget(self.lbl_type) # status label self.lbl_status = QtWidgets.QLabel("stat") self.lbl_status.setObjectName("databaseStatus") self.lbl_status.setFixedWidth(45) self.hbox1.addWidget(self.lbl_status) self._updateStatus(False) # name label self.lbl_name = QtWidgets.QLabel("name") font = self.lbl_name.font() font.setBold(True) self.lbl_name.setFont(font) self.hbox1.addWidget(self.lbl_name) # create all resources ico_rename = QIcon("resources/pencil.png") pix_reload = QtWidgets.qApp.style().standardPixmap(QtWidgets.QStyle.SP_BrowserReload) ico_reload = QIcon(pix_reload) pix_up = QtWidgets.qApp.style().standardPixmap(QtWidgets.QStyle.SP_ArrowUp) ico_up = QIcon(pix_up) pix_down = QtWidgets.qApp.style().standardPixmap(QtWidgets.QStyle.SP_ArrowDown) ico_down = QIcon(pix_down) pix_assign = QtWidgets.qApp.style().standardPixmap(QtWidgets.QStyle.SP_CommandLink) ico_assign = QIcon(pix_assign) ico_remove = QIcon('resources/trash.png') # make all buttons self.pb_rename = QtWidgets.QPushButton(ico_rename, "") self.pb_rename.setFixedSize(30, 30) self.pb_rename.setToolTip("Rename") self.hbox2.addWidget(self.pb_rename) self.pb_remove = QtWidgets.QPushButton(ico_remove, "") self.pb_remove.setFixedSize(30, 30) self.pb_remove.setToolTip("Remove") self.hbox2.addWidget(self.pb_remove) self.pb_reload = QtWidgets.QPushButton(ico_reload, "") self.pb_reload.setFixedSize(30, 30) self.pb_reload.setToolTip("Reload") self.hbox2.addWidget(self.pb_reload) self.pb_up = QtWidgets.QPushButton(ico_up, "") self.pb_up.setFixedSize(30, 30) self.pb_up.setToolTip("Move Up") self.hbox2.addWidget(self.pb_up) self.pb_down = QtWidgets.QPushButton(ico_down, "") self.pb_down.setFixedSize(30, 30) self.pb_down.setToolTip("Move Down") self.hbox2.addWidget(self.pb_down) # some special buttons for HDF5TXT if type == "HDF5TXT": pix_edit = QtWidgets.qApp.style().standardPixmap(QtWidgets.QStyle.SP_FileDialogContentsView) ico_edit = QIcon(pix_edit) self.pb_edit = QtWidgets.QPushButton(ico_edit, "") self.pb_edit.setFixedSize(30, 30) self.pb_edit.setToolTip("Edit Textfile") self.hbox2.addWidget(self.pb_edit) # self.pb_open.clicked.connect(lambda: self._openFile(), Qt.QueuedConnection) self.pb_edit.clicked.connect(lambda: (self.onEditPressed(), Qt.QueuedConnection)) pix_add = QtWidgets.qApp.style().standardPixmap(QtWidgets.QStyle.SP_FileDialogNewFolder) ico_add = QIcon(pix_add) self.pb_add = QtWidgets.QPushButton(ico_add, "") self.pb_add.setFixedSize(30, 30) self.pb_add.setToolTip("Add single Hdf5 File") self.hbox2.addWidget(self.pb_add) self.pb_add.clicked.connect(lambda: self._addHdf5(), Qt.QueuedConnection) self.pb_assign = QtWidgets.QPushButton(ico_assign, "") self.pb_assign.setFixedSize(30, 30) self.pb_assign.setToolTip("Assign to Layer") self.hbox2.addWidget(self.pb_assign) # usage label self.lbl_usage = QtWidgets.QLabel("no layer assigned") font = self.lbl_usage.font() font.setItalic(True) self.lbl_usage.setFont(font) self.hbox2.addWidget(self.lbl_usage) # number of elements in the db self.lbl_count = QtWidgets.QLabel("None") self.lbl_count.setObjectName("databaseLabel") self.hbox4.addWidget(self.lbl_count) self.lbl_count.setMinimumWidth(200) self.lbl_count.setSizePolicy(QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Minimum) self.lbl_count.setAlignment(Qt.AlignTop) # dimensions of elements in the db self.lbl_dim = QtWidgets.QLabel("None") self.lbl_dim.setObjectName("databaseLabel") self.hbox4.addWidget(self.lbl_dim) self.lbl_dim.setMinimumWidth(200) self.lbl_dim.setSizePolicy(QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Minimum) self.lbl_dim.setAlignment(Qt.AlignTop) # connect the buttons self.pb_rename.clicked.connect(self.onRename, Qt.QueuedConnection) self.pb_reload.clicked.connect(self.reload, Qt.QueuedConnection) self.pb_up.clicked.connect(lambda: self.manager.moveUp(self.id), Qt.QueuedConnection) self.pb_down.clicked.connect(lambda: self.manager.moveDown(self.id), Qt.QueuedConnection) self.pb_assign.clicked.connect(lambda: self.manager.assignToLayer(self.id), Qt.QueuedConnection) self.pb_remove.clicked.connect(lambda: self.manager.onButtonDelete(self.id), Qt.QueuedConnection) def getPath(self): '''return the path of this db''' return self.lbl_path.text() def setPath(self, path): '''set the path of this db''' self.lbl_path.setText(path) def getType(self): '''get the type of this db''' return self.lbl_type.text() def setName(self, name): '''set the name of this db''' self.lbl_name.setText(name) def setUsage(self, usage): '''set the usage of this db''' self.lbl_usage.setText(usage) def setId(self, id): '''set the id of this db''' self.id = id def setRemote(self, host, port): '''set this as remote''' self.isRemote = True self.remoteHost = [host, port] self.lbl_remote.setText(host + ":" + str(port)) def showUsage(self, network): """ Set the usage-label. """ # define corresponding layertypes layertypes = [] if self.getType() == "LMDB" or self.getType() == "LEVELDB": layertypes = ["Data"] if self.getType() == "HDF5TXT": layertypes = ["HDF5Data"] # get the layers mylayer = LayerHelper.getNamesAndDBPaths(layertypes, network) usedBy = "" for layername, layerpath in mylayer: if os.path.realpath(self.getPath()) == os.path.realpath(layerpath): usedBy += layername + ", " if usedBy is "": self.setUsage("no layer assigned") else: self.setUsage(usedBy[:-2]) """ Reload this database widget by opening it again and in case of a local HDF5TXT rehashing it. If the file was changed outside of Barista, a warning is shown. @author: j_stru18 """ def reload(self): self.open() type = self.getType() path = self.getPath() if type == "HDF5TXT" and not self.isRemote: dbpaths = db_util.getHdf5List(path) hashedID = db_util.getMultipleHash(dbpaths) if hashedID != self.id: # file was changed outside of Barista QtWidgets.QMessageBox.warning(self, "File was Changed", "This HDF5TXT file's paths were changed " "outside of Barista - check for duplicate " "files may fail.") def open(self): '''try to open this db and fill all the label based on whether or not it is local or remote''' if not self.isRemote: self.openLocal() else: self.openRemote() def openLocal(self): '''try to open this db an fill all the labels with information''' projectPath = self.manager.parent.actions.getProjectPath() if projectPath: # create a DatabaseObject and open it from path self.dbo = DatabaseObject() path = os.path.join(projectPath, self.getPath()) self.dbo.setDB(path, self.getType()) if self.getType() == "HDF5TXT": self.dbo.setProjectPath(projectPath) self.dbo.open() # if DBO is open: status = self.dbo.isOpen() self._updateStatus(status) if status: # get the element count count = self.dbo.getDataCount() self.lbl_count.setText(self._convertDatacount(count)) # get the element dim dim = self.dbo.getDimensions() self.lbl_dim.setText(self._convertDimensions(dim)) else: self.lbl_count.setText("None") self.lbl_dim.setText("None") # close the DBO self.dbo.close() # call the manager to update the size hints # use this threadsave methode to enable updates of the GUI QMetaObject.invokeMethod(self.manager, "_updateSizeHint", Qt.QueuedConnection) else: Log.error("No project set, can't build rel. paths", self.manager.getLogID()) def openRemote(self): ''' try to open this db on a remote host and fill all the label with information''' msg = {"key": Protocol.GETDBSTATUS, "type": self.getType(), "path": self.getPath()} ret = sendMsgToHost(self.remoteHost[0], self.remoteHost[1], msg) if ret and ret["status"]: # getting db status was successful self._updateStatus(True) # get the element count count = ret["dataCount"] self.lbl_count.setText(self._convertDatacount(count)) # get the element dim dim = ret["dimensions"] self.lbl_dim.setText(self._convertDimensions(dim)) else: # getting db status was not successful self._updateStatus(False) self.lbl_count.setText("None") self.lbl_dim.setText("None") QMetaObject.invokeMethod(self.manager, "_updateSizeHint", Qt.QueuedConnection) def onEditPressed(self): self.editor = TableEditor(self.getPath()) self.open() def onRename(self): '''ask for a new name and set it''' name = self.manager._showLineDialog("Rename", "Enter a new name:", self.lbl_name.text()) self.setName(name) self.manager.setName(self.id, name) def _convertDatacount(self, count): strcount = "Number of Elements:" if count: for key in sorted(count.keys()): strcount += "\n" + key + ":" strcount += "\n\t" + str(count[key]) else: strcount += "\nNone" return strcount def _convertDimensions(self, dim): strdim = "Dimensions:" if dim: for key in sorted(dim.keys()): strdim += "\n" + key + ":" l = dim[key] if len(l) == 0: strdim += "\n\tscalar" else: # reformat strdim += "\n" + str(len(l)) + ":\t" for i in range(0, len(l)): strdim += str(l[i]) if i < len(l) - 1: strdim += " x " else: strdim += "\nNone" return strdim def _updateStatus(self, status): '''update the status label''' if status: self.lbl_status.setText("alive") else: self.lbl_status.setText("dead") # Overwrite the status colors by appending them to the current stylesheet. self.lbl_status.setStyleSheet( self.lbl_status.styleSheet() + "QLabel {"+self._statusColor(status)+"}" ) def _typeColor(self, type): '''define the colors for every type as strings''' if type == "LMDB": return "#FFBF00" if type == "LEVELDB": return "#FF8000" if type == "HDF5TXT": return "#2EFEF7" return "#eeeeee" def _statusColor(self, status): '''set the color for the status''' if status: return """background-color: #00EE00;color:#000000;""" return """background-color:#EE0000;color:#EEEEEE;""" def _openFile(self): if self.isRemote: self._openFileRemote() else: self._openFileLocal() def _openFileLocal(self): '''open the file of HDF5TXT in a system prototxt_editor''' if self.getType() == "HDF5TXT": projectPath = self.manager.parent.actions.getProjectPath() if projectPath: path = os.path.join(projectPath, self.getPath()) if os.path.exists(path): url = QUrl(path, QUrl.TolerantMode) QDesktopServices.openUrl(url) else: Log.error("File not found: " + path, self.manager.getLogID()) QtWidgets.QMessageBox.critical(self, "File not Found!", "File not Found at:\n" + path) self.open() def _openFileRemote(self): msg = {"key": Protocol.GETFILECONTENT, "path": self.getPath()} ret = sendMsgToHost(self.remoteHost[0], self.remoteHost[1], msg) if ret["status"]: self.editor = EditorWidget(False, self.manager) self.editor.setText(ret["file"]) self.editor.show() self.editor.setWindowTitle(self.getPath()) self.editor.sgSave.connect(self._editorSave) self.editor.sgClose.connect(self._editorClose) return QtWidgets.QMessageBox.critical(self, "Open File", "Failed to open remote file:\n" + self.getPath()) def _editorClose(self): self.editor.sgSave.disconnect(self._editorSave) self.editor.sgClose.disconnect(self._editorClose) self.editor.deleteLater() self.editor = None def _editorSave(self, text): msg = {"key": Protocol.WRITEFILECONTENT, "path": self.getPath(), "file": text} ret = sendMsgToHost(self.remoteHost[0], self.remoteHost[1], msg) if ret["status"]: return QtWidgets.QMessageBox.critical(self, "Write File", "Failed to write remote file:\n" + self.getPath()) """ adds a new .h5 or .hdf5 file to an existing HDF5TXT file in case of a local db, the file or files are hashed and it is checked if the resulting HDF5TXT already exists @author j_stru18 """ def _addHdf5(self): if self.isRemote: self._addHdf5Remote() else: self._addHdf5Local() """ Adds a new HDF5 File to the local HDF5TXT file: After selecting one or more files , it checks whether or not there is another imported db that contains paths to the same file as the selected HDF5TXT with the newly selected paths would. If the modified file would be no duplicate, it is deleted under its old ID and added again with its new one. If it is a duplicate, the user is asked whether to keep the changes and remove the old file or discard them and modify none of them. When changes were made to either one of the files, the Widgets of the input manager are reloaded. @author j_stru18 """ def _addHdf5Local(self): '''add new HDF5 files to the HDF5TXT file''' if self.getType() == "HDF5TXT": # ask for type ret = QtWidgets.QFileDialog.getOpenFileNames(self, "Select a HDF5 Database", QDir.homePath(), "HDF5 (*.hdf5 *.h5)") path = self.getPath() newh5paths = ret[0] if len(newh5paths) is not 0: if os.path.exists(path): # there are selected files and the HDF5TXT exists h5paths = db_util.getHdf5List(path) oldHash = db_util.getMultipleHash(h5paths) newHash = oldHash + db_util.getMultipleHash(newh5paths) oldID = str(oldHash) newID = str(newHash) isFileDuplicate = self.manager.dbcontains(newID) # does a db with the same id exist? updateNeeded = False if not isFileDuplicate: with open(path, 'a+') as file: for newh5path in newh5paths: # add paths to HDF5TXT file.write("\n" + newh5path) self.manager.deletebyID(oldID) self.manager.addDBbyPathLocal(path) assert self.manager.dbcontains(newID) updateNeeded = True # db entries were changed and the inputs need to be updated else: duplicatePath = self.manager.getPathByID(newID) ret = QtWidgets.QMessageBox.question(self, "HDF5TXT with same Files Found", "Another file with the same ID was found: \n{}\n Do you want to keep the changed " "HDF5TXT and remove the old duplicate from Barista?".format(duplicatePath), QtWidgets.QMessageBox.Apply, QtWidgets.QMessageBox.Discard) if ret == QtWidgets.QMessageBox.Apply: # remove both old databases and add modified new one with open(path, 'a+') as file: for newh5path in newh5paths: # add paths to HDF5TXT file.write("\n" + newh5path) self.manager.deletebyID(newID) self.manager.deletebyID(oldID) self.manager.addDBbyPathLocal(path) assert self.manager.dbcontains(newID) updateNeeded = True # db entries were changed and the inputs need to be updated Log.log("Old HDF5TXT was removed and current file was modified.", self.manager.getLogID()) else: # discard changes Log.log("No changes to HDF5TXT files were made.", self.manager.getLogID()) if updateNeeded: self.manager.updateListWidget() else: Log.error("File not found: " + path, self.manager.getLogID()) QtWidgets.QMessageBox.critical(self, "File not Found!", "File not Found at:\n" + path) else: Log.error("No HDF5TXT file was selected.", self.manager.getLogID()) """ Adds a new HDF5 File to the local HDF5TXT file: After selecting a .h5 or .hdf5 file , it checks whether or not there is another imported db that contains paths to the same file as the selected HDF5TXT with the newly selected path would. If the modified file would be no duplicate, it is deleted under its old ID and added again with its new one. If it is a duplicate, the user is asked whether to keep the changes and remove the old file or discard them and modify none of them. When changes were made to either one of the files, the Widgets of the input manager are reloaded. WARNING: until #401 is implemented, we have to hash the port of the remote host as well, to ensure that duplicate files on different hosts can be added. This has to be changed afterwards. @author j_stru18 """ def _addHdf5Remote(self): hostName = self.remoteHost[0] port = self.remoteHost[1] rfd = RemoteFileDialog(hostName, port, "Select a HDF5 Database", "HDF5 (*.hdf5 *.h5)") rfd.exec_() newPath = rfd.returnvalue if newPath is not "": path = self.getPath() oldHashMsg = {"key": Protocol.GETHASH, "path": path, "type": "HDF5TXT"} oldHashRet = sendMsgToHost(hostName, port, oldHashMsg) if oldHashRet["status"]: oldHashValue = oldHashRet["hashValue"] portHash = db_util.getStringHash(str(port)) oldID = str(oldHashValue ^ portHash) newHashMsg = {"key": Protocol.GETHASH, "path": newPath, "type": "HDF5"} newHashRet = sendMsgToHost(hostName, port, newHashMsg) if newHashRet["status"]: updateNeeded = False newHashValue = oldHashValue + newHashRet["hashValue"] newID = str(newHashValue ^ portHash) isFileDupliate = self.manager.dbcontains(newID) # does a db with the same id exist? if not isFileDupliate: addPathMsg = {"key": Protocol.ADDHDF5, "path": path, "hdf5": newPath} addPathRet = sendMsgToHost(hostName, port, addPathMsg) if addPathRet["status"]: self.manager.deletebyID(oldID) self.manager.addDBbyPathRemote(path, hostName, port) assert self.manager.dbcontains(newID) updateNeeded = True # db entries were changed and the inputs need to be updated else: Log.error("Failed to add HDF5 Path: " + newPath, self.manager.getLogID()) else: duplicatePath = self.manager.getPathByID(newID) ret = QtWidgets.QMessageBox.question(self, "HDF5TXT with same Files Found", "Another file with the same ID was found: \n{}\n Do you want to keep the changed " "HDF5TXT and remove the old duplicate from Barista?".format(duplicatePath), QtWidgets.QMessageBox.Apply, QtWidgets.QMessageBox.Discard) if ret == QtWidgets.QMessageBox.Apply: # remove both old databases and add modified new one addPathMsg = {"key": Protocol.ADDHDF5, "path": path, "hdf5": newPath} addPathRet = sendMsgToHost(hostName, port, addPathMsg) if addPathRet["status"]: self.manager.deletebyID(oldID) self.manager.deletebyID(newID) self.manager.addDBbyPathRemote(path, hostName, port) assert self.manager.dbcontains(newID) updateNeeded = True # db entries were changed and the inputs need to be updated Log.log("Old HDF5TXT was removed and current file was modified.", self.manager.getLogID()) else: Log.error("Failed to add HDF5 Path: " + newPath, self.manager.getLogID()) else: # discard changes Log.log("No changes to HDF5TXT files were made.", self.manager.getLogID()) if updateNeeded: self.manager.updateListWidget() else: Log.error("Hashing of selected HDF5 was not successful", self.manager.getLogID()) else: Log.error("Hashing of old HDF5TXT not successful.", self.manager.getLogID()) else: Log.error("No HDF5TXT file was selected.", self.manager.getLogID()) def disableEditing(self, disable): """ Disable the button to assign a new layer """ self.pb_assign.setDisabled(disable) # class to make the Database List accept Drag and Drop inputs class ScrollList(QtWidgets.QListWidget): dropped = pyqtSignal(list) def __init__(self, parent=None): QtWidgets.QWidget.__init__(self, parent) self.setAcceptDrops(True) def dragEnterEvent(self, event): if event.mimeData().hasUrls: event.accept() else: event.ignore() def dragMoveEvent(self, event): if event.mimeData().hasUrls: event.accept() else: event.ignore() def dropEvent(self, event): if event.mimeData().hasUrls: event.setDropAction(QtCore.Qt.CopyAction) event.accept() links = [] for url in event.mimeData().urls(): links.append(str(url.toLocalFile())) self.dropped.emit(links) else: event.ignore() gui/input_manager/input_manager_dialogs.py0000664ڊ00111170000003363313243274037021340 0ustar klemmso0cvprimport os from PyQt5 import QtWidgets from PyQt5 import QtGui from PyQt5 import QtCore from gui.host_manager.remote_file_dialog import * class CheckBoxDialog(QtWidgets.QDialog): """ A Dialog presenting found files to the user as a List with Checkboxes can return a List with only selected Items in it""" def __init__(self, fileList, root, parent=None): QtWidgets.QDialog.__init__(self, parent) self.setWindowTitle("Choose Databases") self.setWindowModality(Qt.ApplicationModal) self.resize(700, 500) self.root = root # layouts self.layout = QtWidgets.QVBoxLayout(self) self.buttonLayout = QtWidgets.QHBoxLayout() # elements self.label = QtWidgets.QLabel("Choose the Databases you want to add") self.fileList = fileList self.list = QtWidgets.QListView() self.list.setSelectionMode(QtWidgets.QListView.ExtendedSelection) self.listModel = QtGui.QStandardItemModel(self) self.list.setModel(self.listModel) self.pb_ok = QtWidgets.QPushButton("Ok") self.pb_can = QtWidgets.QPushButton("Cancel") self.pb_toggle = QtWidgets.QPushButton("Toggle Selected") self.pb_selall = QtWidgets.QPushButton("Select All") self.pb_selnone = QtWidgets.QPushButton("Select None") # add content to the list if fileList: if isinstance(fileList[0], type(str())): self.fillStr() elif isinstance(fileList[0], type(dict())): self.fillAddSearch() # add elements to layouts self.buttonLayout.addWidget(self.pb_selall) self.buttonLayout.addWidget(self.pb_selnone) self.buttonLayout.addWidget(self.pb_toggle) self.buttonLayout.addStretch(40) self.buttonLayout.addWidget(self.pb_ok) self.buttonLayout.addWidget(self.pb_can) self.layout.addWidget(self.label) self.layout.addWidget(self.list) self.layout.addLayout(self.buttonLayout) # connect the buttons self.pb_selall.clicked.connect(self.selectAll) self.pb_selnone.clicked.connect(self.selectNone) self.pb_toggle.clicked.connect(self.toggleSelected) self.pb_ok.clicked.connect(self.accept) self.pb_can.clicked.connect(self.close) self.pb_ok.setDefault(True) self.pb_ok.setFocus() def selectAll(self): for index in range(self.listModel.rowCount()): item = self.listModel.item(index) item.setCheckState(Qt.Checked) def selectNone(self): for index in range(self.listModel.rowCount()): item = self.listModel.item(index) item.setCheckState(Qt.Unchecked) def toggleSelected(self): selectedRows = self.list.selectedIndexes() if len(selectedRows) > 0: firstItem = self.listModel.item(selectedRows[0].row()) checkState = Qt.Checked if firstItem.checkState() == Qt.Unchecked else Qt.Unchecked for index in selectedRows: item = self.listModel.item(index.row()) item.setCheckState(checkState) def fillAddSearch(self): for file in self.fileList: path = os.path.relpath(file["filepath"], self.root+"/..") item = QtGui.QStandardItem(path) item.setToolTip(file["filepath"]) item.setCheckable(True) item.setCheckState(Qt.Checked) item.setEditable(False) self.listModel.appendRow(item) def fillStr(self): for file in self.fileList: item = QtGui.QStandardItem(file) item.setCheckable(True) item.setCheckState(Qt.Checked) item.setEditable(False) self.listModel.appendRow(item) def getFileList(self): # delete all unchecked items from the list for index in reversed(range(self.listModel.rowCount())): item = self.listModel.item(index) if item.isCheckable() and item.checkState() == QtCore.Qt.Unchecked: del self.fileList[index] return self.fileList class TableEditor(QtWidgets.QDialog): def __init__(self, path=False, parent=None): QtWidgets.QDialog.__init__(self, parent) # get labels whether there is a given path or not if path: windowTitle = "HDF5 Textfile Editor" + " - " + os.path.basename(path) savelabel = "Save" else: windowTitle = "HDF5 Textfile Editor" savelabel = "Save as" # window settings self.setWindowTitle(windowTitle) self.setWindowModality(Qt.ApplicationModal) self.resize(500, 600) # layouts self.layout = QtWidgets.QVBoxLayout(self) self.topLayout = QtWidgets.QHBoxLayout() self.buttonLayout = QtWidgets.QHBoxLayout() # buttons self.label = QtWidgets.QLabel("Add, remove and edit lines below:") self.pb_save = QtWidgets.QPushButton(savelabel) self.pb_cancel = QtWidgets.QPushButton("Cancel") self.pb_addline = QtWidgets.QPushButton("Add Line") self.pb_rmline = QtWidgets.QPushButton("Remove Line") self.pb_addfile = QtWidgets.QPushButton("Add File(s)") # table self.table = self.CustomTable() self.table.insertColumn(0) self.table.insertColumn(1) self.table.setHorizontalHeaderLabels(["Path", "Availibility"]) self.icon_exists = QtGui.QIcon("resources/valid.png") self.icon_missing = QtGui.QIcon("resources/invalid.png") header = self.table.horizontalHeader() header.setSectionResizeMode(0, QtWidgets.QHeaderView.Stretch) header.setSectionResizeMode(1, QtWidgets.QHeaderView.ResizeToContents) # textfile self.path = path if path: try: self.txtfile = open(self.path, "r").read().splitlines() for counter, line in enumerate([line for line in self.txtfile if line]): self.table.insertRow(counter) self.table.setItem(counter, 0, QtWidgets.QTableWidgetItem(line)) self.fillCheckItem(counter, line) except IOError: return # add elements to layouts self.topLayout.addWidget(self.label) self.topLayout.addStretch() self.buttonLayout.addWidget(self.pb_addline) self.buttonLayout.addWidget(self.pb_rmline) self.buttonLayout.addWidget(self.pb_addfile) self.buttonLayout.addStretch() self.buttonLayout.addWidget(self.pb_save) self.buttonLayout.addWidget(self.pb_cancel) self.layout.addLayout(self.topLayout) self.layout.addWidget(self.table) self.layout.addLayout(self.buttonLayout) # connect the buttons and the table self.pb_save.clicked.connect(self.save) self.pb_cancel.clicked.connect(self.close) self.pb_addline.clicked.connect(self.addLine) self.pb_rmline.clicked.connect(self.removeLine) self.pb_addfile.clicked.connect(self.addFiles) self.table.cellChanged.connect(self.reload) self.table.dropped.connect(self.addFiles) self.pb_save.setDefault(True) self.pb_save.setFocus() self.exec_() def addLine(self): """ adds one row to the table """ index = self.table.rowCount() self.table.insertRow(index) self.table.setItem(index, 0, QtWidgets.QTableWidgetItem("")) self.fillCheckItem(index, "") self.reload() def addItem(self, index, path): """ adds an item in a given row to the table """ filepath, ext = os.path.splitext(path) if ext == ".h5" or ext ==".hdf5": self.table.setItem(index, 0, QtWidgets.QTableWidgetItem(path)) return True else: return False def removeLine(self): for index in reversed(self.table.selectedIndexes()): self.table.removeRow(index.row()) def addFiles(self, files=[]): """ gets called when add File button gets clicked """ if not files: fileDialog = self.CustomFileDialog() execAccepted = fileDialog.exec_() files, openClicked = fileDialog.getReturnState() if not (execAccepted or openClicked): return def append(path): nameFound = False if len(self.table.selectedIndexes()) > 0: item = self.table.selectedIndexes()[0] if len(self.table.item(item.row(), 0).text()) == 0: nameFound = True self.addItem(item.row(), path) if not nameFound: emptyFound = False for index in range(self.table.rowCount()): item = self.table.item(index, 0) if item.text() == "": emptyFound = True break if not emptyFound: index = self.table.rowCount() self.addLine() self.addItem(index, path) for path in files: if os.path.isfile(path): append(path) elif os.path.isdir(path): for root, dirs, files in os.walk(path): for name in reversed(files): filepath, extension = os.path.splitext(os.path.join(root, name)) if extension == ".h5" or extension == ".hdf5": append(filepath+extension) self.reload() def checkIfExists(self, path): return os.path.exists(path) def fillCheckItem(self, index, line): """ checks wether a path is valid or missing """ if self.checkIfExists(line): ext = os.path.splitext(line)[1] if ext == ".h5" or ext == ".hdf5": icon = self.icon_exists label = "found" else: icon = self.icon_missing label = "invalid" else: icon = self.icon_missing label = "empty" if line == "" else "missing" self.table.setItem(index, 1, QtWidgets.QTableWidgetItem(icon, label)) item = self.table.item(index, 1) item.setFlags(Qt.NoItemFlags) def reload(self, row=0, column=0): """ reloads all path avalibilities """ if column == 0: for index in range(self.table.rowCount()): item = self.table.item(index, 0) try: self.fillCheckItem(index, item.text()) except AttributeError: self.table.setItem(index, 1, QtWidgets.QTableWidgetItem(self.icon_missing, "missing")) def save(self): if self.path: txtfile = open(self.path, "w+") for index in range(self.table.rowCount()): line = self.table.item(index, 0).text() if self.checkIfExists(line): txtfile.write(line+"\n") txtfile.close() self.accept() else: path, type = QtWidgets.QFileDialog.getSaveFileName(self, "Select a new Filename", QtCore.QDir.homePath(), "HDF5TXT (*.txt)") if len(path) > 0: path, extension = os.path.splitext(path) path = path + ".txt" # ensure .txt is the suffix txtfile = open(path, "w+") for index in range(self.table.rowCount()): line = self.table.item(index, 0).text() if self.checkIfExists(line): txtfile.write(line+"\n") txtfile.close() QtWidgets.QMessageBox.information(self, self.tr("File Saved"), self.tr("File saved at \"{}\"".format(path))) def keyReleaseEvent(self, QKeyEvent): if QKeyEvent.key() == QtCore.Qt.Key_Delete: self.removeLine() class CustomTable(QtWidgets.QTableWidget): dropped = pyqtSignal(list) def __init__(self, parent=None): QtWidgets.QWidget.__init__(self, parent) self.setAcceptDrops(True) def dragEnterEvent(self, event): if event.mimeData().hasUrls: event.accept() else: event.ignore() def dragMoveEvent(self, event): if event.mimeData().hasUrls: event.accept() else: event.ignore() def dropEvent(self, event): if event.mimeData().hasUrls: event.setDropAction(QtCore.Qt.CopyAction) event.accept() links = [] for url in event.mimeData().urls(): links.append(str(url.toLocalFile())) self.dropped.emit(links) else: event.ignore() class CustomFileDialog(QtWidgets.QFileDialog): """ to get Folders OR Files""" def __init__(self, parent=None): QtWidgets.QFileDialog.__init__(self, parent) self.setWindowTitle("Select a Folder or hdf5 files") self.setFileMode(QtWidgets.QFileDialog.ExistingFiles) self.setNameFilter("HDF5 (*.h5 *.hdf5);; Directories") buttons = self.findChildren(QtWidgets.QPushButton) self.openBtn = [x for x in buttons if 'open' in str(x.text()).lower()][0] self.openBtn.clicked.disconnect() self.cancelBtn = [x for x in buttons if 'cancel' in str(x.text().lower())][0] self.cancelBtn.clicked.disconnect() # TODO: Open Button Label could be changed, LookIn however doesnt work yet # self.setLabelText(QtWidgets.QFileDialog.Accept, "Choose") # self.setLabelText(QtWidgets.QFileDialog.LookIn, "Choose") self.openBtn.clicked.connect(self.openClick) self.cancelBtn.clicked.connect(self.close) self.openClicked = 0 def openClick(self): self.openClicked = 1 self.hide() def getReturnState(self): return self.selectedFiles(), self.openClicked gui/main_window/__init__.py0000664ڊ00111170000000000013243274037016204 0ustar klemmso0cvprgui/main_window/default_actions.py0000664ڊ00111170000012274413243274037017635 0ustar klemmso0cvprimport json import os import tempfile import traceback from PyQt5 import QtWidgets, QtGui, QtCore from PyQt5.QtCore import pyqtSignal from PyQt5.QtGui import QPixmap, QIcon, QTransform from PyQt5.QtWidgets import QStyle, QMessageBox from backend.barista.constraints.session_run.training import checkMinimumTrainingRequirements from backend.barista.project import Project, dirIsProject from backend.barista.session.client_session import ClientSession from backend.barista.session.session_utils import State from backend.barista.utils.logger import Log from backend.barista.utils.settings import applicationQSetting from backend.caffe import loader from backend.caffe import saver, path_loader from gui.deployment_dialog import DeploymentDialog from gui.export_solver_dialog import ExportSolverDialog from gui.gui_util import getNewProjectDir, getOpenProjectDir class RecentlyData(QtCore.QObject): """ Model for Recently-Menu """ recentlyChanged = pyqtSignal() def __init__(self, parent=None, maxHistory=5): super(RecentlyData, self).__init__(parent) self._recently = [] self._maxHistory = maxHistory def addRecently(self, text, data): """ Add an entry. It will contains the given data but show the given text. """ # Remove if already exists (because of order) if [text, data] in self._recently: self._recently.remove([text, data]) self._recently.append([text, data]) # Delete oldest if needed if len(self._recently) >= self._maxHistory: self._recently = self._recently[-self._maxHistory:] self.recentlyChanged.emit() def setRecently(self, recentlylist): """ Set entries from a list. recentlylist should look like [[text1,data1],[text2,data2],...] """ if type(recentlylist) != list: return self._recently = recentlylist if len(self._recently) >= self._maxHistory: self._recently = self._recently[-self._maxHistory:] self.recentlyChanged.emit() def getRecently(self): """ Returns the list of recent entries in form of [[text1, data1],[text2,data2],...] """ return self._recently def getMostRecent(self): """ returns: Most recent [text, data] tuple or None (if there are no recent elements) """ try: return self._recently[-1] except IndexError: return None def recentlyFromSerialization(self, string): """ Deserialize recent entries from a json-string """ self.setRecently(json.loads(string)) def serializeRecently(self): """ Serialize recent entries into a json-string and return it""" return json.dumps(self._recently) class RecentMenu(QtWidgets.QMenu): """ QMenu showing recent files with a clear entry """ recentlySelected = pyqtSignal(object) def __init__(self, name, recentlydata, parent=None): super(RecentMenu, self).__init__(name, parent) self._data = recentlydata # type: RecentlyData self._data.recentlyChanged.connect(self.__rebuildMenu) self.__rebuildMenu() def __rebuildMenu(self): self.clear() recently = self._data.getRecently() for i in range(0, len(recently)): # Build Recently-Menu Reverse (text, data) = recently[-i - 1] validPath = os.path.exists(text) if not validPath: text = '(?) ' + text try: action = self.addAction("{}. {}".format(i, text)) except UnicodeEncodeError: action = self.addAction("{}. (?)".format(i)) action.setEnabled(False) else: if not validPath: action.setEnabled(False) action.setData(data) action.triggered.connect(self.__recentlyFound) self.addSeparator() clearAction = self.addAction(self.tr("Clear")) clearAction.triggered.connect(lambda: self._data.setRecently([])) def __recentlyFound(self): sender = self.sender() data = sender.data() self.recentlySelected.emit(data) class DefaultActions(QtCore.QObject): """ DefaultActions implements function which are used by the user. It can expose this function by building menus and toolbars. """ projectChanged = pyqtSignal(object) parentChanged = pyqtSignal(object) netManagerChanged = pyqtSignal(object) viewManagerChanged = pyqtSignal(object) quitTriggered = pyqtSignal() def __init__(self, parentWdg=None): """ Initialize DefaultActions. parentWdg will be used for dialogs to block interaction to it if wanted. """ super(DefaultActions, self).__init__() self._parent = parentWdg self._netManager = None self._viewManager = None self._scene = None self._editor = None self._editMode = None self._project = None self._pathEditor = None self._inputManager = None self._hostManager = None self._solverDialog = None self._deploymentDialog = None # Build all actions self.openNetAction = self._buildAction(self.tr("&Import Network"), None, lambda: self.openNetDialog()) self.openSolverAction = self._buildAction(self.tr("Import Solver"), None, lambda: self.openSolverDialog()) self.saveNetAction = self._buildAction(self.tr("&Export Network"), None, lambda: self.saveNetDialog()) self.saveSolverAction = self._buildAction("&Export Solver", None, lambda: self.saveSolverDialog()) self.exitAction = self._buildAction(self.tr("&Exit"), "Ctrl+Q", self.quitTriggered, None, self.tr("Close Application")) self.undoAction = self._buildAction(self.tr("Undo"), "Ctrl+Z", lambda: self._netManager.undo(), standardPixmap=QStyle.SP_ArrowLeft) self.redoAction = self._buildAction(self.tr("Redo"), "Shift+Ctrl+Z", lambda: self._netManager.redo(), standardPixmap=QStyle.SP_ArrowRight) self.editNetAction = self._buildAction(self.tr("Edit &Network as Prototxt"), "Ctrl+E", self.openNetEditor) self.editSolverAction = self._buildAction(self.tr("Edit &Solver as Prototxt"), "Shift+Ctrl+E", self.openSolverEditor) self.viewInputManager = self._buildAction(self.tr("&Input Manager"), "Ctrl+I", self.openInputManager, icon="resources/input_manager.png") self.viewHostManager = self._buildAction(self.tr("&Host Manager"), "Ctrl+H", self.openHostManager, icon="resources/host_manager.png") self.loadDefaultViewAction = self._buildAction(self.tr("&Load Default View"), None, self._loadDefaultView) self.sortSceneAction = self._buildAction(self.tr("Arrange Layers Horizontally"), None, self._sortScene) self.sortSceneActionVertical = self._buildAction(self.tr("Arrange Layers Vertically"), None, self._sortSceneVertical) self.newProjectAction = self._buildAction(self.tr("New Project"), "Ctrl+N", lambda: self.newProjectDialog(), standardPixmap=QStyle.SP_FileIcon) self.loadProjectAction = self._buildAction(self.tr("Load Project"), "Ctrl+O", lambda: self.openProjectDialog(), standardPixmap=QStyle.SP_DialogOpenButton) self.saveProjectAction = self._buildAction(self.tr("Save Project"), "Ctrl+S", lambda: self.saveProject(), standardPixmap=QStyle.SP_DialogSaveButton) self.changeCaffePathAction = self._buildAction(self.tr("Change Caffe Path"), None, lambda: self.changeCaffePath()) self.deployAction = self._buildAction(self.tr("Deploy and export"), None, lambda: self.openDeploymentDialog()) # Build menus self.dockMenu = QtWidgets.QMenu("Docks", self._parent) self.toolBarMenu = QtWidgets.QMenu("Toolbars", self._parent) # Recently-Menu self.recentNetData = RecentlyData() self.recentSolverData = RecentlyData() self.recentProjectsData = RecentlyData() self._loadActionSettings() self.recentNetData.recentlyChanged.connect(self._saveActionSettings) self.recentSolverData.recentlyChanged.connect(self._saveActionSettings) self.recentProjectsData.recentlyChanged.connect(self._saveActionSettings) self.recentNetMenu = None self.recentSolverMenu = None # Actions for the active session self.startSessionAction = self._buildAction("Start Session", "F5", lambda: self._viewManager.getSessionsDock().onStart(), icon=QIcon(QPixmap('resources/start.png'))) self.pauseSessionAction = self._buildAction("Pause Session", "F6", lambda: self._viewManager.getSessionsDock().onPause(), icon=QIcon(QPixmap('resources/pause.png'))) self.snapshotAction = self._buildAction("Create Snapshot", "F8", lambda: self._viewManager.getSessionsDock().onSnap(), icon=QIcon(QPixmap('resources/snap.png'))) def projectHelper(p): if self._pathEditor and p: self._pathEditor.updateProject(p) self.projectChanged.connect(projectHelper) def setParent(self, parent): """ Sets the parent widget. It is primary used to block interaction when opening a dialog """ if parent == self._parent: return # oldp = self._parent self._parent = parent self.parentChanged.emit(parent) # Recent-Data def _loadActionSettings(self): settings = applicationQSetting() settings.beginGroup("Actions") self.recentNetData.recentlyFromSerialization(settings.value("recent_nets", "[]")) self.recentSolverData.recentlyFromSerialization(settings.value("recent_solvers", "[]")) self.recentProjectsData.recentlyFromSerialization(settings.value("recent_projects", "[]")) settings.endGroup() def _saveActionSettings(self): settings = applicationQSetting() settings.beginGroup("Actions") settings.setValue("recent_nets", self.recentNetData.serializeRecently()) settings.setValue("recent_solvers", self.recentSolverData.serializeRecently()) settings.setValue("recent_projects", self.recentProjectsData.serializeRecently()) settings.endGroup() # Recent-Gui-Menu def buildRecentNetMenu(self, name="Recent Nets", parent=None): return self.__buildRecentMenu(name, self.recentNetData, parent, self.openNet) def buildRecentSolverMenu(self, name="Recent Solvers", parent=None): return self.__buildRecentMenu(name, self.recentSolverData, parent, self.openSolver) def buildRecentProjectMenu(self, name="Recent Projects", parent=None, callback=None): if callback is None: callback = lambda directory: self.setProjectAskUserForSave(directory, parent) return self.__buildRecentMenu(name, self.recentProjectsData, parent, callback) def __buildRecentMenu(self, name, data, parent, action): if parent is None: parent = self._parent res = RecentMenu(name, data, parent) res.recentlySelected.connect(action) return res # Update Action-State def updateActionState(self): """ Update Actions (setup enabled e.g.)""" self._updateUndoRedoState() self.saveProjectAction.setEnabled(not self._project is None) def _updateUndoRedoState(self): if self._netManager: self.undoAction.setEnabled(self._netManager.canUndo()) self.redoAction.setEnabled(self._netManager.canRedo()) # Build-Gui-Elements def buildMenuBar(self, parent=None): """ Build a menubar with all available actions. Set its parent to the given one. """ if parent is None: parent = self._parent bar = QtWidgets.QMenuBar(self._parent) fileMenu = bar.addMenu(self.tr("&File")) # type: QtWidgets.QMenu fileMenu.addAction(self.newProjectAction) fileMenu.addAction(self.loadProjectAction) fileMenu.addMenu(self.buildRecentProjectMenu(parent=parent)) fileMenu.addAction(self.saveProjectAction) fileMenu.addSeparator() fileMenu.addAction(self.openNetAction) fileMenu.addAction(self.openSolverAction) fileMenu.addAction(self.saveNetAction) fileMenu.addAction(self.saveSolverAction) fileMenu.addSeparator() self.recentNetMenu = self.buildRecentNetMenu(parent=parent) fileMenu.addMenu(self.recentNetMenu) self.recentSolverMenu = self.buildRecentSolverMenu(parent=parent) fileMenu.addMenu(self.recentSolverMenu) fileMenu.addSeparator() fileMenu.addAction(self.exitAction) editMenu = bar.addMenu(self.tr("&Edit")) editMenu.addAction(self.undoAction) editMenu.addAction(self.redoAction) editMenu.addSeparator() editMenu.addAction(self.sortSceneAction) editMenu.addAction(self.sortSceneActionVertical) editMenu.addSeparator() editMenu.addAction(self.viewInputManager) editMenu.addAction(self.viewHostManager) editMenu.addSeparator() editMenu.addAction(self.editNetAction) editMenu.addAction(self.editSolverAction) viewMenu = bar.addMenu(self.tr("&View")) viewMenu.addMenu(self.dockMenu) viewMenu.addMenu(self.toolBarMenu) viewMenu.addSeparator() viewMenu.addAction(self.loadDefaultViewAction) settingMenu = bar.addMenu(self.tr("&Deployment")) settingMenu.addAction(self.deployAction) settingMenu = bar.addMenu(self.tr("&Settings")) settingMenu.addAction(self.changeCaffePathAction) viewMenu.aboutToShow.connect(lambda: self.triggerViewMenu()) return bar def triggerViewMenu(self): """ Update dock visibility when ViewMenu was clicked """ self._viewManager.onVisibilityChange() def addDockMenuEntry(self, dockwdg, title=None): """ Add a new menu entry for the dockwidget dockwdg """ if title is None: title = dockwdg.windowTitle() action = self.dockMenu.addAction(title) action.setCheckable(True) action.setChecked(dockwdg.isVisible()) action.triggered.connect(lambda: self._viewManager.dockAction(title)) dockwdg.visibilityChanged.connect(lambda vis: action.setChecked(vis)) return action def addToolBarEntry(self, toolbar): """ Add a new menu entry for the visibility of the given toolbar """ action = self.toolBarMenu.addAction(toolbar.windowTitle()) action.setCheckable(True) action.setChecked(toolbar.isVisible()) action.triggered.connect(toolbar.setVisible) toolbar.visibilityChanged.connect(action.setChecked) return toolbar def buildFileToolbar(self, title="File"): """ Build toolbar with default actions for file""" toolbar = QtWidgets.QToolBar(title) toolbar.setObjectName("FileToolbar") toolbar.addAction(self.newProjectAction) toolbar.addAction(self.loadProjectAction) toolbar.addAction(self.saveProjectAction) toolbar.addAction(self.viewInputManager) toolbar.addAction(self.viewHostManager) return toolbar def buildEditToolbar(self, title="Edit"): """ Build toolbar with default actions for edit""" toolbar = QtWidgets.QToolBar(title) toolbar.setObjectName("EditToolbar") toolbar.addAction(self.undoAction) toolbar.addAction(self.redoAction) return toolbar def buildRunToolbar(self, title="Active Job"): """ Build toolbar with default actions for edit""" toolbar = QtWidgets.QToolBar(title) toolbar.setObjectName("RunToolbar") toolbar.addAction(self.startSessionAction) toolbar.addAction(self.pauseSessionAction) toolbar.addAction(self.snapshotAction) return toolbar def _buildAction(self, text, shortcut, fun, icon=None, statustip=None, standardPixmap=None): """ Build QAction with given parameter """ res = QtWidgets.QAction(text, self._parent) if standardPixmap: icon = QtWidgets.qApp.style().standardIcon(standardPixmap) if icon: if type(icon) == str: icon = QtGui.QIcon(icon) res.setIcon(icon) if statustip: res.setStatusTip(statustip) if shortcut: res.setShortcut(shortcut) if fun: res.triggered.connect(fun) return res # Project def checkRunningSessions(self): """ Check if there are running sessions and if thats the case ask the user to pause them. """ if self._project: if self._project.hasRunningSessions(): QM = QtWidgets.QMessageBox answer = QM.question(self._parent, self.tr("Pause sessions?"), self.tr( "There are running sessions. To proceed all sessions must be paused. Do you want to pause them?"), QM.Yes | QM.Cancel, QM.Cancel) if answer == QM.Yes: self._project.pauseRunningSessions() return True if answer == QM.Cancel: return False def checkSessionState(self, session, showWarnings): valid = checkMinimumTrainingRequirements(session=session, parentGui=self._viewManager.sessionController.sessionGui, reportToUser=showWarnings) if valid: session.setState(session.getState()) else: # if not session.getState() == State.INVALID: # previous = session.getState() if not session.getState() == State.NOTCONNECTED: session.setState(State.INVALID) def setProjectAskUserForSave(self, project_dir, parent=None): """ Load the project from the given dir. The user will be asked for saving the current project if there are some unsaved changes. """ try: if self.checkRunningSessions() is False: return if os.path.exists(project_dir) is False: if self.askRecreateProject(parent): os.makedirs(project_dir) else: return self.recentProjectsData.addRecently(project_dir, project_dir) project = Project(project_dir) if parent is None: parent = self._parent if self.askUserAndSave(parent): self.setProject(project) except: self.errorMsg("Error while trying to use an existing project.", "set_project") return False def setProject(self, project): """ Set the project. It will be loaded automatically in gui if networkmanager is set. """ try: if self._project == project: return # Register project as (most) recently used self.recentProjectsData.addRecently(project.projectRootDir, project.projectRootDir) self.removeProjectFromPlotter(self._project) self._project = project self._inputManager.clearInputManager() self._viewManager.setProject(self._project) self.updateActionState() self.__propagateProjectToNetworkManager() self._project.newSession.connect(self.registerNewSession) self._project.deleteSession.connect(self.disconnectSession) activeSID = self._project.getActiveSID() if activeSID: self.registerNewSession(activeSID) self.selectSessionInList(activeSID) self.projectChanged.emit(project) except: self.errorMsg("Error while trying to use an existing project.", "set_project") return False def removeProjectFromPlotter(self, project): """ Removes the given project from the plotter. """ if project: for session in project.getSessions().values(): self._viewManager.getPlotter().removeLog(str(session.getLogId()), str(session.getLogFileName(True))) def selectSessionInList(self, SID): self._viewManager.getSessionsDock().setSelectedSID(SID) def registerNewSession(self, SID): self._project.getSession(SID).stateDictChanged.connect(self.checkSessionState) def disconnectSession(self, SID): try: self._project.getSession(SID).stateDictChanged.disconnect() except TypeError: # will be raised if disconnect failed # TODO: Not sure if this is the right way to do it. This happens if a non-active session is deleted. pass def changeSession(self, newSID, oldSID=None): """Changes the active session within one project. The current State of the netManager is saved to the old session. The state of the new session is loaded to the netManager.""" self.storeSessionState(SID=oldSID, stateDict=None) self._project.setActiveSID(newSID) if self._project.getActiveSID() == newSID: self.loadSessionState(SID=newSID) else: Log.log("New Session " + str(newSID) + " could not be set. Valid SIDs are: " + ", ".join([str(id) for id in self._project.getValidSIDs()]), self._viewManager.sessionController.getCallerId()) return def storeSessionState(self, SID=None, stateDict=None): """Stores the stateDict to a session. If no SID is provided, the currently active session is used. If no dictionary is provided, the current netManager state is used.""" dict = stateDict if not dict: dict = self._netManager.getStateDictionary() _SID = SID if not _SID: _SID = self._project.getActiveSID() elif _SID not in self._project.getValidSIDs(): Log.log("Could not store state to SID ", _SID, ". Valid SIDs are:" + ", ".join([str(id) for id in self._project.getValidSIDs()]), self._viewManager.sessionController.getCallerId()) self._project.getSession(_SID).setStateDict(dict) return def loadSessionState(self, SID=None): """Loades the state from a session to the netManager. If no Session is provided, the currently active session is used.""" _SID = SID if not _SID: _SID = self._project.getActiveSID() if _SID in self._project.getValidSIDs(): if self._project.getSession(_SID).hasStateDict(): dict = self._project.getSession(_SID).state_dictionary net = None pos = None sel = None if "network" in dict: net = dict["network"] elif ("layerOrder" in dict) and ("layers" in dict): net = dict if "position" in dict: pos = dict["position"] if "selection" in dict: sel = dict["selection"] if net: self._netManager.setStateDictionary(dictionary=dict, clearHistory=True) else: Log.log("Could not load state from Session," + str(_SID) + " is this an old Project?", self._viewManager.sessionController.getCallerId()) else: # TODO: write proper Error to log print("Could not load state from session", _SID) Log.log("Could not load state from session " + str(_SID), self._viewManager.sessionController.getCallerId()) return def __propagateProjectToNetworkManager(self): """ Setup the current Project in NetworkManager """ if self._project is None: return if self._netManager is None: return state = self._project.getActiveSession().state_dictionary inputdb = self._project.getInputManagerState() transform = self._project.getViewTransform() self._inputManager.importdb(inputdb) self._netManager.setStateDictionary(state, clearHistory=True) self._netManager.resetModifiedFlag() if transform is None: return self._netManager.nodeEditor.getView().setTransform( QTransform(transform[0], transform[1], transform[2], transform[3], transform[4], transform[5])) # method called when creating a new project, either from start window # or from project menu bar def newProjectDialog(self, dir="", parent=None): """ Open a new dialog for selecting a new project. Parent is the parent for the dialogs. If None the set parent in this instance is used. Returns true if successfully selecting a project dir, false otherwise. """ try: if parent is None: parent = self._parent if not self.askUserAndSave(parent): return # directory alreay selected? if dir == "": dir = getNewProjectDir(self, parent) # directory valid? if not abort if len(dir) == 0: return if self.checkRunningSessions() is False: return False # Create project os.mkdir(dir) self.setProject(Project(dir)) self.recentProjectsData.addRecently(dir, dir) self.saveProject() return True except: self.errorMsg("Error while trying to create a new project.", "create_project") return False def openProjectDialog(self, parent=None): """ Open a new dialog for selecting an existing project. Parent is the parent for the dialog. If None the set parent in this instance is used. Returns true if successfully selecting a project, false otherwise. """ try: if parent is None: parent = self._parent if not self.askUserAndSave(parent): return False dir = getOpenProjectDir(self, parent) if len(dir) == 0: return False if self.checkRunningSessions() is False: return False self.recentProjectsData.addRecently(dir, dir) self.setProject(Project(dir)) return True except: self.errorMsg("Error while trying to open an existing project.", "open_project") return False def saveProject(self): """ Save the project in the directory it lives """ try: if self._project is None or self._netManager is None: return False dict = self._netManager.getStateDictionary() inputs = self._inputManager.getDBDict() transform = self._netManager.nodeEditor.getView().transform() self._project.saveProject(dict, inputs, transform) self._netManager.resetModifiedFlag() return True except: self.errorMsg("Error while trying to save the project.", "save_project") return False def getProjectPath(self): """returns the project path for external classes""" if self._project: return self._project.projectRootDir return None # Network Manager def saveNetDialog(self, parent=None): """ Open a save dialog and saves the network. Parent is the parent for the dialog. If None the set parent in this instance is used. Returns True if successfully selecting an place to save, False otherwise. """ try: if parent is None: parent = self._parent if self._netManager is None: return False filename, _ = QtWidgets.QFileDialog.getSaveFileName(parent, self.tr("Export Network")) if len(filename) == 0: return False self._netManager.saveNetworkToFile(filename) self.recentNetData.addRecently(filename, filename) return True except: self.errorMsg("Error while trying to export the network.", "network_export") return False def saveSolverDialog(self, parent=None): """ Open a save dialog and saves the solver. Parent is the parent for the dialog. If None the set parent in this instance is used. Returns True if successfully selecting an place to save, False otherwise. """ try: if parent is None: parent = self._parent if self._netManager is None: return False self._solverDialog = ExportSolverDialog(self._netManager.solver, self._netManager.network, parent, self) self._solverDialog.show() return True except: self.errorMsg("Error while trying to export the solver.", "solver_export") return False def openSolverDialog(self, parent=None): """ Open a open dialog and load the solver. Parent is the parent for the dialog. If None the set parent in this instance is used. Returns True if successfully selecting a file to load, False otherwise. """ if parent is None: parent = self._parent if self._netManager is None: return False filename, _ = QtWidgets.QFileDialog.getOpenFileName(parent, self.tr("Import Solver File"), "", "Prototxt (*.prototxt) ;; All Files (*)") if len(filename) == 0: return False try: self.openSolver(filename) return True except loader.ParseException as ex: QtWidgets.QMessageBox.critical(parent, self.tr("Error parsing prototxt"), str(ex)) return False def openSolver(self, filename): if not os.path.exists(filename): QtWidgets.QMessageBox.critical(None, self.tr("File does not exists"), self.tr("Cannot load {}. File does not exists.").format(filename)) return # open solver self._netManager.openSolverFromFile(filename) self.recentSolverData.addRecently(filename, filename) # check whether the loaded solver does contain a network specification, too # (we need to load the solver once again, as the redundant network specification has been removed automatically) with open(filename, 'r') as file: solverOriginalString = file.read() solverOriginalDictionary = loader.loadSolver(solverOriginalString) solverRefersToNetFile = "net" in solverOriginalDictionary solverIncludesNet = "net_param" in solverOriginalDictionary # Theoretically, the solver definition might contain both: a file reference and an inline definition # We do support this case, but as it will be very seldom, we do not create a new main_window handling both options. # Instead we just show two separate windows after each other if solverRefersToNetFile: netPath = solverOriginalDictionary["net"] # if the given path is not an absolute one.. if not os.path.isabs(netPath): # it's either relative to the caffe root directory netPathAlternative1 = None if path_loader.checkIfFileIsCaffeRoot(self._project.getCaffeRoot()): netPathAlternative1 = os.path.join(self._project.getCaffeRoot(), netPath) else: callerId = Log.getCallerId('solver_load') Log.log("CAFFE_ROOT path is not set", callerId) # or relative to the solver file (e.g. for Barista sessions) netPathAlternative2 = os.path.join(os.path.dirname(filename), netPath) if netPathAlternative1 and os.path.isfile(netPathAlternative1): netPath = netPathAlternative1 else: netPath = netPathAlternative2 self._promptForAutoNetImport(netPath) if solverIncludesNet: # get the network definition as a string netString = loader.extractNetFromSolver(solverOriginalString) # save inline definition to temporary file, so we can use the same algorithm as above tempFilePath = tempfile.NamedTemporaryFile(delete=False) tempFilePath.write(netString) tempFilePath.close() self._promptForAutoNetImport(tempFilePath.name) os.remove(tempFilePath.name) def _promptForAutoNetImport(self, netPath): """Open a message box and ask the user if he/she wants to import the net definition given in netPath. netPath contains a net definition that has been extracted from a loaded solver definition. """ # check whether the specified file does exist if os.path.isfile(netPath): # get the file content with open(netPath, 'r') as file: netPrototxt = file.read() msgBox = QMessageBox() msgBox.setWindowTitle("Barista") msgBox.setText(self.tr("Solver definition contains a network reference.")) msgBox.setInformativeText(self.tr("Do you want to import the network, too?")) msgBox.setDetailedText("File:\n{}\n\nNet definition:\n{}".format(netPath, netPrototxt)) msgBox.setStandardButtons(QMessageBox.Yes | QMessageBox.No) msgBox.setDefaultButton(QMessageBox.Yes) ret = msgBox.exec_() if ret == QMessageBox.Yes: self.openNet(netPath) else: callerId = Log.getCallerId('file_loader') Log.log("Found network reference in loaded solver definition, but the file {} does not exist.".format( netPath ), callerId) def openNet(self, filename): if not os.path.exists(filename): QtWidgets.QMessageBox.critical(None, self.tr("File does not exists"), self.tr("Cannot load {}. File does not exists.").format(filename)) return self._netManager.openNetworkFromFile(filename) self.recentNetData.addRecently(filename, filename) def openNetDialog(self, parent=None): """ Open a open dialog and load the network. Parent is the parent for the dialog. If None the set parent in this instance is used. Returns True if successfully selecting a file to load, False otherwise. """ if parent is None: parent = self._parent if self._netManager is None: return False filename, _ = QtWidgets.QFileDialog.getOpenFileName(parent, self.tr("Import Network File"), "", "Prototxt (*.prototxt) ;; All Files (*)") if len(filename) == 0: return False try: self.openNet(filename) return True except loader.ParseException as ex: QtWidgets.QMessageBox.critical(parent, self.tr("Error parsing prototxt"), str(ex)) return False def setNetworkManager(self, networkManager): """ Set the Networkmanager the actions should work with""" if networkManager == self._netManager: return if self._netManager: self._netManager.stateChanged.disconnect(self.updateActionState) self._netManager = networkManager if self._netManager: self._netManager.stateChanged.connect(self.updateActionState) self.updateActionState() self.netManagerChanged.emit(networkManager) # View Manager def setViewManager(self, viewManager): """Set the ViewManager the actions should work with""" if viewManager == self._viewManager: return self._viewManager = viewManager self.updateActionState() self.viewManagerChanged.emit(viewManager) def _loadDefaultView(self): if self._viewManager is None: return self._viewManager.loadDefaultView() def _sortScene(self): self._netManager.nodeEditor.rearrangeNodes() def _sortSceneVertical(self): self._netManager.nodeEditor.rearrangeNodesVertical() # SCENE def setScene(self, scene): self._scene = scene self.updateActionState() # EDITOR def openNetEditor(self): """ Open a Editor for editing the prototxt of the network """ if self._editor is None: return False self._editMode = "Network" self._editor.setText(saver.saveNet(self._netManager.network)) self._editor.show() return True def openSolverEditor(self): """ Open a Editor for editing the prototxt of the solver """ if self._editor is None: return False self._editMode = "Solver" self._editor.setText(saver.saveSolver(self._netManager.solver)) self._editor.show() return True def openInputManager(self): if self._inputManager is None: return False self._inputManager.updateListWidget() self._inputManager.show() return True def openHostManager(self): if self._hostManager is None: return False self._hostManager.show() return True def _editorChangedText(self, netproto): try: netproto = netproto.encode("utf-8") if self._netManager: if self._editMode == "Network": self._netManager.openNetworkFromString(netproto, False) if self._editMode == "Solver": self._netManager.openSolverFromString(netproto) except loader.ParseException as ex: QtWidgets.QMessageBox.critical(self._parent, "Error parsing prototxt", str(ex)) def setEditor(self, editor): """Set the Editor the actions should work with""" if self._editor: self._editor.sgSave.disconnect(self._editorChangedText) self._editor = editor if self._editor: self._editor.sgSave.connect(self._editorChangedText) self.updateActionState() def setInputManager(self, manager): """sets the input Manager""" self._inputManager = manager self.updateActionState() def setHostManager(self, manager): """sets the host Manager""" self._hostManager = manager self.updateActionState() def changeCaffePath(self): if self._pathEditor is None: return False self._pathEditor.show() def openDeploymentDialog(self): """Show a file selection main_window and export a deployment network to the selected location.""" self._deploymentDialog = DeploymentDialog(self._project.getSessions(), self._parent) self._deploymentDialog.show() return True def setPathEditor(self, editor): self._pathEditor = editor self.updateActionState() def askUserAndSave(self, parent=None): """ Ask the user for saving if something changes since last save. If the user want to save, the project get saved. If the user want to abort, return False else return True """ if parent is None: parent = self._parent if not self._netManager or not self._netManager.isModified(): return True QM = QtWidgets.QMessageBox answer = QM.question(parent, self.tr("Are you sure?"), self.tr("Do you want to save the changes you made?"), QM.Yes | QM.No | QM.Abort, QM.Abort) if answer == QM.Abort: return False if answer == QM.Yes: self.saveProject() return True def askRecreateProject(self, parent=None): """ Ask the user if he wants to create a new project in the directory. """ QM = QtWidgets.QMessageBox answer = QM.question(parent, self.tr("Are you sure?"), self.tr( "The directory does not exists. Do you want to create a new project in this directory?"), QM.Yes | QM.Abort, QM.Abort) if answer == QM.Yes: self.saveProject() return True if answer == QM.Abort: return False return False def errorMsg(self, errorMsg, loggerIdString=None, addStacktrace=True): """Show an error message in the Logger as well as in an additional GUI popup.""" # use the logger if loggerIdString is not None: callerId = Log.getCallerId(loggerIdString) else: callerId = None Log.log(errorMsg, callerId) # show message in the GUI msgBox = QMessageBox() msgBox.setWindowTitle("Barista - Error") msgBox.setText(self.tr(errorMsg)) if addStacktrace: stacktrace = traceback.format_exc() msgBox.setDetailedText(stacktrace) msgBox.setStandardButtons(QMessageBox.Ok) _ = msgBox.exec_() def disableEditing(self, disable): """ Disable actions for loading a new net or solver. """ self.openNetAction.setDisabled(disable) self.openSolverAction.setDisabled(disable) if self.recentNetMenu: self.recentNetMenu.setDisabled(disable) if self.recentSolverMenu: self.recentSolverMenu.setDisabled(disable) gui/main_window/main_window.py0000664ڊ00111170000002153513243274037017000 0ustar klemmso0cvpr# To change this license header, choose License Headers in Project Properties. # To change this template file, choose Tools | Templates # and open the template in the prototxt_editor. from PyQt5 import QtWidgets, QtCore from PyQt5.QtGui import QIcon from PyQt5.QtCore import pyqtSignal import gui.main_window.default_actions import gui.main_window.status_bar from backend.barista.session.session_utils import State from backend.caffe import dict_helper from gui import path_editor_dialog from gui.input_manager import input_manager from gui.host_manager import host_manager from gui.main_window import view_manager from gui.main_window.node_editor.node_editor import NodeEditor from gui.network_manager import network_manager from gui.prototxt_editor import editor_widget from backend.caffe import path_loader class UiMainWindow(QtWidgets.QMainWindow): sessionSelectionChanged = pyqtSignal(object) def __init__(self, actions=None, desktop = None): QtWidgets.QMainWindow.__init__(self) self.setUnifiedTitleAndToolBarOnMac(True) self.disabled = False #self.setGeometry(desktop.screenGeometry()) #fullscreen self.setWindowTitle("Barista") self.setWindowIcon(QIcon('./resources/icon.png')) # Initialize menu and status bar if actions: self.actions = actions self.actions.setParent(self) else: self.actions = gui.main_window.default_actions.DefaultActions(self) self.editor = editor_widget.EditorWidget(parent=self) self.editor.setWindowTitle("Barista Network Editor") self.actions.setEditor(self.editor) self.inManager = input_manager.InputManager(self) self.actions.setInputManager(self.inManager) self.hostManager = host_manager.HostManager(self) self.actions.setHostManager(self.hostManager) pathEditor = path_editor_dialog.PathEditorDialog(self) pathEditor.setWindowTitle("Barista Path Editor") self.actions.setPathEditor(pathEditor) self.setMenuBar(self.actions.buildMenuBar()) filetoolbar = self.actions.buildFileToolbar() self.addToolBar(filetoolbar) self.actions.addToolBarEntry(filetoolbar) edittoolbar=self.actions.buildEditToolbar() self.addToolBar(edittoolbar) self.actions.addToolBarEntry(edittoolbar) #runtoolbar=self.actions.buildRunToolbar() #self.addToolBar(runtoolbar) #self.actions.addToolBarEntry(runtoolbar) # Create the view manager self.viewManager = view_manager.ViewManager(self, self.actions) self.actions.setViewManager(self.viewManager) self.setStatusBar(gui.main_window.status_bar.UiStatusBar(self)) # Create the view self.viewManager.createView() # Create the node prototxt_editor as the central widget self.nodeEditor = NodeEditor(self) self.setCentralWidget(self.nodeEditor.getView()) self.actions.setScene(self.nodeEditor.getView().scene()) # Connect the node editor context menu. self.nodeEditor.getView().arrangeVerticallyClicked.connect(self.actions.sortSceneActionVertical.trigger) self.nodeEditor.getView().arrangeHorizontallyClicked.connect(self.actions.sortSceneAction.trigger) # Create the network manager to save all layers and handle layer-related events self.networkManager = network_manager.NetworkManager(self.viewManager.getActiveLayersDock(), self.viewManager.getLayerPropertyDock(), self.viewManager.getSolverPropertyDock, self.nodeEditor) # Link networkManager to menuBar self.actions.setNetworkManager(self.networkManager) # As a test, open the dummy network file self.networkManager.setNetwork(dict_helper.bareNet("default")) #connect all signals self.actions.quitTriggered.connect(self.close) self.actions.projectChanged.connect(self._onProjectChanged) self.networkManager.modifiedChanged.connect(self._onNetworkChanged) self.viewManager.sessionController.sessionSelectionChanged.connect(self._onSessionChanged) self.networkManager.newStateData.connect(self._onStateDictChanged) self.viewManager.sessionController.sessionStateRefreshed.connect(self.disableEditing) self.rec_installEventfilter() def eventFilter(self, source, event): if event.type() == QtCore.QEvent.MouseButtonRelease and event.button() == QtCore.Qt.LeftButton: self.viewManager.onVisibilityChange() return super(UiMainWindow, self).eventFilter(source, event) def rec_installEventfilter(self): def recursive_set(parent): for child in parent.findChildren(QtCore.QObject): try: child.installEventFilter(self) except: pass recursive_set(child) self.installEventFilter(self) recursive_set(self) def availableActions(self): return self.actions def keyPressEvent(self, QKeyEvent): # Send signal to network manager when ctrl key is pressed in application if QKeyEvent.key() == QtCore.Qt.Key_Control: self.networkManager.setControlHold(True) # Send signal to network manager when shift key is pressed in application elif QKeyEvent.key() == QtCore.Qt.Key_Shift: self.networkManager.setShiftHold(True) super(UiMainWindow, self).keyPressEvent(QKeyEvent) def keyReleaseEvent(self, QKeyEvent): # Send signal to network manager when ctrl key is released in application if QKeyEvent.key() == QtCore.Qt.Key_Control: self.networkManager.setControlHold(False) # Send signal to network manager when ctrl key is released in application elif QKeyEvent.key() == QtCore.Qt.Key_Shift: self.networkManager.setShiftHold(False) super(UiMainWindow, self).keyReleaseEvent(QKeyEvent) def showPathWarning(self): self.warningBox = QtWidgets.QMessageBox() self.warningBox.setText("caffe-path does not exists on your PC, please edit caffe-path in settings and restart the program") self.warningBox.setIcon(2) self.warningBox.show() def closeEvent(self, event): # check if there are running sessions and ask the user to pause them if self.actions.checkRunningSessions() is False: event.ignore() return # Save the current view self.viewManager.saveView() # Ask the user if he wants to save project changes if necessary. if self.actions.askUserAndSave(self): # We need to process events here, because when the project is saved # an event is emitted. If this doesn't get processed, the main window # will not be closed. QtWidgets.QApplication.processEvents() event.accept() else: # The user decided to abort the close event. event.ignore() def _onProjectChanged(self, project): self.setWindowTitle("Barista - {}".format(project.getProjectName())) # TODO: Those lines should not be commented, right? #reload the caffe information # path_loader.reloadCaffeModule(project.getCaffeRoot()) # path_loader.reloadProtoModule(project.getCaffeRoot()) # self.editor = editor_widget.EditorWidget(self) # self.editor.setWindowTitle("Barista Network Editor") # network_manager.helper.bareNet(project.getProjectName()) # self.viewManager.loadView() def _onStateDictChanged(self, stateDict): self.viewManager.project.getActiveSession().state_dictionary = stateDict def _onSessionChanged(self): activeSession = self.viewManager.project.getActiveSession() self.networkManager.setState(activeSession.state_dictionary if activeSession is not None else None) def _onNetworkChanged(self, changeSinceLastSave): #update the status bar self.statusBar().showModifiedFlag(changeSinceLastSave) #update the current session self.viewManager.project.setActiveSessionStateDict(self.networkManager.getStateDictionary()) def disableEditing(self, state): """ Handle event on state change of a session and disable/enable editing. """ disable = False if state in [State.RUNNING, State.PAUSED, State.FINISHED, State.NOTCONNECTED, State.FAILED]: disable = True self.actions.disableEditing(disable) self.viewManager.disableEditing(disable) self.nodeEditor.disableEditing(disable) self.inManager.disableEditing(disable) self.editor.disableEditing(disable) self.show() self.disabled = disable def getDisabled(self): return self.disabled gui/main_window/status_bar.py0000664ڊ00111170000000113213243274037016623 0ustar klemmso0cvpr# To change this license header, choose License Headers in Project Properties. # To change this template file, choose Tools | Templates # and open the template in the prototxt_editor. from PyQt5 import QtWidgets class UiStatusBar(QtWidgets.QStatusBar): def __init__(self, parent=None): QtWidgets.QStatusBar.__init__(self, parent) self.modifyWdg = QtWidgets.QLabel() self.addPermanentWidget(self.modifyWdg) def showModifiedFlag(self, modified): if modified: self.modifyWdg.setText("Changes *") else: self.modifyWdg.setText("") gui/main_window/view_manager.py0000664ڊ00111170000002160513243274037017127 0ustar klemmso0cvprfrom PyQt5.QtCore import Qt from backend.barista.session.session_utils import State from gui.main_window.docks import ( dock_available_layers, dock_active_layers ) from gui.main_window.docks.session import dock_session_list from backend.barista.utils.settings import applicationQSetting from gui.main_window.docks.dock_console import DockElementConsole from gui.main_window.docks.plotter.qt5_plotter_gui import DockElementPlotter from gui.main_window.docks.properties import dock_layer_properties, dock_solver_properties from gui.main_window.docks.session.session_controller import SessionController from gui.main_window.docks.weight_visualization.dock_weight_visualization import DockElementWeightPlotter class ViewManager(): """Management of the current view state of the main main_window""" def __init__(self, mainWindow, actions): # Store the variables for the class self.mainWindow = mainWindow self.actions = actions self._defaultState = None self._isDefaultStateSaved = False self.dicDockWidgets = {} self.dicDockActions = {} self.sessionController = SessionController() def _addDock(self, element, title): """Creates the Dock-Element element with title and mainwindow and add it to Menu""" dock = element(self.mainWindow, title) self.dicDockWidgets[title] = dock self.dicDockActions[title] = self.actions.addDockMenuEntry(dock, title) return dock def createView(self): """Creates the docks of the application and loads the view settings""" # Create the dock widgets and add the menu bar actions self._addDock(dock_available_layers.DockElementLayers, 'Available Layers') self._addDock(dock_active_layers.DockElementActivLayers, "Network Layers") self._addDock(dock_session_list.DockElementSessions, "Sessions") self._addDock(dock_layer_properties.LayerProperties, "Layer Properties") self._addDock(dock_solver_properties.SolverProperties, "Solver Properties") bottomDock=self._addDock(DockElementConsole, "Console") bottomDock.setMinimumHeight(100) plotterDock=self._addDock(DockElementPlotter, "Plotter") plotterDock.setMinimumHeight(100) weightPlotterDock=self._addDock(DockElementWeightPlotter, "Weight Visualization") weightPlotterDock.setMinimumHeight(100) self.sessionController.setWeightPlotter(weightPlotterDock) self.sessionController.setPlotterGui(self.dicDockWidgets['Plotter']) self.dicDockWidgets['Sessions'].setController(self.sessionController) self.loadDefaultView() # Try to load the view if the user has saved one self.loadView() def getLayersDock(self): return self.dicDockWidgets['Available Layers'] def getActiveLayersDock(self): return self.dicDockWidgets['Network Layers'] def getLayerPropertyDock(self): return self.dicDockWidgets['Layer Properties'] def getSolverPropertyDock(self): return self.dicDockWidgets['Solver Properties'] def getSessionsDock(self): return self.dicDockWidgets['Sessions'] def dockAction(self, title): 'Shows or hides a dock based on input from menu bar action' if self.dicDockWidgets[title].isHidden(): self.dicDockWidgets[title].show() else: self.dicDockWidgets[title].hide() #reload the hooks in the view menu for the docks self.onVisibilityChange() def onVisibilityChange(self): 'Shows or hides a hook for the docks in the view menu' # widget is only the title as string for widget in self.dicDockWidgets: if self.dicDockWidgets[widget].isVisible(): self.dicDockActions[widget].setChecked(True) else: self.dicDockActions[widget].setChecked(False) def loadDefaultView(self): 'Loads the standard view settings' if self._isDefaultStateSaved == False: # Add layers dock self.mainWindow.addDockWidget(Qt.LeftDockWidgetArea, self.dicDockWidgets['Available Layers']) self.dicDockWidgets['Available Layers'].setFloating(False) self.dicDockWidgets['Available Layers'].show() # Add network layers dock # self.mainWindow.addDockWidget(Qt.LeftDockWidgetArea, self.dicDockWidgets['Network Layers']) self.mainWindow.tabifyDockWidget(self.dicDockWidgets['Available Layers'], self.dicDockWidgets['Network Layers']) self.dicDockWidgets['Network Layers'].setFloating(False) self.dicDockWidgets['Network Layers'].show() self.dicDockActions['Network Layers'].setChecked(True) self.dicDockWidgets['Available Layers'].raise_() # Add running tasks dock self.mainWindow.addDockWidget(Qt.RightDockWidgetArea, self.dicDockWidgets['Sessions']) self.dicDockWidgets['Sessions'].setFloating(False) self.dicDockWidgets['Sessions'].show() self.dicDockActions['Sessions'].setChecked(True) # Add solver properties dock # self.mainWindow.addDockWidget(Qt.RightDockWidgetArea, self.dicDockWidgets['Solver Properties']) self.mainWindow.tabifyDockWidget(self.dicDockWidgets['Sessions'], self.dicDockWidgets['Solver Properties']) self.dicDockWidgets['Solver Properties'].setFloating(False) self.dicDockWidgets['Solver Properties'].show() self.dicDockActions['Solver Properties'].setChecked(True) # Add layer properties dock # self.mainWindow.addDockWidget(Qt.RightDockWidgetArea, self.dicDockWidgets['Layer Properties']) self.mainWindow.tabifyDockWidget(self.dicDockWidgets['Solver Properties'], self.dicDockWidgets['Layer Properties']) self.dicDockWidgets['Layer Properties'].setFloating(False) self.dicDockWidgets['Layer Properties'].show() self.dicDockActions['Layer Properties'].setChecked(True) self.dicDockWidgets['Sessions'].raise_() # Add Console dock self.mainWindow.addDockWidget(Qt.BottomDockWidgetArea, self.dicDockWidgets['Console']) self.dicDockWidgets['Console'].setFloating(False) self.dicDockWidgets['Console'].show() self.dicDockActions['Console'].setChecked(True) # Add plotter dock self.mainWindow.addDockWidget(Qt.BottomDockWidgetArea, self.dicDockWidgets['Plotter']) self.dicDockWidgets['Plotter'].setFloating(False) self.dicDockWidgets['Plotter'].show() self.dicDockActions['Plotter'].setChecked(True) # Add weight dock # self.mainWindow.addDockWidget(Qt.BottomDockWidgetArea, self.dicDockWidgets['Weight Visualization']) self.mainWindow.tabifyDockWidget(self.dicDockWidgets['Plotter'], self.dicDockWidgets['Weight Visualization']) self.dicDockWidgets['Weight Visualization'].setFloating(False) self.dicDockWidgets['Weight Visualization'].show() self.dicDockActions['Weight Visualization'].setChecked(True) self.dicDockWidgets['Plotter'].raise_() # Save this state as default so that multiple clicks on the restore default view action do not cause changes self._isDefaultStateSaved = True self._defaultState = self.mainWindow.saveState() else: self.mainWindow.restoreState(self._defaultState) self.onVisibilityChange() def loadView(self): 'Loads the user-defined view settings from view_settings' settings = applicationQSetting() settings.beginGroup("ViewManager") state = settings.value("guistate", None) settings.endGroup() if state: self.mainWindow.restoreState(state) self.onVisibilityChange() def saveView(self): 'Saves the current view settings to view_settings file' settings = applicationQSetting() settings.beginGroup("ViewManager") settings.setValue("guistate", self.mainWindow.saveState()) settings.endGroup() def setProject(self, project): """ Set the current project of the appliction. """ self.project = project self.sessionController.setProject(project) self.dicDockWidgets['Plotter'].setProject(project) def getProject(self): """ Return the current project. """ return self.project def disableEditing(self, disable): """ Disable editing on all docks. """ self.dicDockWidgets['Solver Properties'].disableEditing(disable) self.dicDockWidgets['Layer Properties'].disableEditing(disable) self.dicDockWidgets['Available Layers'].disableEditing(disable) self.dicDockWidgets['Network Layers'].disableEditing(disable) def getPlotter(self): """ Return the plotter. """ return self.dicDockWidgets['Plotter'] gui/network_manager/__init__.py0000664ڊ00111170000000000013243274037017054 0ustar klemmso0cvprgui/network_manager/history_manager.py0000664ڊ00111170000000630013243274037020521 0ustar klemmso0cvprclass HistoryManager(): def __init__(self, maxSize=100): """ Initialize HistoryManager which saved maximal maxSize states. The maxSize+1 insertion removes the first """ self.history = [] # Position is a value contains the index of the state in a continues way. # That means even when the first entries get deleted because of maxSize # self.position do not get decreased by this. self.position = -1 # the index of the current position in history list self.__idx = -1 # self.listLen contains maxSize self.listLen = maxSize # If this gets true, no other entry can be make in history self.__lockHistory = False def canUndo(self): return self.__idx >0 def canRedo(self): return self.__idx < len(self.history)-1 def undo(self): if self.canUndo(): self.__idx -=1 self.position -= 1 def redo(self): if self.canRedo(): self.__idx +=1 self.position += 1 def currentState(self): """ Get latest state saved in history""" if len(self.history) > 0: return self.history[self.__idx] else: return None def _insert(self,element): """ Insert element at the current position """ # remove newer elements del self.history[self.__idx + 1:] # Remove the oldest element if there are too many elements if self.__idx == self.listLen: del self.history[0] else: self.__idx += 1 self.history.append(element) self.position += 1 def lockHistory(self, fun): """ Lock history for the whole method fun. As a result no history can be inserted whil fun is executed. """ if self.__lockHistory: return self.__lockHistory = True fun() self.__lockHistory = False def insertFunc(self, fun): """ Insert an element in history using the function fun. While fun is working insertFunc is locked, so no other element can be added in history. As a result recursivly insertion of history is stopped. The function fun will be called with in insertion function which can be called to insert an element in history. E.g.: def createNewElement(text): # Nothing happend, because insertFunc is locked historymanager.insertFunc(lambda insertF: insertF(text)) return text historymananger.insertFunc(lambda insertF: insertF(createNewElement("bla")) # Only one string will be insert When inserting a new state old state gets removed if the limit of entries is reached. Also states newer than the current one (e.g. by using undo()) get removed (so you can't do a redo() anymore) """ if self.__lockHistory: return self.__lockHistory = True fun(self._insert) self.__lockHistory = False def clear(self): self.position = -1 self.__idx = -1 self.history = [] gui/network_manager/layer_helper.py0000664ڊ00111170000002142013243274037020001 0ustar klemmso0cvprclass LayerHelper: def __init__(self): return PHASE_TRAIN = u'TRAIN' PHASE_TEST = u'TEST' @staticmethod def getLayerPhase(layer): # TODO implement further rules to decide the phase of a layer (see issue #201) # -> not only "include", but also "exclude" # -> (include|exclude).min_level # -> (include|exclude).max_level # -> (include|exclude).stage # -> (include|exclude).not_stage # -> combination of multiple rules. e.g. include.phase == TRAIN and include.phase == TEST equals no value # -> unclear: what does LayerParameter.phase (instead of LayerParameter.[include|exclude].phase) do? if "include" in layer["parameters"]: for item in layer["parameters"]["include"]: if "phase" in item: return item["phase"] return "" @staticmethod def isLayerIncludedInTrainingPhase(layer): """Check whether this layer is included in the training phase.""" # TODO see TODO in LayerHelper.getLayerPhase(). return LayerHelper.getLayerPhase(layer) != LayerHelper.PHASE_TEST @staticmethod def isLayerInPlace(layer): if "top" in layer["parameters"] and "bottom" in layer["parameters"]: for bottomBlob in layer["parameters"]["bottom"]: for topBlob in layer["parameters"]["top"]: if bottomBlob == topBlob: return True return False @staticmethod def isLayerInPlaceForBlob(layer, blobName): if "top" in layer["parameters"] and "bottom" in layer["parameters"]: if not blobName in layer["parameters"]["top"]: return False if not blobName in layer["parameters"]["bottom"]: return False return True else: return False @staticmethod def getTopBlobIndex(layer, blobName): if "top" in layer["parameters"]: for index in range(0, len(layer["parameters"]["top"])): if blobName == layer["parameters"]["top"][index]: return index return -1 @staticmethod def buildConnectionList(network): layerDict = network["layers"] layerOrder = network["layerOrder"] orderedLayersByBlob = OrderedLayersByBlob(layerDict, layerOrder) blobOrder = orderedLayersByBlob.blobLayerDict connections = list() for layerID in layerOrder: layer = layerDict[layerID] LayerHelper.__addConnectionsForLayer(layer, layerID, blobOrder, connections, layerDict) return connections @staticmethod def getIdandName(layertypes, network): '''go over all layers in the network and search for layers of a type''' mylayer = [] for nid in network["layerOrder"]: if network["layers"][nid]["type"].name() in layertypes: layername = network["layers"][nid]["parameters"]["name"] params = network["layers"][nid]["parameters"] # since names can be ambiguous append the phase if unicode("include") in params: for incl in params["include"]: if unicode("phase") in incl: layername += " (" + incl["phase"] + ")" mylayer.append([nid,layername]) return mylayer @staticmethod def getNamesAndDBPaths(layertypes, network): """ Go over all layers in the network, search for layers of a type and return their name and database path. """ mylayer = [] for nid in network["layerOrder"]: if network["layers"][nid]["type"].name() in layertypes: params = network["layers"][nid]["parameters"] layername = params["name"] # since names can be ambiguous append the phase if unicode("include") in params: for incl in params["include"]: if unicode("phase") in incl: layername += " (" + incl["phase"] + ")" if unicode("data_param") in params and unicode("source") in params["data_param"]: dbpath = params["data_param"]["source"] mylayer.append([layername, dbpath]) return mylayer @staticmethod def setPathforType(id, path, type, network): '''set the path and db type to layer basend on the ID''' params = network["layers"][id]["parameters"] if type in ["LMDB", "LEVELDB"]: if "data_param" not in params: params.giveProperty("data_param",{}) params["data_param"]["source"] = path if type == "LMDB": params["data_param"]["backend"] = u'LMDB' if type == "LEVELDB": params["data_param"]["backend"] = u'LEVELDB' if type == "HDF5TXT": if "hdf5_data_param" not in params: params.giveProperty("hdf5_data_param",{}) params["hdf5_data_param"]["source"] = path @staticmethod def __addConnectionsForLayer(layer, layerID, blobOrder, connections, layerDict): if "bottom" in layer["parameters"]: bottoms = layer["parameters"]["bottom"] for bottomIndex in range(0, len(bottoms)): bottom = bottoms[bottomIndex] if bottom in blobOrder: topBlobList = blobOrder[bottom] phaseDict = topBlobList[0] if "" in phaseDict: LayerHelper.__addConnection(phaseDict[""], layerID, bottomIndex, bottom, connections, layerDict) LayerHelper.__removeBlobPhaseFromDict("", bottom, blobOrder) elif LayerHelper.getLayerPhase(layer) == "": for topPhase, topLayerID in phaseDict.iteritems(): LayerHelper.__addConnection(topLayerID, layerID, bottomIndex, bottom, connections, layerDict) LayerHelper.__removeBlobPhaseFromDict(topPhase, bottom, blobOrder) else: phase = LayerHelper.getLayerPhase(layer) for topPhase, topLayerID in phaseDict.iteritems(): if phase == topPhase: LayerHelper.__addConnection(topLayerID, layerID, bottomIndex, bottom, connections, layerDict) LayerHelper.__removeBlobPhaseFromDict(topPhase, bottom, blobOrder) @staticmethod def __addConnection(topLayerID, bottomLayerID, bottomBlobIndex, blobName, connections, layerDict): topBlobIndex = LayerHelper.getTopBlobIndex(layerDict[topLayerID], blobName) connections.append((topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex)) return @staticmethod def __removeBlobPhaseFromDict(phase, top, blobOrder): blobList = blobOrder[top] if len(blobList) > 1: phaseDict = blobList[0] if "" in phaseDict: del phaseDict[""] elif phase in blobList[1]: del phaseDict[phase] if len(phaseDict) == 0: del blobList[0] class OrderedLayersByBlob: def __init__(self, layerDict, layerOrder): self.blobLayerDict = dict() self.__createOrderedLayerListPerBlob(layerDict, layerOrder) def __createOrderedLayerListPerBlob(self, layerDict, layerOrder): for layerID in layerOrder: layer = layerDict[layerID] self.__addLayerToDict(layer, layerID) def __addLayerToDict(self, layer, layerID): if "top" in layer["parameters"]: for top in layer["parameters"]["top"]: self.__addTopBlobLayer(top, layer, layerID) def __addTopBlobLayer(self, top, layer, layerID): if top in self.blobLayerDict: if LayerHelper.isLayerInPlaceForBlob(layer, top): if len(self.blobLayerDict[top]) > 1: currentLevelDict = self.blobLayerDict[top][-1] if "" in currentLevelDict or LayerHelper.getLayerPhase(layer) in currentLevelDict: self.__createDictForTop(top, layer, layerID) else: currentLevelDict[LayerHelper.getLayerPhase(layer)] = layerID else: self.__createDictForTop(top, layer, layerID) else: self.blobLayerDict[top][0][LayerHelper.getLayerPhase(layer)] = layerID else: self.blobLayerDict[top] = list() self.__createDictForTop(top, layer, layerID) def __createDictForTop(self, top, layer, layerID): tempDict = dict() tempDict[LayerHelper.getLayerPhase(layer)] = layerID self.blobLayerDict[top].append(tempDict) gui/network_manager/network_manager.py0000664ڊ00111170000014671313243274037020526 0ustar klemmso0cvprimport copy import re import sys import uuid from PyQt5.QtCore import QObject, pyqtSignal, Qt from gui.main_window.docks.properties.property_widgets.network_dict_info import NetworkDictInfoBuilder from gui.main_window.docks.properties.property_widgets.program_state_info import ProgramStateInfoBuilder, defaultState from gui.network_manager.history_manager import HistoryManager import gui.main_window.docks.properties.property_widgets.data as PropData from backend.barista.constraints.permanent.solver import ensureSolverConstraints from backend.barista.utils.logger import Log from backend.caffe import loader, saver, proto_info, dict_helper as helper from gui.main_window.docks.properties.property_widgets.data import ReasonUpdated, ReasonKeyDeleted, ReasonKeyAdded, ReasonRemoved from gui.network_manager.layer_helper import LayerHelper def buildNetData(network): """ Build PropertyData for the network-dictionary network """ netinfo = NetworkDictInfoBuilder(network) return PropData.PropertyData(network, netinfo.buildRootInfo().subparameter(), netinfo) def buildStateData(stateDict): """ Build PropertyData for the state-dictionary stateDict """ if stateDict == None: stateDict = defaultState() info = ProgramStateInfoBuilder(stateDict) return PropData.PropertyData(stateDict, info.buildRoot().subparameter(), info, signalcontroller=PropData.SignalController()) class HistoryWriting(object): """ Class which should give methods to avoid recursive history writings """ def __init__(self, state, callback): """ Creates this object. state is the state-object which can be changed through the makeEntry-call callback is a function which will be called if an entry in history should be make """ self.__state = state self.__callback = callback self.__lock = False def valueOf(self,uri): return self.__state.valueOf(uri) def makeEntry(self, func): """ Make an entry in history. func is a function which will be called with the state object (e.g. historywriter.makeEntry(lambda state: state["name"] = 2) as long as func is called, no other function can make an entry in history, but they still can change the state through this function """ prevLocked = self.__lock self.__lock = True res = None if func: res = func(self.__state) if not prevLocked: self.__lock = False self.__callback() return res def entryWithoutHistory(self,func): """ Change state without touching the history. See makeEntry for the argument """ prevLocked = self.__lock self.__lock = True res = None if func: res = func(self.__state) if not prevLocked: self.__lock = False return res class NetworkManager(QObject): """ stateChanged is triggered if the state of the network or solver has changed. modifiedChanged is used to indicate a change of the network to other components. The boolean value indicates whether the current state differs from the last saved one. """ stateChanged = pyqtSignal() modifiedChanged = pyqtSignal(bool) newStateData = pyqtSignal(object) 'The network manager stores the information for all layers currently in ui and handles all layer-related events' def __init__(self, dockElementActiveLayers, dockLayerProperties, dockSolverProperties, nodeEditor): 'Creates a new network manager' super(NetworkManager, self).__init__() self.callerId = Log.getCallerId('networkmanager') # Set the reference to the active layers dock, as well as reference to network manager in active layers dock self.dockElementActiveLayers = dockElementActiveLayers.getLayersListWidget() self.dockElementActiveLayers.setNetworkManager(self) # Set the reference to property dock, as well as reference to network manager in property dock self.dockLayerProperties = dockLayerProperties self.dockLayerProperties.setNetworkManager(self) # Set the reference to the solver property dock self.dockSolverProperties = dockSolverProperties #self.dockSolverProperties().solverChanged.connect(self.makeHistory) self.dockSolverProperties().solverChanged.connect(self.validateChangedSolver, Qt.QueuedConnection) # Set the reference to the node edtior scene, as well as reference to network manager in node prototxt_editor self.nodeEditor = nodeEditor self.nodeEditor.setNetworkManager(self) # Get the caffe metainformation self.caffeInfo = proto_info.CaffeMetaInformation() # Set up a counter for overall number of layers in the application, to use as id for each layer self.layerCount = 0 # Create an empty network, to be filled by both opening a file and user actions self.stateData = None self.historyWriter = None # type: HistoryWriting self.setState(None) # self.setNetwork(helper.bareNet("Unnamed")) # Set up a list that contains all currently selected layer ids # Setup historyManager self.__historyDepth = 0 self._historymanager = HistoryManager() self.makeHistory() self.__lastModified = 0 self.stateChanged.connect(lambda: self.modifiedChanged.emit(self.isModified())) self.networkHelper = None # type: helper.DictHelper def validateChangedSolver(self): """This method is triggered automatically whenever the solver has been changed (in the GUI).""" # prevent manipulation of solver constraints (changes self.solver) data = self.dockSolverProperties().data() if data is None: data = self.solver ensureSolverConstraints(data) # ensureSolverConstraints(self.solver) # backpropagate changed changes to the GUI by replacing the complete solver object # (it's already up to date inside of this class, but not in the GUI) # self.dockSolverProperties().setSolver(self.solver) @property def network(self): return self.stateData["network"] @property def solver(self): return self.stateData["solver"] """Method that adds a Layer in to the scene, called when drag and dropping a layer from layer selection""" def addLayer(self, type, scenePosX, scenePosY): # Check if a layer have been moved self.checkLayersForMovement(self.nodeEditor.getNodeIds()) """ Add a layer from the type (a string) at the given position """ def intern(stateData): # generate new Layer Name state = self.stateData.getBoringDict() boringNetwork = state["network"] nethelper = helper.DictHelper(boringNetwork) # get all present Layer Names in a list allLayerNames = nethelper.nameOfLayers() # set new Name as Typename and layercount = 0 newName = type count = 0 for name in allLayerNames: if newName == re.sub(' #[0-9]+$', '', name): try: number = int(re.sub(newName + ' #', '', name)) count = number if number > count else count except ValueError: count = 0 # increment layertype count count += 1 # append layertypecount to new Layername newName = newName + " #" + str(count) network = stateData["network"] networkHelper = helper.DictHelper(network) # Add a layer to the network #self.stateData.signalController().lock() newlayer, id = networkHelper.addLayer(self.caffeInfo.availableLayerTypes()[type], newName, 0) #self.stateData.signalController().unlock() # Add the node item to the graphics scene stateData["position"][id] = [scenePosX, scenePosY] #self.nodeEditor.addLayer(id, network['layers'][id], scenePosX, scenePosY) # Get the order and add the layer to the dock of active layers order = network["layerOrder"].index(id) #self.dockElementActiveLayers.addItem(id, name, order, type) self.updateOrder() # Set the selection to the new element self.setSelection(id) # Set the new overall layer count self.layerCount += 1 self.historyWriter.makeEntry(intern) def focusLayer(self, id): self.checkLayerForMovement(id) self.nodeEditor.focusLayer(id) def deleteLayer(self, id): # Check if a layer have been moved self.checkLayersForMovement(self.nodeEditor.getNodeIds()) """ Removes the layer with the given id """ def intern(stateDict): self.setSelection(id) self.deleteSelectedLayers() self.historyWriter.makeEntry(intern) def deleteSelectedLayers(self): # Check if a layer have been moved self.checkLayersForMovement(self.nodeEditor.getNodeIds()) 'Deletes all Layers currently selected' def intern(stateData): stateData.signalController().lock() self.nodeEditor.disconnectConnectionsOfSelection() network = stateData["network"] networkHelper = helper.DictHelper(network) selected = list(stateData["selection"]) stateData["selection"].setValue([]) for id in selected: self.__checkLayerForMovement(id) if networkHelper.hasLayerWithId(id): networkHelper.removeLayer(id) del stateData["position"][id] stateData.signalController().unlock() self.updateOrder() self.historyWriter.makeEntry(intern) def onStateUpdate(self, uri, reason): """ Get called if something is changed in stateData """ def intern(stateData): # # Debugging output: #stateData.signalController().enableLoging() #print("Updating {} because {}".format(uri, reason)) # If something in selection changed if uri[0] == "selection": # If we have an complete new list of selected docks if isinstance(reason, ReasonUpdated): ids = reason.oldValue # Trigger if layers moved self.__checkLayerForMovement(ids) # If a node get deselected if isinstance(reason, ReasonRemoved): ids = [id for id in stateData["selection"]] ids.append(reason.value) # Trigger if layers moved self.__checkLayerForMovement(ids) # Set the selection in prototxt_editor and the propertywdg ids = [id for id in stateData["selection"]] self.nodeEditor.setSelectedLayers(ids) self.dockLayerProperties.setTabs(ids) # The whole network changed elif uri == ["network"]: # Refresh node-items and connections self.refreshItems() self.refreshConnections() elif uri == ["hidden_connections"]: self.updateHiddenConnectionsInNodeEditor() # Some layer information changed elif uri[:2] == ["network","layers"] and len(uri) > 2: id = uri[2] # Some parameter changed if len(uri) > 4 and uri[3] == "parameters": self.updateToolTip(id) reluri = uri[4:] # The top blobs changed -> tell the node prototxt_editor if reluri == ["top"]: # Some blob added if isinstance(reason, PropData.ReasonAdded): idx = reason.idx content = self.historyWriter.valueOf(uri+[idx]) self.nodeEditor.addTopBlob(id, content) # Some blob removed elif isinstance(reason, PropData.ReasonRemoved): idx = reason.idx self.nodeEditor.disconnectTopBlob(id, idx) self.nodeEditor.removeTopBlob(id, idx) self.updateOrder() # The bottom blobs changed -> tell the node prototxt_editor elif reluri == ["bottom"]: # Some blob added if isinstance(reason, PropData.ReasonAdded): idx = reason.idx content = self.historyWriter.valueOf(uri+[idx]) self.nodeEditor.addBottomBlob(id, idx) # Some blob removed elif isinstance(reason, PropData.ReasonRemoved): idx = reason.idx self.nodeEditor.disconnectBottomBlob(id, idx) self.nodeEditor.removeBottomBlob(id, idx) self.updateOrder() # One top blobs changed -> tell the node prototxt_editor elif reluri[0] == "top" and isinstance(reason, PropData.ReasonUpdated): newVal = reason.newValue self.nodeEditor.renameTopBlob(id, reluri[1], newVal) # One bottom blobs changed -> tell the node prototxt_editor elif reluri[0] == "bottom" and isinstance(reason, PropData.ReasonUpdated): newVal = reason.newValue self.nodeEditor.renameBottomBlob(id, reluri[1], newVal) # The include phase changed -> tell the node prototxt_editor elif reluri in [["name"]] or len(reluri) == 3 and (reluri[0], reluri[2]) in [("include","phase")]: self.nodeEditor.updateLayerData(id) # Layers are changed elif uri == ["network", "layers"]: # If a layer is removed -> tell the node prototxt_editor if isinstance(reason, ReasonKeyDeleted): # Check if some node has moved id = reason.key layerstocheck = list(stateData["selection"]) if id in layerstocheck: layerstocheck.remove(id) self.__checkLayerForMovement(layerstocheck) self.nodeEditor.deleteItemsByID([id]) self.updateOrder() # If a layer is added -> tell the node prototxt_editor if isinstance(reason,ReasonKeyAdded): # Check if some node has moved self.__checkLayerForMovement(list(stateData["selection"])) id = reason.key self.nodeEditor.addLayer(id, stateData["network"]["layers"][id], 0.0,0.0) # Positions get updated -> tell the node prototxt_editor elif uri[0] == "position" and (isinstance(reason, ReasonUpdated) or isinstance(reason,ReasonKeyAdded)): self.nodeEditor.applyLayerPositionDict(stateData["position"]) # We've got a new solver -> set solver to the propertywdg elif uri == ["solver"]: self.setSolver(stateData["solver"]) #stateData.signalController().disableLoging() self.historyWriter.entryWithoutHistory(intern) self.newStateData.emit(self.getStateDictionary()) def setSelection(self, id): """ Select the layer with the given id. Note: you can also manipulate stateData["selection"] instead. """ self.setSelectionList([id]) def setSelectionList(self, ids): """ Select all layers in ids Note: you can also manipulate stateData["selection"] instead. """ def intern(stateData): stateData["selection"].setValue(ids) self.historyWriter.makeEntry(intern) def addLayerToSelection(self, id): """ Add the layer with the given id to the selection. Note: you can also manipulate stateData["selection"] instead. """ self.historyWriter.makeEntry(lambda stateData: stateData["selection"].pushBack(id)) # Notify all elements #self.dockElementActiveLayers.addItemToSelection(id) #self.nodeEditor.addLayerToSelection(id) #self.dockLayerProperties.addTab(id) # Save the history to include this step in undo and redo #self.__historyDepth -= 1 #self.makeHistory() def __checkLayerForMovement(self,ids): """ Checks if the layer with ids (a list of string or a string) has a different position in prototxt_editor than in stateData. If so the stateData will be updated. """ if type(ids) != list: ids = [ids] def intern(stateData): # mayUpdate contains ids which are valid # because some if ids could already be deleted mayUpdate = set(ids).intersection(stateData["position"].keys()) oldValues = dict([(id, stateData["position"][id]) for id in mayUpdate]) newValue = dict([(id,self.nodeEditor.getPositionOfLayer(id)) for id in mayUpdate]) for id in mayUpdate: oldx, oldy = oldValues[id] newx, newy = newValue[id] if oldx != newx or oldy != newy: stateData["position"][id].setValue([newx,newy]) self.historyWriter.makeEntry(intern) def checkLayersForMovement(self,ids): self.__checkLayerForMovement(ids) def removeLayerFromSelection(self, id): """ Deselect the layer with the given id. Note: you can also manipulate stateData["selection"] instead. """ def intern(stateData): stateData["selection"].remove(id) self.historyWriter.makeEntry(intern) def clearSelection(self): """ Deselect all layers. Note: you can also manipulate stateData["selection"] instead. """ if len(self.stateData["selection"]) == 0: return def intern(stateData): stateData["selection"].setValue([]) self.historyWriter.makeEntry(intern) def clearSelectionWithoutSavingHistory(self): """ Deselect all layers without an entry in history. Note: you can also manipulate stateData["selection"] through historyWriter.entryWithoutHistory instead. """ self.historyWriter.entryWithoutHistory(lambda state: self.clearSelection()) def updateNet(self): """ Force an entry in history """ self._historymanager.insertFunc(None) # todo: this should really do something def updateName(self, id, name): """ Sets the name of the layer with the given id to the given name. Note: you can also manipulate stateData["network"]["layers"][id]["parameters"]["name"] instead """ def intern(stateDict): network = stateDict["network"] networkHelper = helper.DictHelper(network) networkHelper.layerParams(id)["name"] = name self.historyWriter.makeEntry(intern) def updateLayerInNodeEditor(self, layerID): """ Tell nodeEditor to update the layerdata of the given layer """ self.nodeEditor.updateLayerData(layerID) def updateOrder(self): """ Recalculate the order of the layer """ if set(self.nodeEditor.getNodeIds()) == set(self.network["layers"].keys()): self.network["layerOrder"].setValue(self.nodeEditor.calculateLayerOrder()) def refreshConnections(self): """ Refresh the connections of all layers """ # Connect layers connections = LayerHelper.buildConnectionList(self.network.getBoringDict()) #connections = LayerHelper.buildConnectionList(self.network) for connection in connections: self.nodeEditor.createConnection(connection[0], connection[1], connection[2], connection[3]) def updateHiddenConnectionsInNodeEditor(self): """ Update the NodeEditor to hide connections which should be hidden in the current state """ # There are no hidden_connections if not "hidden_connections" in self.stateData: self.nodeEditor.setHiddenConnectionsFromList([]) return connectionList = [] for el in self.stateData["hidden_connections"]: topId = el["topLayerId"] topIdx = el["topBlobIdx"] bottomId = el["bottomLayerId"] bottomIdx = el["bottomBlobIdx"] connectionList.append(( topId, topIdx, bottomId, bottomIdx )) self.nodeEditor.setHiddenConnectionsFromList(connectionList) def connectionsHiddenStateChanged(self, listOfConnections): """ Gets called when a connection gets hidden/shown. listOfConnections is a list of tupels (topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex) """ hidden_connections = [] for (topid, topidx, bottomid, bottomidx) in listOfConnections: hidden_connections.append({ "topLayerId": topid, "topBlobIdx": topidx, "bottomLayerId": bottomid, "bottomBlobIdx": bottomidx }) self.historyWriter.makeEntry(lambda stateData: stateData.giveProperty("hidden_connections", hidden_connections)) def refreshItems(self): # Clear everything, if a previous network has been opened self.nodeEditor.clearAll() self.dockLayerProperties.clearProperties() self.dockElementActiveLayers.clearTable() for id, layer in self.getLayerDict().iteritems(): # todo: Set an ID for each layer in input dictionary # Create a new node item in the view and set focus to it self.nodeEditor.addLayer(id, layer) # Create a new entry in the active layers list self.dockElementActiveLayers.addItem(id, layer["parameters"]["name"], self.network["layerOrder"].index(id), layer["parameters"]["type"]) # Add one to the count of all layers in the application self.layerCount += 1 def addBottomBlob(self, layerID): 'Adds a bottom entry to the layer with the given id' def intern(stateDict): network = stateDict["network"] networkHelper = helper.DictHelper(network) layerParams = networkHelper.layerParams(layerID) if "bottom" not in layerParams: layerParams.giveProperty("bottom",[]) layerParams["bottom"].pushBack("") # TODO: Instead update in onStateUpdate #self.nodeEditor.addBottomBlob(layerID, "") # update property dock self.dockLayerProperties.updateDock() self.historyWriter.makeEntry(intern) def removeBottomBlob(self, layerID, blobIndex): def intern(stateDict): network = stateDict["network"] networkHelper = helper.DictHelper(network) self.nodeEditor.disconnectBottomBlob(layerID, blobIndex) del networkHelper.layerParameter(layerID, "bottom")[blobIndex] # TODO: Instead update in onStateUpdate # self.nodeEditor.removeBottomBlob(layerID, blobIndex) self.updateOrder() # update property dock self.dockLayerProperties.updateDock() self.historyWriter.makeEntry(intern) def addTopBlob(self, layerID, blobName): 'Adds a top entry to the layer with the given id' def intern(stateDict): network = stateDict["network"] networkHelper = helper.DictHelper(network) layerParams = networkHelper.layerParams(layerID) if "top" not in layerParams: layerParams.giveProperty("top", []) layerParams["top"].pushBack(blobName) # TODO: Instead update in onStateUpdate #self.nodeEditor.addTopBlob(layerID, blobName) # update property dock self.dockLayerProperties.updateDock() self.historyWriter.makeEntry(intern) def renameTopBlob(self, layerID, blobIndex, newName): def intern(stateDict): network = stateDict["network"] networkHelper = helper.DictHelper(network) networkHelper.layerParams(layerID)["top"][blobIndex] = newName # TODO: Instead update in onStateUpdate #self.nodeEditor.renameTopBlob(layerID, blobIndex, newName) # update property dock self.dockLayerProperties.updateDock() self.historyWriter.makeEntry(intern) def renameBottomBlob(self, layerID, blobIndex, newName): def intern(stateDict): network = stateDict["network"] networkHelper = helper.DictHelper(network) networkHelper.layerParams(layerID)["bottom"][blobIndex] = newName # TODO: Instead update in onStateUpdate #self.nodeEditor.renameBottomBlob(layerID, blobIndex, newName) # update property dock self.dockLayerProperties.updateDock() self.historyWriter.makeEntry(intern) def removeTopBlob(self, layerID, blobIndex): def intern(stateDict): network = stateDict["network"] networkHelper = helper.DictHelper(network) self.nodeEditor.disconnectTopBlob(layerID, blobIndex) del networkHelper.layerParams(layerID)["top"][blobIndex] # TODO: Instead update in onStateUpdate #self.nodeEditor.removeTopBlob(layerID, blobIndex) self.updateOrder() # update property dock self.dockLayerProperties.updateDock() self.historyWriter.makeEntry(intern) def connectLayers(self, topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex): def intern(stateDict): network = stateDict["network"] networkHelper = helper.DictHelper(network) blobName = networkHelper.layerParams(topLayerID)["top"][topBlobIndex] networkHelper.layerParams(bottomLayerID)["bottom"][bottomBlobIndex] = blobName # TODO: Instead update in onStateUpdate self.nodeEditor.updateLayerData(bottomLayerID) self.nodeEditor.createConnection(topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex) # self.updateNet() self.updateOrder() # update property dock self.dockLayerProperties.updateDock() self.historyWriter.makeEntry(intern) def disconnectLayer(self, bottomLayerID, bottomBlobIndex): def intern(stateDict): network = stateDict["network"] networkHelper = helper.DictHelper(network) networkHelper.layerParams(bottomLayerID)["bottom"][bottomBlobIndex] = unicode("") # TODO: Instead update in onStateUpdate self.nodeEditor.updateLayerData(bottomLayerID) #self.updateNet() self.updateOrder() # update property dock self.dockLayerProperties.updateDock() self.historyWriter.makeEntry(intern) def isValidLayerType(self, type): if type in self.caffeInfo.availableLayerTypes(): return True return False def __historyEntry(self, uri, reason): self.historyWriter.makeEntry(None) def setState(self, statedict): """ Sets a new stateData and refresh the gui. statedict has to be from type dict and will be wrapped with PropertyData through this function. """ # Disconnect old signals if self.stateData: self.stateData.someChildPropertyChanged.disconnect(self.onStateUpdate) self.stateData.someChildPropertyChanged.disconnect(self.__historyEntry) # Build PropertyData self.stateData = buildStateData(statedict) # Refresh everynthing self.__connectReactiveThings() self.refreshItems() self.refreshConnections() self.nodeEditor.applyLayerPositionDict(self.stateData["position"]) self.nodeEditor.setSelectedLayers(self.stateData["selection"]) self.updateHiddenConnectionsInNodeEditor() if self.stateData: self.historyWriter = HistoryWriting(self.stateData, self.makeHistory) def intern(stateData): self.networkHelper = helper.DictHelper(stateData["network"]) self.dockLayerProperties.setTabs(stateData["selection"]) # Reconnect self.stateData.someChildPropertyChanged.connect(self.__historyEntry) self.stateData.someChildPropertyChanged.connect(self.onStateUpdate) if "solver" in self.stateData: self.setSolver(stateData["solver"]) else: self.setSolver(None) self.historyWriter.entryWithoutHistory(intern) def makeHistory(self): """ Make an entry in history. Note: If possible use historyWriter.makeEntry instead as it avoid making unnecessary entries. """ #if self.__historyDepth > 0: # return boring = self.stateData.getBoringDict() network = self.stateData["network"].getBoringDict() valid, msg = helper.isNetDictValid(network) if not valid: Log.error( "WARNING!!!! Network-Dictionary is not valid:" +" " + msg +"\nInfo: Current Dictionary looks like this:\n" + str(self.network.getBoringDict()), self.callerId ) self._historymanager.insertFunc(lambda insert: insert(copy.deepcopy(self.stateData.getBoringDict()))) netids = set(network["layerOrder"]) posids = set(self.stateData["position"].getBoringDict()) if netids != posids: Log.error("WARNING!!! NetworkManager has invalid History-State",self.callerId) for id in netids.difference(posids): Log.error("Id {} is in net, but has no position".format(id),self.callerId) for id in posids.difference(netids): Log.error("Id {} has position, but is not in net".format(id),self.callerId) Log.log("Info: Current State-Dictionary looks like this:",self.callerId) Log.log(str(self._historymanager.currentState()),self.callerId) self.stateChanged.emit() def _refreshStateFromHistory(self): """ Rebuild State of network, selection, position from latest point in history """ state = copy.deepcopy(self._historymanager.currentState()) def updateFun(): self.setState(state) #self.setNetwork(state["network"]) #self.refreshItems() #self.refreshConnections() #self.setSelectionList(state["selection"]) #self.setLayerPositionDictionary(state["position"]) #self.setSolver(state["solver"]) #if not state is None: self._historymanager.lockHistory(updateFun) self.stateChanged.emit() def canUndo(self): return self._historymanager.canUndo() def undo(self): self._historymanager.undo() self._refreshStateFromHistory() def canRedo(self): return self._historymanager.canRedo() def redo(self): self._historymanager.redo() self._refreshStateFromHistory() def isModified(self): """ Return True iff anything changes since last reset of changes """ return self.__lastModified != self._historymanager.position def resetModifiedFlag(self): """ Reset the change flag, so the program things nothing has changes since last save. Then there is no need to ask the user for example. """ self.__lastModified = self._historymanager.position self.modifiedChanged.emit(self.isModified()) def setModifiedFlag(self): """ Set the change flag, so the program things something has changes since last save. So there is need to ask the user for example. """ self.__lastModified = -1 self.modifiedChanged.emit(self.isModified()) def openNetworkFromFile(self, filePath): with open(filePath, 'r') as file: inputString = file.read() self.openNetworkFromString(inputString) def saveNetworkToFile(self, filePath): with open(filePath, 'w') as file: file.write(saver.saveNet(self.network)) def openNetworkFromString(self, inputString, clearHistory = True): if clearHistory: self._historymanager.clear() def intern(stateData): self.setNetwork(loader.loadNet(inputString)) #self.refreshItems() #self.refreshConnections() # Rearrange the layers self.nodeEditor.rearrangeNodes() self.updateOrder() self.historyWriter.makeEntry(intern) def setNetwork(self, network): def intern(stateData): stateData.signalController().lock() stateData.giveProperty("network", network) stateData["position"].setValue(dict([(id, [0.0,0.0]) for id in network["layerOrder"]])) stateData["selection"].setValue([]) stateData.signalController().unlock() self.historyWriter.entryWithoutHistory(intern) self.networkHelper = helper.DictHelper(self.stateData["network"]) def __connectReactiveThings(self): if self.dockElementActiveLayers: self.dockElementActiveLayers.setReactiveState(self.stateData) def openSolverFromFile(self, filePath): with open(filePath, 'r') as file: inputString = file.read() self.openSolverFromString(inputString) def openSolverFromString(self,inputString, clearHistory=False): if clearHistory: self._historymanager.clear() def intern(stateDict): stateDict.giveProperty("solver",loader.loadSolver(inputString)) self.historyWriter.makeEntry(intern) def setSolver(self, solver): # prevent manipulation of solver constraints if solver: solver = ensureSolverConstraints(solver) # save data self.dockSolverProperties().setSolver(solver) self.historyWriter.makeEntry(None) def setLayerPositionDictionary(self, posDict): self.nodeEditor.applyLayerPositionDict(posDict) def getLayerPositionDictionary(self): return self.nodeEditor.createLayerPositionDict() def setControlHold(self, bool): self.dockElementActiveLayers.setControlHold(bool) self.nodeEditor.setControlHold(bool) def setShiftHold(self, bool): self.dockElementActiveLayers.setShiftHold(bool) # Node Editor does not support shift functionality at this point # self.nodeEditor.setShiftHold(bool) def getLayerDict(self): return self.network["layers"] def getLayer(self, layerId): return self.networkHelper.layerPerId(layerId) def getLayerCount(self): return self.layerCount def getLayerTypeCount(self, type): return self.dicLayerTypeCounter[type] def setStateDictionary(self, dictionary, clearHistory = False): """ Sets the dictionary of docks state and update gui """ if clearHistory: self._historymanager.clear() self._historymanager.insertFunc(lambda inserter: inserter(dictionary)) self._refreshStateFromHistory() def getStateDictionary(self): """ Returns a dictionary which represents the current state of the project """ #return self._historymanager.currentState() return self.stateData.getBoringDict() def getToolTip(self, id): 'Returns the standard tooltip for an item representing the layer given by id' # Check if name exists, since it is optional network = self.stateData["network"].getBoringDict() if "name" in network["layers"][id]["parameters"]: tooltip = network["layers"][id]["parameters"]["name"] + "\n" # Add type and other parameters tooltip += "Type: " + network["layers"][id]["parameters"]["type"] + "\n" tooltip += "\n" tooltip += "Parameters: " + "\n" # Add all other parameters to the tooltip for key in network["layers"][id]["parameters"]: if not key in ["name", "type", "top", "bottom"]: tooltip += self._getToolTipHelper(0, key, network["layers"][id]["parameters"][key]) tooltip += "\n" # Add Top and Bottom to the bottom of the tooltip if "top" in network["layers"][id]["parameters"]: tooltip += self._getToolTipHelper(0, "top", network["layers"][id]["parameters"]["top"]) if "bottom" in network["layers"][id]["parameters"]: tooltip += self._getToolTipHelper(0, "bottom", network["layers"][id]["parameters"]["bottom"]) return tooltip def _getToolTipHelper(self, tabs, key, data): # Initialize the partial tooltip as a string containing the given key toolTipPart = tabs * " " toolTipPart += key # Use recursion if the data is a dict if isinstance(data, dict): toolTipPart += " { \n" for innerKey in data: # Add a tabulator as starter string for inner parts of the dict toolTipPart += self._getToolTipHelper(tabs + 1, innerKey, data[innerKey]) toolTipPart += tabs * " " + "} \n" elif isinstance(data, list): # Set up a special case for the representation of top and bottom lists if (key == "top") | (key == "bottom"): toolTipPart += ": " for i in range(0, len(data)): # The string may contain some spacial characters which can not be displayed in unicode. # The encode method returns a bytes representation of the unicode string and ignores errors. # Then the decode method convertes the string using the utf-8 codec. # This string can be displayed even if it is containing special characters. toolTipPart += "\t" + data[i].encode('utf-8', 'ignore').decode('utf-8') + "\n" else: # Proceed based on the type of the list elements, catch empty lists if len(data) > 0: # Assumption: All elements of the parameter list are of the same type # always true in current network definitions. If that is changed, this code must be revised if isinstance(data[0], int) | isinstance(data[0], float): # In case of simple int or float values, just concatenate the elements into one line of string toolTipPart += ": " for i in range(0, len(data)): toolTipPart += str(data[i]) + " " toolTipPart += "\n" elif isinstance(data[0], dict): # In case of more complex element types, create a new subset of values containing the elements toolTipPart += " [ \n" # Type is dictionary, use recursion to get all values from that dicationary for i in range(0, len(data)): for key in data[i]: toolTipPart += self._getToolTipHelper(tabs + 1, key, data[i][key]) toolTipPart += tabs * " " + "] \n" elif isinstance(data[0], list): # In case of more complex element types, create a new subset of values containing the elements toolTipPart += " [ \n" # Type is list, user recursion to get all values from that list for i in range(0, len(data)): toolTipPart += self._getToolTipHelper(tabs, "", data[i]) toolTipPart += tabs * " " + "] \n" elif isinstance(data, str) or isinstance(data, unicode): # Add string by simple concatenation toolTipPart += ": " + "\"" + data + "\"" + "\n" elif isinstance(data, int) or isinstance(data, float): # Add int by simple concatenation toolTipPart += ": " + str(data) + "\n" return toolTipPart def updateToolTip(self, layerID): self.nodeEditor.updateTooltip(layerID) def duplicateLayers(self, layerIDs, point=None): """ Duplicates the layer referenced by layerIDs. If these are interconnected, then the connection is duplicated as well. """ # Get the current data state = self.stateData.getBoringDict() network = state["network"] nethelper = helper.DictHelper(network) # Check if a layer have been moved self.checkLayersForMovement(self.nodeEditor.getNodeIds()) layerIDs = self._checkValidIDs(layerIDs, nethelper) newLayerNames = self._generateNewLayerNames(layerIDs, nethelper) newIDs = self._generateNewLayers(layerIDs, newLayerNames, nethelper, state) self._addBlobs(layerIDs, newIDs, newLayerNames, nethelper) self._addConnections(layerIDs, newIDs) self._translateLayers(layerIDs, newIDs, point) def _checkValidIDs(self, layerIDs, nethelper): """ check if every id is valid, returns a list of all valid IDs in layerIDs. """ tempLayerIds = [] for index, id in enumerate(layerIDs): # encapsulate getting the layer per id in a try block try: nethelper.layerPerId(id) tempLayerIds.append(id) except KeyError: Log.error( "Could not copy element with id " + str(id), self.callerId ) return tempLayerIds def _generateNewLayerNames(self, layerIDs, nethelper): """ This function generates new Layer names based on a list of layerIds to be duplicated and the already given names in the network, to prevent duplicate names or unreasonable enumeration. The names are generated as followed: 1. Check if the layer to be copied ends in a number. If it does: add 1 to this number If it does not: append #1 at the end 2. Increase the last number until the string does not match any given layer name in the network or of the new names already generated in the loop. """ # get all already distributed names, including the new generated names in the following loop allLayerNames = nethelper.nameOfLayers() # list to save the new generated Names newNames = [] for id in layerIDs: # Get the old layer and its name oldLayer = nethelper.layerPerId(id) oldName = oldLayer["parameters"]["name"] # Check if oldName ends with a number, then strips the number # baseName stored the name without the number at the end baseName = re.sub('[0-9]+$', '', oldName) # check if there was a number attached at the end, store it in counter try: counter = int(oldName[len(baseName):]) counter += 1 except ValueError: # Set up new counter, add a '#' at the basename to indicate a new enumeration counter = 1 baseName = baseName + "#" newName = "{}{}".format(baseName, counter) # Check if there already is a name with that number, if yes, add one to counter for new name while newName in allLayerNames: counter += 1 newName = "{}{}".format(baseName, counter) # Append the chosen Name allLayerNames.append(newName) newNames.append(newName) return newNames def _generateNewLayers(self, layerIDs, newLayerNames, nethelper, state): """ Given a list of given layerIDs, and a list of new layer names, duplicate the layers in the first list by using the names of the latter. The i-th elements of the two lists correspond to one another. Returns a list of the IDs of the new layers. """ # Get the current data positions = state["position"] network = state["network"] newIDs = [] for index in range(len(layerIDs)): id = layerIDs[index] newName = newLayerNames[index] # Get the old layer and its name oldLayer = nethelper.layerPerId(id) # Create a new layer with the type and parameters of the old layer newLayer = { "type": oldLayer["type"], "parameters": copy.deepcopy(oldLayer["parameters"]) } # Overwrite the values for name, top and bottom newLayer["parameters"]["name"] = newName newLayer["parameters"]["top"] = [] newLayer["parameters"]["bottom"] = [] # Generate random new ID and set the new idx newId = str(uuid.uuid4()) idx = network["layerOrder"].index(id) + 1 # Add the new layer to the network network["layers"][newId] = newLayer network["layerOrder"].insert(idx, newId) newIDs.append(newId) # Set the new position for each copied element, also replace selection with duplicates newPositions = positions.copy() for i in range(0, len(layerIDs)): (x, y) = positions[layerIDs[i]] newPositions[newIDs[i]] = (x + 20, y + 20) # Update the network manager self.setStateDictionary({ "position": newPositions, "selection": newIDs, "network": network, "solver": state["solver"] }) return newIDs def _addBlobs(self, layerIDs, newIDs, newLayerNames, nethelper): """ Add bottom and uniquely named top blobs to the layers referenced by newIDs, by using the blobs in the layers referenced by layerIDs as a template. The naming of the top blobs goes as follows: 1. If the name of the blob in the old layer matches the name of the layer: The new blob gets the name of the new layer (which always ends in a number). Else: If the old blob end with a number: Add one to that number Else: Append "#1" to the name 2. Increase the last number until the string does not match any given layer name in the network or of the new names already generated in the loop. """ allTopNames = nethelper.nameOfTopBlobs() # Add the right top and bottom blobs to all new layers for index in range(0, len(layerIDs)): newLayerName = newLayerNames[index] oldLayer = nethelper.layerPerId(layerIDs[index]) oldLayerName = oldLayer["parameters"]["name"] newID = newIDs[index] oldID = layerIDs[index] # Create the right amount of bottom blobs for the new layer if "bottom" in nethelper.layerPerId(oldID)["parameters"]: for j in range(0, len(nethelper.layerPerId(oldID)["parameters"]["bottom"])): self.addBottomBlob(newID) # Create the right amount of top blobs for the new layer, using the names from the old one if "top" in nethelper.layerPerId(oldID)["parameters"]: for j in range(0, len(nethelper.layerPerId(oldID)["parameters"]["top"])): topBlobName = nethelper.layerPerId(oldID)["parameters"]["top"][j] # Check if top blob name matches the layername if topBlobName == oldLayerName: # check if newLayerName is candidate for top blob name candidateName = newLayerName if candidateName in allTopNames: candidateName += "#1" # Since newName ends with a number (either as an iteration based on a previous name or by adding # #1), we can assert here that candidateName ends with a number else: candidateName = topBlobName # Check if there is no number at the end of the top blob name baseName = re.sub('[0-9]+$', '', candidateName) if baseName >= candidateName: candidateName += '#1' # Set up the current counter, either by setting it to 1 or by getting the suffix, if it is a number baseName = re.sub('[0-9]+$', '', candidateName) try: counter = int(candidateName[len(baseName):]) except ValueError: # Set up new counter, add a '#' at the basename to indicate a new enumeration counter = 1 baseName = baseName + "#" newTopName = "{}{}".format(baseName, counter) # increase pending number of candidateName, until it is unique. while newTopName in allTopNames: baseName = re.sub('[0-9]+$', '', newTopName) counter += 1 newTopName = "{}{}".format(baseName, counter) self.addTopBlob(newIDs[index], newTopName) # Add new name to the list to prevent duplicates allTopNames.append(newTopName) def _addConnections(self, layerIDs, newIDs): """ Check if copied layers were interconnected, if yes, add these connections to the new layers. """ for i in range(0, len(layerIDs)): # Get the corresponding layer item nodeItem = self.nodeEditor.getNodes()[layerIDs[i]] for j in range(0, len(nodeItem.getBottomConnectors())): for k in range(0, len(nodeItem.getBottomConnectors()[j].getConnections())): for n in range(0, len(layerIDs)): if nodeItem.getBottomConnectors()[j].getConnections()[k].getTopConnector() in \ self.nodeEditor.getNodes()[layerIDs[n]].getTopConnectors(): self.connectLayers(newIDs[n], nodeItem.getBottomConnectors()[j].getConnections()[ k].getTopConnector().getIndex(), newIDs[i], nodeItem.getBottomConnectors()[j].getIndex()) def _translateLayers(self, layerIDs, newIDs, point): """ If a position has been given by user, translate the new layers. """ if point != None: # Find the smallest x coordinate from the old layers minX = sys.maxint nodes = self.nodeEditor.getNodes() for id in layerIDs: if nodes[id].x() < minX: minX = nodes[id].x() correspondingY = nodes[id].y() # Translate all new elements accordingly for i in range(0, len(newIDs)): nodes[newIDs[i]].setPos(nodes[layerIDs[i]].x() - minX + point.x(), nodes[layerIDs[i]].y() - correspondingY + point.y()) # todo: IMPORTANT how do i save these positions so that they won't get reversed? gui/prototxt_editor/__init__.py0000664ڊ00111170000000000013243274037017142 0ustar klemmso0cvprgui/prototxt_editor/editor_search_bar.py0000664ڊ00111170000000571113243274037021060 0ustar klemmso0cvprfrom PyQt5.QtWidgets import QLineEdit, QPlainTextEdit from PyQt5.QtCore import Qt from PyQt5.QtGui import QTextCursor, QTextCharFormat, QTextDocument, QTextCursor, QColor, QFontMetrics class SearchBar(QLineEdit): def __init__(self, editor, parent=None): QLineEdit.__init__(self, parent) self.editor = editor self.setAttribute(Qt.WA_TintedBackground) self.setHidden(True) self.defaultstyle = self.styleSheet() self.textChanged.connect(self.onTextChange) def disp(self): '''display the searchbar''' self.setHidden(False) self.setFocus() def hide(self): '''hide the searchbar''' self.setHidden(True) def onF3(self): '''open or jump to next''' if self.isHidden(): self.disp() else: self.findText() def getText(self): return self.text().lower() def keyPressEvent(self, event): # find the text on enter if event.key() == Qt.Key_Return or event.key() == Qt.Key_Enter: self.first = True self.findText() return # clear or hide the searchbar if event.key() == Qt.Key_Escape: if self.text() == "": self.hide() self.editor.setFocus() else: self.clear() return # jump to next if event.key() == Qt.Key_F3: self.onF3() self.editor.setFocus() return super(SearchBar, self).keyPressEvent(event) def findText(self): '''search the prototxt_editor for text''' text = self.getText() if text != "": self.onTextChange() doc = self.editor.document() # first search from cursor position hig = doc.find(text, self.editor.textCursor()) if not hig.isNull(): self.editor.setTextCursor(hig) return # if not found: search from start hig = doc.find(text, QTextCursor(doc)) if not hig.isNull(): self.editor.setTextCursor(hig) return # not found, make red self.setStyleSheet("background:#FFaaaa;") def updatePosition(self): '''set position of searchbar''' rect = self.editor.rect() dx = 0 if self.editor.verticalScrollBar().isVisible(): dx = self.editor.verticalScrollBar().width() mx = rect.width() - self.width() - dx - 1 self.move(mx, 2) def resizeToFont(self): '''set font of searchbar''' font = self.font() fm = QFontMetrics(font) # adjust height to fontsize self.setFixedSize(300, fm.height() * 1.2) def paintEvent(self, event): self.updatePosition() super(SearchBar, self).paintEvent(event) def onTextChange(self): # reset the style back to default self.setStyleSheet(self.defaultstyle) gui/prototxt_editor/editor_side_bar.py0000664ڊ00111170000000060213243274037020531 0ustar klemmso0cvprfrom PyQt5.QtWidgets import QWidget from PyQt5.QtCore import QSize class Sidebar(QWidget): # constructor class for sidebar def __init__(self, parent): QWidget.__init__(self, parent) self.editor = parent def sizeHint(self): return QSize(self.editor.sidebarWidth(), 0) def paintEvent(self, event): self.editor.sidebarPaintEvent(event) gui/prototxt_editor/editor_slider.py0000664ڊ00111170000000316013243274037020245 0ustar klemmso0cvprfrom PyQt5.QtWidgets import QWidget, QSlider, QHBoxLayout from PyQt5.QtCore import QMargins from PyQt5.QtCore import QObject, pyqtSignal class EditorSlider(QWidget): # widget containig the slider # direct subclassing of QSlider leads to errors valueChanged = pyqtSignal(int) def __init__(self, vmin, vcur, vmax, parent): QWidget.__init__(self, parent) self.editor = parent self.setFocusPolicy(0) #layouts self.layout = QHBoxLayout(self) self.layout.setContentsMargins(0, 0, 0, 0) #add the slider self.slider = QSlider() self.slider.setRange(vmin, vmax) self.slider.setValue(vcur) self.slider.setOrientation(1) self.slider.setFocusPolicy(0) self.layout.addWidget(self.slider) self.slider.valueChanged.connect(self._passThrough) def updatePosition(self): '''set the position of the slider''' rect = self.editor.rect() dy = 0 dx = 0 #check for scrollbars if self.editor.verticalScrollBar().isVisible(): dx = self.editor.verticalScrollBar().width() if self.editor.horizontalScrollBar().isVisible(): dy = self.editor.horizontalScrollBar().height() mx = rect.width() - self.width() - dx - 5 my = rect.height() - self.height() - dy - 5 self.move(mx, my) def paintEvent(self, event): self.updatePosition() super(EditorSlider, self).paintEvent(event) def _passThrough(self, value): '''pass through the valueChanged signal of the QSlider''' self.valueChanged.emit(value) gui/prototxt_editor/editor_syntax_highlight.py0000664ڊ00111170000001142313243274037022341 0ustar klemmso0cvprfrom PyQt5.QtGui import QSyntaxHighlighter, QColor, QTextCharFormat, QFont from PyQt5.QtCore import QRegExp import backend.caffe.proto_info as info class EditorSyntaxHighlighter(QSyntaxHighlighter): ''' Based on http://doc.qt.io/qt-5/qtwidgets-richtext-syntaxhighlighter-example.html https://wiki.python.org/moin/PyQt/Python%20syntax%20highlighting ''' def __init__(self, document): QSyntaxHighlighter.__init__(self, document) self.rules = [] self.enums = [] self.params = [] self.paramgroups = [] self._setupRules() def _setupRules(self): # strings self.tri_single = self._createRule('"', "darkorange", "", 1) self.tri_double = self._createRule("'", "darkorange", "", 2) # numbers self.rules.append(self._createRule('\\b\\d+(\\.\\d+)?(e(\\-|\\+?)\\d+)?\\b', "darkmagenta")) # get the parameters from caffe self._extractParams() # enums for e in self.enums: self.rules.append(self._createRule("\\b" + e + "\\b", "cornflowerblue", "bold")) # parameter for p in self.params: self.rules.append(self._createRule("\\b" + p + "\\b", "darkcyan", "")) # parameter groups for g in self.paramgroups: self.rules.append(self._createRule("\\b" + g + "\\b", "indigo", "italic")) def highlightBlock(self, p_str): '''apply rules to block''' # for each rule for exp, n, form in self.rules: # find in text index = exp.indexIn(p_str, 0) while index >= 0: index = exp.pos(n) l = len(exp.cap(n)) # change format till end of string self.setFormat(index, l, form) # search for next index = exp.indexIn(p_str, index + l) self.setCurrentBlockState(0) # multiline handling for strings multiline = self.match_multiline(p_str, *self.tri_single) if not multiline: self.match_multiline(p_str, *self.tri_double) def match_multiline(self, text, exp, n, style): '''handling of multiline matches''' # start from beginning start = 0 add = 0 # if not previously found search anew if self.previousBlockState() != n: start = exp.indexIn(text) add = exp.matchedLength() while start >= 0: # find the end end = exp.indexIn(text, start + add) if end >= add: length = end - start + add + exp.matchedLength() self.setCurrentBlockState(0) else: self.setCurrentBlockState(n) length = len(text) - start + add # set style self.setFormat(start, length, style) # find next start = exp.indexIn(text, start + length) if self.currentBlockState() == n: return True else: return False def _createRule(self, regex, color="black", style="", index=0): return QRegExp(regex), index, self._format(color, style) def _format(self, color, style): # refer to https://www.w3.org/TR/SVG/types.html#ColorKeywords for colors # create color _c = QColor() _c.setNamedColor(color) # create font _f = QTextCharFormat() _f.setForeground(_c) # set font style if 'bold' in style: _f.setFontWeight(QFont.Bold) if 'italic' in style: _f.setFontItalic(True) return _f def _extractParams(self): '''open caffe meta information''' info.resetCaffeProtoModulesvar() info.CaffeMetaInformation().updateCaffeMetaInformation() allSolver = info.CaffeMetaInformation().availableSolverTypes() for solver in allSolver: self._iterParams(allSolver[solver].parameters()) allLayer = info.CaffeMetaInformation().availableLayerTypes() for layer in allLayer: self._iterParams(allLayer[layer].parameters()) def _iterParams(self, dict): '''recursive param search''' for param in dict: obj = dict[param] # sort if obj.isParameterGroup(): if param not in self.paramgroups: self.paramgroups.append(param) self._iterParams(obj.parameter()) elif obj.isEnum(): if param not in self.params: self.params.append(param) values = obj.availableValues() for v in values: if v not in self.enums: self.enums.append(v) else: if param not in self.params: self.params.append(param) gui/prototxt_editor/editor_widget.py0000664ڊ00111170000000744713243274037020262 0ustar klemmso0cvprfrom PyQt5.QtCore import pyqtSignal, Qt from PyQt5.QtWidgets import QDialog, QPushButton, QMessageBox, QAction, QGridLayout from gui.prototxt_editor import text_edit class EditorWidget(QDialog): '''This Editor enables text-based changes of Prototxt files. Connect to the sinal sgSave to get the savedata''' def __init__(self, syntax = True, parent=None): # main_window settings QDialog.__init__(self, parent) self.setModal(True) self.resize(600, 600) self.setMinimumWidth(500) self.layout = QGridLayout(self) # buttons self.pbSave = QPushButton("Save") self.layout.addWidget(self.pbSave, 0, 0) self.pbSaveClose = QPushButton("Save && Close") self.layout.addWidget(self.pbSaveClose, 0, 1) self.pbClose = QPushButton("Close") self.layout.addWidget(self.pbClose, 0, 2) # the prototxt_editor itself if syntax: self.textwidget = text_edit.EditorSyntax() else: self.textwidget = text_edit.Editor() self.layout.addWidget(self.textwidget, 1, 0, 1, 3) self.containsChanges = False # connections self.pbSave.clicked.connect(self._onSave) self.pbSaveClose.clicked.connect(self._onSaveClose) self.pbClose.clicked.connect(self._onClose) self.textwidget.textChanged.connect(self._onChange) # actions self.saveaction = QAction(self) self.saveaction.setShortcut('Ctrl+S') self.addAction(self.saveaction) self.saveaction.triggered.connect(self._onSave) self.quitaction = QAction(self) self.quitaction.setShortcut('Ctrl+Q') self.addAction(self.quitaction) self.quitaction.triggered.connect(self._onClose) sgSave = pyqtSignal(['QString']) sgClose = pyqtSignal() def closeEvent(self, event): '''if closing with changes ask if changes should be discarded''' if self.containsChanges: if self._showWarning(): self.sgClose.emit() event.accept() else: event.ignore() else: self.sgClose.emit() event.accept() def getText(self): '''convert prototxt_editor to text''' return self.textwidget.toPlainText() def setText(self, insertText): '''replace the current content of the prototxt_editor with the new one''' self.textwidget.setPlainText(insertText) self.containsChanges = False def clear(self): '''clear the prototxt_editor''' self.textwidget.clear() self.textwidget.setFocus() self.containsChanges = False def _onSave(self): '''emit save sinal''' self.sgSave.emit(self.getText()) self.containsChanges = False def _onSaveClose(self): '''save and close''' self._onSave() self.close() def _onClose(self): self.close() def _onChange(self): '''set echange flag''' self.containsChanges = True def _showWarning(self): '''show message box if changes are discarded''' if QMessageBox.warning( self, "Warning", "Close without saving?", QMessageBox.Yes, QMessageBox.No) \ == QMessageBox.Yes: return True else: return False # workaround # previously pressing the ESC key would close the main_window and always discard changes # therefor catch the key and close the main_window manualy def keyPressEvent(self, event): if event.key() == Qt.Key_Escape: self.close() return super(EditorWidget, self).keyPressEvent(event) def disableEditing(self, disable): ''' Set textwidget in read-only mode ''' self.textwidget.setReadOnly(disable) gui/prototxt_editor/text_edit.py0000664ڊ00111170000001216713243274037017415 0ustar klemmso0cvprfrom PyQt5.QtCore import QRect, Qt from PyQt5.QtGui import QFont, QFontMetrics, QColor, QPainter, QFontDatabase from PyQt5.QtWidgets import QPlainTextEdit from gui.prototxt_editor import editor_side_bar, editor_search_bar, editor_syntax_highlight from gui.prototxt_editor import editor_slider class Editor(QPlainTextEdit): '''based on http://doc.qt.io/qt-5/qtwidgets-widgets-codeeditor-example.html''' def __init__(self, parent=None): QPlainTextEdit.__init__(self, parent) # default settings self._fontsize = 12 self._fmin = 8 self._fmax = 30 # setup of prototxt_editor self.setLineWrapMode(0) # loading of subwidgets self.sidebar = editor_side_bar.Sidebar(self) # line numbers self.searchbar = editor_search_bar.SearchBar(self, self) # search self.slider = editor_slider.EditorSlider(self._fmin, self._fontsize, self._fmax, self) # fontsize slider # conntions self.slider.valueChanged.connect(self.setFontsize) self.blockCountChanged.connect(self._updateSidebarWidth) self.updateRequest.connect(self._updateSidebar) # show self._updateFont() self._updateSidebarWidth() self.sidebar.show() def setFontsize(self, value): '''set the fontsize''' if self._fmin <= value <= self._fmax: self._fontsize = value self._updateFont() def sidebarWidth(self): '''calculate the width of the sidebar''' # get the number of digits displayed digits = 1 maxd = max(1, self.blockCount()) while maxd >= 10: maxd /= 10 digits += 1 # calc the width space = 3 + QFontMetrics(self._font).width(*'9') * digits return space def _updateSidebarWidth(self, arg=0): '''refresh the width of the sidebar''' self.setViewportMargins(self.sidebarWidth(), 0, 0, 0) def _updateSidebar(self, rect, dy): '''realign the sidebar''' if dy is not 0: self.sidebar.scroll(0, dy) else: self.sidebar.update(0, rect.y(), self.sidebarWidth(), rect.height()) if rect.contains(self.viewport().rect()): self._updateSidebarWidth() def sidebarPaintEvent(self, event): '''draw the sidebar''' # define the colors _c1 = QColor() _c1.setNamedColor("lightgrey") _c2 = QColor() _c2.setNamedColor("darkgrey") _c3 = QColor() _c3.setNamedColor("black") # setup the painter painter = QPainter(self.sidebar) painter.fillRect(event.rect(), _c1) painter.setFont(self._font) # calculate the lines block = self.firstVisibleBlock() blocknumber = block.blockNumber() btop = self.blockBoundingGeometry(block).translated(self.contentOffset()).top() bbottom = btop + self.blockBoundingRect(block).height() # write the line numbers while block.isValid() and btop <= event.rect().bottom(): if block.isVisible() and bbottom >= event.rect().top(): number = str(blocknumber + 1) # change color for "current" line if self.textCursor().block().blockNumber() != blocknumber: painter.setPen(_c2) else: painter.setPen(_c3) painter.drawText(0, btop, self.sidebar.width(), QFontMetrics(self._font).height(), 2, number) block = block.next() btop = bbottom bbottom = btop + self.blockBoundingRect(block).height() blocknumber += 1 def resizeEvent(self, event): '''update all subwidgets on resize of the main_window''' super(Editor, self).resizeEvent(event) self.slider.updatePosition() self.searchbar.updatePosition() c = self.contentsRect() self.sidebar.setGeometry(QRect(c.left(), c.top(), self.sidebarWidth(), c.height())) def _updateFont(self): '''set the font and fontsize across all widgets''' self._font = QFontDatabase.systemFont(QFontDatabase.FixedFont) # QFont("DejaVu Sans Mono", self._fontsize) self._font.setPointSize(self._fontsize) # Change Fontsize if using QFontDatabase self.setFont(self._font) self.searchbar.setFont(self._font) self.searchbar.resizeToFont() self.update() def keyPressEvent(self, event): # search if event.key() == Qt.Key_F: if event.modifiers() & Qt.ControlModifier: self.searchbar.disp() return if event.key() == Qt.Key_F3: self.searchbar.onF3() return # close if event.key() == Qt.Key_Escape: if not self.searchbar.isHidden(): self.searchbar.hide() return super(Editor, self).keyPressEvent(event) class EditorSyntax(Editor): def __init__(self, parent = None): Editor.__init__(self, parent) self.highlight = editor_syntax_highlight.EditorSyntaxHighlighter(self.document()) # syntax highlighting self.highlight._extractParams() gui/__init__.py0000664ڊ00111170000000000013243274037013671 0ustar klemmso0cvprgui/caffepath_dialog.py0000664ڊ00111170000000526513243274037015414 0ustar klemmso0cvpr# -*- coding: utf-8 -*- from PyQt5 import QtWidgets, QtCore from PyQt5.QtGui import QIcon from backend.caffe import path_loader import sys import signal class CaffepathDialog(QtWidgets.QDialog): def __init__(self, parent = None): super(CaffepathDialog, self).__init__(parent) self.setWindowFlags(self.windowFlags() & ~QtCore.Qt.WindowContextHelpButtonHint) self.setWindowTitle("Caffepath") self.changes = False self.newPath = "" caffepathttext = """To run Barista you need a valid Caffepath.""" lay = QtWidgets.QGridLayout(self) lay.addWidget(QtWidgets.QLabel(caffepathttext, self),0,0,1,1) self.pbSaveStart = QtWidgets.QPushButton("Save and start Barista") lay.addWidget(self.pbSaveStart, 1, 0, 1, 2) self.textCurrentPath = QtWidgets.QLabel("Current default path: " + path_loader.getCaffePath()) lay.addWidget(self.textCurrentPath, 2, 0, 1, 4) self.textChangeDefault = QtWidgets.QLabel("Change default path:") lay.addWidget(self.textChangeDefault, 3, 0, 1, 4) self.textwidget = QtWidgets.QLineEdit() lay.addWidget(self.textwidget, 4, 0, 1, 2) pix = QtWidgets.qApp.style().standardPixmap(QtWidgets.QStyle.SP_DialogOpenButton) #Maybe you can do this better ico = QIcon(pix) self.pbFile = QtWidgets.QPushButton(ico, "") lay.addWidget(self.pbFile, 4, 3, 1, 1) self.pbFile.clicked.connect(self._onFile) self.pbSaveStart.clicked.connect(self._onSave) def _onFile(self): """if default path should be changed, show file dialog to chose a path """ ret = QtWidgets.QFileDialog.getExistingDirectory(self, "Select the caffe path", "/home", QtWidgets.QFileDialog.ShowDirsOnly) if len(ret) != 0: self.textwidget.setText(ret) def _onSave(self): """if path in textwidget is correct caffepath, change caffepath and close window, otherwise show warning """ self.newPath = str(self.textwidget.text()) if (path_loader.checkIfFileIsCaffeRoot(self.newPath)): path_loader.changeCaffePath(self.newPath) self.changes = True self.close() else: QtWidgets.QMessageBox.warning( self, self.tr("Error"), self.tr("The path provided as default path is not a valid caffe root. The files ./src/caffe/proto/caffe.proto and ./python/caffe/pycaffe.py shall exist in that path.")) def closeEvent(self, event): #if nothing has changed(no new Caffepath) just exit complete programm if not self.changes: exit(1) gui/deployment_dialog.py0000664ڊ00111170000003032113243274037015642 0ustar klemmso0cvpr# -*- coding: utf-8 -*- import os import ntpath from PyQt5 import QtWidgets from backend.barista.utils.logger import Log from backend.barista.deployed_net import DeployedNet from backend.barista.project import Project from backend.barista.session.session_utils import Paths class DeploymentDialog(QtWidgets.QDialog): COMBO_INDEX_INLINE = 0 COMBO_INDEX_SEPARATE_FILE = 1 def __init__(self, sessions, parent = None): super(DeploymentDialog, self).__init__(parent) # Save given params. self._sessions = sessions self._setupUi() self.caller_id = None def _setupUi(self): self.setWindowTitle(self.tr("Deployment - Options")) mainLayout = QtWidgets.QVBoxLayout(self) # browse destination folder mainLayout.addWidget(QtWidgets.QLabel(self.tr("Export deployment files to"), self)) self._textDestination = QtWidgets.QLineEdit() btnBrowseDestination = QtWidgets.QPushButton(self.tr("Browse")) btnBrowseDestination.clicked.connect(lambda: self.selectFolder()) layout = QtWidgets.QHBoxLayout() layout.addWidget(self._textDestination) layout.addWidget(btnBrowseDestination) mainLayout.addLayout(layout) # snapshot selection self._snapshotInfoByComboIndex = [] mainLayout.addWidget(QtWidgets.QLabel(self.tr("Snapshot (determines caffemodel file and " " network version)."), self)) self._comboNet = QtWidgets.QComboBox() for sessionKey in sorted(self._sessions, reverse=True): session = self._sessions[sessionKey] snapshots = session.getSnapshots() for snapshotKey in sorted(snapshots, reverse=True): snapshot = snapshots[snapshotKey] self._comboNet.addItem(self.tr("Session " + str(session.getSessionId()) + " - " + os.path.basename(snapshot))) self._snapshotInfoByComboIndex.append({"session": session, "snapshot": snapshot}) mainLayout.addWidget(self._comboNet) # bottom button bar layout = QtWidgets.QHBoxLayout() btnDeployAndExport = QtWidgets.QPushButton(self.tr("Deploy and Export")) btnDeployAndExport.clicked.connect(lambda: self.deployAndExport()) btnCancel = QtWidgets.QPushButton(self.tr("Cancel")) btnCancel.clicked.connect(lambda: self.cancel()) layout.addWidget(btnCancel) layout.addWidget(btnDeployAndExport) mainLayout.addLayout(layout) # Fix width and height. self.layout().setSizeConstraint(QtWidgets.QLayout.SetFixedSize) def getCallerId(self): """ Return the unique caller id for this session """ if self.caller_id is None: self.caller_id = Log.getCallerId('Deployment') return self.caller_id def selectFolder(self): """ Opens a directory selection dialog and sets the line edit's value to the selected path. This will be triggered when the user clicks the browse button. """ dir_path = QtWidgets.QFileDialog.getExistingDirectory( parent=self ) self._textDestination.setText(dir_path) def cancel(self): """ Closes the dialog. This will be triggered when the user clicks the cancel button. """ self.close() def _replaceLast(self, source_string, replace_what, replace_with): """ Replace the very last occurrence of replace_what with replace_with in source_string. Code taken from: http://stackoverflow.com/a/3675423 """ head, sep, tail = source_string.rpartition(replace_what) return head + replace_with + tail def deployAndExport(self): """ Wrapper around _deployAndExportUnsafe that catches exceptions and shows them to the user in an error dialog. """ try: self._deployAndExportUnsafe() except Exception as e: QtWidgets.QMessageBox.critical(self, self.tr("Can't use specified destination"), self.tr("Error: " + str(e))) self.close() def _deployAndExportUnsafe(self): """ This will be triggered when the user clicks the deploy button. It validates the user input and displays messages and errors if additional input is required. If everything is validated successfully, the session is exported to the destination directory. """ # If no snapshot exists, we show an error message and close the dialog. if not self._hasSnapshotsOrDisplayError(): self.close() return # Get the destination folder from the current user input. If the input # is empty, we show an error message and cancel. destinationFolder = self._getDestinationOrDisplayError() if not destinationFolder: return # Check if the path already exsists. If it doesn't exist yet, let the user # decide whether to create all missing folders and abort otherwise. folderExists = os.path.isdir(destinationFolder) if not folderExists and not self._askDirectoryCreatePermission(): return # Ensure that the full path points to a folder and not a file. if not self._ensurePathIsFolderOrDisplayError(destinationFolder): return # Determine the destination file paths for destinationPrototxtFile, caffemodelDestination = self._getDestinationFilePaths(destinationFolder) # Check if any of the destination files already exsist and ask the user # if they should be replaced. Abort if the user decides not to replace one # of the files. if not self._checkFilesDontExistOrAskReplacePermission([destinationPrototxtFile, caffemodelDestination]): return # Export files. session = self._selectedSession() snapshot = self._selectedSnapshot() caffemodelContents = session.readCaffemodelFile(self._replaceLast(snapshot, 'solverstate', 'caffemodel')) # Start deployment. deployedNet = session.readDeployedNetAsString() # Write prototxt file. with open(destinationPrototxtFile, 'w') as file: file.write(deployedNet) # Write caffemodel file. with open(caffemodelDestination, 'w') as caffemodelFile: caffemodelFile.write(caffemodelContents) Log.log("Deployment files have been saved successfully to {}.".format(destinationPrototxtFile), self.getCallerId()) # Close the current dialog. self.close() def _selectedSession(self): """ Returns the session that belongs to the currently selected snapshot. """ return self._snapshotInfoByComboIndex[self._comboNet.currentIndex()]["session"] def _selectedSnapshot(self): """ Returns the currently selected snapshot name. """ return self._snapshotInfoByComboIndex[self._comboNet.currentIndex()]["snapshot"] def _hasSnapshotsOrDisplayError(self): """ Displays an error QMessageBox and returns False if no snapshots exist. Returns True otherwise. """ if len(self._snapshotInfoByComboIndex) <= 0: QtWidgets.QMessageBox.critical(self, self.tr("Deployment not available"), self.tr("There needs to be at least one saved snapshot to allow deployment.")) return False return True def _getDestinationOrDisplayError(self): """ If the user has not input a destination directory, this shows an error QMessageBox and returns False. Returns the destination path otherwise. """ destinationFolder = self._textDestination.text() if len(destinationFolder) <= 0: QtWidgets.QMessageBox.critical(self, self.tr("Can't use specified destination"), self.tr("Please select a valid destination to export deployment files.")) return False return destinationFolder def _ensurePathIsFolderOrDisplayError(self, path): """ Checks if the given path is a folder and not a file. Displays an error QMessageBox if it is not a folder and returns False. Returns True otherwise. """ # Ensure that the full path points to a folder and not a file. destinationIsFolder = not os.path.exists(path) or os.path.isdir(path) # If it does point to a file, we show an error and return False. if not destinationIsFolder: QtWidgets.QMessageBox.critical(self, self.tr("Can't use specified destination"), self.tr("The given path points to an existing file instead of " "a folder.")) return False return True def _checkFilesDontExistOrAskReplacePermission(self, files): """ Takes a list of filepaths as an input and for each of those paths checks if the file already exists. If it does, it asks the user if it should be replaced. If the user clicks 'No' on any of the dialogs it returns False. Returns True otherwise. """ for f in files: fileAlreadyExists = os.path.exists(f) basename = ntpath.basename(f) if fileAlreadyExists and not self._askFileReplacePermission(basename): return False return True def _askFileReplacePermission(self, filename): """, Shows a QMessageBox that asks the user if the file with name filename should be replaced. Returns True if the user clicks 'Yes' and False otherwise. """ reply = QtWidgets.QMessageBox.question(self, self.tr("File does already exist."), self.tr( "Do you want to replace the existing " "%s file?" % (filename)), QtWidgets.QMessageBox.Yes, QtWidgets.QMessageBox.No) return reply == QtWidgets.QMessageBox.Yes def _askDirectoryCreatePermission(self): """ Shows a QMessageBox that asks the user if the destination directories should be created. If the users clicks 'Yes', the directories will be created. Returns True if the user clicked 'Yes', False otherwise. """ reply = QtWidgets.QMessageBox.question(self, self.tr("Destination does not exist."), self.tr("Do you want to create all non-existing folders in the given path?"), QtWidgets.QMessageBox.Yes, QtWidgets.QMessageBox.No) # If the user agrees, we create the directory. if reply == QtWidgets.QMessageBox.Yes: os.makedirs(destinationFolder) return reply == QtWidgets.QMessageBox.Yes def _getDestinationFilePaths(self, directory): """ Returns a tuple where the first element is the filepath for the destination prototxt file and the second element is the filepath for the destination caffemodel file. """ # Determine the destination file paths for destinationPrototxtFile = os.path.join(directory, "deployed_net.prototxt") caffemodelFilename = self._replaceLast(self._selectedSnapshot(), "solverstate", "caffemodel") caffemodelDestination = os.path.join(os.path.dirname(destinationPrototxtFile), caffemodelFilename) return destinationPrototxtFile, caffemodelDestination gui/export_solver_dialog.py0000664ڊ00111170000002225613243274037016405 0ustar klemmso0cvpr# -*- coding: utf-8 -*- import copy import os from PyQt5 import QtWidgets from backend.barista.utils.logger import Log from backend.caffe import saver class ExportSolverDialog(QtWidgets.QDialog): COMBO_INDEX_INLINE = 0 COMBO_INDEX_SEPARATE_FILE = 1 def __init__(self, solver, network, parent = None, defaultActions=None): super(ExportSolverDialog, self).__init__(parent) # create a deep copy of the solver, as we will need to manipulate it a little bit self._solver = copy.deepcopy(solver.valuedict) self._network = network # this will be used to add paths to the recent file list self._defaultActions = defaultActions self.PROTOTXT_FILTER = self.tr("Prototxt Files (*.prototxt);;All files (*)") self.setWindowTitle(self.tr("Export Solver - Options")) mainLayout = QtWidgets.QVBoxLayout(self) # browse solver file mainLayout.addWidget(QtWidgets.QLabel(self.tr("Export solver definition to"), self)) self._textPathSolver = QtWidgets.QLineEdit() btnBrowseSolver = QtWidgets.QPushButton(self.tr("Browse")) btnBrowseSolver.clicked.connect(lambda: self.selectFileSolver()) layout = QtWidgets.QHBoxLayout() layout.addWidget(self._textPathSolver) layout.addWidget(btnBrowseSolver) mainLayout.addLayout(layout) # options to include the net # pay attention to the order in which the combo items are added. # they must fit to COMBO_INDEX_SEPARATE_FILE and COMBO_INDEX_INLINE mainLayout.addWidget(QtWidgets.QLabel(self.tr("Include net definition"), self)) self._comboNet = QtWidgets.QComboBox() self._comboNet.addItem(self.tr("inline (parameter: net_param)")) self._comboNet.addItem(self.tr("by pointing to the following file (parameter: net)")) self._comboNet.currentIndexChanged.connect(self.selectionChange) mainLayout.addWidget(self._comboNet) # browse net file self._textPathNet = QtWidgets.QLineEdit() self._textPathNet.setDisabled(True) self._btnBrowseNet = QtWidgets.QPushButton(self.tr("Browse")) self._btnBrowseNet.clicked.connect(lambda: self.selectFileNet()) self._btnBrowseNet.setDisabled(True) layout = QtWidgets.QHBoxLayout() layout.addWidget(self._textPathNet) layout.addWidget(self._btnBrowseNet) mainLayout.addLayout(layout) # bottom button bar layout = QtWidgets.QHBoxLayout() btnSave = QtWidgets.QPushButton(self.tr("Save")) btnSave.clicked.connect(lambda: self.save()) btnCancel = QtWidgets.QPushButton(self.tr("Cancel")) btnCancel.clicked.connect(lambda: self.cancel()) layout.addWidget(btnCancel) layout.addWidget(btnSave) mainLayout.addLayout(layout) # use fixed size self.layout().setSizeConstraint(QtWidgets.QLayout.SetFixedSize) def selectFileNet(self): filename, _ = QtWidgets.QFileDialog.getOpenFileName( parent=self, filter=self.PROTOTXT_FILTER ) self._textPathNet.setText(filename) def selectFileSolver(self): filename, _ = QtWidgets.QFileDialog.getSaveFileName( parent=self, filter=self.PROTOTXT_FILTER ) # if the file does not yet exist and no file extension has been specified, # set it to the default .prototxt if len(filename) > 0 and not os.path.exists(filename) and os.path.splitext(filename)[1] == "": filename += ".prototxt" self._textPathSolver.setText(filename) def selectionChange(self, selectedIndex): disableNetFileSelection = (selectedIndex != self.COMBO_INDEX_SEPARATE_FILE) self._textPathNet.setDisabled(disableNetFileSelection) self._btnBrowseNet.setDisabled(disableNetFileSelection) def cancel(self): self.close() def save(self): try: # get user input netFullPath = self._textPathNet.text() solverFullPath = self._textPathSolver.text() solverDirPath = os.path.dirname(solverFullPath) netInSeparateFile = (self._comboNet.currentIndex() == self.COMBO_INDEX_SEPARATE_FILE) netPathIsValid = len(netFullPath) > 0 and os.path.exists(netFullPath) \ and os.path.isfile(netFullPath) # ensure that the input isn't empty if len(solverFullPath) > 0: # ensure that the path (except the base name) does already exist folderExists = os.path.isdir(solverDirPath) # if it doesn't exist yet, let the user decide whether to create all missing folders if not folderExists: reply = QtWidgets.QMessageBox.question(self, self.tr("Destination doesn't exist yet."), self.tr("Do you want to create all non-existing folders " "in the given path?"), QtWidgets.QMessageBox.Yes, QtWidgets.QMessageBox.No) if reply == QtWidgets.QMessageBox.Yes: folderExists = True os.makedirs(solverDirPath) if folderExists: # ensure that the full path does point to a file and not a folder fileIsNoFolder = not os.path.exists(solverFullPath) or not os.path.isdir(solverFullPath) # input is valid, go ahead and start the actual export if fileIsNoFolder: if netInSeparateFile: # network path does not need to be valid, as we are not doing anything with the referenced file # anyway: let the user decide whether an invalid path is used on purpose if not netPathIsValid: reply = QtWidgets.QMessageBox.question(self, self.tr("Network path seems to be invalid."), "Do you want to continue anyway?", QtWidgets.QMessageBox.Yes, QtWidgets.QMessageBox.No) if reply == QtWidgets.QMessageBox.Yes: netPathIsValid = True if netPathIsValid: # point to selected network file self._solver["net"] = netFullPath # remove any other references to a network definition if "net_param" in self._solver: del self._solver["net_param"] else: # include inline definition of the network self._solver["net_param"] = self._network # remove any other references to a network file if "net" in self._solver: del self._solver["net"] # finally, save solver prototxt if not netInSeparateFile or netPathIsValid: with open(solverFullPath, 'w') as file: file.write(saver.saveSolver(self._solver)) callerId = Log.getCallerId('export_solver') Log.log( "The solver has been exported successfully to {}.".format( solverFullPath ), callerId) # add used file paths to the recent file list if self._defaultActions is not None: self._defaultActions.recentSolverData.addRecently(solverFullPath, solverFullPath) if netInSeparateFile and netPathIsValid: self._defaultActions.recentNetData.addRecently(netFullPath, netFullPath) self.close() else: QtWidgets.QMessageBox.critical(self, self.tr("Can't save solver"), self.tr("The given path points to an existing folder instead of " "a file.")) else: QtWidgets.QMessageBox.critical(self, self.tr("Can't save solver"), self.tr("Please select a valid destination for the solver file.")) except: QtWidgets.QMessageBox.critical(self, self.tr("Can't save solver"), self.tr("Unknown error.")) self.close() gui/gui_util.py0000664ڊ00111170000000770213243274037013773 0ustar klemmso0cvprfrom PyQt5 import QtWidgets from PyQt5.QtCore import Qt, QRegExp from backend.barista.project import dirIsProject import os def askFromList(parent, layerlist, title, text, defaultSelection=True): '''given a list of layernames, select one''' # create a QDialog dialog = QtWidgets.QDialog(parent) dialog.setWindowTitle(title) # set some layouts layout = QtWidgets.QVBoxLayout(dialog) sublayout = QtWidgets.QHBoxLayout() # set Widgets label = QtWidgets.QLabel(text) layout.addWidget(label) list = QtWidgets.QListWidget(dialog) layout.addWidget(list) # add the listitems for id, name in layerlist: item = QtWidgets.QListWidgetItem(list) item.setData(Qt.UserRole, id) item.setData(0, name) list.addItem(item) # add the buttons pb_ok = QtWidgets.QPushButton("Ok") pb_can = QtWidgets.QPushButton("Cancel") sublayout.addWidget(pb_ok) sublayout.addWidget(pb_can) layout.addLayout(sublayout) # connect the buttons list.doubleClicked.connect(dialog.close) pb_ok.clicked.connect(dialog.close) pb_can.clicked.connect(lambda: (list.clearSelection(), dialog.close())) # select the first item of the list by default if defaultSelection: list.setCurrentRow(0) # exec the dialog dialog.exec_() # process the selection selection = list.selectedItems() if len(selection) == 0: return None # return the ID of the selected layer return selection[0].data(Qt.UserRole) def getRecentProjectsFolder(defAct): """ Get the folder containing (!) the most recent project. defAct: DefaultActions to use returns: folder path or None """ # Find starting directory to start new project dialog recentProjectTuple = defAct.recentProjectsData.getMostRecent() if recentProjectTuple is None: return else: [_, mostRecentProjectPath] = recentProjectTuple return os.path.dirname(mostRecentProjectPath) def getOpenProjectDir(defAct, parent): """ Interactively select an existing project directory defAct: DefaultActions to use parent: a Qt parent object returns: folder path (may be empty if selection fails) """ directory = QtWidgets.QFileDialog.getExistingDirectory(parent = parent, caption = defAct.tr("Directoy of the project"), directory = getRecentProjectsFolder(defAct)) if len(directory) == 0: return "" if not dirIsProject(directory): answer = QtWidgets.QMessageBox.question(parent, defAct.tr("No Project"), defAct.tr("The selected folder does not seem to be a valid project. Load this dir anyway?")) if answer == QtWidgets.QMessageBox.No: return "" return directory def getNewProjectDir(defAct, parent): """ Interactively select a potential new project directory defAct: DefaultActions to use parent: a Qt parent object returns: folder path (may be empty if selection fails) """ # Select new project dir path newProjectDir, _ = QtWidgets.QFileDialog.getSaveFileName(parent = parent, caption = defAct.tr("New Project Directory"), directory = getRecentProjectsFolder(defAct)) if len(newProjectDir) == 0: return "" name = os.path.basename(newProjectDir) # If name contains special characters regExpName = QRegExp("[A-Za-z0-9_()-][A-Za-z0-9_ #()-]*") if not regExpName.exactMatch(name): QtWidgets.QMessageBox.warning(parent, defAct.tr("Invalid character"), defAct.tr("Project name is invalid. You may only use letters, digits, whitespace and the following special characters: _ # ( ) -")) return "" # if the path already exists if os.path.exists(newProjectDir): QtWidgets.QMessageBox.warning(parent, defAct.tr("Already exists"), defAct.tr("Project with this name already exists")) return "" # if everything is alright return newProjectDir gui/ipy_widget.py0000664ڊ00111170000000334513243274037014315 0ustar klemmso0cvpr # https://github.com/jupyter/qtconsole/blob/master/examples/inprocess_qtconsole.py # https://stackoverflow.com/questions/26666583/embedding-ipython-qtconsole-and-passing-objects import sys qtconsole = True if qtconsole: from qtconsole.qt import QtGui from qtconsole.rich_jupyter_widget import RichJupyterWidget from qtconsole.inprocess import QtInProcessKernelManager from ipykernel.inprocess import InProcessKernelManager def show(parent): global ipython_widget # Prevent from being garbage collected # Create an in-process kernel kernel_manager = QtInProcessKernelManager() kernel_manager.start_kernel(show_banner=False) kernel = kernel_manager.kernel kernel.gui = 'qt' kernel_client = kernel_manager.client() kernel_client.start_channels() kernel_client.namespace = parent ipython_widget = RichJupyterWidget() ipython_widget.kernel_manager = kernel_manager ipython_widget.kernel_client = kernel_client ipython_widget.show() kernel.shell.push({"manager": parent}) else: from IPython.kernel.inprocess import InProcessKernelManager from IPython.terminal.console.interactiveshell import ZMQTerminalInteractiveShell #https://github.com/ipython/ipykernel/blob/master/examples/embedding/inprocess_terminal.py def show(parent): kernel_manager = InProcessKernelManager() kernel_manager.start_kernel() kernel = kernel_manager.kernel # kernel.gui = 'qt' kernel.shell.push({"manager": parent}) client = kernel_manager.client() client.start_channels() shell = ZMQTerminalInteractiveShell(manager=kernel_manager, client=client) shell.mainloop() gui/path_editor_dialog.py0000664ڊ00111170000002142013243274037015764 0ustar klemmso0cvprfrom PyQt5.QtCore import pyqtSlot as Slot, QMetaObject,Qt from PyQt5.QtGui import QIcon from PyQt5.QtWidgets import QDialog, QPushButton, QLabel, QLineEdit, QMessageBox, QGridLayout, QStyle, QFileDialog, qApp from backend.barista.utils.logger import Log from backend.caffe import path_loader class PathEditorDialog(QDialog): """This Editor enables text-based changes of the caffe path.""" def __init__(self, parent = None): QDialog.__init__(self, parent) #check if the caffe path exists if not (path_loader.checkIfFileIsCaffeRoot(path_loader.getCaffePath())): callerId = Log.getCallerId("PathEditorWidget") Log.error("Caffe-path is not saved in program settings, or it is an invalid path. Change caffe-path in settings",callerId) self.projectCaffePath = "" #project.getProjectCaffePath self.project = None self.newPath = "" self.setModal(True) self.resize(100,30) self.layout = QGridLayout(self) self.pbSaveClose = QPushButton("Save and Close") self.layout.addWidget(self.pbSaveClose, 0, 0, 1, 2) self.pbClose = QPushButton("Close without Saving") self.layout.addWidget(self.pbClose, 0, 2, 1, 2) self.textCurrentPath = QLabel("Current default path: " + path_loader.getCaffePath()) self.layout.addWidget(self.textCurrentPath, 1, 0, 1, 4) self.textChangeDefault = QLabel("Change default path:") self.layout.addWidget(self.textChangeDefault, 2, 0, 1, 4) self.textwidget = QLineEdit() self.layout.addWidget(self.textwidget, 3, 0, 1, 3) pix = qApp.style().standardPixmap(QStyle.SP_DialogOpenButton) #Maybe you can do this better ico = QIcon(pix) self.pbFile = QPushButton(ico, "") self.layout.addWidget(self.pbFile, 3, 3, 1, 1) self.textCurrentProjectPath = QLabel("Current project path: " + path_loader.getCaffePath()) self.layout.addWidget(self.textCurrentProjectPath, 5, 0, 1, 4) self.textProjectDefault = QLabel("Change project path:") self.layout.addWidget(self.textProjectDefault, 6, 0, 1, 4) self.textwidget1 = QLineEdit() self.layout.addWidget(self.textwidget1, 7, 0, 1, 3) pix = qApp.style().standardPixmap(QStyle.SP_DialogOpenButton) #Maybe you can do this better ico = QIcon(pix) self.pbFile1 = QPushButton(ico, "") self.layout.addWidget(self.pbFile1, 7, 3, 1, 1) self.pbSaveClose.clicked.connect(self._onSave) self.pbClose.clicked.connect(self._onClose) self.pbFile.clicked.connect(self._onFile) self.pbFile1.clicked.connect(self._onFile1) @Slot() def showPathWarning(self): """"if the project path does not exist, show this warning, and give the user the possibility to change the path""" quit_msg = "caffe-path from project file is does not exists on your PC. Click \"Yes\" to chose a " \ "project path. Click \"No\" to set default path as project path" reply = QMessageBox.question(self, 'Caffe Path Warning', quit_msg, QMessageBox.Yes, QMessageBox.No) if reply == QMessageBox.No: self.changeProjectPath("") # TODO: See if re-start is required # self.showRestartProgramWarning() else: self.changeOnlyProjectPath() def showRestartProgramWarning(self): """shows warning that the program needs to be restarted if something is changed """ quit_msg = "You have to restart for the changes to have effect." reply = QMessageBox.warning(self, "Warning", quit_msg, QMessageBox.Ok) def changeText1(self, text): """change the shown text of the project path""" self.textCurrentProjectPath.setText("Current project path:" + text) def getText(self): """returns the text from the textwidget, to get the new path """ return self.textwidget.text() def getText1(self): """returns the text from the textwidget, to get the new path """ return self.textwidget1.text() def clear(self): """clears the textwidgets, after a path has changed""" self.textwidget.clear() self.textwidget.setFocus() self.textwidget1.clear() self.textwidget1.setFocus() def _onFile(self): """if default path should be changed, show file dialog to chose a path """ ret = QFileDialog.getExistingDirectory(self, "Select the caffe path", "/home", QFileDialog.ShowDirsOnly) if len(ret) != 0: self.textwidget.setText(ret) def _onFile1(self): """if project path should be changed, show file dialog to chose a path """ ret = QFileDialog.getExistingDirectory(self, "Select the caffe path", "/home", QFileDialog.ShowDirsOnly) if len(ret) != 0: self.textwidget1.setText(ret) def changeOnlyProjectPath(self): """show file dialog and change the project path directly""" ret = QFileDialog.getExistingDirectory(self, "Select the caffe path", "/home", QFileDialog.ShowDirsOnly) if len(ret) != 0 and path_loader.checkIfFileIsCaffeRoot(ret): self.changeProjectPath(ret) self.showRestartProgramWarning() else: self.changeProjectPath("") QMessageBox.warning( self, self.tr("Error"), self.tr( "The path provided as project path is not a valid caffe root. The files ./src/caffe/proto/caffe.proto and ./python/caffe/pycaffe.py shall exist in that path. \n" "the path is now set to the default path, you can change this in the settings")) def _onSave(self): """reads the paths that need to be changed""" changes = False if (path_loader.checkIfFileIsCaffeRoot(str(self.getText())) or self.getText() == ""): if not (self.getText() == ""): path_loader.changeCaffePath(self.getText()) changes = True self.updateDefaultPath() self.textwidget.clear() #QMessageBox.warning( # self, self.tr("Info"), self.tr("You have to save the project and restart this program to make changes available")) self._onClose() else: QMessageBox.warning( self, self.tr("Error"), self.tr("The path provided as default path is not a valid caffe root. The files ./src/caffe/proto/caffe.proto and ./python/caffe/pycaffe.py shall exist in that path.")) if (path_loader.checkIfFileIsCaffeRoot(str(self.getText1())) or self.getText1() == ""): if not (self.getText1() == ""): self.changeProjectPath(self.getText1()) changes = True self.updateProjectPath() # proto_description.changeCaffePath(self.getText1()) # QMessageBox.warning( # self, self.tr("Info"), self.tr("You have to save the project and restart this program to make changes available")) self.textwidget1.clear() self._onClose() else: QMessageBox.warning( self, self.tr("Error"), self.tr( "The path provided as project path is not a valid caffe root. The files ./src/caffe/proto/caffe.proto and ./python/caffe/pycaffe.py shall exist in that path.")) if changes: self.showRestartProgramWarning() def _onClose(self): self.close() def changeProjectPath(self, path): """change the project path""" self.project.changeProjectCaffePath(path) self.changeText1(self.project.getCaffeRoot()) def updateDefaultPath(self): """updates the shown default path""" self.textCurrentPath.setText(path_loader.getCaffePath()) def updateProjectPath(self): """updates the shown project path""" self.changeText1(self.project.getCaffeRoot()) def updateProject(self, project): """updates the project, and reads the the path from it, if it does not exists or is not set, the path warining is shown""" self.project= project if not (path_loader.checkIfFileIsCaffeRoot(self.project.getCaffeRoot())): QMetaObject.invokeMethod(self, "showPathWarning", Qt.QueuedConnection) # Todo: show warning if the path does not exists... # give the user the possibility to 1. set project path to default path # 2. set project path manual # 3. ignore -> maybe disable training? else: self.updateProjectPath() gui/start_dialog.py0000664ڊ00111170000001330713243274037014624 0ustar klemmso0cvpr# -*- coding: utf-8 -*- from PyQt5 import QtWidgets, QtCore from PyQt5.QtGui import QIcon from backend.barista.utils.settings import applicationQSetting from gui.main_window import main_window from backend.caffe import path_loader from backend.caffe.proto_info import resetCaffeProtoModulesvar from gui.main_window.default_actions import DefaultActions, dirIsProject from backend.barista.project import Project from gui_util import getNewProjectDir, getOpenProjectDir # True => Enable Development Feature like skip project loading DEVELOP_MODE = False class StartDialog(QtWidgets.QDialog): def __init__(self, parent = None): super(StartDialog, self).__init__(parent) #reload the caffe und proto Modules from the program settings to get a working version #path_loader.reloadCaffeModule(path_loader.getCaffePath()) #path_loader.reloadProtoModule(path_loader.getCaffePath()) self.setWindowFlags(self.windowFlags() & ~QtCore.Qt.WindowContextHelpButtonHint) self.setWindowTitle("Barista") self.setWindowIcon(QIcon('./resources/icon.png')) self._defAct = DefaultActions(self) # self._win = main_window.UiMainWindow() # self._defAct.projectChanged.connect(self.__switchMainWindow) lay = QtWidgets.QVBoxLayout(self) welcomeText="Barista is a graphical user interface for caffe, a deep neural network framework.

\n\ Additional information about caffe and barista can be found under:
\n\ Caffe: http://caffe.berkeleyvision.org
\n\ Barista: http://barista.uni-muenster.de

\n\ We sincerely appreciate your interest and contributions!
\n\ Just follow the link above, if you like to contribute or want to send feedback.

" welcomeLabel=QtWidgets.QLabel(self) welcomeLabel.setText(welcomeText) welcomeLabel.setOpenExternalLinks(True) lay.addWidget(welcomeLabel) lay.addLayout(self.__buildButton(self.tr("New Project"), self.newProject)) lay.addLayout(self.__buildButton(self.tr("Open Project"), self.openProject, menu=self._defAct.buildRecentProjectMenu(callback=self.loadProject))) if DEVELOP_MODE: lay.addLayout(self.__buildButton("Skip", descr="I'm professional! Just want to develop without some strange " "loading (Development Feature)", fun=self.skip)) lay.addLayout(self.__buildButton(self.tr("Exit"), fun=self.close)) self.setFixedSize(self.sizeHint()) #Development-Feature if DEVELOP_MODE: skipping = self._read_skip_setting() skipBox = QtWidgets.QCheckBox("Fast skipping on start (development-feature)") skipBox.setToolTip("Skip automatically on next start within 1 second") skipBox.setChecked(skipping) skipBox.stateChanged.connect(lambda: self._write_skip_setting(skipBox.isChecked())) lay.addWidget(skipBox) if skipping: QtCore.QTimer.singleShot(1000, self.skip) else: self._write_skip_setting(False) # method called when pushing "new project" button in Start window def newProject(self): dir = getNewProjectDir(defAct=self._defAct, parent=self) if len(dir) == 0: return else: win = main_window.UiMainWindow(self._defAct) # self._defAct.setProject(proj) win.availableActions().newProjectDialog(dir, parent=self) self.__switchMainWindow(win) def openProject(self): proj = getOpenProjectDir(defAct=self._defAct, parent=self) if len(proj) == 0: return resetCaffeProtoModulesvar() self.loadProject(proj) def loadProject(self, dir): resetCaffeProtoModulesvar() proj = Project(dir) win = main_window.UiMainWindow(self._defAct) self._defAct.setProject(proj) self.__switchMainWindow(win) def skip(self): win = main_window.UiMainWindow(self._defAct) self.__switchMainWindow(win) def __buildButton(self, text, fun, descr="", menu=None): """ Build Button with given text and description. Connect click event to fun. Add an extra Button for Menu if menu is not None """ lay = QtWidgets.QHBoxLayout() wdg = QtWidgets.QCommandLinkButton(text, self) lay.addWidget(wdg) wdg.setDescription(descr) if menu: menuButton = QtWidgets.QCommandLinkButton("Open Recent") menuButton.setIcon(QIcon(None)) #menuButton.setFlat(True) menuButton.setMenu(menu) lay.addWidget(menuButton) pol = QtWidgets.QSizePolicy() pol.setHorizontalPolicy(QtWidgets.QSizePolicy.Expanding) wdg.setSizePolicy(pol) wdg.clicked.connect(fun) return lay def __switchMainWindow(self, win): # Resize and show the main main_window win.showMaximized() win.show() win.setFocus() self.close() def _read_skip_setting(self): setting = applicationQSetting() setting.beginGroup("StartDialog") res = setting.value("skip_always", "false") setting.endGroup() return res == "true" def _write_skip_setting(self, shouldSkip ): setting = applicationQSetting() setting.beginGroup("StartDialog") setting.setValue("skip_always", "true" if shouldSkip else "false") setting.endGroup() backend/barista/constraints/permanent/__init__.py0000664ڊ00111170000000050613243274037022474 0ustar klemmso0cvpr""" This subpackage contains all constraints that need to be ensured permanently. So the given methods need to be called each time one of the handled constraints might get broken, which is usually identically to all situations in which the user could edit the raw prototxt file or equivalent data (as in the project file). """backend/barista/constraints/permanent/project.py0000664ڊ00111170000000235013243274037022402 0ustar klemmso0cvpr""" This module defines permanent constraints for a Barista project. """ from backend.barista.utils.logger import Log from backend.barista.constraints.permanent.solver import ensureSolverConstraints logID = Log.getCallerId('constraints/project') def ensureProjectDataConstraints(projectData): """Take the current state of projectData and manipulate it to handle some special cases. This method should be called each time when the project data has been changed. It will adjust some values, which would be valid for the prototxt-syntax in general, but underlay further constraints especially for Barista. """ if not hasattr(projectData, '__getItem__'): Log.error('Project data is empty.', logID) return projectData jsonKeys = ("activeSession", "environment", "inputdb", "projectid", "transform") keyNotFound = False for key in jsonKeys: if key not in projectData: Log.error("Project information is invalid! Key '" + key + "' is missing!", logID) keyNotFound = True if keyNotFound: Log.error("Session at %s is invalid. Key %s in 'sessionstate.json' is missing", logID) # add further constraint types like e.g.: return projectData backend/barista/constraints/permanent/solver.py0000664ڊ00111170000000364113243274037022252 0ustar klemmso0cvpr""" This module defines permanent solver constraints. """ import os import backend from backend.barista.utils.logger import Log logID = Log.getCallerId('constraints/solver') def ensureSolverConstraints(solverDictionary): """Ensure that all constraints for the given solverDictionary are valid. Sets static values and removes invalid values. """ # The file names inside of a session are static and must not be altered by the user if "net" not in solverDictionary or solverDictionary["net"] != backend.barista.session.session_utils.Paths.FILE_NAME_NET_INTERNAL: Log.log("The solver property 'net' must point to the generated network file. " "Value has been changed from '{}' to '{}'.".format( solverDictionary["net"] if "net" in solverDictionary else "None", backend.barista.session.session_utils.Paths.FILE_NAME_NET_INTERNAL ), logID) solverDictionary["net"] = backend.barista.session.session_utils.Paths.FILE_NAME_NET_INTERNAL # An additional net definition inside of the solver would be inconsistent to the separately handled network if "net_param" in solverDictionary: Log.log("The solver property 'net_param' is not supported as it would be inconsistent with the separately " "handled network. Property has been removed.", logID) del solverDictionary["net_param"] # a snapshot_prefix containing a path is not supported either if "snapshot_prefix" in solverDictionary: head, tail = os.path.split(solverDictionary["snapshot_prefix"]) if len(head) > 0: Log.log("The solver property 'snapshot_prefix' contained an unsupported path. " "Property was shortened from '{}' to '{}'.".format( solverDictionary["snapshot_prefix"], tail ), logID) solverDictionary["snapshot_prefix"] = tail return solverDictionary backend/barista/constraints/session_run/__init__.py0000664ڊ00111170000000030013243274037023042 0ustar klemmso0cvpr""" This subpackage contains all constraints that need to be ensured before/during a new session run. So the given methods need to be called each time a new session is about to be started. """backend/barista/constraints/session_run/training.py0000664ڊ00111170000005571013243274037023135 0ustar klemmso0cvpr""" This subpackage contains all constraints that need to be ensured before/during the training only. So the given methods need to be called each time a new training session is about to be started. """ import os from PyQt5 import QtWidgets from backend.barista.constraints import common from backend.caffe.proto_info import CaffeMetaInformation from gui.network_manager.layer_helper import LayerHelper def checkMinimumTrainingRequirements(session, parentGui=None, reportToUser=True): """ Check whether the minimum requirements for starting a new training session are met. This method should be called whenever a new training session is about to be started. If the returned value isn't True, the session start should be aborted. TODO implement the same for a evaluation-only net. If reportToUser is set to True, which is the default value, the user will be informed about unmet requirements by a message box, including an option to ignore all listed errors. :return: True, if everything is okay. False otherwise. """ # validate requirements = MinimumTrainingRequirements(session) valid = requirements.valid() session.setErrorList(requirements.getErrors()) # list errors and let the user decide whether to ignore the errors if (parentGui is not None): if reportToUser and not valid: valid = requirements.showErrors(parentGui) return valid else: # if there is no GUI, return the Error Messages return requirements.getErrors() class MinimumTrainingRequirements: """This class is only required inside of checkMinimumTrainingRequirements. Do not use as a public interface. """ def __init__(self, session): self._session = session self._stateData = None if session.state_dictionary: self._stateData = session.state_dictionary # store one error message per unmet requirement in the following list self._errorMessages = [] # start validation self._check() def showErrors(self, parentGui): """ If any errors have been collected, show them in a message box. Additionally, to the listed errors, the user will get an option to ignore all errors. :param parentGui: The parent GUI to which the shown message box will be bounded. :return: Return true, iff no errors occured or the user decided to ignore them. """ valid = self.valid() if not valid: # accumulate collected error data msg = "" for msg_part in self._errorMessages: if len(msg) > 0: msg += "\n\n" msg += msg_part # configure message box msgBox = QtWidgets.QMessageBox(parentGui) msgBox.setIcon(QtWidgets.QMessageBox.Warning) msgBox.setText(parentGui.tr("Please check the following constraints.\nIf you decide to ignore this message, " "it might result in a program crash.")) msgBox.setInformativeText(parentGui.tr(msg)) msgBox.setWindowTitle(parentGui.tr("Minimum training requirements aren't met.")) msgBox.setParent(parentGui) msgBox.setStandardButtons(QtWidgets.QMessageBox.Ignore | QtWidgets.QMessageBox.Ok) msgBox.setDefaultButton(QtWidgets.QMessageBox.Ok) # show message box and receive user input clickedButton = msgBox.exec_() # if the user clicked "ignore", all errors will be ignored valid = clickedButton == QtWidgets.QMessageBox.Ignore return valid def valid(self): """Check whether any constraint has been broken.""" return len(self._errorMessages) == 0 def getErrors(self): return self._errorMessages def _check(self): """Start all validations.""" # TODO check type(self._session) for # temporary change the current working dir to allow evaluation of relative paths if hasattr(self._session, 'checkTraining'): self._errorMessages.extend(self._session.checkTraining()) else: self._checkSolver() self._checkDataLayerExistence() self._checkDataLayerParameters() self._checkInputLayer() self._checkUniqueBlobNames() def _checkSolver(self): """Check whether all solver constraints are valid.""" # the base learning rate should be a positive number if not hasattr(self._stateData, '__getitem__'): self._errorMessages.append(("State data is empty!","...")) return if self._stateData["solver"] == {u'net': u'net-internal.prototxt'}: self._errorMessages.append(("The solver seems to be empty. Please import or define a solver.\n" "(You can define a solver with the help of the dock Solver Properties)","No solver.")) if not ("base_lr" in self._stateData["solver"] and common.isPositiveNumber(self._stateData["solver"]["base_lr"])): self._errorMessages.append(("The base learning rate (base_lr) must be a positive number.","Invalid base_lr.")) # the maximum number of iterations should be a positive integer if not ("max_iter" in self._stateData["solver"] and common.isPositiveInteger(self._stateData["solver"]["max_iter"])): self._errorMessages.append(("The maximum number of iterations (max_iter) must be a positive integer.","Invalid max_iter.")) def _checkUniqueBlobNames(self): """Checks for duplicate top blob names in all layers and emits error message if duplicates found except if in-place is permitted (Activation Layer)""" nonActivationTopBlobNames = [] activationTopBlobNames = [] if hasattr(self._stateData, '__getitem__'): # inefficient double looping but activation layers can only be checked after non-activation layers for layer_id, layer in self._stateData["network"]["layers"].iteritems(): if LayerHelper.isLayerIncludedInTrainingPhase(layer) and ("top" in layer["parameters"]): if not layer["type"].isActivationLayer(): for name in layer["parameters"]["top"]: if name not in nonActivationTopBlobNames: nonActivationTopBlobNames.append(name) else: self._errorMessages.append((name + " is used more than once as connection name!", "Duplicate connector name in " + layer["parameters"]["name"])) for layer_id, layer in self._stateData["network"]["layers"].iteritems(): if LayerHelper.isLayerIncludedInTrainingPhase(layer) and ("top" in layer["parameters"]): if layer["type"].isActivationLayer(): for name in layer["parameters"]["top"]: if name not in activationTopBlobNames: activationTopBlobNames.append(name) if name in nonActivationTopBlobNames: if not LayerHelper.isLayerInPlace(layer): self._errorMessages.append( (name + " is used more than once as connection name!", "Duplicate connector name in " + layer["parameters"]["name"])) else: self._errorMessages.append((name + " is used more than once as connection name!", "Duplicate connector name in " + layer["parameters"]["name"])) def _checkDataLayerExistence(self): """Check whether at least one data layer exists (during the training phase).""" nDataLayerTotal = 0 nDataLayerTraining = 0 if not hasattr(self._stateData, '__getitem__'): self._errorMessages.append(("State data is empty!","Empty state data.")) return if self._stateData["network"]["layers"] == {}: self._errorMessages.append(("There is no net defined. Please import or create a network.","No net.")) try: for layer_id, layer in self._stateData["network"]["layers"].iteritems(): if layer["type"].isDataLayer(): nDataLayerTotal += 1 if LayerHelper.isLayerIncludedInTrainingPhase(layer): nDataLayerTraining += 1 except KeyError: self._errorMessages.append(("No network data available.","No network data.")) # list all available data layers to inform the user about possible options typeListMsg = "" allTypes = CaffeMetaInformation().availableLayerTypes() for key in sorted(allTypes): if allTypes[key].isDataLayer(): if typeListMsg != "": typeListMsg += ", " typeListMsg += key typeListMsg = " Available types of such a layer are: " + typeListMsg + "." if nDataLayerTotal == 0: self._errorMessages.append(("There should be at least one data layer in your network.\n" + typeListMsg,"No/invalid input data.")) elif nDataLayerTraining == 0: self._errorMessages.append(("There should be at least one data layer in your network that is included in " "the training phase.\n" + typeListMsg,"No/invalid input data.")) def _checkDataLayerParameters(self): """Check whether the existing data layers (of the training phase) provide (valid) data. required parameters were determined based on http://caffe.berkeleyvision.org/tutorial/layers.html """ if not hasattr(self._stateData, '__getitem__'): self._errorMessages.append(("State data is empty!","Empty state data.")) return # a list of pairs. first element of a pair is the name of a layer (type). # The second one is the name of the missing parameter. self._missingLayerParams = [] for layer_id, layer in self._stateData["network"]["layers"].iteritems(): if layer["type"].isDataLayer() and LayerHelper.isLayerIncludedInTrainingPhase(layer): if layer["type"].name() == "Data": self._checkDataLayer(layer) elif layer["type"].name() == "MemoryData": self._checkMemoryDataLayer(layer) elif layer["type"].name() == "HDF5Data": self._checkHDF5DataLayer(layer) elif layer["type"].name() == "ImageData": self._checkImageDataLayer(layer) elif layer["type"].name() == "WindowData": self._checkWindowDataLayer(layer) elif layer["type"].name() == "DummyData": self._checkDummyDataLayer(layer) for pair in self._missingLayerParams: self._errorMessages.append(("The {} layer must provide the {} parameter.".format( pair[0], pair[1] ),"Missing parameter.")) def _checkDataLayer(self, layer): """Validate a layer of type Data.""" specificParamKey = "data_param" if specificParamKey not in layer["parameters"]: self._missingLayerParams.append([layer["type"].name(), specificParamKey]) else: specificParams = layer["parameters"][specificParamKey] # source if "source" not in specificParams: self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".source"]) elif not os.path.isdir(self._makeAbsPath(specificParams["source"])): self._errorMessages.append(( "The parameter {} of the layer {}\ndoes not seem to point to a valid directory. \n" "Please import and connect databases (via Input Manager).".format(specificParamKey + ".source", layer["type"].name()), "No/invalid input data.")) # TODO check whether the given directory contains files that fit to the given database type. # batch_size if "batch_size" not in specificParams: self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".batch_size"]) elif not common.isPositiveInteger(specificParams["batch_size"]): self._errorMessages.append(("The batch size (of a {} Layer) must be a positive integer.".format( layer["type"].name() ),"Invalid batch size.")) def _checkMemoryDataLayer(self, layer): """Validate a layer of type MemoryData.""" specificParamKey = "memory_data_param" if specificParamKey not in layer["parameters"]: self._missingLayerParams.append([layer["type"].name(), specificParamKey]) else: specificParams = layer["parameters"][specificParamKey] # batch_size if "batch_size" not in specificParams: self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".batch_size"]) elif not common.isPositiveInteger(specificParams["batch_size"]): self._errorMessages.append(("The batch size (of a {} Layer) must be a positive integer.".format( layer["type"].name() ),"Invalid batch size.")) # channels if "channels" not in specificParams: self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".channels"]) elif not common.isPositiveInteger(specificParams["channels"]): self._errorMessages.append(("The number of channels (of a {} Layer) must be a positive integer.".format( layer["type"].name() ),"Invalid number of channels.")) # height if "height" not in specificParams: self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".height"]) elif not common.isPositiveInteger(specificParams["height"]): self._errorMessages.append(("The height parameter (of a {} Layer) must be a positive integer.".format( layer["type"].name() ),"Invalid height parameter.")) # width if "width" not in specificParams: self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".width"]) elif not common.isPositiveInteger(specificParams["width"]): self._errorMessages.append(("The width parameter (of a {} Layer) must be a positive integer.".format( layer["type"].name() ),"Invalid width parameter.")) # actually, this layer isn't supported in the training at all. however, the above validation is kept to # allow using Barista to build(!) a valid net definition without(!) the possibility to start training. self._errorMessages.append(('Layers of type "MemoryData" aren\'t supported yet, as the in-memory data\n' 'reference needs to be specified in the code. So you can\'t run a Barista\n' 'session right now. However, you can use Barista to\n' 'build and export a valid net definition containing this layer type.',"...")) def _checkHDF5DataLayer(self, layer): """Validate a layer of type HDF5Data.""" specificParamKey = "hdf5_data_param" if specificParamKey not in layer["parameters"]: self._missingLayerParams.append([layer["type"].name(), specificParamKey]) else: specificParams = layer["parameters"][specificParamKey] # source if "source" not in specificParams: self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".source"]) elif not os.path.isfile(self._makeAbsPath(specificParams["source"])): self._errorMessages.append(( "The parameter {} of the layer {}\ndoes not seem to point to a valid " "file.".format(specificParamKey + ".source", layer["type"].name()),"No/invalid input data.")) # TODO check whether the given file is really a (valid) HDF5 file. # batch_size if "batch_size" not in specificParams: self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".batch_size"]) elif not common.isPositiveInteger(specificParams["batch_size"]): self._errorMessages.append(("The batch size (of a {} Layer) must be a positive integer.".format( layer["type"].name() ),"Invalid batch size.")) def _checkImageDataLayer(self, layer): """Validate a layer of type ImageData.""" specificParamKey = "image_data_param" if specificParamKey not in layer["parameters"]: self._missingLayerParams.append([layer["type"].name(), specificParamKey]) else: specificParams = layer["parameters"][specificParamKey] # source if "source" not in specificParams: self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".source"]) elif not os.path.isfile(self._makeAbsPath(specificParams["source"])): self._errorMessages.append(( "The parameter {} of the layer {}\ndoes not seem to point to a valid " "file.".format(specificParamKey + ".source", layer["type"].name()),"No/invalid input data.")) # TODO check whether the given file is really a (valid) text file that contains (valid) image paths. # batch_size if "batch_size" not in specificParams: self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".batch_size"]) elif not common.isPositiveInteger(specificParams["batch_size"]): self._errorMessages.append(("The batch size (of a {} Layer) must be a positive integer.".format( layer["type"].name() ),"Invalid batch size.")) def _checkWindowDataLayer(self, layer): """Validate a layer of type WindowData. Constraints are described here: http://caffe.berkeleyvision.org/doxygen/classcaffe_1_1WindowDataLayer.html#details """ specificParamKey = "window_data_param" if specificParamKey not in layer["parameters"]: self._missingLayerParams.append([layer["type"].name(), specificParamKey]) else: specificParams = layer["parameters"][specificParamKey] # source if "source" not in specificParams: self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".source"]) elif not os.path.isfile(self._makeAbsPath(specificParams["source"])): self._errorMessages.append(( "The parameter {} of the layer {}\ndoes not seem to point to a valid " "file.".format(specificParamKey + ".source", layer["type"].name()),"No/invalid input data.")) # TODO add further validation checks for this type (problem: documentation lacks a lot). # batch_size if "batch_size" not in specificParams: self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".batch_size"]) elif not common.isPositiveInteger(specificParams["batch_size"]): self._errorMessages.append(("The batch size (of a {} Layer) must be a positive integer.".format( layer["type"].name() ),"Invalid batch size.")) def _checkDummyDataLayer(self, layer): """Validate a layer of type DummyData. Constraints are described in the comments of the caffe.proto file. """ specificParamKey = "dummy_data_param" if specificParamKey not in layer["parameters"]: self._missingLayerParams.append([layer["type"].name(), specificParamKey]) else: specificParams = layer["parameters"][specificParamKey] # shape if "shape" not in specificParams: self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".shape"]) elif not len(specificParams["shape"]) >= 1: self._errorMessages.append(( "The parameter {} of the layer {}\nneeds at least one value.".format( specificParamKey + ".shape", layer["type"].name() ),"Missing parameter.")) def _checkInputLayer(self): """Check for existence of an input layer. Note that, this is about the specific layer type called "Input". This is not about the general group of Data layers (see tests above). """ if not hasattr(self._stateData, '__getitem__'): self._errorMessages.append(("State data is empty!","Empty state data.")) return for layer_id, layer in self._stateData["network"]["layers"].iteritems(): if layer["type"].name() == "Input": self._errorMessages.append(('Layers of type "Input" aren\'t supported yet, as they are usually not meant\n' 'to be included before deploying the net. So you can\'t run a Barista\n' 'session right now. However, you can use Barista to build and export a valid\n' 'net definition containing this layer type.',"...")) break def _makeAbsPath(self, path): # TODO: remove the print once errorMessages are shown to the user if os.path.isabs(path): return os.path.normpath(path) else: # if path is a relative path, see if it is relative to the session folder sessionDir = self._session.getDirectory() offsetDir = os.path.normpath(os.path.abspath(sessionDir)) candidate = os.path.normpath(os.path.join(offsetDir, path)) lastCandidate = '' while (not os.path.exists(candidate)) and not (candidate == lastCandidate): lastCandidate = candidate offsetDir = os.path.normpath(os.path.join(offsetDir, os.pardir)) candidate = os.path.normpath(os.path.join(offsetDir, path)) if os.path.exists(candidate): return candidate else: print("Could not create an absolute path from: {} and sessionDir {}".format(path, sessionDir)) return path # TODO: remove if not called from anywhere else def _makeSessionPath(self, path): raise NotImplementedError """add ../../ to path to move from project to session level""" for i in range(0, self.sessionDirDepth): path = os.path.join(os.pardir, path) return path backend/barista/constraints/__init__.py0000664ڊ00111170000000054613243274037020507 0ustar klemmso0cvpr""" This package groups some additional constraints required for the Barista program. Those constraints might not be required to create valid caffe protoxt files, but they are required for the Barista program due to whatever. Depending on the subpackages, the included constraints need to be ensured permanently or only at specific times during runtime. """backend/barista/constraints/common.py0000664ڊ00111170000000077513243274037020244 0ustar klemmso0cvpr"""Common (helper) functions to check constraints.""" def isNumber(input): """Check whether input is a valid (float) number.""" try: float(input) return True except ValueError: return False def isPositiveNumber(input): """Check whether input is a valid (float) number > 0.""" return isNumber(input) and input > 0 def isPositiveInteger(input): """Check whether input is a valid integer > 0.""" return isPositiveNumber(input) and float(input).is_integer()backend/barista/session/__init__.py0000664ڊ00111170000000000013243274037017604 0ustar klemmso0cvprbackend/barista/session/client_session.py0000664ڊ00111170000006066613243274037021116 0ustar klemmso0cvprfrom PyQt5.QtCore import QTimer from backend.barista.session.session import * from backend.barista.session.session_utils import State from backend.barista.utils.logger import Log from backend.parser.parser_dummy import ParserDummy from backend.networking.net_util import buildTransaction from backend.networking.protocol import Protocol, SessionProtocol from gui.main_window.docks.weight_visualization.weights import loadNetParamFromString class ClientSession(QObject): # event signals stateChanged = pyqtSignal(object) stateDictChanged = pyqtSignal(object, bool) iterationChanged = pyqtSignal() snapshotAdded = pyqtSignal(object) def __init__(self, project, remote, uid, sid): super(self.__class__, self).__init__() self.options = { SessionProtocol.UPDATESTATE: self._updateState, SessionProtocol.PRINTLOG: self._printLog, SessionProtocol.UPDATEPARSER: self._updateParser, SessionProtocol.UPDATEKEYS: self._updateParserKeys, SessionProtocol.ADDSNAPSHOT: self._addSnapshot, SessionProtocol.PARSEHANDLE: self._parseHandle } self.firstConnect = True self.remote = remote self.isConnected = False self.transaction = None self.caller_id = None self.project = project self.parser = ParserDummy() # One ID for client, on ID for server self.sid = sid self.uid = uid self.lastStateDict = None self.state = State.NOTCONNECTED self.invalidErrorsList = [] # setup timer # see details in setStateDict() self.timer = QTimer() self.timer.timeout.connect(self.transmit) self.timer.setSingleShot(True) self.timer.setInterval(10) self.getStateFirstTime = True self.lastIter = 0 self.lastMaxIter = 0 # maybe get log def processMessage(self): # key = self.transaction.messageOutput[0]["key"] for subkey in self.options.keys(): if self.transaction.containsAttr(["subkey", subkey]): self.options[subkey]() return self.transaction.stage() def _assertConnection(self): if self.isConnected: return True return self._connect() def _connect(self): if not self.isConnected: trans = buildTransaction(self.remote[0], self.remote[1]) if trans: trans.send({"key": Protocol.CONNECTTOSESSION, "uid": self.uid}) ret = trans.asyncRead(attr=("key", Protocol.CONNECTTOSESSION)) if ret: if ret["status"]: self.transaction = trans self.isConnected = True self.transaction.socketClosed.connect(self._disconnect) self.transaction.bufferReady.connect(self.processMessage) self.firstConnect = False if not self.firstConnect: self.setState(self.getState(False)) return True else: # connected but session not found self._handleErrors(ret["error"]) trans.close() return False else: # connected but no answer self._handleErrors(["Communication error on connecting to session."]) # TODO better error msg trans.close() return False else: # can't connect to host self._handleErrors(["Host not found"]) # TODO add more info return False return True def _disconnect(self): if self.isConnected: self.transaction.socketClosed.disconnect() self.transaction.bufferReady.disconnect() self.transaction.close() self.isConnected = False self.transaction = None self.setState(State.NOTCONNECTED) self.getStateFirstTime = True self._handleErrors(["Session: Connection to host lost!"]) def _handleErrors(self, errors): for error in errors: Log.error(error, self.getCallerId()) def _printLog(self): msg = self.transaction.asyncRead(attr=("subkey", SessionProtocol.PRINTLOG)) logs = msg["log"] for log, error in logs: if not isinstance(log, list): log = [log] # if msg["isError"]: if error: self._handleErrors(log) else: for l in log: Log.log(l, self.getCallerId()) def _updateIter(self): msg = self.transaction.asyncRead(attr=("subkey", SessionProtocol.UPDATEITER)) self.lastIter = msg["iteration"] self.iterationChanged.emit() def _updateParser(self): msg = self.transaction.asyncRead(attr=("subkey", SessionProtocol.UPDATEPARSER)) phase = msg["phase"] row = msg["row"] self.lastIter = row["NumIters"] self.iterationChanged.emit() self.parser.sendParserUpdate(phase, row) #TODO plotter def _updateParserKeys(self): msg = self.transaction.asyncRead(attr=("subkey", SessionProtocol.UPDATEKEYS)) phase = msg["phase"] key = msg["parserkey"] self.parser.sendParserRegisterKeys(phase, key) # Log.log("Found key: " + key, self.getCallerId()) def _addSnapshot(self): msg = self.transaction.asyncRead(attr=("subkey", SessionProtocol.ADDSNAPSHOT)) snapshot = msg["snapshot"] self.snapshotAdded.emit(snapshot) def _parseHandle(self): msg = self.transaction.asyncRead(attr=("subkey", SessionProtocol.PARSEHANDLE)) event = msg["event"] message = msg["message"] groups = msg["groups"] self.parser.sendParserHandle(event, message, groups) def getCallerId(self): """ Return the unique caller id for this session """ if self.caller_id is None: self.caller_id = Log.getCallerId(self._getLogFileName()) return self.caller_id def _getLogFileName(self): """ Return the name of the logfile with session id. """ log_file = self.project.getProjectName() + '_' + str(self.getSessionId()) + '.log' return log_file def getSessionId(self): return self.sid def checkFiles(self): if self._assertConnection(): msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.CHECKFILES} self.transaction.send(msg) ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.CHECKFILES)) if ret: self._handleErrors(ret["error"]) return self._handleErrors(["Failed to connect to remote session to check files."]) def getSnapshots(self): """ Return all snapshot files, keyed by iteration number. """ if self._assertConnection(): msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.GETSNAPSHOTS} self.transaction.send(msg) ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.GETSNAPSHOTS)) if ret: return ret["snapshots"] self._handleErrors(["Failed to connect to remote session to acquire Snapshots."]) return {} def getIteration(self): # Maybe let this be local + signal if self._assertConnection(): msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.GETITERATION} self.transaction.send(msg) ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.GETITERATION)) if ret: self.lastIter = ret["iteration"] return self.lastIter self._handleErrors(["Failed to connect to remote session to acquire current iteration."]) return 0 def getMaxIteration(self): # maybe let this be local + signal if self._assertConnection(): msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.GETMAXITERATION} self.transaction.send(msg) ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.GETMAXITERATION)) if ret: self.lastMaxIter = ret["iteration"] return self.lastMaxIter self._handleErrors(["Failed to connect to remote session to acquire max iteration."]) return 0 def delete(self): if self._assertConnection(): msg = {"key": Protocol.DELETESESSION, "uid": self.uid} self.transaction.send(msg) ret = self.transaction.asyncRead(staging=True, attr=("key", Protocol.DELETESESSION)) if ret: if ret["status"]: self.project.deleteSession.emit(self.getSessionId()) else: self._handleErrors(["Failed to connect to remote session to acquire current state."]) self.setState(State.NOTCONNECTED, True) def reset(self): if self._assertConnection(): msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.RESET} self.transaction.send(msg) ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.RESET)) if ret: if ret["status"]: self.iterationChanged.emit() self.project.resetSession.emit(self.getSessionId()) return True else: self._handleErrors(ret["error"]) self._handleErrors(["Could not reset session in state " + str(self.getState())]) return False def getState(self, local=True): if local and not self.getStateFirstTime: return self.state self.getStateFirstTime = False if self._assertConnection(): msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.GETSTATE} self.transaction.send(msg) ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.GETSTATE)) if ret: if ret["status"]: self.setState(ret["state"], True) return ret["state"] else: self._handleErrors(ret["error"]) self.setState(State.UNDEFINED, True) return State.UNDEFINED self._handleErrors(["Failed to connect to remote session to acquire current state."]) self.setState(State.NOTCONNECTED, True) return State.NOTCONNECTED def setState(self, state, silent=False): self.state = state if not silent: self.stateChanged.emit(self.state) def setErrorList(self, errorList): self.invalidErrorsList = errorList def _updateState(self): msg = self.transaction.asyncRead(attr=("subkey", SessionProtocol.UPDATESTATE)) self.setState(msg["state"]) def getErrorList(self): return self.invalidErrorsList def save(self): Log.log("Saving current Session status to disk.", self.getCallerId()) if self._assertConnection(): msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.SAVE} self.transaction.send(msg) ret = self.transaction.asyncRead(attr=("subkey", SessionProtocol.SAVE)) if ret: if ret["status"]: return True else: self._handleErrors(ret["error"]) self._handleErrors(["Could not save session."]) return False def setStateDict(self, stateDict): self.lastStateDict = stateDict # to prevent subsequent writes, wait until some time passed before sending the statedict self.timer.stop() self.timer.start() def transmit(self): if self._assertConnection(): if not self._validateLayerOrder(self.lastStateDict): self._handleErrors(["Warning: layerorder does not match layers"]) return msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.SETSTATEDICT} # remove types for transfer transferDict = copy.deepcopy(self.lastStateDict) layers = transferDict["network"]["layers"] for id in layers: del layers[id]["type"] msg["statedict"] = transferDict self.transaction.send(msg) ret = self.transaction.asyncRead(attr=("subkey", SessionProtocol.SETSTATEDICT)) if ret: if ret["status"]: self.stateDictChanged.emit(self, False) return self._handleErrors(ret["error"]) self._handleErrors(["SetStateDict: Failed to send StateDict"]) # TODO improve warnings self.stateDictChanged.emit(self, False) def _validateLayerOrder(self, dict): """There is a fundamental problem in how the setStateDict function is designed. In theory there should by only one write one write access. But in reality layerorder and the layer itself are writen in subsequent operations. this leads to a statedict with a layer in the layerorder, but no corresponding layer in the net. thus the save function produces a key error since they iterate over the layerorder. simply waiting until the layer is in layerorder and layer solves this problem. this function verifies this for all IDs. """ for id in dict["network"]["layerOrder"]: if id not in dict["network"]["layers"]: return False return True @property def state_dictionary(self): if self.lastStateDict is None: if self._assertConnection(): msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.GETSTATEDICT} self.transaction.send(msg) ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.GETSTATEDICT)) if ret: if ret["status"]: self.lastStateDict = ret["statedict"] try: layers = self.lastStateDict["network"]["layers"] for id in layers: if "parameters" in layers[id]: if "type" in layers[id]["parameters"]: typename = layers[id]["parameters"]["type"] layers[id]["type"] = info.CaffeMetaInformation().availableLayerTypes()[typename] except KeyError: pass return self.lastStateDict else: self._handleErrors(ret["error"]) if self.lastStateDict is None: return {u'position': {}, u'selection': [], u'hidden_connections': {}, u'network': {u'layers': {}, u'name': u'default', u'layerOrder': []}, u'solver': {u'net': u'net-internal.prototxt'}} return self.lastStateDict @state_dictionary.setter def state_dictionary(self, value): self.setStateDict(value) def hasStateDict(self): return True def checkTraining(self): if self._assertConnection(): msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.CHECKTRAINING} self.transaction.send(msg) ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.CHECKTRAINING)) if ret: if ret["status"]: return ret["check"] else: self._handleErrors(msg["error"]) return [(msg["error"], "")] else: self._handleErrors(["CheckTraining: Did not receive a reply from host."]) return [("CheckTraining: Did not receive a reply from host.", "No reply")] else: self._handleErrors(["CheckTraining: Failed to connect to host to check session validity!"]) return [("CheckTraining: Failed to connect to host to check session validity!", "Connection failed")] def start(self, solverstate=None, caffemodel=None): if self._assertConnection(): msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.START} if solverstate is not None: msg["solverstate"] = solverstate if caffemodel is not None: msg["caffemodel"] = caffemodel self.transaction.send(msg) ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.START)) if ret: if ret["status"]: return True else: self._handleErrors(ret["error"]) self._handleErrors(["Could not start session in state " + str(self.getState())]) return False def pause(self): if self._assertConnection(): msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.PAUSE} self.transaction.send(msg) ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.PAUSE)) if ret: if ret["status"]: self.iterationChanged.emit() return True else: self._handleErrors(ret["error"]) self._handleErrors(["Could not pause session in state " + str(self.getState())]) return False def snapshot(self): if self._assertConnection(): msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.TAKESNAPSHOT} self.transaction.send(msg) ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.TAKESNAPSHOT)) if ret: if ret["status"]: return True else: self._handleErrors(ret["error"]) self._handleErrors(["Could not take snapshot of session in state " + str(self.getState())]) return False def proceed(self, snapshot=None): # TODO snapshot if self._assertConnection(): msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.PROCEED} if snapshot is not None: msg["snapshot"] = snapshot print "proceed with snapshot", msg self.transaction.send(msg) ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.PROCEED)) if ret: if ret["status"]: return True else: self._handleErrors(ret["error"]) self._handleErrors(["Could not proceed session in state " + str(self.getState())]) return False def getParser(self): return self.parser def getLogFileName(self, basename=False): if basename: return self.project.getProjectName() + "_" + str(self.sid) + ".log" else: print "There is no 'getLogFileName' for clientsessions!" return None def getLogId(self): return self.uid def isRemote(self): return True def fetchParserData(self): if self._assertConnection() and self.transaction is not None: msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.FETCHPARSERDATA} self.transaction.send(msg) ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.FETCHPARSERDATA)) if ret: if ret["status"]: parserRows = ret["ParserRows"] parserKeys = ret["ParserKeys"] parserHandle = ret["ParserHandle"] parserLog = ret["ParserLogs"] for phase, key in parserKeys: self.parser.sendParserRegisterKeys(phase, key) for phase, row in parserRows: self.parser.sendParserUpdate(phase, row) for event, line, groups in parserHandle: self.parser.sendParserHandle(event, line, groups) for log, error in parserLog: if error: Log.error(log, self.getCallerId()) else: Log.log(log, self.getCallerId()) return else: self._handleErrors(ret["error"]) self._handleErrors(["Failed to fetch parser data from host."]) def readInternalNetFile(self): """ Returns the contents of the internal net prototxt file. Wrapper around loadInternalNetFile, to provide a coherent interface together with the Session class. """ return self.loadInternalNetFile() def readDeployedNetAsString(self): """ Returns the contents of the deployable net prototxt file. Wrapper around loadDeployedNetFile, to provide a coherent interface together with the Session class. """ return self.loadDeployedNetAsString() def readCaffemodelFile(self, snapshot): """ Returns the contents of the .caffemodel file that belongs to snapshot. snapshot: string Filename without path of the snapshot file. """ return self.loadCaffemodel(snapshot) def loadInternalNetFile(self): if self._assertConnection(): msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.LOADINTERNALNET} self.transaction.send(msg) ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.LOADINTERNALNET)) if ret: if ret["status"]: return ret["NetInternal"] else: self._handleErrors(ret["error"]) self._handleErrors(["Failed to load InternalNet Prototxt"]) return "" def loadDeployedNetAsString(self): if self._assertConnection(): msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.LOADDEPLOYEDNET} self.transaction.send(msg) ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.LOADDEPLOYEDNET)) if ret: if ret["status"]: return ret["Net"] else: self._handleErrors(ret["error"]) self._handleErrors(["Failed to load Deployed Net"]) return "" def loadNetParameter(self, snapshot): if self._assertConnection(): msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.LOADNETPARAMETER} msg["snapshot"] = snapshot self.transaction.send(msg) ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.LOADNETPARAMETER)) if ret: if ret["status"]: net = ret["NetParams"] net = loadNetParamFromString(net) if net is not None: return net else: self._handleErrors(["Failed to load " + str(snapshot), "HDF5 snapshot format is not supported for weight " "visualization. This can be changed by setting the " "snapshot_format in the solver properties."]) else: self._handleErrors(ret["error"]) self._handleErrors(["Failed to load NetParameter for snapshot '" + snapshot + "'"]) return None def getCaffemodelContents(self, snapshot): """ Wrapper around the loadCaffemodel function to provide a coherent interface between local and remote sessions. """ return self.loadCaffemodel(snapshot) def loadCaffemodel(self, snapshot): """ Loads the contents of the caffemodel file belonging to the given snapshot from the server. """ if self._assertConnection(): msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.LOADCAFFEMODEL} msg["snapshot"] = snapshot self.transaction.send(msg) ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.LOADCAFFEMODEL)) if ret: if ret["status"]: return ret["caffemodel"] else: self._handleErrors(ret["error"]) self._handleErrors(["Failed to load caffemodel contents for snapshot '" + snapshot + "'"]) return None backend/barista/session/server_session.py0000664ڊ00111170000012151213243274037021132 0ustar klemmso0cvprimport copy import json import os import re import signal import sys import time import uuid import shutil from collections import OrderedDict from subprocess import Popen, PIPE, STDOUT from PyQt5.Qt import QObject from PyQt5.QtCore import Qt, pyqtSignal import backend.caffe.dict_helper as helper import backend.caffe.proto_info as info import backend.caffe.saver as saver from backend.barista.constraints.session_run.training import checkMinimumTrainingRequirements from backend.barista.session.session import State from backend.barista.session.session_common import SessionCommon from backend.barista.session.session_pool import SessionPool from backend.barista.session.session_utils import Paths, Events from backend.networking.protocol import Protocol, SessionProtocol from backend.parser.concatenator import Concatenator from backend.parser.parser import Parser from backend.parser.parser_listener import ParserListener from backend.barista.deployed_net import DeployedNet from PyQt5.QtCore import QTimer from threading import Lock class ServerSession(QObject, ParserListener, SessionCommon): logsig = pyqtSignal(str, bool) # This is the way to do threadsafe logging while the session is running parssig = pyqtSignal(str, OrderedDict) keysig = pyqtSignal(str, str) statesig = pyqtSignal(int) snapshotsig = pyqtSignal(str) handlesig = pyqtSignal(str, str, list) # a runnning session is moved to another thread (which is not a QThread and has no main-loop) # using the network feature causes warnings unless every operation is connected with a signal to the # original network method with Qt.Autoconnect (theoretically Qt.QueuedConnection should work as well) def __init__(self, manager, directory, parse_old=False, pid=""): super(self.__class__, self).__init__() self.options = {SessionProtocol.CHECKFILES: self._msgCheckFiles, SessionProtocol.CHECKTRAINING: self._msgCheckTraining, SessionProtocol.GETSNAPSHOTS: self._msgGetSnapshots, SessionProtocol.GETITERATION: self._msgGetIteration, SessionProtocol.GETMAXITERATION: self._msgGetMaxIteration, SessionProtocol.GETSTATE: self._msgGetState, SessionProtocol.SETSTATEDICT: self._msgSetStateDict, SessionProtocol.GETSTATEDICT: self._msgGetStateDict, SessionProtocol.SAVE: self._msgSave, SessionProtocol.START: self._start, SessionProtocol.PAUSE: self._pause, SessionProtocol.PROCEED: self._proceed, SessionProtocol.TAKESNAPSHOT: self._takeSnapshot, SessionProtocol.FETCHPARSERDATA: self._msgFetchParserData, SessionProtocol.LOADINTERNALNET: self._msgLoadInternalNet, SessionProtocol.LOADDEPLOYEDNET: self._msgLoadDeployedNet, SessionProtocol.LOADNETPARAMETER: self._msgLoadNetParameter, SessionProtocol.LOADCAFFEMODEL: self._msgLoadCaffemodel, SessionProtocol.RESET: self._reset, SessionProtocol.DELETE: self.delete} self.logsig.connect(self.Log, Qt.AutoConnection) self.parssig.connect(self.addParserRow, Qt.AutoConnection) self.keysig.connect(self.addParserKey, Qt.AutoConnection) self.statesig.connect(self.setState, Qt.AutoConnection) self.snapshotsig.connect(self.addSnapshot, Qt.AutoConnection) self.handlesig.connect(self.addHandle, Qt.AutoConnection) self.transaction = None self.isConnected = False self.manager = manager self.directory = directory self.pid = None self.sid = None self.uid = str(uuid.uuid4()) self.snapshot_dir = None self.snapshot_prefix = None self.iteration = 0 self.max_iter = 1 self.state_dictionary = {} self.state = State.WAITING self.invalidErrorsList = [] self.last_solverstate = None self.logs = os.path.join(directory, 'logs') # run stuff self.rid = 0 self.tee = None self.proc = None self.parser = None self.lock = Lock() self.parse_old = False self.parser_initialized = False self.parserRows = [] self.parserKeys = [] self.parserHandle = [] self.parserLogs = [] self.timer = QTimer() self.timer.timeout.connect(self.transmitLog) self.timer.setSingleShot(True) self.timer.setInterval(10) self.logBuffer = [] self.logLock = Lock() if not parse_old: self.pid = pid else: self.rebuild() self._parseSessionAndRunID() self.parse_old = True def __lt__(self, other): if type(other) is not ServerSession: return True return other.sid - self.sid < 0 def __gt__(self, other): if type(other) is not ServerSession: return False return other.sid - self.sid > 0 def __eq__(self, other): if type(other) is not ServerSession: return False return other.sid - self.sid == 0 def __le__(self, other): if type(other) is not ServerSession: return True return other.sid - self.sid <= 0 def __ge__(self, other): if type(other) is not ServerSession: return False return other.sid - self.sid >= 0 def __ne__(self, other): if type(other) is not ServerSession: return True return other.sid - self.sid != 0 def rebuild(self): filename = os.path.join(self.directory, Paths.FILE_NAME_SESSION_JSON) if os.path.isfile(filename): with open(filename, 'r') as f: try: res = json.load(f) self._parseSetting(res) except Exception: sys.stderr.write("ERROR on reading json data from: " + filename + "\n") def connect(self, transaction): if not self.isConnected: self.isConnected = True self.transaction = transaction self.transaction.bufferReady.connect(self.processMessage) self.transaction.socketClosed.connect(self.disconnect) return True return False def disconnect(self): self.isConnected = False if self.transaction is not None: self.transaction.bufferReady.disconnect() self.transaction.socketClosed.disconnect() self.transaction = None def processMessage(self): # key = self.transaction.messageOutput[0]["key"] key, subkey = self.transaction.getAttrOfFirst(["key", "subkey"]) if key is Protocol.SESSION: # if "subkey" in self.transaction.messageOutput[0]: # subkey = self.transaction.messageOutput[0]["subkey"] if subkey is not None: if subkey in self.options: self.options[subkey]() else: sys.stderr.write("Wrong subkey for session: " + str(subkey) + "\n") msg = self.transaction.asyncRead() msg["status"] = False msg["error"] = ["Wrong subkey for session: " + str(subkey)] self.transaction.send(msg) return else: sys.stderr.write("Missing subkey for session\n") msg = self.transaction.asyncRead() msg["status"] = False msg["error"] = ["Missing subkey for session"] self.transaction.send(msg) return def setErrorList(self, errorList): self.invalidErrorsList = errorList def setState(self, state): self.state = state if (self.isConnected): msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.UPDATESTATE} msg["state"] = state self.transaction.send(msg) def Log(self, log, error=False): self.logLock.acquire() self.logBuffer.append((log, error)) self.logLock.release() self.parserLogs.append((log, error)) self.timer.stop() self.timer.start() # if self.isConnected: # msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.PRINTLOG} # msg["log"] = log # msg["isError"] = error # self.transaction.send(msg) def transmitLog(self): self.logLock.acquire() log = self.logBuffer self.logBuffer = [] self.logLock.release() if self.isConnected: msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.PRINTLOG} msg["log"] = log self.transaction.send(msg) def addParserRow(self, phase, row): self.parserRows.append((phase, row)) if self.isConnected: msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.UPDATEPARSER, "phase": phase, "row": row} self.transaction.send(msg) def addParserKey(self, phase, key): self.parserKeys.append((phase, key)) if self.isConnected: msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.UPDATEKEYS, "phase": phase, "parserkey": key} self.transaction.send(msg) def addSnapshot(self, snapshot): if self.isConnected: msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.ADDSNAPSHOT, "snapshot": snapshot} self.transaction.send(msg) def addHandle(self, event, message, groups): self.parserHandle.append([event, message, groups]) if self.isConnected: msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.PARSEHANDLE, "event": event, "message": message, "groups": groups} self.transaction.send(msg) def getDirectory(self): return self.directory def getErrorList(self): return self.invalidErrorsList def setInitialSid(self, sid): self.sid = sid def _msgCheckFiles(self): """ Check for the existence of the session directories and files. """ msg = self.transaction.asyncRead() errors = [] if os.path.exists(self.getDirectory()) is False: errors.append('Session directory does not exists: ' + self.directory) if os.path.exists(self.logs) is False: errors.append('Log directory does not exists: ' + self.logs) if os.path.exists(self.snapshot_dir) is False: errors.append('Snapshot directory does not exists: ' + self.snapshot_dir) if os.path.exists(self.manager.parent.caffePath) is False: errors.append('CAFFE_ROOT directory does not exists: ' + self.manager.parent.caffePath) msg["error"] = errors self.transaction.send(msg) def _msgCheckTraining(self): msg = self.transaction.asyncRead() req = checkMinimumTrainingRequirements(self) self.setState(self._getState()) msg["status"] = True msg["check"] = req self.transaction.send(msg) def delete(self): self.pause() # make sure the session is not running try: shutil.rmtree(self.getDirectory()) except OSError as e: sys.stderr.write(str(e)+'\n') # python docs say, that rmtree should raise an OSError code 66 if dir is not empty. However, this does not seem to happen. self.disconnect() return def _msgGetSnapshots(self): """ Return all snapshot files, keyed by iteration number. """ msg = self.transaction.asyncRead() msg["snapshots"] = self._getSnapshots() self.transaction.send(msg) def _getSnapshots(self): regex_snapshot = re.compile('iter_([\d]+)\.solverstate') snaps = {} if os.path.exists(self.getSnapshotDirectory()): for entry in os.listdir(self.getSnapshotDirectory()): snap_match = regex_snapshot.search(entry) if snap_match: try: iter_num = int(snap_match.group(1)) snaps[iter_num] = entry except Exception: pass return snaps def _msgGetIteration(self): msg = self.transaction.asyncRead() msg["iteration"] = self.getIteration() self.transaction.send(msg) def _msgGetMaxIteration(self): msg = self.transaction.asyncRead() msg["iteration"] = self.getMaxIteration() self.transaction.send(msg) def _msgGetState(self): msg = self.transaction.asyncRead() msg["state"] = self._getState() msg["status"] = True self.transaction.send(msg) def getIteration(self): return self.iteration def getMaxIteration(self): return self.max_iter def _getState(self): """ Return the state of the session. """ if self.state == State.FAILED: return self.state if self.proc is not None: if self.iteration == self.max_iter: self.state = State.FINISHED else: self.state = State.RUNNING else: if len(checkMinimumTrainingRequirements(self)) > 0: self.state = State.INVALID elif self.iteration == self.max_iter: self.state = State.FINISHED elif len(self._getSnapshots()) > 0: self.state = State.PAUSED else: self.state = State.WAITING return self.state def _msgSetStateDict(self): msg = self.transaction.asyncRead() self.state_dictionary = msg["statedict"] self._parseSetting(self.state_dictionary) # restore lost types if hasattr(self.state_dictionary, '__getitem__'): if "network" in self.state_dictionary: if "layers" in self.state_dictionary["network"]: layers = self.state_dictionary["network"]["layers"] for id in layers: if "parameters" in layers[id]: if "type" in layers[id]["parameters"]: typename = layers[id]["parameters"]["type"] layers[id]["type"] = info.CaffeMetaInformation().availableLayerTypes()[typename] del msg["statedict"] msg["status"] = True self.transaction.send(msg) def _msgGetStateDict(self): msg = self.transaction.asyncRead() transferDict = copy.deepcopy(self.state_dictionary) try: layers = transferDict["network"]["layers"] for id in layers: del layers[id]["type"] msg["statedict"] = transferDict msg["status"] = True except KeyError as e: msg["statedict"] = transferDict msg["status"] = True self.transaction.send(msg) def save(self, includeProtoTxt = False): """Saves the current session to prototxt files and session_settings json file.""" res = self.__ensureDirectory() if len(res) > 0: return res toSave = {"SessionState": self.state, "Iteration": self.iteration, "MaxIter": self.max_iter} toSave["UID"] = self.uid toSave["SID"] = self.sid toSave["ProjectID"] = self.pid if self.last_solverstate: toSave["LastSnapshot"] = self.last_solverstate if self.state_dictionary: serializedDict = copy.deepcopy(self.state_dictionary) if "network" in serializedDict: if includeProtoTxt: netDict = copy.deepcopy(self.state_dictionary["network"]) net = saver.saveNet(netdict=netDict) with open(os.path.join(self.directory, Paths.FILE_NAME_NET_ORIGINAL), 'w') as f: f.write(net) if "layers" in serializedDict["network"]: layers = serializedDict["network"]["layers"] for id in layers: del layers[id]["type"] toSave["NetworkState"] = serializedDict filename = os.path.join(self.directory, Paths.FILE_NAME_SESSION_JSON) # clear the file. sometimes json.dump does not exitclean and causes valuerrors on load open(filename, 'w').close() with open(filename, "w") as f: json.dump(toSave, f, sort_keys=True, indent=4) return [] def prepairInternalPrototxt(self): error = [] serializedDict = copy.deepcopy(self.state_dictionary) if "solver" in self.state_dictionary: solverDict = self.state_dictionary["solver"] if self.manager.parent.trainOnHW == 0: solverDict["solver_mode"] = 'CPU' else: solverDict["solver_mode"] = 'GPU' solver = saver.saveSolver(solverdict=solverDict) with open(os.path.join(self.directory, Paths.FILE_NAME_SOLVER), 'w') as f: f.write(solver) else: error.append("There is no solver") if "network" in serializedDict: net, error = self._modifyNetDictionaryToInternalVersion(serializedDict["network"]) net = saver.saveNet(net) with open(os.path.join(self.directory, Paths.FILE_NAME_NET_INTERNAL), 'w') as f: f.write(net) else: error.append("There is no network") return error def _modifyNetDictionaryToInternalVersion(self, net): # define all layer parameters that can contain (relative) paths # (use a dot to separate nested parameters) layerParamsContainingPaths = [ "data_param.source", "hdf5_data_param.source", "image_data_param.source", "window_data_param.source", "data_param.mean_file", "hdf5_output_param.file_name", "image_data_param.mean_file", "window_data_param.mean_file", "transform_param.mean_file"] error = [] # evaluate layer after layer h = helper.DictHelper(net) for layerId, layer in net.get("layers", {}).iteritems(): # evaluate parameter after parameter for paramKey in layerParamsContainingPaths: # if the current layer does have the current parameter.. if h.layerParameterIsSet(layerId, paramKey): paramValue = h.layerParameter(layerId, paramKey) # ..and its value is a relative path: modify it if paramValue is not None and not os.path.isabs(paramValue): if paramKey != "hdf5_data_param.source": newPath = os.path.join(os.pardir, paramValue) else: newFilename = str(layerId) + ".txt" newPath = os.path.join(os.curdir, newFilename) parentPath = self.manager.parent.sessionPath oldPath = os.path.join(parentPath, paramValue) # this builds the new hdf5 files if os.path.exists(oldPath): with open(os.path.join(self.directory, newPath), 'w') as f: lines = [line.rstrip('\n') for line in open(oldPath)] for line in lines: if line is not "": if line[:1] == '.': line = os.path.join(os.pardir, line) f.write("\n" + line) else: self.Log("Failed to copy hdf5txt file. File does not exists: " + oldPath, True) error.append("Failed to copy hdf5txt file. File does not exists: " + oldPath) h.setLayerParameter(layerId, paramKey, newPath) return net, error def _parseSetting(self, settings): if not hasattr(settings, '__getitem__'): return else: if "UID" in settings: self.uid = settings["UID"] if "SID" in settings: self.sid = settings["SID"] if "ProjectID" in settings: self.pid = settings["ProjectID"] if "SessionState" in settings: self.state = settings["SessionState"] if self.state == State.RUNNING: self.state = State.PAUSED self.previousState = self.state if "Iteration" in settings: self.iteration = settings["Iteration"] if "solver" in settings: if "max_iter" in settings["solver"]: self.max_iter = settings["solver"]["max_iter"] elif "MaxIter" in settings: self.max_iter = settings["MaxIter"] if "LastSnapshot" in settings: self.last_solverstate = settings["LastSnapshot"] if "NetworkState" in settings: self.state_dictionary = settings["NetworkState"] layers = self.state_dictionary["network"]["layers"] for id in layers: if "parameters" in layers[id]: if "type" in layers[id]["parameters"]: typename = layers[id]["parameters"]["type"] layers[id]["type"] = info.CaffeMetaInformation().availableLayerTypes()[typename] solver = self.state_dictionary["solver"] if solver: if "snapshot_prefix" in solver: self.snapshot_prefix = solver["snapshot_prefix"] def _parseSessionAndRunID(self): if self.sid is None: regex_sid = re.compile('[\d]{8}_[\d]{6}_([\d]+)') sid_match = regex_sid.search(self.directory) session_id = None if sid_match: try: self.sid = int(sid_match.group(1)) except: pass if self.sid is None: self.sid = 0 regex_rid = re.compile('([\d]+)\.([\d]+)\.log') run_id = 0 if os.path.exists(self.logs): for entry in os.listdir(self.logs): rid_match = regex_rid.search(entry) if rid_match: try: _run_id = int(rid_match.group(2)) if _run_id > run_id: run_id = _run_id except: pass self.rid = run_id def getSnapshotDirectory(self): """ Return the snapshot directory. """ return self.getDirectory() # if self.snapshot_dir: # return self.snapshot_dir # snapshot_prefix = self._getSnapshotPrefix() # sdir = os.path.dirname(snapshot_prefix) # self.snapshot_dir = os.path.join(self.directory, sdir) # return self.snapshot_dir def _getSnapshotPrefix(self): """ Return the snapshot prefix which is used for snapshots of this session. """ if not self.snapshot_prefix: pass # self.__parseSnapshotPrefix() # TODO if not self.snapshot_prefix: self.snapshot_prefix = '' return self.snapshot_prefix def _start(self): msg = self.transaction.asyncRead() if self.manager.isTraining(): msg["error"] = ["Could not start Session. There is already one running Session on this Server."] msg["status"] = False self.transaction.send(msg) return solverstate = None if "solverstate" in msg: solverstate = msg["solverstate"] caffemodel = None if "caffemodel" in msg: caffemodel = msg["caffemodel"] msg["error"] = self.start(solverstate, caffemodel) msg["status"] = len(msg["error"]) == 0 self.transaction.send(msg) def _pause(self): msg = self.transaction.asyncRead() msg["status"] = self.pause() msg["iteration"] = self.iteration if not msg["status"]: msg["error"] = ["Could not pause a session in state " + str(self.state)] self.transaction.send(msg) self.save() def _reset(self): msg = self.transaction.asyncRead() msg["status"] = self.reset() msg["iteration"] = 0 if not msg["status"]: msg["error"] = ["Could not reset a session in state " + str(self.state)] self.transaction.send(msg) self.save() def _proceed(self): msg = self.transaction.asyncRead() if self.manager.isTraining(): msg["error"] = ["Could not proceed Session. There is already one running Session on this Server."] msg["status"] = False self.transaction.send(msg) return snapshot = None if "snapshot" in msg: snapshot = msg["snapshot"] msg["error"] = self.proceed(snapshot) msg["status"] = len(msg["error"]) == 0 self.transaction.send(msg) def _msgSave(self): msg = self.transaction.asyncRead() res = self.save() msg["status"] = len(res) == 0 msg["error"] = res self.transaction.send(msg) def _takeSnapshot(self): msg = self.transaction.asyncRead() msg["status"] = self.snapshot() if not msg["status"]: msg["error"] = ["Could not take snapshot of session."] self.transaction.send(msg) def getRunLogFileName(self, basename=False): """ Return the name of the logfile with session and run id. """ log_file = 'server_' + str(self.sid) + '.' + str(self.rid) + '.log' if basename is True: return log_file log_file = os.path.join(self.getLogs(), log_file) return log_file def getParser(self): """ Return the log parser of this session. """ if self.parser is None: self.parser = Parser(None, Events.events) self.parser.setLogging(True) self.parser.printLogSignl.connect(self.Log, Qt.AutoConnection) self.parser.addListener(self) return self.parser def getLogs(self): """ Return the log directory. """ logs = os.path.join(self.directory, "logs") if os.path.exists(logs) is False: os.makedirs(logs) return logs def startParsing(self): """ Create a parser and run it in a newly dispatched thread. """ self.parseOldLogs() logs = self.getStream() self.getParser().addLogStream(logs) pool = SessionPool() pool.addSession(self) def parseOldLogs(self): """ Parse all log files in the log directory. """ locked = self.lock.acquire() if locked is False: return try: if self.parse_old: self.parse_old = False log_files = {} regex_filename = re.compile('[\d]+\.([\d]+)\.log$') for entry in os.listdir(self.getLogs()): filename_match = regex_filename.search(entry) if filename_match: # key files by run id try: run_id = int(filename_match.group(1)) log_files[run_id] = entry except: pass log_list = [] for run_id in sorted(log_files.keys()): log_file = os.path.join(self.getLogs(), log_files[run_id]) log_list.append(log_file) con = Concatenator(log_list) logs = con.concate() for log in logs: try: self.getParser().addLogStream(log) except Exception as e: self.Log('Failed to parse log file ' + self.getLogFileName(True) + ": " + str(e), True) except Exception as e: self.Log('Failed to parse old log ' + str(e), True) finally: if locked: self.lock.release() def getLogFileName(self, basename=False): """ Return the name of the logfile with session id. """ log_file = 'server_' + str(self.getSessionId()) + '.log' if basename is True: return log_file log_file = os.path.join(self.getLogs(), log_file) return log_file def getStream(self): """ Return the log stream of this session. This is an iterator over stdout of the subprocess. """ if self.tee is not None: return iter(self.tee.stdout.readline, '') if self.proc is not None: return iter(self.proc.stdout.readline, '') return iter([]) def update(self, phase, row): self.iteration = row['NumIters'] self.parssig.emit(phase, row) def parsingFinished(self): """ Called when the parser has processed all available streams. """ if self.parser_initialized: self.logsig.emit("Parsing Finished", False) if self.proc is not None: # Wait for caffe process, kill tee and respond to return code assert self.state is State.RUNNING rcode = self.proc.wait() self.proc = None try: self.tee.kill() except Exception: pass self.tee = None if rcode is 0: self.setFinished() else: self.setState(State.FAILED) self.Log('Session failed with return code ' + str(rcode), True) self.parser_initialized = True def registerKey(self, phase, key): # TODO save the keys and send them on connection self.keysig.emit(phase, key) def handle(self, event, message, groups): self.handlesig.emit(event, message, groups) if event == 'OptimizationDone': self.save() self.setState(State.FINISHED) self.proc = None elif event == 'max_iter': self.max_iter = int(groups[0]) # todo update max_iter elif event == 'state_snapshot': self.last_solverstate = groups[0] #if self.parser_initialized: self.snapshotsig.emit(self.last_solverstate) elif event == 'model_snapshot': self.last_caffemodel = groups[0] def setFinished(self): self.iteration = self.max_iter self.setState(State.FINISHED) def start(self, solverstate=None, caffemodel=None): # TODO only one training per server error = [] # (re-)write all session files error.extend(self.save(includeProtoTxt=True)) if not os.path.exists(self.logs): try: os.makedirs(self.logs) except OSError as e: error.extend(['Failed to create Folder: ' + str(e)]) if self.rid is None: self._parseSessionAndRunID() if self.state is State.WAITING: self.rid += 1 error.extend(self.prepairInternalPrototxt()) if len(error) > 0: return error caffepath = self.manager.caffePath if not os.path.exists(caffepath): return ["No CaffePath"] try: self.getParser().setLogging(True) # TODO set hardware trainOnHW cmd = [ os.path.join(caffepath, 'build/tools/caffe'), 'train', '-solver', os.path.join(self.directory, Paths.FILE_NAME_SOLVER)] if solverstate is not None: cmd.append('-snapshot') cmd.append(str(solverstate)) if caffemodel is not None: cmd.append('-weights') cmd.append(str(caffemodel)) if self.manager.parent.trainOnHW > 0: cmd.append('-gpu') cmd.append(str(self.manager.parent.trainOnHW-1)) self.proc = Popen( cmd, stdout=PIPE, stderr=STDOUT, cwd=self.getSnapshotDirectory()) try: self.tee = Popen( ['tee', '-a', self.getRunLogFileName()], stdin=self.proc.stdout, stdout=PIPE) except Exception as e: self.Log("Failed to start tee: " + str(e), error=True) self.setState(State.RUNNING) self.Log('Session ' + self.getRunLogFileName(True) + " was started.") self.startParsing() return [] except Exception as e: return ["Failed to start session: " + str(e)] else: self.statesig.emit(self.state) error.append("Failed to start session in state " + str(self.state)) return error def pause(self): if self.state is State.RUNNING: if self.proc: self.snapshot() time.sleep(1) try: if self.proc: self.proc.kill() except Exception as e: self.logsig.emit('Pausing session failed: ' + str(e), True) try: if self.tee: self.tee.kill() except Exception: pass self.proc = None self.tee = None self.last_solverstate = None self.last_solverstate = self._getLastSnapshotFromSnapshotDirectory(True) regex_iter = re.compile('iter_([\d]+)\.solverstate[\.\w-]*$') iter_match = None if self.last_solverstate is not None: iter_match = regex_iter.search(self.last_solverstate) if iter_match is not None: self.iteration = int(iter_match.group(1)) # self.iterationChanged.emit() # implicit in _pause self.setState(State.PAUSED) self.logsig.emit("Session " + self.getRunLogFileName(True) + " was paused", False) return True return False def snapshot(self): if self.proc: self.last_solverstate = None try: self.proc.send_signal(signal.SIGHUP) except Exception as e: self.Log('Failed to take snapshot: ' + str(e), True) return False self.Log('Snapshot was saved for session ' + self.getRunLogFileName(True)) return True else: self.statesig.emit(self.state) self.Log('Could not take a snapshot in state ' + str(self.state)) return False def proceed(self, snapshot=None): if self.state is State.PAUSED: if snapshot is None: snapshot = self.getLastSnapshot() caffepath = self.manager.caffePath if not os.path.exists(caffepath): return ["No CaffePath"] self.rid += 1 try: self.getParser().setLogging(True) cmd = [ os.path.join(caffepath, 'build/tools/caffe'), 'train', '-solver', os.path.join(self.directory, Paths.FILE_NAME_SOLVER)] cmd.append('-snapshot') cmd.append(snapshot) if self.manager.parent.trainOnHW > 0: cmd.append('-gpu') cmd.append(str(self.manager.parent.trainOnHW-1)) self.proc = Popen( cmd, stdout=PIPE, stderr=STDOUT, cwd=self.getDirectory()) try: self.tee = Popen( ['tee', '-a', self.getRunLogFileName()], stdin=self.proc.stdout, stdout=PIPE) except Exception as e: self.Log("Failed to start tee: " + str(e), True) self.setState(State.RUNNING) self.Log('Session ' + self.getRunLogFileName(True) + " was proceeded.") self.startParsing() return [] except Exception as e: return ["Failed to proceed session: " + str(e)] else: self.statesig.emit(self.state) return ["Can't proceed a session in state " + str(self.state)] def __ensureDirectory(self): if os.path.exists(self.directory) is False: try: os.makedirs(self.directory) return [] except Exception as e: return ["Can't create Session Directory '" + self.directory + "': "+str(e)] return [] def _msgFetchParserData(self): msg = self.transaction.asyncRead() msg["ParserRows"] = self.parserRows msg["ParserKeys"] = self.parserKeys msg["ParserHandle"] = self.parserHandle msg["ParserLogs"] = self.parserLogs self.transaction.send(msg) def _msgLoadInternalNet(self): msg = self.transaction.asyncRead() try: path = os.path.join(self.directory, Paths.FILE_NAME_NET_ORIGINAL) file = open(path, 'r') msg["NetInternal"] = file.read() except IOError: msg["status"] = False msg["error"] = ["Failed to read " + path] self.transaction.send(msg) def _msgLoadDeployedNet(self): msg = self.transaction.asyncRead() try: path = os.path.join(self.directory, Paths.FILE_NAME_NET_ORIGINAL) file = open(path, 'r') dn = DeployedNet(file.read()) msg["Net"] = dn.getProtoTxt() except IOError: msg["status"] = False msg["error"] = ["Failed to read " + path] self.transaction.send(msg) def _msgLoadNetParameter(self): msg = self.transaction.asyncRead() msg["status"] = False if "snapshot" in msg.keys(): path = msg["snapshot"] path = os.path.join(self.directory, path) if os.path.exists(path): try: with open(path, 'rb') as f: msg["NetParams"] = f.read() msg["status"] = True except: msg["error"] = ["Failed to load " + str(path)] else: msg["error"] = ["File not found " + str(path)] else: msg["error"] = ["No Snapshot provided"] self.transaction.send(msg) def _msgLoadCaffemodel(self): msg = self.transaction.asyncRead() msg["status"] = False print("Load caffe model received.") if "snapshot" in msg.keys(): path = msg["snapshot"] print(path) path = os.path.join(self.directory, path) if os.path.exists(path): try: with open(path, 'rb') as f: msg["caffemodel"] = f.read() msg["status"] = True except: msg["error"] = ["Failed to load " + str(path)] else: msg["error"] = ["File not found " + str(path)] else: msg["error"] = ["No Snapshot provided"] self.transaction.send(msg) def reset(self): self.pause() for dirpath, dirnames, filenames in os.walk(self.directory, topdown=True): for dirname in dirnames: if os.path.join(dirpath, dirname) == self.logs: try: shutil.rmtree(self.logs) except shutil.Error as e: self.logsig.emit('Failed to delete logs folder: ' + str(e), self.getCallerId(), True) for filename in filenames: if filename.endswith(".solverstate") or filename.endswith(".caffemodel"): try: os.remove(os.path.join(dirpath, filename)) except OSError as e: self.logsig.emit('Failed to delete ' + str(filename) + ': ' + str(e), self.getCallerId(), True) if filename in ["net-internal.prototxt", "net-original.prototxt", "solver.prototxt"]: try: os.remove(os.path.join(dirpath, filename)) except OSError as e: self.logsig.emit('Failed to delete ' + str(filename) + ': ' + str(e), self.getCallerId(), True) break self.iteration = 0 self.state = State.UNDEFINED self._getState() self.last_caffemodel = None self.last_solverstate = None self.save() self.statesig.emit(self.state) return True backend/barista/session/session.py0000664ڊ00111170000011453213243274037017550 0ustar klemmso0cvprimport os import re import shutil import signal import time import json import copy from subprocess import Popen, PIPE, STDOUT from threading import Lock from datetime import datetime from PyQt5.Qt import QObject from PyQt5.QtCore import pyqtSignal from backend.parser.concatenator import Concatenator from backend.parser.parser_listener import ParserListener from backend.parser.parser import Parser from backend.barista.constraints.session_run.training import checkMinimumTrainingRequirements from backend.barista.session.session_pool import SessionPool from backend.barista.session.session_utils import * from backend.barista.utils.logger import Log from backend.barista.utils.logger import LogCaller from backend.barista.deployed_net import DeployedNet from backend.barista.session.session_common import SessionCommon import backend.caffe.saver as saver import backend.caffe.proto_info as info import backend.caffe.dict_helper as helper class Session(QObject, LogCaller, ParserListener, SessionCommon): """ A session is a caffe training process. Objects of this class encapsulate the state of a process, control the process (starting, pausing etc.), handle events and provide the session output stream. """ # event signals stateChanged = pyqtSignal(object) stateDictChanged = pyqtSignal(object, bool) iterationChanged = pyqtSignal() snapshotAdded = pyqtSignal(object) def __init__(self, project, directory=None, sid=None, parse_old=False, caffe_root=None, last_solverstate=None, last_caffemodel=None, state_dictionary=None): super(Session, self).__init__() self.caller_id = None self.state = State.UNDEFINED self.invalidErrorsList = [] self.sid = sid self.directory = directory self.rid = 0 self.project = project self.logs = None if self.directory is None: if self.sid is None: raise ValueError('Either directory or sid must be provided to create a session.') self.directory = self.__createSessionDirectoryName() self.logs = os.path.join(self.directory, 'logs') else: self.logs = os.path.join(directory, 'logs') dir_sid, self.rid = self.__parseSessionId() if self.sid is None: self.sid = dir_sid else: Log.log('Provided sid and directory do not match ('+str(self.sid) + ' vs. ' + str(dir_sid) + '), ' + 'provided sid is used.', self.getCallerId()) self.parse_old = parse_old self.caffe_root = caffe_root # overrides project caffe_root if necessary, i.e. if deployed to another system self.last_solverstate = last_solverstate self.last_caffemodel = last_caffemodel self.state_dictionary = state_dictionary # state as saved from the network manager, such it can be restored self.start_time = self.__parseStartTime() self.snapshot_dir = None self.snapshot_prefix = None self.proc = None self.tee = None self.parser = None self.iteration = 0 self.max_iter = 1 self.parser_initialized = False self.__getSettingsFromSessionFile() if self.state_dictionary is not None: self.__parseSettings(self.state_dictionary) self.__solverFile = os.path.join(self.directory, Paths.FILE_NAME_SOLVER) self.__netInternalFile = os.path.join(self.directory, Paths.FILE_NAME_NET_INTERNAL) self.__netOriginalFile = os.path.join(self.directory, Paths.FILE_NAME_NET_ORIGINAL) self.lock = Lock() self.getSnapshotDirectory() # comparison methods for the priorization of sessions in the thread pool def __lt__(self, other): if type(other) is not Session: return True return other.sid - self.sid < 0 def __gt__(self, other): if type(other) is not Session: return False return other.sid - self.sid > 0 def __eq__(self, other): if type(other) is not Session: return False return other.sid - self.sid == 0 def __le__(self, other): if type(other) is not Session: return True return other.sid - self.sid <= 0 def __ge__(self, other): if type(other) is not Session: return False return other.sid - self.sid >= 0 def __ne__(self, other): if type(other) is not Session: return True return other.sid - self.sid != 0 def __createSessionDirectoryName(self): """ Return a new session directory with the format YYYYMMDD_hh24mmss_SID """ directory = self.__getDateTimeString() directory += '_'+str(self.sid) return os.path.join(self.project.getSessionsDirectory(), directory) def __ensureDirectory(self, directory): """ Creates a directory if it does not exist. """ if directory == '': return if os.path.exists(directory) is False: try: os.makedirs(directory) Log.log('Created directory: ' + directory, self.getCallerId()) except Exception as e: Log.error('Failed to create directory '+directory+')'+str(e), self.getCallerId()) def __getSettingsFromSessionFile(self): filename = os.path.join(self.directory, baristaSessionFile(self.directory)) if os.path.isfile(filename): with open(filename, 'r') as f: res = json.load(f) self.__parseSettings(res) def checkFiles(self): """ Check for the existence of the session directories and files. """ if os.path.exists(self.directory) is False: Log.error('Session directory does not exists: '+self.directory, self.getCallerId()) if os.path.exists(self.logs) is False: Log.error('Log directory does not exists: '+self.logs, self.getCallerId()) if os.path.exists(self.snapshot_dir) is False: Log.error('Snapshot directory does not exists: '+self.snapshot_dir, self.getCallerId()) if os.path.exists(self.project.getCaffeRoot()) is False: Log.error('CAFFE_ROOT directory does not exists: ' + self.project.getCaffeRoot(), self.getCallerId()) def __parseSnapshotPrefix(self): """ Parse the snapshot prefix from the solver file. """ if self.snapshot_prefix is None: sf = os.path.join(self.getDirectory(), self.getSolver()) if os.path.isfile(sf): self.snapshot_prefix = self.project.parseSnapshotPrefixFromFile(sf) return def getSnapshotPrefix(self): """ Return the snapshot prefix which is used for snapshots of this session. """ if not self.snapshot_prefix: self.__parseSnapshotPrefix() if not self.snapshot_prefix: self.snapshot_prefix = '' return self.snapshot_prefix def getSnapshotExtension(self): """ Return the file name extension for snapshot files. The extensions differ between binaryproto and hdf5 formats. """ extension = None if self.last_caffemodel: # try to parse the extension from the last_caffemodel regex_snapshot = re.compile('(\.caffemodel[\.\w-]*)') snapshot_match = regex_snapshot.search(self.last_caffemodel) if snapshot_match: return snapshot_match.group(1) else: # try to parse the suffix from the last_solverstate extension = self.getLastSnapshot() if extension: regex_snapshot = re.compile('\.solverstate([\.\w-]*)') snapshot_match = regex_snapshot.search(extension) if snapshot_match: ext = snapshot_match.group(1) if ext: # append suffix return '.caffemodel' + ext return '.caffemodel' def getSolverstateExtension(self): """ Return the file name extension for solverstate files. The extensions differ between binaryproto and hdf5 formats. """ extension = self.getLastSnapshot() if extension: regex_snapshot = re.compile('(\.solverstate[\.\w-]*)') snapshot_match = regex_snapshot.search(extension) if snapshot_match: return snapshot_match.group(1) return '.solverstate' def getState(self): """ Return the state of the session. """ if self.state == State.FAILED: return self.state if self.proc is not None: if self.iteration == self.max_iter: self.state = State.FINISHED else: self.state = State.RUNNING else: if len(checkMinimumTrainingRequirements(self)) > 0: self.state = State.INVALID elif self.iteration == self.max_iter: self.state = State.FINISHED elif len(self.getSnapshots()) > 0: self.state = State.PAUSED else: self.state = State.WAITING return self.state def getErrorList(self): return self.invalidErrorsList def setState(self, state): """ Set the state of the session and emit a stateChanged signal. """ self.state = state self.stateChanged.emit(self.state) def setErrorList(self, errorList): self.invalidErrorsList = errorList def setStateDict(self, stateDict): self.state_dictionary = stateDict self.stateDictChanged.emit(self, False) return def getSessionId(self): """ Return the id of the session. """ return self.sid def getRunId(self): """ Return the run id of the session. This id increases every time the process is started/proceeded. It enables the user to distinguish between different runs of the session. """ return self.rid def start(self, solverstate=None, caffemodel=None): """ Start the process. Return True if the process was started False if the start failed """ if self.getState() is State.WAITING: self.rid += 1 # (re-)write all session files self.save(includeProtoTxt=True) # check if the session has its own caffeRoot caffeRoot = self.caffe_root if not caffeRoot: # else take the project's caffeRoot path caffeRoot = self.project.getCaffeRoot() try: self.getParser().setLogging(True) cmd = [ os.path.join(caffeRoot, 'build/tools/caffe'), 'train', '-solver', self.getSolver()] if solverstate: cmd.append('-snapshot') cmd.append(str(solverstate)) elif caffemodel: cmd.append('-weights') cmd.append(str(caffemodel)) self.proc = Popen( cmd, stdout=PIPE, stderr=STDOUT, cwd=self.getDirectory()) try: self.tee = Popen( ['tee', '-a', self.getRunLogFileName()], stdin=self.proc.stdout, stdout=PIPE) except Exception as e: # continue without tee Log.error('Failed to start tee: '+str(e), self.getCallerId()) self.setState(State.RUNNING) Log.log('Session '+self.getRunLogFileName(True) + ' was started', self.getCallerId()) self.__startParsing() return True except Exception as e: # check if caffe root exists Log.error('Failed to start session: '+str(e), self.getCallerId()) if os.path.exists(self.project.getCaffeRoot()) is False: Log.error('CAFFE_ROOT directory does not exists: ' + caffeRoot + '! Please set CAFFE_ROOT to run a session.', self.getCallerId()) else: Log.error( 'Could not start a session in state '+str(self.getState()), self.getCallerId()) # self.setState(State.UNDEFINED) return False def pause(self): """ Pause the process. Return True if the process was paused False if the pause failed """ if self.getState() is State.RUNNING: if self.proc: self.snapshot() # give the caffe process a second to save the state time.sleep(1) try: if self.proc: self.proc.kill() except Exception as e: Log.error('Pausing session failed: '+str(e)) try: if self.tee: self.tee.kill() except Exception: pass self.proc = None self.tee = None self.last_solverstate = None snap = self._getLastSnapshotFromSnapshotDirectory() if snap is not None: self.last_solverstate = os.path.basename(self._getLastSnapshotFromSnapshotDirectory()) regex_iter = re.compile('iter_([\d]+)\.solverstate[\.\w-]*$') iter_match = regex_iter.search(self.last_solverstate) if iter_match: self.iteration = int(iter_match.group(1)) self.iterationChanged.emit() self.setState(State.PAUSED) Log.log('Session ' + self.getRunLogFileName(True) + ' was paused', self.getCallerId()) else: self.setState(State.WAITING) Log.log('Session ' + self.getRunLogFileName(True) + ' was halted', self.getCallerId()) self.save() return True else: Log.error( 'Could not pause a session in state '+str(self.getState()), self.getCallerId()) self.setState(State.UNDEFINED) return False def proceed(self, snapshot=None): """ Continue training from the (last) snapshot. Return True if the process was continued False if the continuation failed """ if self.getState() is State.PAUSED: self.__ensureDirectory(self.getSnapshotDirectory()) self.__ensureDirectory(self.logs) if snapshot is None: snapshot = self.getLastSnapshot() self.rid += 1 try: self.getParser().setLogging(True) self.proc = Popen( [os.path.join(self.project.getCaffeRoot(), 'build/tools/caffe'), 'train', '-solver', self.getSolver(), '-snapshot', snapshot], stdout=PIPE, stderr=STDOUT, cwd=self.getDirectory()) try: self.tee = Popen( ['tee', '-a', self.getRunLogFileName()], stdin=self.proc.stdout, stdout=PIPE) except Exception as e: # continue without tee Log.error('Failed to start tee: '+str(e), self.getCallerId()) self.setState(State.RUNNING) Log.log('Session '+self.getRunLogFileName(True) + ' was proceeded', self.getCallerId()) self.__startParsing() return True except Exception as e: # check if caffe root exists Log.error('Failed to continue session: '+str(e), self.getCallerId()) if os.path.exists(self.project.getCaffeRoot()) is False: Log.error('CAFFE_ROOT directory does not exists: ' + self.project.getCaffeRoot() + '! Please set CAFFE_ROOT to run a session.', self.getCallerId()) elif self.getState() in (State.FAILED, State.FINISHED): Log.error('Could not continue a session in state ' + str(self.getState()), self.getCallerId()) return False def snapshot(self): """ Create a snapshot from the training state. Return True if the snapshot was created False if the snapshot could not be created """ if self.getState() is State.RUNNING: if self.proc: self.last_solverstate = None try: self.proc.send_signal(signal.SIGHUP) except Exception as e: Log.error('Failed to take snapshot: '+str(e), self.getCallerId()) return False Log.log('Snapshot was saved for session ' + self.getRunLogFileName(True)+'', self.getCallerId()) return True else: Log.error( 'Could not take a session snapshot in state ' + str(self.getState()), self.getCallerId()) return False def setFinished(self): """ Mark this session as finished. """ self.save() self.setState(State.FINISHED) self.proc = None def getStream(self): """ Return the log stream of this session. This is an iterator over stdout of the subprocess. """ if self.tee: return iter(self.tee.stdout.readline, '') if self.proc: return iter(self.proc.stdout.readline, '') return iter([]) def getDirectory(self): """ Return the session directory, usually a directory of the form YYYYMMDD_hh24mmss_SID. """ return self.directory def getLogs(self): """ Return the log directory. """ if os.path.exists(self.logs) is False: os.makedirs(self.logs) return self.logs def isRemote(self): return False def getSnapshotDirectory(self): """ Return the snapshot directory. """ if self.snapshot_dir: return self.snapshot_dir snapshot_prefix = self.getSnapshotPrefix() sdir = os.path.dirname(snapshot_prefix) self.snapshot_dir = os.path.join(self.getDirectory(), sdir) return self.snapshot_dir def getSnapshots(self): """ Return all snapshot files, keyed by iteration number. """ regex_snapshot = re.compile('iter_([\d]+)\.solverstate') snaps = {} if os.path.exists(self.getSnapshotDirectory()): for entry in os.listdir(self.getSnapshotDirectory()): snap_match = regex_snapshot.search(entry) if snap_match: try: iter_num = int(snap_match.group(1)) snaps[iter_num] = entry except Exception: pass return snaps def getLastModel(self): """ Return the name of the last saved caffe model. """ return self.last_caffemodel def setLastModel(self, lcm): self.last_caffemodel = lcm def setLastSnapshot(self, lss): self.last_solverstate = lss def getSolver(self, log=False): """ Returns the solver prototxt file name. When the log flag is set, a message will be sent to the logger console if the file does not exist. """ if log: if not os.path.isfile(self.__solverFile): Log.log("This sessions Solverfile: " + self.__solverFile + " does not exist.", self.caller_id) return self.__solverFile def getOriginalNetFile(self, log=False): """ Returns the original net prototxt file name. When the log flag is set, a message will be sent to the logger console if the file does not exist. """ if log: if not os.path.isfile(self.__netOriginalFile): Log.log("This sessions net file: " + self.__netOriginalFile + " does not exist.", self.caller_id) return self.__netOriginalFile def getInternalNetFile(self, log=False): """ Returns the original net prototxt file name. When the log flag is set, a message will be sent to the logger console if the file does not exist. """ if log: if not os.path.isfile(self.__netInternalFile): Log.log("This sessions net file: " + self.__netInternalFile + " does not exist.", self.caller_id) return self.__netInternalFile def readInternalNetFile(self): """ Returns the contents of the internal net prototxt file. """ path = self.getInternalNetFile() with open(path, 'r') as f: contents = f.read() return contents def readDeployedNetAsString(self): """ Returns the contents of the deployable net prototxt file. """ path = self.getInternalNetFile() dn = DeployedNet(open(path).read()) return dn.getProtoTxt() def readCaffemodelFile(self, snapshot): """ Returns the contents of the .caffemodel file that belongs to snapshot. snapshot: string Filename without path of the snapshot file. """ path = os.path.join(self.getSnapshotDirectory(), snapshot) with open(path, 'r') as f: contents = f.read() return contents def getRunLogFileName(self, basename=False): """ Return the name of the logfile with session and run id. """ log_file = self.project.getProjectName()+'_'+str(self.getSessionId())+'.'+str(self.getRunId())+'.log' if basename is True: return log_file log_file = os.path.join(self.getLogs(), log_file) return log_file def getLogFileName(self, basename=False): """ Return the name of the logfile with session id. """ log_file = self.project.getProjectName()+'_'+str(self.getSessionId())+'.log' if basename is True: return log_file log_file = os.path.join(self.getLogs(), log_file) return log_file def getLogId(self): """ Return the id of the logfile. For local sessions same as the name of the logfile. """ return self.getLogFileName(True) def parseOldLogs(self): """ Parse all log files in the log directory. """ locked = self.lock.acquire() if locked is False: return try: if self.parse_old: self.parse_old = False log_files = {} regex_filename = re.compile('[\d]+\.([\d]+)\.log$') for entry in os.listdir(self.getLogs()): filename_match = regex_filename.search(entry) if filename_match: # key files by run id try: run_id = int(filename_match.group(1)) log_files[run_id] = entry except: pass log_list = [] for run_id in sorted(log_files.keys()): log_file = os.path.join(self.getLogs(), log_files[run_id]) log_list.append(log_file) con = Concatenator(log_list) logs = con.concate() for log in logs: try: self.getParser().addLogStream(log) except Exception as e: Log.error('Failed to parse log file '+self.getLogFileName(True)+": "+str(e), self.getCallerId()) except Exception as e: Log.error('Failed to parse old log '+str(e), self.getCallerId()) finally: if locked: self.lock.release() def __startParsing(self): """ Create a parser and run it in a newly dispatched thread. """ self.parseOldLogs() logs = self.getStream() self.getParser().addLogStream(logs) pool = SessionPool() pool.addSession(self) def getParser(self): """ Return the log parser of this session. """ if self.parser is None: self.parser = Parser(None, Events.events, self.getCallerId()) self.parser.addListener(self) return self.parser def getIteration(self): """ Return the current training iteration of this session. """ return self.iteration def getMaxIteration(self): """ Return the maximum training iteration of this session. """ return self.max_iter def setParserInitialized(self): """ Should be called after the parser finished the inital parsing of log files. """ #if self.parser_initialized is False: # self.snapshotAdded.emit(self.last_solverstate) self.parser_initialized = True def isParserInitialized(self): return self.parser_initialized def delete(self): """ Delete the session directory and disconnect signals. """ self.pause() try: shutil.rmtree(self.getDirectory()) except Exception as e: Log.error('Could not remove session directory: '+str(e), self.getCallerId()) try: self.stateChanged.disconnect() self.iterationChanged.disconnect() self.snapshotAdded.disconnect() self.project.deleteSession.emit(self.sid) except Exception as e: pass Log.removeCallerId(self.caller_id, False) # LogCaller def getCallerId(self): """ Return the unique caller id for this session """ if self.caller_id is None: self.caller_id = Log.getCallerId(self.getLogFileName(True)) return self.caller_id # ParserListener def update(self, phase, row): """ Called when the parser has parsed a new record. """ self.iteration = row['NumIters'] self.iterationChanged.emit() def handle(self, event, message, groups): """ Called when the parser has parsed a registered event. """ if event == 'OptimizationDone': self.setFinished() elif event == 'max_iter': self.max_iter = int(groups[0]) elif event == 'state_snapshot': self.last_solverstate = groups[0] #if self.parser_initialized: self.snapshotAdded.emit(self.last_solverstate) elif event == 'model_snapshot': self.last_caffemodel = groups[0] def parsingFinished(self): """ Called when the parser has processed all available streams. """ if self.proc is not None: # Wait for caffe process, kill tee and respond to return code assert self.state is State.RUNNING rcode = self.proc.wait() self.proc = None try: self.tee.kill() except Exception: pass self.tee = None if rcode is 0: self.setFinished() else: self.setState(State.FAILED) Log.error('Session failed with return code '+str(rcode), self.getCallerId()) self.setParserInitialized() def hasStateDict(self): return self.state_dictionary is not None def save(self, includeProtoTxt = False): """Saves the current session to a json file. If includeProtoTxt is True, prototxt files are saved as well.""" toSave = {"SessionState": self.state, "Iteration": self.iteration, "MaxIter": self.max_iter} toSave["ProjectID"] = self.project.projectId self.__ensureDirectory(self.directory) Log.log("Saving current Session status to disk.", self.getCallerId()) if self.last_solverstate: toSave["LastSnapshot"] = self.last_solverstate if self.state_dictionary: serializedDict = copy.deepcopy(self.state_dictionary) if includeProtoTxt: if "solver" in self.state_dictionary: solver = self.buildSolverPrototxt() with open(self.getSolver(log=False), 'w') as f: f.write(solver) else: Log.error("Could not save a solver prototxt file, because no solver settings are defined.", self.getCallerId()) if "network" in serializedDict: if includeProtoTxt: net = self.buildNetPrototxt(internalVersion=False) with open(self.getOriginalNetFile(log=False), 'w') as f: f.write(net) net = self.buildNetPrototxt(internalVersion=True) with open(self.getInternalNetFile(log=False), 'w') as f: f.write(net) if "layers" in serializedDict["network"]: layers = serializedDict["network"]["layers"] for id in layers: del layers[id]["type"] else: Log.error("Could not save the network state because no state was defined.", self.getCallerId()) toSave["NetworkState"] = serializedDict with open(baristaSessionFile(self.directory), "w") as f: json.dump(toSave, f, sort_keys=True, indent=4) def _modifyNetDictionaryToInternalVersion(self, net): """ Take an original net dictionary and apply all changes necessary for the internal version. Mainly, this will change all relative paths to be relative to a session folder. Hint: This will change net itself. You might want to create a (deep) copy of the dictionary before calling this method. """ # define all layer parameters that can contain (relative) paths # (use a dot to separate nested parameters) layerParamsContainingPaths = [ "data_param.source", "hdf5_data_param.source", "image_data_param.source", "window_data_param.source", "data_param.mean_file", "hdf5_output_param.file_name", "image_data_param.mean_file", "window_data_param.mean_file", "transform_param.mean_file"] # evaluate layer after layer h = helper.DictHelper(net) for layerId, layer in net.get("layers", {}).iteritems(): # evaluate parameter after parameter for paramKey in layerParamsContainingPaths: # if the current layer does have the current parameter.. if h.layerParameterIsSet(layerId, paramKey): paramValue = h.layerParameter(layerId, paramKey) # ..and its value is a relative path: modify it if paramValue is not None and not os.path.isabs(paramValue): if paramKey != "hdf5_data_param.source": newPath = os.path.join( os.pardir, os.path.join(os.pardir, paramValue) ) else: newFilename = str(layerId) + ".txt" newPath = os.path.join(os.curdir, newFilename) h.setLayerParameter(layerId, paramKey, newPath) return net def buildSolverPrototxt(self): """ Load the current solver dictionary and return the corresponding message object. :return: A solver message object. """ if not self.state_dictionary: return None elif "solver" not in self.state_dictionary: return None else: solver = saver.saveSolver(self.state_dictionary["solver"]) return solver def buildNetPrototxt(self, internalVersion=False): """ Load the current net dictionary and return the corresponding message object. :param internalVersion: Iff true, the loaded net will be modified to the internal version. :return: A net message object. """ currentState = self.state_dictionary netDictionary = None if currentState: netDictionary = currentState["network"] if internalVersion: netDictionary = self._modifyNetDictionaryToInternalVersion(copy.deepcopy(netDictionary)) solver = saver.saveNet(netDictionary) return solver def __parseSettings(self, settings): if settings: if "SessionState" in settings: self.state = settings["SessionState"] self.__previousState = self.state if "Iteration" in settings: self.iteration = settings["Iteration"] if "solver" in settings: if "max_iter" in settings["solver"]: self.max_iter = settings["solver"]["max_iter"] elif "MaxIter" in settings: self.max_iter = settings["MaxIter"] if "LastSnapshot" in settings: self.last_solverstate = settings["LastSnapshot"] if "NetworkState" in settings: self.state_dictionary = settings["NetworkState"] layers = self.state_dictionary["network"]["layers"] for id in layers: if "parameters" in layers[id]: if "type" in layers[id]["parameters"]: typename = layers[id]["parameters"]["type"] layers[id]["type"] = info.CaffeMetaInformation().availableLayerTypes()[typename] solver = self.state_dictionary["solver"] if solver: if "snapshot_prefix" in solver: self.snapshot_prefix = solver["snapshot_prefix"] def __parseSessionId(self): """ Return a tuple (session_id, run_id). The ids are parsed from the directory. session_id is the id of the session and run_id the highest found run id. """ regex_sid = re.compile('[\d]{8}_[\d]{6}_([\d]+)') sid_match = regex_sid.search(self.directory) session_id = None if sid_match: try: session_id = int(sid_match.group(1)) except: pass run_id = 0 regex_rid = re.compile('([\d]+)\.([\d]+)\.log') if os.path.exists(self.logs): for entry in os.listdir(self.logs): rid_match = regex_rid.search(entry) if rid_match: try: _run_id = int(rid_match.group(2)) if run_id: if _run_id > run_id: run_id = _run_id else: run_id = _run_id if session_id is None: session_id = int(rid_match.group(1)) except: pass return session_id, run_id def __parseStartTime(self): """ Parse the start time with second precision from the directory name. Return the parsed datetime. """ regex_time = re.compile( '([\d]{4})([\d]{2})([\d]{2})_([\d]{2})([\d]{2})([\d]{2})') time_match = regex_time.search(self.directory) time = datetime.now() if time_match: try: year = int(time_match.group(1)) month = int(time_match.group(2)) day = int(time_match.group(3)) hour = int(time_match.group(4)) minute = int(time_match.group(5)) second = int(time_match.group(6)) time = time.replace(year=year, month=month, day=day, hour=hour, minute=minute, second=second) except Exception: pass return time def __getDateTimeString(self): """ Return the current datetime in the format YYYYMMDD_hh24mmss """ now = datetime.now() date_string = '' date_string += str(now.year) date_string += str(now.month).zfill(2) date_string += str(now.day).zfill(2) date_string += '_' date_string += str(now.hour).zfill(2) date_string += str(now.minute).zfill(2) date_string += str(now.second).zfill(2) return date_string def reset(self): self.pause() for dirpath, dirnames, filenames in os.walk(self.directory, topdown=True): for dirname in dirnames: if os.path.join(dirpath, dirname) == self.logs: try: shutil.rmtree(self.logs) except shutil.Error as e: Log.error('Failed to delete logs folder: ' + str(e), self.getCallerId()) for filename in filenames: if filename.endswith(".solverstate") or filename.endswith(".caffemodel"): try: os.remove(os.path.join(dirpath, filename)) except OSError as e: Log.error('Failed to delete ' + str(filename) + ': ' + str(e), self.getCallerId()) if filename in ["net-internal.prototxt", "net-original.prototxt", "solver.prototxt"]: try: os.remove(os.path.join(dirpath, filename)) except OSError as e: Log.error('Failed to delete ' + str(filename) + ': ' + str(e), self.getCallerId()) break self.iteration = 0 self.iterationChanged.emit() self.state = State.UNDEFINED self.setState(self.getState()) self.setLastModel(None) self.setLastSnapshot(None) self.project.resetSession.emit(self.getSessionId()) self.save() backend/barista/session/session_common.py0000664ڊ00111170000000636113243274037021120 0ustar klemmso0cvprimport os import re from abc import abstractmethod class SessionCommon(): def __init__(self): self.lastSolverState = None self.directory = None @abstractmethod def getLogs(self): pass @abstractmethod def getSnapshotDirectory(self): pass def getLastSnapshot(self): """ Return the last snapshot/solverstate for this session. The last snapshot name is searched in the log files. """ if self.last_solverstate: if os.path.isfile(os.path.join(self.directory, self.last_solverstate)): return self.last_solverstate self.last_solverstate = self._getLastSnapshotFromLogFiles() if self.last_solverstate: if os.path.isfile(os.path.join(self.directory, self.last_solverstate)): return self.last_solverstate self.last_solverstate = self._getLastSnapshotFromSnapshotDirectory() if self.last_solverstate: if os.path.isfile(os.path.join(self.directory, self.last_solverstate)): return self.last_solverstate return None def _getLastSnapshotFromLogFiles(self): """ Try to find the last snapshot name in the log file. Return the name of the solverstate file if it was found. """ # get all log files log_files = {} regex_filename = re.compile('[\d]+\.([\d]+)\.log$') for entry in os.listdir(self.getLogs()): filename_match = regex_filename.search(entry) if filename_match: # key files by run id try: run_id = int(filename_match.group(1)) log_files[run_id] = entry except: pass last_solverstate = None for run_id in reversed(sorted(log_files.keys())): with open(os.path.join(self.getLogs(), log_files[run_id])) as f: # find the last snapshot in the file regex_snapshot = re.compile( 'Snapshotting solver state to (?:binary proto|HDF5) file (.+\.solverstate[\.\w-]*)') for line in f: snapshot_match = regex_snapshot.search(line) if snapshot_match: last_solverstate = snapshot_match.group(1) if last_solverstate: return last_solverstate def _getLastSnapshotFromSnapshotDirectory(self, basename=False): """ Try to find the last snapshot in the snapshot directory. Return the name of the solverstate file if it was found. """ solver_state = None max_iter = -1 regex_iter = re.compile('iter_([\d]+)\.solverstate[\.\w-]*$') for entry in os.listdir(self.getSnapshotDirectory()): iter_match = regex_iter.search(entry) if iter_match: try: iter_id = int(iter_match.group(1)) if iter_id > max_iter: max_iter = iter_id solver_state = entry except: pass if solver_state: if basename: return solver_state return os.path.join(self.getSnapshotDirectory(), solver_state) backend/barista/session/session_pool.py0000664ڊ00111170000000610513243274037020575 0ustar klemmso0cvprimport multiprocessing import sys if sys.version_info[0] == 2: from Queue import PriorityQueue, Empty else: from queue import PriorityQueue, Empty from threading import Thread class Singleton(type): """This metaclass is used to provide the singleton pattern in a generic way. See http://stackoverflow.com/a/6798042 for source and further explanation. TODO outsource this class to use the same pattern in the complete project? """ _instances = {} def __call__(cls, *args, **kwargs): if cls not in cls._instances: cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) return cls._instances[cls] class SessionPool: """ A simple thread pool implementation for parsing log files of sessions. A job is added by adding a session. Jobs are ordered by the session_id(priority), so new sessions have a higher priority. The pool allocates and starts threads on demand. It allocates no more then CPU_COUNT - 2 threads, but at least 1. """ __metaclass__ = Singleton def __init__(self): self.sessions = PriorityQueue() self.pool = [] self.__start_jobs = False self.MAX_THREADS = multiprocessing.cpu_count() - 2 if self.MAX_THREADS <= 0: self.MAX_THREADS = 1 def addSession(self, session): """ Add a session to to queue and start a thread. """ self.sessions.put(session) if self.__start_jobs: self.__startJob() def activate(self, emptyJob): """ Activates this pool by starting threads. """ self.__start_jobs = True self.emptyJob = emptyJob qs = self.sessions.qsize() starts = min(qs, self.MAX_THREADS) for i in range(0, starts): self.__startJob() # private methods def __startJob(self): """ Remove inactive threads from the pool and create a new for the new job. """ if self.__start_jobs is False: return # clean thread pool jc = len(self.pool) for ri in range(jc, 0, -1): i = ri - 1 thread = self.pool[i] if thread is not None: if thread.is_alive() is False: self.pool.pop(i) # create a new thread jc = len(self.pool) if jc < self.MAX_THREADS: t = Thread(target=self.__executeJob) t.start() self.pool.append(t) def __executeJob(self): """ The target method of a thread. Poll for sessions and execute their parser. """ try: while True: session = self.sessions.get(True, 1) if session: session.parseOldLogs() parser = session.getParser() parser.parseLog() self.sessions.task_done() else: break except Empty: if self.emptyJob: self.emptyJob() return except Exception as e: print(str(e)) backend/barista/session/session_utils.py0000664ڊ00111170000000307513243274037020767 0ustar klemmso0cvprclass State: """ Pseudo Enumeration for the states a session could be in. """ UNDEFINED = 0 WAITING = 1 RUNNING = 2 PAUSED = 3 FAILED = 5 FINISHED = 6 INVALID = 7 NOTCONNECTED = 8 def baristaSessionFile(directory): import os """ Returns the filename of the config-json file in the given directory """ return os.path.join(directory, "sessionstate.json") class Paths: # class "constants" defining common file names inside of a session FILE_NAME_SOLVER = "solver.prototxt" FILE_NAME_NET_ORIGINAL = "net-original.prototxt" FILE_NAME_NET_INTERNAL = "net-internal.prototxt" FILE_NAME_SESSION_JSON = "sessionstate.json" class Events: import re events = { # exceptions 'FileNotFound': re.compile('Check failed: mdb_status == 0 \(([\d]+) vs\. 0\) No such file or directory'), 'OutOfGPU': re.compile('Check failed: error == cudaSuccess \(([\d]+) vs\. 0\) out of memory'), 'NoSnapshotPrefix': re.compile('Check failed: param_\.has_snapshot_prefix\(\) In solver params, snapshot is specified but snapshot_prefix is not'), # session finished 'OptimizationDone': re.compile('Optimization Done'), # iterations 'max_iter': re.compile('max_iter:[\s]+([\d]+)'), # snapshots 'state_snapshot': re.compile('Snapshotting solver state to (?:binary proto|HDF5) file (.+\.solverstate[\.\w-]*)'), 'model_snapshot': re.compile('Snapshotting to (?:binary proto|HDF5) file (.+\.caffemodel[\.\w-]*)') }backend/barista/utils/__init__.py0000664ڊ00111170000000000013243274037017261 0ustar klemmso0cvprbackend/barista/utils/db_util.py0000664ڊ00111170000001132113243274037017154 0ustar klemmso0cvprimport os import hashlib from backend.barista.utils.logger import * """ This file contains functions that deal with the path handling and hashing of database files @author: j_stru18 """ """ Logger ID used in functions below. """ DBUTIL_LOGGER_ID = Log.getCallerId("dbutil") """ returns a hash value by calling a hash function for the opened file of the given path Warning: This is not meant to be used for HDF5TXT files. You need to use getHdf5List() and call getMultipleHash with the result. @author ..., f_prob02, j_stru18, l_hu0002 @param db_path: a path to a database file, for example a data.mdb @return a hash value for the hashed database as int @pre dbPath is a valid path """ def getHash(dbPath): hashObject = hashlib.sha256() with open(dbPath, "rb") as openDB: for chunk in iter(lambda: openDB.read(4096), b""): hashObject.update(chunk) return int(hashObject.hexdigest(), 16) """ hashes all files in the given paths @author j_stru18 @param dbPaths a list of paths @return a hash value for all files in the given paths as int """ def getMultipleHash(dbPaths): hashValue = 0 for path in dbPaths: # hash all the old .h5 and .hdf5 files in HDF5TXT if os.path.exists(path): hashValue += getHash(path) return hashValue def getStringHash(inputStr): hashObject = hashlib.sha256() hashObject.update(inputStr) return int(hashObject.hexdigest(), 16) """ returns a list containing the path to a lmdb, if this file exists @author j_stru18 @param path : a string that is supposed to be a valid path for a lmdb @return a list that contains the given path """ def getLMDBList(path): pathList = [] if os.path.exists(path): pathList.append(path) return pathList """ @author j_stru18 @param path : a string that is supposed to be a valid path for a leveldb (directory path containing a CURRENT file) @return a list that contains all the .sst and .ldb files in the directory """ def getLEVELDBList(directoryPath): pathList = [] if os.path.exists(os.path.join(directoryPath, "CURRENT")): for db_file in os.listdir(directoryPath): if db_file.endswith(".sst") or db_file.endswith(".ldb"): pathList.append(os.path.join(directoryPath, db_file)) return pathList """ returns a list containing the paths to all .h5 or .hdf5 files that are specified in a HDF5TXT file, if this file exists @author j_stru18 @param path : a string that is supposed to be a valid path for a HDF5TXT file @return a list that contains all .h5 and .hdf5 files in the HDF5TXT """ def getHdf5List(filePath): pathList = [] if os.path.exists(filePath): lines = [line.rstrip('\n') for line in open(filePath)] for line in lines: if line.endswith(".h5") or line.endswith(".hdf5"): if os.path.exists(line): # absolute path in txt pathList.append(line) elif os.path.exists(os.path.normpath(os.path.join(filePath, line))): # path relative to HDF5 location pathList.append(os.path.normpath(os.path.join(filePath, line))) else: Log.error("HDF5 textfile contained invalid path {}".format(line), DBUTIL_LOGGER_ID) return pathList """ Given the name of a file, this method returns its type @author j_stru18 @param filename a string that represents the file's name @return its type as a string """ def getType(filename): type = "" if filename == "CURRENT": type = "LEVELDB" else: extension = filename[-4:] if extension == ".mdb": type = "LMDB" elif extension == ".ldb": type = "LEVELDB" elif extension == ".txt": type = "HDF5TXT" elif filename[-5:] == ".hdf5": type = "HDF5" elif filename[-3:] == ".h5": type = "HDF5" return type """ Given the path and type of a file, this method returns the paths to all files that this db file uses to store data. @author: j_stru18 @param filePath : the path to a db file as string @param type : the db file's type as string @return : all paths of this db file that are used to store data """ def getDBPathsByType(filePath, type): dbpaths = [] if type == 'LMDB': filePath = os.path.normpath(os.path.join(filePath, "data.mdb")) dbpaths = getLMDBList(filePath) elif type == 'LEVELDB': directoryPath = os.path.normpath(filePath) dbpaths = getLEVELDBList(directoryPath) elif type == 'HDF5TXT': filePath = os.path.normpath(filePath) dbpaths = getHdf5List(filePath) elif type == 'HDF5': dbpaths.append(filePath) else: Log.error("File has the wrong type: {}".format(type), DBUTIL_LOGGER_ID) return dbpaths backend/barista/utils/logger.py0000664ڊ00111170000001523713243274037017023 0ustar klemmso0cvprfrom PyQt5.QtCore import Qt from PyQt5.Qt import QObject import PyQt5.QtCore as QtCore import inspect import datetime import time as tim ''' Model/Helper-Classes ''' class MessageType: def __init__(self, id, description): self.typeId = id if(id == -1): # DockElementConsole.ALL self.color = Logger.COLORS[Logger.TEXT.typeId] # not existiing now else: self.color = Logger.COLORS[id] self.description = description class LogLine: def __init__(self, line, caller, msgType, time): self.line = line self.caller = caller self.msgType = msgType self.time = time class Caller: def __init__(self, id, description, customColor): # if no id is given, default to 0 if not id: id = 0 self.callerId = id id += 2 # color offset for reserved colors black and red if id >= len(Logger.COLORS) or not customColor: self.color = Logger.COLORS[ Logger.TEXT.typeId] else: self.color = Logger.COLORS[id] self.description = description self.used = False def setUsed(self): self.used = True class LogCaller: """ A interface for users of the Logger class. If a class implements this interface, it can use the Logger methods log and error without delivering the callerId """ def getCallerId(self): pass class Logger(QObject): """ TODO: Selection is removed on insertion, fix needed? TODO: Logfile schreiben """ newLine = QtCore.pyqtSignal(object) sigRefreshGui = QtCore.pyqtSignal(object) sigRefreshCallers = QtCore.pyqtSignal(object) # Color Constants COLORS = [Qt.red, Qt.black, Qt.blue, Qt.green, Qt.yellow] # Type Constants, used for color and combobox index # initalized at the bottom of this module ERROR = None TEXT = None ALL = None def __init__(self): super(Logger, self).__init__() self.__loglines = [] self.__callerIdCount = 0 self.__removedCallers = [] self.__callers = {} self.__guiConsole = None self.__filePath = None self.__defaultCaller = Caller(-1, "Caller could not be identified", False) def getCallerFromIdWithFallback(self, callerId): if callerId is None or callerId not in self.__callers: return self.__defaultCaller else: return self.__callers[callerId] def log(self, line, callerId=None): ''' use to log simple text ''' # inspired by http://stackoverflow.com/a/7272464/2129327 if callerId is None: try: caller = inspect.currentframe().f_back.f_locals['self'] if caller: log_id = getattr(caller, "getCallerId()", None) if callable(log_id): callerId = caller.getCallerId()() except Exception: # TODO: Warn properly that something went wrong print('From unknown callerID: ' + line) pass self.appendLine(line, callerId) def error(self, line, callerId=None): ''' use to log errors ''' # inspired by http://stackoverflow.com/a/7272464/2129327 if callerId is None: try: caller = inspect.currentframe().f_back.f_locals['self'] if caller: log_id = getattr(caller, "getCallerId()", None) if callable(log_id): callerId = caller.getCallerId()() except Exception: pass self.appendLine(line, callerId, Logger.ERROR) def appendLine(self, line, callerId, msgType=None): ''' appends a sing line to the logger and if existing to console and file line: String , the string to append callerId: Int, the id created by getCallerId msgType: MessageType, An Object indicating which Type this logline is ''' if(msgType is None): msgType = Logger.TEXT ts = tim.time() st = datetime.datetime.fromtimestamp(ts).strftime("%H:%M:%S") caller = self.getCallerFromIdWithFallback(callerId) caller.setUsed() logline = LogLine(line, caller, msgType, st) self.__loglines.append(logline) # notify all consoles etc about new line self.newLine.emit(logline) if self.__filePath is not None: self.__guiConsole.appendLineToConsole(logline) def appendLines(self, lines, callerId): ''' TODO: Possible optimization for big amount of data, like logfiles ''' for line in lines: self.appendLine(line, callerId) def getCallerId(self, description, customColor=False): ''' creates a new Caller id which identifies the using object to group their log messages ''' if(self.__removedCallers): # list not empty callerId = self.__removedCallers.pop() else: callerId = self.__callerIdCount self.__callerIdCount += 1 caller = Caller(callerId, description, customColor) self.__callers[callerId] = caller self.refreshCallers() return callerId def removeCallerId(self, callerId, keepLines=True): if(callerId in self.__removedCallers): return self.__removedCallers.append(callerId) self.__callers.pop(callerId, None) if(keepLines == False): filterLines = lambda logline: logline.caller.callerId != callerId self.__loglines = list(filter(filterLines, self.__loglines)) self.refreshGui() else: self.refreshCallers() def setFile(self, filePath): self.__filePath = filePath def refreshGui(self): ''' refreshs/validates the whole console and filter values ''' self.refreshConsole() self.refreshCallers() def refreshConsole(self): ''' refreshs/validates the whole console ''' self.sigRefreshGui.emit(self.__loglines) def refreshCallers(self): ''' refreshs/validates the filter values/callers ''' self.sigRefreshCallers.emit(self.__callers) def __getPrefix(self, caller, time): ''' the prefix that have to be added ''' prefix = "" prefix = "[" + time + ", " + caller.description + "]" return prefix # Add Constant Values to Logger Logger.ERROR = MessageType(0, "ERROR") Logger.TEXT = MessageType(1, "TEXT") Logger.ALL = MessageType(-1, "ALL") # "Singelton" instance Log = Logger() backend/barista/utils/settings.py0000664ڊ00111170000000037613243274037017402 0ustar klemmso0cvprfrom PyQt5.QtCore import QSettings def applicationQSetting(): """ Returns the default QSettings-Object for this application """ # return QSettings(QSettings.IniFormat,QSettings.UserScope, "wwu", "Barista") return QSettings("wwu", "Barista") backend/barista/__init__.py0000664ڊ00111170000000000013243274037016121 0ustar klemmso0cvprbackend/barista/deployed_net.py0000664ڊ00111170000003371613243274037017061 0ustar klemmso0cvprimport copy import os from shutil import copyfile import backend.caffe.proto_info as info from backend.barista.utils.logger import Log from backend.caffe import loader from backend.caffe import saver from backend.caffe.dict_helper import DictHelper from gui.input_manager.database_object import DatabaseObject class DeployedNet: """Based on a given network dictionary, an instance of this class (creates and) provides all data necessary to export a deployed version of the network. The applied rules are based on the information provided in the following caffe wiki page: https://github.com/BVLC/caffe/wiki/Using-a-Trained-Network:-Deploy In summary, the following actions are performed automatically: - remove all data layers - remove all layers that require any labels - insert new input layers - append a new Softmax layer (only if none does exist yet and a SoftmaxWithLoss layer used to be included) Finally, by calling saveProtoTxtFile() the following is done: - copy the modified prototxt file and the associated caffe model file to the Restrictions: - Each data layer must not have more than two top blobs (a general restriction of caffe). Raises a warning. - At least the label blob name must follow the naming conventions (so it's always called "label"). Raises a warning. - The shape of a Data layer can only be determined automatically, if the layer type is either "Data" (LMDB or LEVELDB) or "HDF5Data". Otherwise, a warning will inform the user about necessary manual changes. """ def __init__(self, netPrototxtContents): """Create a deployment version of the given network. netPrototxtContents: string The contents of the prototxt file to deploy. """ # create a logger id self._logId = Log.getCallerId('deployment') self._originalNetworkDictionary = loader.loadNet(netPrototxtContents) # init further attributes self._dataLayers = dict() # a dictionary containing all data layers. keys are the layer ids. self._labelBlobNames = [] # a list containing all names of blobs, that represent any labels self._dataBlobNames = [] # a list containing all names of blobs, that represent any input data self._inputBlobShapes = dict() # the keys of this dictionary equal self._dataBlobNames. self._deployedNetworkDictionary = copy.deepcopy(self._originalNetworkDictionary) self._dHelper = DictHelper(self._deployedNetworkDictionary) # start deployment self._createDeployedNetwork() def _createDeployedNetwork(self): """Create a deployed version of self._originalNetworkDictionary and save it in self._deployedNetworkDictionary. Requires self._originalNetworkDictionary to be set before. """ # ensure that at least one layer does exist if len(self._deployedNetworkDictionary["layers"]) <= 0: raise ValueError("Can not create deployment network, because the original one does not contain any layers.") self._searchDataLayers() self._removeDataLayers() self._removeLayersUsingLabels() self._insertInputLayers() self._addSoftmax() def getProtoTxt(self): """return the content of self._deployedNetworkDictionary as a prototxt string.""" return saver.saveNet(self._deployedNetworkDictionary) def _searchDataLayers(self): """Search for all data layers in self._originalNetworkDictionary and store them in self._dataLayers. Additionally, the names of all top blobs containing label data will be saved in self._labelBlobNames. """ nLabelBlobUndetermined = 0 # check whether some label blobs couldn't be determined automatically nTooManyTopBlobs = 0 # check whether some data layers do have too many top blobs for id, layer in self._originalNetworkDictionary["layers"].iteritems(): if layer["type"].isDataLayer(): self._dataLayers[id] = layer # validate number of top blobs if "top" in layer["parameters"] and len(layer["parameters"]["top"]) > 2: nTooManyTopBlobs += 1 # remember the associated label blob name labelBlobName = self._getLabelBlobName(layer) if labelBlobName is not None and labelBlobName not in self._labelBlobNames: self._labelBlobNames.append(labelBlobName) elif labelBlobName is None: nLabelBlobUndetermined += 1 # remember the associated data blob name and its shape dataBlobName = self._getDataBlobName(layer) if dataBlobName is not None and dataBlobName not in self._dataBlobNames: self._dataBlobNames.append(dataBlobName) # calculate the shape (this assumes that data layers with the same name are using the same shape) blobShape = [] # will be left empty, if shape cannot be calculated automatically if layer["type"].name() in ["Data", "HDF5Data"]: if layer["type"].name() == "Data": path = layer["parameters"].get("data_param", {}).get("source") type = layer["parameters"].get("data_param", {}).get("backend") elif layer["type"].name() == "HDF5Data": path = layer["parameters"].get("hdf5_data_param", {}).get("source") type = "HDF5TXT" if path is not None and type is not None: db = DatabaseObject() db.openFromPath(path, type) blobShapeTupel = db.getDimensions() if blobShapeTupel is not None: blobShapeTupel = blobShapeTupel.get(dataBlobName) if blobShapeTupel is not None: blobShape = list(blobShapeTupel) self._inputBlobShapes[dataBlobName] = blobShape # show warning, if some label blobs do not have the correct name if nLabelBlobUndetermined > 0: Log.log("{} data layers might have been handled incorrectly, because their top blobs are named " "unconventionally. Please change the name of the blobs which provide labels to \"label\".".format( nLabelBlobUndetermined ), self._logId) # show warning, if some data layers have too many top blobs if nTooManyTopBlobs > 0: Log.log("{} data layers have too many top blobs. The native caffe version does support only a maximum of 2 " "top blobs per data layer. Deployment result might be incorrect.".format( nTooManyTopBlobs ), self._logId) def _removeDataLayers(self): """Remove all data layers from the deployed Network.""" for id, layer in self._dataLayers.iteritems(): self._dHelper.removeLayer(id) def _removeLayersUsingLabels(self): """Remove all layers that require labels as an input.""" # iterate over the original(!) net, but delete from the deployment(!) net for id, layer in self._originalNetworkDictionary["layers"].iteritems(): if self._isLayerRequiringLabels(layer): self._dHelper.removeLayer(id) def _isLayerRequiringLabels(self, layer): """Return true, iff the given layer requires any labels as an input.""" if "bottom" in layer["parameters"]: for bottomBlob in layer["parameters"]["bottom"]: if bottomBlob in self._labelBlobNames: return True return False def _getLabelBlobName(self, layer): """Get the name of the layer's top blob that represents any labels. Require: layer must be a data layer. According to http://caffe.berkeleyvision.org/tutorial/data.html, the label blob is always called "label". Note: This method might need to be adjusted to be able handling further special cases. """ if "top" in layer["parameters"] and "label" in layer["parameters"]["top"]: return "label" else: return None def _getDataBlobName(self, layer): """Get the name of the layer's top blob, that represents any input data. Require: layer must be a data layer. According to http://caffe.berkeleyvision.org/tutorial/data.html, the data blob is always called "data". However, the following implementation allows handling (some) special cases by changing only the implementation of _getLabelBlobName(). """ labelBlobName = self._getLabelBlobName(layer) if "top" in layer["parameters"]: for topBlobName in layer["parameters"]["top"]: if topBlobName != labelBlobName: return topBlobName return None def _insertInputLayers(self): """Insert new input layers with fixed dimensions. Note that, the number of newly-added input layer might be lower than the number of previously-removed data layers. On the one hand, we will add only one new input layer for each unique data blob name, while multiple data layers might have used the same data blob name. On the other hand, input layers will only be added, if at least one other layer is using the provided data. """ inputLayerType = info.CaffeMetaInformation().availableLayerTypes()["Input"] inputLayerNr = 1 for dataBlobName in self._dataBlobNames: # create a new input layer with default values and add it to the deployment net name = self._getNewInputLayerName(inputLayerNr) inputLayer, inputLayerId = self._dHelper.addLayer(inputLayerType, name, inputLayerNr-1) # modify the layer template inputLayer["parameters"]["top"] = [dataBlobName] # set input_param.shape with batch size 1 and the dimensions of the first data element inputLayer["parameters"]["input_param"] = dict() inputLayer["parameters"]["input_param"]["shape"] = [] inputLayer["parameters"]["input_param"]["shape"].append(dict()) inputLayer["parameters"]["input_param"]["shape"][0]["dim"] = self._inputBlobShapes[dataBlobName] inputLayer["parameters"]["input_param"]["shape"][0]["dim"].insert(0, 1) # prepare next input layer inputLayerNr += 1 # check whether there is any shape that could not be determined automatically and needs to be set manually inputShapeWarning = False for inputShape in self._inputBlobShapes: if len(inputShape) < 1: inputShapeWarning = True break if inputShapeWarning: Log.log("At least one input shape could not be determined automatically. Please open the created prototxt " "and manually fix all input shapes which include only the batch size (1).", self._logId) def _getNewInputLayerName(self, inputLayerNr): """Get the name for a new input layer. If only one input layer will be added, the name will always be "data". Otherwise, inputLayerNr will be added as a suffix. """ name = "data" if len(self._dataBlobNames) > 1: name += str(inputLayerNr) return name def _addSoftmax(self): """Add a softmax layer to the very end of the net, but only if a SoftmaxWithLoss layer was used before.""" # check whether the net used to contain a SoftmaxWithLoss layer softmaxWithLossWasUsed = False for id, layer in self._originalNetworkDictionary["layers"].iteritems(): if layer["type"].name() == "SoftmaxWithLoss": softmaxWithLossWasUsed = True break if softmaxWithLossWasUsed: # ensure that the remaining deployment net has at least one layer if len(self._deployedNetworkDictionary["layers"]) > 0: softmaxLayerType = info.CaffeMetaInformation().availableLayerTypes()["Softmax"] # do not add another softmax, if the current deployment network already contains one softmaxAlreadyIncluded = False for id, layer in self._deployedNetworkDictionary["layers"].iteritems(): if layer["type"].name() == softmaxLayerType.name(): softmaxAlreadyIncluded = True break if not softmaxAlreadyIncluded: # get the very last layer lastLayerId = self._deployedNetworkDictionary["layerOrder"][-1] lastLayer = self._deployedNetworkDictionary["layers"][lastLayerId] # ensure that the determined last layer does have a top blob if "top" in lastLayer["parameters"] and len(lastLayer["parameters"]["top"]) > 0: # create new softmax layer with default values and add it to the deployment net name = "softmax" position = len(self._deployedNetworkDictionary["layers"]) softmaxLayer, softmaxLayerId = self._dHelper.addLayer(softmaxLayerType, name, position) # connect the softmax layer with the existing network softmaxLayer["parameters"]["bottom"] = [lastLayer["parameters"]["top"][0]] # name the output softmaxLayer["parameters"]["top"] = ["probabilities"] else: Log.log("Could not add Softmax layer as the very last layer of the deployment net does not have any " "top blobs.", self._logId) else: Log.log("Could not add Softmax layer as the remaining deployment net does not have any layers.", self._logId) backend/barista/project.py0000664ڊ00111170000006605613243274037016057 0ustar klemmso0cvprimport copy import json import os import re import shutil import uuid from datetime import datetime from PyQt5.Qt import QObject from PyQt5.QtCore import pyqtSignal from backend.barista.constraints.permanent.project import ensureProjectDataConstraints from backend.barista.session.session import Session from backend.barista.session.client_session import ClientSession from backend.barista.session.session import State from backend.barista.session.session_utils import Paths from backend.networking.protocol import Protocol from backend.networking.net_util import sendMsgToHost import backend.caffe.dict_helper as helper import backend.caffe.loader as loader import backend.caffe.proto_info as info import backend.caffe.saver as saver from backend.barista.session.session_pool import SessionPool from backend.barista.utils.logger import Log from backend.barista.utils.logger import LogCaller from backend.caffe import path_loader def dirIsProject(dirname): """ Checks if the dir with the name dirname contains a projectfile """ if os.path.exists(_baristaProjectFile(dirname)): return True return False def _baristaProjectFile(directory): """ Returns the filename of the config-json file in the given directory """ return os.path.join(directory, "barista_project.json") class Project(QObject, LogCaller): """ This class is a abstraction for a caffe project. A project has a directory which contains the following dependencies: - a settings file with general settings for the project, e.g. the caffe version - a layout file with information about the layout of the net in the prototxt_editor - a caffe solver file - two versions of the caffe net file -> original one: a copy of the net state just as it would appear by using the "export net" feature -> internal one: a slightly modified version of the original file. Relative paths are changed to allow executing caffe inside of a session folder, although the user defines paths relative to the project folder. - a log directory - a directory for training sessions containing log files and snapshots A project directory has the following structure: /project_name |---barista_project.json |---/logs |---/sessions | |---/20161118_153000_1 | | |---solver.prototxt (see FILE_NAME_SOLVER) | | |---net-original.prototxt (see FILE_NAME_NET_ORIGINAL) | | |---net-internal.prototxt (see FILE_NAME_NET_INTERNAL) | | |---/logs | | | |---project_name_1.1.log | | | |---project_name_1.2.log | | |---/snapshots | | | |---project_name_iter_1000.caffemodel | | | |---project_name_iter_1000.solverstate | |---/20161118_161000_2 | | |---solver-internal.prototxt (see FILE_NAME_SOLVER) | | |---net-original.prototxt (see FILE_NAME_NET_ORIGINAL) | | |---net-internal.prototxt (see FILE_NAME_NET_INTERNAL) | | |---/logs | | | |---project_name_2.1.log | | |---/snapshots ... """ # signals newSession = pyqtSignal(object) # The deleteSession signal is called from within backend/barista/session/session.py # in the Session.delete() function. deleteSession = pyqtSignal(object) resetSession = pyqtSignal(object) activeSessionChanged = pyqtSignal(object) def __init__(self, directory): super(Project, self).__init__() # Make sure needed directoies exists self.projectRootDir = directory self.projectId = None self.current_sid = None self.callerId = None self.__activeSID = None self.__sessions = {} self.__inputManagerState = dict() self.__transform = None self.sessions = os.path.join(self.projectRootDir, 'sessions') # self.logs = os.path.join(self.projectRootDir, 'logs') self.__ensureDirectory(self.sessions) # self.__ensureDirectory(self.logs) self.settings = self.__loadDefaultSettings() self.loadProject(True) def projectConfigFileName(self): """ Returns the filename of the config-json file """ return _baristaProjectFile(self.projectRootDir) def loadProject(self, set_settings=False): """ Loads the settings of the project. Returns a state-dictionary for the NetworkManager. State looks like: { "projectid": "xyz...", "inputdb": { .. }, "network": { .. }, "solver": {...}, "selection": [], "position": {} } """ filename = self.projectConfigFileName() projectData = None if not os.path.exists(filename): self.__loadDefaultSettings() self.projectId = str(uuid.uuid4()) sessionID = self.createSession(state_dictionary=None) self.setActiveSID(sessionID) else: with open(filename, "r") as file: res = json.load(file) if set_settings and ("environment" in res): self.settings = res["environment"] #path_loader.reloadCaffeModule(self.getCaffeRoot()) #path_loader.reloadProtoModule(self.getCaffeRoot()) if "inputdb" in res: self.__inputManagerState = res["inputdb"] if "projectdata" in res: # this is for backward compatibility, only # Fill missing type-instances allLayers = info.CaffeMetaInformation().availableLayerTypes() layers = res["projectdata"]["network"]["layers"] for id in layers: typename = layers[id]["parameters"]["type"] layers[id]["type"] = allLayers[typename] projectData = res["projectdata"] if "transform" in res: self.__transform = res["transform"] if "projectid" in res: self.projectId = res["projectid"] else: self.projectId = str(uuid.uuid4()) # backwards compatibility self.loadSessions() if len(self.__sessions) == 0: Log.error("Could not find a valid Session, creating new empty Session. Maybe the project " "contains only remote Sessions?", self.getCallerId()) sessionID = self.createSession() #check which session was last active, if this is not available, take the last Session in SID order, if no # valid sessions are available, create a new empty session and set this active if "activeSession" in res: self.setActiveSID(res["activeSession"]) elif len(self.getValidSIDs())>0: self.setActiveSID(self.getValidSIDs()[-1]) else: # this branch should be reached only if an old projekt format is opened, where no sessions are stored sessionID = self.createSession(state_dictionary=projectData) self.setActiveSID(sessionID) if projectData: self.setActiveSessionStateDict(projectData) if self.getActiveSession().hasStateDict(): projectData = self.getActiveSession().state_dictionary else: if projectData: # this branch should be reached only if an old projekt format is opened, # where sessions do not have their own state dictionary self.setActiveSessionStateDict(projectData) return projectData, self.__inputManagerState, self.__transform def getInputManagerState(self): return self.__inputManagerState def getViewTransform(self): return self.__transform def saveProject(self, networkManagerState=None, inputManagerState=None, transform=None): """ Saves the settings of this project using the given State of the NetworkManager.abs State should looks like: { "projectid": "xyz...", "network": { .. }, "inputdb": { .. }, "solver": {...}, "selection": [], "position": {} } """ if inputManagerState: self.__inputManagerState = copy.deepcopy(inputManagerState) if transform: self.__transform = copy.deepcopy(transform) for session in self.__sessions.itervalues(): session.save() # Serialize tosave = { "projectid": self.projectId, "activeSession": self.getActiveSID(), "inputdb": self.__inputManagerState, "environment": self.settings, "transform": [self.__transform.m11(), self.__transform.m12(), self.__transform.m21(), self.__transform.m22(), self.__transform.dx(), self.__transform.dy()] } with open(self.projectConfigFileName(), "w") as file: json.dump(tosave, file, sort_keys=True, indent=4) def __loadDefaultSettings(self): """ Load settings with sensitive defaults. """ self.settings = { 'CAFFE_ROOT': "", 'plotter': { 'logFiles': {}, 'checkBoxes': {} } } def changeProjectCaffePath(self, path): """change the project caffe Path, and if a project settings jason exists, the path is saved in this""" if not os.path.exists(self.projectConfigFileName()): self.__loadDefaultSettings() inputManagerState = dict() transform = None self.settings["CAFFE_ROOT"] = path else: with open(self.projectConfigFileName(), "r") as file: res = json.load(file) res["environment"] = self.settings with open(self.projectConfigFileName(), "w") as file: json.dump(res, file, sort_keys=True, indent=4) def deletePlotterSettings(self, logId): """deletes the plotter settings for a given log, e.g. if a session was reset""" try: #check if settings for given logId exist: if "logId" in self.settings["plotter"]["checkBoxes"]: return except Exception: return res = None #check if a settings json exists: if os.path.exists(self.projectConfigFileName()): with open(self.projectConfigFileName(), "r") as file: res = json.load(file) #determine how much of the json is already created and write accordingly: if res is None: res ={"environment": {"plotter": {"checkBoxes": {}}}} elif "environment" not in res: res["environment"] = {"plotter": {"checkBoxes": {}}} elif "plotter" not in res["environment"]: res["environment"]["plotter"] = {"checkBoxes": {}} elif logId in res["environment"]["plotter"]["checkBoxes"]: res["environment"]["plotter"]["checkBoxes"].pop(logId) #save to file with open(self.projectConfigFileName(), "w") as file: json.dump(res, file, sort_keys=True, indent=4) def getCaffeRoot(self): """returns the caffe path, saved in the project file or, if this isn't set, from Barista default settings""" path = None if self.settings: if "CAFFE_ROOT" in self.settings: if self.settings["CAFFE_ROOT"] != "": path = self.settings["CAFFE_ROOT"] if not path: path = path_loader.getCaffePath() return path def getCallerId(self): """ Return the unique caller id for this project """ if self.callerId is None: self.callerId = Log.getCallerId(self.getProjectName()) return self.callerId def getProjectName(self): """ Return the name of the project. """ return os.path.basename(os.path.normpath(self.projectRootDir)) def getProjectDirectory(self): """ Return the project directory. """ return self.projectRootDir def buildSolverPrototxt(self): """ Load the current solver dictionary and return the corresponding message object. :return: A solver message object. """ res, inp, _ = self.loadProject() solver = saver.saveSolver(res["solver"]) return solver def buildNetPrototxt(self, internalVersion=False): """ Load the current net dictionary and return the corresponding message object. :param internalVersion: Iff true, the loaded net will be modified to the internal version. :return: A net message object. """ currentState = self.getActiveSession().state_dictionary netDictionary = None if currentState: netDictionary = currentState["network"] else: res, _, _= self.loadProject() netDictionary = res["network"] if internalVersion: netDictionary = self._modifyNetDictionaryToInternalVersion(copy.deepcopy(netDictionary)) solver = saver.saveNet(netDictionary) return solver def _modifyH5TxtFile(self, dir, state=None): net = None if state: net = state["network"] else: session = self.getActiveSession() if session: state_dict = session.state_dictionary if state_dict: if "network" in state_dict: net = state_dict["network"] if net: h = helper.DictHelper(net) for layerId, layer in net.get("layers", {}).iteritems(): paramKey = "hdf5_data_param.source" if h.layerParameterIsSet(layerId, paramKey): paramValue = h.layerParameter(layerId, paramKey) if paramValue is not None and not os.path.isabs(paramValue): newFilename = str(layerId) + ".txt" newFilepath = os.path.join(dir, newFilename) oldPath = os.path.join(dir, os.path.join(os.pardir, os.path.join(os.pardir, paramValue))) if os.path.exists(oldPath): with open(newFilepath, "w") as f: lines = [line.rstrip('\n') for line in open(oldPath)] for line in lines: if line is not "": if line[:1] == '.': line = os.path.join(os.pardir, os.path.join(os.pardir, line)) f.write("\n" + line) else: Log.error('Failed to copy hdf5txt file. File does not exists: ' + oldPath, self.getCallerId()) def getActiveSession(self): sid = self.getActiveSID() if sid: return self.getSession(sid) else: return None def getActiveSID(self): if self.__activeSID: if len(self.__sessions) > 0: if self.__activeSID not in self.__sessions: Log.log("The Active Session is no longer available. The Project seems to be broken. The active Session is set to the highest ID available.", self.callerId) self.setActiveSID(self.__sessions.keys()[-1]) Log.log("Active Session set to " + str(self.__activeSID), self.callerId) return self.__activeSID def getSession(self, SID): if SID in self.getValidSIDs(): return self.__sessions[SID] else: Log.log("Session " + str(SID) + " could not be loaded. Valid IDs are: " + ", ".join([str(i) for i in self.getValidSIDs()]), self.getCallerId()) return None def setActiveSessionStateDict(self, state): self.__sessions[self.__activeSID].setStateDict(stateDict=state) def setActiveSID(self, sid): validSIDs = self.getValidSIDs() if sid in validSIDs: self.__activeSID = sid self.activeSessionChanged.emit(sid) else: Log.error("Could not set active session to " + str(sid) + " valid Session-IDs are: "+ ", ".join([str(s) for s in validSIDs]), self.getCallerId()) if not self.__activeSID: self.__activeSID = validSIDs[-1] if len(validSIDs) > 0 else None Log.log("Active session set to " + str(self.__activeSID), self.getCallerId()) def getValidSIDs(self): validSIDs = [session.getSessionId() for session in self.__sessions.itervalues()] return validSIDs def getSettings(self): """ Return the settings for this project. """ return self.settings def getSnapshots(self): """ Return the directory for snapshots. """ return self.snapshots def getLogs(self): """ Return the directory for log files. """ return self.logs def getSessions(self): """ Return a dictionary of finished and running caffe sessions, keyed by session id. """ return self.__sessions def getSessionsDirectory(self): """ Return the directory which contains the sessions. """ return self.sessions def createSession(self, state_dictionary=None): """ Return a new session instance with a new directory """ sid = self.getNextSessionId() session = Session(self, sid=sid, parse_old=False, state_dictionary=state_dictionary) self.__sessions[sid] = session # copy HDF5TXT files # TODO: This should probably be moved to the Session? self._modifyH5TxtFile(session.getDirectory(), state=state_dictionary) self.newSession.emit(sid) return sid def createRemoteSession(self, remote, state_dictionary=None): """use this only to create entirely new sessions. to load existing use the loadRemoteSession command""" sid = self.getNextSessionId() msg = {"key": Protocol.CREATESESSION, "pid": self.projectId, "sid": sid} ret = sendMsgToHost(remote[0], remote[1], msg) if ret: if ret["status"]: uid = ret["uid"] else: for e in ret["error"]: Log.error(e, self.getCallerId()) return None else: Log.error('Failed to create remote session! No connection to Host', self.getCallerId()) return None session = ClientSession(self, remote, uid, sid) if state_dictionary is not None: session.state_dictionary = state_dictionary self.__sessions[sid] = session self.newSession.emit(sid) return sid def loadRemoteSession(self, remote, uid): sid = self.getNextSessionId() session = ClientSession(self, remote, uid, sid) self.__sessions[sid] = session self.newSession.emit(sid) return sid def cloneSession(self, solverstate, old_session=None): """ Return a new session cloned from the solverstate. If old_session is None the solverstate is expected to be a valid path to a solverstate. If the old_session exists the solverstate is searched within the sessions snapshot directory. Copies the last snapshot to the new sessions snapshot directory. """ if old_session: snapshot_dir = old_session.getSnapshotDirectory() solverstate = os.path.basename(solverstate) old_lss = None if os.path.isabs(snapshot_dir): old_lss = os.path.join(snapshot_dir, solverstate) else: old_sdir = old_session.getDirectory() old_lss = os.path.join(old_sdir, snapshot_dir, solverstate) else: old_lss = solverstate sessionID = self.createSession() self.__sessions[sessionID].setParserInitialized() if sessionID is None: Log.error('Failed to create session!', self.getCallerId()) return None new_sdir = self.__sessions[sessionID].getSnapshotDirectory() self.__ensureDirectory(self.__sessions[sessionID].getDirectory()) self.__ensureDirectory(new_sdir) if os.path.isdir(new_sdir): # copy solverstate new_lss = os.path.join(new_sdir, os.path.basename(solverstate)) try: shutil.copy2(old_lss, new_lss) self.__sessions[sessionID].setLastSnapshot(new_lss) except Exception as e: Log.error('Failed to copy solverstate to new session: '+str(e), self.getCallerId()) # copy caffemodel model_file = loader.getCaffemodelFromSolverstate(old_lss) iterations = loader.getIterFromSolverstate(old_lss) if model_file is None: model_file = loader.getCaffemodelFromSolverstateHdf5(old_lss) if model_file is None: Log.error('Could not load model from solverstate '+old_lss, self.getCallerId()) self.__sessions[sessionID].delete() return None model_dir = os.path.dirname(old_lss) old_lcm = os.path.join(model_dir, model_file) new_caffemodel = os.path.basename(model_file) new_lcm = os.path.join(new_sdir, new_caffemodel) try: shutil.copy2(old_lcm, new_lcm) self.__sessions[sessionID].setLastModel(new_lcm) self.__sessions[sessionID].iteration = iterations if old_session is not None: self.__sessions[sessionID].max_iter = old_session.max_iter self.__sessions[sessionID].caffe_root = old_session.caffe_root except Exception as e: Log.error('Failed to copy caffemodel to new session: '+str(e), self.getCallerId()) # copy state dictionary if old_session is not None: #TODO: This will cause troubles if the current session was changed during training (See #270) self.__sessions[sessionID].setStateDict(old_session.state_dictionary) if self.__sessions[sessionID].iteration > 0: self.__sessions[sessionID].setState(State.PAUSED) else: self.__sessions[sessionID].setState(State.WAITING) self.__sessions[sessionID].save(includeProtoTxt=True) return sessionID else: self.__sessions[sessionID].delete() Log.error('Snapshot directory '+new_sdir+' does not exist!', self.getCallerId()) return None def rebuildSession(self, directory): """ Rebuild the session from the given directory. Return: The session if the directory is a valid session directory. """ session = Session(self, directory=directory, parse_old=False) return session def loadSessions(self): """ Load sessions which are found in the sessions directory. Create a session object for every session and key it by the session id. """ pool = SessionPool() sdir = self.getSessionsDirectory() count = 0 for entry in os.listdir(sdir): sess_dir = os.path.join(sdir, entry) if self.isSession(sess_dir): session = self.rebuildSession(sess_dir) if session: self.__sessions[session.getSessionId()] = session pool.addSession(session) self.newSession.emit(session.getSessionId()) count += 1 return count def removeSessions(self, sessions): """ Remove the sessions from the project. """ for session in sessions: sid = session.getSessionId() session.delete() self.__sessions.pop(sid) del session def isSession(self, directory): """ Checks if the directory contains a valid session.""" if not os.path.exists(directory): Log.error('Session directory '+directory+' does not exist', self.getCallerId()) return False session_file = os.path.join(directory, Paths.FILE_NAME_SESSION_JSON) if not os.path.exists(session_file): Log.error('Session directory '+directory+' does not contain a session file.', self.getCallerId()) return False with open(session_file, 'r') as file: d = json.load(file) jsonKeys = ("Iteration", "MaxIter", "NetworkState", "ProjectID", "SessionState") keyNotFound=False for key in jsonKeys: if key not in d: Log.error("Session directory " + os.path.basename(os.path.normpath(dir)) + " is invalid! Key '" + key + "' is missing in sessionstate!", self.getCallerId()) keyNotFound = True if keyNotFound: Log.error("Session at %s is invalid. Key %s in 'sessionstate.json' is missing", self.getCallerId()) return False return True def hasRunningSessions(self): """ Return True if there are running sessions in this project. """ for sid, session in self.__sessions.iteritems(): if session.getState() is State.RUNNING: return True return False def pauseRunningSessions(self): """ Pause all running sessions. """ for sid, session in self.__sessions.iteritems(): if session.getState() is State.RUNNING: session.pause() def getNextSessionId(self): """ Return the next session id from the sequence of ids. """ sIDs = [s for s in self.__sessions.iterkeys()] if len(sIDs) > 0: return max(max(sIDs)+1, 1) else: return 1 #def getCaffeRoot(self): # """ Return the caffe installation directory. # """ # if 'CAFFE_ROOT' in self.settings: # print("bla") # self.__loadDefaultSettings() # else: # print("bla2") # return self.settings['CAFFE_ROOT'] def __ensureDirectory(self, directory): """ Creates a directory if it does not exist. """ if directory == '': return if not os.path.exists(directory): try: os.makedirs(directory) Log.log('Created directory: '+directory, self.getCallerId()) except Exception as e: Log.error('Failed to create directory '+directory+')'+str(e), self.getCallerId()) def parseSnapshotPrefixFromFile(self, filename): """ Return the snapshot prefix of the solver. """ with open(filename) as f: regex_prefix = re.compile( 'snapshot_prefix:[\s]+"(.+)"') for line in f: prefix_match = regex_prefix.search(line) if prefix_match: return prefix_match.group(1) backend/caffe/__init__.py0000664ڊ00111170000000000013243274037015540 0ustar klemmso0cvprbackend/caffe/check_hardware.py0000664ڊ00111170000000546213243274037016754 0ustar klemmso0cvprfrom subprocess import Popen, PIPE, STDOUT from sys import stdout from os import path import logging from backend.networking.protocol import Protocol def checkHardware(cafferoot, silent=False, transaction=None): """ probe caffe continuously for incrementing until missing id structure: [ { "id": 0, "name": "..", "log": ["..", "..", "..", ... ] }, { "id": 1, "name": "..", "log": ["..", "..", "..", ... ] }, ... ] """ gid = 0 hw = [] if not silent: stdout.write("Checking Hardware...\n") logging.info("Checking Hardware...") cpu = _getCPU() name = _getCPUName(cpu) hw.append({"name": name, "log": cpu}) if not silent: stdout.write("CPU found: " + name + "\n") logging.info("CPU found: %s", name) if transaction: msg = {"key": Protocol.SCANHARDWARE, "finished": False, "name": name} transaction.send(msg) while True: log = _getId(gid, cafferoot) if not _isValid(log) or _isCpuOnly(log): if not silent and gid is 0: stdout.write("No GPU found, CPU mode\n") logging.info("No GPU found, CPU mode") break name = _getName(log) if not silent: stdout.write("GPU " + str(gid) + " found: " + name + "\n") if transaction: msg = {"key": Protocol.SCANHARDWARE, "finished": False, "name": name, "id": gid} transaction.send(msg) hw.append({"id": gid, "name": name, "log": _parseLog(log)}) gid += 1 return hw def _getId(gid, cafferoot): """probe caffe for gpu id""" caffe = path.join(cafferoot, "build/tools/caffe") proc = Popen([caffe, "device_query", "-gpu", str(gid)], stdout=PIPE, stderr=STDOUT) it = iter(proc.stdout.readline, '') log = [] for i in it: log.append(i[:-1]) return log def _getCPU(): proc = Popen(["cat", "/proc/cpuinfo"], stdout=PIPE) it = iter(proc.stdout.readline, '') log = [] for i in it: log.append(i[:-1]) return log def _isValid(log): """check if no error has occurred""" for l in log: if "error" in l.lower(): return False return True def _isCpuOnly(log): """check for CPU-Only mode""" for l in log: if "cpu" in l.lower(): return True return False def _getName(log): """extract the Name""" for l in log: if "name:" in l.lower(): return l[l.index("Name:") + 5:].strip() return "" def _getCPUName(log): """extract the CPU name""" for l in log: if "model name" in l.lower(): return l.split(":")[1].strip() def _parseLog(log): """clean up logs to show to user""" return [l[l.index("]")+2:] for l in log[1:]] backend/caffe/dict_helper.py0000664ڊ00111170000002605313243274037016303 0ustar klemmso0cvprfrom backend.caffe.loader import _extract_param import backend.caffe.proto_info as info import uuid import copy from google.protobuf.internal.containers import RepeatedScalarFieldContainer as ProtoList def isNetDictValid(netdic): """ This function returns a tuple (valid, msg). Value is true iff netdic has a valid format. If the format is not valid, msg contains a text which describes the problem. """ if not {"layers", "layerOrder"}.issubset(netdic.keys()): missing = {"layers","layerOrder"}.difference(netdic.keys()) return False, "Needed parameters {} not in dictionary".format(list(missing)) layers = netdic["layers"] order = netdic["layerOrder"] #Check if netparameters are valid (except layers) netparam = info.CaffeMetaInformation().availableNetParameters().copy() del netparam["layers"] tmp = netdic.copy() del tmp["layers"] del tmp["layerOrder"] if not _areParameterValid(netparam, tmp): return False, "Invalid net paremters" # Check for correct types of this special values if type(order) != list or type(layers) != dict: return False, "layerOrder or layers are invalid types" # Test if all ids are in the layerOrder list and vise versa if set(layers.keys()) != set(order): missingOrder = set(layers.keys()).difference(order) missingLayer = set(order).difference(layers.keys()) return False, "layerOrder and layers does not have same ids - missing ids in layers: {} ; missing ids in order: {}"\ .format(missingLayer, missingOrder) # Check every sublayer for id in order: if not {"parameters","type"}.issubset(layers[id].keys()): return False, "parameters or type are not in layers subdictionary" params = layers[id]["parameters"] if params is None: return False, "parameters is None, should be at least {'name': '..', 'type': '...'}" layer = layers[id]["type"] if not isinstance(layer, info.LayerType): return False, "layers is {}, expected instance of protoinfo.LayerType".format(layer) validparam, msg = _areParameterValid(layer.parameters(), params) if not validparam: return False, "Invalid parameters for {}: {}".format(id, msg) return True, "" def _areParameterValid(allparams, paramdict, prefix=""): """ Checks if all parameters in paramdict are valid. allparams should contain all available parameters names as key and the proto-meta-info-class as value. """ def checkType(meta, value, typeprefix): pytype = None if meta.isParameterGroup(): if type(value) != dict: return False, "Expected dictionary , found {}".format(type(value)) return _areParameterValid(meta.parameter(), value, typeprefix) elif meta.isBool(): pytype = [bool] elif meta.isString(): pytype = [unicode, str] elif meta.isInt(): pytype = [int, long] elif meta.isBytes(): pass # TODO: What type are bytes? elif meta.isFloat() or meta.isDouble(): pytype = [float] elif meta.isEnum(): pytype = [str, unicode] res = type(value) in pytype if not res: return False, "Expected Type {}, found {}".format(pytype, type(value)) return True, "" for param in paramdict: typeprefix=prefix+"."+param val = paramdict[param] if not allparams.has_key(param): return False, "Parameter {}: Parameter not available in Caffe".format(typeprefix) meta = allparams[param] if meta.isRepeated(): if not (type(val) is ProtoList or type(val) is list): return False, "Parameter {}: Expected repeated entry, found single one".format(typeprefix) for i,entry in enumerate(val): valid, msg = checkType(meta,entry, typeprefix) if not valid: return False, "Parameter {} - idx {}: {}".format(typeprefix, i,msg) else: valid, msg = checkType(meta,val,typeprefix) if not valid: return False, "Parameter {}: {}".format(typeprefix,msg) return True, "" def bareNet(name): """ Creates a dictionary of a networks with default values where required. """ import backend.caffe.path_loader as caffeLoader proto = caffeLoader.importProto() net = proto.NetParameter() descr = info.ParameterGroupDescriptor(net) params = descr.parameter().copy() del params["layer"] del params["layers"] res = _extract_param(net, params) res["layers"] = {} res["layerOrder"] = [] res["name"] = unicode(name) return res def _bareLayer(layertype, name): """ Creates a dictionary of the given layertype with the given name initialized with default values if required. """ import backend.caffe.path_loader as caffeLoader proto = caffeLoader.importProto() res = {"type": layertype} layerParamInst = proto.LayerParameter() res["parameters"] = _extract_param(layerParamInst, info.CaffeMetaInformation().commonParameters()) res["parameters"]["name"] = unicode(name) layerparamkeys = layertype.layerParamKeys() for layerparamkey in layerparamkeys: layerTypeParam = layertype.parameters()[layerparamkey] layerTypeParamInst = layerTypeParam.protoclass()() res["parameters"][layerparamkey] = _extract_param(layerTypeParamInst, layerTypeParam.parameter()) res["parameters"]["type"] = unicode(layertype.name()) return res class DictHelper: """ A helper class to simplify the using of a network dictionary """ def __init__(self,netdic): self._netdic = netdic def nameOfLayers(self): """ Return a list with all layers of the networks """ return [x["parameters"]["name"] for x in self._netdic["layers"].values()] def nameOfTopBlobs(self): """ Return a list with all top blob names in use of the current network """ names = [] for id in self._netdic["layers"]: if "top" in self.layerPerId(id)["parameters"]: for j in range(0, len(self.layerPerId(id)["parameters"]["top"])): names.append(self.layerPerId(id)["parameters"]["top"][j]) return names def netParameters(self): """ Return all parameters for the network (excluding layers) """ return dict([(k,self._netdic[k]) for k in self._netdic if not k in ["layers","layerOrder"]]) def layerIdsForName(self,name): """ Return a list of the ids for every layer with the given name """ return [x for x in self._netdic["layers"] if self._netdic["layers"][x]["parameters"]["name"] == name] def layerType(self,id): """ Return the type of the layer with the given id """ return self._netdic["layers"][id]["type"] def layerParams(self,id): """ Return the parameter of the layer with the given id """ return self._netdic["layers"][id]["parameters"] def layerPerId(self, id): """ Return the layer with the given id """ return self._netdic["layers"][id] def hasLayerWithId(self,id): """ Return true if there is a layer with the given id""" return id in self._netdic["layers"] def addLayer(self, layertype, name, idx): """ Add a new Layer from layertype with the given name and initialize it with the default values if required. This new layer is added to the intern dictionary at position idx and the layer subdictionary will be returned with the id. import backend.caffe.protoinfo as info convLayer = info.CaffeMetaInformation().availableLayerTypes()["Convolution"] newLayer, layerId = dichelperinstance.addLayer(convLayer, "Convolution", 3) """ newLayer = _bareLayer(layertype, name) id = str(uuid.uuid4()) self._netdic["layerOrder"].insert(idx, id) self._netdic["layers"][id] = newLayer return newLayer, id def removeLayer(self, id): del self._netdic["layers"][id] self._netdic["layerOrder"].remove(id) def duplicateLayer(self, id, newName, idx=None): """ Duplicate Layer with given id. The new layer has a new random id and the new given name. The layer will be added at position idx in the layerOrder. If idx==None, the idx will be directly beside the old layer. This function returns the new layer as dictionary and its id. E.g. newLayerDict, newId = dicthelper.duplicateLayer("somerandomid", "Some awesome name") """ oldLayer = self.layerPerId(id) newLayer = { "type": oldLayer["type"], "parameters": copy.deepcopy(oldLayer["parameters"]) } newLayer["parameters"]["name"] = newName newId = str(uuid.uuid4()) if idx is None: idx = self._netdic["layerOrder"].index(id)+1 self._netdic["layerOrder"].insert(idx, newId) self._netdic["layers"][newId] = newLayer return newLayer, newId def dictionary(self): return self._netdic def layerParameterIsSet(self, layerId, parameterKey): """ Check whether the given parameter is set for the given layer. :param layerId: The id of the layer to check. :param parameterKey: The key of the parameter to check. Might represent a nested parameter concatenated with dots. :return: """ isSet = "parameters" in self._netdic["layers"][layerId] if isSet: paramKeyParts = parameterKey.split(".") value = self._netdic["layers"][layerId]["parameters"] for paramKeyPart in paramKeyParts: if paramKeyPart in value: value = value[paramKeyPart] else: isSet = False break return isSet def layerParameter(self, layerId, parameterKey): """ Get a specific parameter for one of the layers. :param layerId: The id of the layer. :param parameterKey: The key of the requested parameter. Might represent a nested parameter concatenated with dots. :return: """ paramKeyParts = parameterKey.split(".") value = self._netdic["layers"][layerId]["parameters"] for paramKeyPart in paramKeyParts: if paramKeyPart in value: value = value[paramKeyPart] else: value = None break return value def setLayerParameter(self, layerId, parameterKey, value): """ Set a specific parameter for one of the layers. :param layerId: The id of the layer. :param parameterKey: The key of the requested parameter. Might represent a nested parameter concatenated with dots. :return: """ paramKeyParts = parameterKey.split(".") dic = self._netdic["layers"][layerId]["parameters"] for key in paramKeyParts[:-1]: dic = dic.setdefault(key, {}) dic[paramKeyParts[-1]] = value backend/caffe/loader.py0000664ڊ00111170000001756513243274037015277 0ustar klemmso0cvprimport uuid import h5py as h5 from google.protobuf import text_format from google.protobuf.text_format import ParseError import backend.caffe.proto_info as info from backend.barista.utils.logger import Log class ParseException(Exception): def __init__(self, msg): self._msg = msg def __str__(self): return self._msg def loadSolver(solverstring): """ Return a dictionary which represent the caffe-solver-prototxt solverstring """ import backend.caffe.path_loader as caffeLoader proto = caffeLoader.importProto() solver = proto.SolverParameter() # Get DESCRIPTION for meta infos descr = info.ParameterGroupDescriptor(solver) # "Parse" the solver-definition in prototxt-format try: text_format.Merge(solverstring,solver) except ParseError as ex: raise ParseException(str(ex)) params = descr.parameter().copy() # All Parameters of the solver return copy.deepcopy(_extract_param(solver,params)) import copy def loadNet(netstring): """ Load the prototxt string "netstring" into a dictionary. The dictionary has the following form { "name": "Somenetwork", "input_dim": [1,2,1,1], "state": { "phase": "TRAIN" }, ... "layers": { "somerandomid1": { "type": LayerType Instance of Pooling-Layer, "parameters": { "pooling_param": [ "kernel_size": 23, "engine": "DEFAULT" ] .... "input_param": [ {"shape": {"dim": [...], .... }, {"shape": {"dim": [...], .... }, ] } }, "somerandomid2": {"type": ..., "parameters": ....} }, "layerOrder": ["somerandomid1", "somerandomid2", ....] } """ import backend.caffe.path_loader as caffeLoader proto = caffeLoader.importProto() # Load Protoclass for parsing net = proto.NetParameter() # Get DESCRIPTION for meta infos descr = info.ParameterGroupDescriptor(net) # "Parse" the netdefinition in prototxt-format try: text_format.Merge(netstring,net) except ParseError as ex: raise ParseException(str(ex)) params = descr.parameter().copy() # All Parameters of the network # add logger output if deprecated layers have been found, to inform the user that those can't be parsed yet if len(net.layers) > 0: callerId = Log.getCallerId('protoxt-parser') Log.log("The given network contains deprecated layer definitions which are not supported and will be dropped.", callerId) # Layers is deprecated, Layer will be handled seperatly and linked to "Layers" key del params["layers"] del params["layer"] if params.has_key("layerOrder"): raise ValueError('Key layerOrder not expected!') # Extract every other parameters res = _extract_param(net,params) res["layers"], res["layerOrder"] = _load_layers(net.layer) res = copy.deepcopy(res) return res def _load_layers(layerlist): """ Build the dictionary of all layers in layerlist. The dictionary has the form loadNet needs. """ allLayers = info.CaffeMetaInformation().availableLayerTypes() order = [] res = {} dicLayerTypeCounter = {} for layer in allLayers: dicLayerTypeCounter[layer] = 1 for layer in layerlist: typename = layer.type if not allLayers.has_key(typename): raise ParseException("Layer with type {} not available in caffe".format(typename)) layerinfo = allLayers[typename] id = str(uuid.uuid4()) res[id]={ "type": layerinfo, "parameters": _extract_param(layer,layerinfo.parameters()) } order.append(id) for id in order: if "name" not in res[id]["parameters"]: typeName = res[id]["parameters"]["type"] newName = typeName + " #" + str(dicLayerTypeCounter[typeName]) changed = True; while changed == True: newName = typeName + " #" + str(dicLayerTypeCounter[typeName]) changed = False for id2 in order: if "name" in res[id2]["parameters"] and res[id2]["parameters"]["name"] == newName: dicLayerTypeCounter[typeName] += 1 changed = True res[id]["parameters"]["name"] = newName return res, order def _extract_param(value,parameters): """ Build the dictionary of all paramters, e.g. for layer. The parameters will be constructed recursivly. "value" is the loaded class from caffe.proto.caffe_pb2 and "parameters" the dictionary of parameter-descriptions (wrapped with protoinfo.py-Classes). """ res = {} for paramname in parameters: val = getattr(value, paramname) parameter = parameters[paramname] if parameter.isRepeated(): if len(val) == 0: # probably not set continue if parameter.isParameterGroup(): res[paramname] = [_extract_param(subval, parameter.parameter()) for subval in val] elif parameter.isEnum(): res[paramname] = [copy.deepcopy(parameter.availableValues()[x]) for x in val] else: res[paramname] = list(val) else: if not value.HasField(paramname): #Not explicit set -> ignore continue if parameter.isParameterGroup(): res[paramname] = _extract_param(val, parameter.parameter()) elif parameter.isEnum(): res[paramname] = copy.deepcopy(parameter.availableValues()[val]) else: res[paramname] = val return res def extractNetFromSolver(solverstring): """Read a protoxt string(!) of a solver and return the network protoxt string(!). This works only, if the solver specifies a network using the "net_param" parameter. A reference to a file using the "net" parameter can not be handled by this method. """ import backend.caffe.path_loader as caffeLoader proto = caffeLoader.importProto() # create empty solver message solver = proto.SolverParameter() # "Parse" the solver-definition in prototxt-format try: text_format.Merge(solverstring, solver) # extract net as a message and convert it into a string netString = text_format.MessageToString(solver.net_param) return netString except ParseError as ex: raise ParseException(str(ex)) def getCaffemodelFromSolverstate(solverstate): """ Parse the filename of the caffemodel file from the solverstate file. """ import backend.caffe.path_loader as caffeLoader proto = caffeLoader.importProto() try: state = proto.SolverState() with open(solverstate, 'rb') as f: state.ParseFromString(f.read()) return state.learned_net except Exception as e: print(str(e)) def getCaffemodelFromSolverstateHdf5(filename): """ Extract the filename of the caffemodel file from the solverstate hdf5 file. """ try: file = h5.File(filename, 'r') if "learned_net" in file.keys(): return str(file["learned_net"].value) return None except: return None def getIterFromSolverstate(solverstate): """ Parse the iterations from the solverstate file. """ import backend.caffe.path_loader as caffeLoader proto = caffeLoader.importProto() try: state = proto.SolverState() with open(solverstate, 'rb') as f: state.ParseFromString(f.read()) return state.iter except Exception as e: print(str(e)) backend/caffe/path_loader.py0000664ڊ00111170000000267713243274037016311 0ustar klemmso0cvprimport imp from backend.barista.utils.settings import applicationQSetting import os #todo: move this function to an new class def changeCaffePath(path): """changes the default path in settings""" settings = applicationQSetting() settings.beginGroup("Path") settings.setValue("caffePath", path) settings.endGroup() def getCaffePath(): """returns the default path from settings""" settings = applicationQSetting() settings.beginGroup("Path") path = settings.value("caffePath","") settings.endGroup() return path def checkIfFileIsCaffeRoot(fileName): """checks if the given file is a accurate path""" path = os.path.join(str(fileName), "src/caffe/proto/caffe.proto") if os.path.isfile(path): return True else: return False def importCaffe(): import importlib import sys path = getCaffePath() path += "/python" sys.path.insert(0, path) # stellt sicher, dass der angegebene Pfad als erstes durchsucht wird try: return importlib.import_module("caffe") except ImportError as e: print(e) sys.exit(1) def importProto(): import importlib import sys path = getCaffePath() path += "/python" sys.path.insert(0, path) # stellt sicher, dass der angegebene Pfad als erstes durchsucht wird try: return importlib.import_module("caffe.proto.caffe_pb2") except ImportError as e: print(e) sys.exit(1) backend/caffe/proto_description.py0000664ڊ00111170000002430713243274037017567 0ustar klemmso0cvprimport os import re from backend.barista.utils.settings import applicationQSetting from backend.barista.utils.logger import Log from backend.caffe import path_loader class Singleton(type): """This metaclass is used to provide the singleton pattern in a generic way. See http://stackoverflow.com/a/6798042 for source and further explanation. TODO outsource this class to use the same pattern in the complete project? """ _instances = {} def __call__(cls, *args, **kwargs): if cls not in cls._instances: cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) return cls._instances[cls] class CaffeProtoParser: """This class parses the caffe.proto file. This is different from what is done in other files of this package, as this is the first time we really parse the text file itself on our own. Other methods rely on meta data provided after(!) parsing/compiling that file. """ __metaclass__ = Singleton def __init__(self, replaceLineBreaks=True): """ replaceLineBreaks: If True, all line breaks will be replaced by spaces. This is the default behavior as most line breaks are only caused by the limited code width rather than for semantic reasons. Anyway, there are also some parameter descriptions using linebreaks on purpose (e.g. lr_policy). (This does not affect cases in which there is an empty comment line. The latter will be kept anyway.) """ self._replaceLineBreaks = replaceLineBreaks self._fileContent = self._readFile() self._fieldDescriptions, self._messageDescriptions = self._parseComments() def _readFile(self): """Read the content of the caffe.proto file.""" import backend.caffe.path_loader as caffeLoader caffe = caffeLoader.importCaffe() # first of all, try to use the environment variable CAFFE_ROOT caffeRootPath = path_loader.getCaffePath() # if that variable isn't available, try to reconstruct the caffe root path based on the loaded module if caffeRootPath is None: caffeRootPath = os.path.join(os.path.dirname(caffe.__file__), "../..") # extend root path by file name caffeProtoFilePath = os.path.join(caffeRootPath, "src/caffe/proto/caffe.proto") if os.path.isfile(caffeProtoFilePath): # open caffe.proto file with open(caffeProtoFilePath) as f: # read complete file at once content = f.read() else: callerId = Log.getCallerId("parameter_descriptions") Log.error("ERROR: Unable to load caffe.proto file. Parameter descriptions and deprecated flags " "will be left empty. Did you configure the environment variable CAFFE_ROOT?", callerId) content = "" return content def _parseComments(self): """Parse all parameter descriptions provided in the C-style comments. Results will be provided by the fieldDescriptions() and messageDescriptions() methods. """ # our result cache fieldDescr = {} msgDescr = {} # get all defined messages and their content to be able grouping their parameters # regex inside of outer msg brackets: # repeat (any chars except brackets) or (opening bracket + any chars except brackets + closing brackets) msgBlocks = re.findall('((//[^\n]*\n)*)message\s*(\w+)\s*\{(' # begin of message definition '([^\{\}]|(\{[^\{\}]*\}))*' ')\}', self._fileContent) # end of message definition for block in msgBlocks: # only some of the groups are relevant. ignore the rest. msgName = block[2] msgContent = block[3] # first of all: get the comment right above the message itself. not the ones included. msgCommentLines = block[0].splitlines() msgComment = "" for line in msgCommentLines: msgComment = self._proccessCommentLine(line, msgComment) if len(msgComment) > 0: msgDescr[msgName] = self._finalizeComment(msgComment) # collect all parameters of this message, before adding them to the global result msgParams = {} # parse line by line of the message's content and add it to either a comment, parameter, or nothing msgLines = msgContent.splitlines() currComment = "" for line in msgLines: # check whether this is a comment line or a parameter line # (if a comment has been added next to a param definition, it might be both) isComment = re.search(r'.*//\s*(.*)\s*', line) is not None # do not restrict to leading whitespaces! nonEnumParamMatch = re.search(r'\s*(repeated|optional|required)\s*(\w+)\s*(\w+)\s*=', line) enumParamMatch = re.search(r'\s*enum\s*(\w+)\s*\{', line) isParam = nonEnumParamMatch is not None or enumParamMatch is not None # if this is a comment line.. if isComment: # add line to current comment currComment = self._proccessCommentLine(line, currComment) # if the last lines have build a comment and this line contains a new parameter definition.. if len(currComment) > 0 and isParam: # add the current parameter to the list of the current message if nonEnumParamMatch is not None: paramName = nonEnumParamMatch.group(3) else: paramName = enumParamMatch.group(1) msgParams[paramName] = self._finalizeComment(currComment) # reset current comment, if this line isn't one anymore or if it is a inline-comment that belongs # to the parameter in the same line => this param was already processed if not isComment or isParam: currComment = "" # if any parameter descriptions have been found for this message, add it to the global result if len(msgParams) > 0: fieldDescr[msgName] = msgParams return fieldDescr, msgDescr def _proccessCommentLine(self, line, currComment): """Process line as a comment line and append the result to currComment.""" # the content of this line without everything which comes before the comment slashes commentReducible = re.search(r'.*//\s*(.*)\s*', line) if commentReducible is not None: lineContent = commentReducible.group(1) else: # might be empty or reduced already lineContent = line # check whether this comment line does only contain whitespaces isNotEmpty = re.search(r'\S+', lineContent) # recreate linebreaks or replace them with spaces # (don't add anything at the beginning or if the last char is a newline, which might happen in all configs) if len(currComment) > 0 and currComment[-1] != "\n": if isNotEmpty is not None: if self._replaceLineBreaks: currComment += " " else: currComment += "\n" else: # if this is an empty comment line, add an empty line to the full comment, too. currComment += "\n\n" # add text of current line to current comment if isNotEmpty is not None: currComment += lineContent return currComment def _finalizeComment(self, comment): """Call this method for each comment after each line has been processed by _proccessCommentLine().""" # remove multiple spaces following each other while " " in comment: comment = comment.replace(" ", " ") # unify descriptions: add a "." at the end of each non-empty comment. if len(comment) > 0 and comment[-1] != ".": comment += "." # capitalize first letter comment = comment.capitalize() return comment def fieldDescriptions(self): """Get all descriptions provided in the caffe.proto file's comments, that belong to a field inside of a message. Returned value is a dictionary of dictionaries. Keys of the first dimension are names of messages, e.g. 'AccuracyParameter'. Keys of the second dimension are parameter names of those messages, such as 'top_k'. Finally, each of those keys is connected to a string containing the actual description. If a message/parameter does not have any descriptions, there won't be a dictionary entry for that element at all. """ return self._fieldDescriptions def messageDescriptions(self): """Get all descriptions provided in the caffe.proto file's comments, that belong to a message itself. The returned value is a dictionary. Keys are message names and values their descriptions as strings. """ return self._messageDescriptions def description(self, msgName, fieldName=None, fieldMsgDefault=None): """Get a single description for either a field of a message or a message itself. fieldName: If None, the message description for msgName will be returned. Otherwise, the description for fieldName of msgName. fieldMsgDefault: If field is also a message, the general message description of messageDescriptions()[fieldMsgDefault] will be used as a default value for field. """ description = "" if fieldName is None: # message description if msgName in self.messageDescriptions(): description = self.messageDescriptions()[msgName] else: # field description if msgName in self.fieldDescriptions() and fieldName in self.fieldDescriptions()[msgName]: description = self.fieldDescriptions()[msgName][fieldName] elif fieldMsgDefault is not None and fieldMsgDefault in self.messageDescriptions(): # field description based on message description description = self.messageDescriptions()[fieldMsgDefault] return description backend/caffe/proto_info.py0000664ڊ00111170000005313013243274037016173 0ustar klemmso0cvprfrom google.protobuf.descriptor import FieldDescriptor import sys, inspect from backend.caffe import proto_description, path_loader import re import copy """This module dynamically provides meta information about the available caffe parameters. If the compiled caffe version is changed, the provided meta information will change, too. Use these information to show available options in the gui. Example to get all layer types: types = CaffeMetaInformation().availableLayerTypes() """ moreLayerNameParameter = { "Deconvolution": "ConvolutionParameter" } class Singleton(type): """This metaclass is used to provide the singleton pattern in a generic way. See http://stackoverflow.com/a/6798042 for source and further explanation. TODO outsource this class to use the same pattern in the complete project? """ _instances = {} def __call__(cls, *args, **kwargs): if cls not in cls._instances: cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) return cls._instances[cls] class CaffeMetaInformation: """This class is the main interface for any usage outside of this module. It provides various meta information about the compiled caffe version and its available features. Singleton is used to ensure that all constant data is generated only once. """ __metaclass__ = Singleton def __init__(self): self.updateCaffeMetaInformation() def updateCaffeMetaInformation(self): self.__initAvailableParameterGroupDescriptors() self.__initAvailableLayerTypes() self.__initAvailableSolverTypes() def __initAvailableParameterGroupDescriptors(self): """Generate information about available parameter only once using this method. See description of self.availableParameterGroupDescriptors(). """ import backend.caffe.path_loader as caffeLoader proto = caffeLoader.importProto() current_module = sys.modules[proto.__name__] res = {} for (el,val) in inspect.getmembers(current_module, inspect.isclass): res[el] = ParameterGroupDescriptor(val) self._availableParameterGroupDescriptors = res def __initAvailableLayerTypes(self): """Generate information about available layer types only once. self.__initAvailableParameterGroupDescriptors() needs to be called before this method.""" import backend.caffe.path_loader as caffeLoader caffe = caffeLoader.importCaffe() layerNameMainParts = list(caffe.layer_type_list()) res = {} paramsPerLayerType = {} # calculate common parameters of all layer types # by removing all which will be used for one specific layer type only # also keep in mind which ones have been removed to readd them to specific layers commonParams = self._availableParameterGroupDescriptors["LayerParameter"].parameter() #use .parameter() on purpose layerSpecificParameters = set() for nameMainPart in layerNameMainParts: specificParamsName = [nameMainPart + "Parameter"] if moreLayerNameParameter.has_key(nameMainPart): specificParamsName.append( moreLayerNameParameter[nameMainPart]) paramsPerLayerType[nameMainPart] = {} for key, value in commonParams.items(): if value.isParameterGroup() and value.parameterName() in specificParamsName: paramsPerLayerType[nameMainPart][key] = value layerSpecificParameters.add(key) # special case: shared params for loss layers key = "loss_param" value = commonParams[key] del commonParams[key] for nameMainPart in layerNameMainParts: if LayerType.getCategoryByName(nameMainPart) == LayerType.CATEGORY_LOSS: paramsPerLayerType[nameMainPart][key] = value # TODO is there a special case for the TransformationParameter? # create each layer type after one another for nameMainPart in layerNameMainParts: # add common params to the specific ones layerTypeParam = paramsPerLayerType[nameMainPart].keys() paramsPerLayerType[nameMainPart].update(commonParams) irrelevant = layerSpecificParameters.difference(layerTypeParam) res[nameMainPart] = LayerType(nameMainPart, paramsPerLayerType[nameMainPart], layerTypeParam, irrelevant) self._commonParams = commonParams self._availableLayerTypes = res def __initAvailableSolverTypes(self): """Generate information about available solver types only once. self.__initAvailableParameterGroupDescriptors() needs to be called before this method.""" import backend.caffe.path_loader as caffeLoader caffe = caffeLoader.importCaffe() # DO NOT REMOVE the following import statement, although your IDE might say it's unused. It's not! from caffe._caffe import Solver as SolverBaseClassInfo # Unfortunately, there isn't a preexisting method to get all solver (names). # But, there is a base class which is subclassed by all existing solvers. # Use those subclasses to gain the names of all solvers. solverNamesFull = [cls.__name__ for cls in vars()['SolverBaseClassInfo'].__subclasses__()] # All those names end with the suffix "Solver". # Remove that suffix to be consistent with the names of available LayerTypes. solverNameMainParts = [] for name in solverNamesFull: solverNameMainParts.append(name.replace("Solver","")) # Get all (common) params of a solver type # TODO try to separate params which should be available only for a specific SolverType. # Unfortunately, there doesn't seem to exist any way to automatically do that. There are only a few comments # in the caffe.proto file which would allow hardcoding some special cases. But is that a good idea? commonParams = self._availableParameterGroupDescriptors[ "SolverParameter"].parameter() # use .parameter() on purpose # create each solver type after one another res = {} for nameMainPart in solverNameMainParts: res[nameMainPart] = SolverType(nameMainPart, commonParams) self._availableSolverTypes = res def availableParameterGroupDescriptors(self): """Return all available Parameter (e.g. Layer-Parameter) for Caffe as an dictionary. The key is the name of the parameter and the value is an instance of the helper class ParameterGroupDescriptor. This is the most general way of getting any caffe meta information. """ return self._availableParameterGroupDescriptors def availableLayerTypes(self): """Return a dictionary containing all layer types which are available in caffe. The key is the name of the layer type (without the suffix 'Layer'. e.g. 'SoftMax' instead of 'SoftMaxLayer') and the value is an object of the class LayerType. """ return self._availableLayerTypes def commonParameters(self): """ Return parameters all layers share. It is missing specific paremeters like "inner_product_param" or "convolution_param". """ return self._commonParams def availableSolverTypes(self): """Return a dictionary containing all solver types which are available in caffe. The key is the name of the solver type (without the suffix 'Solver'. e.g. 'SGD' instead of 'SGDSolver') and the value is an object of the class SolverType. """ return self._availableSolverTypes def availableNetParameters(self): """Return a list containing all available parameters of a net. Note that, this differs a little bit from the way you get the parameters of a solver or layer, as we don't got different net types. Also pay attention to the fact, that the returned list will of course contain the parameter "layer". Because this parameter is only a general description of all layers at once, you will probably want to use availableLayerTypes() instead. """ return self._availableParameterGroupDescriptors['NetParameter'].parameter() class TopLevelEntityType: """This class represents entities of special importance for our project. According to the caffe.proto file, those entities are normal ParameterGroupDescriptor (/Message) docks. But the proto file doesn't add a hierarchy for the top level. This hierarchy will be added by inheriting from this class. The already complete list of top level entities is probably: LayerType, SolverType """ def __init__(self, name, parameters): self._name = name self._parameters = parameters def name(self): """The name extended by the correct suffix (e.g. 'Layer') equals the name of its C++ class.""" return self._name def parameters(self): """Return a dictionary containing all available parameters for this TopLevelEntityType. Each parameter will be an instance of a subclass of Parameter. """ return self._parameters class LayerType(TopLevelEntityType): """An object of this class represents one type of layer that is available in caffe. The name of such an object is for instance "Softmax" or "InnerProduct". The parameters of a LayerType include general parameters which are available for all LayerTypes as well as specific ones which are only available for this particular LayerType. """ CATEGORY_NONE = 0 CATEGORY_LOSS = 1 CATEGORY_DATA = 2 # == input CATEGORY_ACTIVATION = 3 # TODO (try to) implement further categories def __init__(self, name, parameters, layerParamKeys=[], seemsNotRelevantParams=set()): TopLevelEntityType.__init__(self, name, parameters) self._category = self.getCategoryByName(self._name) self._layerParamKeys = layerParamKeys self._irrelevant = seemsNotRelevantParams def category(self): """Return the category of this LayerType. The category is described by one of the class constants CATEGORY_. """ return self._category def layerParamKeys(self): """The paremter key for the specific parmeters of this Layertype E.g. ["inner_product_param"] for the InnerProduct-Layer """ return self._layerParamKeys def probablyIrrelevantParameter(self): """ Return a set of parameter which seems not to belong to this layer """ return self._irrelevant def isLossLayer(self): return self._category == self.CATEGORY_LOSS def isDataLayer(self): return self._category == self.CATEGORY_DATA def isActivationLayer(self): return self._category == self.CATEGORY_ACTIVATION @staticmethod def getCategoryByName(layerTypeName): """Given its name, determine which category a LayerType does belong to.""" # !!! hard coded list of activation layers could cause problems when more layers are added in caffe if layerTypeName in ["ReLU", "PReLU", "ELU", "Sigmoid", "TanH", "AbsVal", "Power", "Exp", "Log", "BNLL", "Threshold", "Bias", "Scale" ]: return LayerType.CATEGORY_ACTIVATION elif "Loss" in layerTypeName: return LayerType.CATEGORY_LOSS elif "Data" in layerTypeName: return LayerType.CATEGORY_DATA else: return LayerType.CATEGORY_NONE def __copy__(self): """See __deepcopy__.""" result = CaffeMetaInformation().availableLayerTypes()[self._name] return result def __deepcopy__(self, memo): """Override the deepcopy behavior used in copy.deepcopy() and pickle(). The protobuf object stored in self._field can't be pickeled, which will also result in an exception each time we try to call copy.deepcopy() on any list/dict/etc. that contains an object of this class. We need to do this e.g. in network_manager._refreshStateFromHistory(). This method solves the problem by just recreating an object with the same information as the original one without copying anything else. That's actually everything we need. """ result = CaffeMetaInformation().availableLayerTypes()[self._name] memo[id(self)] = result return result class SolverType(TopLevelEntityType): """Currently, the following SolverTypes do exist: 'SGD', 'Nesterov', 'AdaGrad', 'RMSProp', 'AdaDelta', 'Adam'""" class ParameterGroupDescriptor: """A ParameterGroupDescriptor is a general description of a ParameterGroup without being used as a concrete parameter. The idea is equivalent to a Message in the Google Protobuf definition. """ def __init__(self,protoclass, descriptor=None): self._protoclass = protoclass if descriptor is None and protoclass is not None: self._desc = protoclass.DESCRIPTOR else: self._desc = descriptor def protoclass(self): """ Return the protoclass associated with this descriptor """ return self._protoclass def name(self): return self._desc.name def parameter(self): """Return a dictionary containing all parameters of this group. The parameter name is used as the key while each value is an instance of the class Parameter. This instance can be used too gain further information. """ res = dict() for x in self._desc.fields: if x.type == FieldDescriptor.TYPE_MESSAGE: parameter = ParameterGroup(x) elif x.type == FieldDescriptor.TYPE_ENUM: parameter = ParameterEnum(x) else: parameter = ParameterPrimitive(x) res[x.name] = parameter return res def __eq__(self, other): return self._desc.full_name == other._desc.full_name class Parameter: """This class represents a single parameter which can be specified in the .prototxt file. There are various use cases for a parameter of this type. It might be usable only for the net definition, the solver definition, or both. A parameter also does not necessarily need to be a top-level parameter, meaning it might only be used as the value of another parameter, too. A Parameter is also a single property of a ParameterGroupDescriptor and therefore sometimes referred to as a 'property', too. An object of this (abstract) class will always be an instance of one of the following subclasses: ParameterPrimitive, ParameterEnum or ParameterGroup. TODO this base class should not be allowed to instantiate and should therefore be an abstract class, but that doesn't make sense in python, does it? """ string_dict = { FieldDescriptor.TYPE_BOOL: "bool", FieldDescriptor.TYPE_BYTES: "bytes", FieldDescriptor.TYPE_DOUBLE: "double", FieldDescriptor.TYPE_ENUM: "enum", FieldDescriptor.TYPE_FIXED32: "fixed32", FieldDescriptor.TYPE_FIXED64: "fixed64", FieldDescriptor.TYPE_FLOAT: "float", # FieldDescriptor.TYPE_GROUP: "group", FieldDescriptor.TYPE_INT32: "int32", FieldDescriptor.TYPE_INT64: "int64", FieldDescriptor.TYPE_MESSAGE: "message", FieldDescriptor.TYPE_SFIXED32: "sfixed32", FieldDescriptor.TYPE_SFIXED64: "sfixed64", FieldDescriptor.TYPE_SINT32: "sint32", FieldDescriptor.TYPE_SINT64: "sint64", FieldDescriptor.TYPE_STRING: "string", FieldDescriptor.TYPE_UINT32: "uint32", FieldDescriptor.TYPE_UINT64: "uint64", } def __init__(self,field): self._field = field self._description = proto_description.CaffeProtoParser().description(field.containing_type.name, field.name) self._initIsDeprecated() def _initIsDeprecated(self): """Call this whenever a new description has been set.""" self._isDeprecated = re.search("deprecated", self._description, re.IGNORECASE) is not None def isDeprecated(self): return self._isDeprecated def description(self): """Get a string describing this parameter in a human-readable way.""" return self._description def fieldName(self): """Return the name of this property (with underscores, not camelcase). This is not the same as the name of the parameter's type. Therefore this method is not called 'name()'. Pay additional attention to possible confusion with the name of the ParameterGroupDescriptor, if this element is a ParameterGroup. """ return self._field.name def isOptional(self): """Return true iff the parameter is optional, this includes if it's repeated. (Repeated mean 0-n) """ return self._field.label == FieldDescriptor.LABEL_OPTIONAL or self.isRepeated() def isRequired(self): """ Return the opposite of isOptional""" return not self.isOptional() # return self._field.label == FieldDescriptor.LABEL_REQUIRED def isRepeated(self): """ Return true iff the parameter is repeated. Repeated values may contain 0 entries""" return self._field.label == FieldDescriptor.LABEL_REPEATED def isParameterGroup(self): """Return true iff this parameter is a combination of other parameter.""" return self._field.type == FieldDescriptor.TYPE_MESSAGE def defaultValue(self): # We make a copy, so no one can override the default value return copy.deepcopy(self._field.default_value) def __eq__(self, other): return self._field.full_name == other._field.full_name and self.fieldName() == other.fieldName() def type(self): """The type is descript by one of the constants FieldDescriptor.TYPE_.""" return self._field.type def typeString(self): """Return a human-readable string describing the type (e.g. "bool", "int32", ..). """ return Parameter.string_dict[self._field.type] def isBool(self): return self.type() == FieldDescriptor.TYPE_BOOL def isString(self): return self.type() == FieldDescriptor.TYPE_STRING def isInt(self): """Combine all possible tests of integer (does not differ: uint vs int vs sint).""" return self.type() == FieldDescriptor.TYPE_INT32 or self.type() == FieldDescriptor.TYPE_INT64 or \ self.type() == FieldDescriptor.TYPE_SINT32 or self.type() == FieldDescriptor.TYPE_SINT64 or \ self.type() == FieldDescriptor.TYPE_UINT32 or self.type() == FieldDescriptor.TYPE_UINT64 or \ self.type() == FieldDescriptor.TYPE_FIXED32 or self.type() == FieldDescriptor.TYPE_FIXED64 or \ self.type() == FieldDescriptor.TYPE_SFIXED32 or self.type() == FieldDescriptor.TYPE_SFIXED64 def isSInt(self): return self.type() == FieldDescriptor.TYPE_SINT32 or self.type() == FieldDescriptor.TYPE_SINT64 or \ self.type() == FieldDescriptor.TYPE_SFIXED32 or self.type() == FieldDescriptor.TYPE_SFIXED64 def isUInt(self): return self.type() == FieldDescriptor.TYPE_UINT32 or self.type() == FieldDescriptor.TYPE_UINT64 or \ self.type() == FieldDescriptor.TYPE_FIXED32 or self.type() == FieldDescriptor.TYPE_FIXED64 def isBytes(self): return self.type() == FieldDescriptor.TYPE_BYTES def isFloat(self): return self.type() == FieldDescriptor.TYPE_FLOAT def isDouble(self): return self.type() == FieldDescriptor.TYPE_DOUBLE def isEnum(self): return self.type() == FieldDescriptor.TYPE_ENUM class ParameterPrimitive(Parameter): """An Object of this class represents a primitive data type like int, bool, string etc.""" class ParameterEnum(Parameter): """An Object of this class represents an enumeration. TODO throw exception in constructor if self._field.type != FieldDescriptor.TYPE_ENUM """ def availableValues(self): """Return all available values possible for this enum.""" return [x.name for x in self._field.enum_type.values] def defaultValue(self): return self.availableValues()[self._field.default_value] def resetCaffeProtoModulesvar(): global _caffeprotomodulesvar _caffeprotomodulesvar = None def _caffeProtobufModules(): """ Returns all available Classes of caffe_pb2 in a dictionary """ import backend.caffe.path_loader as caffeLoader proto = caffeLoader.importProto() global _caffeprotomodulesvar if _caffeprotomodulesvar is None: current_module = sys.modules[proto.__name__] _caffeprotomodulesvar = dict(inspect.getmembers(current_module, inspect.isclass)) return _caffeprotomodulesvar class ParameterGroup(ParameterGroupDescriptor, Parameter): """An Object of this class is a ParameterGroupDescriptor which is used as a Parameter of another ParameterGroupDescriptor. TODO throw exception in constructor if self._field.type != FieldDescriptor.TYPE_MESSAGE: """ def __init__(self, field): Parameter.__init__(self, field) ParameterGroupDescriptor.__init__(self, None, descriptor=field.message_type) self._protoclass = _caffeProtobufModules()[self.parameterName()] # refresh description as this type of parameter got additional default values self._description = proto_description.CaffeProtoParser().description(field.containing_type.name, field.name, field.message_type.name) self._initIsDeprecated() def parameterName(self): """Get the ParameterGroupDescriptor name. Just an alias for the name() method to prevent confusion caused by the polymorphism. """ return self.name() backend/caffe/saver.py0000664ڊ00111170000000703213243274037015135 0ustar klemmso0cvprfrom google.protobuf import text_format import backend.caffe.proto_info as info def saveSolver(solverdict): solver= _import_solver(solverdict) solverproto = text_format.MessageToString(solver) return solverproto def _import_solver(solverdict): import backend.caffe.path_loader as caffeLoader proto = caffeLoader.importProto() solver = proto.SolverParameter() for entry in solverdict: # special case: inline-net definition in solver # (required to handle layers and layerOrder the same way it is handled for a standalone net definition) if entry == "net_param": net = _import_dictionary(solverdict["net_param"]) solver.net_param.MergeFrom(net) else: _insert(entry, solverdict[entry], solver) return solver def saveNet(netdict): """ Save the dictionary into a prototxt string. The dictionary has the following form { "name": "Somenetwork", "input_dim": [1,2,1,1], "state": { "phase": "TRAIN" }, ... "layers": { "somerandomid1": { "type": LayerType Instance of Pooling-Layer, "parameters": { "pooling_param": [ "kernel_size": 23, "engine": "DEFAULT" ] .... "input_param": [ {"shape": {"dim": [...], .... }, {"shape": {"dim": [...], .... }, ] } }, "somerandomid2": {"type": ..., "parameters": ....} }, "layerOrder": ["somerandomid1", "somerandomid2", ....] } """ net = _import_dictionary(netdict) netproto = text_format.MessageToString(net) return netproto def _import_dictionary(netdict): """fill the ProtoTxt-Net with data from the dictionary""" import backend.caffe.path_loader as caffeLoader proto = caffeLoader.importProto() net = proto.NetParameter() for entry in netdict: if entry == "layerOrder": continue if entry == "layers": _extract_layer(netdict["layers"],netdict["layerOrder"],net) continue _insert(entry,netdict[entry],net) return net def _extract_layer(layers, layerorder,net): for order in layerorder: index = len(net.layer) net.layer.add() currlayer = layers[order] for param in currlayer["parameters"]: _insert(param,currlayer["parameters"][param],net.layer[index]) return def _insert(key, value, insert): descr = info.ParameterGroupDescriptor(insert) param = descr.parameter()[key] if param.isParameterGroup(): if param.isRepeated(): for l in range(0, len(value)): index = len(getattr(insert,key)) getattr(insert,key).add() for v in value[l]: _insert(v,value[l][v],getattr(insert,key)[index]) else: for v in value: _insert(v,value[v],getattr(insert,key)) return if param.isRepeated(): for v in value: w = v if param.isEnum(): w = param.availableValues().index(v) getattr(insert,key).append(w) return if param.isEnum(): value = param.availableValues().index(value) setattr(insert, key, value) backend/input_db/__init__.py0000664ڊ00111170000000000013243274037016300 0ustar klemmso0cvprbackend/input_db/hdf5_input.py0000664ڊ00111170000000451713243274037016627 0ustar klemmso0cvprimport os.path import h5py as h5 import numpy as np from backend.barista.utils.logger import Log class Hdf5Input: def __init__(self): self._db = None self._path = None self.logid = Log.getCallerId('HDF5 Input') def __del__(self): self.close() def setPath(self, path): '''set the path of the database.''' self._path = path def open(self): '''open the database from the set path.''' if self._db: self.close() if self._path: if os.path.exists(self._path): try: self._db = h5.File(self._path, 'r') except: Log.error("File not valid HDF5: " + self._path, self.logid) self._db = None else: Log.error("File does not exist: " + self._path, self.logid) def close(self): if self._db: self._db.close() self._db = None def getDataCount(self): if self._db: d = {} for i in range(0, len(self._db.keys())): d[self._db.keys()[i]] = len(self._db.values()[i]) return d def getDimensions(self): if self._db: d = {} for i in range(0, len(self._db.keys())): ar = np.array(self._db.values()[i][0]) d[self._db.keys()[i]] = ar.shape return d def verifyConsistency(self): if self._db: for i in range(0, len(self._db.keys())): ar = np.array(self._db.values()[i][0]) for j in range(0, len(self._db.values()[i])): ar2 = np.array(self._db.values()[i][j]) if not self._dimCompare(ar.shape, ar2.shape): return False return True def isOpen(self): if self._db: return True return False def _dimCompare(self, d1, d2): if not d1 and not d2: # both empty return True if d2 and not d1: # d1 empty return False if d1 and not d2: # d2 empty return False if len(d1) != len(d2): # different dim count return False for i in range(0, len(d1)): # dims not matching if d1[i] != d2[i]: return False return True backend/input_db/hdf5_txt_input.py0000664ڊ00111170000000601113243274037017515 0ustar klemmso0cvprfrom os import path from backend.barista.utils.logger import Log from backend.input_db.hdf5_input import * class Hdf5TxtInput: def __init__(self): self._db = [] self._path = None self.logid = Log.getCallerId("HDF5TXT Input") self._projectPath = None def __del__(self): self.close() def setPath(self, path): self._path = os.path.normpath(path) def open(self): if self._db: self.close() if self._path is not None: if os.path.exists(self._path): lines = [line.rstrip('\n') for line in open(self._path)] for line in lines: if line is not "": if line[:1] == '.': line = self._makepath(line) i = len(self._db) self._db.append(Hdf5Input()) self._db[i].setPath(line) self._db[i].open() if not self._db[i].isOpen(): Log.error("File contains invalid HDF5: " + self._path, self.logid) self._db = None return else: Log.error("File does not exist: " + self._path, self.logid) def close(self): if self._db: for db in self._db: db.close() self._db = [] def getDataCount(self): if self._db: sum = {} for db in self._db: sum = self._sumDataCount(sum, db.getDataCount()) return sum def getDimensions(self): if self._db: d = {} for db in self._db: d = self._combineDim(d, db.getDimensions()) return d def verifyConsistency(self): if self._db: d = self._db[0].getDimensions() for db in self._db: if not db.verifyConsistency(): return False if not self._compareDim(d, db.getDimensions()): return False return True def isOpen(self): if self._db: return True return False def setProjectPath(self, path): self._projectPath = path def _sumDataCount(self, d1, d2): return {k: d1.get(k, 0) + d2.get(k, 0) for k in set(d1) | set(d2)} def _combineDim(self, d1, d2): return {k: max(d1.get(k, 0), d2.get(k, 0)) for k in set(d1) | set(d2)} def _compareDim(self, d1, d2): l = [d1.get(k) == d2.get(k) for k in set(d1) | set(d2)] if False in l: return False return True """ Converts a path that is considered to be relative to the project path into an absolute path. @author: ..., j_stru18 @param relpath a path that should be relative to the project path """ def _makepath(self, relpath): if self._projectPath: p = path.abspath(path.join(self._projectPath, relpath)) return p return relpath backend/input_db/leveldb_input.py0000664ڊ00111170000000620213243274037017407 0ustar klemmso0cvprimport leveldb # pip install leveldb import os.path from backend.barista.utils.logger import Log class LeveldbInput: def __init__(self): self._db = None self._path = None self.logid = Log.getCallerId("LEVELDB Input") def __del__(self): self.close() def setPath(self, path): '''set the path of the database.''' self._path = path def open(self): '''open the database from the set path.''' if self._db: self.close() if self._path: if os.path.exists(self._path + "CURRENT") or os.path.exists(self._path + "/CURRENT"): self._db = leveldb.LevelDB(self._path) else: Log.error("Dir is not valid LEVELDB: " + self._path, self.logid) def close(self): '''close and clean up the database.''' self._db = None def getDataCount(self): '''get the number of entries in the database''' if self._db: iter = self._getIter() if iter: number = 0 for elem in iter: number = number + 1 return {"data": number} def getDimensions(self): '''get the dimensions of the first entry''' datum = self._getFirstDatum() if datum: dim = (datum.channels, datum.height, datum.width) return {"label": (), "data": dim} # TODO dim of label def verifyConsistency(self): '''check if all entries have the same channel number and size. This may take a lot of time since every entry has to be checked.''' import backend.caffe.path_loader as caffeLoader caffe = caffeLoader.importCaffe() if self.getDataCount() < 2: return True first = self._getFirstDatum() if first: channels = first.channels width = first.width height = first.height iter = self._getIter() if iter: for key, value in iter: raw_datum = value datum = caffe.proto.caffe_pb2.Datum() datum.ParseFromString(raw_datum) if channels is not datum.channels \ or width is not datum.width \ or height is not datum.height: return False return True def isOpen(self): '''check if db is open.''' if self._db: return True return False def _getIter(self): if self._db: return self._db.RangeIter() def _getFirstDatum(self): import backend.caffe.path_loader as caffeLoader caffe = caffeLoader.importCaffe() iter = self._getIter() if iter: for key, value in iter: raw_datum = value datum = caffe.proto.caffe_pb2.Datum() try: datum.ParseFromString(raw_datum) except: Log.error("LEVELDB does not contain valid data: " + self._path, self.logid) return None return datum backend/input_db/lmdb_input.py0000664ڊ00111170000000650313243274037016714 0ustar klemmso0cvprimport os.path import lmdb # pip install lmdb from backend.barista.utils.logger import Log class LmdbInput: def __init__(self): self._env = None self._path = None self._cursor = None self._txn = None self.logid = Log.getCallerId("LMDB Input") def __del__(self): self.close() def setPath(self, path): '''set the path of the database.''' self._path = path def open(self): '''open the database from the set path.''' if self._env: self.close() if self._path: if os.path.exists(self._path + "data.mdb") or os.path.exists(self._path + "/data.mdb"): self._env = lmdb.open(self._path, max_dbs=2) else: Log.error("Dir is not valid LMDB: " + self._path, self.logid) def close(self): '''close and clean up the database.''' if self._env: self._env.close() self._env = None self._txn = None self._cursor = None def getDataCount(self): '''get the number of entries in the database''' if self._env: return {"data": self._env.stat()['entries']} def getDimensions(self): '''get the dimensions of the first entry''' if self._env: datum = self._getFirstDatum() if datum: dim = (datum.channels, datum.height, datum.width) return {"label": (), "data": dim} # TODO dim of label def verifyConsistency(self): '''check if all entries have the same channel number and size. This may take a lot of time since every entrie has to be checked.''' if self._env: if self.getDataCount() < 2: return True first = self._getFirstDatum() if first: channels = first.channels width = first.width height = first.height while self._cursor.next(): if channels is not self._getCurrentDatum().channels \ or width is not self._getCurrentDatum().width \ or height is not self._getCurrentDatum().height: return False return True def isOpen(self): '''check if db is open:''' if self._env: return True return False def _openTransaction(self): if self._env: if self._txn is None: self._txn = self._env.begin() def _openCursor(self): if self._env: if self._txn: if self._cursor is None: self._cursor = self._txn.cursor() def _getFirstDatum(self): self._openTransaction() self._openCursor() if self._cursor: self._cursor.first() return self._getCurrentDatum() def _getCurrentDatum(self): import backend.caffe.path_loader as caffeLoader caffe = caffeLoader.importCaffe() if self._cursor: raw_datum = self._cursor.value() datum = caffe.proto.caffe_pb2.Datum() try: datum.ParseFromString(raw_datum) except: Log.error("LMDB does not contain valid data: " + self._path, self.logid) return None return datum backend/networking/__init__.py0000664ڊ00111170000000000013243274037016663 0ustar klemmso0cvprbackend/networking/barista_server.py0000664ڊ00111170000001765213243274037020164 0ustar klemmso0cvprimport json import os import socket import sys import logging from PyQt5.QtNetwork import QTcpServer, QHostAddress from backend.caffe import path_loader from backend.caffe.check_hardware import checkHardware from backend.barista.session.session_utils import State from backend.networking.protocol import Protocol from backend.networking.server_session_manager import ServerSessionManager from backend.networking.server_transaction import ServerTransaction class BaristaServer(): def __init__(self, application, ip, port, sessionPath): self.application = application self.server = None self.port = int(port) self.ip = QHostAddress.Any if ip is not None: self.ip = QHostAddress(ip) self.sessionPath = sessionPath self.configpath = os.path.join(self.sessionPath, "barista.conf") self.caffePath = "" self.hardware = [] self.trainOnHW = 0 self.transactionList = [] self._loadConfig() self.checkHardware() state = self._checkConfig() self.start() self.sessionManager = ServerSessionManager(self, self.sessionPath) if state: self.sessionManager.setCaffePath(self.caffePath) def start(self): self.server = QTcpServer() self.server.listen(self.ip, self.port) self.server.newConnection.connect(self._newConnection) if self.server.isListening(): hostIP = str(self.server.serverAddress().toString()) if hostIP == '0.0.0.0': hostIP = '' sys.stdout.write("Hostname: " + socket.gethostname() + "\tIP: " + hostIP + ":" + str(self.server.serverPort()) + "\n") else: sys.stderr.write("Something went wrong. Server is not listening.\n") logging.error("Something went wrong. Server is not listening.") exit(1) def getBaristaStatus(self, pid=""): hardware = [k["name"] for k in self.hardware] statusdict = {"caffepath": self.caffePath, "trainOnHW": self.trainOnHW, "hardware": hardware} statusdict["connections"] = len(self.transactionList) statusdict["config"] = self._checkConfig(False) statusdict["training"] = len(self.sessionManager.findSessionIDsWithState(State.RUNNING)) is 0 statusdict["sessioncount"] = len(self.sessionManager.sessions) statusdict["sessionpath"] = self.sessionPath if pid is not "": pidses = {} sessions = set(self.sessionManager.findSessionIDsByProjectId(pid)) pidses["pid"] = pid pidses["count"] = len(sessions) pidses["running"] = len(set(self.sessionManager.findSessionIDsWithState(State.RUNNING)) & sessions) pidses["running"] += len(set(self.sessionManager.findSessionIDsWithState(State.PAUSED)) & sessions) pidses["waiting"] = len(set(self.sessionManager.findSessionIDsWithState(State.WAITING)) & sessions) pidses["finished"] = len(set(self.sessionManager.findSessionIDsWithState(State.FINISHED)) & sessions) statusdict["projectsessions"] = pidses return statusdict def setCaffePath(self, path): oldpath = self.caffePath self.caffePath = path if self._checkConfig(): self.trainOnHW = 0 # reset Hardware self._saveConfig() self.sessionManager.setCaffePath(path) sys.stdout.write("Caffepath changed to: " + str(path) + "\n") logging.info("Caffepath changed to: %s", str(path)) return True else: self.caffePath = oldpath return False def _newConnection(self): transaction = ServerTransaction(self) self.transactionList.append(transaction) transaction.socketClosed.connect(lambda item=transaction: self._deleteConnection(item)) transaction.acceptClient(self.server.nextPendingConnection()) def _deleteConnection(self, transaction): index = self.transactionList.index(transaction) del self.transactionList[index] def _saveConfig(self): tosave = dict() tosave["caffepath"] = self.caffePath tosave["trainOnHW"] = self.trainOnHW with open(self.configpath, "w") as file: json.dump(tosave, file, sort_keys=True, indent=4) def _loadConfig(self): if not os.path.exists(self.configpath): # TODO Default pass else: with open(self.configpath, "r") as file: res = json.load(file) if "caffepath" in res: self.caffePath = res["caffepath"] if "trainOnHW" in res: self.trainOnHW = res["trainOnHW"] else: # TODO default pass def _checkConfig(self, verbose=True): state = True if self.caffePath is "": if verbose: sys.stderr.write("Warning: There is no Caffepath set. Use the Hostmanager to set the Path.\n") logging.warning("There is no Caffepath set. Use the Hostmanager to set the Path.") else: if path_loader.checkIfFileIsCaffeRoot(self.caffePath): pass else: if verbose: sys.stderr.write("Error: " + self.caffePath + " is not a valid Caffepath\n") logging.error("%s is not a valid Caffepath", self.caffePath) state = False if self.trainOnHW >= len(self.hardware): self.trainOnHW = 0 if verbose: sys.stderr.write("Warning: Currently selected Hardware could not be matched against detected Hardware." " Set to CPU mode!\n") logging.warning("Currently selected Hardware could not be matched against detected Hardware." " Set to CPU mode!") return state def checkHardware(self, verbose=True, transaction=None): if self.caffePath is "": if verbose: sys.stderr.write("Warning: Can't check hardware without caffepath\n") logging.warning("Can't check hardware without caffepath") if transaction: msg = {"key": Protocol.SCANHARDWARE, "status": False, "error": "Can't check hardware without caffepath"} transaction.send(msg) return if not path_loader.checkIfFileIsCaffeRoot(self.caffePath): if verbose: sys.stderr.write("Warning: Can't check hardware without valid caffepath\n") logging.warning("Can't check hardware without valid caffepath") if transaction: msg = {"key": Protocol.SCANHARDWARE, "status": False, "error": "Can't check hardware without valid caffepath"} transaction.send(msg) return try: self.hardware = checkHardware(self.caffePath, not verbose, transaction) if verbose: sys.stdout.write("Finished scanning Hardware. " + str(len(self.hardware)) + " devices found.\n") logging.info("Finished scanning Hardware. %s devices found.", str(len(self.hardware))) if transaction: msg = {"key": Protocol.SCANHARDWARE, "status": True, "finished": True, "hardware": self.hardware, "current": self.trainOnHW} transaction.send(msg) except: if verbose: sys.stderr.write("Error: Failed to check hardware!\n") logging.error("Failed to check hardware!") if transaction: msg = {"key": Protocol.SCANHARDWARE, "status": False, "error": "Failed to check hardware!"} transaction.send(msg) def setHardware(self, hid): if hid >= len(self.hardware): return False self.trainOnHW = hid self._saveConfig() return True backend/networking/client_transaction.py0000664ڊ00111170000000121313243274037021016 0ustar klemmso0cvprfrom PyQt5.QtCore import pyqtSignal from backend.networking.transaction import Transaction from backend.networking.protocol import Protocol class ClientTransaction(Transaction): showDirSig = pyqtSignal(list, bool) def __init__(self): Transaction.__init__(self) # self.options = {Protocol.ECHO: self.msgEcho} def processMessage(self): """process the received message and call further processing""" pass # msg = self.messageOutput[0] # if msg["key"] in self.options: # self.options[msg["key"]]() # def msgEcho(self): # msg = self.asyncRead() # print(msg) backend/networking/net_util.py0000664ڊ00111170000000137313243274037016765 0ustar klemmso0cvprfrom backend.barista.utils.logger import * from backend.networking.client_transaction import ClientTransaction def sendMsgToHost(host, port, msg): ct = ClientTransaction() ct.connect(host, port) ct.waitForConnection() if ct.isConnected(): ct.send(msg) ret = ct.asyncRead() ct.close() if ret: return ret else: Log.error("No answer from " + host + ":" + str(port), Log.getCallerId("Network Connection")) else: Log.error("Failed to connect to " + host + ":" + str(port), Log.getCallerId("Network Connection")) def buildTransaction(host, port): ct = ClientTransaction() ct.connect(host, port) ct.waitForConnection() if ct.isConnected(): return ct backend/networking/protocol.py0000664ڊ00111170000000227013243274037017000 0ustar klemmso0cvpr class Protocol: """ Pseudo Enumeration for the communication protocol """ # GENERAL ECHO = 0 # GET GETDIR = 10 GETSTATUS = 11 GETDBSTATUS = 12 # INPUTDB MAKEPATHRELATIVE = 20 SETCAFFEPATH = 21 MAKEHDF5TXT = 22 ADDHDF5 = 23 GETFILECONTENT = 24 WRITEFILECONTENT = 25 GETHASH = 26 # HARDWARE GETHARDWARE = 30 SCANHARDWARE = 31 SETHARDWARE = 32 # SESSION SESSION = 40 CREATESESSION = 41 CONNECTTOSESSION = 42 GETSESSIONS = 43 DELETESESSION = 44 class SessionProtocol: """ Pseudo Enumeration for the communication protocol for subkeys in remote sessions """ GETITERATION = 0 GETMAXITERATION = 1 GETSNAPSHOTS = 2 GETSTATE = 3 SAVE = 10 CHECKFILES = 11 CHECKTRAINING = 12 GETSTATEDICT = 20 SETSTATEDICT = 21 PRINTLOG = 30 UPDATEPARSER = 31 UPDATESTATE = 32 UPDATEKEYS = 33 ADDSNAPSHOT = 34 PARSEHANDLE = 35 FETCHPARSERDATA = 36 START = 40 PAUSE = 41 PROCEED = 42 TAKESNAPSHOT = 43 LOADINTERNALNET = 50 LOADDEPLOYEDNET = 51 LOADNETPARAMETER = 52 LOADCAFFEMODEL = 53 RESET = 98 DELETE = 99 backend/networking/server_session_manager.py0000664ڊ00111170000002566613243274037021720 0ustar klemmso0cvprimport os import re import sys import json from datetime import datetime import logging from backend.barista.session.server_session import ServerSession from backend.barista.session.session_pool import SessionPool from backend.barista.session.session_utils import Paths, State from backend.caffe.proto_info import resetCaffeProtoModulesvar from PyQt5.QtCore import pyqtSignal from PyQt5.Qt import QObject class ServerSessionManager(QObject): poolEmptyJob = pyqtSignal(str) jsonKeys = ("Iteration", "MaxIter", "NetworkState", "ProjectID", "SID", "SessionState", "UID") def __init__(self, parent, sessionPath): super(ServerSessionManager, self).__init__() resetCaffeProtoModulesvar() self.parent = parent self.sessionPath = sessionPath self.caffePath = "" # TODO use caffepath for CaffeMetaInfo self.sessions = [] self.loadSessions() def connectToSession(self, transaction): """connect to an existing session with UID""" msg = transaction.asyncRead() msg["error"] = [] msg["status"] = False if "uid" in msg: session = self.findSessionBySessionUid(msg["uid"]) if session: if session.connect(transaction): msg["status"] = True logging.info("Session connected.") else: msg["error"].append("ConnectToSession: Failed to connect. Session is already connected!") logging.error("Failed to connect. Session is already connected") else: msg["error"].append("ConnectToSession: No session with UID '" + msg["uid"] + "' found.") logging.error("No session with UID '%s' found.", msg["uid"]) else: msg["error"].append("ConnectToSession: No UID provided.") logging.error("No UID provided.") transaction.send(msg) def createNewSession(self, transaction): """create a new session with SID and PID""" msg = transaction.asyncRead() if "pid" in msg and "sid" in msg: dirp = self._createDirName(msg["sid"]) uid = self._createSession(msg["pid"], dirp) self.findSessionBySessionUid(uid).setInitialSid(msg["sid"]) msg["status"] = True msg["uid"] = uid logging.info("Session created with UID '%s'", uid) else: msg["status"] = False msg["error"] = ["CreateNewSession: No PID or SID provided."] logging.error("Could not create session. No PID or SID provided.") transaction.send(msg) def deleteSession(self, transaction): """delete an existing session with UID""" msg = transaction.asyncRead() msg["error"] = [] msg["status"] = False if "uid" in msg: session = self.findSessionBySessionUid(msg["uid"]) if session: for sess in self.sessions: if sess["uid"] == msg["uid"]: self.sessions.remove(sess) session.delete() del session msg["status"] = True logging.info("Session deleted.") else: msg["error"].append("DeleteSession: No session with UID '" + msg["uid"] + "' found.") logging.error("No session with UID '%s' found.", msg["uid"]) else: msg["error"].append("DeleteSession: No UID provided.") logging.error("No UID provided.") transaction.send(msg) def getSessions(self, transaction): """Get all session for a project with PID""" msg = transaction.asyncRead() if "pid" in msg: ret = [] uids = self.findSessionIDsByProjectId(msg["pid"]) for uid in uids: session = self.findSessionBySessionUid(uid) name = session.directory name = os.path.basename(name) if session.isConnected: name = "(connected) " + name state = session.state if state is State.FINISHED: name += " [FINISHED]" if state is State.WAITING: name += " [READY]" if state is State.RUNNING: name += " [RUNNING]" if state is State.PAUSED: name += " [PAUSED]" if state is State.FAILED: name += " [FAILED]" ret.append((uid, name)) ret.sort(key=lambda s: s[1], reverse=True) msg["sessions"] = ret msg["status"] = True logging.debug("Found %s Session/s", str(len(ret))) else: msg["status"] = False msg["error"] = ["GetSessions: No PID provided"] logging.error("Sessions not found. No PID provided.") transaction.send(msg) def setCaffePath(self, path): self.caffePath = path logging.debug("Set Caffepath for ServerSessionManager") def findSessionBySessionUid(self, uid): """find a single session with UID""" for sessiondict in self.sessions: if sessiondict["uid"] == uid: logging.debug("Found session for UID '%s'", uid) return sessiondict["session"] logging.debug("No sessions found for UID '%s'", uid) return None def findSessionIDsByProjectId(self, pid): """list all sessionids with PID""" uids = [] for sessiondict in self.sessions: if sessiondict["pid"] == pid: uids.append(sessiondict["uid"]) logging.info("Found %s session/s for PID '%s'", str(len(uids)), pid) return uids def findSessionIDsWithState(self, state): """find all session UIDs with state""" sessions = [] for sessiondict in self.sessions: if sessiondict["session"].state == state: sessions.append(sessiondict["uid"]) logging.debug("There are %i session/s with state %i", len(sessions), state) return sessions def _createSession(self, pid, dir): """Create a new session with PID in directory""" # TODO session = ServerSession(self, dir, pid=pid) uid = session.uid self.sessions.append({"pid": pid, "uid": uid, "session": session}) return uid def _loadSession(self, dir): """load a session from directory""" session = ServerSession(self, dir, parse_old=True) return {"pid": session.pid, "uid": session.uid, "session": session} def _createDirName(self, sid): """create a directory for session with SID""" directory = self.__getDateTimeString() directory += "_" + str(sid) sdir = os.path.join(self.sessionPath, directory) return sdir def __getDateTimeString(self): """ Return the current datetime in the format YYYYMMDD_hh24mmss """ now = datetime.now() date_string = '' date_string += str(now.year) date_string += str(now.month).zfill(2) date_string += str(now.day).zfill(2) date_string += '_' date_string += str(now.hour).zfill(2) date_string += str(now.minute).zfill(2) date_string += str(now.second).zfill(2) return date_string def loadSessions(self): """load all session from sessionpath and add them to the pool""" pool = SessionPool() count = 0 for entry in os.listdir(self.sessionPath): if entry == "barista.conf": continue sdir = os.path.join(self.sessionPath, entry) if self._isSession(sdir): sessiondict = self._loadSession(sdir) if sessiondict is not None: self.sessions.append(sessiondict) pool.addSession(sessiondict["session"]) count += 1 self.poolEmptyJob.connect(self.emptyJob) pool.activate(lambda: self.poolEmptyJob.emit("All jobs finished. SessionPool Queue empty.")) logging.info("Loaded %s session/s", str(count)) return count def emptyJob(self, msg): logging.info("All jobs finished. SessionPool Queue empty.") print msg def _isSession(self, dir): """Check if directory contains a valid session.""" session_json = os.path.join(dir,Paths.FILE_NAME_SESSION_JSON) if os.path.isdir(dir) is False: sys.stderr.write("Session directory " + dir + " is invalid! No directory!\n") logging.error("Session directory %s is invalid. No directory", dir) return False if not os.path.exists(session_json): sys.stderr.write("Session directory " + os.path.basename(os.path.normpath(dir)) + " is invalid!\n File 'sessionstate.json' does not exist!\n") logging.error("Session directory %s is invalid. 'sessionstate.json' does not exist!", dir) return False with open(session_json,"r") as file: try: dict = json.load(file) except ValueError: sys.stderr.write("Session file " + session_json + " is invalid!\n File 'sessionstate.json' could not be parsed!\n") logging.error("Session file %s is invalid. 'sessionstate.json' could not be parsed!", session_json) return False for key in self.jsonKeys: if key not in dict: sys.stderr.write("Session directory " + os.path.basename(os.path.normpath(dir))+" is invalid!\n Key '" +key +"' is missing in sessionstate!\n") logging.error("Session at %s is invalid. Key %s in 'sessionstate.json' is missing.",dir,key) return False if "LastSnapshot" in dict: if dict["LastSnapshot"]: if not os.path.exists(os.path.join(dir,dict["LastSnapshot"])): sys.stderr.write("Session directory " + os.path.basename(os.path.normpath(dir)) + " is invalid!\n Snapshot " "was set but not found in directory\n") logging.error("Session at %s is invalid. Snapshot was set but not found.", dir) return False logging.debug("Session directory %s is valid!", dir) return True def _parseSnapshotPrefixFromFile(self, filename): """ Return the snapshot prefix of the solver. """ with open(filename) as f: regex_prefix = re.compile( 'snapshot_prefix:[\s]+"(.+)"') for line in f: prefix_match = regex_prefix.search(line) if prefix_match: return prefix_match.group(1) def isTraining(self): """check if there is already a session running on this server""" return len(self.findSessionIDsWithState(State.RUNNING)) > 0 backend/networking/server_transaction.py0000664ڊ00111170000002471713243274037021064 0ustar klemmso0cvprimport os import sys import logging from PyQt5.QtCore import QFileInfo, QDirIterator, QDateTime, QRegExp, pyqtSignal from PyQt5.QtCore import QTimer from backend.networking.transaction import Transaction from gui.input_manager.database_object import DatabaseObject from backend.networking.protocol import Protocol import backend.barista.utils.db_util as db_util class ServerTransaction(Transaction): def __init__(self, parent): self.options = { Protocol.ECHO: self.msgEcho, Protocol.GETDIR: self.getDir, Protocol.GETSTATUS: self._getBaristaStatus, Protocol.GETDBSTATUS: self._getDBStatus, Protocol.MAKEPATHRELATIVE: self._makeRelative, Protocol.SETCAFFEPATH: self._setCaffePath, Protocol.MAKEHDF5TXT: self._makeHDF5Txt, Protocol.ADDHDF5: self._addHdf5, Protocol.GETFILECONTENT: self._getFileContent, Protocol.WRITEFILECONTENT: self._writeFileContent, Protocol.GETHASH: self._getHash, Protocol.GETHARDWARE: self._getHardware, Protocol.SCANHARDWARE: self._scanHardware, Protocol.SETHARDWARE: self._setHardware, Protocol.CREATESESSION: self._createSession, Protocol.CONNECTTOSESSION: self._connectToSession, Protocol.GETSESSIONS: self._getSessions, Protocol.DELETESESSION: self._deleteSession, Protocol.SESSION: self._doNothing } Transaction.__init__(self) self.parent = parent def processMessage(self): """process the received message and call further processing""" # key = self.messageOutput[0]["key"] key = self.getAttrOfFirst(["key"])[0] # Threadsafe if key in self.options: self.options[key]() else: sys.stderr.write("Wrong Key: " + str(key) + "\n") def msgEcho(self): """return the message back to client. this is for testing.""" msg = self.asyncRead() self.send(msg) def getDir(self): """list all files and subdirectories for a given dir with regex.""" msg = self.asyncRead() if msg: path = msg["path"] dirselect = False if "dirSelect" in msg: dirselect = msg["dirSelect"] if path == "": path = "/home" path = os.path.normpath(path) filters = ["*"] if "filter" in msg: filters = msg["filter"] dirIter = QDirIterator(path) data = list() while dirIter.hasNext(): dirIter.next() if dirIter.fileName() in [".", ".."]: continue if not dirIter.fileInfo().isDir(): if dirselect: continue match = False for fi in filters: reg = QRegExp(fi) reg.setPatternSyntax(QRegExp.Wildcard) if reg.exactMatch(dirIter.fileName()): match = True break if not match: continue elem = dict() elem["name"] = dirIter.fileName() elem["path"] = dirIter.filePath() inf = dirIter.fileInfo() # type: QFileInfo elem["isDir"] = inf.isDir() last = inf.lastModified() # type: QDateTime elem["lastChange"] = last.toString("yyyy.MM.dd - HH:MM") if inf.isDir(): elem["fileSize"] = "" else: elem["fileSize"] = self._sizeHumanReadableStr(inf.size()) data.append(elem) data = sorted(data, key=lambda di: (not di["isDir"], di["name"].lower())) msg["data"] = data logging.debug("There is/are %i element/s in %s", len(data), path) self.send(msg) def _sizeHumanReadableStr(self, size): """convert integer to human readable size""" sizes = ["B", "kB", "MB", "GB", "TB"] curSize = 0 unit = sizes[curSize] while (size > 1000 and curSize < len(sizes) - 1): curSize = curSize + 1 unit = sizes[curSize] size = round(size / 1000.0, 1) return str(size) + " " + unit def _getBaristaStatus(self): """start the Timer for Heartbeat""" msg = self.asyncRead() if "projectid" in msg.keys(): self.projectID = msg["projectid"] if not hasattr(self, "timer"): self.timer = QTimer() self.timer.setInterval(5000) self.timer.setSingleShot(True) self.timer.timeout.connect(self._sendBaristaStatus) self.socketClosed.connect(lambda : self.timer.stop()) logging.debug("start heartbeat timer") self._sendBaristaStatus() def _sendBaristaStatus(self): """return the server status""" if hasattr(self, "timer"): self.timer.stop() if self.isConnected(): self.timer.start() msg = {"key": Protocol.GETSTATUS} if hasattr(self, "projectID"): state = self.parent.getBaristaStatus(self.projectID) else: state = self.parent.getBaristaStatus() msg["data"] = state logging.debug("got state") self.send(msg) def _getDBStatus(self): """open a database, check for status and get properties""" msg = self.asyncRead() dbo = DatabaseObject() path = msg["path"] if "absolute" not in msg: path = os.path.join(self.parent.sessionPath, path) dbo.setDB(path, msg["type"]) if msg["type"] == "HDF5TXT": dbo.setProjectPath(self.parent.sessionPath) dbo.open() status = dbo.isOpen() msg["status"] = status if status: msg["dataCount"] = dbo.getDataCount() msg["dimensions"] = dbo.getDimensions() dbo.close() logging.debug("Database: %s, Status: %s", path, str(status)) self.send(msg) def _makeRelative(self): """make a given path relative to sessionpath""" msg = self.asyncRead() path = msg["path"] sesPath = self.parent.sessionPath msg["path"] = os.path.relpath(path, sesPath) logging.debug("old: '%s', new: '%s'", path, msg["path"]) self.send(msg) def _setCaffePath(self): """set the caffepath""" msg = self.asyncRead() if self.parent.setCaffePath(msg["path"]): msg["status"] = True else: msg["status"] = False logging.debug("change of caffepath: %s", str(msg["status"])) self.send(msg) def _makeHDF5Txt(self): """create the hdf5txt file""" msg = self.asyncRead() msg["status"] = False path = msg["path"] if not os.path.exists(path): with open(path, 'w') as file: file.write(msg["hdf5"]) msg["status"] = True logging.debug("File created: %s", str(msg["status"])) self.send(msg) def _addHdf5(self): """add a line to HDF5txt""" msg = self.asyncRead() msg["status"] = False path = os.path.join(self.parent.sessionPath, msg["path"]) if os.path.exists(path): with open(path, 'a') as file: file.write("\n" + msg["hdf5"]) msg["status"] = True logging.debug("Line written: %s", str(msg["status"])) self.send(msg) def _getFileContent(self): """read a file and return contents""" # TODO this can be a security flaw since it allows to read arbitrary files. msg = self.asyncRead() msg["status"] = False path = os.path.join(self.parent.sessionPath, msg["path"]) if os.path.exists(path): with open(path, 'r') as file: msg["file"] = file.read() msg["status"] = True else: logging.warning("File does not exist: %s", path) self.send(msg) def _writeFileContent(self): """write a file""" # TODO this can be a security flaw since it allows to write arbitrary files. msg = self.asyncRead() msg["status"] = False path = os.path.join(self.parent.sessionPath, msg["path"]) if os.path.exists(path): with open(path, 'w') as file: file.write(msg["file"]) msg["status"] = True else: logging.warning("File does not exist: %s", path) self.send(msg) """ Computes and sends the hashValue of a db file, if this file exists. @author : j_stru18 """ def _getHash(self): msg = self.asyncRead() msg["status"] = False if "path" in msg and "type" in msg: path = msg["path"] type = msg["type"] hashValue = 0 if path is not None and type is not None: if os.path.exists(path): dbPaths = db_util.getDBPathsByType(path, type) hashValue = db_util.getMultipleHash(dbPaths) msg["status"] = True else: logging.warning("File does not exist: %s", path) else: logging.warning("Internal error: Remote message gave no path and type.") msg["hashValue"] = hashValue else: logging.warning("Internal error: Remote message was faulty.") self.send(msg) def _getHardware(self): """get the hardware specs""" msg = self.asyncRead() msg["hardware"] = self.parent.hardware msg["current"] = self.parent.trainOnHW self.send(msg) def _scanHardware(self): """launch a hardware scan""" self.parent.checkHardware(True, self) def _setHardware(self): msg = self.asyncRead() status = self.parent.setHardware(msg["hid"]) msg["status"] = status logging.info("Set current hardware to HID %i: %s", msg["hid"], str(status)) self.send(msg) def _createSession(self): self.parent.sessionManager.createNewSession(self) def _deleteSession(self): self.parent.sessionManager.deleteSession(self) def _connectToSession(self): self.parent.sessionManager.connectToSession(self) def _getSessions(self): self.parent.sessionManager.getSessions(self) def _doNothing(self): pass backend/networking/transaction.py0000664ڊ00111170000003175413243274037017475 0ustar klemmso0cvprimport pickle import sys from abc import abstractmethod from threading import Lock import logging import zlib from PyQt5.QtCore import QByteArray, QDataStream, QIODevice, QObject, pyqtSignal from PyQt5.QtNetwork import QTcpSocket, QAbstractSocket from PyQt5.QtTest import QSignalSpy from PyQt5.QtWidgets import qApp COMPRESS = True class Transaction(QObject): """Use parent class for server and client transactions. Abstracts TCPSocket-Access""" # empirical benchmarks using the echo command (send + recieve) shows following performance characteristics: # - a server can handle thousands of messages on one open socket per second, more CPU performance --> faster server # - as long as you don't saturate your network connection by increasing the message size there # should be no problems at all # - increasing the message size beyond the saturation of the network connection may lead to message drops # and crash the server # # some examples: # - a server on a laptop connected over a 2-3 mb/s wifi would crash if the message size is greater then aprox 6mb # - the same laptop with an ethernet cable can handle up to aprox 20 mb per message # - a server over localhost and the loopback interface could handle up to aprox 100mb # TODO longer testing # TODO close and reopen the socket after each / after n messages # TODO investigate message drops and server crashed # TODO maybe rewrite the class to split messages over certain size # splitting messages refers to writing at most n bytes to the QDataStream and flushing regulary # alternatively split the message altogether # this depends if the QTcpSocket or the QDataStream is the bottleneck bufferReady = pyqtSignal() socketClosed = pyqtSignal() _stopWaiting = pyqtSignal() _staging = pyqtSignal() def __init__(self): # QObject.__init__(self) super(Transaction, self).__init__() self.tcpsocket = None # type: QTcpSocket self.messageBuffer = QByteArray() self.recieveBuffer = QByteArray() self.messageOutput = list() self.messageSize = 0 self.isRecieving = False self.bufferReady.connect(self.processMessage) self._isConnected = False self._hasError = False self.lock = Lock() def connect(self, host, port): """connect to host on port""" logging.debug("Connecting to %s:%i", host, port) self._hasError = False self.tcpsocket = QTcpSocket() self.tcpsocket.error.connect(self.processError) self.tcpsocket.connected.connect(self._connected) self.tcpsocket.connected.connect(lambda: self._stopWaiting.emit()) self.tcpsocket.readyRead.connect(self.receive) self.tcpsocket.connectToHost(host, port) self.waitForConnection() def processError(self, error): """ Process the errors on the tcpsocket See: http://doc.qt.io/qt-5/qabstractsocket.html#SocketError-enum """ logging.debug("Error %i", error) if error == 0: # Remote Host can't be resolved self.close() elif error == 1: # Remote Host closed the connection. self.close() else: # TODO: somehow handle errors # Code 7 is detected far too late, it waits till the socket itself times out errortxt = "" for k in dir(QAbstractSocket): v = getattr(QAbstractSocket, k) if type(v) is type(error) and str(v) is str(error): errortxt = k sys.stderr.write("QTcpSocket@Transaction Error Code: " + str(error) + " " + errortxt + "\n") self.close() self._isConnected = False self._hasError = True self._stopWaiting.emit() def _connected(self): """On Connect mark this transactions as connected""" logging.debug("Socket connected.") self._isConnected = True self._hasError = False def acceptClient(self, socket): """set external socket""" logging.debug("Client socket connected.") self.tcpsocket = socket self._isConnected = True self._hasError = False self.tcpsocket.error.connect(self.processError) self.tcpsocket.readyRead.connect(self.receive) def receive(self): """On recieve packet load message into buffer. But first get message length""" logging.debug("Receiving message") while self.tcpsocket.bytesAvailable() > 0: # while some unread bytes available stream = QDataStream(self.tcpsocket) stream.setVersion(QDataStream.Qt_5_3) if not self.isRecieving: # For a new Message get the Message Size if self.tcpsocket.bytesAvailable >= 4: # since reading UInt32 self.messageSize = stream.readUInt32() logging.debug("Start of new message of size %i", self.messageSize) self.isRecieving = True else: break else: # For a continued message keep reding until whole message is in buffer s = min(self.tcpsocket.bytesAvailable(), self.messageSize - len(self.messageBuffer)) self.messageBuffer.append(stream.readRawData(s)) if len(self.messageBuffer) == self.messageSize: logging.debug("Finished receiving message of size %i", self.messageSize) self._processBuffer() def send(self, msg): """Send first the message size, then the message in Pickle""" if self.isConnected(): pmsg = pickle.dumps(msg) if COMPRESS: pmsg = zlib.compress(pmsg) buffer = QByteArray() stream = QDataStream(buffer, QIODevice.WriteOnly) stream.setVersion(QDataStream.Qt_5_3) stream.writeUInt32(len(pmsg)) stream.writeRawData(pmsg) bytesWritten = self.tcpsocket.write(buffer) self.tcpsocket.flush() self.tcpsocket.waitForBytesWritten() # qApp.processEvents() # send data immediately and don't wait for next mainloop logging.debug("Bytes written: %i", bytesWritten) if bytesWritten > 0: return True else: logging.debug("Message not send. Not connected") return False def isConnected(self): """return the connected state""" return self._isConnected def close(self): """close this socket and fire signal socketClosed to notify slots to delete this obj""" if self.tcpsocket: logging.debug("socket closing") self._isConnected = False self.tcpsocket.close() # self.tcpsocket = None self.socketClosed.emit() def _processBuffer(self): """If message is complete: read Pickle and move to output buffer""" pmsg = str(self.messageBuffer) if COMPRESS: pmsg = zlib.decompress(pmsg) msg = pickle.loads(pmsg) logging.debug("Message received: %s", str(msg)) self.lock.acquire() self.messageOutput.append(msg) self.lock.release() # reset state self.isRecieving = False self.messageBuffer.clear() self._stopWaiting.emit() self.bufferReady.emit() @abstractmethod def processMessage(self): """implement in server/client""" pass def asyncRead(self, timeout=5000, staging=False, attr=None): """For direct reading access, pop the first message in buffer Explanation: timeout: stop after waiting for 5s and return NONE object. this is usefull if the server does not answer (e.g. crash) or the networkconnection breaks staging: If True the asynchronous wait will wait for a the staging signal instead of the messageOutput signal. the staging signal should be used if you have a class which uses both: a signal/slot based processMessage methode and asyncRead calls. This gives the processMessage calls priority. !!!Don't forget to call the stage() method if your processMessage ignores the message!!! attr: In some cases you have multiple asyncRead calls on the same transaction Item. This may lead to preemtive behaviour resulting in reading the wrong messages and key errors. (e.g. READ#1 reads messages destined for READ#2) By providing a tupel (attr, value) the message is parsed and only poped if the value matches. Not matching messages are appended back into the buffer and other waiting asyncReads are notified """ if self.isConnected(): turns = 0 while True: turns += 1 if turns > 100: logging.debug("Timeout on read after 100 iterations") return None result = True logging.debug("MessageOutput size: %i", len(self.messageOutput)) if len(self.messageOutput) == 0 and not self.containsAttr(attr): logging.debug("Waiting for new message.") if not staging: # spy = QSignalSpy(self._stopWaiting) spy = QSignalSpy(self.bufferReady) else: spy = QSignalSpy(self._staging) result = spy.wait(timeout) # Asynchronous wait, Timeout 5s if result and not self._hasError: self.lock.acquire() if len(self.messageOutput) == 0: self.lock.release() logging.debug("Race condition triggered. Wait for next message.") continue found = False result = self.messageOutput[0] if attr is not None: for msg in self.messageOutput: if attr[0] in msg: if attr[1] == msg[attr[0]]: found = True result = msg break if found or attr is None: del self.messageOutput[self.messageOutput.index(result)] logging.debug("MessageOutput size: %i", len(self.messageOutput)) self.lock.release() if "error" not in result: result["error"] = [] if "status" not in result: result["status"] = True return result else: logging.debug("Message not found. Release of lock.") if attr is not None: logging.debug("Miss '%s' with value '%s'", str(attr[0]), str(attr[1])) self.lock.release() self.bufferReady.emit() qApp.processEvents() else: logging.debug("Nothing to read.") break else: logging.debug("Not connected. Did not read.") return None def stage(self): logging.debug("Stage!") self._staging.emit() def waitForConnection(self): """sometimes it takes some time to establish the connection. wait half a second""" logging.debug("Establishing connection") if self.tcpsocket is not None: result = True if not self.isConnected() and not self._hasError: # spy = QSignalSpy(self.tcpsocket.connected) spy = QSignalSpy(self._stopWaiting) result = spy.wait(5000) # Asynchronous wait, Timeout 5 s if not result: # it is bad if the socket needs longer than the timeout and connects # after the connection is deemed dead self.close() logging.debug("Connection not established, manually closing socket.") return result and not self._hasError return False def getAttrOfFirst(self, attrlist): """check the attributes of the first item in the output buffer""" self.lock.acquire() result = [] for attr in attrlist: if attr in self.messageOutput[0]: result.append(self.messageOutput[0][attr]) else: result.append(None) self.lock.release() return result def containsAttr(self, attr): """check if any message in the output buffer contains the desired attribute""" if attr is None: return False self.lock.acquire() found = False for msg in self.messageOutput: if attr[0] in msg: if msg[attr[0]] is attr[1]: found = True break self.lock.release() return found backend/parser/__init__.py0000664ڊ00111170000000000013243274037015770 0ustar klemmso0cvprbackend/parser/concatenator.py0000664ڊ00111170000002007413243274037016726 0ustar klemmso0cvpr''' Created on 18.11.2016 @author: k.Hartz ''' import itertools import os import re from backend.parser.parser_listener import ParserListener from backend.barista.utils.logger import Log from backend.parser.parser import Parser class Concatenator: ''' Concatenates multiple logfiles (with same snapshot-file) to one iterator ''' def __init__(self, listOfLogfiles): ''' Initializes the concatinator with a list of Logfiles to get a list of iterators concerning to the merged logs call concate() ''' self.__listeners = [] events = { Concatenator.finish: re.compile('Optimization Done.'), Concatenator.interrupt: re.compile( "((?:Snapshotting solver state to (?:binary proto|HDF5) file ).*)"), Concatenator.resume: re.compile("((?<= Resuming from ).*)"), } for logFile in listOfLogfiles: log_iter = self.__createIterFromFile(logFile) parser = Parser(log_iter, events) listener = Concatenator.SnapshotListener(logFile) parser.addListener(listener) try: parser.parseLog() self.__listeners.append(listener) except: callerId = Log.getCallerId("Error Parsing Log File:" + str(logFile)) Log.error("", callerId) def createSingleLogfile(self, filePath, logfileName, stream=None): ''' creates one logfile for each Training returns amount of created files or -1 for none ''' if(stream is not None): fullIter = [self.concateWithStream(stream)] else: fullIter = self.concate() i = -1 for itera in fullIter: i += 1 with open(filePath + logfileName + str(i) + '.log', 'w') as f: for line in itera: f.write(str(line)) return i+1 # TODO: Test def concateWithStream(self, stream): ''' concatenates (only!) the first (concatenated) logfile with the given stream to an iterator, first the logfiles than the stream ''' return itertools.chain(iter(self.concate()[0]), stream) def concate(self): ''' concatenate the logfiles into one iterator returns an array of concatenated iterators every iterator represents one training ''' if(len(self.__listeners) == 1): return [self.__listeners[0].filePath] listenerRange = range(0, len(self.__listeners)) for i in listenerRange: for j in listenerRange: # TODO can start from i ... length if(i != j and (self.__listeners[i].groupId == -1 or self.__listeners[j].groupId == -1 ) and self.__fromSameTraining(i, j)): self.__setGroupId(i, j) for i in listenerRange: if(self.__listeners[i].groupId == -1): self.__setGroupId(i, i) listenerGroups = self.__groupListeners() return self.__createAndConcateIters(listenerGroups) # Training Interrupted, snapshot is made __groupIdCounter = 0 # ------EVENT CONSTANTS----- # Training Completet (Caffe exited) finish = 'Finished' # Training Resumed from snapshot resume = 'Resume' # Training Interrupted, snapshot is made interrupt = 'Interrupted' class SnapshotListener(ParserListener): ''' Listener for Logfile-Parser, looks for properties that are needed to determine which logs should concate ''' def __init__(self, logfilePath): # id to put listeners together self.groupId = -1 # to be able to create an iter from this listener self.filePath = logfilePath # to identify the listeners ancestor self.logname = self.__getFileName(logfilePath) # has finished parsing self.finish = False # has been resumed from another session self.resume = False # hasSnapshot self.hasSnapshot = False self.snapshotName = "" # the name of the resumed snapshot file self.snapshotResumeName = "" # start iteration count self.iterationMin = -1 # finished iteration count self.iterationMax = -1 def update(self, phase, row): if phase == "TRAIN": iterations = row.get('NumIters') if iterations > self.iterationMax: self.iterationMax = int(iterations) if self.iterationMin == -1: self.iterationMin = int(iterations) def handle(self, event, message, groups): ''' handles events: finish,interrupt,resume ''' if(event == Concatenator.finish): self.finish = True elif(event == Concatenator.interrupt): self.hasSnapshot = True self.snapshotName = groups[0].strip() elif(event == Concatenator.resume): self.hasSnapshot = True self.snapshotResumeName = groups[0].strip() def __getFileName(self, filePath): ''' extracts the filename from a filepath ''' _, tail = os.path.split(str(filePath)) return tail # ----------Private Methods------------- def __fromSameTraining(self, i, j): ''' returns boolean indicating whether index i and j are from the same training ''' iListener = self.__listeners[i] jListener = self.__listeners[j] # Merge every log if(False): return True # have snapshots? if not(iListener.hasSnapshot and jListener.hasSnapshot): return False # Same snapshotname? if(iListener.snapshotName != jListener.snapshotResumeName and jListener.snapshotName != iListener.snapshotResumeName): return False range1 = range(iListener.iterationMin, iListener.iterationMax) range2 = range(jListener.iterationMin, jListener.iterationMax) # Iterations doesnt have intersect if(len(set(range1).intersection(range2)) > 1): return False return True def __setGroupId(self, i, j): ''' Sets the same group id for index i,j in self.__listeners ''' if self.__listeners[i].groupId != -1: self.__listeners[j].groupId = self.__listeners[i].groupId return if self.__listeners[i].groupId != -1: self.__listeners[j].groupId = self.__listeners[i].groupId return Concatenator.__groupIdCounter += 1 self.__listeners[j].groupId = Concatenator.__groupIdCounter self.__listeners[j].groupId = Concatenator.__groupIdCounter def __groupListeners(self): ''' groups the listeners by id and sorts them by the startiterations ''' self.__listeners.sort(key=lambda listener: (listener.groupId, listener.iterationMin)) listenerGroups = itertools.groupby(self.__listeners, lambda listener: (listener.groupId)) sortedList = [] for _, group in listenerGroups: group = list(group) sortedList.append(group) return sortedList def __createAndConcateIters(self, listenerGroups): ''' creates an iterator for every ''' iterators = [] for group in listenerGroups: for listener in group: itera = listener.filePath iterators.append(itera) # for i in range(0, len(iterators)): # iterators[i] = fileinput.input(iterators[i]) return iterators def __createIterFromFile(self, filePath): ''' creates an iterator for a logfile for a given path ''' return (open(filePath)) backend/parser/dummy_std_out.py0000664ڊ00111170000000132413243274037017137 0ustar klemmso0cvpr''' Created on 08.11.2016 @author: k.hartz ''' import os import time import sys # Logfile to pass through stdout logFile = "test/logfiles/test_log_1.log" # delay per line in sec, 1=1sec delayPerLine = 0.01 # Seperator to seperate the logfile output from print seperator = "------------------" # directory where this file is contained dir_path = os.path.dirname(os.path.realpath(__file__)) # working directory cwd = os.getcwd() print("Start Output:\n\n") print("working directory:", cwd) print("directory where this file is contained:", dir_path) print(seperator) f = open(logFile) for line in f: sys.stdout.write(line) sys.stdout.flush() time.sleep(delayPerLine) print(seperator) print("Finished log") backend/parser/extract_seconds.py0000775ڊ00111170000000715113243274037017442 0ustar klemmso0cvpr""" COPYRIGHT All contributions by the University of California: Copyright (c) 2014, 2015, The Regents of the University of California (Regents) All rights reserved. All other contributions: Copyright (c) 2014, 2015, the respective contributors All rights reserved. Caffe uses a shared copyright model: each contributor holds copyright over their contributions to Caffe. The project versioning records all such contribution and copyright details. If a contributor wants to further mark their specific copyright on a particular contribution, they should indicate their copyright solely in the commit message of the change when it is committed. LICENSE Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. CONTRIBUTION AGREEMENT By contributing to the BVLC/caffe repository through pull-request, comment, or otherwise, the contributor releases their content to the license and copyright terms herein. """ import re from datetime import datetime def extractDatetimeFromLine(line, year): """Extract the datetime from the line. Returns the parsed datetime with the added year. """ # Expected format: # I0210 13:39:22.381027 2521 solver.cpp:204] line = line.strip().split() month = int(line[0][1:3]) day = int(line[0][3:]) timestamp = line[1] pos = timestamp.rfind('.') ts = [int(x) for x in timestamp[:pos].split(':')] hour = ts[0] minute = ts[1] second = ts[2] microsecond = int(timestamp[pos + 1:]) dt = datetime(year, month, day, hour, minute, second, microsecond) return dt def getLogCreatedYear(input_file): """Get year from log file start time. Search for 'Log file created at:' Use current year if the string can not be found. """ regex_year = re.compile('Log file created at: ([\d]{4})') for line in input_file: year_match = regex_year.search(line) if year_match: return int(year_match.group(1)) if isLogFormat(line): now = datetime.now() return now.year def getStartTime(line, year): """Find start time in the line. """ if line.find('Solving') != -1: return extractDatetimeFromLine(line, year) regex_log_msg = re.compile( '[IWEF][\d]{4} [\d]{2}:[\d]{2}:[\d]{2}\.[\d]{6}[\s]+[\d]+[\s]+[\w\.]+:[\d]+\]') def isLogFormat(line): """ Return True if the line has the log format. """ log_match = regex_log_msg.search(line) if log_match: return True else: return False backend/parser/parser.py0000664ڊ00111170000002770013243274037015545 0ustar klemmso0cvpr""" COPYRIGHT All contributions by the University of California: Copyright (c) 2014, 2015, The Regents of the University of California (Regents) All rights reserved. All other contributions: Copyright (c) 2014, 2015, the respective contributors All rights reserved. Caffe uses a shared copyright model: each contributor holds copyright over their contributions to Caffe. The project versioning records all such contribution and copyright details. If a contributor wants to further mark their specific copyright on a particular contribution, they should indicate their copyright solely in the commit message of the change when it is committed. LICENSE Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. CONTRIBUTION AGREEMENT By contributing to the BVLC/caffe repository through pull-request, comment, or otherwise, the contributor releases their content to the license and copyright terms herein. """ import re import sys if sys.version_info[0] == 2: from Queue import Queue else: from queue import Queue from collections import OrderedDict from threading import Lock from backend.parser import extract_seconds as exs from backend.barista.utils.logger import Log from backend.parser.parser_common import ParserCommon from PyQt5.QtCore import pyqtSignal from PyQt5.Qt import QObject class Parser(QObject, ParserCommon): """ Log File Parser. This class takes a stream of log messages and parses them on parseLog(). A listener could be registered for parsing events like log records, exceptions and new keys. """ printLogSignl = pyqtSignal(str, bool) def __init__(self, log_stream, events, log_id=None): super(Parser, self).__init__() self.train_dict_list = [] self.test_dict_list = [] self.log_stream = log_stream self.thread_id = None self.start_time = None self.log_id = log_id self.logging = False self.streams = Queue() self.lock = Lock() if self.log_stream: self.streams.put(self.log_stream) self.events = events if self.events is None: self.events = {} def printLog(self, log, error=False): if self.log_id is not None: if not error: Log.log(log, self.log_id) else: Log.error(log, self.log_id) else: self.printLogSignl.emit(log, error) def getTestEvents(self): """Return the list of parsed test records.""" return self.test_dict_list def getTrainEvents(self): """Return the list of parsed training records.""" return self.train_dict_list def getThreadID(self): """ Return the thread id of the caffe process. """ return self.thread_id def addLogStream(self, log_stream): """ Add a log stream to to queue of parsing tasks. """ if log_stream: self.streams.put(log_stream) def setLogging(self, log): self.logging = log def parseLog(self): """Parse log file. Returns (train_dict_list, test_dict_list) train_dict_list and test_dict_list are lists of dicts that define the table rows """ locked = self.lock.acquire() if locked is False: return self.train_dict_list, self.test_dict_list regex_iteration = re.compile('Iteration (\d+)') regex_train_output = re.compile( 'Train net output #(\d+): (\S+) = ([\.\deE+-]+)') regex_test_output = re.compile( 'Test net output #(\d+): (\S+) = ([\.\deE+-]+)') regex_learning_rate = re.compile( 'lr = ([-+]?[0-9]*\.?[0-9]+([eE]?[-+]?[0-9]+)?)') # Pick out lines of interest iteration = -1 learning_rate = float('NaN') train_dict_list = self.train_dict_list test_dict_list = self.test_dict_list train_row = None test_row = None try: while self.streams.empty() is False: head = self.streams.get() log_file = None ht = type(head) if ht is unicode or ht is str: # open file log_file = open(head, 'r') log_stream = log_file.readlines() else: log_stream = head try: logfile_year = exs.getLogCreatedYear(log_stream) for line in log_stream: line = line.strip() if self.logging: self.printLog(line) if self.thread_id is None: self.extractThreadID(line) if self.start_time is None: self.start_time = exs.getStartTime(line, logfile_year) self.parseEvent(line) iteration_match = regex_iteration.search(line) if iteration_match: iteration = float(iteration_match.group(1)) if iteration == -1: # Only start parsing for other stuff if we've # found the first iteration continue if exs.isLogFormat(line) is False: continue if self.start_time is None: self.start_time = exs.extractDatetimeFromLine(line, logfile_year) time = exs.extractDatetimeFromLine(line, logfile_year) seconds = (time - self.start_time).total_seconds() lr_match = regex_learning_rate.search(line) if lr_match: self.checkKey(Parser.TRAIN, 'LearningRate') learning_rate = float(lr_match.group(1)) train_dict_list, train_row = self.parseLine( regex_train_output, train_row, train_dict_list, line, iteration, seconds, learning_rate, time, Parser.TRAIN ) test_dict_list, test_row = self.parseLine( regex_test_output, test_row, test_dict_list, line, iteration, seconds, learning_rate, time, Parser.TEST ) except Exception as e: self.printLog('Parser error: '+str(e)) finally: if log_file: log_file.close() except Exception as e: if self.log_id: self.printLog('Failed to parse log '+str(e), True) else: print('Failed to parse log '+str(e)) finally: if locked: self.lock.release() for lis in self.listener: lis.parsingFinished() return train_dict_list, test_dict_list def parseLine(self, regex_obj, row, row_dict_list, line, iteration, seconds, learning_rate, time, phase): """Parse a single line for training or test output. Returns a a tuple with (row_dict_list, row) row: may be either a new row or an augmented version of the current row row_dict_list: may be either the current row_dict_list or an augmented version of the current row_dict_list """ output_match = regex_obj.search(line) if output_match: if not row or row['NumIters'] != iteration: # Push the last row and start a new one if row: # If we're on a new iteration, push the last row # This will probably only happen for the first row; # otherwisethe full row checking logic below will push and # clear full rows row_dict_list.append(row) row = OrderedDict([ ('NumIters', iteration), ('Seconds', seconds), ('LearningRate', learning_rate), ('DateTime', time) ]) # output_num is not used; may be used in the future # output_num = output_match.group(1) output_name = output_match.group(2) output_val = output_match.group(3) row[output_name] = float(output_val) self.checkKey(phase, output_name) # append a new row to the dict if (row and len(row_dict_list) >= 1 and len(row) == len(row_dict_list[0])): # fix the learning rate of the first row if len(row_dict_list) == 1: row_dict_list[0]['LearningRate'] = row['LearningRate'] for lis in self.listener: lis.update(phase, row_dict_list[0]) # The row is full, based on the fact that it has the same number of # columns as the first row; append it to the list row_dict_list.append(row) # notify the listener about the new row for lis in self.listener: lis.update(phase, row) row = None return row_dict_list, row def parseEvent(self, line): """Parse the line for events. Notifies the listener about every event found. """ for event, regex in self.events.iteritems(): event_match = regex.search(line) if event_match: size = len(event_match.groups()) groups = [] for i in range(0, size): groups.append(event_match.group(i+1)) for lis in self.listener: lis.handle(event, line, groups) def extractThreadID(self, line): """ Extract the thread id from the file. """ regex_thread_id = re.compile( '[IWEF][\d]{4} [\d]{2}:[\d]{2}:[\d]{2}\.[\d]{6}[\s]+([\d]+)') line = line.strip() thread_id_match = regex_thread_id.search(line) if thread_id_match: self.thread_id = thread_id_match.group(1) def checkKey(self, phase, key): """ Check wheter the key is new and notify the listener in this case. """ if key == "lr": key = "LearningRate" key_registry = self.getKeys(phase) if key not in key_registry: key_registry.append(key) for lis in self.listener: lis.registerKey(phase, key) def hasKey(self, phase, key): """ Return true if the key was registered for the phase. """ return key in self.getKeys(phase) backend/parser/parser_common.py0000664ڊ00111170000000161713243274037017114 0ustar klemmso0cvpr class ParserCommon(object): # keys for train/test phase TRAIN = 'TRAIN' TEST = 'TEST' def __init__(self): super(ParserCommon, self).__init__() self.listener = [] self.train_keys = [] self.test_keys = [] def addListener(self, listener): """Add the listener to the list of listener. Every listener is notified about every parsing event. """ if listener not in self.listener: self.listener.append(listener) def removeListener(self, listener): """ Remove the listener from the list of listener. """ self.listener.remove(listener) def getKeys(self, phase): """ Return all registered keys for the given phase. """ if phase == ParserCommon.TEST: return self.test_keys elif phase == ParserCommon.TRAIN: return self.train_keys backend/parser/parser_dummy.py0000664ڊ00111170000000134713243274037016757 0ustar klemmso0cvprfrom backend.parser.parser_common import ParserCommon class ParserDummy(ParserCommon): def __init__(self): super(ParserDummy, self).__init__() def sendParserFinished(self): for lis in self.listener: lis.parsingFinished() def sendParserUpdate(self, phase, row): for lis in self.listener: lis.update(phase, row) def sendParserHandle(self, event, line, groups): for lis in self.listener: lis.handle(event, line, groups) def sendParserRegisterKeys(self, phase, key): keyRegistry = self.getKeys(phase) if key not in keyRegistry: keyRegistry.append(key) for lis in self.listener: lis.registerKey(phase, key) backend/parser/parser_listener.py0000664ڊ00111170000000103113243274037017437 0ustar klemmso0cvpr class ParserListener: def update(self, phase, row): """ Called when the parser has parsed a new record. """ pass def handle(self, event, message, groups): """ Called when the parser has parsed a registered event. """ pass def registerKey(self, phase, key): """ Called when a new key was found in the log data. """ pass def parsingFinished(self): """ Called when the parser has processed all available streams. """ pass backend/__init__.py0000664ڊ00111170000000000013243274037014474 0ustar klemmso0cvprresources/0000775ڊ00111170000000000013243274037013020 5ustar klemmso0cvprgui/main_window/docks/properties/property_widgets/0000775ڊ00111170000000000013243274037023016 5ustar klemmso0cvprgui/main_window/node_editor/items/renderer/0000775ڊ00111170000000000013243274037021327 5ustar klemmso0cvprgui/main_window/docks/plotter/0000775ڊ00111170000000000013243274037016701 5ustar klemmso0cvprgui/main_window/docks/properties/0000775ڊ00111170000000000013243274037017404 5ustar klemmso0cvprgui/main_window/docks/session/0000775ڊ00111170000000000013243274037016673 5ustar klemmso0cvprgui/main_window/docks/weight_visualization/0000775ڊ00111170000000000013243274037021460 5ustar klemmso0cvprgui/main_window/node_editor/items/0000775ڊ00111170000000000013243274037017521 5ustar klemmso0cvprgui/main_window/docks/0000775ڊ00111170000000000013243274037015210 5ustar klemmso0cvprgui/main_window/node_editor/0000775ڊ00111170000000000013243274037016400 5ustar klemmso0cvprgui/host_manager/0000775ڊ00111170000000000013243274037014241 5ustar klemmso0cvprgui/input_manager/0000775ڊ00111170000000000013243274037014423 5ustar klemmso0cvprgui/main_window/0000775ڊ00111170000000000013243274037014105 5ustar klemmso0cvprgui/network_manager/0000775ڊ00111170000000000013243274037014755 5ustar klemmso0cvprgui/prototxt_editor/0000775ڊ00111170000000000013243274037015043 5ustar klemmso0cvprgui/0000775ڊ00111170000000000013243274037011572 5ustar klemmso0cvprbackend/barista/constraints/permanent/0000775ڊ00111170000000000013243274037020362 5ustar klemmso0cvprbackend/barista/constraints/session_run/0000775ڊ00111170000000000013243274037020740 5ustar klemmso0cvprbackend/barista/constraints/0000775ڊ00111170000000000013243274037016371 5ustar klemmso0cvprbackend/barista/session/0000775ڊ00111170000000000013243274037015505 5ustar klemmso0cvprbackend/barista/utils/0000775ڊ00111170000000000013243274037015162 5ustar klemmso0cvprbackend/barista/0000775ڊ00111170000000000013243274037014022 5ustar klemmso0cvprbackend/caffe/0000775ڊ00111170000000000013243274037013441 5ustar klemmso0cvprbackend/input_db/0000775ڊ00111170000000000013243274037014201 5ustar klemmso0cvprbackend/networking/0000775ڊ00111170000000000013243274037014564 5ustar klemmso0cvprbackend/parser/0000775ڊ00111170000000000013243274037013671 5ustar klemmso0cvprbackend/0000775ڊ00111170000000000013243274037012375 5ustar klemmso0cvpr